diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 4c7c0ec9bb35..31e90c30cf17 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -68,6 +68,13 @@ - The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`. +- The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though: + - The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below. + - The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed. + - These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["redis"]; }` for `hiredis` & `txredisapi` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if `services.matrix-synapse.settings.redis.enabled` is set to `true`, `"redis"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`. + - A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras). + - In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only. + - `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides - `consul` has been updated to `1.16.0`. See the [release note](https://github.com/hashicorp/consul/releases/tag/v1.16.0) for more details. Once a new Consul version has started and upgraded its data directory, it generally cannot be downgraded to the previous version. diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index 3dca3ff94f21..ef69a8adebb0 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -9,11 +9,6 @@ let # remove null values from the final configuration finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings; configFile = format.generate "homeserver.yaml" finalSettings; - logConfigFile = format.generate "log_config.yaml" cfg.logConfig; - - pluginsEnv = cfg.package.python.buildEnv.override { - extraLibs = cfg.plugins; - }; usePostgresql = cfg.settings.database.name == "psycopg2"; hasLocalPostgresDB = let args = cfg.settings.database.args; in @@ -50,6 +45,29 @@ let "${bindAddress}" }:${builtins.toString listener.port}/" ''; + + defaultExtras = [ + "systemd" + "postgres" + "url-preview" + "user-search" + ]; + + wantedExtras = cfg.extras + ++ lib.optional (cfg.settings ? oidc_providers) "oidc" + ++ lib.optional (cfg.settings ? jwt_config) "jwt" + ++ lib.optional (cfg.settings ? saml2_config) "saml2" + ++ lib.optional (cfg.settings ? opentracing) "opentracing" + ++ lib.optional (cfg.settings ? redis) "redis" + ++ lib.optional (cfg.settings ? sentry) "sentry" + ++ lib.optional (cfg.settings ? user_directory) "user-search" + ++ lib.optional (cfg.settings.url_preview_enabled) "url-preview" + ++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres"; + + wrapped = pkgs.matrix-synapse.override { + extras = wantedExtras; + inherit (cfg) plugins; + }; in { imports = [ @@ -151,10 +169,53 @@ in { package = mkOption { type = types.package; - default = pkgs.matrix-synapse; - defaultText = literalExpression "pkgs.matrix-synapse"; + readOnly = true; description = lib.mdDoc '' - Overridable attribute of the matrix synapse server package to use. + Reference to the `matrix-synapse` wrapper with all extras + (e.g. for `oidc` or `saml2`) added to the `PYTHONPATH` of all executables. + + This option is useful to reference the "final" `matrix-synapse` package that's + actually used by `matrix-synapse.service`. For instance, when using + workers, it's possible to run + `''${config.services.matrix-synapse.package}/bin/synapse_worker` and + no additional PYTHONPATH needs to be specified for extras or plugins configured + via `services.matrix-synapse`. + + However, this means that this option is supposed to be only declared + by the `services.matrix-synapse` module itself and is thus read-only. + In order to modify `matrix-synapse` itself, use an overlay to override + `pkgs.matrix-synapse-unwrapped`. + ''; + }; + + extras = mkOption { + type = types.listOf (types.enum (lib.attrNames pkgs.matrix-synapse-unwrapped.optional-dependencies)); + default = defaultExtras; + example = literalExpression '' + [ + "cache-memory" # Provide statistics about caching memory consumption + "jwt" # JSON Web Token authentication + "opentracing" # End-to-end tracing support using Jaeger + "oidc" # OpenID Connect authentication + "postgres" # PostgreSQL database backend + "redis" # Redis support for the replication stream between worker processes + "saml2" # SAML2 authentication + "sentry" # Error tracking and performance metrics + "systemd" # Provide the JournalHandler used in the default log_config + "url-preview" # Support for oEmbed URL previews + "user-search" # Support internationalized domain names in user-search + ] + ''; + description = lib.mdDoc '' + Explicitly install extras provided by matrix-synapse. Most + will require some additional configuration. + + Extras will automatically be enabled, when the relevant + configuration sections are present. + + Please note that this option is additive: i.e. when adding a new item + to this list, the defaults are still kept. To override the defaults as well, + use `lib.mkForce`. ''; }; @@ -193,7 +254,7 @@ in { default = {}; description = mdDoc '' The primary synapse configuration. See the - [sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml) + [sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml) for possible values. Secrets should be passed in by using the `extraConfigFiles` option. @@ -706,6 +767,10 @@ in { ]; services.matrix-synapse.configFile = configFile; + services.matrix-synapse.package = wrapped; + + # default them, so they are additive + services.matrix-synapse.extras = defaultExtras; users.users.matrix-synapse = { group = "matrix-synapse"; @@ -729,9 +794,7 @@ in { --keys-directory ${cfg.dataDir} \ --generate-keys ''; - environment = { - PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ]; - } // optionalAttrs (cfg.withJemalloc) { + environment = optionalAttrs (cfg.withJemalloc) { LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so"; }; serviceConfig = { diff --git a/nixos/tests/matrix/synapse.nix b/nixos/tests/matrix/synapse.nix index 698d67c793e3..98b077469192 100644 --- a/nixos/tests/matrix/synapse.nix +++ b/nixos/tests/matrix/synapse.nix @@ -65,7 +65,7 @@ in { nodes = { # Since 0.33.0, matrix-synapse doesn't allow underscores in server names - serverpostgres = { pkgs, nodes, ... }: let + serverpostgres = { pkgs, nodes, config, ... }: let mailserverIP = nodes.mailserver.config.networking.primaryIPAddress; in { @@ -77,6 +77,11 @@ in { name = "psycopg2"; args.password = "synapse"; }; + redis = { + enabled = true; + host = "localhost"; + port = config.services.redis.servers.matrix-synapse.port; + }; tls_certificate_path = "${cert}"; tls_private_key_path = "${key}"; registration_shared_secret = registrationSharedSecret; @@ -107,6 +112,11 @@ in { ''; }; + services.redis.servers.matrix-synapse = { + enable = true; + port = 6380; + }; + networking.extraHosts = '' ${mailserverIP} ${mailerDomain} ''; @@ -208,6 +218,9 @@ in { serverpostgres.wait_until_succeeds( "curl --fail -L --cacert ${ca_pem} https://localhost:8448/" ) + serverpostgres.wait_until_succeeds( + "journalctl -u matrix-synapse.service | grep -q 'Connected to redis'" + ) serverpostgres.require_unit_state("postgresql.service") serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} https://localhost:8448/") serverpostgres.succeed("obtain-token-and-register-email") diff --git a/pkgs/servers/matrix-synapse/default.nix b/pkgs/servers/matrix-synapse/default.nix index 56650531a65d..2d4455d322a7 100644 --- a/pkgs/servers/matrix-synapse/default.nix +++ b/pkgs/servers/matrix-synapse/default.nix @@ -1,7 +1,12 @@ -{ lib, stdenv, fetchFromGitHub, python3, openssl, cargo, rustPlatform, rustc -, enableSystemd ? lib.meta.availableOn stdenv.hostPlatform python3.pkgs.systemd +{ lib +, stdenv +, fetchFromGitHub +, python3 +, openssl +, cargo +, rustPlatform +, rustc , nixosTests -, enableRedis ? true , callPackage }: @@ -9,8 +14,7 @@ let plugins = python3.pkgs.callPackage ./plugins { }; tools = callPackage ./tools { }; in -with python3.pkgs; -buildPythonApplication rec { +python3.pkgs.buildPythonApplication rec { pname = "matrix-synapse"; version = "1.89.0"; format = "pyproject"; @@ -34,7 +38,7 @@ buildPythonApplication rec { sed -i '/^setuptools_rust =/d' pyproject.toml ''; - nativeBuildInputs = [ + nativeBuildInputs = with python3.pkgs; [ poetry-core rustPlatform.cargoSetupHook setuptools-rust @@ -42,47 +46,90 @@ buildPythonApplication rec { rustc ]; - buildInputs = [ openssl ]; + buildInputs = [ + openssl + ]; - propagatedBuildInputs = [ - authlib + propagatedBuildInputs = with python3.pkgs; [ + attrs bcrypt bleach canonicaljson - daemonize + cryptography ijson immutabledict jinja2 jsonschema - lxml matrix-common msgpack netaddr + packaging phonenumbers pillow prometheus-client - psutil - psycopg2 pyasn1 + pyasn1-modules pydantic - pyicu pymacaroons - pynacl pyopenssl - pysaml2 pyyaml - requests - setuptools + service-identity signedjson sortedcontainers treq twisted typing-extensions unpaddedbase64 - ] ++ lib.optional enableSystemd systemd - ++ lib.optionals enableRedis [ hiredis txredisapi ]; + ] + ++ twisted.optional-dependencies.tls; - nativeCheckInputs = [ mock parameterized openssl ]; + passthru.optional-dependencies = with python3.pkgs; { + postgres = if isPyPy then [ + psycopg2cffi + ] else [ + psycopg2 + ]; + saml2 = [ + pysaml2 + ]; + oidc = [ + authlib + ]; + systemd = [ + systemd + ]; + url-preview = [ + lxml + ]; + sentry = [ + sentry-sdk + ]; + opentracing = [ + jaeger-client + opentracing + ]; + jwt = [ + authlib + ]; + redis = [ + hiredis + txredisapi + ]; + cache-memory = [ + pympler + ]; + user-search = [ + pyicu + ]; + }; + + nativeCheckInputs = [ + openssl + ] ++ (with python3.pkgs; [ + mock + parameterized + ]) + ++ lib.flatten (lib.attrValues passthru.optional-dependencies); doCheck = !stdenv.isDarwin; @@ -112,6 +159,7 @@ buildPythonApplication rec { meta = with lib; { homepage = "https://matrix.org"; + changelog = "https://github.com/matrix-org/synapse/releases/tag/v${version}"; description = "Matrix reference homeserver"; license = licenses.asl20; maintainers = teams.matrix.members; diff --git a/pkgs/servers/matrix-synapse/plugins/ldap3.nix b/pkgs/servers/matrix-synapse/plugins/ldap3.nix index b29dc21422e9..feac6f084726 100644 --- a/pkgs/servers/matrix-synapse/plugins/ldap3.nix +++ b/pkgs/servers/matrix-synapse/plugins/ldap3.nix @@ -4,7 +4,7 @@ , fetchPypi , ldap3 , ldaptor -, matrix-synapse +, matrix-synapse-unwrapped , pytestCheckHook , service-identity , setuptools @@ -33,7 +33,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ service-identity ldap3 twisted ]; - nativeCheckInputs = [ ldaptor matrix-synapse pytestCheckHook ]; + nativeCheckInputs = [ ldaptor matrix-synapse-unwrapped pytestCheckHook ]; pythonImportsCheck = [ "ldap_auth_provider" ]; diff --git a/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix b/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix index 2d0fa4115cc5..e1a9f7285920 100644 --- a/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix +++ b/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse }: +{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped }: buildPythonPackage rec { pname = "matrix-synapse-mjolnir-antispam"; @@ -13,7 +13,7 @@ buildPythonPackage rec { sourceRoot = "./${src.name}/synapse_antispam"; - buildInputs = [ matrix-synapse ]; + buildInputs = [ matrix-synapse-unwrapped ]; doCheck = false; # no tests pythonImportsCheck = [ "mjolnir" ]; diff --git a/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix b/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix index a6e22db34fe5..b5be02a4b21a 100644 --- a/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix +++ b/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse, twisted }: +{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped, twisted }: buildPythonPackage rec { pname = "matrix-synapse-shared-secret-auth"; @@ -14,7 +14,7 @@ buildPythonPackage rec { doCheck = false; pythonImportsCheck = [ "shared_secret_authenticator" ]; - buildInputs = [ matrix-synapse ]; + buildInputs = [ matrix-synapse-unwrapped ]; propagatedBuildInputs = [ twisted ]; meta = with lib; { diff --git a/pkgs/servers/matrix-synapse/wrapper.nix b/pkgs/servers/matrix-synapse/wrapper.nix new file mode 100644 index 000000000000..930142c5f55e --- /dev/null +++ b/pkgs/servers/matrix-synapse/wrapper.nix @@ -0,0 +1,44 @@ +{ lib +, stdenv +, makeWrapper +, matrix-synapse-unwrapped +, extras ? [ + "postgres" + "url-preview" + "user-search" + ] ++ lib.optional (lib.meta.availableOn stdenv.hostPlatform matrix-synapse-unwrapped.python.pkgs.systemd) "systemd" +, plugins ? [ ] +, ... +}: + +let + extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) (lib.unique extras); + + pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override { + extraLibs = plugins; + }; + + searchPath = lib.makeSearchPathOutput "lib" matrix-synapse-unwrapped.python.sitePackages (extraPackages ++ [ pluginsEnv ]); +in +stdenv.mkDerivation { + name = (lib.appendToName "wrapped" matrix-synapse-unwrapped).name; + + nativeBuildInputs = [ + makeWrapper + ]; + + buildCommand = '' + for bin in ${matrix-synapse-unwrapped}/bin/*; do + echo $bin + makeWrapper "$bin" "$out/bin/$(basename $bin)" \ + --set PYTHONPATH ${searchPath} + done; + ''; + + passthru = { + unwrapped = matrix-synapse-unwrapped; + + # for backward compatibility + inherit (matrix-synapse-unwrapped) plugins tools; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 70fffbe6dc38..0ce71f677689 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9839,11 +9839,10 @@ with pkgs; matrix-sliding-sync = callPackage ../servers/matrix-synapse/sliding-sync { }; - matrix-synapse = callPackage ../servers/matrix-synapse { }; - - matrix-synapse-plugins = recurseIntoAttrs matrix-synapse.plugins; - - matrix-synapse-tools = recurseIntoAttrs matrix-synapse.tools; + matrix-synapse = callPackage ../servers/matrix-synapse/wrapper.nix { }; + matrix-synapse-unwrapped = callPackage ../servers/matrix-synapse/default.nix { }; + matrix-synapse-plugins = recurseIntoAttrs matrix-synapse-unwrapped.plugins; + matrix-synapse-tools = recurseIntoAttrs matrix-synapse-unwrapped.tools; matrix-appservice-irc = callPackage ../servers/matrix-synapse/matrix-appservice-irc { };