colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #221318 from mweinelt/synapse-extras

Browse Source 
nixos/matrix-synapse: Allow passing extras, discover extras from config
This commit is contained in:
Maximilian Bosch 2023-08-14 18:43:05 +02:00 committed by GitHub
commit a471325eaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 219 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nixos/doc/manual/release-notes/rl-2311.section.md
View File

@ -68,6 +68,13 @@
- The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`.
- The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though:
- The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below.
- The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed.
- These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["redis"]; }` for `hiredis` & `txredisapi` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if `services.matrix-synapse.settings.redis.enabled` is set to `true`, `"redis"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`.
- A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras).
- In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only.
- `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides
- `consul` has been updated to `1.16.0`. See the [release note](https://github.com/hashicorp/consul/releases/tag/v1.16.0) for more details. Once a new Consul version has started and upgraded its data directory, it generally cannot be downgraded to the previous version.

87
nixos/modules/services/matrix/synapse.nix
View File

@ -9,11 +9,6 @@ let
# remove null values from the final configuration
finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
configFile = format.generate "homeserver.yaml" finalSettings;
logConfigFile = format.generate "log_config.yaml" cfg.logConfig;
pluginsEnv = cfg.package.python.buildEnv.override {
extraLibs = cfg.plugins;
};
usePostgresql = cfg.settings.database.name == "psycopg2";
hasLocalPostgresDB = let args = cfg.settings.database.args; in
@ -50,6 +45,29 @@ let
"${bindAddress}"
}:${builtins.toString listener.port}/"
'';
defaultExtras = [
"systemd"
"postgres"
"url-preview"
"user-search"
];
wantedExtras = cfg.extras
++ lib.optional (cfg.settings ? oidc_providers) "oidc"
++ lib.optional (cfg.settings ? jwt_config) "jwt"
++ lib.optional (cfg.settings ? saml2_config) "saml2"
++ lib.optional (cfg.settings ? opentracing) "opentracing"
++ lib.optional (cfg.settings ? redis) "redis"
++ lib.optional (cfg.settings ? sentry) "sentry"
++ lib.optional (cfg.settings ? user_directory) "user-search"
++ lib.optional (cfg.settings.url_preview_enabled) "url-preview"
++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres";
wrapped = pkgs.matrix-synapse.override {
extras = wantedExtras;
inherit (cfg) plugins;
};
in {
imports = [
@ -151,10 +169,53 @@ in {
package = mkOption {
type = types.package;
default = pkgs.matrix-synapse;
defaultText = literalExpression "pkgs.matrix-synapse";
readOnly = true;
description = lib.mdDoc ''
Overridable attribute of the matrix synapse server package to use.
Reference to the `matrix-synapse` wrapper with all extras
(e.g. for `oidc` or `saml2`) added to the `PYTHONPATH` of all executables.
This option is useful to reference the "final" `matrix-synapse` package that's
actually used by `matrix-synapse.service`. For instance, when using
workers, it's possible to run
`''${config.services.matrix-synapse.package}/bin/synapse_worker` and
no additional PYTHONPATH needs to be specified for extras or plugins configured
via `services.matrix-synapse`.
However, this means that this option is supposed to be only declared
by the `services.matrix-synapse` module itself and is thus read-only.
In order to modify `matrix-synapse` itself, use an overlay to override
`pkgs.matrix-synapse-unwrapped`.
'';
};
extras = mkOption {
type = types.listOf (types.enum (lib.attrNames pkgs.matrix-synapse-unwrapped.optional-dependencies));
default = defaultExtras;
example = literalExpression ''
[
"cache-memory" # Provide statistics about caching memory consumption
"jwt" # JSON Web Token authentication
"opentracing" # End-to-end tracing support using Jaeger
"oidc" # OpenID Connect authentication
"postgres" # PostgreSQL database backend
"redis" # Redis support for the replication stream between worker processes
"saml2" # SAML2 authentication
"sentry" # Error tracking and performance metrics
"systemd" # Provide the JournalHandler used in the default log_config
"url-preview" # Support for oEmbed URL previews
"user-search" # Support internationalized domain names in user-search
]
'';
description = lib.mdDoc ''
Explicitly install extras provided by matrix-synapse. Most
will require some additional configuration.
Extras will automatically be enabled, when the relevant
configuration sections are present.
Please note that this option is additive: i.e. when adding a new item
to this list, the defaults are still kept. To override the defaults as well,
use `lib.mkForce`.
'';
};
@ -193,7 +254,7 @@ in {
default = {};
description = mdDoc ''
The primary synapse configuration. See the
[sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml)
[sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml)
for possible values.
Secrets should be passed in by using the `extraConfigFiles` option.
@ -706,6 +767,10 @@ in {
];
services.matrix-synapse.configFile = configFile;
services.matrix-synapse.package = wrapped;
# default them, so they are additive
services.matrix-synapse.extras = defaultExtras;
users.users.matrix-synapse = {
group = "matrix-synapse";
@ -729,9 +794,7 @@ in {
--keys-directory ${cfg.dataDir} \
--generate-keys
'';
environment = {
PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
} // optionalAttrs (cfg.withJemalloc) {
environment = optionalAttrs (cfg.withJemalloc) {
LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
};
serviceConfig = {

15
nixos/tests/matrix/synapse.nix
View File

@ -65,7 +65,7 @@ in {
nodes = {
# Since 0.33.0, matrix-synapse doesn't allow underscores in server names
serverpostgres = { pkgs, nodes, ... }: let
serverpostgres = { pkgs, nodes, config, ... }: let
mailserverIP = nodes.mailserver.config.networking.primaryIPAddress;
in
{
@ -77,6 +77,11 @@ in {
name = "psycopg2";
args.password = "synapse";
};
redis = {
enabled = true;
host = "localhost";
port = config.services.redis.servers.matrix-synapse.port;
};
tls_certificate_path = "${cert}";
tls_private_key_path = "${key}";
registration_shared_secret = registrationSharedSecret;
@ -107,6 +112,11 @@ in {
'';
};
services.redis.servers.matrix-synapse = {
enable = true;
port = 6380;
};
networking.extraHosts = ''
${mailserverIP} ${mailerDomain}
'';
@ -208,6 +218,9 @@ in {
serverpostgres.wait_until_succeeds(
"curl --fail -L --cacert ${ca_pem} https://localhost:8448/"
)
serverpostgres.wait_until_succeeds(
"journalctl -u matrix-synapse.service | grep -q 'Connected to redis'"
)
serverpostgres.require_unit_state("postgresql.service")
serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} https://localhost:8448/")
serverpostgres.succeed("obtain-token-and-register-email")

90
pkgs/servers/matrix-synapse/default.nix
View File

@ -1,7 +1,12 @@
{ lib, stdenv, fetchFromGitHub, python3, openssl, cargo, rustPlatform, rustc
, enableSystemd ? lib.meta.availableOn stdenv.hostPlatform python3.pkgs.systemd
{ lib
, stdenv
, fetchFromGitHub
, python3
, openssl
, cargo
, rustPlatform
, rustc
, nixosTests
, enableRedis ? true
, callPackage
}:
@ -9,8 +14,7 @@ let
plugins = python3.pkgs.callPackage ./plugins { };
tools = callPackage ./tools { };
in
with python3.pkgs;
buildPythonApplication rec {
python3.pkgs.buildPythonApplication rec {
pname = "matrix-synapse";
version = "1.89.0";
format = "pyproject";
@ -34,7 +38,7 @@ buildPythonApplication rec {
sed -i '/^setuptools_rust =/d' pyproject.toml
'';
nativeBuildInputs = [
nativeBuildInputs = with python3.pkgs; [
poetry-core
rustPlatform.cargoSetupHook
setuptools-rust
@ -42,47 +46,90 @@ buildPythonApplication rec {
rustc
];
buildInputs = [ openssl ];
buildInputs = [
openssl
];
propagatedBuildInputs = [
authlib
propagatedBuildInputs = with python3.pkgs; [
attrs
bcrypt
bleach
canonicaljson
daemonize
cryptography
ijson
immutabledict
jinja2
jsonschema
lxml
matrix-common
msgpack
netaddr
packaging
phonenumbers
pillow
prometheus-client
psutil
psycopg2
pyasn1
pyasn1-modules
pydantic
pyicu
pymacaroons
pynacl
pyopenssl
pysaml2
pyyaml
requests
setuptools
service-identity
signedjson
sortedcontainers
treq
twisted
typing-extensions
unpaddedbase64
] ++ lib.optional enableSystemd systemd
++ lib.optionals enableRedis [ hiredis txredisapi ];
]
++ twisted.optional-dependencies.tls;
nativeCheckInputs = [ mock parameterized openssl ];
passthru.optional-dependencies = with python3.pkgs; {
postgres = if isPyPy then [
psycopg2cffi
] else [
psycopg2
];
saml2 = [
pysaml2
];
oidc = [
authlib
];
systemd = [
systemd
];
url-preview = [
lxml
];
sentry = [
sentry-sdk
];
opentracing = [
jaeger-client
opentracing
];
jwt = [
authlib
];
redis = [
hiredis
txredisapi
];
cache-memory = [
pympler
];
user-search = [
pyicu
];
};
nativeCheckInputs = [
openssl
] ++ (with python3.pkgs; [
mock
parameterized
])
++ lib.flatten (lib.attrValues passthru.optional-dependencies);
doCheck = !stdenv.isDarwin;
@ -112,6 +159,7 @@ buildPythonApplication rec {
meta = with lib; {
homepage = "https://matrix.org";
changelog = "https://github.com/matrix-org/synapse/releases/tag/v${version}";
description = "Matrix reference homeserver";
license = licenses.asl20;
maintainers = teams.matrix.members;

4
pkgs/servers/matrix-synapse/plugins/ldap3.nix
View File

@ -4,7 +4,7 @@
, fetchPypi
, ldap3
, ldaptor
, matrix-synapse
, matrix-synapse-unwrapped
, pytestCheckHook
, service-identity
, setuptools
@ -33,7 +33,7 @@ buildPythonPackage rec {
propagatedBuildInputs = [ service-identity ldap3 twisted ];
nativeCheckInputs = [ ldaptor matrix-synapse pytestCheckHook ];
nativeCheckInputs = [ ldaptor matrix-synapse-unwrapped pytestCheckHook ];
pythonImportsCheck = [ "ldap_auth_provider" ];

4
pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix
View File

@ -1,4 +1,4 @@
{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse }:
{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped }:
buildPythonPackage rec {
pname = "matrix-synapse-mjolnir-antispam";
@ -13,7 +13,7 @@ buildPythonPackage rec {
sourceRoot = "./${src.name}/synapse_antispam";
buildInputs = [ matrix-synapse ];
buildInputs = [ matrix-synapse-unwrapped ];
doCheck = false; # no tests
pythonImportsCheck = [ "mjolnir" ];

4
pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix
View File

@ -1,4 +1,4 @@
{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse, twisted }:
{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped, twisted }:
buildPythonPackage rec {
pname = "matrix-synapse-shared-secret-auth";
@ -14,7 +14,7 @@ buildPythonPackage rec {
doCheck = false;
pythonImportsCheck = [ "shared_secret_authenticator" ];
buildInputs = [ matrix-synapse ];
buildInputs = [ matrix-synapse-unwrapped ];
propagatedBuildInputs = [ twisted ];
meta = with lib; {

44
pkgs/servers/matrix-synapse/wrapper.nix Normal file
View File

@ -0,0 +1,44 @@
{ lib
, stdenv
, makeWrapper
, matrix-synapse-unwrapped
, extras ? [
"postgres"
"url-preview"
"user-search"
] ++ lib.optional (lib.meta.availableOn stdenv.hostPlatform matrix-synapse-unwrapped.python.pkgs.systemd) "systemd"
, plugins ? [ ]
, ...
}:
let
extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) (lib.unique extras);
pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override {
extraLibs = plugins;
};
searchPath = lib.makeSearchPathOutput "lib" matrix-synapse-unwrapped.python.sitePackages (extraPackages ++ [ pluginsEnv ]);
in
stdenv.mkDerivation {
name = (lib.appendToName "wrapped" matrix-synapse-unwrapped).name;
nativeBuildInputs = [
makeWrapper
];
buildCommand = ''
for bin in ${matrix-synapse-unwrapped}/bin/*; do
echo $bin
makeWrapper "$bin" "$out/bin/$(basename $bin)" \
--set PYTHONPATH ${searchPath}
done;
'';
passthru = {
unwrapped = matrix-synapse-unwrapped;
# for backward compatibility
inherit (matrix-synapse-unwrapped) plugins tools;
};
}

9
pkgs/top-level/all-packages.nix
View File

@ -9839,11 +9839,10 @@ with pkgs;
matrix-sliding-sync = callPackage ../servers/matrix-synapse/sliding-sync { };
matrix-synapse = callPackage ../servers/matrix-synapse { };
matrix-synapse-plugins = recurseIntoAttrs matrix-synapse.plugins;
matrix-synapse-tools = recurseIntoAttrs matrix-synapse.tools;
matrix-synapse = callPackage ../servers/matrix-synapse/wrapper.nix { };
matrix-synapse-unwrapped = callPackage ../servers/matrix-synapse/default.nix { };
matrix-synapse-plugins = recurseIntoAttrs matrix-synapse-unwrapped.plugins;
matrix-synapse-tools = recurseIntoAttrs matrix-synapse-unwrapped.tools;
matrix-appservice-irc = callPackage ../servers/matrix-synapse/matrix-appservice-irc { };