From aea4ba847a8ebaa452424ce7654f5127e37efb17 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 3 Mar 2024 11:48:15 +0100 Subject: [PATCH] postgresql: remove thisAttr argument by calling tests directly Previously, it was not possible to run tests on an overridden derivation, because the derivation under test was always pulled from pkgs. With this change, the following will return the same test: postgresql_jit.tests and (postgresql.override { jitSupport = true; }).tests --- nixos/tests/postgresql-jit.nix | 15 +- nixos/tests/postgresql-wal-receiver.nix | 205 ++++++++++++------------ pkgs/servers/sql/postgresql/default.nix | 1 - pkgs/servers/sql/postgresql/generic.nix | 14 +- 4 files changed, 128 insertions(+), 107 deletions(-) diff --git a/nixos/tests/postgresql-jit.nix b/nixos/tests/postgresql-jit.nix index baf26b8da2b3..f4b1d07a7faf 100644 --- a/nixos/tests/postgresql-jit.nix +++ b/nixos/tests/postgresql-jit.nix @@ -1,6 +1,7 @@ { system ? builtins.currentSystem , config ? {} , pkgs ? import ../.. { inherit system config; } +, package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,14 +10,17 @@ let inherit (pkgs) lib; packages = builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs); - mkJitTest = packageName: makeTest { - name = "${packageName}"; + mkJitTestFromName = name: + mkJitTest pkgs.${name}; + + mkJitTest = package: makeTest { + name = package.name; meta.maintainers = with lib.maintainers; [ ma27 ]; nodes.machine = { pkgs, lib, ... }: { services.postgresql = { + inherit package; enable = true; enableJIT = true; - package = pkgs.${packageName}; initialScript = pkgs.writeText "init.sql" '' create table demo (id int); insert into demo (id) select generate_series(1, 5); @@ -45,4 +49,7 @@ let ''; }; in -lib.genAttrs packages mkJitTest +if package == null then + lib.genAttrs packages mkJitTestFromName +else + mkJitTest package diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix index b0bd7711dbcd..33c3945f0f79 100644 --- a/nixos/tests/postgresql-wal-receiver.nix +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -1,6 +1,7 @@ { system ? builtins.currentSystem, config ? {}, - pkgs ? import ../.. { inherit system config; } + pkgs ? import ../.. { inherit system config; }, + package ? null }: with import ../lib/testing-python.nix { inherit system pkgs; }; @@ -9,111 +10,117 @@ let lib = pkgs.lib; # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`. - makePostgresqlWalReceiverTest = postgresqlPackage: + makeTestAttribute = name: { - name = postgresqlPackage; - value = - let - pkg = pkgs."${postgresqlPackage}"; - postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; - replicationUser = "wal_receiver_user"; - replicationSlot = "wal_receiver_slot"; - replicationConn = "postgresql://${replicationUser}@localhost"; - baseBackupDir = "/tmp/pg_basebackup"; - walBackupDir = "/tmp/pg_wal"; - atLeast12 = lib.versionAtLeast pkg.version "12.0"; + inherit name; + value = makePostgresqlWalReceiverTest pkgs."${name}"; + }; - recoveryFile = if atLeast12 - then pkgs.writeTextDir "recovery.signal" "" - else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'"; + makePostgresqlWalReceiverTest = pkg: + let + postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}"; + replicationUser = "wal_receiver_user"; + replicationSlot = "wal_receiver_slot"; + replicationConn = "postgresql://${replicationUser}@localhost"; + baseBackupDir = "/tmp/pg_basebackup"; + walBackupDir = "/tmp/pg_wal"; + atLeast12 = lib.versionAtLeast pkg.version "12.0"; - in makeTest { - name = "postgresql-wal-receiver-${postgresqlPackage}"; - meta.maintainers = with lib.maintainers; [ pacien ]; + recoveryFile = if atLeast12 + then pkgs.writeTextDir "recovery.signal" "" + else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'"; - nodes.machine = { ... }: { - services.postgresql = { - package = pkg; - enable = true; - settings = lib.mkMerge [ - { - wal_level = "archive"; # alias for replica on pg >= 9.6 - max_wal_senders = 10; - max_replication_slots = 10; - } - (lib.mkIf atLeast12 { - restore_command = "cp ${walBackupDir}/%f %p"; - recovery_end_command = "touch recovery.done"; - }) - ]; - authentication = '' - host replication ${replicationUser} all trust - ''; - initialScript = pkgs.writeText "init.sql" '' - create user ${replicationUser} replication; - select * from pg_create_physical_replication_slot('${replicationSlot}'); - ''; - }; + in makeTest { + name = "postgresql-wal-receiver-${pkg.name}"; + meta.maintainers = with lib.maintainers; [ pacien ]; - services.postgresqlWalReceiver.receivers.main = { - postgresqlPackage = pkg; - connection = replicationConn; - slot = replicationSlot; - directory = walBackupDir; - }; - # This is only to speedup test, it isn't time racing. Service is set to autorestart always, - # default 60sec is fine for real system, but is too much for a test - systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; + nodes.machine = { ... }: { + services.postgresql = { + package = pkg; + enable = true; + settings = lib.mkMerge [ + { + wal_level = "archive"; # alias for replica on pg >= 9.6 + max_wal_senders = 10; + max_replication_slots = 10; + } + (lib.mkIf atLeast12 { + restore_command = "cp ${walBackupDir}/%f %p"; + recovery_end_command = "touch recovery.done"; + }) + ]; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; }; - testScript = '' - # make an initial base backup - machine.wait_for_unit("postgresql") - machine.wait_for_unit("postgresql-wal-receiver-main") - # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other - # required only for 9.4 - machine.sleep(5) - machine.succeed( - "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" - ) - - # create a dummy table with 100 records - machine.succeed( - "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" - ) - - # stop postgres and destroy data - machine.systemctl("stop postgresql") - machine.systemctl("stop postgresql-wal-receiver-main") - machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") - - # restore the base backup - machine.succeed( - "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" - ) - - # prepare WAL and recovery - machine.succeed("chmod a+rX -R ${walBackupDir}") - machine.execute( - "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" - ) # make use of partial segments too - machine.succeed( - "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" - ) - - # replay WAL - machine.systemctl("start postgresql") - machine.wait_for_file("${postgresqlDataDir}/recovery.done") - machine.systemctl("restart postgresql") - machine.wait_for_unit("postgresql") - - # check that our records have been restored - machine.succeed( - "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" - ) - ''; + services.postgresqlWalReceiver.receivers.main = { + postgresqlPackage = pkg; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + # This is only to speedup test, it isn't time racing. Service is set to autorestart always, + # default 60sec is fine for real system, but is too much for a test + systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5; }; + + testScript = '' + # make an initial base backup + machine.wait_for_unit("postgresql") + machine.wait_for_unit("postgresql-wal-receiver-main") + # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other + # required only for 9.4 + machine.sleep(5) + machine.succeed( + "${pkg}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}" + ) + + # create a dummy table with 100 records + machine.succeed( + "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'" + ) + + # stop postgres and destroy data + machine.systemctl("stop postgresql") + machine.systemctl("stop postgresql-wal-receiver-main") + machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}") + + # restore the base backup + machine.succeed( + "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}" + ) + + # prepare WAL and recovery + machine.succeed("chmod a+rX -R ${walBackupDir}") + machine.execute( + "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done" + ) # make use of partial segments too + machine.succeed( + "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*" + ) + + # replay WAL + machine.systemctl("start postgresql") + machine.wait_for_file("${postgresqlDataDir}/recovery.done") + machine.systemctl("restart postgresql") + machine.wait_for_unit("postgresql") + + # check that our records have been restored + machine.succeed( + "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100" + ) + ''; }; -# Maps the generic function over all attributes of PostgreSQL packages -in builtins.listToAttrs (map makePostgresqlWalReceiverTest (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) +in +if package == null then + # all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages + builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs))) +else + # Called directly from .tests + makePostgresqlWalReceiverTest package diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index d11a2d06b2d2..ff9f4f4150a2 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -15,7 +15,6 @@ let in self.lib.nameValuePair attrName (import path { inherit jitSupport self; - thisAttr = attrName; }) ) versions; diff --git a/pkgs/servers/sql/postgresql/generic.nix b/pkgs/servers/sql/postgresql/generic.nix index 7f9e561f877b..863d8c3b4e2f 100644 --- a/pkgs/servers/sql/postgresql/generic.nix +++ b/pkgs/servers/sql/postgresql/generic.nix @@ -19,7 +19,7 @@ let , version, hash, muslPatches # for tests - , testers, nixosTests, thisAttr + , testers, nixosTests # JIT , jitSupport @@ -254,10 +254,18 @@ let this.pkgs; tests = { - postgresql = nixosTests.postgresql-wal-receiver.${thisAttr}; + postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; } // lib.optionalAttrs jitSupport { - postgresql-jit = nixosTests.postgresql-jit.${thisAttr}; + postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; }; };