From f8ba8be54b7277ee284850119dc156297633b6b3 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Tue, 19 Sep 2023 23:30:52 -0400 Subject: [PATCH 1/6] testing-instrumentation: Factor backdoor service out to variable --- .../modules/testing/test-instrumentation.nix | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix index c91e54f5a4d7..6c4b4f13d04b 100644 --- a/nixos/modules/testing/test-instrumentation.nix +++ b/nixos/modules/testing/test-instrumentation.nix @@ -7,48 +7,51 @@ with lib; let qemu-common = import ../../lib/qemu-common.nix { inherit lib pkgs; }; + + backdoorService = { + wantedBy = [ "multi-user.target" ]; + requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; + after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; + script = + '' + export USER=root + export HOME=/root + export DISPLAY=:0.0 + + source /etc/profile + + # Don't use a pager when executing backdoor + # actions. Because we use a tty, commands like systemctl + # or nix-store get confused into thinking they're running + # interactively. + export PAGER= + + cd /tmp + exec < /dev/hvc0 > /dev/hvc0 + while ! exec 2> /dev/${qemu-common.qemuSerialDevice}; do sleep 0.1; done + echo "connecting to host..." >&2 + stty -F /dev/hvc0 raw -echo # prevent nl -> cr/nl conversion + # The following line is essential since it signals to + # the test driver that the shell is ready. + # See: the connect method in the Machine class. + echo "Spawning backdoor root shell..." + # Passing the terminal device makes bash run non-interactively. + # Otherwise we get errors on the terminal because bash tries to + # setup things like job control. + # Note: calling bash explicitly here instead of sh makes sure that + # we can also run non-NixOS guests during tests. + PS1= exec /usr/bin/env bash --norc /dev/hvc0 + ''; + serviceConfig.KillSignal = "SIGHUP"; + }; + in { config = { - systemd.services.backdoor = - { wantedBy = [ "multi-user.target" ]; - requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; - after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; - script = - '' - export USER=root - export HOME=/root - export DISPLAY=:0.0 - - source /etc/profile - - # Don't use a pager when executing backdoor - # actions. Because we use a tty, commands like systemctl - # or nix-store get confused into thinking they're running - # interactively. - export PAGER= - - cd /tmp - exec < /dev/hvc0 > /dev/hvc0 - while ! exec 2> /dev/${qemu-common.qemuSerialDevice}; do sleep 0.1; done - echo "connecting to host..." >&2 - stty -F /dev/hvc0 raw -echo # prevent nl -> cr/nl conversion - # The following line is essential since it signals to - # the test driver that the shell is ready. - # See: the connect method in the Machine class. - echo "Spawning backdoor root shell..." - # Passing the terminal device makes bash run non-interactively. - # Otherwise we get errors on the terminal because bash tries to - # setup things like job control. - # Note: calling bash explicitly here instead of sh makes sure that - # we can also run non-NixOS guests during tests. - PS1= exec /usr/bin/env bash --norc /dev/hvc0 - ''; - serviceConfig.KillSignal = "SIGHUP"; - }; + systemd.services.backdoor = backdoorService # Prevent agetty from being instantiated on the serial device, since it # interferes with the backdoor (writes to it will randomly fail From 9a0f523372cf836662421ef2603bfb93f568f399 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Tue, 19 Sep 2023 23:39:51 -0400 Subject: [PATCH 2/6] systemd-stage-1: Enable backdoor in nixos tests --- .../manual/release-notes/rl-2311.section.md | 2 + nixos/lib/test-driver/test_driver/machine.py | 16 +++++ .../modules/testing/test-instrumentation.nix | 71 ++++++++++++++++--- nixos/tests/systemd-initrd-simple.nix | 12 ++-- 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 22bc4c31618e..e33f6b337e2d 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -554,3 +554,5 @@ The module update takes care of the new config syntax and the data itself (user - `teleport` has been upgraded from major version 12 to major version 14. Please see upstream [upgrade instructions](https://goteleport.com/docs/management/operations/upgrading/) and release notes for versions [13](https://goteleport.com/docs/changelog/#1300-050823) and [14](https://goteleport.com/docs/changelog/#1400-092023). Note that Teleport does not officially support upgrades across more than one major version at a time. If you're running Teleport server components, it is recommended to first upgrade to an intermediate 13.x version by setting `services.teleport.package = pkgs.teleport_13`. Afterwards, this option can be removed to upgrade to the default version (14). - The Linux kernel module `msr` (see [`msr(4)`](https://man7.org/linux/man-pages/man4/msr.4.html)), which provides an interface to read and write the model-specific registers (MSRs) of an x86 CPU, can now be configured via `hardware.cpu.x86.msr`. + +- There is a new NixOS option when writing NixOS tests `testing.initrdBackdoor`, that enables `backdoor.service` in initrd. Requires `boot.initrd.systemd.enable` to be enabled. Boot will pause in stage 1 at `initrd.target`, and will listen for commands from the `Machine` python interface, just like stage 2 normally does. This enables commands to be sent to test and debug stage 1. Use `machine.switch_root()` to leave stage 1 and proceed to stage 2. diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py index 529de41d892a..f430321bb607 100644 --- a/nixos/lib/test-driver/test_driver/machine.py +++ b/nixos/lib/test-driver/test_driver/machine.py @@ -1278,3 +1278,19 @@ class Machine: def run_callbacks(self) -> None: for callback in self.callbacks: callback() + + def switch_root(self) -> None: + """ + Transition from stage 1 to stage 2. This requires the + machine to be configured with `testing.initrdBackdoor = true` + and `boot.initrd.systemd.enable = true`. + """ + self.wait_for_unit("initrd.target") + self.execute( + "systemctl isolate --no-block initrd-switch-root.target 2>/dev/null >/dev/null", + check_return=False, + check_output=False, + ) + self.wait_for_console_text(r"systemd\[1\]:.*Switching root\.") + self.connected = False + self.connect() diff --git a/nixos/modules/testing/test-instrumentation.nix b/nixos/modules/testing/test-instrumentation.nix index 6c4b4f13d04b..abe68dd6eae6 100644 --- a/nixos/modules/testing/test-instrumentation.nix +++ b/nixos/modules/testing/test-instrumentation.nix @@ -6,10 +6,15 @@ with lib; let + cfg = config.testing; + qemu-common = import ../../lib/qemu-common.nix { inherit lib pkgs; }; backdoorService = { - wantedBy = [ "multi-user.target" ]; + wantedBy = [ "sysinit.target" ]; + unitConfig.DefaultDependencies = false; + conflicts = [ "shutdown.target" "initrd-switch-root.target" ]; + before = [ "shutdown.target" "initrd-switch-root.target" ]; requires = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; after = [ "dev-hvc0.device" "dev-${qemu-common.qemuSerialDevice}.device" ]; script = @@ -18,7 +23,9 @@ let export HOME=/root export DISPLAY=:0.0 - source /etc/profile + if [[ -e /etc/profile ]]; then + source /etc/profile + fi # Don't use a pager when executing backdoor # actions. Because we use a tty, commands like systemctl @@ -49,9 +56,59 @@ in { + options.testing = { + + initrdBackdoor = lib.mkEnableOption (lib.mdDoc '' + enable backdoor.service in initrd. Requires + boot.initrd.systemd.enable to be enabled. Boot will pause in + stage 1 at initrd.target, and will listen for commands from the + Machine python interface, just like stage 2 normally does. This + enables commands to be sent to test and debug stage 1. Use + machine.switch_root() to leave stage 1 and proceed to stage 2. + ''); + + }; + config = { - systemd.services.backdoor = backdoorService + assertions = [ + { + assertion = cfg.initrdBackdoor -> config.boot.initrd.systemd.enable; + message = '' + testing.initrdBackdoor requires boot.initrd.systemd.enable to be enabled. + ''; + } + ]; + + systemd.services.backdoor = backdoorService; + + boot.initrd.systemd = lib.mkMerge [ + { + contents."/etc/systemd/journald.conf".text = '' + [Journal] + ForwardToConsole=yes + MaxLevelConsole=debug + ''; + + extraConfig = config.systemd.extraConfig; + } + + (lib.mkIf cfg.initrdBackdoor { + # Implemented in machine.switch_root(). Suppress the unit by + # making it a noop without removing it, which would break + # initrd-parse-etc.service + services.initrd-cleanup.serviceConfig.ExecStart = [ + # Reset + "" + # noop + "/bin/true" + ]; + + services.backdoor = backdoorService; + + contents."/usr/bin/env".source = "${pkgs.coreutils}/bin/env"; + }) + ]; # Prevent agetty from being instantiated on the serial device, since it # interferes with the backdoor (writes to it will randomly fail @@ -107,12 +164,6 @@ in MaxLevelConsole=debug ''; - boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = '' - [Journal] - ForwardToConsole=yes - MaxLevelConsole=debug - ''; - systemd.extraConfig = '' # Don't clobber the console with duplicate systemd messages. ShowStatus=no @@ -126,8 +177,6 @@ in DefaultDeviceTimeoutSec=300 ''; - boot.initrd.systemd.extraConfig = config.systemd.extraConfig; - boot.consoleLogLevel = 7; # Prevent tests from accessing the Internet. diff --git a/nixos/tests/systemd-initrd-simple.nix b/nixos/tests/systemd-initrd-simple.nix index a6a22e9d48e0..2b7283a82193 100644 --- a/nixos/tests/systemd-initrd-simple.nix +++ b/nixos/tests/systemd-initrd-simple.nix @@ -2,16 +2,19 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: { name = "systemd-initrd-simple"; nodes.machine = { pkgs, ... }: { - boot.initrd.systemd = { - enable = true; - emergencyAccess = true; - }; + testing.initrdBackdoor = true; + boot.initrd.systemd.enable = true; virtualisation.fileSystems."/".autoResize = true; }; testScript = '' import subprocess + with subtest("testing initrd backdoor"): + machine.wait_for_unit("initrd.target") + machine.succeed("systemctl status initrd-fs.target") + machine.switch_root() + with subtest("handover to stage-2 systemd works"): machine.wait_for_unit("multi-user.target") machine.succeed("systemd-analyze | grep -q '(initrd)'") # direct handover @@ -37,6 +40,7 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: { subprocess.check_call(["qemu-img", "resize", "vm-state-machine/machine.qcow2", "+1G"]) machine.start() + machine.switch_root() newAvail = machine.succeed("df --output=avail / | sed 1d") assert int(oldAvail) < int(newAvail), "File system did not grow" From 90e26586937dc2bb5bb4c1a825ee1d8682a62d94 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 20 Sep 2023 05:58:30 -0400 Subject: [PATCH 3/6] nixos/tests/systemd-initrd-networkd: Separate into different tests --- nixos/tests/systemd-initrd-networkd.nix | 75 +++++++++++++++---------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/nixos/tests/systemd-initrd-networkd.nix b/nixos/tests/systemd-initrd-networkd.nix index 8376276d8f63..22a570520545 100644 --- a/nixos/tests/systemd-initrd-networkd.nix +++ b/nixos/tests/systemd-initrd-networkd.nix @@ -1,9 +1,19 @@ -import ./make-test-python.nix ({ pkgs, lib, ... }: { - name = "systemd-initrd-network"; - meta.maintainers = [ lib.maintainers.elvishjerricco ]; +{ system ? builtins.currentSystem +, config ? {} +, pkgs ? import ../.. { inherit system config; } +, lib ? pkgs.lib +}: - nodes = let - mkFlushTest = flush: script: { ... }: { +with import ../lib/testing-python.nix { inherit system pkgs; }; + +let + inherit (lib.maintainers) elvishjerricco; + + mkFlushTest = flush: script: makeTest { + name = "systemd-initrd-network-${lib.optionalString (!flush) "no-"}flush"; + meta.maintainers = [ elvishjerricco ]; + + nodes.machine = { boot.initrd.systemd.enable = true; boot.initrd.network = { enable = true; @@ -19,8 +29,18 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { inherit script; }; }; - in { - basic = { ... }: { + + testScript = '' + machine.wait_for_unit("multi-user.target") + ''; + }; + +in { + basic = makeTest { + name = "systemd-initrd-network"; + meta.maintainers = [ elvishjerricco ]; + + nodes.machine = { boot.initrd.network.enable = true; boot.initrd.systemd = { @@ -47,29 +67,26 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { }; }; - doFlush = mkFlushTest true '' - if ip addr | grep 10.0.2.15; then - echo "Network configuration survived switch-root; flushBeforeStage2 failed" - exit 1 - fi - ''; - - dontFlush = mkFlushTest false '' - if ! (ip addr | grep 10.0.2.15); then - echo "Network configuration didn't survive switch-root" - exit 1 - fi + testScript = '' + machine.wait_for_unit("multi-user.target") + # Make sure the systemd-network user was set correctly in initrd + machine.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]") + machine.succeed("ip addr show >&2") + machine.succeed("ip route show >&2") ''; }; - testScript = '' - start_all() - basic.wait_for_unit("multi-user.target") - doFlush.wait_for_unit("multi-user.target") - dontFlush.wait_for_unit("multi-user.target") - # Make sure the systemd-network user was set correctly in initrd - basic.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]") - basic.succeed("ip addr show >&2") - basic.succeed("ip route show >&2") + doFlush = mkFlushTest true '' + if ip addr | grep 10.0.2.15; then + echo "Network configuration survived switch-root; flushBeforeStage2 failed" + exit 1 + fi ''; -}) + + dontFlush = mkFlushTest false '' + if ! (ip addr | grep 10.0.2.15); then + echo "Network configuration didn't survive switch-root" + exit 1 + fi + ''; +} From 8900b027c73c389e4f3802225a790ef91e7a907f Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 20 Sep 2023 06:20:05 -0400 Subject: [PATCH 4/6] nixos/tests/systemd-initrd-networkd: Use initrdBackdoor --- nixos/tests/systemd-initrd-networkd.nix | 65 +++++++++++++------------ 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/nixos/tests/systemd-initrd-networkd.nix b/nixos/tests/systemd-initrd-networkd.nix index 22a570520545..9c4ddb6e4b36 100644 --- a/nixos/tests/systemd-initrd-networkd.nix +++ b/nixos/tests/systemd-initrd-networkd.nix @@ -9,16 +9,28 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; let inherit (lib.maintainers) elvishjerricco; + common = { + boot.initrd.systemd = { + enable = true; + network.wait-online.timeout = 10; + network.wait-online.anyInterface = true; + targets.network-online.requiredBy = [ "initrd.target" ]; + services.systemd-networkd-wait-online.requiredBy = + [ "network-online.target" ]; + initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ]; + }; + testing.initrdBackdoor = true; + boot.initrd.network.enable = true; + }; + mkFlushTest = flush: script: makeTest { name = "systemd-initrd-network-${lib.optionalString (!flush) "no-"}flush"; meta.maintainers = [ elvishjerricco ]; nodes.machine = { - boot.initrd.systemd.enable = true; - boot.initrd.network = { - enable = true; - flushBeforeStage2 = flush; - }; + imports = [ common ]; + + boot.initrd.network.flushBeforeStage2 = flush; systemd.services.check-flush = { requiredBy = ["multi-user.target"]; before = ["network-pre.target" "multi-user.target"]; @@ -31,6 +43,13 @@ let }; testScript = '' + machine.wait_for_unit("network-online.target") + machine.succeed( + "ip addr | grep 10.0.2.15", + "ping -c1 10.0.2.2", + ) + machine.switch_root() + machine.wait_for_unit("multi-user.target") ''; }; @@ -40,36 +59,18 @@ in { name = "systemd-initrd-network"; meta.maintainers = [ elvishjerricco ]; - nodes.machine = { - boot.initrd.network.enable = true; - - boot.initrd.systemd = { - enable = true; - # Enable network-online to fail the test in case of timeout - network.wait-online.timeout = 10; - network.wait-online.anyInterface = true; - targets.network-online.requiredBy = [ "initrd.target" ]; - services.systemd-networkd-wait-online.requiredBy = - [ "network-online.target" ]; - - initrdBin = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ]; - services.check = { - requiredBy = [ "initrd.target" ]; - before = [ "initrd.target" ]; - after = [ "network-online.target" ]; - serviceConfig.Type = "oneshot"; - path = [ pkgs.iproute2 pkgs.iputils pkgs.gnugrep ]; - script = '' - ip addr | grep 10.0.2.15 || exit 1 - ping -c1 10.0.2.2 || exit 1 - ''; - }; - }; - }; + nodes.machine = common; testScript = '' - machine.wait_for_unit("multi-user.target") + machine.wait_for_unit("network-online.target") + machine.succeed( + "ip addr | grep 10.0.2.15", + "ping -c1 10.0.2.2", + ) + machine.switch_root() + # Make sure the systemd-network user was set correctly in initrd + machine.wait_for_unit("multi-user.target") machine.succeed("[ $(stat -c '%U,%G' /run/systemd/netif/links) = systemd-network,systemd-network ]") machine.succeed("ip addr show >&2") machine.succeed("ip route show >&2") From b41cbee0fd2531b00369c787b20f5b20f94cfc15 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 20 Sep 2023 18:19:22 -0400 Subject: [PATCH 5/6] nixos/tests/systemd-initrd-networkd-ssh: Test with backdoor not LUKS --- nixos/tests/systemd-initrd-networkd-ssh.nix | 52 ++++++--------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/nixos/tests/systemd-initrd-networkd-ssh.nix b/nixos/tests/systemd-initrd-networkd-ssh.nix index 6aaa6c828f7b..d4c168f40e29 100644 --- a/nixos/tests/systemd-initrd-networkd-ssh.nix +++ b/nixos/tests/systemd-initrd-networkd-ssh.nix @@ -4,34 +4,16 @@ import ./make-test-python.nix ({ lib, ... }: { nodes = { server = { config, pkgs, ... }: { - environment.systemPackages = [ pkgs.cryptsetup ]; - boot.loader.systemd-boot.enable = true; - boot.loader.timeout = 0; - virtualisation = { - emptyDiskImages = [ 4096 ]; - useBootLoader = true; - # Booting off the encrypted disk requires an available init script from - # the Nix store - mountHostNixStore = true; - useEFIBoot = true; - }; - - specialisation.encrypted-root.configuration = { - virtualisation.rootDevice = "/dev/mapper/root"; - virtualisation.fileSystems."/".autoFormat = true; - boot.initrd.luks.devices = lib.mkVMOverride { - root.device = "/dev/vdb"; - }; - boot.initrd.systemd.enable = true; - boot.initrd.network = { + testing.initrdBackdoor = true; + boot.initrd.systemd.enable = true; + boot.initrd.systemd.contents."/etc/msg".text = "foo"; + boot.initrd.network = { + enable = true; + ssh = { enable = true; - ssh = { - enable = true; - authorizedKeys = [ (lib.readFile ./initrd-network-ssh/id_ed25519.pub) ]; - port = 22; - # Terrible hack so it works with useBootLoader - hostKeys = [ { outPath = "${./initrd-network-ssh/ssh_host_ed25519_key}"; } ]; - }; + authorizedKeys = [ (lib.readFile ./initrd-network-ssh/id_ed25519.pub) ]; + port = 22; + hostKeys = [ ./initrd-network-ssh/ssh_host_ed25519_key ]; }; }; }; @@ -63,24 +45,16 @@ import ./make-test-python.nix ({ lib, ... }: { status, _ = client.execute("nc -z server 22") return status == 0 - server.wait_for_unit("multi-user.target") - server.succeed( - "echo somepass | cryptsetup luksFormat --type=luks2 /dev/vdb", - "bootctl set-default nixos-generation-1-specialisation-encrypted-root.conf", - "sync", - ) - server.shutdown() - server.start() - client.wait_for_unit("network.target") with client.nested("waiting for SSH server to come up"): retry(ssh_is_up) - client.succeed( - "echo somepass | ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'systemd-tty-ask-password-agent' & exit" + msg = client.succeed( + "ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'cat /etc/msg'" ) + assert "foo" in msg + server.switch_root() server.wait_for_unit("multi-user.target") - server.succeed("mount | grep '/dev/mapper/root on /'") ''; }) From e9e2240763e409b9b634aa73712578be3fda6b1e Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 20 Sep 2023 06:26:03 -0400 Subject: [PATCH 6/6] nixos/tests/systemd-initrd-modprobe: Test parameter in stage 1 --- nixos/tests/systemd-initrd-modprobe.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixos/tests/systemd-initrd-modprobe.nix b/nixos/tests/systemd-initrd-modprobe.nix index bf635a10d0e9..0f93492176b4 100644 --- a/nixos/tests/systemd-initrd-modprobe.nix +++ b/nixos/tests/systemd-initrd-modprobe.nix @@ -2,6 +2,7 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: { name = "systemd-initrd-modprobe"; nodes.machine = { pkgs, ... }: { + testing.initrdBackdoor = true; boot.initrd.systemd.enable = true; boot.initrd.kernelModules = [ "loop" ]; # Load module in initrd. boot.extraModprobeConfig = '' @@ -10,6 +11,12 @@ import ./make-test-python.nix ({ lib, pkgs, ... }: { }; testScript = '' + machine.wait_for_unit("initrd.target") + max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop") + assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules" + + # Make sure it sticks in stage 2 + machine.switch_root() machine.wait_for_unit("multi-user.target") max_loop = machine.succeed("cat /sys/module/loop/parameters/max_loop") assert int(max_loop) == 42, "Parameter should be respected for initrd kernel modules"