nixos/grub: Name initrd-secrets by system, not by initrd

Previously, secrets were named according to the initrd they were
associated with. This created a problem: If secrets were changed whilst
the initrd remained the same, there were two versions of the secrets
with one initrd. The result was that only one version of the secrets would
by recorded into the /boot partition and get used. AFAICT this would
only be the oldest version of the secrets for the given initrd version.

This manifests as #114594, which I found frustrating while trying to use
initrd secrets for the first time. While developing the secrets I found
I could not get new versions of the secrets to take effect.
Additionally, it's a nasty issue to run into if you had cause to change
the initrd secrets for credential rotation, etc, if you change them and
discover you cannot, or alternatively that you can't roll back as you
would expect.

Additional changes in this patch.

* Add a regression test that switching to another grub configuration
  with the alternate secrets works. This test relies on the fact that it
  is not changing the initrd. I have checked that the test fails if I
  undo my change.

* Persist the useBootLoader disk state, similarly to other boot state.
  * I had to do this, otherwise I could not find a route to testing the
    alternate boot configuration. I did attempt a few different ways of
    testing this, including directly running install-grub.pl, but what
    I've settled on is most like what a user would do and avoids
    depending on lots of internal details.
  * Making tests that test the boot are a bit tricky (see hibernate.nix
    and installer.nix for inspiration), I found that in addition to
    having to copy quite a bit of code I still couldn't get things to
    work as desired since the bootloader state was being clobbered.

My change to persist the useBootLoader state could break things,
conceptually. I need some help here discovering if that is the case,
possibly by letting this run through a staging CI if there is one.

Fix #114594.

cc potential reviewers:

@lopsided98 (original implementer) @joachifm (original reviewer),
@wkennington (numerous fixes to grub-install.pl), @lheckemann (wrote
original secrets test).

nixos/modules/system/boot/loader/grub/install-grub.pl

@@ -450,8 +450,9 @@ sub addEntry {
 
     # Include second initrd with secrets
     if (-e -x "$path/append-initrd-secrets") {
-        my $initrdName = basename($initrd);
-        my $initrdSecretsPath = "$bootPath/kernels/$initrdName-secrets";
+        # Name the initrd secrets after the system from which they're derived.
+        my $systemName = basename(Cwd::abs_path("$path"));
+        my $initrdSecretsPath = "$bootPath/kernels/$systemName-secrets";
 
         mkpath(dirname($initrdSecretsPath), 0, 0755);
         my $oldUmask = umask;
@@ -463,7 +464,7 @@ sub addEntry {
         if (-e $initrdSecretsPathTemp && ! -z _) {
             rename $initrdSecretsPathTemp, $initrdSecretsPath or die "failed to move initrd secrets into place: $!\n";
             $copied{$initrdSecretsPath} = 1;
-            $initrd .= " " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/kernels/$initrdName-secrets";
+            $initrd .= " " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/kernels/$systemName-secrets";
         } else {
             unlink $initrdSecretsPathTemp;
             rmdir dirname($initrdSecretsPathTemp);

nixos/modules/virtualisation/qemu-vm.nix

@@ -152,9 +152,11 @@ let
 
       ${lib.optionalString cfg.useBootLoader
       ''
-        # Create a writable copy/snapshot of the boot disk.
-        # A writable boot disk can be booted from automatically.
-        ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${bootDisk}/disk.img "$TMPDIR/disk.img"
+        if ${if !cfg.persistBootDevice then "true" else "! test -e $TMPDIR/disk.img"}; then
+          # Create a writable copy/snapshot of the boot disk.
+          # A writable boot disk can be booted from automatically.
+          ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${bootDisk}/disk.img "$TMPDIR/disk.img"
+        fi
 
         NIX_EFI_VARS=$(readlink -f "''${NIX_EFI_VARS:-${cfg.efiVars}}")
 
@@ -367,6 +369,17 @@ in
           '';
       };
 
+    virtualisation.persistBootDevice =
+      mkOption {
+        type = types.bool;
+        default = false;
+        description =
+          lib.mdDoc ''
+            If useBootLoader is specified, whether to recreate the boot device
+            on each instantiaton or allow it to persist.
+            '';
+      };
+
     virtualisation.emptyDiskImages =
       mkOption {
         type = types.listOf types.ints.positive;
@@ -835,6 +848,8 @@ in
     # * The disks are attached in `virtualisation.qemu.drives`.
     #   Their order makes them appear as devices `a`, `b`, etc.
     # * `fileSystems."/boot"` is adjusted to be on device `b`.
+    # * The disk.img is recreated each time the VM is booted unless
+    #   virtualisation.persistBootDevice is set.
 
     # If `useBootLoader`, GRUB goes to the second disk, see
     # note [Disk layout with `useBootLoader`].

nixos/tests/all-tests.nix

@@ -279,6 +279,7 @@ in {
   initrd-network-ssh = handleTest ./initrd-network-ssh {};
   initrdNetwork = handleTest ./initrd-network.nix {};
   initrd-secrets = handleTest ./initrd-secrets.nix {};
+  initrd-secrets-changing = handleTest ./initrd-secrets-changing.nix {};
   input-remapper = handleTest ./input-remapper.nix {};
   inspircd = handleTest ./inspircd.nix {};
   installer = handleTest ./installer.nix {};

nixos/tests/initrd-secrets-changing.nix

@@ -0,0 +1,58 @@
+{ system ? builtins.currentSystem
+, config ? {}
+, pkgs ? import ../.. { inherit system config; }
+, lib ? pkgs.lib
+, testing ? import ../lib/testing-python.nix { inherit system pkgs; }
+}:
+
+let
+  secret1InStore = pkgs.writeText "topsecret" "iamasecret1";
+  secret2InStore = pkgs.writeText "topsecret" "iamasecret2";
+in
+
+testing.makeTest {
+  name = "initrd-secrets-changing";
+
+  nodes.machine = { ... }: {
+    virtualisation.useBootLoader = true;
+    virtualisation.persistBootDevice = true;
+
+    boot.loader.grub.device = "/dev/vda";
+
+    boot.initrd.secrets = {
+      "/test" = secret1InStore;
+      "/run/keys/test" = secret1InStore;
+    };
+    boot.initrd.postMountCommands = "cp /test /mnt-root/secret-from-initramfs";
+
+    specialisation.secrets2System.configuration = {
+      boot.initrd.secrets = lib.mkForce {
+        "/test" = secret2InStore;
+        "/run/keys/test" = secret2InStore;
+      };
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    machine.wait_for_unit("multi-user.target")
+    print(machine.succeed("cat /run/keys/test"))
+    machine.succeed(
+        "cmp ${secret1InStore} /secret-from-initramfs",
+        "cmp ${secret1InStore} /run/keys/test",
+    )
+    # Select the second boot entry corresponding to the specialisation secrets2System.
+    machine.succeed("grub-reboot 1")
+    machine.shutdown()
+
+    with subtest("Check that the specialisation's secrets are distinct despite identical kernels"):
+        machine.wait_for_unit("multi-user.target")
+        print(machine.succeed("cat /run/keys/test"))
+        machine.succeed(
+            "cmp ${secret2InStore} /secret-from-initramfs",
+            "cmp ${secret2InStore} /run/keys/test",
+        )
+        machine.shutdown()
+  '';
+}