From 77bc27ccdb87b1a3400fc59f097f8218982a55ed Mon Sep 17 00:00:00 2001 From: Robert Obryk Date: Sun, 4 Dec 2022 21:40:54 +0100 Subject: [PATCH] nixos/backups/restic: handle cases when both dynamicFileFrom and paths are set Also, add a test to verify that it works. This change also removes the part of custom package test that verifies that the correct paths are provided. This is already tested by restore tests. Before this change, setting both paths and dynamicFileFrom would cause paths to be silently ignored. Making that actually apply the obvious interpretation seems to me to be strictly better than prohibiting the two from being set at the same time. --- nixos/modules/services/backup/restic.nix | 25 +++++++++++++----------- nixos/tests/restic.nix | 18 ++++++++++++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index 1620770e5b56..9c5d3b984a87 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -113,12 +113,15 @@ in }; paths = mkOption { + # This is nullable for legacy reasons only. We should consider making it a pure listOf + # after some time has passed since this comment was added. type = types.nullOr (types.listOf types.str); - default = null; + default = [ ]; description = lib.mdDoc '' - Which paths to backup. If null or an empty array, no - backup command will be run. This can be used to create a - prune-only job. + Which paths to backup, in addition to ones specified via + `dynamicFilesFrom`. If null or an empty array and + `dynamicFilesFrom` is also null, no backup command will be run. + This can be used to create a prune-only job. ''; example = [ "/var/lib/postgresql" @@ -231,7 +234,7 @@ in description = lib.mdDoc '' A script that produces a list of files to back up. The results of this command are given to the '--files-from' - option. + option. The result is merged with paths specified via `paths`. ''; example = "find /home/matt/git -type d -name .git"; }; @@ -300,10 +303,7 @@ in resticCmd = "${backup.package}/bin/restic${extraOptions}"; excludeFlags = optional (backup.exclude != []) "--exclude-file=${pkgs.writeText "exclude-patterns" (concatStringsSep "\n" backup.exclude)}"; filesFromTmpFile = "/run/restic-backups-${name}/includes"; - backupPaths = - if (backup.dynamicFilesFrom == null) - then optionalString (backup.paths != null) (concatStringsSep " " backup.paths) - else "--files-from ${filesFromTmpFile}"; + doBackup = (backup.dynamicFilesFrom != null) || (backup.paths != null && backup.paths != []); pruneCmd = optionals (builtins.length backup.pruneOpts > 0) [ (resticCmd + " forget --prune " + (concatStringsSep " " backup.pruneOpts)) (resticCmd + " check " + (concatStringsSep " " backup.checkOpts)) @@ -335,7 +335,7 @@ in restartIfChanged = false; serviceConfig = { Type = "oneshot"; - ExecStart = (optionals (backupPaths != "") [ "${resticCmd} backup ${concatStringsSep " " (backup.extraBackupArgs ++ excludeFlags)} ${backupPaths}" ]) + ExecStart = (optionals doBackup [ "${resticCmd} backup ${concatStringsSep " " (backup.extraBackupArgs ++ excludeFlags)} --files-from=${filesFromTmpFile}" ]) ++ pruneCmd; User = backup.user; RuntimeDirectory = "restic-backups-${name}"; @@ -353,8 +353,11 @@ in ${optionalString (backup.initialize) '' ${resticCmd} snapshots || ${resticCmd} init ''} + ${optionalString (backup.paths != null && backup.paths != []) '' + cat ${pkgs.writeText "staticPaths" (concatStringsSep "\n" backup.paths)} >> ${filesFromTmpFile} + ''} ${optionalString (backup.dynamicFilesFrom != null) '' - ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} > ${filesFromTmpFile} + ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} >> ${filesFromTmpFile} ''} ''; } // optionalAttrs (backup.dynamicFilesFrom != null || backup.backupCleanupCommand != null) { diff --git a/nixos/tests/restic.nix b/nixos/tests/restic.nix index 626049e73341..a66919fded19 100644 --- a/nixos/tests/restic.nix +++ b/nixos/tests/restic.nix @@ -21,7 +21,10 @@ import ./make-test-python.nix ( unpackPhase = "true"; installPhase = '' mkdir $out - touch $out/some_file + echo some_file > $out/some_file + echo some_other_file > $out/some_other_file + mkdir $out/a_dir + echo a_file > $out/a_dir/a_file ''; }; @@ -53,9 +56,13 @@ import ./make-test-python.nix ( initialize = true; }; remote-from-file-backup = { - inherit passwordFile paths exclude pruneOpts; + inherit passwordFile exclude pruneOpts; initialize = true; repositoryFile = pkgs.writeText "repositoryFile" remoteFromFileRepository; + paths = [ "/opt/a_dir" ]; + dynamicFilesFrom = '' + find /opt -mindepth 1 -maxdepth 1 ! -name a_dir # all files in /opt except for a_dir + ''; }; rclonebackup = { inherit passwordFile paths exclude pruneOpts; @@ -123,13 +130,18 @@ import ./make-test-python.nix ( "systemctl start restic-backups-remote-from-file-backup.service", '${pkgs.restic}/bin/restic -r ${remoteFromFileRepository} -p ${passwordFile} snapshots --json | ${pkgs.jq}/bin/jq "length | . == 1"', + # test that restoring that snapshot produces the same directory + "mkdir /tmp/restore-2", + "${pkgs.restic}/bin/restic -r ${remoteRepository} -p ${passwordFile} restore latest -t /tmp/restore-2", + "diff -ru ${testDir} /tmp/restore-2/opt", + # test that rclonebackup produces a snapshot "systemctl start restic-backups-rclonebackup.service", '${pkgs.restic}/bin/restic -r ${rcloneRepository} -p ${passwordFile} snapshots --json | ${pkgs.jq}/bin/jq "length | . == 1"', # test that custompackage runs both `restic backup` and `restic check` with reasonable commandlines "systemctl start restic-backups-custompackage.service", - "grep 'backup.* /opt' /root/fake-restic.log", + "grep 'backup' /root/fake-restic.log", "grep 'check.* --some-check-option' /root/fake-restic.log", # test that we can create four snapshots in remotebackup and rclonebackup