diff --git a/TODO.md b/TODO.md index ed98fd9f2..e1381bf71 100644 --- a/TODO.md +++ b/TODO.md @@ -60,7 +60,6 @@ - has better multi-stream perf (e.g. `sane-sync-music` should be able to copy N items in parallel) ### security/resilience -- validate duplicity backups! - encrypt more ~ dirs (~/archives, ~/records, ..?) - best to do this after i know for sure i have good backups - /mnt/desko/home, etc, shouldn't include secrets (~/private) diff --git a/hosts/by-name/desko/default.nix b/hosts/by-name/desko/default.nix index 58e999fe9..1bcc8df75 100644 --- a/hosts/by-name/desko/default.nix +++ b/hosts/by-name/desko/default.nix @@ -28,7 +28,6 @@ sane.services.wg-home.ip = config.sane.hosts.by-name."desko".wg-home.ip; sane.ovpn.addrV4 = "172.26.55.21"; # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:20c1:a73c"; - sane.services.duplicity.enable = true; sane.services.rsync-net.enable = true; sane.nixcache.remote-builders.desko = false; diff --git a/hosts/by-name/servo/default.nix b/hosts/by-name/servo/default.nix index 7328c33ed..3f89603fd 100644 --- a/hosts/by-name/servo/default.nix +++ b/hosts/by-name/servo/default.nix @@ -31,7 +31,6 @@ # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:8df3:14b0"; sane.nixcache.remote-builders.desko = false; sane.nixcache.remote-builders.servo = false; - # sane.services.duplicity.enable = true; # TODO: re-enable after HW upgrade sane.services.rsync-net.enable = true; # automatically log in at the virtual consoles. diff --git a/hosts/common/programs/assorted.nix b/hosts/common/programs/assorted.nix index 9cec78166..d8909942f 100644 --- a/hosts/common/programs/assorted.nix +++ b/hosts/common/programs/assorted.nix @@ -109,9 +109,6 @@ in # "zfs" # doesn't cross-compile (requires samba) ]; sysadminExtraUtils = declPackageSet [ - "backblaze-b2" - "duplicity" - "sane-scripts.backup" "sqlite" # to debug sqlite3 databases ]; diff --git a/hosts/common/programs/sane-scripts.nix b/hosts/common/programs/sane-scripts.nix index 6b5cd82c5..25285f95a 100644 --- a/hosts/common/programs/sane-scripts.nix +++ b/hosts/common/programs/sane-scripts.nix @@ -7,10 +7,6 @@ let in { sane.programs = { - "sane-scripts.backup" = declPackageSet [ - "sane-scripts.backup-ls" - "sane-scripts.backup-restore" - ]; "sane-scripts.bittorrent" = declPackageSet [ "sane-scripts.bt-add" "sane-scripts.bt-rm" @@ -46,9 +42,6 @@ in "sane-scripts.sync-music" ]; - "sane-scripts.backup-ls" = {}; - "sane-scripts.backup-restore" = {}; - "sane-scripts.bt-add".sandbox = { method = "bwrap"; autodetectCliPaths = "existing"; #< for adding a .torrent from disk diff --git a/hosts/modules/services/default.nix b/hosts/modules/services/default.nix index acf614d86..a49de6b81 100644 --- a/hosts/modules/services/default.nix +++ b/hosts/modules/services/default.nix @@ -1,7 +1,6 @@ { ... }: { imports = [ - ./duplicity.nix ./rsync-net ]; } diff --git a/hosts/modules/services/duplicity.nix b/hosts/modules/services/duplicity.nix deleted file mode 100644 index 62a796961..000000000 --- a/hosts/modules/services/duplicity.nix +++ /dev/null @@ -1,98 +0,0 @@ -# docs: https://search.nixos.org/options?channel=21.11&query=duplicity -{ config, lib, pkgs, ... }: - -with lib; -let - cfg = config.sane.services.duplicity; -in -{ - options = { - sane.services.duplicity.enable = mkOption { - default = false; - type = types.bool; - }; - }; - - config = mkIf cfg.enable { - # we need this mostly because of the size of duplicity's cache - sane.persist.sys.byStore.ephemeral = [{ - path = "/var/lib/duplicity"; - user = "root"; - group = "root"; - mode = "0700"; - }]; - - services.duplicity.enable = true; - services.duplicity.targetUrl = "$DUPLICITY_URL"; - # format: PASSPHRASE= \n DUPLICITY_URL=b2://... - # two sisters - # PASSPHRASE: remote backups will be encrypted using this passphrase (using gpg) - # DUPLICITY_URL: b2://$key_id:$app_key@$bucket - # create key with: backblaze-b2 create-key --bucket uninsane-host-duplicity uninsane-host-duplicity-safe listBuckets,listFiles,readBuckets,readFiles,writeFiles - # ^ run this until you get a key with no forward slashes :upside_down: - # web-created keys are allowed to delete files, which you probably don't want for an incremental backup program - # you need to create a new application key from the web in order to first get a key which can create new keys (use env vars in the above command) - # TODO: s/duplicity_passphrase/duplicity_env/ - services.duplicity.secretFile = config.sops.secrets."duplicity_passphrase.env".path; - # NB: manually trigger with `systemctl start duplicity` - services.duplicity.frequency = "daily"; - - services.duplicity.extraFlags = [ - # without --allow-source-mismatch, duplicity will abort if you change the hostname between backups - "--allow-source-mismatch" - - # includes/exclude ordering matters, so we explicitly control it here. - # the first match decides a file's treatment. so here: - # - /nix/persist/home/colin/tmp is excluded - # - *other* /nix/persist/ files are included by default - # - anything else under `/` are excluded by default - "--exclude" "/nix/persist/home/colin/dev/home-logic/coremem/out" # this can reach > 1 TB - "--exclude" "/nix/persist/home/colin/use/iso" # might want to re-enable... but not critical - "--exclude" "/nix/persist/home/colin/.local/share/sublime-music" # music cache. better to just keep the HQ sources - "--exclude" "/nix/persist/home/colin/.local/share/Steam" # can just re-download games - "--exclude" "/nix/persist/home/colin/.bitmonero/lmdb" # monero blockchain - "--exclude" "/nix/persist/home/colin/.rustup" - "--exclude" "/nix/persist/home/colin/ref" # publicly available data: no point in duplicating it - "--exclude" "/nix/persist/home/colin/tmp" - "--exclude" "/nix/persist/home/colin/Videos" - "--exclude" "/nix/persist/var/lib/duplicity" # don't back up our own backup state! - "--include" "/nix/persist" - "--exclude" "/" - ]; - - # set this for the FIRST backup, then remove it to enable incremental backups - # (that the first backup *isn't* full i think is a defect) - # services.duplicity.fullIfOlderThan = "always"; - - systemd.services.duplicity.serviceConfig = { - # rate-limit the read bandwidth in an effort to thereby prevent net upload saturation - # this could perhaps be done better by adding a duplicity config option to replace the binary with `trickle` - IOReadBandwidthMax = [ - "/dev/sda1 5M" - "/dev/nvme0n1 5M" - "/dev/mmc0 5M" - ]; - }; - - # based on with changes: - # - remove the cleanup step: API key doesn't have delete perms - # - don't escape the targetUrl: it comes from an env var set in the secret file - systemd.services.duplicity.script = let - cfg = config.services.duplicity; - target = cfg.targetUrl; - extra = escapeShellArgs ([ "--archive-dir" "/var/lib/duplicity" ] ++ cfg.extraFlags); - dup = "${pkgs.duplicity}/bin/duplicity"; - in lib.mkForce '' - set -x - # ${dup} cleanup ${target} --force ${extra} - # ${lib.optionalString (cfg.cleanup.maxAge != null) "${dup} remove-older-than ${lib.escapeShellArg cfg.cleanup.maxAge} ${target} --force ${extra}"} - # ${lib.optionalString (cfg.cleanup.maxFull != null) "${dup} remove-all-but-n-full ${builtins.toString cfg.cleanup.maxFull} ${target} --force ${extra}"} - # ${lib.optionalString (cfg.cleanup.maxIncr != null) "${dup} remove-all-inc-of-but-n-full ${toString cfg.cleanup.maxIncr} ${target} --force ${extra}"} - exec ${dup} ${if cfg.fullIfOlderThan == "always" then "full" else "incr"} ${lib.escapeShellArg cfg.root} ${target} ${lib.escapeShellArgs ([] - ++ concatMap (p: [ "--include" p ]) cfg.include - ++ concatMap (p: [ "--exclude" p ]) cfg.exclude - ++ (lib.optionals (cfg.fullIfOlderThan != "never" && cfg.fullIfOlderThan != "always") [ "--full-if-older-than" cfg.fullIfOlderThan ]) - )} ${extra} - ''; - }; -} diff --git a/pkgs/additional/sane-scripts/default.nix b/pkgs/additional/sane-scripts/default.nix index 8ce696712..3e220d7c0 100644 --- a/pkgs/additional/sane-scripts/default.nix +++ b/pkgs/additional/sane-scripts/default.nix @@ -58,16 +58,6 @@ let sane-bin = { # anything added to this attrset gets symlink-joined into `sane-scripts` # and is made available through `sane-scripts.passthru` - backup-ls = static-nix-shell.mkBash { - pname = "sane-backup-ls"; - srcRoot = ./src; - pkgs = [ "duplicity" ]; - }; - backup-restore = static-nix-shell.mkBash { - pname = "sane-backup-restore"; - srcRoot = ./src; - pkgs = [ "duplicity" ]; - }; bt-add = static-nix-shell.mkPython3 { pname = "sane-bt-add"; srcRoot = ./src; diff --git a/pkgs/additional/sane-scripts/src/sane-backup-ls b/pkgs/additional/sane-scripts/src/sane-backup-ls deleted file mode 100755 index 06ba8b884..000000000 --- a/pkgs/additional/sane-scripts/src/sane-backup-ls +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p duplicity - -# N.B. must be run as root - -set -ex - -# source the URL; hack to satisfy resholve -external_cmd="source /run/secrets/duplicity_passphrase.env" -$external_cmd -duplicity list-current-files --archive-dir /var/lib/duplicity $DUPLICITY_URL diff --git a/pkgs/additional/sane-scripts/src/sane-backup-restore b/pkgs/additional/sane-scripts/src/sane-backup-restore deleted file mode 100755 index 248a02bda..000000000 --- a/pkgs/additional/sane-scripts/src/sane-backup-restore +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash -p duplicity - -# N.B. must be run as root - -set -ex - -dest_path="$1" -source_path="$2" - -# source the URL; hack to satisfy resholve -external_cmd="source /run/secrets/duplicity_passphrase.env" -$external_cmd -duplicity restore --archive-dir /var/lib/duplicity --file-to-restore "$source_path" $DUPLICITY_URL "$dest_path" diff --git a/secrets/desko/duplicity_passphrase.env.bin b/secrets/desko/duplicity_passphrase.env.bin deleted file mode 100644 index ed3ab9cb9..000000000 --- a/secrets/desko/duplicity_passphrase.env.bin +++ /dev/null @@ -1,28 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:dVL/VccL8BhOidlecaQydxKFDYSpt8U+ZczEr5e+a3e+IeOe0+7Ns6QQuM47fg69Kng+7hhTLxaW9wCPCTNFtI0x44OsXvWT70xRXW9z7mGajrGOFjMZ3tJC2OP9R3M+2FDUa1BsHf/jSqwrLyrh0/uNvpJHuTTQ6EhFkjNkPd7nV4S8,iv:mKfrn4xi3lk4Q7L/Y/DqaceZJ2+2JxX6R8m384Q9qfc=,tag:dLFbqCCu8CzDi06xniBX/w==,type:str]", - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": [ - { - "recipient": "age1tnl4jfgacwkargzeqnhzernw29xx8mkv73xh6ufdyde6q7859slsnzf24x", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2V2lTdTJRYnZERW1IbUZW\ncmd5MzNmRWVTRWJLU1dZeHhpZWVSbTAwVjF3Ck40TWVjaEpIUWxPazZQNVR3Ylp1\nM0N2VFNtSS9tOVlJdFVwQjdPVVhyN1UKLS0tIHNOenZCYkV3YWJVT3NTeVpOdVVF\neXkrZ083QzZTV0VHM1ZYR2N5S1ZWRVUKHyCKYFtrLNLFSQBWDs2tPTajM4QS5YcM\n8SsgVEFEExy0b8Zd8nmvntVHBaeazoYr0e2Rt97YoujjaAa+7RdLCg==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1j2pqnl8j0krdzk6npe93s4nnqrzwx978qrc0u570gzlamqpnje9sc8le2g", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPam5iSWc5SUxvbC9MUjZy\nL0RKYjZTYWpWSDhJZjMzbGZacVJ2Wm5kemlZCmxLWkFiMjZOM2JKNWFCczQyc08r\nYnJHZTF4RFV1cFdGUE1sOGFoRDB5U00KLS0tIEVuVGdRUkt4aG1pQWc2VG1QYXpO\nMytJV2RjQXE5Sk80bzM2bjIvalFIQ0UKllABQMTHe0dGOFG4sbFJgjer31QvY0o8\nw0zo3OgUQV/TZZDcQR9w8R/ZKNiaKPP38uhgppZEKqm8Y1Hnuj0Nng==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1vnw7lnfpdpjn62l3u5nyv5xt2c965k96p98kc43mcnyzpetrts9q54mc9v", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6R2Z6VE1PMkk2ZlpGR2py\nNkloZHR5NStvVi9XUG41d3pqYjRuMzgwWlF3ClFkQWVYZUN3NDJDYUNmT2srZDFm\nVGJVQnlDbkVYK1hMNXgydmFTTzlJR00KLS0tIHBOS21SSXhaOElzMDYwL0lQbFh1\nenBmQityc1pBR1ZpK3pPbm5NeTdUbkEKSzkUt2RAOzgPEIV4K0KajuMvsS2e5mFR\n93IRYE3nRFH/h26GxfYp9+C2yhfbpDpjc0dH7/fVxOTssOeekpTeXQ==\n-----END AGE ENCRYPTED FILE-----\n" - } - ], - "lastmodified": "2023-05-14T02:12:47Z", - "mac": "ENC[AES256_GCM,data:l6d9xLwaVcfrbVMNHtbX921gBAfYWCP6xZhVkc7MtDe8BumdX2+78nHV7+FC0NTMbMBfPTU26dVpf3udVf19H6X1XZ41IBLpuIWwnM4biluNgfSBQU7MSf3+B2DMaJqDqB1sUwPiwiIHPVyTvIXiK0BRxpwfo+PGzaVCc+rPpX8=,iv:mwIRnNdz7a7pabfy8SkdVbWkjCqKQbtMpuBoKYyDKB4=,tag:aWKI0Ax/0YSKAMf0EE+cKQ==,type:str]", - "pgp": null, - "unencrypted_suffix": "_unencrypted", - "version": "3.7.3" - } -} \ No newline at end of file diff --git a/secrets/servo/duplicity_passphrase.env.bin b/secrets/servo/duplicity_passphrase.env.bin deleted file mode 100644 index c36d678c2..000000000 --- a/secrets/servo/duplicity_passphrase.env.bin +++ /dev/null @@ -1,32 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:FckJhlRs49hD6xvbAQI+fZxRUDrym1rTPeIVOnivgBGX5NX6SAhYD1ZRgpqbVmTyUOui8UnGcx/XKTfMrEqQF21KtX7FSDY6lmu6A8eupLfd/W9Is66N+3jZfSWoCDyLORONwpTiH6e1E6Jtx4ZmCZo2fFg1LJ/IQrPpmEPcvu24DoRPCUwX65k7aw4=,iv:+ewWmraJJW/+zmg1rWgMk7IH3nUd2nhq5X13MtlGBTs=,tag:7jlZ7w6z5VJR0JYVh+aarQ==,type:str]", - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": [ - { - "recipient": "age1tnl4jfgacwkargzeqnhzernw29xx8mkv73xh6ufdyde6q7859slsnzf24x", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIM0pxb3AxK3E3cTNVM0hk\nbTYvU0w5T1BlRHd0OXViK3Y4MUtQSnZnMkVjCk81OHFMQ0dyaHVlS3daMHlBQm1i\nT1JJd3QyUkJ1ck5Hd0JyaTRpZmNjWEkKLS0tIEpOWWVaMDYyTGRCNnFlZHF6OVJW\nRFhYYXZnelF6SjFmYldxZUJ3dU12MGsKKx4cCVau5B+nCjcfD5OgGhn7ePL2qD9S\n9z/xCQbj3tEyRiHZKifnjBjvP0d4q7EfiYO6cpIsWlGQjwJvaUqh/w==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1j2pqnl8j0krdzk6npe93s4nnqrzwx978qrc0u570gzlamqpnje9sc8le2g", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsRkFzL2o0L1dUVWJlRkxV\nbU9ZRjQxR1dDUTJEWHhZTFJ1cjhHamVKR3lnCjlpTUVGVmk3TjVGNzBTLzVkeXRQ\nM1l5aHNMczVYUU9kVCtwRzZQL1ZtREkKLS0tIGtuZzJxU3l4T1lVUGx3T0xaTmxD\nRTQ5WEEwTWowZnlvbkNOZVdCbUVUZE0KifE0v05OLZGOHDw1xq/ioDjW/KvHLoU0\n7l/8tl4vPM9K2I/ibIBslg9aGdzw78wgOxO7VhKXobHmswlKDje6Jw==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1z8fauff34cdecr6sjkre260luzxcca05kpcwvhx988d306tpcejsp63znu", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWTU9OdW9EaGdzNzlkWkFM\nUWhtRGNCSWpwQ0dkUkJUYlNPSnI4SEpDZ1VBCmU5WmtWS2gvZ2hpRFNRYmVXbTRV\naGlGSktPa0JUbmpNSUt2bHh6OFB5QVUKLS0tIG4ySlV4YnhRYm0ra1JXRlMyMERP\nL3V5N3kxeUF5a1JaaUZFQ1Y5MzVHd3cK93TkOJaHlRWrsUVGJ/Cgx3lw/aFHVJlX\nasTHY9xSiot4son3HJKClAwJhSJGj83LkBcyTc98vmNRrf5BTL4lsw==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1tzlyex2z6t88tg9h82943e39shxhmqeyr7ywhlwpdjmyqsndv3qq27x0rf", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqdnhmV1BRckJIV0xwMUVr\nVnlrQVhzT0RJdEFiUjYvaWFkTjdqZjNvSzI4CkU5UGlYbmNCNTR2OHJ4djMraGdw\na2h1TFBkYlE5Z1BYaW9QWXIwWHFna2MKLS0tIDFRVk5TeHBmU1hwZHhhR0ppSUhN\nZVNvdkVWMXZqSWJNdGh0YXRZdnVzS1EK8mD2Vn4e310sh+Ei6djtD8//wuuf3u/l\nQ1U+wY/N91POEqPJ4Uw56chS4BMzR+XVepVvJZwZnY4fBL01Pr+0CA==\n-----END AGE ENCRYPTED FILE-----\n" - } - ], - "lastmodified": "2023-05-14T08:33:49Z", - "mac": "ENC[AES256_GCM,data:7sdsrb2T8YkejitwFQBb17FriPsqMWX3+dVl255EG/k9futJzXoR2Y1TpHH0A+09i5+Q7dxReWsIQ2REafyPy2NAtkc2GZgMHMdKrVfbYFr63VIcayS7vlDWEjnOLH7znLtzWn09dwSAJzbuinQ1EoPavh5dqZeiwcWBoD2sYk8=,iv:VdMTjYnbkJqMeLLu7eOUoMzmuqtP+NRayx6nZUVgRho=,tag:64kz11YYBMUtlkhLFa+k4Q==,type:str]", - "pgp": null, - "unencrypted_suffix": "_unencrypted", - "version": "3.7.3" - } -} \ No newline at end of file