colin/nix-files
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

duplicity: purge

Browse Source
This commit is contained in:
Colin
2024-08-07 01:25:09 +00:00
parent cc6ed6c0ec
commit 6dad290cd5
12 changed files with 0 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
TODO.md
View File

@@ -60,7 +60,6 @@
- has better multi-stream perf (e.g. `sane-sync-music` should be able to copy N items in parallel)
### security/resilience
- validate duplicity backups!
- encrypt more ~ dirs (~/archives, ~/records, ..?)
- best to do this after i know for sure i have good backups
- /mnt/desko/home, etc, shouldn't include secrets (~/private)

1
hosts/by-name/desko/default.nix
View File

@@ -28,7 +28,6 @@
sane.services.wg-home.ip = config.sane.hosts.by-name."desko".wg-home.ip;
sane.ovpn.addrV4 = "172.26.55.21";
# sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:20c1:a73c";
sane.services.duplicity.enable = true;
sane.services.rsync-net.enable = true;
sane.nixcache.remote-builders.desko = false;

1
hosts/by-name/servo/default.nix
View File

@@ -31,7 +31,6 @@
# sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:8df3:14b0";
sane.nixcache.remote-builders.desko = false;
sane.nixcache.remote-builders.servo = false;
# sane.services.duplicity.enable = true; # TODO: re-enable after HW upgrade
sane.services.rsync-net.enable = true;
# automatically log in at the virtual consoles.

3
hosts/common/programs/assorted.nix
View File

@@ -109,9 +109,6 @@ in
# "zfs" # doesn't cross-compile (requires samba)
];
sysadminExtraUtils = declPackageSet [
"backblaze-b2"
"duplicity"
"sane-scripts.backup"
"sqlite" # to debug sqlite3 databases
];

7
hosts/common/programs/sane-scripts.nix
View File

@@ -7,10 +7,6 @@ let
in
{
sane.programs = {
"sane-scripts.backup" = declPackageSet [
"sane-scripts.backup-ls"
"sane-scripts.backup-restore"
];
"sane-scripts.bittorrent" = declPackageSet [
"sane-scripts.bt-add"
"sane-scripts.bt-rm"
@@ -46,9 +42,6 @@ in
"sane-scripts.sync-music"
];
"sane-scripts.backup-ls" = {};
"sane-scripts.backup-restore" = {};
"sane-scripts.bt-add".sandbox = {
method = "bwrap";
autodetectCliPaths = "existing"; #< for adding a .torrent from disk

1
hosts/modules/services/default.nix
View File

@@ -1,7 +1,6 @@
{ ... }:
{
imports = [
./duplicity.nix
./rsync-net
];
}

98
hosts/modules/services/duplicity.nix
View File

@@ -1,98 +0,0 @@
# docs: https://search.nixos.org/options?channel=21.11&query=duplicity
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.sane.services.duplicity;
in
{
options = {
sane.services.duplicity.enable = mkOption {
default = false;
type = types.bool;
};
};
config = mkIf cfg.enable {
# we need this mostly because of the size of duplicity's cache
sane.persist.sys.byStore.ephemeral = [{
path = "/var/lib/duplicity";
user = "root";
group = "root";
mode = "0700";
}];
services.duplicity.enable = true;
services.duplicity.targetUrl = "$DUPLICITY_URL";
# format: PASSPHRASE=<cleartext> \n DUPLICITY_URL=b2://...
# two sisters
# PASSPHRASE: remote backups will be encrypted using this passphrase (using gpg)
# DUPLICITY_URL: b2://$key_id:$app_key@$bucket
# create key with: backblaze-b2 create-key --bucket uninsane-host-duplicity uninsane-host-duplicity-safe listBuckets,listFiles,readBuckets,readFiles,writeFiles
# ^ run this until you get a key with no forward slashes :upside_down:
# web-created keys are allowed to delete files, which you probably don't want for an incremental backup program
# you need to create a new application key from the web in order to first get a key which can create new keys (use env vars in the above command)
# TODO: s/duplicity_passphrase/duplicity_env/
services.duplicity.secretFile = config.sops.secrets."duplicity_passphrase.env".path;
# NB: manually trigger with `systemctl start duplicity`
services.duplicity.frequency = "daily";
services.duplicity.extraFlags = [
# without --allow-source-mismatch, duplicity will abort if you change the hostname between backups
"--allow-source-mismatch"
# includes/exclude ordering matters, so we explicitly control it here.
# the first match decides a file's treatment. so here:
# - /nix/persist/home/colin/tmp is excluded
# - *other* /nix/persist/ files are included by default
# - anything else under `/` are excluded by default
"--exclude" "/nix/persist/home/colin/dev/home-logic/coremem/out" # this can reach > 1 TB
"--exclude" "/nix/persist/home/colin/use/iso" # might want to re-enable... but not critical
"--exclude" "/nix/persist/home/colin/.local/share/sublime-music" # music cache. better to just keep the HQ sources
"--exclude" "/nix/persist/home/colin/.local/share/Steam" # can just re-download games
"--exclude" "/nix/persist/home/colin/.bitmonero/lmdb" # monero blockchain
"--exclude" "/nix/persist/home/colin/.rustup"
"--exclude" "/nix/persist/home/colin/ref" # publicly available data: no point in duplicating it
"--exclude" "/nix/persist/home/colin/tmp"
"--exclude" "/nix/persist/home/colin/Videos"
"--exclude" "/nix/persist/var/lib/duplicity" # don't back up our own backup state!
"--include" "/nix/persist"
"--exclude" "/"
];
# set this for the FIRST backup, then remove it to enable incremental backups
# (that the first backup *isn't* full i think is a defect)
# services.duplicity.fullIfOlderThan = "always";
systemd.services.duplicity.serviceConfig = {
# rate-limit the read bandwidth in an effort to thereby prevent net upload saturation
# this could perhaps be done better by adding a duplicity config option to replace the binary with `trickle`
IOReadBandwidthMax = [
"/dev/sda1 5M"
"/dev/nvme0n1 5M"
"/dev/mmc0 5M"
];
};
# based on <nixpkgs:nixos/modules/services/backup/duplicity.nix> with changes:
# - remove the cleanup step: API key doesn't have delete perms
# - don't escape the targetUrl: it comes from an env var set in the secret file
systemd.services.duplicity.script = let
cfg = config.services.duplicity;
target = cfg.targetUrl;
extra = escapeShellArgs ([ "--archive-dir" "/var/lib/duplicity" ] ++ cfg.extraFlags);
dup = "${pkgs.duplicity}/bin/duplicity";
in lib.mkForce ''
set -x
# ${dup} cleanup ${target} --force ${extra}
# ${lib.optionalString (cfg.cleanup.maxAge != null) "${dup} remove-older-than ${lib.escapeShellArg cfg.cleanup.maxAge} ${target} --force ${extra}"}
# ${lib.optionalString (cfg.cleanup.maxFull != null) "${dup} remove-all-but-n-full ${builtins.toString cfg.cleanup.maxFull} ${target} --force ${extra}"}
# ${lib.optionalString (cfg.cleanup.maxIncr != null) "${dup} remove-all-inc-of-but-n-full ${toString cfg.cleanup.maxIncr} ${target} --force ${extra}"}
exec ${dup} ${if cfg.fullIfOlderThan == "always" then "full" else "incr"} ${lib.escapeShellArg cfg.root} ${target} ${lib.escapeShellArgs ([]
++ concatMap (p: [ "--include" p ]) cfg.include
++ concatMap (p: [ "--exclude" p ]) cfg.exclude
++ (lib.optionals (cfg.fullIfOlderThan != "never" && cfg.fullIfOlderThan != "always") [ "--full-if-older-than" cfg.fullIfOlderThan ])
)} ${extra}
'';
};
}

10
pkgs/additional/sane-scripts/default.nix
View File

@@ -58,16 +58,6 @@ let
sane-bin = {
# anything added to this attrset gets symlink-joined into `sane-scripts`
# and is made available through `sane-scripts.passthru`
backup-ls = static-nix-shell.mkBash {
pname = "sane-backup-ls";
srcRoot = ./src;
pkgs = [ "duplicity" ];
};
backup-restore = static-nix-shell.mkBash {
pname = "sane-backup-restore";
srcRoot = ./src;
pkgs = [ "duplicity" ];
};
bt-add = static-nix-shell.mkPython3 {
pname = "sane-bt-add";
srcRoot = ./src;

11
pkgs/additional/sane-scripts/src/sane-backup-ls
View File

@@ -1,11 +0,0 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p bash -p duplicity
# N.B. must be run as root
set -ex
# source the URL; hack to satisfy resholve
external_cmd="source /run/secrets/duplicity_passphrase.env"
$external_cmd
duplicity list-current-files --archive-dir /var/lib/duplicity $DUPLICITY_URL

14
pkgs/additional/sane-scripts/src/sane-backup-restore
View File

@@ -1,14 +0,0 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p bash -p duplicity
# N.B. must be run as root
set -ex
dest_path="$1"
source_path="$2"
# source the URL; hack to satisfy resholve
external_cmd="source /run/secrets/duplicity_passphrase.env"
$external_cmd
duplicity restore --archive-dir /var/lib/duplicity --file-to-restore "$source_path" $DUPLICITY_URL "$dest_path"

28
secrets/desko/duplicity_passphrase.env.bin
View File

@@ -1,28 +0,0 @@
{
"data": "ENC[AES256_GCM,data:dVL/VccL8BhOidlecaQydxKFDYSpt8U+ZczEr5e+a3e+IeOe0+7Ns6QQuM47fg69Kng+7hhTLxaW9wCPCTNFtI0x44OsXvWT70xRXW9z7mGajrGOFjMZ3tJC2OP9R3M+2FDUa1BsHf/jSqwrLyrh0/uNvpJHuTTQ6EhFkjNkPd7nV4S8,iv:mKfrn4xi3lk4Q7L/Y/DqaceZJ2+2JxX6R8m384Q9qfc=,tag:dLFbqCCu8CzDi06xniBX/w==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": [
{
"recipient": "age1tnl4jfgacwkargzeqnhzernw29xx8mkv73xh6ufdyde6q7859slsnzf24x",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2V2lTdTJRYnZERW1IbUZW\ncmd5MzNmRWVTRWJLU1dZeHhpZWVSbTAwVjF3Ck40TWVjaEpIUWxPazZQNVR3Ylp1\nM0N2VFNtSS9tOVlJdFVwQjdPVVhyN1UKLS0tIHNOenZCYkV3YWJVT3NTeVpOdVVF\neXkrZ083QzZTV0VHM1ZYR2N5S1ZWRVUKHyCKYFtrLNLFSQBWDs2tPTajM4QS5YcM\n8SsgVEFEExy0b8Zd8nmvntVHBaeazoYr0e2Rt97YoujjaAa+7RdLCg==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1j2pqnl8j0krdzk6npe93s4nnqrzwx978qrc0u570gzlamqpnje9sc8le2g",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPam5iSWc5SUxvbC9MUjZy\nL0RKYjZTYWpWSDhJZjMzbGZacVJ2Wm5kemlZCmxLWkFiMjZOM2JKNWFCczQyc08r\nYnJHZTF4RFV1cFdGUE1sOGFoRDB5U00KLS0tIEVuVGdRUkt4aG1pQWc2VG1QYXpO\nMytJV2RjQXE5Sk80bzM2bjIvalFIQ0UKllABQMTHe0dGOFG4sbFJgjer31QvY0o8\nw0zo3OgUQV/TZZDcQR9w8R/ZKNiaKPP38uhgppZEKqm8Y1Hnuj0Nng==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1vnw7lnfpdpjn62l3u5nyv5xt2c965k96p98kc43mcnyzpetrts9q54mc9v",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6R2Z6VE1PMkk2ZlpGR2py\nNkloZHR5NStvVi9XUG41d3pqYjRuMzgwWlF3ClFkQWVYZUN3NDJDYUNmT2srZDFm\nVGJVQnlDbkVYK1hMNXgydmFTTzlJR00KLS0tIHBOS21SSXhaOElzMDYwL0lQbFh1\nenBmQityc1pBR1ZpK3pPbm5NeTdUbkEKSzkUt2RAOzgPEIV4K0KajuMvsS2e5mFR\n93IRYE3nRFH/h26GxfYp9+C2yhfbpDpjc0dH7/fVxOTssOeekpTeXQ==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2023-05-14T02:12:47Z",
"mac": "ENC[AES256_GCM,data:l6d9xLwaVcfrbVMNHtbX921gBAfYWCP6xZhVkc7MtDe8BumdX2+78nHV7+FC0NTMbMBfPTU26dVpf3udVf19H6X1XZ41IBLpuIWwnM4biluNgfSBQU7MSf3+B2DMaJqDqB1sUwPiwiIHPVyTvIXiK0BRxpwfo+PGzaVCc+rPpX8=,iv:mwIRnNdz7a7pabfy8SkdVbWkjCqKQbtMpuBoKYyDKB4=,tag:aWKI0Ax/0YSKAMf0EE+cKQ==,type:str]",
"pgp": null,
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}

32
secrets/servo/duplicity_passphrase.env.bin
View File

@@ -1,32 +0,0 @@
{
"data": "ENC[AES256_GCM,data:FckJhlRs49hD6xvbAQI+fZxRUDrym1rTPeIVOnivgBGX5NX6SAhYD1ZRgpqbVmTyUOui8UnGcx/XKTfMrEqQF21KtX7FSDY6lmu6A8eupLfd/W9Is66N+3jZfSWoCDyLORONwpTiH6e1E6Jtx4ZmCZo2fFg1LJ/IQrPpmEPcvu24DoRPCUwX65k7aw4=,iv:+ewWmraJJW/+zmg1rWgMk7IH3nUd2nhq5X13MtlGBTs=,tag:7jlZ7w6z5VJR0JYVh+aarQ==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": [
{
"recipient": "age1tnl4jfgacwkargzeqnhzernw29xx8mkv73xh6ufdyde6q7859slsnzf24x",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIM0pxb3AxK3E3cTNVM0hk\nbTYvU0w5T1BlRHd0OXViK3Y4MUtQSnZnMkVjCk81OHFMQ0dyaHVlS3daMHlBQm1i\nT1JJd3QyUkJ1ck5Hd0JyaTRpZmNjWEkKLS0tIEpOWWVaMDYyTGRCNnFlZHF6OVJW\nRFhYYXZnelF6SjFmYldxZUJ3dU12MGsKKx4cCVau5B+nCjcfD5OgGhn7ePL2qD9S\n9z/xCQbj3tEyRiHZKifnjBjvP0d4q7EfiYO6cpIsWlGQjwJvaUqh/w==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1j2pqnl8j0krdzk6npe93s4nnqrzwx978qrc0u570gzlamqpnje9sc8le2g",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsRkFzL2o0L1dUVWJlRkxV\nbU9ZRjQxR1dDUTJEWHhZTFJ1cjhHamVKR3lnCjlpTUVGVmk3TjVGNzBTLzVkeXRQ\nM1l5aHNMczVYUU9kVCtwRzZQL1ZtREkKLS0tIGtuZzJxU3l4T1lVUGx3T0xaTmxD\nRTQ5WEEwTWowZnlvbkNOZVdCbUVUZE0KifE0v05OLZGOHDw1xq/ioDjW/KvHLoU0\n7l/8tl4vPM9K2I/ibIBslg9aGdzw78wgOxO7VhKXobHmswlKDje6Jw==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1z8fauff34cdecr6sjkre260luzxcca05kpcwvhx988d306tpcejsp63znu",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWTU9OdW9EaGdzNzlkWkFM\nUWhtRGNCSWpwQ0dkUkJUYlNPSnI4SEpDZ1VBCmU5WmtWS2gvZ2hpRFNRYmVXbTRV\naGlGSktPa0JUbmpNSUt2bHh6OFB5QVUKLS0tIG4ySlV4YnhRYm0ra1JXRlMyMERP\nL3V5N3kxeUF5a1JaaUZFQ1Y5MzVHd3cK93TkOJaHlRWrsUVGJ/Cgx3lw/aFHVJlX\nasTHY9xSiot4son3HJKClAwJhSJGj83LkBcyTc98vmNRrf5BTL4lsw==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1tzlyex2z6t88tg9h82943e39shxhmqeyr7ywhlwpdjmyqsndv3qq27x0rf",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqdnhmV1BRckJIV0xwMUVr\nVnlrQVhzT0RJdEFiUjYvaWFkTjdqZjNvSzI4CkU5UGlYbmNCNTR2OHJ4djMraGdw\na2h1TFBkYlE5Z1BYaW9QWXIwWHFna2MKLS0tIDFRVk5TeHBmU1hwZHhhR0ppSUhN\nZVNvdkVWMXZqSWJNdGh0YXRZdnVzS1EK8mD2Vn4e310sh+Ei6djtD8//wuuf3u/l\nQ1U+wY/N91POEqPJ4Uw56chS4BMzR+XVepVvJZwZnY4fBL01Pr+0CA==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2023-05-14T08:33:49Z",
"mac": "ENC[AES256_GCM,data:7sdsrb2T8YkejitwFQBb17FriPsqMWX3+dVl255EG/k9futJzXoR2Y1TpHH0A+09i5+Q7dxReWsIQ2REafyPy2NAtkc2GZgMHMdKrVfbYFr63VIcayS7vlDWEjnOLH7znLtzWn09dwSAJzbuinQ1EoPavh5dqZeiwcWBoD2sYk8=,iv:VdMTjYnbkJqMeLLu7eOUoMzmuqtP+NRayx6nZUVgRho=,tag:64kz11YYBMUtlkhLFa+k4Q==,type:str]",
"pgp": null,
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}