From 121936620a53982c5349cea7518b866ed19a81fa Mon Sep 17 00:00:00 2001 From: colin Date: Wed, 28 Dec 2022 03:42:14 +0000 Subject: [PATCH] impermanence: add support for encrypted clear-on-boot storage this is useful for when we need to store files to disk purely due to their size, but don't actually want them to be persisted. --- hosts/common/machine-id.nix | 13 +- hosts/common/users.nix | 2 +- hosts/servo/fs.nix | 2 +- hosts/servo/services/ejabberd.nix | 2 +- hosts/servo/services/freshrss.nix | 2 +- hosts/servo/services/gitea.nix | 2 +- hosts/servo/services/ipfs.nix | 2 +- hosts/servo/services/jackett.nix | 2 +- hosts/servo/services/jellyfin.nix | 2 +- hosts/servo/services/matrix/default.nix | 2 +- .../servo/services/matrix/discord-puppet.nix | 2 +- hosts/servo/services/matrix/irc.nix | 2 +- hosts/servo/services/navidrome.nix | 2 +- hosts/servo/services/nginx.nix | 2 +- hosts/servo/services/pleroma.nix | 2 +- hosts/servo/services/postfix.nix | 2 +- hosts/servo/services/postgres.nix | 2 +- hosts/servo/services/prosody.nix | 2 +- hosts/servo/services/transmission.nix | 2 +- modules/impermanence.nix | 225 +++++++++++------- modules/services/duplicity.nix | 2 +- 21 files changed, 169 insertions(+), 107 deletions(-) diff --git a/hosts/common/machine-id.nix b/hosts/common/machine-id.nix index 08cf8eab..80148a51 100644 --- a/hosts/common/machine-id.nix +++ b/hosts/common/machine-id.nix @@ -1,9 +1,14 @@ { ... }: { - # we wan't an /etc/machine-id which is consistent across boot so that `journalctl` will actually show us - # logs from previous boots. - # maybe there's a config option for this (since persistent machine-id is bad for reasons listed in impermanence.nix), - # but for now generate it from ssh keys. + # /etc/machine-id is a globally unique identifier used for: + # - systemd-networkd: DHCP lease renewal (instead of keying by the MAC address) + # - systemd-journald: to filter logs by host + # - chromium (potentially to track re-installations) + # - gdbus; system services that might upgrade to AF_LOCAL if both services can confirm they're on the same machine + # because of e.g. the chromium use, we *don't want* to persist this. + # however, `journalctl` won't show logs from previous boots unless the machine-ids match. + # so for now, generate something unique from the host ssh key. + # TODO: move this into modules? system.activationScripts.machine-id = { deps = [ "persist-ssh-host-keys" ]; text = "sha256sum /etc/ssh/host_keys/ssh_host_ed25519_key | cut -c 1-32 > /etc/machine-id"; diff --git a/hosts/common/users.nix b/hosts/common/users.nix index 36fcfceb..34b0d53c 100644 --- a/hosts/common/users.nix +++ b/hosts/common/users.nix @@ -75,7 +75,7 @@ in ".local/share/keyrings" ]; - sane.impermanence.service-dirs = mkIf cfg.guest.enable [ + sane.impermanence.dirs = mkIf cfg.guest.enable [ { user = "guest"; group = "users"; directory = "/home/guest"; } ]; users.users.guest = mkIf cfg.guest.enable { diff --git a/hosts/servo/fs.nix b/hosts/servo/fs.nix index bd099ccc..835883a6 100644 --- a/hosts/servo/fs.nix +++ b/hosts/servo/fs.nix @@ -36,7 +36,7 @@ ]; }; - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: this is overly broad; only need media and share directories to be persisted { user = "colin"; group = "users"; directory = "/var/lib/uninsane"; } ]; diff --git a/hosts/servo/services/ejabberd.nix b/hosts/servo/services/ejabberd.nix index 60f990d9..0d84246a 100644 --- a/hosts/servo/services/ejabberd.nix +++ b/hosts/servo/services/ejabberd.nix @@ -19,7 +19,7 @@ # XXX: avatar support works in MUCs but not DMs # lib.mkIf false { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "ejabberd"; group = "ejabberd"; directory = "/var/lib/ejabberd"; } ]; networking.firewall.allowedTCPPorts = [ diff --git a/hosts/servo/services/freshrss.nix b/hosts/servo/services/freshrss.nix index 2484349b..a999b3c2 100644 --- a/hosts/servo/services/freshrss.nix +++ b/hosts/servo/services/freshrss.nix @@ -16,7 +16,7 @@ owner = config.users.users.freshrss.name; mode = "400"; }; - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "freshrss"; group = "freshrss"; directory = "/var/lib/freshrss"; } ]; diff --git a/hosts/servo/services/gitea.nix b/hosts/servo/services/gitea.nix index 97e95357..594d05f6 100644 --- a/hosts/servo/services/gitea.nix +++ b/hosts/servo/services/gitea.nix @@ -1,7 +1,7 @@ { config, pkgs, lib, ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? could be more granular { user = "git"; group = "gitea"; directory = "/var/lib/gitea"; } ]; diff --git a/hosts/servo/services/ipfs.nix b/hosts/servo/services/ipfs.nix index 993195fb..ee8df852 100644 --- a/hosts/servo/services/ipfs.nix +++ b/hosts/servo/services/ipfs.nix @@ -10,7 +10,7 @@ lib.mkIf false # i don't actively use ipfs anymore { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? could be more granular { user = "261"; group = "261"; directory = "/var/lib/ipfs"; } ]; diff --git a/hosts/servo/services/jackett.nix b/hosts/servo/services/jackett.nix index a1093e73..31c2d023 100644 --- a/hosts/servo/services/jackett.nix +++ b/hosts/servo/services/jackett.nix @@ -1,7 +1,7 @@ { ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? we only need this to save Indexer creds ==> migrate to config? { user = "root"; group = "root"; directory = "/var/lib/jackett"; } ]; diff --git a/hosts/servo/services/jellyfin.nix b/hosts/servo/services/jellyfin.nix index c3cb8e71..20234cf1 100644 --- a/hosts/servo/services/jellyfin.nix +++ b/hosts/servo/services/jellyfin.nix @@ -7,7 +7,7 @@ lib.mkIf false networking.firewall.allowedUDPPorts = [ 1900 7359 # DLNA: https://jellyfin.org/docs/general/networking/index.html ]; - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? could be more granular { user = "jellyfin"; group = "jellyfin"; directory = "/var/lib/jellyfin"; } ]; diff --git a/hosts/servo/services/matrix/default.nix b/hosts/servo/services/matrix/default.nix index a53fe2cb..346bbd75 100644 --- a/hosts/servo/services/matrix/default.nix +++ b/hosts/servo/services/matrix/default.nix @@ -8,7 +8,7 @@ # ./irc.nix ]; - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "matrix-synapse"; group = "matrix-synapse"; directory = "/var/lib/matrix-synapse"; } ]; services.matrix-synapse.enable = true; diff --git a/hosts/servo/services/matrix/discord-puppet.nix b/hosts/servo/services/matrix/discord-puppet.nix index 05622ef9..c6f3a3c2 100644 --- a/hosts/servo/services/matrix/discord-puppet.nix +++ b/hosts/servo/services/matrix/discord-puppet.nix @@ -1,6 +1,6 @@ { lib, ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "matrix-synapse"; group = "matrix-synapse"; directory = "/var/lib/mx-puppet-discord"; } ]; diff --git a/hosts/servo/services/matrix/irc.nix b/hosts/servo/services/matrix/irc.nix index 72b2f15f..440ecf0a 100644 --- a/hosts/servo/services/matrix/irc.nix +++ b/hosts/servo/services/matrix/irc.nix @@ -1,7 +1,7 @@ { config, lib, ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? # user and group are both "matrix-appservice-irc" { user = "993"; group = "992"; directory = "/var/lib/matrix-appservice-irc"; } diff --git a/hosts/servo/services/navidrome.nix b/hosts/servo/services/navidrome.nix index 48e21ed6..3bf5b0aa 100644 --- a/hosts/servo/services/navidrome.nix +++ b/hosts/servo/services/navidrome.nix @@ -1,7 +1,7 @@ { ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "navidrome"; group = "navidrome"; directory = "/var/lib/private/navidrome"; } ]; services.navidrome.enable = true; diff --git a/hosts/servo/services/nginx.nix b/hosts/servo/services/nginx.nix index 5a274254..49f6162f 100644 --- a/hosts/servo/services/nginx.nix +++ b/hosts/servo/services/nginx.nix @@ -122,7 +122,7 @@ in users.users.acme.uid = config.sane.allocations.acme-uid; users.groups.acme.gid = config.sane.allocations.acme-gid; - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? { user = "acme"; group = "acme"; directory = "/var/lib/acme"; } { user = "colin"; group = "users"; directory = "/var/www/sites"; } diff --git a/hosts/servo/services/pleroma.nix b/hosts/servo/services/pleroma.nix index 0b07bb18..c2c104e3 100644 --- a/hosts/servo/services/pleroma.nix +++ b/hosts/servo/services/pleroma.nix @@ -6,7 +6,7 @@ { config, pkgs, ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? could be more granular { user = "pleroma"; group = "pleroma"; directory = "/var/lib/pleroma"; } ]; diff --git a/hosts/servo/services/postfix.nix b/hosts/servo/services/postfix.nix index 89045bcc..f5daca03 100644 --- a/hosts/servo/services/postfix.nix +++ b/hosts/servo/services/postfix.nix @@ -16,7 +16,7 @@ let }; in { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? could be more granular { user = "opendkim"; group = "opendkim"; directory = "/var/lib/opendkim"; } { user = "root"; group = "root"; directory = "/var/lib/postfix"; } diff --git a/hosts/servo/services/postgres.nix b/hosts/servo/services/postgres.nix index 9dc05d23..c3ccde5a 100644 --- a/hosts/servo/services/postgres.nix +++ b/hosts/servo/services/postgres.nix @@ -1,7 +1,7 @@ { ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? { user = "postgres"; group = "postgres"; directory = "/var/lib/postgresql"; } ]; diff --git a/hosts/servo/services/prosody.nix b/hosts/servo/services/prosody.nix index 0b0c5360..6ad7dfc0 100644 --- a/hosts/servo/services/prosody.nix +++ b/hosts/servo/services/prosody.nix @@ -9,7 +9,7 @@ # nixnet runs ejabberd, so revisiting that. lib.mkIf false { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ { user = "prosody"; group = "prosody"; directory = "/var/lib/prosody"; } ]; networking.firewall.allowedTCPPorts = [ diff --git a/hosts/servo/services/transmission.nix b/hosts/servo/services/transmission.nix index 48860e44..db9b75bb 100644 --- a/hosts/servo/services/transmission.nix +++ b/hosts/servo/services/transmission.nix @@ -1,7 +1,7 @@ { pkgs, ... }: { - sane.impermanence.service-dirs = [ + sane.impermanence.dirs = [ # TODO: mode? we need this specifically for the stats tracking in .config/ { user = "transmission"; group = "transmission"; directory = "/var/lib/transmission"; } ]; diff --git a/modules/impermanence.nix b/modules/impermanence.nix index 63b8e7b6..77dede58 100644 --- a/modules/impermanence.nix +++ b/modules/impermanence.nix @@ -2,13 +2,77 @@ # https://xeiaso.net/blog/paranoid-nixos-2021-07-18 # https://elis.nu/blog/2020/05/nixos-tmpfs-as-root/ # https://github.com/nix-community/impermanence -{ lib, config, impermanence, ... }: +{ config, impermanence, lib, pkgs, ... }: with lib; let cfg = config.sane.impermanence; # taken from sops-nix code: checks if any secrets are needed to create /etc/shadow secretsForUsers = (lib.filterAttrs (_: v: v.neededForUsers) config.sops.secrets) != {}; + persist-base = "/nix/persist"; + encrypted-clear-on-boot-base = "/var/lib/impermanence/cleared-on-boot"; + home-dir-defaults = { + user = "colin"; + group = "users"; + mode = "0755"; + relativeTo = "/home/colin/"; + }; + sys-dir-defaults = { + user = "root"; + group = "root"; + mode = "0755"; + relativeTo = ""; + }; + + dir-options = defaults: types.submodule { + options = { + encryptedClearOnBoot = mkOption { + default = false; + type = types.bool; + }; + directory = mkOption { + type = types.str; + }; + user = mkOption { + type = types.str; + default = defaults.user; + }; + group = mkOption { + type = types.str; + default = defaults.group; + }; + mode = mkOption { + type = types.str; + default = defaults.mode; + }; + }; + }; + + ingest-dir-option = defaults: opt: + if isString opt then + ingest-dir-option defaults { directory = opt; } + else + { + encryptedClearOnBoot = opt.encryptedClearOnBoot or false; + directory = defaults.relativeTo + opt.directory; + user = opt.user or defaults.user; + group = opt.group or defaults.group; + mode = opt.mode or defaults.mode; + } + ; + ingest-dir-options = defaults: opts: builtins.map (ingest-dir-option defaults) opts; + ingested-home-dirs = ingest-dir-options home-dir-defaults cfg.home-dirs; + ingested-sys-dirs = ingest-dir-options sys-dir-defaults cfg.dirs; + ingested-default-dirs = ingest-dir-options sys-dir-defaults [ + "/var/log" + "/var/backup" # for e.g. postgres dumps + # TODO: move elsewhere + "/var/lib/alsa" # preserve output levels, default devices + "/var/lib/bluetooth" # preserve bluetooth handshakes + "/var/lib/colord" # preserve color calibrations (?) + "/var/lib/machines" # maybe not needed, but would be painful to add a VM and forget. + ]; + ingested-dirs = ingested-home-dirs ++ ingested-sys-dirs ++ ingested-default-dirs; in { options = { @@ -21,100 +85,93 @@ in type = types.bool; description = "define / to be a tmpfs. make sure to mount some other device to /nix"; }; + sane.impermanence.encrypted-clear-on-boot = mkOption { + default = builtins.any (opt: opt.encryptedClearOnBoot) ingested-dirs; + type = types.bool; + description = "define /nix/persist/crypt/cleared-on-boot to be an encrypted filesystem which is unreadable after power-off"; + }; sane.impermanence.home-dirs = mkOption { default = []; - type = types.listOf (types.either types.str (types.attrsOf types.str)); + type = types.listOf (types.either types.str (dir-options home-dir-defaults)); }; - sane.impermanence.service-dirs = mkOption { + sane.impermanence.dirs = mkOption { default = []; - type = types.listOf (types.either types.str (types.attrsOf types.str)); + type = types.listOf (types.either types.str (dir-options sys-dir-defaults)); }; }; - config = let - map-dir = defaults: dir: if isString dir then - map-dir defaults { directory = "${defaults.directory}${dir}"; } - else - defaults // dir - ; - map-dirs = defaults: dirs: builtins.map (map-dir defaults) dirs; + config = mkIf cfg.enable (lib.mkMerge [ + (lib.mkIf cfg.root-on-tmpfs { + fileSystems."/" = { + device = "none"; + fsType = "tmpfs"; + options = [ + "mode=755" + "size=1G" + "defaults" + ]; + }; + }) - map-home-dirs = map-dirs { user = "colin"; group = "users"; mode = "0755"; directory = "/home/colin/"; }; - map-sys-dirs = map-dirs { user = "root"; group = "root"; mode = "0755"; directory = ""; }; + (lib.mkIf cfg.encrypted-clear-on-boot { + system.activationScripts.mountEncryptedClearedOnBoot.text = let + gocryptfs = "${pkgs.gocryptfs}/bin/gocryptfs"; + backing = "/nix/persist/crypt/cleared-on-boot"; + mountpt = encrypted-clear-on-boot-base; + pass-template = "/tmp/encrypted-clear-on-boot.XXXXXXXX"; + tmpdir = "/tmp/impermanence"; + in '' + if !(test -e ${mountpt}/init) + then + mkdir -p ${backing} ${mountpt} ${tmpdir} + rm -rf ${backing}/* + passfile=$(mktemp ${pass-template}) + dd if=/dev/random bs=128 count=1 | base64 --wrap=0 > $passfile + ${gocryptfs} -quiet -passfile $passfile -init ${backing} + ${gocryptfs} -quiet -passfile $passfile ${backing} ${mountpt} + rm $passfile + unset passfile + touch ${mountpt}/init + fi + ''; - in mkIf cfg.enable { - fileSystems."/" = lib.mkIf cfg.root-on-tmpfs { - device = "none"; - fsType = "tmpfs"; - options = [ - "mode=755" - "size=1G" - "defaults" - ]; - }; + system.activationScripts.createPersistentStorageDirs.deps = [ "mountEncryptedClearedOnBoot" ]; + }) - # XXX: why is this necessary? - sane.image.extraDirectories = [ "/nix/persist/var/log" ]; - environment.persistence."/nix/persist" = { - directories = (map-home-dirs cfg.home-dirs) ++ (map-sys-dirs [ - # NB: this `0700` here clobbers the perms for /persist/etc, breaking boot on freshly-deployed devices - # { mode = "0700"; directory = "/etc/NetworkManager/system-connections"; } - # "/etc/nixos" - # "/etc/ssh" # persist only the specific files we want, instead - "/var/log" - "/var/backup" # for e.g. postgres dumps - # "/var/lib/AccountsService" # not sure what this is, but it's empty - "/var/lib/alsa" # preserve output levels, default devices - # "/var/lib/blueman" # files aren't human readable - # TODO: if we changed the bluetooth installer to auto-discover the host MAC address, we could de-persist this - "/var/lib/bluetooth" # preserve bluetooth handshakes - "/var/lib/colord" # preserve color calibrations (?) - # "/var/lib/dhclient" # empty on lappy; dunno about desko - # "/var/lib/fwupd" # not sure why this would need persistent state - # "/var/lib/geoclue" # empty on lappy - # "/var/lib/lockdown" # empty on desko; might store secrets after iOS handshake? - # "/var/lib/logrotate.status" # seems redundant with what's in /var/log? - "/var/lib/machines" # maybe not needed, but would be painful to add a VM and forget. - # "/var/lib/misc" # empty on lappy - # "/var/lib/NetworkManager" # looks to be mostly impermanent state? - # "/var/lib/NetworkManager-fortisslvpn" # empty on lappy - # "/var/lib/nixos" # has some uid/gid maps, but we enforce these to be deterministic. - # "/var/lib/PackageKit" # wtf is this? - # "/var/lib/power-profiles-daemon" # redundant with nixos declarations - # "/var/lib/private" # empty on lappy - # "/var/lib/systemd" # nothing obviously necessary - # "/var/lib/udisks2" # empty on lappy - # "/var/lib/upower" # historic charge data. unnecessary, but maybe used somewhere? - # - # servo additions: - ] ++ cfg.service-dirs); - # /etc/machine-id is a globally unique identifier used for: - # - systemd-networkd: DHCP lease renewal (instead of keying by the MAC address) - # - systemd-journald: to filter logs by host - # - chromium (potentially to track re-installations) - # - gdbus; system services that might upgrade to AF_LOCAL if both services can confirm they're on the same machine - # of these, systemd-networkd is the only legitimate case to persist the machine-id. - # depersisting it should be "safe"; edge-cases like systemd-networkd can be directed to use some other ID if necessary. - # nixos-impermanence shows binding the host ssh priv key to this; i could probably hash the host key into /etc/machine-id if necessary. - # files = [ "/etc/machine-id" ]; - }; + ({ + # XXX: why is this necessary? + sane.image.extraDirectories = [ "/nix/persist/var/log" ]; - # for each edge in a mount path, impermanence gives that target directory the same permissions - # as the matching folder in /nix/persist. - # /nix/persist is often created with poor permissions. so patch them to get the desired directory permissions. - system.activationScripts.fixImpermanencePerms = { - text = "chmod ${config.users.users.colin.homeMode} /nix/persist/home/colin"; - deps = [ "users" ]; - }; - system.activationScripts.createPersistentStorageDirs.deps = [ "fixImpermanencePerms" ]; + environment.persistence = lib.mkMerge (builtins.map (opt: + let + base = if opt.encryptedClearOnBoot + then encrypted-clear-on-boot-base + else persist-base + ; + in { + "${base}".directories = [ + { inherit (opt) directory user group mode; } + ]; + } + ) ingested-dirs); - # secret decoding depends on /etc/ssh keys, which may be persisted - system.activationScripts.setupSecrets.deps = [ "persist-ssh-host-keys" ]; - system.activationScripts.setupSecretsForUsers = lib.mkIf secretsForUsers { - deps = [ "persist-ssh-host-keys" ]; - }; - # populated by ssh.nix, which persists /etc/ssh/host_keys - system.activationScripts.persist-ssh-host-keys.text = lib.mkDefault ""; - }; + # for each edge in a mount path, impermanence gives that target directory the same permissions + # as the matching folder in /nix/persist. + # /nix/persist is often created with poor permissions. so patch them to get the desired directory permissions. + system.activationScripts.fixImpermanencePerms = { + text = "chmod ${config.users.users.colin.homeMode} /nix/persist/home/colin"; + deps = [ "users" ]; + }; + system.activationScripts.createPersistentStorageDirs.deps = [ "fixImpermanencePerms" ]; + + # secret decoding depends on /etc/ssh keys, which may be persisted + system.activationScripts.setupSecrets.deps = [ "persist-ssh-host-keys" ]; + system.activationScripts.setupSecretsForUsers = lib.mkIf secretsForUsers { + deps = [ "persist-ssh-host-keys" ]; + }; + # populated by ssh.nix, which persists /etc/ssh/host_keys + system.activationScripts.persist-ssh-host-keys.text = lib.mkDefault ""; + }) + ]); } diff --git a/modules/services/duplicity.nix b/modules/services/duplicity.nix index b0b93e4c..ef28e80f 100644 --- a/modules/services/duplicity.nix +++ b/modules/services/duplicity.nix @@ -15,7 +15,7 @@ in config = mkIf cfg.enable { # we need this mostly because of the size of duplicity's cache - sane.impermanence.service-dirs = [ "/var/lib/duplicity" ]; + sane.impermanence.dirs = [ "/var/lib/duplicity" ]; services.duplicity.enable = true; services.duplicity.targetUrl = "$DUPLICITY_URL";