hosts/common: persist systemd journal in a way thats encrypted AND doesnt break boot

hosts/common/systemd.nix

@@ -46,17 +46,46 @@ in
     })
   '';
 
-  # hard to wrangle systemd early-logging and my persistence.
+  # persist to PLAINTEXT, i.e. WORLD-READABLE
-  # instead, don't have systemd-journald try to persist its logs at all --
+  # sane.persist.sys.byStore.plaintext = [
-  # use a separate program like rsyslogd (configured elsewhere) to ingest the journal into persistent storage
+  #   { mode = "0755"; path = "/var/log/journal"; method = "bind"; }
-  services.journald.storage = "volatile";
+  # ];
-  # services.journald.extraConfig = ''
+
-  #   # docs: `man journald.conf`
+  # persist to private:
-  #   # merged journald config is deployed to /etc/systemd/journald.conf
+  sane.persist.sys.byStore.private = [
-  #   [Journal]
+    { mode = "0755"; path = "/var/log/journal"; }
-  #   # disable journal compression because the underlying fs is compressed
+  ];
-  #   Compress=no
+
-  # '';
+  # systemd-journal-flush.service switches journald from volatile to persistent storage partway through boot
+  #   ExecStart=journalctl --flush
+  #   ExecStop=journalctl --smart-relinquish-var
+  # - before `--flush`, data is at /run/systemd/journal
+  #   flush _copies_ data from /run... to /var/log/journal
+  #
+  # BUT: systemd-journal-flush.service ships with `Before=systemd-tmpfiles-setup.service`, which causes a circular dependency
+  # that's low-level enough to break boot (sysinit.target).
+  #
+  # no way to _remove_ an upstream `Before` attribute.
+  # instead, neuter the service and re-implement the key bits.
+  systemd.services.systemd-journal-flush.serviceConfig.ExecStart = [
+    ""  #< clear original `ExecStart`
+  ];
+
+  systemd.services.systemd-journal-flush-sane = {
+    description = "flush early logging data to persistent storage, but later in the boot than normal to avoid cycles";
+    wantedBy = [ "systemd-journal-flush.service" ];
+    serviceConfig.ExecStart = "journalctl --flush";
+    unitConfig.DefaultDependencies = false;
+    unitConfig.RequiresMountsFor = [ "/mnt/persist/private" ];
+  };
+
+  services.journald.extraConfig = ''
+    # docs: `man journald.conf`
+    # merged journald config is deployed to /etc/systemd/journald.conf
+    [Journal]
+    # disable journal compression for better debugging (besides, my fs is already compressed)
+    Compress=no
+  '';
 
   # see: `man logind.conf`
   # don’t shutdown when power button is short-pressed (commonly done an accident, or by cats).
@@ -75,7 +104,7 @@ in
     DefaultTimeoutStopSec=${builtins.toString haltTimeout}
   '';
 
-  # hard base systemd services
+  # harden base systemd services
   # see: `systemd-analyze security`
   systemd.services.systemd-rfkill.serviceConfig = {
     AmbientCapabilities = "";