lemoa: inline the cross compilation fix

i don't think any auto-updating bit me, i'm just being pre-emptive

.gitignore

@@ -1,4 +1,5 @@
-.working
+/build
+/.working
 result
 result-*
 /secrets/local.nix

.sops.yaml

@@ -1,9 +1,12 @@
 keys:
   - &user_desko_colin age1tnl4jfgacwkargzeqnhzernw29xx8mkv73xh6ufdyde6q7859slsnzf24x
+  - &user_flowy_colin age1nw3z25gn6l8gxneqw43tp8d2354c83d9sn3r0dqy5tapakdwhyvse0j2cc
   - &user_lappy_colin age1j2pqnl8j0krdzk6npe93s4nnqrzwx978qrc0u570gzlamqpnje9sc8le2g
   - &user_servo_colin age1z8fauff34cdecr6sjkre260luzxcca05kpcwvhx988d306tpcejsp63znu
   - &user_moby_colin age1zsrsvd7j6l62fjxpfd2qnhqlk8wk4p8r0dtxpe4sdgnh2474095qdu7xj9
+  - &host_crappy age1hl50ufuxnqy0jnk8fqeu4tclh4vte2xn2d59pxff0gun20vsmv5sp78chj
   - &host_desko age1vnw7lnfpdpjn62l3u5nyv5xt2c965k96p98kc43mcnyzpetrts9q54mc9v
+  - &host_flowy age1azm6carlm6tdjup37u5dr40585vjujajev70u4glwd9sv7swa99sk6mswx
   - &host_lappy age1w7mectcjku6x3sd8plm8wkn2qfrhv9n6zhzlf329e2r2uycgke8qkf9dyn
   - &host_servo age1tzlyex2z6t88tg9h82943e39shxhmqeyr7ywhlwpdjmyqsndv3qq27x0rf
   - &host_moby age18vq5ktwgeaysucvw9t67drqmg5zd5c5k3le34yqxckkfj7wqdqgsd4ejmt
@@ -12,10 +15,13 @@ creation_rules:
     key_groups:
     - age:
       - *user_desko_colin
+      - *user_flowy_colin
       - *user_lappy_colin
       - *user_servo_colin
       - *user_moby_colin
+      - *host_crappy
       - *host_desko
+      - *host_flowy
       - *host_lappy
       - *host_servo
       - *host_moby
@@ -23,6 +29,7 @@ creation_rules:
     key_groups:
     - age:
       - *user_desko_colin
+      - *user_flowy_colin
       - *user_lappy_colin
       - *user_servo_colin
       - *host_servo
@@ -30,18 +37,28 @@ creation_rules:
     key_groups:
     - age:
       - *user_desko_colin
+      - *user_flowy_colin
       - *user_lappy_colin
       - *host_desko
+  - path_regex: secrets/flowy*
+    key_groups:
+    - age:
+      - *user_lappy_colin
+      - *user_flowy_colin
+      - *user_desko_colin
+      - *host_flowy
   - path_regex: secrets/lappy*
     key_groups:
     - age:
       - *user_lappy_colin
+      - *user_flowy_colin
       - *user_desko_colin
       - *host_lappy
   - path_regex: secrets/moby*
     key_groups:
     - age:
       - *user_desko_colin
+      - *user_flowy_colin
       - *user_lappy_colin
       - *user_moby_colin
       - *host_moby

README.md

@@ -2,6 +2,8 @@
 
 # .❄️≡We|_c0m3 7o m`/ f14k≡❄️.
 
+(er, it's not a flake anymore. welcome to my nix files.)
+
 ## What's Here
 
 this is the top-level repo from which i configure/deploy all my NixOS machines:
@@ -15,60 +17,53 @@ the only hard dependency for my exported pkgs/modules should be [nixpkgs][nixpkg
 building [hosts/](./hosts/) will require [sops][sops].
 
 you might specifically be interested in these files (elaborated further in #key-points-of-interest):
-- ~~[`sxmo-utils`](./pkgs/additional/sxmo-utils/default.nix)~~
-  - ~~[example SXMO deployment](./hosts/modules/gui/sxmo/default.nix)~~
-  - these files will remain until my config settles down, but i no longer use or maintain SXMO.
+- [my packages](./pkgs/by-name)
 - [my implementation of impermanence](./modules/persist/default.nix)
 - my way of deploying dotfiles/configuring programs per-user:
   - [modules/fs/](./modules/fs/default.nix)
   - [modules/programs/](./modules/programs/default.nix)
   - [modules/users/](./modules/users/default.nix)
 
+if you find anything here genuinely useful, message me so that i can work to upstream it!
+
 [nixpkgs]: https://github.com/NixOS/nixpkgs
 [sops]: https://github.com/Mic92/sops-nix
 [uninsane-org]: https://uninsane.org
 
+
 ## Using This Repo In Your Own Config
 
-this should be a pretty "standard" flake. just reference it, and import either
-- `nixosModules.sane` (for the modules)
-- `overlays.pkgs` (for the packages)
-
-or follow the instructions [here][NUR] to use it via the Nix User Repositories.
+follow the instructions [here][NUR] to access my packages through the Nix User Repositories.
 
 [NUR]: https://nur.nix-community.org/
 
+
 ## Layout
 - `doc/`
   - instructions for tasks i find myself doing semi-occasionally in this repo.
 - `hosts/`
-  - the bulk of config which isn't factored with external use in mind.
+  - configs which aren't factored with external use in mind.
   - that is, if you were to add this repo to a flake.nix for your own use,
     you won't likely be depending on anything in this directory.
 - `integrations/`
-  - code intended for consumption by external tools (e.g. the Nix User Repos)
+  - code intended for consumption by external tools (e.g. the Nix User Repos).
 - `modules/`
-  - config which is gated behind `enable` flags, in similar style to nixpkgs'
-    `nixos/` directory.
-  - if you depend on this repo, it's most likely for something in this directory.
-- `nixpatches/`
-  - literally, diffs i apply atop upstream nixpkgs before performing further eval.
+  - config which is gated behind `enable` flags, in similar style to nixpkgs' `nixos/` directory.
+  - if you depend on this repo for anything besides packages, it's most likely for something in this directory.
 - `overlays/`
-  - exposed via the `overlays` output in `flake.nix`.
   - predominantly a list of `callPackage` directives.
 - `pkgs/`
   - derivations for things not yet packaged in nixpkgs.
   - derivations for things from nixpkgs which i need to `override` for some reason.
-  - inline code for wholly custom packages (e.g. `pkgs/additional/sane-scripts/` for CLI tools
+  - inline code for wholly custom packages (e.g. `pkgs/by-name/sane-scripts/` for CLI tools
     that are highly specific to my setup).
 - `scripts/`
-  - scripts which aren't reachable on a deployed system, but may aid manual deployments
+  - scripts which aren't reachable on a deployed system, but may aid manual deployments.
 - `secrets/`
   - encrypted keys, API tokens, anything which one or more of my machines needs
     read access to but shouldn't be world-readable.
-  - not much to see here
+  - not much to see here.
 - `templates/`
-  - exposed via the `templates` output in `flake.nix`.
   - used to instantiate short-lived environments.
   - used to auto-fill the boiler-plate portions of new packages.
 
@@ -87,44 +82,40 @@ i.e. you might find value in using these in your own config:
     - populated with some statically-defined data
     - populated according to some script
     - created as a dependency of some service (e.g. `nginx`)
-  - values defined here are applied neither at evaluation time _nor_ at activation time.
-    - rather, they become systemd services.
-    - systemd manages dependencies
-    - e.g. link `/var/www -> /mnt/my-drive/www` only _after_ `/mnt/my-drive/www` appears)
   - this is akin to using [Home Manager's][home-manager] file API -- the part which lets you
     statically define `~/.config` files -- just with a different philosophy.
+    namely, it avoids any custom activation scripts by leveraging `systemd-tmpfiles`.
 - `modules/persist/`
-  - my alternative to the Impermanence module.
-  - this builds atop `modules/fs/` to achieve things stock impermanence can't:
-    - persist things to encrypted storage which is unlocked at login time (pam_mount).
+  - my implementation of impermanence, built atop the above `fs` module, with a few notable features:
+    - no custom activation scripts or services (uses `systemd-tmpfiles` and `.mount` units)
     - "persist" cache directories -- to free up RAM -- but auto-wipe them on mount
       and encrypt them to ephemeral keys so they're unreadable post shutdown/unmount.
+    - persist to encrypted storage which is unlocked at login time.
 - `modules/programs/`
   - like nixpkgs' `programs` options, but allows both system-wide or per-user deployment.
   - allows `fs` and `persist` config values to be gated behind program deployment:
     - e.g. `/home/<user>/.mozilla/firefox` is persisted only for users who
       `sane.programs.firefox.enableFor.user."<user>" = true;`
   - allows aggressive sandboxing any program:
-    - `sane.programs.firefox.sandbox.method = "bwrap";  # sandbox with bubblewrap`
+    - `sane.programs.firefox.sandbox.enable = true;  # wraps the program so that it isolates itself into a new namespace when invoked`
     - `sane.programs.firefox.sandbox.whitelistWayland = true;  # allow it to render a wayland window`
     - `sane.programs.firefox.sandbox.extraHomePaths = [ "Downloads" ];  # allow it read/write access to ~/Downloads`
     - integrated with `fs` and `persist` modules so that programs' config files and persisted data stores are linked into the sandbox w/o any extra involvement.
 - `modules/users/`
   - convenience layer atop the above modules so that you can just write
     `fs.".config/git"` instead of `fs."/home/colin/.config/git"`
-  - per-user services managed by [s6-rc](https://www.skarnet.org/software/s6-rc/)
-
-some things in here could easily find broader use. if you would find benefit in
-them being factored out of my config, message me and we could work to make that happen.
+  - simplified `systemd.services` API
 
 [home-manager]: https://github.com/nix-community/home-manager
 
+
 ## Mirrors
 
 this repo exists in a few known locations:
 - primary: <https://git.uninsane.org/colin/nix-files>
 - mirror: <https://github.com/nix-community/nur-combined/tree/master/repos/colinsane>
 
+
 ## Contact
 
 if you want to contact me for questions, or collaborate to split something useful into a shared repo, etc,

TODO.md

@@ -1,95 +1,129 @@
 ## BUGS
-- moby: my mobile ISP is adding spoofed AAAA records that break things like wireguard
-  - it only does this when i use their DNS resolvers though: if i run my own recursive resolver, they won't mess with it.
-- moby: mpv uosc always starts at 40% volume
-  - is this just mpv remembering its last-played volume?
-- moby: rofi crashes sporadically
-- mpv: no way to exit fullscreen video on moby
-  - uosc hides controls on FS, and touch doesn't support unhiding
-- i accidentally create sub-splits in sway all the time
-  - especially on moby => unusable
-  - like toplevel is split L/R, and then the L is a tabbed view and the R is a tabbed view
-- Signal restart loop drains battery
-  - decrease s6 restart time?
-- `ssh` access doesn't grant same linux capabilities as login
-- ringer (i.e. dino incoming call) doesn't prevent moby from sleeping
-- sway mouse/kb hotplug doesn't work
-- `nix` operations from lappy hang when `desko` is unreachable
-  - could at least direct the cache to `http://desko-hn:5001`
-- sysvol (volume overlay): when casting with `blast`, sysvol doesn't react to volume changes
+- alacritty Ctrl+N frequently fails to `cd` to the previous directory
+- bunpen dbus sandboxing can't be *nested* (likely a problem in xdg-dbus-proxy)
+- dissent has a memory leak (3G+ after 24hr)
+  - set a max memory use in the systemd service, to force it to restart as it leaks?
+- `rmDbusServices` may break sandboxing
+  - e.g. if the package ships a systemd unit which references $out, then make-sandboxed won't properly update that unit.
+  - `rmDbusServicesInPlace` is not affected
+- mpv: audiocast has mpv sending its output to the builtin speakers unless manually changed
+- syshud (volume overlay): when casting with `blast`, syshud doesn't react to volume changes
+- dissent: if i launch it without net connectivity, it gets stuck at the login, and never tries again
+- newsflash on moby can't play videos
+  - "open in browser" works though -- in mpv
+- gnome-maps can't use geoclue *and* openstreetmap at the same time
+  - get gnome-maps to speak xdg-desktop-portal, and this will be fixed
+- epiphany can't save cookies
+  - see under "preferences", cookies are disabled
+  - prevents logging into websites (OpenStreetMap)
+  - works when sandbox is disabled
+- rsync to ssh target fails because of restrictive sandboxing
+- `/mnt/.servo_ftp` retries every 10s, endlessly, rather than doing a linear backoff
+  - repro by `systemctl stop sftpgo` on servo, then watching `mnt-.servo_ftp.{mount,timer}` on desko
+- `ovpns` (and presumably `doof`) net namespaces aren't firewalled
+  - not great because things like `bitmagnet` expose unprotected admin APIs by default!
+- moby: NetworkManager doesn't connect to network until _after_ `systemctl restart NetworkManager`
+  - probably a dependency ordering issue
+    - e.g. we try to bring up NetworkManager before bringing up `lo`
+  - could be a perms issue (over-restrictive sandboxing)
 
 ## REFACTORING:
-- REMOVE DEPRECATED `crypt` from sftpgo_auth_hook
+- fold hosts/modules/ into toplevel modules/
+- add import checks to my Python nix-shell scripts
 - consolidate ~/dev and ~/ref
   - ~/dev becomes a link to ~/ref/cat/mine
 - fold hosts/common/home/ssh.nix -> hosts/common/users/colin.nix
+- don't hardcode IP addresses so much in servo
+- modules/netns: migrate `sane.netns.$NS.services = [ FOO ]` option to be `systemd.services.$FOO.sane.netns = NS`
+  - then change the ExecStartPre check to not ping `ipinfo.net` or whatever.
+    either port all of `sane-ip-check` to use a self-hosted reflector,
+    or settle for something like `test -eq "$(ip route get ...)" "$expectedGateway"`
 
 ### sops/secrets
-- rework secrets to leverage `sane.fs`
-- remove sops activation script as it's covered by my systemd sane.fs impl
 - user secrets could just use `gocryptfs`, like with ~/private?
   - can gocryptfs support nested filesystems, each with different perms (for desko, moby, etc)?
 
-### roles
-- allow any host to take the role of `uninsane.org`
-  - will make it easier to test new services?
-
 ### upstreaming
-- add updateScripts to all my packages in nixpkgs
-- REVIEW/integrate jellyfin dataDir config: <https://github.com/NixOS/nixpkgs/pull/233617>
+- upstream blueprint-compiler cross fixes -> nixpkgs
+- upstream cargo cross fixes -> nixpkgs
+- upstream `gps-share` package -> nixpkgs
 
 #### upstreaming to non-nixpkgs repos
+- gnome-calls: retry net connection when DNS is down
 - gtk: build schemas even on cross compilation: <https://github.com/NixOS/nixpkgs/pull/247844>
+- linux: upstream PinePhonePro device trees
+- nwg-panel: configurable media controls
+- nwg-panel / playerctl hang fix (i think nwg-panel is what should be patched here)
 
 
 ## IMPROVEMENTS:
+- servo: expand /boot to 2 GiB like all other hosts
+- moby: port to systemd-boot
+- sane-deadlines: show day of the week for upcoming items
+  - and only show on "first" terminal opened; not on Ctrl+N terminals
+- curlftpfs: replace with something better
+  - safer (rust? actively maintained? sandboxable?)
+  - handles spaces/symbols in filenames
+  - has better multi-stream perf (e.g. `sane-sync-music` should be able to copy N items in parallel)
+- firefox: open *all* links (http, https, ...) with system handler
+  - removes the need for open-in-mpv, firefox-xdg-open, etc.
+  - matrix room links *just work*.
+  - `network.protocol-handler.external.https = true` in about:config *seems* to do this,
+    but breaks some webpages (e.g. Pleroma)
+- associate http(s)://*.pdf with my pdf handler
+  - can't do that because lots of applications don't handle URIs
+  - could workaround using a wrapper that downloads the file and then passes it to the program
+- geary: replace with envelope
+  - likely requires updating envelope to a more recent version (for multi-accounting), and therefore updating libadwaita...
+
 ### security/resilience
-- add FTPS support for WAN users of uninsane.org (and possibly require it?)
-- validate duplicity backups!
-- encrypt more ~ dirs (~/archives, ~/records, ..?)
-  - best to do this after i know for sure i have good backups
 - /mnt/desko/home, etc, shouldn't include secrets (~/private)
   - 95% of its use is for remote media access and stuff which isn't in VCS (~/records)
+- harden systemd services:
+  - servo: `coturn.service`
+  - servo: `postgresql.service`
+  - servo: `postfix.service`
+  - servo: `prosody.service`
+  - servo: `slskd.service`
+  - desko: `usbmuxd.service`
+  - servo: `backup-torrents.service`
+  - servo: `dedupe-media.service`
+  - remove SGID /run/wrappers/bin/sendmail, and just add senders to `postdrop` group
 - port all sane.programs to be sandboxed
+  - sandbox `nix`
   - enforce that all `environment.packages` has a sandbox profile (or explicitly opts out)
-  - revisit "non-sandboxable" apps and check that i'm not actually just missing mountpoints
-    - LL_FS_RW=/ isn't enough -- need all mount points like `=/:/proc:/sys:...`.
-  - ensure non-bin package outputs are linked for sandboxed apps
-    - i.e. `outputs.man`, `outputs.debug`, `outputs.doc`, ...
-  - lock down dbus calls within the sandbox
-    - otherwise anyone can `systemd-run --user ...` to potentially escape a sandbox
-    - <https://github.com/flatpak/xdg-dbus-proxy>
-  - remove `.ssh` access from Firefox!
-    - limit access to `~/knowledge/secrets` through an agent that requires GUI approval, so a firefox exploit can't steal all my logins
-  - port sane-sandboxed to a compiled language (hare?)
-    - it adds like 50-70ms launch time _on my laptop_. i'd hate to know how much that is on the pinephone.
-  - remove /run/wrappers from the sandbox path
-    - they're mostly useless when using no-new-privs, just an opportunity to forget to specify deps
+  - enforce granular dbus sandboxing (bunpen-dbus-*)
+- make gnome-keyring-daemon less monolithic
+  - no reason every application with _a_ secret needs to see _all_ secrets
+  - check out oo7-daemon?
+  - also unix-pass based provider: <https://github.com/mdellweg/pass_secret_service>
 - make dconf stuff less monolithic
   - i.e. per-app dconf profiles for those which need it. possible static config.
-- canaries for important services
-  - e.g. daily email checks; daily backup checks
-  - integrate `nix check` into Gitea actions?
+  - flatpak/spectrum has some stuff to proxy dconf per-app
+- rework `programs` API to be just an overlay which wraps each binary in an env with XDG_DATA_DIRS etc set & the config/state links placed in /nix/store instead of $HOME.
 
 ### user experience
+- setup a real calendar system, for recurring events
 - rofi: sort items case-insensitively
-- give `mpv` better `nice`ness?
-- xdg-desktop-portal shouldn't kill children on exit
-  - *maybe* a job for `setsid -f`?
+- rofi: enable mouse mode?
+- mpv: add media looping controls (e.g. loop song, loop playlist)
+- mpv: add/implement an extension to search youtube
+  - apparently `yt-dlp` does searching!
 - replace starship prompt with something more efficient
   - watch `forkstat`: it does way too much
-- cleanup waybar so that it's not invoking playerctl every 2 seconds
+- cleanup nwg-panel so that it's not invoking swaync every second
+  - nwg-panel: doesn't know that virtual-desktop 10/TV exists
 - install apps:
   - display QR codes for WiFi endpoints: <https://linuxphoneapps.org/apps/noappid.wisperwind.wifi2qr/>
   - shopping list (not in nixpkgs): <https://linuxphoneapps.org/apps/ro.hume.cosmin.shoppinglist/>
   - offline Wikipedia (or, add to `wike`)
-  - offline docs viewer (gtk): <https://github.com/workbenchdev/Biblioteca>
   - some type of games manager/launcher
     - Gnome Highscore (retro games)?: <https://gitlab.gnome.org/World/highscore>
-  - better maps for mobile (Osmin (QtQuick)? Pure Maps (Qt/Kirigami)? Gnome Maps is improved in 45)
   - note-taking app: <https://linuxphoneapps.org/categories/note-taking/>
+    - Folio is nice, uses standard markdown, though it only supports flat repos
   - OSK overlay specifically for mobile gaming
     - i.e. mock joysticks, for use with SuperTux and SuperTuxKart
+  - game: Hedgewars
 - install mobile-friendly games:
   - Shattered Pixel Dungeon (nixpkgs `shattered-pixel-dungeon`; doesn't cross-compile b/c openjdk/libIDL) <https://github.com/ebolalex/shattered-pixel-dungeon>
   - UnCiv (Civ V clone; nixpkgs `unciv`; doesn't cross-compile):  <https://github.com/yairm210/UnCiv>
@@ -100,54 +134,49 @@
   - blurble (https://linuxphoneapps.org/games/app.drey.blurble/). nix: not as of 2024-02-05
   - Trivia Quiz (https://linuxphoneapps.org/games/io.github.nokse22.trivia-quiz/)
 - sane-sync-music: remove empty dirs
+- soulseek: install a CLI app usable over ssh
+- moby: replace `spot` with its replacement, `riff` (<https://github.com/Diegovsky/riff>)
 
 #### moby
+- moby: port battery support to something upstreamable
+- moby: install transito/mobroute public transit app: <https://sr.ht/~mil/mobroute/> <https://git.sr.ht/~mil/transito>
+  - see: <https://github.com/NixOS/nixpkgs/pull/335613>
+- moby: consider honeybee instead of gnome-calls for calling? <https://git.sr.ht/~anjan/honeybee>
+  - uses XMPP, so more NAT/WoWLAN-friendly
 - fix cpuidle (gets better power consumption): <https://xnux.eu/log/077.html>
+- fix cpupower for better power/perf
+  - `journalctl -u cpupower --boot` (problem is present on lappy, at least)
+- use dynamic DRAM clocking to reduce power by 0.5W: <https://xnux.eu/log/083.html>
+  - coreboot implements DRAM training for rk3399: <https://gitlab.com/vicencb/kevinboot/-/blob/master/cb/sdram.c>
 - moby: tune keyboard layout
-- SwayNC:
-  - don't show MPRIS if no players detected
-    - this is a problem of playerctld, i guess
-  - add option to change audio output
-  - fix colors (red alert) to match overall theme
-- moby: tune GPS
-  - run only geoclue, and not gpsd, to save power?
-  - tune QGPS setting in eg25-control, for less jitter?
-  - direct mepo to prefer gpsd, with fallback to geoclue, for better accuracy?
-  - configure geoclue to do some smoothing?
-  - manually do smoothing, as some layer between mepo and geoclue/gpsd?
-- moby: show battery state on ssh login
-- moby: improve gPodder launch time
-- moby: theme GTK apps (i.e. non-adwaita styles)
-  - especially, make the menubar collapsible
-  - try Gradience tool specifically for theming adwaita? <https://linuxphoneapps.org/apps/com.github.gradienceteam.gradience/>
+- SwayNC/nwg-panel: add option to change audio output
+- Newsflash: sync OPML on start, same way i do with gpodder
+- better podcasting client?
+- hardware upgrade (OnePlus)?
 
 #### non-moby
 - RSS: integrate a paywall bypass
   - e.g. self-hosted [ladder](https://github.com/everywall/ladder) (like 12ft.io)
-- neovim: set up language server (lsp; rnix-lsp; nvim-lspconfig)
-- neovim: integrate LLMs
-- Helix: make copy-to-system clipboard be the default
-- firefox/librewolf: persist history
+- RSS: have podcasts get downloaded straight into ~/Videos/...
+  - and strip the ads out using Whisper transcription + asking a LLM where the ad breaks are
+- neovim: integrate ollama
+- neovim: better docsets (e.g. c++, glib)
+- firefox: persist history
   - just not cookies or tabs
-- package Nix/NixOS docs for Zeal
-  - install [doc-browser](https://github.com/qwfy/doc-browser)
-  - this supports both dash (zeal) *and* the datasets from <https://devdocs.io> (which includes nix!)
-  - install [devhelp](https://wiki.gnome.org/Apps/Devhelp)  (gnome)
 - have xdg-open parse `<repo:...> URIs (or adjust them so that it _can_ parse)
 - sane-bt-search: show details like 5.1 vs stereo, h264 vs h265
   - maybe just color these "keywords" in all search results?
+- transmission: apply `sane-tag-media` path fix in `torrent-done` script
+  - many .mkv files do appear to be tagged: i'd just need to add support in my own tooling
+  - more aggressively cleanup non-media files after DL (ripper logos, info txts)
 - uninsane.org: make URLs relative to allow local use (and as offline homepage)
 - email: fix so that local mail doesn't go to junk
   - git sendmail flow adds the DKIM signatures, but gets delivered locally w/o having the sig checked, so goes into Junk
   - could change junk filter from "no DKIM success" to explicit "DKIM failed"
-
-### perf
-- debug nixos-rebuild times
-- add `pkgs.impure-cached.<foo>` package set to build things with ccache enabled
-  - every package here can be auto-generated, and marked with some env var so that it doesn't pollute the pure package set
-  - would be super handy for package prototyping!
+  - add an auto-reply address (e.g. `reply-test@uninsane.org`) which reflects all incoming mail; use this (or a friend running this) for liveness checks
 
 ## NEW FEATURES:
+- migrate Kodi box to nix
 - migrate MAME cabinet to nix
   - boot it from PXE from servo?
 - enable IPv6

default.nix

@@ -1,9 +1,5 @@
-# limited, non-flake interface to this repo.
-# this file exposes the same view into `pkgs` which the flake would see when evaluated.
-#
-# the primary purpose of this file is so i can run `updateScript`s which expect
-# the root to be `default.nix`
-{ pkgs ? import <nixpkgs> {} }:
-pkgs.appendOverlays [
-  (import ./overlays/all.nix)
-]
+{ ... }@args:
+let
+  sane-nix-files = import ./pkgs/by-name/sane-nix-files/package.nix { };
+in
+  import "${sane-nix-files}/impure.nix" args

doc/adding-a-host.md

@@ -0,0 +1,33 @@
+to add a host:
+- create the new nix targets
+  - hosts/by-name/HOST
+  - let the toplevel (impure.nix) know about HOST
+  - let the other hosts know about this host (hosts/common/hosts.nix)
+  - let sops know about the host's pubkey (.sops.yaml)
+    - re-encrypt all sops keys in secrets/common
+- build and flash an image
+- optionally expand the rootfs
+  - `cfdisk /dev/sda2` -> resize partition
+  - `mount /dev/sda2 boot`
+  - `btrfs filesystem resize max root`
+- setup required persistent directories
+  - `mkdir -p root/persist/private`
+  - `gocryptfs -init root/persist/private`
+  - then boot the device, and for every dangling symlink in ~/.local/share, ~/.cache, do `mkdir -p` on it
+- setup host ssh
+  - `mkdir -p root/persist/plaintext/etc/ssh/host_keys`
+  - boot the machine and let it create its own ssh keys
+  - add the pubkey to `hosts/common/hosts.nix`
+- setup user ssh
+  - `ssh-keygen`. don't enter any password; it's stored in a password-encrypted fs.
+  - add the pubkey to `hosts/common/hosts.nix`
+- allow the new host to view secrets
+  - instructions in hosts/common/secrets.nix
+  - run `ssh-to-age` on user/host pubkeys
+  - add age key to .sops.yaml
+  - update encrypted secrets: `find secrets -type f -exec sops updatekeys -y '{}' ';'`
+- setup wireguard keys
+  - `pk=$(wg genkey)`
+  - `echo "$pk" | sops encrypt --filename-override secrets/$(hostname)/wg-home.priv.bin --output secrets/$(hostname)/wg-home.priv.bin`
+  - `pub=$(echo "$pk" | wg pubkey)`
+  - add pubkey to hosts/common/hosts.nix

doc/migrating-storage-device.md

@@ -0,0 +1,49 @@
+## migrating a host to a new drive
+### 1. copy persistent data off of the host:
+```sh
+$ mkdir -p mnt old/persist
+$ mount /dev/$old mnt
+$ rsync -arv mnt/persist/ old/persist/
+```
+
+### 2. flash the new drive
+```
+$ nix-build -A hosts.moby.img
+$ dd if=$(readlink ./result) of=/dev/$new bs=4M oflag=direct conv=sync status=progress
+```
+
+### 3.1. expand the partition
+```sh
+$ cfdisk /dev/$new
+# scroll to the last partition
+> Resize
+  leave at default (max)
+> Write
+  type "yes"
+> Quit
+```
+### 3.2. expand the filesystem
+```
+$ mkdir -p /mnt/$new
+$ mount /dev/$new /mnt/$new
+$ btrfs filesystem resize max /mnt/$new
+```
+
+### 4. copy data onto the new host
+```
+$ mkdir /mnt/$new
+$ mount /dev/$new /mnt/$new
+# if you want to use btrfs snapshots (e.g. snapper), then create the data directory as a subvolume:
+$ btrfs subvolume create /mnt/$new/persist
+# restore the data
+$ rsync -arv old/persist/ /mnt/$new/persist/
+```
+
+### 5. ensure/fix ownership
+```
+$ chmod -R a+rX /mnt/$new/nix
+# or, let the nix daemon do it:
+$ nix copy --no-check-sigs --to /mnt/$new $(nix-build -A hosts.moby)
+```
+
+### 6. insert the disk into the system, and boot!

doc/recovery.md

@@ -0,0 +1,12 @@
+## deploying to SD card
+- build a toplevel config: `nix build '.#hosts.moby.img'`
+- mount a system:
+  - `mkdir -p root/{nix,boot}`
+  - `mount /dev/sdX1 root/boot`
+  - `mount /dev/sdX2 root/nix`
+- copy the config:
+  - `sudo nix copy --no-check-sigs --to root/ $(readlink result)`
+    - nix will copy stuff to `root/nix/store`
+- install the boot files:
+  - `sudo /nix/store/sbwpwngjlgw4f736ay9hgi69pj3fdwk5-extlinux-conf-builder.sh -d ./root/boot -t 5 -c $(readlink ./result)`
+    - extlinux-conf-builder can be found in `/run/current-system/bin/switch-to-configuration`

flake.lock

@@ -1,330 +0,0 @@
-{
-  "nodes": {
-    "flake-compat": {
-      "locked": {
-        "lastModified": 1688025799,
-        "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=",
-        "owner": "nix-community",
-        "repo": "flake-compat",
-        "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "nixpkgs-wayland",
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712014858,
-        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "lib-aggregate": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "nixpkgs-lib": "nixpkgs-lib"
-      },
-      "locked": {
-        "lastModified": 1715515815,
-        "narHash": "sha256-yaLScMHNFCH6SbB0HSA/8DWDgK0PyOhCXoFTdHlWkhk=",
-        "owner": "nix-community",
-        "repo": "lib-aggregate",
-        "rev": "09883ca828e8cfaacdb09e29190a7b84ad1d9925",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "lib-aggregate",
-        "type": "github"
-      }
-    },
-    "mobile-nixos": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1694749521,
-        "narHash": "sha256-MiVokKlpcJmfoGuWAMeW1En7gZ5hk0rCQArYm6P9XCc=",
-        "owner": "nixos",
-        "repo": "mobile-nixos",
-        "rev": "d25d3b87e7f300d8066e31d792337d9cd7ecd23b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "d25d3b87e7f300d8066e31d792337d9cd7ecd23b",
-        "repo": "mobile-nixos",
-        "type": "github"
-      }
-    },
-    "nix-eval-jobs": {
-      "inputs": {
-        "flake-parts": "flake-parts",
-        "nix-github-actions": "nix-github-actions",
-        "nixpkgs": "nixpkgs",
-        "treefmt-nix": "treefmt-nix"
-      },
-      "locked": {
-        "lastModified": 1715248291,
-        "narHash": "sha256-npC9Swu4VIlRIiEP0XFGoIukd6vOufS/M3PdHk6rQpc=",
-        "owner": "nix-community",
-        "repo": "nix-eval-jobs",
-        "rev": "63154bdfb22091041b307d17863bdc0e01a32a00",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-eval-jobs",
-        "type": "github"
-      }
-    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-wayland",
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1703863825,
-        "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1715037484,
-        "narHash": "sha256-OUt8xQFmBU96Hmm4T9tOWTu4oCswCzoVl+pxSq/kiFc=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "ad7efee13e0d216bf29992311536fce1d3eefbef",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-lib": {
-      "locked": {
-        "lastModified": 1715474941,
-        "narHash": "sha256-CNCqCGOHdxuiVnVkhTpp2WcqSSmSfeQjubhDOcgwGjU=",
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
-        "rev": "58e03b95f65dfdca21979a081aa62db0eed6b1d8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
-        "type": "github"
-      }
-    },
-    "nixpkgs-next-unpatched": {
-      "locked": {
-        "lastModified": 1715601680,
-        "narHash": "sha256-Gmz6U8NMZVVnP6AGX4sMl4X6RcQBASPl/2Gj9R5k1Pk=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "eda36d7cf3391ad06097009b08822fb74acd5e00",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "staging-next",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1715458492,
-        "narHash": "sha256-q0OFeZqKQaik2U8wwGDsELEkgoZMK7gvfF6tTXkpsqE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "8e47858badee5594292921c2668c11004c3b0142",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-unpatched": {
-      "locked": {
-        "lastModified": 1715616096,
-        "narHash": "sha256-rxh2XECb5hRzgNR4Xqj3aAjg6821LmNTVRfF6sUW6fI=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "0a949cf2618e8eab83aa008f1f8e03db137ed36c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "master",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-wayland": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "lib-aggregate": "lib-aggregate",
-        "nix-eval-jobs": "nix-eval-jobs",
-        "nixpkgs": [
-          "nixpkgs-unpatched"
-        ]
-      },
-      "locked": {
-        "lastModified": 1715609745,
-        "narHash": "sha256-z2lQ7G1AxljvYeqrHWjc1ctOI4QZP06vPtvLYJWfZSc=",
-        "owner": "nix-community",
-        "repo": "nixpkgs-wayland",
-        "rev": "ed18785b8816fa878bdd9df7f2e8722695401ef8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nixpkgs-wayland",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "mobile-nixos": "mobile-nixos",
-        "nixpkgs-next-unpatched": "nixpkgs-next-unpatched",
-        "nixpkgs-unpatched": "nixpkgs-unpatched",
-        "nixpkgs-wayland": "nixpkgs-wayland",
-        "sops-nix": "sops-nix",
-        "uninsane-dot-org": "uninsane-dot-org"
-      }
-    },
-    "sops-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-unpatched"
-        ],
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1715482972,
-        "narHash": "sha256-y1uMzXNlrVOWYj1YNcsGYLm4TOC2aJrwoUY1NjQs9fM=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "b6cb5de2ce57acb10ecdaaf9bbd62a5ff24fa02e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-wayland",
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1711963903,
-        "narHash": "sha256-N3QDhoaX+paWXHbEXZapqd1r95mdshxToGowtjtYkGI=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "49dc4a92b02b8e68798abd99184f228243b6e3ac",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
-    },
-    "uninsane-dot-org": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-unpatched"
-        ]
-      },
-      "locked": {
-        "lastModified": 1713198740,
-        "narHash": "sha256-8SUaqMJdAkMOI9zhvlToL7eCr5Sl+2o2pDQ7nq+HoJU=",
-        "ref": "refs/heads/master",
-        "rev": "af8420d1c256d990b5e24de14ad8592a5d85bf77",
-        "revCount": 239,
-        "type": "git",
-        "url": "https://git.uninsane.org/colin/uninsane"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.uninsane.org/colin/uninsane"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

flake.nix

@@ -1,659 +0,0 @@
-# FLAKE FEEDBACK:
-# - if flake inputs are meant to be human-readable, a human should be able to easily track them down given the URL.
-#   - this is not the case with registry URLs, like `nixpkgs/nixos-22.11`.
-#   - this is marginally the case with schemes like `github:nixos/nixpkgs`.
-#   - given the *existing* `git+https://` scheme, i propose expressing github URLs similarly:
-#     - `github+https://github.com/nixos/nixpkgs/tree/nixos-22.11`
-#     - this would allow for the same optimizations as today's `github:nixos/nixpkgs`, but without obscuring the source.
-#       a code reader could view the source being referenced simply by clicking the https:// portion of that URI.
-# - need some way to apply local patches to inputs.
-#
-#
-# DEVELOPMENT DOCS:
-# - Flake docs: <https://nixos.wiki/wiki/Flakes>
-# - Flake RFC: <https://github.com/tweag/rfcs/blob/flakes/rfcs/0049-flakes.md>
-#   - Discussion: <https://github.com/NixOS/rfcs/pull/49>
-# - <https://serokell.io/blog/practical-nix-flakes>
-#
-#
-# COMMON OPERATIONS:
-# - update a specific flake input:
-#   - `nix flake lock --update-input nixpkgs`
-
-{
-  # XXX: use the `github:` scheme instead of the more readable git+https: because it's *way* more efficient
-  # preferably, i would rewrite the human-readable https URLs to nix-specific github: URLs with a helper,
-  # but `inputs` is required to be a strict attrset: not an expression.
-  inputs = {
-    # branch workflow:
-    # - daily:
-    #   - nixos-unstable cut from master after enough packages have been built in caches.
-    # - every 6 hours:
-    #   - master auto-merged into staging and staging-next
-    #   - staging-next auto-merged into staging.
-    # - manually, approximately once per month:
-    #   - staging-next is cut from staging.
-    #   - staging-next merged into master.
-    #
-    # which branch to source from?
-    # - nixos-unstable: for everyday development; it provides good caching
-    # - master: temporarily if i'm otherwise cherry-picking lots of already-applied patches
-    # - staging-next: if testing stuff that's been PR'd into staging, i.e. base library updates.
-    # - staging: maybe if no staging-next -> master PR has been cut yet?
-    #
-    # <https://github.com/nixos/nixpkgs/tree/nixos-unstable>
-    # nixpkgs-unpatched.url = "github:nixos/nixpkgs?ref=nixos-unstable";
-    nixpkgs-unpatched.url = "github:nixos/nixpkgs?ref=master";
-    # nixpkgs-unpatched.url = "github:nixos/nixpkgs?ref=nixos-staging";
-    # nixpkgs-unpatched.url = "github:nixos/nixpkgs?ref=nixos-staging-next";
-    nixpkgs-next-unpatched.url = "github:nixos/nixpkgs?ref=staging-next";
-
-    nixpkgs-wayland = {
-      url = "github:nix-community/nixpkgs-wayland";
-      inputs.nixpkgs.follows = "nixpkgs-unpatched";
-    };
-
-    mobile-nixos = {
-      # <https://github.com/nixos/mobile-nixos>
-      # only used for building disk images, not relevant after deployment
-      # TODO: replace with something else. commit `0f3ac0bef1aea70254a3bae35e3cc2561623f4c1`
-      # replaces the imageBuilder with a "new implementation from celun" and wildly breaks my use.
-      # pinning to d25d3b... is equivalent to holding at 2023-09-15
-      url = "github:nixos/mobile-nixos?ref=d25d3b87e7f300d8066e31d792337d9cd7ecd23b";
-      flake = false;
-    };
-    sops-nix = {
-      # <https://github.com/Mic92/sops-nix>
-      # used to distribute secrets to my hosts
-      url = "github:Mic92/sops-nix";
-      # inputs.nixpkgs.follows = "nixpkgs";
-      inputs.nixpkgs.follows = "nixpkgs-unpatched";
-    };
-    uninsane-dot-org = {
-      # provides the package to deploy <https://uninsane.org>, used only when building the servo host
-      url = "git+https://git.uninsane.org/colin/uninsane";
-      # inputs.nixpkgs.follows = "nixpkgs";
-      inputs.nixpkgs.follows = "nixpkgs-unpatched";
-    };
-  };
-
-  outputs = {
-    self,
-    nixpkgs-unpatched,
-    nixpkgs-next-unpatched ? nixpkgs-unpatched,
-    nixpkgs-wayland,
-    mobile-nixos,
-    sops-nix,
-    uninsane-dot-org,
-    ...
-  }@inputs:
-    let
-      inherit (builtins) attrNames elem listToAttrs map mapAttrs;
-      # redefine some nixpkgs `lib` functions to avoid the infinite recursion
-      # of if we tried to use patched `nixpkgs.lib` as part of the patching process.
-      mapAttrs' = f: set:
-        listToAttrs (map (attr: f attr set.${attr}) (attrNames set));
-      optionalAttrs = cond: attrs: if cond then attrs else {};
-      # mapAttrs but without the `name` argument
-      mapAttrValues = f: mapAttrs (_: f);
-
-      # rather than apply our nixpkgs patches as a flake input, do that here instead.
-      # this (temporarily?) resolves the bad UX wherein a subflake residing in the same git
-      # repo as the main flake causes the main flake to have an unstable hash.
-      patchNixpkgs = variant: nixpkgs: (import ./nixpatches/flake.nix).outputs {
-        inherit variant nixpkgs;
-        self = patchNixpkgs variant nixpkgs;
-      };
-
-      nixpkgs' = patchNixpkgs "master" nixpkgs-unpatched;
-      nixpkgsCompiledBy = system: nixpkgs'.legacyPackages."${system}";
-
-      evalHost = { name, local, target, variant ? null, nixpkgs ? nixpkgs' }: nixpkgs.lib.nixosSystem {
-        system = target;
-        modules = [
-          {
-            nixpkgs.buildPlatform.system = local;
-            # nixpkgs.config.replaceStdenv = { pkgs }: pkgs.ccacheStdenv;
-          }
-          (optionalAttrs (local != target) {
-            # XXX(2023/12/11): cache.nixos.org uses `system = ...` instead of `hostPlatform.system`, and that choice impacts the closure of every package.
-            # so avoid specifying hostPlatform.system on non-cross builds, so i can use upstream caches.
-            nixpkgs.hostPlatform.system = target;
-          })
-          (optionalAttrs (variant == "light") {
-            sane.maxBuildCost = 2;
-          })
-          (optionalAttrs (variant == "min") {
-            sane.maxBuildCost = 0;
-          })
-          (import ./hosts/instantiate.nix { hostName = name; })
-          self.nixosModules.default
-          self.nixosModules.passthru
-          {
-            nixpkgs.overlays = [
-              self.overlays.passthru
-              self.overlays.sane-all
-            ];
-          }
-        ];
-      };
-    in {
-      nixosConfigurations = let
-        hosts = {
-          servo       = { name = "servo";  local = "x86_64-linux"; target = "x86_64-linux";  };
-          desko       = { name = "desko";  local = "x86_64-linux"; target = "x86_64-linux";  };
-          desko-light = { name = "desko";  local = "x86_64-linux"; target = "x86_64-linux";  variant = "light"; };
-          lappy       = { name = "lappy";  local = "x86_64-linux"; target = "x86_64-linux";  };
-          lappy-light = { name = "lappy";  local = "x86_64-linux"; target = "x86_64-linux";  variant = "light"; };
-          lappy-min =   { name = "lappy";  local = "x86_64-linux"; target = "x86_64-linux";  variant = "min"; };
-          moby        = { name = "moby";   local = "x86_64-linux"; target = "aarch64-linux"; };
-          moby-light  = { name = "moby";   local = "x86_64-linux"; target = "aarch64-linux"; variant = "light"; };
-          moby-min  =   { name = "moby";   local = "x86_64-linux"; target = "aarch64-linux"; variant = "min"; };
-          rescue      = { name = "rescue"; local = "x86_64-linux"; target = "x86_64-linux";  };
-        };
-        hostsNext = mapAttrs' (h: v: {
-          name = "${h}-next";
-          value = v // { nixpkgs = patchNixpkgs "staging-next" nixpkgs-next-unpatched; };
-        }) hosts;
-      in mapAttrValues evalHost (
-        hosts // hostsNext
-      );
-
-      # unofficial output
-      # this produces a EFI-bootable .img file (GPT with a /boot partition and a system (/ or /nix) partition).
-      # after building this:
-      #   - flash it to a bootable medium (SD card, flash drive, HDD)
-      #   - resize the root partition (use cfdisk)
-      #   - mount the part
-      #      - chown root:nixbld <part>/nix/store
-      #      - chown root:root -R <part>/nix/store/*
-      #      - chown root:root -R <part>/persist  # if using impermanence
-      #      - populate any important things (persist/, home/colin/.ssh, etc)
-      #   - boot
-      #   - if fs wasn't resized automatically, then `sudo btrfs filesystem resize max /`
-      #   - checkout this flake into /etc/nixos AND UPDATE THE FS UUIDS.
-      #   - `nixos-rebuild --flake './#<host>' switch`
-      imgs = mapAttrValues (host: host.config.system.build.img) self.nixosConfigurations;
-
-      # unofficial output
-      hostConfigs = mapAttrValues (host: host.config) self.nixosConfigurations;
-      hostSystems = mapAttrValues (host: host.config.system.build.toplevel) self.nixosConfigurations;
-      hostPkgs = mapAttrValues (host: host.config.system.build.pkgs) self.nixosConfigurations;
-      hostPrograms = mapAttrValues (host: mapAttrValues (p: p.package) host.config.sane.programs) self.nixosConfigurations;
-
-      patched.nixpkgs = nixpkgs';
-
-      overlays = {
-        # N.B.: `nix flake check` requires every overlay to take `final: prev:` at defn site,
-        #   hence the weird redundancy.
-        default = final: prev: self.overlays.pkgs final prev;
-        sane-all = final: prev: import ./overlays/all.nix final prev;
-        pkgs = final: prev: import ./overlays/pkgs.nix final prev;
-        pins = final: prev: import ./overlays/pins.nix final prev;
-        preferences = final: prev: import ./overlays/preferences.nix final prev;
-        passthru = final: prev:
-          let
-            mobile = (import "${mobile-nixos}/overlay/overlay.nix");
-            uninsane = uninsane-dot-org.overlays.default;
-            wayland = final: prev: {
-              # default is to dump the packages into `waylandPkgs` *and* the toplevel.
-              # but i just want the `waylandPkgs` set
-              inherit (nixpkgs-wayland.overlays.default final prev)
-                waylandPkgs
-                new-wayland-protocols  #< 2024/03/10: nixpkgs-wayland assumes this will be in the toplevel
-              ;
-            };
-          in
-            (mobile final prev)
-            // (uninsane final prev)
-            // (wayland final prev)
-          ;
-      };
-
-      nixosModules = rec {
-        default = sane;
-        sane = import ./modules;
-        passthru = { ... }: {
-          imports = [
-            sops-nix.nixosModules.sops
-          ];
-        };
-      };
-
-      # this includes both our native packages and all the nixpkgs packages.
-      legacyPackages =
-        let
-          allPkgsFor = sys: (nixpkgsCompiledBy sys).appendOverlays [
-            self.overlays.passthru self.overlays.pkgs
-          ];
-        in {
-          x86_64-linux = allPkgsFor "x86_64-linux";
-          aarch64-linux = allPkgsFor "aarch64-linux";
-        };
-
-      # extract only our own packages from the full set.
-      # because of `nix flake check`, we flatten the package set and only surface x86_64-linux packages.
-      packages = mapAttrs
-        (system: passthruPkgs: passthruPkgs.lib.filterAttrs
-          (name: pkg:
-            # keep only packages which will pass `nix flake check`, i.e. keep only:
-            # - derivations (not package sets)
-            # - packages that build for the given platform
-            (! elem name [ "feeds" "pythonPackagesExtensions" ])
-            && (passthruPkgs.lib.meta.availableOn passthruPkgs.stdenv.hostPlatform pkg)
-          )
-          (
-            # expose sane packages and chosen inputs (uninsane.org)
-            (import ./pkgs { pkgs = passthruPkgs; }) // {
-              inherit (passthruPkgs) uninsane-dot-org;
-            }
-          )
-        )
-        # self.legacyPackages;
-        {
-          x86_64-linux = (nixpkgsCompiledBy "x86_64-linux").appendOverlays [
-            self.overlays.passthru
-          ];
-        }
-      ;
-
-      apps."x86_64-linux" =
-        let
-          pkgs = self.legacyPackages."x86_64-linux";
-          sanePkgs = import ./pkgs { inherit pkgs; };
-          deployScript = host: addr: action: pkgs.writeShellScript "deploy-${host}" ''
-            set -e
-
-            host="${host}"
-            addr="${addr}"
-            action="${if action != null then action else ""}"
-            runOnTarget() {
-              # run the command ($@) on the machine we're deploying to.
-              # if that's a remote machine, then do it via ssh, else local shell.
-              if [ -n "$addr" ]; then
-                ssh "$addr" "$@"
-              else
-                "$@"
-              fi
-            }
-
-            nix build ".#nixosConfigurations.$host.config.system.build.toplevel" --out-link "./build/result-$host" "$@"
-            storePath="$(readlink ./build/result-$host)"
-
-            # mimic `nixos-rebuild --target-host`, in effect:
-            # - nix-copy-closure ...
-            # - nix-env --set ...
-            # - switch-to-configuration <boot|dry-activate|switch|test|>
-            # avoid the actual `nixos-rebuild` for a few reasons:
-            # - fewer nix evals
-            # - more introspectability and debuggability
-            # - sandbox friendliness (especially: `git` doesn't have to be run as root)
-
-            if [ -n "$addr" ]; then
-              sudo nix store sign -r -k /run/secrets/nix_serve_privkey "$storePath"
-              # add more `-v` for more verbosity (up to 5).
-              # builders-use-substitutes false: optimizes so that the remote machine doesn't try to get paths from its substituters.
-              #   we already have all paths here, and the remote substitution is slow to check and SERIOUSLY flaky on moby in particular.
-              nix copy -vv --option builders-use-substitutes false --to "ssh-ng://$addr" "$storePath"
-            fi
-
-            if [ -n "$action" ]; then
-              runOnTarget sudo nix-env -p /nix/var/nix/profiles/system --set "$storePath"
-              runOnTarget sudo "$storePath/bin/switch-to-configuration" "$action"
-            fi
-          '';
-          deployApp = host: addr: action: {
-            type = "app";
-            program = ''${deployScript host addr action}'';
-          };
-
-          # pkg updating.
-          # a cleaner alternative lives here: <https://discourse.nixos.org/t/how-can-i-run-the-updatescript-of-personal-packages/25274/2>
-          # mkUpdater :: [ String ] -> { type = "app"; program = path; }
-          mkUpdater = attrPath: {
-            type = "app";
-            program = let
-              pkg = pkgs.lib.getAttrFromPath attrPath sanePkgs;
-              strAttrPath = pkgs.lib.concatStringsSep "." attrPath;
-              commandArgv = pkg.updateScript.command or pkg.updateScript;
-              command = pkgs.lib.escapeShellArgs commandArgv;
-            in builtins.toString (pkgs.writeShellScript "update-${strAttrPath}" ''
-              export UPDATE_NIX_NAME=${pkg.name}
-              export UPDATE_NIX_PNAME=${pkg.pname}
-              export UPDATE_NIX_OLD_VERSION=${pkg.version}
-              export UPDATE_NIX_ATTR_PATH=${strAttrPath}
-              ${command}
-            '');
-          };
-          mkUpdatersNoAliases = opts: basePath: pkgs.lib.concatMapAttrs
-            (name: pkg:
-              if pkg.recurseForDerivations or false then {
-                "${name}" = mkUpdaters opts (basePath ++ [ name ]);
-              } else if pkg.updateScript or null != null then {
-                "${name}" = mkUpdater (basePath ++ [ name ]);
-              } else {}
-            )
-            (pkgs.lib.getAttrFromPath basePath sanePkgs);
-          mkUpdaters = { ignore ? [], flakePrefix ? [] }@opts: basePath:
-            let
-              updaters = mkUpdatersNoAliases opts basePath;
-              invokeUpdater = name: pkg:
-                let
-                  fullPath = basePath ++ [ name ];
-                  doUpdateByDefault = !builtins.elem fullPath ignore;
-
-                  # in case `name` has a `.` in it, we have to quote it
-                  escapedPath = builtins.map (p: ''"${p}"'') fullPath;
-                  updatePath = builtins.concatStringsSep "." (flakePrefix ++ escapedPath);
-                in pkgs.lib.optionalString doUpdateByDefault (
-                  pkgs.lib.escapeShellArgs [
-                    "nix" "run" ".#${updatePath}"
-                  ]
-                );
-            in {
-              type = "app";
-              # top-level app just invokes the updater of everything one layer below it
-              program = builtins.toString (pkgs.writeShellScript
-                (builtins.concatStringsSep "-" (flakePrefix ++ basePath))
-                (builtins.concatStringsSep
-                  "\n"
-                  (pkgs.lib.mapAttrsToList invokeUpdater updaters)
-                )
-              );
-            } // updaters;
-        in {
-          help = {
-            type = "app";
-            program = let
-              helpMsg = builtins.toFile "nixos-config-help-message" ''
-                commands:
-                - `nix run '.#help'`
-                  - show this message
-                - `nix run '.#update.pkgs'`
-                  - updates every package
-                - `nix run '.#update.feeds'`
-                  - updates metadata for all feeds
-                - `nix run '.#init-feed' <url>`
-                - `nix run '.#deploy.{desko,lappy,moby,servo}[-light|-test]' [nix args ...]`
-                  - build and deploy the host
-                - `nix run '.#preDeploy.{desko,lappy,moby,servo}[-light]' [nix args ...]`
-                  - copy closures to a host, but don't activate it
-                  - or `nix run '.#preDeploy'` to target all hosts
-                - `nix run '.#check'`
-                  - make sure all systems build; NUR evaluates
-                - `nix run '.#bench'`
-                  - benchmark the eval time of common targets this flake provides
-
-                specific build targets of interest:
-                - `nix build '.#imgs.rescue'`
-              '';
-            in builtins.toString (pkgs.writeShellScript "nixos-config-help" ''
-              cat ${helpMsg}
-              echo ""
-              echo "complete flake structure:"
-              nix flake show --option allow-import-from-derivation true
-            '');
-          };
-          # wrangle some names to get package updaters which refer back into the flake, but also conditionally ignore certain paths (e.g. sane.feeds).
-          # TODO: better design
-          update = rec {
-            _impl.pkgs.sane = mkUpdaters { flakePrefix = [ "update" "_impl" "pkgs" ]; ignore = [ [ "sane" "feeds" ] ]; } [ "sane" ];
-            pkgs = _impl.pkgs.sane;
-            _impl.feeds.sane.feeds = mkUpdaters { flakePrefix = [ "update" "_impl" "feeds" ]; } [ "sane" "feeds" ];
-            feeds = _impl.feeds.sane.feeds;
-          };
-
-          init-feed = {
-            type = "app";
-            program = "${pkgs.feeds.init-feed}";
-          };
-
-          deploy = {
-            desko       = deployApp "desko"       "desko" "switch";
-            desko-light = deployApp "desko-light" "desko" "switch";
-            lappy       = deployApp "lappy"       "lappy" "switch";
-            lappy-light = deployApp "lappy-light" "lappy" "switch";
-            lappy-min   = deployApp "lappy-min"   "lappy" "switch";
-            moby        = deployApp "moby"        "moby"  "switch";
-            moby-light  = deployApp "moby-light"  "moby"  "switch";
-            moby-min  =   deployApp "moby-min"    "moby"  "switch";
-            moby-test   = deployApp "moby"        "moby"  "test";
-            servo       = deployApp "servo"       "servo" "switch";
-
-            # like `nixos-rebuild --flake . switch`
-            self        = deployApp "$(hostname)"       "" "switch";
-            self-light  = deployApp "$(hostname)-light" "" "switch";
-            self-min    = deployApp "$(hostname)-min"   "" "switch";
-
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "deploy-all" ''
-              nix run '.#deploy.lappy'
-              nix run '.#deploy.moby'
-              nix run '.#deploy.desko'
-              nix run '.#deploy.servo'
-            '');
-          };
-          preDeploy = {
-            # build the host and copy the runtime closure to that host, but don't activate it.
-            desko       = deployApp "desko"       "desko" null;
-            desko-light = deployApp "desko-light" "desko" null;
-            lappy       = deployApp "lappy"       "lappy" null;
-            lappy-light = deployApp "lappy-light" "lappy" null;
-            lappy-min   = deployApp "lappy-min"   "lappy" null;
-            moby        = deployApp "moby"        "moby"  null;
-            moby-light  = deployApp "moby-light"  "moby"  null;
-            moby-min    = deployApp "moby-min"    "moby"  null;
-            servo       = deployApp "servo"       "servo" null;
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "predeploy-all" ''
-              # copy the -min/-light variants first; this might be run while waiting on a full build. or the full build failed.
-              nix run '.#preDeploy.moby-min' -- "$@"
-              nix run '.#preDeploy.lappy-min' -- "$@"
-              nix run '.#preDeploy.moby-light' -- "$@"
-              nix run '.#preDeploy.lappy-light' -- "$@"
-              nix run '.#preDeploy.desko-light' -- "$@"
-              nix run '.#preDeploy.lappy' -- "$@"
-              nix run '.#preDeploy.servo' -- "$@"
-              nix run '.#preDeploy.moby' -- "$@"
-              nix run '.#preDeploy.desko' -- "$@"
-            '');
-          };
-
-          sync = {
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "sync-all" ''
-              RC_lappy=$(nix run '.#sync.lappy' -- "$@")
-              RC_moby=$(nix run '.#sync.moby' -- "$@")
-              RC_desko=$(nix run '.#sync.desko' -- "$@")
-
-              echo "lappy: $RC_lappy"
-              echo "moby: $RC_moby"
-              echo "desko: $RC_desko"
-            '');
-          };
-
-          sync.desko = {
-            # copy music from servo to desko
-            # can run this from any device that has ssh access to desko and servo
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "sync-to-desko" ''
-              sudo mount /mnt/desko/home
-              ${pkgs.sane-scripts.sync-music}/bin/sane-sync-music --compat /mnt/servo/media/Music /mnt/desko/home/Music "$@"
-            '');
-          };
-
-          sync.lappy = {
-            # copy music from servo to lappy
-            # can run this from any device that has ssh access to lappy and servo
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "sync-to-lappy" ''
-              sudo mount /mnt/lappy/home
-              ${pkgs.sane-scripts.sync-music}/bin/sane-sync-music --compress --compat /mnt/servo/media/Music /mnt/lappy/home/Music "$@"
-            '');
-          };
-
-          sync.moby = {
-            # copy music from servo to moby
-            # can run this from any device that has ssh access to moby and servo
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "sync-to-moby" ''
-              sudo mount /mnt/moby/home
-              sudo mount /mnt/desko/home
-              ${pkgs.rsync}/bin/rsync -arv --exclude servo-macros /mnt/moby/home/Pictures/ /mnt/desko/home/Pictures/moby/
-              # N.B.: limited by network/disk -> reduce job count to improve pause/resume behavior
-              ${pkgs.sane-scripts.sync-music}/bin/sane-sync-music --compress --compat --jobs 4 /mnt/servo/media/Music /mnt/moby/home/Music "$@"
-            '');
-          };
-
-          check = {
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "check-all" ''
-              nix run '.#check.nur'
-              RC0=$?
-              nix run '.#check.hostConfigs'
-              RC1=$?
-              nix run '.#check.rescue'
-              RC2=$?
-              echo "nur: $RC0"
-              echo "hostConfigs: $RC1"
-              echo "rescue: $RC2"
-              exit $(($RC0 | $RC1 | $RC2))
-            '');
-          };
-
-          check.nur = {
-            # `nix run '.#check-nur'`
-            # validates that my repo can be included in the Nix User Repository
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "check-nur" ''
-              cd ${./.}/integrations/nur
-              NIX_PATH= NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM=1 nix-env -f . -qa \* --meta --xml \
-                --allowed-uris https://static.rust-lang.org \
-                --option restrict-eval true \
-                --option allow-import-from-derivation true \
-                --drv-path --show-trace \
-                -I nixpkgs=${nixpkgs-unpatched} \
-                -I nixpkgs-overlays=${./.}/hosts/common/nix/overlay \
-                -I ../../ \
-                | tee  # tee to prevent interactive mode
-            '');
-          };
-
-          check.hostConfigs = {
-            type = "app";
-            program = let
-              checkHost = host: let
-                shellHost = pkgs.lib.replaceStrings [ "-" ] [ "_" ] host;
-              in ''
-                nix build -v '.#nixosConfigurations.${host}.config.system.build.toplevel' --out-link ./build/result-${host} -j2 "$@"
-                RC_${shellHost}=$?
-              '';
-            in builtins.toString (pkgs.writeShellScript
-              "check-host-configs"
-              ''
-                # build minimally-usable hosts first, then their full image.
-                # this gives me a minimal image i can deploy or copy over, early.
-                ${checkHost "lappy-min"}
-                ${checkHost "moby-min"}
-
-                ${checkHost "desko-light"}
-                ${checkHost "moby-light"}
-                ${checkHost "lappy-light"}
-
-                ${checkHost "desko"}
-                ${checkHost "lappy"}
-                ${checkHost "servo"}
-                ${checkHost "moby"}
-                ${checkHost "rescue"}
-
-                # still want to build the -light variants first so as to avoid multiple simultaneous webkitgtk builds
-                ${checkHost "desko-light-next"}
-                ${checkHost "moby-light-next"}
-
-                ${checkHost "desko-next"}
-                ${checkHost "lappy-next"}
-                ${checkHost "servo-next"}
-                ${checkHost "moby-next"}
-                ${checkHost "rescue-next"}
-
-                echo "desko: $RC_desko"
-                echo "lappy: $RC_lappy"
-                echo "servo: $RC_servo"
-                echo "moby: $RC_moby"
-                echo "rescue: $RC_rescue"
-
-                echo "desko-next: $RC_desko_next"
-                echo "lappy-next: $RC_lappy_next"
-                echo "servo-next: $RC_servo_next"
-                echo "moby-next: $RC_moby_next"
-                echo "rescue-next: $RC_rescue_next"
-
-                # i don't really care if the -next hosts fail. i build them mostly to keep the cache fresh/ready
-                exit $(($RC_desko | $RC_lappy | $RC_servo | $RC_moby | $RC_rescue))
-              ''
-            );
-          };
-
-          check.rescue = {
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "check-rescue" ''
-              nix build -v '.#imgs.rescue' --out-link ./build/result-rescue-img -j2
-            '');
-          };
-
-          bench = {
-            type = "app";
-            program = builtins.toString (pkgs.writeShellScript "bench" ''
-              doBench() {
-                attrPath="$1"
-                shift
-                echo -n "benchmarking eval of '$attrPath'... "
-                /run/current-system/sw/bin/time -f "%e sec" -o /dev/stdout \
-                  nix eval --no-eval-cache --quiet --raw ".#$attrPath" --apply 'result: if result != null then "" else "unexpected null"' $@ 2> /dev/null
-              }
-
-              if [ -n "$1" ]; then
-                doBench "$@"
-              else
-                doBench hostConfigs
-                doBench hostConfigs.lappy
-                doBench hostConfigs.lappy.sane.programs
-                doBench hostConfigs.lappy.sane.users.colin
-                doBench hostConfigs.lappy.sane.fs
-                doBench hostConfigs.lappy.environment.systemPackages
-              fi
-            '');
-          };
-        };
-
-      templates = {
-        env.python-data = {
-          # initialize with:
-          # - `nix flake init -t '/home/colin/dev/nixos/#env.python-data'`
-          # then enter with:
-          # - `nix develop`
-          path = ./templates/env/python-data;
-          description = "python environment for data processing";
-        };
-        pkgs.rust-inline = {
-          # initialize with:
-          # - `nix flake init -t '/home/colin/dev/nixos/#pkgs.rust-inline'`
-          path = ./templates/pkgs/rust-inline;
-          description = "rust package and development environment (inline rust sources)";
-        };
-        pkgs.rust = {
-          # initialize with:
-          # - `nix flake init -t '/home/colin/dev/nixos/#pkgs.rust'`
-          path = ./templates/pkgs/rust;
-          description = "rust package fit to ship in nixpkgs";
-        };
-        pkgs.make = {
-          # initialize with:
-          # - `nix flake init -t '/home/colin/dev/nixos/#pkgs.make'`
-          path = ./templates/pkgs/make;
-          description = "default Makefile-based derivation";
-        };
-      };
-    };
-}
-

hosts/by-name/cadey/default.nix

@@ -0,0 +1,19 @@
+# MAME arcade cabinet
+# Raspberry Pi 400:
+# - quad-core Cortex-A72 @ 1.8 GHz  (ARMv8-A 64; BCM2711)
+# - 4GiB RAM
+{ ... }:
+{
+  imports = [
+    ./fs.nix
+  ];
+
+  sane.hal.rpi-400.enable = true;
+  sane.roles.client = true;  # for WiFi creds
+
+  # TODO: port to `sane.programs` interface
+  services.xserver.desktopManager.kodi.enable = true;
+
+  # /boot space is at a premium, especially with uncompressed kernels. default was 20.
+  # boot.loader.generic-extlinux-compatible.configurationLimit = 10;
+}

hosts/by-name/cadey/fs.nix

@@ -0,0 +1,17 @@
+{ ... }:
+
+{
+  fileSystems."/nix" = {
+    device = "/dev/disk/by-uuid/cccccccc-aaaa-dddd-eeee-000020250621";
+    fsType = "btrfs";
+    options = [
+      "compress=zstd"
+      "defaults"
+    ];
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/2025-0621";
+    fsType = "vfat";
+  };
+}

hosts/by-name/crappy/default.nix

@@ -0,0 +1,33 @@
+# Samsung chromebook XE303C12
+# - <https://wiki.postmarketos.org/wiki/Samsung_Chromebook_(google-snow)>
+{ ... }:
+{
+  imports = [
+    ./fs.nix
+  ];
+
+  sane.hal.samsung.enable = true;
+  sane.roles.client = true;
+  # sane.roles.pc = true;
+
+  users.users.colin.initialPassword = "147147";
+  sane.programs.sway.enableFor.user.colin = true;
+
+  sane.programs.calls.enableFor.user.colin = false;
+  sane.programs.consoleMediaUtils.enableFor.user.colin = true;
+  sane.programs.epiphany.enableFor.user.colin = true;
+  sane.programs.geary.enableFor.user.colin = false;
+  # sane.programs.firefox.enableFor.user.colin = true;
+  sane.programs.portfolio-filemanager.enableFor.user.colin = true;
+  sane.programs.signal-desktop.enableFor.user.colin = false;
+  sane.programs.wike.enableFor.user.colin = true;
+
+  sane.programs.dino.config.autostart = false;
+  sane.programs.dissent.config.autostart = false;
+  sane.programs.fractal.config.autostart = false;
+  sane.programs.sway.config.mod = "Mod1";  #< alt key instead of Super
+
+  # sane.programs.guiApps.enableFor.user.colin = false;
+
+  # sane.programs.pcGuiApps.enableFor.user.colin = false;  #< errors!
+}

hosts/by-name/crappy/fs.nix

@@ -0,0 +1,16 @@
+{ ... }:
+{
+  fileSystems."/nix" = {
+    device = "/dev/disk/by-uuid/55555555-0303-0c12-86df-eda9e9311526";
+    fsType = "btrfs";
+    options = [
+      "compress=zstd"
+      "defaults"
+    ];
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/303C-5A37";
+    fsType = "vfat";
+  };
+}

hosts/by-name/desko/default.nix

@@ -1,65 +1,57 @@
-{ config, pkgs, ... }:
+{ config, lib, ... }:
 {
   imports = [
     ./fs.nix
   ];
 
+  # firewall has to be open to allow clients to use services hosted on this device,
+  # like `ollama`
+  sane.ports.openFirewall = true;
+
+  # sane.programs.devPkgs.enableFor.user.colin = true;
   # sane.guest.enable = true;
 
-  # services.distccd.enable = true;
-  # sane.programs.distcc.enableFor.user.guest = true;
-
-  # TODO: remove emulation, but need to fix nixos-rebuild to moby for that.
-  # sane.roles.build-machine.emulation = true;
+  # don't enable wifi by default: it messes with connectivity.
+  # systemd.services.iwd.enable = false;
+  # networking.wireless.enable = false;
+  # systemd.services.wpa_supplicant.enable = false;
+  # sane.programs.wpa_supplicant.enableFor.user.colin = lib.mkForce false;
+  # sane.programs.wpa_supplicant.enableFor.system = lib.mkForce false;
+  # don't auto-connect to wifi networks
+  # see: <https://networkmanager.dev/docs/api/latest/NetworkManager.conf.html#device-spec>
+  networking.networkmanager.unmanaged = [ "type:wifi" ];
 
   sops.secrets.colin-passwd.neededForUsers = true;
 
-  sane.ports.openFirewall = true;  # for e.g. nix-serve
-
   sane.roles.build-machine.enable = true;
   sane.roles.client = true;
-  sane.roles.dev-machine = true;
   sane.roles.pc = true;
+  sane.roles.work = true;
+  sane.services.ollama.enable = lib.mkIf (config.sane.maxBuildCost >= 3) true;
   sane.services.wg-home.enable = true;
-  sane.services.wg-home.ip = config.sane.hosts.by-name."desko".wg-home.ip;
-  sane.services.duplicity.enable = true;
-  sane.services.nixserve.secretKeyFile = config.sops.secrets.nix_serve_privkey.path;
+  sane.ovpn.addrV4 = "172.26.55.21";
+  # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:20c1:a73c";
+  sane.services.rsync-net.enable = true;
 
-  sane.nixcache.substituters.desko = false;
   sane.nixcache.remote-builders.desko = false;
 
-  sane.programs.cups.enableFor.user.colin = true;
+  sane.programs.firefox.config.formFactor = "desktop";
+
+  sane.programs.sane-private-unlock-remote.enableFor.user.colin = true;
+  sane.programs.sane-private-unlock-remote.config.hosts = [ "servo" ];
+
   sane.programs.sway.enableFor.user.colin = true;
-  sane.programs.iphoneUtils.enableFor.user.colin = true;
   sane.programs.steam.enableFor.user.colin = true;
 
-  # sane.programs.devPkgs.enableFor.user.colin = true;
+  sane.programs.nwg-panel.config = {
+    battery = false;
+    brightness = false;
+  };
 
-  sane.programs."gnome.geary".config.autostart = true;
-  sane.programs.signal-desktop.config.autostart = true;
-
-  boot.loader.efi.canTouchEfiVariables = false;
-  sane.image.extraBootFiles = [ pkgs.bootpart-uefi-x86_64 ];
+  sane.programs.mpv.config.defaultProfile = "high-quality";
 
   # needed to use libimobiledevice/ifuse, for iphone sync
   services.usbmuxd.enable = true;
 
-  # don't enable wifi by default: it messes with connectivity.
-  systemd.services.iwd.enable = false;
-  systemd.services.wpa_supplicant.enable = false;
-
-  # default config: https://man.archlinux.org/man/snapper-configs.5
-  # defaults to something like:
-  #   - hourly snapshots
-  #   - auto cleanup; keep the last 10 hourlies, last 10 daylies, last 10 monthlys.
-  services.snapper.configs.nix = {
-    # TODO: for the impermanent setup, we'd prefer to just do /nix/persist,
-    # but that also requires setting up the persist dir as a subvol
-    SUBVOLUME = "/nix";
-    # TODO: ALLOW_USERS doesn't seem to work. still need `sudo snapper -c nix list`
-    ALLOW_USERS = [ "colin" ];
-  };
-
-  # docs: https://nixos.org/manual/nixos/stable/options.html#opt-system.stateVersion
-  system.stateVersion = "21.05";
+  hardware.amdgpu.opencl.enable = true;  # desktop (AMD's opencl implementation AKA "ROCM"); probably required for ollama
 }

hosts/by-name/desko/fs.nix

@@ -3,10 +3,10 @@
 {
   # increase /tmp space (defaults to 50% of RAM) for building large nix things.
   # a cross-compiled kernel, particularly, will easily use 30+GB of tmp
-  fileSystems."/tmp".options = [ "size=64G" ];
+  fileSystems."/tmp".options = [ "size=128G" ];
 
   fileSystems."/nix" = {
-    device = "/dev/disk/by-uuid/845d85bf-761d-431b-a406-e6f20909154f";
+    device = "/dev/disk/by-uuid/dddddddd-eeee-5555-cccc-000020250527";
     fsType = "btrfs";
     options = [
       "compress=zstd"
@@ -15,7 +15,7 @@
   };
 
   fileSystems."/boot" = {
-    device = "/dev/disk/by-uuid/5049-9AFD";
+    device = "/dev/disk/by-uuid/2025-0527";
     fsType = "vfat";
   };
 }

hosts/by-name/flowy/default.nix

@@ -0,0 +1,58 @@
+{ lib, pkgs, ... }:
+{
+  imports = [
+    ./fs.nix
+  ];
+
+  sane.roles.client = true;
+  sane.roles.pc = true;
+  sane.roles.work = true;
+  sane.services.wg-home.enable = true;
+  # sane.ovpn.addrV4 = "172.23.119.72";
+
+  # sane.guest.enable = true;
+
+  sane.programs.sane-private-unlock-remote.enableFor.user.colin = true;
+  sane.programs.sane-private-unlock-remote.config.hosts = [ "servo" ];
+
+  sane.programs.firefox.config.formFactor = "laptop";
+  sane.programs.itgmania.enableFor.user.colin = true;
+  sane.programs.sway.enableFor.user.colin = true;
+
+  sops.secrets.colin-passwd.neededForUsers = true;
+
+  sane.services.rsync-net.enable = true;
+
+  # add an entry to boot into Windows, as if it had been launched directly from the BIOS.
+  boot.loader.systemd-boot.rebootForBitlocker = true;
+  boot.loader.systemd-boot.windows.primary.efiDeviceHandle = "HD0b";
+
+  system.activationScripts.makeDefaultBootEntry = {
+    text = let
+      makeDefaultBootEntry = pkgs.writeShellApplication {
+        name = "makeDefaultBootEntry";
+        runtimeInputs = with pkgs; [
+          efibootmgr
+          gnugrep
+        ];
+        text = ''
+          # configure the EFI firmware to boot into NixOS by default.
+          # do this by querying the active boot entry, and just making that be the default.
+          # this is needed on flowy because enabling secure boot / booting into Windows
+          # resets the default boot order; manually reconfiguring that is tiresome.
+          efi=$(efibootmgr)
+          bootCurrent=$(echo "$efi" | grep '^BootCurrent: ')
+          bootCurrent=''${bootCurrent/BootCurrent: /}
+          bootOrder=$(echo "$efi" | grep '^BootOrder: ')
+          bootOrder=''${bootOrder/BootOrder: /}
+          if ! [[ "$bootOrder" =~ ^"$bootCurrent", ]]; then
+            # booted entry was not the default,
+            # so prepend it to the boot order:
+            newBootOrder="$bootCurrent,$bootOrder"
+            (set -x; efibootmgr -o "$newBootOrder")
+          fi
+        '';
+      };
+    in lib.getExe makeDefaultBootEntry;
+  };
+}

hosts/by-name/flowy/fs.nix

@@ -0,0 +1,17 @@
+{ ... }:
+
+{
+  fileSystems."/nix" = {
+    device = "/dev/disk/by-uuid/ffffffff-1111-0000-eeee-000020250531";
+    fsType = "btrfs";
+    options = [
+      "compress=zstd"
+      "defaults"
+    ];
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/2025-0531";
+    fsType = "vfat";
+  };
+}

hosts/by-name/lappy/default.nix

@@ -1,39 +1,37 @@
-{ config, pkgs, ... }:
+{ lib, ... }:
 {
   imports = [
     ./fs.nix
   ];
 
   sane.roles.client = true;
-  sane.roles.dev-machine = true;
   sane.roles.pc = true;
   sane.services.wg-home.enable = true;
-  sane.services.wg-home.ip = config.sane.hosts.by-name."lappy".wg-home.ip;
+  sane.ovpn.addrV4 = "172.23.119.72";
+  # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:0332:aa96/128";
 
   # sane.guest.enable = true;
-  boot.loader.efi.canTouchEfiVariables = false;
-  sane.image.extraBootFiles = [ pkgs.bootpart-uefi-x86_64 ];
 
-  sane.programs.cups.enableFor.user.colin = true;
-  sane.programs.stepmania.enableFor.user.colin = true;
+  sane.programs.sane-private-unlock-remote.enableFor.user.colin = true;
+  sane.programs.sane-private-unlock-remote.config.hosts = [ "servo" ];
+
+  sane.programs.firefox.config.formFactor = "laptop";
+  sane.programs.itgmania.enableFor.user.colin = true;
+  # sane.programs.stepmania.enableFor.user.colin = true;  #< TODO: fix build
   sane.programs.sway.enableFor.user.colin = true;
 
-  sane.programs."gnome.geary".config.autostart = true;
-  sane.programs.signal-desktop.config.autostart = true;
-
   sops.secrets.colin-passwd.neededForUsers = true;
 
-  # default config: https://man.archlinux.org/man/snapper-configs.5
-  # defaults to something like:
-  #   - hourly snapshots
-  #   - auto cleanup; keep the last 10 hourlies, last 10 daylies, last 10 monthlys.
-  services.snapper.configs.nix = {
-    # TODO: for the impermanent setup, we'd prefer to just do /nix/persist,
-    # but that also requires setting up the persist dir as a subvol
-    SUBVOLUME = "/nix";
-    ALLOW_USERS = [ "colin" ];
-  };
+  sane.services.rsync-net.enable = true;
 
-  # docs: https://nixos.org/manual/nixos/stable/options.html#opt-system.stateVersion
-  system.stateVersion = "21.05";
+  # starting 2024/09, under default settings (apparently 256 quantum), audio would crackle under load.
+  # 1024 solves *most* crackles, but still noticable under heavier loads.
+  sane.programs.pipewire.config.min-quantum = 2048;
+
+  # limit how many snapshots we keep, due to extremely limited disk space (TODO: remove this override after upgrading lappy hard drive)
+  services.snapper.configs.root.TIMELINE_LIMIT_HOURLY = lib.mkForce 2;
+  services.snapper.configs.root.TIMELINE_LIMIT_DAILY = lib.mkForce 2;
+  services.snapper.configs.root.TIMELINE_LIMIT_WEEKLY = lib.mkForce 0;
+  services.snapper.configs.root.TIMELINE_LIMIT_MONTHLY = lib.mkForce 0;
+  services.snapper.configs.root.TIMELINE_LIMIT_YEARLY = lib.mkForce 0;
 }

hosts/by-name/lappy/xkb_mobile_normal_buttons

@@ -1,7 +0,0 @@
-xkb_keymap {
-	xkb_keycodes  { include "evdev+aliases(qwerty)"	};
-	xkb_types     { include "complete"	};
-	xkb_compat    { include "complete"	};
-	xkb_symbols   { include "pc+us+inet(evdev)"	};
-	xkb_geometry  { include "pc(pc105)"	};
-};

hosts/by-name/moby/bootloader.nix

@@ -1,22 +0,0 @@
-# tow-boot: <https://tow-boot.org>
-# docs (pinephone specific): <https://github.com/Tow-Boot/Tow-Boot/tree/development/boards/pine64-pinephoneA64>
-# LED and button behavior is defined here: <https://github.com/Tow-Boot/Tow-Boot/blob/development/modules/tow-boot/phone-ux.nix>
-# - hold VOLDOWN: enter recovery mode
-#   - LED will turn aqua instead of yellow
-#   - recovery mode would ordinarily allow a selection of entries, but for pinephone i guess it doesn't do anything?
-# - hold VOLUP: force it to load the OS from eMMC?
-#   - LED will turn blue instead of yellow
-# boot LEDs:
-# - yellow = entered tow-boot
-# - 10 red flashes => poweroff means tow-boot couldn't boot into the next stage (i.e. distroboot)
-#   - distroboot: <https://source.denx.de/u-boot/u-boot/-/blob/v2022.04/doc/develop/distro.rst>)
-{ config, pkgs, ... }:
-{
-  # we need space in the GPT header to place tow-boot.
-  # only actually need 1 MB, but better to over-allocate than under-allocate
-  sane.image.extraGPTPadding = 16 * 1024 * 1024;
-  sane.image.firstPartGap = 0;
-  sane.image.installBootloader = ''
-    dd if=${pkgs.tow-boot-pinephone}/Tow-Boot.noenv.bin of=$out/nixos.img bs=1024 seek=8 conv=notrunc
-  '';
-}

hosts/by-name/moby/default.nix

@@ -1,7 +1,4 @@
 # Pinephone
-# other setups to reference:
-# - <https://hamblingreen.gitlab.io/2022/03/02/my-pinephone-setup.html>
-#   - sxmo Arch user. lots of app recommendations
 #
 # wikis, resources, ...:
 # - Linux Phone Apps: <https://linuxphoneapps.org/>
@@ -9,59 +6,40 @@
 # - Mobian wiki: <https://wiki.mobian-project.org/doku.php?id=start>
 #   - recommended apps, chatrooms
 
-{ config, pkgs, lib, ... }:
+{ ... }:
 {
   imports = [
-    ./bootloader.nix
     ./fs.nix
-    ./gps.nix
-    ./kernel.nix
-    ./polyfill.nix
   ];
 
+  sane.hal.pine64-pinephone-pro.enable = true;
   sane.roles.client = true;
   sane.roles.handheld = true;
-  sane.zsh.showDeadlines = false;  # unlikely to act on them when in shell
   sane.services.wg-home.enable = true;
-  sane.services.wg-home.ip = config.sane.hosts.by-name."moby".wg-home.ip;
-
-  # for some reason desko -> moby deploys are super flaky when desko is also a nixcache (not true of desko -> lappy deploys, though!)
-  # > unable to download 'http://desko:5001/<hash>.narinfo': Server returned nothing (no headers, no data) (52)
-  sane.nixcache.substituters.desko = false;
+  sane.ovpn.addrV4 = "172.24.87.255";
+  # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:18cd:a72b";
 
   # XXX colin: phosh doesn't work well with passwordless login,
   # so set this more reliable default password should anything go wrong
   users.users.colin.initialPassword = "147147";
-  # services.getty.autologinUser = "root";  # allows for emergency maintenance?
 
   sops.secrets.colin-passwd.neededForUsers = true;
 
-  # sane.gui.sxmo.enable = true;
+  sane.services.rsync-net.enable = true;
+
   sane.programs.sway.enableFor.user.colin = true;
-  sane.programs.swaylock.enableFor.user.colin = false;  #< not usable on touch
-  sane.programs.schlock.enableFor.user.colin = true;
-  sane.programs.swayidle.config.actions.screenoff.delay = 300;
-  sane.programs.swayidle.config.actions.screenoff.enable = true;
-  sane.programs.sane-input-handler.enableFor.user.colin = true;
-  sane.programs.blueberry.enableFor.user.colin = false;  # bluetooth manager: doesn't cross compile!
-  sane.programs.fcitx5.enableFor.user.colin = false;  # does not cross compile
-  sane.programs.mercurial.enableFor.user.colin = false;  # does not cross compile
-  sane.programs.nvme-cli.enableFor.system = false;  # does not cross compile (libhugetlbfs)
+  sane.programs.sway.config.mod = "Mod1";  #< alt key instead of Super
 
   # enabled for easier debugging
   sane.programs.eg25-control.enableFor.user.colin = true;
-  sane.programs.rtl8723cs-wowlan.enableFor.user.colin = true;
+  # sane.programs.rtl8723cs-wowlan.enableFor.user.colin = true;
+  # sane.programs.eg25-manager.enableFor.user.colin = true;
 
   # sane.programs.ntfy-sh.config.autostart = true;
   sane.programs.dino.config.autostart = true;
-  # sane.programs.signal-desktop.config.autostart = true;  # TODO: enable once electron stops derping.
-  # sane.programs."gnome.geary".config.autostart = true;
-  # sane.programs.calls.config.autostart = true;
+  sane.programs.signal-desktop.config.autostart = false;
+  sane.programs.geary.config.autostart = false;
 
-  sane.programs.firefox.mime.priority = 300;  # prefer other browsers when possible
-  # HACK/TODO: make `programs.P.env.VAR` behave according to `mime.priority`
-  sane.programs.firefox.env = lib.mkForce {};
-  sane.programs.epiphany.env.BROWSER = "epiphany";
   sane.programs.pipewire.config = {
     # tune so Dino doesn't drop audio
     # there's seemingly two buffers for the mic (see: <https://gitlab.freedesktop.org/pipewire/pipewire/-/wikis/FAQ#pipewire-buffering-explained>)
@@ -78,53 +56,12 @@
     max-quantum = 8192;
   };
 
-  boot.loader.efi.canTouchEfiVariables = false;
-  # /boot space is at a premium. default was 20.
-  # even 10 can be too much
-  boot.loader.generic-extlinux-compatible.configurationLimit = 8;
-  # mobile.bootloader.enable = false;
-  # mobile.boot.stage-1.enable = false;
-  # boot.initrd.systemd.enable = false;
-  # boot.initrd.services.swraid.enable = false;  # attempt to fix dm_mod stuff
+  sane.programs.mpv.config.defaultProfile = "fast";
 
-  # hardware.firmware makes the referenced files visible to the kernel, for whenever a driver explicitly asks for them.
-  # these files are visible from userspace by following `/sys/module/firmware_class/parameters/path`
-  #
-  # mobile-nixos' /lib/firmware includes:
-  #   rtl_bt          (bluetooth)
-  #   anx7688-fw.bin  (USB-C chip: power negotiation, HDMI/dock)
-  #   ov5640_af.bin   (camera module)
-  # hardware.firmware = [ config.mobile.device.firmware ];
-  # hardware.firmware = [ pkgs.rtl8723cs-firmware ];
-  hardware.firmware = [
-    (pkgs.linux-firmware-megous.override {
-      # rtl_bt = false probably means no bluetooth connectivity.
-      # N.B.: DON'T RE-ENABLE without first confirming that wake-on-lan works during suspend (rtcwake).
-      # it seems the rtl_bt stuff ("bluetooth coexist") might make wake-on-LAN radically more flaky.
-      rtl_bt = false;
-    })
-  ];
+  # /boot space is at a premium, especially with uncompressed kernels. default was 20.
+  # boot.loader.generic-extlinux-compatible.configurationLimit = 10;
 
-  system.stateVersion = "21.11";
-
-  # defined: https://www.freedesktop.org/software/systemd/man/machine-info.html
-  # XXX colin: not sure which, if any, software makes use of this
-  environment.etc."machine-info".text = ''
-    CHASSIS="handset"
-  '';
-
-  # enable rotation sensor
-  hardware.sensor.iio.enable = true;
-
-  services.udev.extraRules = let
-    chmod = "${pkgs.coreutils}/bin/chmod";
-    chown = "${pkgs.coreutils}/bin/chown";
-  in ''
-    # make Pinephone flashlight writable by user.
-    # taken from postmarketOS: <repo:postmarketOS/pmaports:device/main/device-pine64-pinephone/60-flashlight.rules>
-    SUBSYSTEM=="leds", DEVPATH=="*/*:flash", RUN+="${chmod} g+w /sys%p/brightness /sys%p/flash_strobe", RUN+="${chown} :video /sys%p/brightness /sys%p/flash_strobe"
-
-    # make Pinephone front LEDs writable by user.
-    SUBSYSTEM=="leds", DEVPATH=="*/*:indicator", RUN+="${chmod} g+w /sys%p/brightness", RUN+="${chown} :video /sys%p/brightness"
-  '';
+  # TODO: switch to systemd-boot
+  boot.loader.generic-extlinux-compatible.enable = true;
+  boot.loader.systemd-boot.enable = false;
 }

hosts/by-name/moby/gps.nix

@@ -1,68 +0,0 @@
-# pinephone GPS happens in EG25 modem
-# serial control interface to modem is /dev/ttyUSB2
-# after enabling GPS, readout is /dev/ttyUSB1
-#
-# minimal process to enable modem and GPS:
-# - `echo 1 > /sys/class/modem-power/modem-power/device/powered`
-# - `screen /dev/ttyUSB2 115200`
-#   - `AT+QGPSCFG="nmeasrc",1`
-#   - `AT+QGPS=1`
-# this process is automated by my `eg25-control` program and services (`eg25-control-powered`, `eg25-control-gps`)
-# - see the `modules/` directory further up this repository.
-#
-# now, something like `gpsd` can directly read from /dev/ttyUSB1,
-# or geoclue can query the GPS directly through modem-manager
-#
-# initial GPS fix can take 15+ minutes.
-# meanwhile, services like eg25-manager or eg25-control-freshen-agps can speed this up by uploading assisted GPS data to the modem.
-#
-# support/help:
-# - geoclue, gnome-maps
-#   - irc: #gnome-maps on irc.gimp.org
-#   - Matrix: #gnome-maps:gnome.org  (unclear if bridged to IRC)
-#
-# programs to pair this with:
-# - `satellite-gtk`: <https://codeberg.org/tpikonen/satellite>
-#   - shows/tracks which satellites the GPS is connected to; useful to understand fix characteristics
-# - `gnome-maps`: uses geoclue, has route planning
-# - `mepo`: uses gpsd, minimalist, flaky, and buttons are kinda hard to activate on mobile
-# - puremaps?
-# - osmin?
-#
-# known/outstanding bugs:
-# - `systemctl start eg25-control-gps` can the hang the whole system (2023/10/06)
-#   - i think it's actually `eg25-control-powered` which does this (started by the gps)
-#   - best guess is modem draws so much power at launch that other parts of the system see undervoltage
-#   - workaround is to hard power-cycle the system. the modem may not bring up after reboot: leave unpowered for 60s and boot again.
-#
-# future work:
-# - integrate with [wigle](https://www.wigle.net/) for offline equivalent to Mozilla Location Services
-
-{ config, lib, ... }:
-{
-  # test gpsd with `gpspipe -w -n 10 2> /dev/null | grep -m 1 TPV | jq '.lat, .lon' | tr '\n' ' '`
-  # ^ should return <lat> <long>
-  services.gpsd.enable = true;
-  services.gpsd.devices = [ "/dev/ttyUSB1" ];
-
-  # test geoclue2 by building `geoclue2-with-demo-agent`
-  # and running "${geoclue2-with-demo-agent}/libexec/geoclue-2.0/demos/where-am-i"
-  # note that geoclue is dbus-activated, and auto-stops after 60s with no caller
-  services.geoclue2.enable = true;
-  services.geoclue2.appConfig.where-am-i = {
-    # this is the default "agent", shipped by geoclue package: allow it to use location
-    isAllowed = true;
-    isSystem = false;
-    # XXX: setting users != [] might be causing `where-am-i` to time out
-    users = [
-      # restrict to only one set of users. empty array (default) means "allow any user to access geolocation".
-      (builtins.toString config.users.users.colin.uid)
-    ];
-  };
-  systemd.services.geoclue.after = lib.mkForce [];  #< defaults to network-online, but not all my sources require network
-  users.users.geoclue.extraGroups = [
-    "dialout"  # TODO: figure out if dialout is required. that's for /dev/ttyUSB1, but geoclue probably doesn't read that?
-  ];
-
-  sane.programs.where-am-i.enableFor.user.colin = true;
-}

hosts/by-name/moby/kernel.nix

@@ -1,91 +0,0 @@
-{ pkgs, ... }:
-let
-  dmesg = "${pkgs.util-linux}/bin/dmesg";
-  grep = "${pkgs.gnugrep}/bin/grep";
-  modprobe = "${pkgs.kmod}/bin/modprobe";
-  ensureHWReady = ''
-    # common boot failure:
-    # blank screen (no backlight even), with the following log:
-    # ```syslog
-    # sun8i-dw-hdmi 1ee0000.hdmi: Couldn't get the HDMI PHY
-    # ...
-    # sun4i-drm display-engine: Couldn't bind all pipelines components
-    # ...
-    # sun8i-dw-hdmi: probe of 1ee0000.hdmi failed with error -17
-    # ```
-    #
-    # in particular, that `probe ... failed` occurs *only* on failed boots
-    # (the other messages might sometimes occur even on successful runs?)
-    #
-    # reloading the sun8i hdmi driver usually gets the screen on, showing boot text.
-    # then restarting display-manager.service gets us to the login.
-    #
-    # NB: the above log is default level. though less specific, there's a `err` level message that also signals this:
-    # sun4i-drm display-engine: failed to bind 1ee0000.hdmi (ops sun8i_dw_hdmi_ops [sun8i_drm_hdmi]): -17
-    # NB: this is the most common, but not the only, failure mode for `display-manager`.
-    # another error seems characterized by these dmesg logs, in which reprobing sun8i_drm_hdmi does not fix:
-    # ```syslog
-    # sun6i-mipi-dsi 1ca0000.dsi: Couldn't get the MIPI D-PHY
-    # sun4i-drm display-engine: Couldn't bind all pipelines components
-    # sun6i-mipi-dsi 1ca0000.dsi: Couldn't register our component
-    # ```
-
-    if (${dmesg} --kernel --level err --color=never --notime | ${grep} -q 'sun4i-drm display-engine: failed to bind 1ee0000.hdmi')
-    then
-      echo "reprobing sun8i_drm_hdmi"
-      # if a command here fails it errors the whole service, so prefer to log instead
-      ${modprobe} -r sun8i_drm_hdmi || echo "failed to unload sun8i_drm_hdmi"
-      ${modprobe} sun8i_drm_hdmi || echo "failed to load sub8i_drm_hdmi"
-    fi
-  '';
-in
-{
-  boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux-megous;
-  # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux-manjaro;
-  # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_latest;
-
-  # alternatively, apply patches directly to stock nixos kernel:
-  # boot.kernelPatches = manjaroPatches ++ [
-  #   (patchDefconfig kernelConfig)
-  # ];
-
-  # configure nixos to build a compressed kernel image, since it doesn't usually do that for aarch64 target.
-  # without this i run out of /boot space in < 10 generations
-  nixpkgs.hostPlatform.linux-kernel = {
-    # defaults:
-    name = "aarch64-multiplatform";
-    baseConfig = "defconfig";
-    DTB = true;
-    autoModules = true;
-    preferBuiltin = true;
-    # extraConfig = ...
-    # ^-- raspberry pi stuff: we don't need it.
-
-    # target = "Image";  # <-- default
-    target = "Image.gz";  # <-- compress the kernel image
-    # target = "zImage";  # <-- confuses other parts of nixos :-(
-  };
-
-  # disable proximity sensor.
-  # the filtering/calibration is bad that it causes the screen to go fully dark at times.
-  boot.blacklistedKernelModules = [ "stk3310" ];
-
-  boot.kernelParams = [
-    # without this some GUI apps fail: `DRM_IOCTL_MODE_CREATE_DUMB failed: Cannot allocate memory`
-    # this is because they can't allocate enough video ram.
-    # see related nixpkgs issue: <https://github.com/NixOS/nixpkgs/issues/260222>
-    # TODO(2023/12/03): remove once mesa 23.3.1 lands: <https://github.com/NixOS/nixpkgs/pull/265740>
-    #
-    # the default CMA seems to be 32M.
-    # i was running fine with 256MB from 2022/07-ish through 2022/12-ish, but then the phone quit reliably coming back from sleep (phosh): maybe a memory leak?
-    # `cat /proc/meminfo` to see CmaTotal/CmaFree if interested in tuning this.
-    "cma=512M"
-    # 2023/10/20: potential fix for the lima (GPU) timeout bugs:
-    # - <https://gitlab.com/postmarketOS/pmaports/-/issues/805#note_890467824>
-    "lima.sched_timeout_ms=2000"
-  ];
-
-  # services.xserver.displayManager.job.preStart = ensureHWReady;
-  # systemd.services.greetd.preStart = ensureHWReady;
-  systemd.services.unl0kr.preStart = ensureHWReady;
-}

hosts/by-name/moby/polyfill.nix

@@ -1,45 +0,0 @@
-# this file configures preferences per program, without actually enabling any programs.
-# the goal is to separate the place where we decide *what* to use (i.e. `sane.programs.firefox.enable = true` -- at the toplevel)
-# from where we specific how that thing should behave *if* it's in use.
-#
-# NixOS backgrounds:
-# - <https://github.com/NixOS/nixos-artwork>
-#   - <https://github.com/NixOS/nixos-artwork/issues/50>  (colorful; unmerged)
-#   - <https://github.com/NixOS/nixos-artwork/pull/60/files>  (desktop-oriented; clean; unmerged)
-# - <https://itsfoss.com/content/images/2023/04/nixos-tutorials.png>
-
-{ lib, pkgs, sane-lib, ... }:
-{
-  sane.programs.firefox.config = {
-    # compromise impermanence for the sake of usability
-    persistCache = "private";
-    persistData = "private";
-
-    # i don't do crypto stuff on moby
-    addons.ether-metamask.enable = false;
-    # sidebery UX doesn't make sense on small screen
-    addons.sidebery.enable = false;
-  };
-  sane.programs.swaynotificationcenter.config = {
-    backlight = "backlight";  # /sys/class/backlight/*backlight*/brightness
-  };
-
-  sane.programs.alacritty.config.fontSize = 9;
-
-  sane.programs.sway.config = {
-    font = "pango:monospace 10";
-    mod = "Mod1";  # prefer Alt
-    workspace_layout = "tabbed";
-  };
-
-  sane.programs.waybar.config = {
-    fontSize = 14;
-    height = 26;
-    persistWorkspaces = [ "1" "2" "3" "4" "5" ];
-    modules.media = false;
-    modules.network = false;
-    modules.perf = false;
-    modules.windowTitle = false;
-    # TODO: show modem state
-  };
-}

hosts/by-name/rescue/default.nix

@@ -1,18 +1,13 @@
-{ pkgs, ... }:
+{ ... }:
 {
   imports = [
     ./fs.nix
   ];
 
-  boot.loader.efi.canTouchEfiVariables = false;
-  sane.image.extraBootFiles = [ pkgs.bootpart-uefi-x86_64 ];
   sane.persist.enable = false;  # what we mean here is that the image is immutable; `/` is still tmpfs.
   sane.nixcache.enable = false;  # don't want to be calling out to dead machines that we're *trying* to rescue
 
   # auto-login at shell
   services.getty.autologinUser = "colin";
   # users.users.colin.initialPassword = "colin";
-
-  # docs: https://nixos.org/manual/nixos/stable/options.html#opt-system.stateVersion
-  system.stateVersion = "21.05";
 }

hosts/by-name/servo/default.nix

@@ -1,57 +1,40 @@
-{ config, pkgs, ... }:
+{ ... }:
 
 {
   imports = [
     ./fs.nix
-    ./net.nix
+    ./net
     ./services
+    ./users
   ];
 
-  sane.programs = {
-    # for administering services
-    freshrss.enableFor.user.colin = true;
-    matrix-synapse.enableFor.user.colin = true;
-    signaldctl.enableFor.user.colin = true;
-  };
+  # for administering services
+  sane.programs.clightning-sane.enableFor.user.colin = true;
+  # sane.programs.freshrss.enableFor.user.colin = true;
+  # sane.programs.signaldctl.enableFor.user.colin = true;
+  # sane.programs.matrix-synapse.enableFor.user.colin = true;
 
   sane.roles.build-machine.enable = true;
-  sane.zsh.showDeadlines = false;  # ~/knowledge doesn't always exist
+  sane.programs.sane-deadlines.config.showOnLogin = false;  # ~/knowledge doesn't always exist
   sane.programs.consoleUtils.suggestedPrograms = [
     "consoleMediaUtils"  # notably, for go2tv / casting
     "pcConsoleUtils"
     "sane-scripts.stop-all-servo"
   ];
   sane.services.dyn-dns.enable = true;
-  sane.services.wg-home.enable = true;
-  sane.services.wg-home.visibleToWan = true;
-  sane.services.wg-home.forwardToWan = true;
-  sane.services.wg-home.routeThroughServo = false;
-  sane.services.wg-home.ip = config.sane.hosts.by-name."servo".wg-home.ip;
-  sane.nixcache.substituters.servo = false;
-  sane.nixcache.substituters.desko = false;
   sane.nixcache.remote-builders.desko = false;
   sane.nixcache.remote-builders.servo = false;
-  # sane.services.duplicity.enable = true;  # TODO: re-enable after HW upgrade
+  sane.services.rsync-net.enable = true;
 
   # automatically log in at the virtual consoles.
-  # using root here makes sure we always have an escape hatch
+  # using root here makes sure we always have an escape hatch.
+  # XXX(2024-07-27): this is incompatible if using s6, which needs to auto-login as `colin` to start its user services.
   services.getty.autologinUser = "root";
 
-  boot.loader.efi.canTouchEfiVariables = false;
-  sane.image.extraBootFiles = [ pkgs.bootpart-uefi-x86_64 ];
-
   # both transmission and ipfs try to set different net defaults.
   # we just use the most aggressive of the two here:
   boot.kernel.sysctl = {
     "net.core.rmem_max" = 4194304;  # 4MB
   };
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "21.11";
 }
 

hosts/by-name/servo/fs.nix

@@ -1,60 +1,9 @@
-# zfs docs:
-# - <https://nixos.wiki/wiki/ZFS>
-# - <repo:nixos/nixpkgs:nixos/modules/tasks/filesystems/zfs.nix>
-#
-# zfs check health: `zpool status`
-#
-# zfs pool creation (requires `boot.supportedFilesystems = [ "zfs" ];`
-# - 1. identify disk IDs: `ls -l /dev/disk/by-id`
-# - 2. pool these disks: `zpool create -f -m legacy pool raidz ata-ST4000VN008-2DR166_WDH0VB45 ata-ST4000VN008-2DR166_WDH17616 ata-ST4000VN008-2DR166_WDH0VC8Q ata-ST4000VN008-2DR166_WDH17680`
-#   - legacy documented: <https://superuser.com/questions/790036/what-is-a-zfs-legacy-mount-point>
-# - 3. enable acl support: `zfs set acltype=posixacl pool`
-#
-# import pools: `zpool import pool`
-# show zfs datasets: `zfs list` (will be empty if haven't imported)
-# show zfs properties (e.g. compression): `zfs get all pool`
-# set zfs properties: `zfs set compression=on pool`
-{ ... }:
+{ lib, pkgs, ... }:
 
 {
   # hostId: not used for anything except zfs guardrail?
   #   [hex(ord(x)) for x in 'serv']
-  networking.hostId = "73657276";
-  boot.supportedFilesystems = [ "zfs" ];
-  # boot.zfs.enabled = true;
-  boot.zfs.forceImportRoot = false;
-  # scrub all zfs pools weekly:
-  services.zfs.autoScrub.enable = true;
-  boot.extraModprobeConfig = ''
-    ### zfs_arc_max tunable:
-    # ZFS likes to use half the ram for its own cache and let the kernel push everything else to swap.
-    # so, reduce its cache size
-    # see: <https://askubuntu.com/a/1290387>
-    # see: <https://serverfault.com/a/1119083>
-    # see: <https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Module%20Parameters.html#zfs-arc-max>
-    # for all tunables, see: `man 4 zfs`
-    # to update these parameters without rebooting:
-    # - `echo '4294967296' | sane-sudo-redirect /sys/module/zfs/parameters/zfs_arc_max`
-    ### zfs_bclone_enabled tunable
-    # this allows `cp --reflink=always FOO BAR` to work. i.e. shallow copies.
-    # it's unstable as of 2.2.3. led to *actual* corruption in 2.2.1, but hopefully better by now.
-    # - <https://github.com/openzfs/zfs/issues/405>
-    # note that `du -h` won't *always* show the reduced size for reflink'd files (?).
-    # `zpool get all | grep clone` seems to be the way to *actually* see how much data is being deduped
-    options zfs zfs_arc_max=4294967296 zfs_bclone_enabled=1
-  '';
-  # to be able to mount the pool like this, make sure to tell zfs to NOT manage it itself.
-  # otherwise local-fs.target will FAIL and you will be dropped into a rescue shell.
-  # - `zfs set mountpoint=legacy pool`
-  # if done correctly, the pool can be mounted before this `fileSystems` entry is created:
-  # - `sudo mount -t zfs pool /mnt/persist/pool`
-  fileSystems."/mnt/pool" = {
-    device = "pool";
-    fsType = "zfs";
-    options = [ "acl" ];  #< not sure if this `acl` flag is actually necessary. it mounts without it.
-  };
-  # services.zfs.zed = ... # TODO: zfs can send me emails when disks fail
-  sane.programs.sysadminUtils.suggestedPrograms = [ "zfs" ];
+  # networking.hostId = "73657276";
 
   sane.persist.stores."ext" = {
     origin = "/mnt/pool/persist";
@@ -67,7 +16,7 @@
   fileSystems."/tmp".options = [ "size=32G" ];
 
   fileSystems."/nix" = {
-    device = "/dev/disk/by-uuid/cc81cca0-3cc7-4d82-a00c-6243af3e7776";
+    device = "/dev/disk/by-uuid/55555555-eeee-ffff-bbbb-000020250820";
     fsType = "btrfs";
     options = [
       "compress=zstd"
@@ -76,23 +25,39 @@
   };
 
   fileSystems."/boot" = {
-    device = "/dev/disk/by-uuid/6EE3-4171";
+    device = "/dev/disk/by-uuid/2025-0820";
     fsType = "vfat";
   };
 
-  # slow, external storage (for archiving, etc)
-  fileSystems."/mnt/usb-hdd" = {
-    device = "/dev/disk/by-uuid/aa272cff-0fcc-498e-a4cb-0d95fb60631b";
+  fileSystems."/mnt/pool" = {
+    # all btrfs devices of the same RAID volume use the same UUID.
+    device = "UUID=40fc6e1d-ba41-44de-bbf3-1aa02c3441df";
     fsType = "btrfs";
     options = [
-      "compress=zstd"
+      # "compress=zstd"  #< not much point in compressing... mostly videos and music; media.
       "defaults"
+      # `device=...` only needed if `btrfs scan` hasn't yet been run
+      # see: <https://askubuntu.com/a/484374>
+      # i don't know what guarantees NixOS/systemd make about that, so specifying all devices for now
+      # "device=/dev/disk/by-partuuid/14a7d00a-be53-2b4e-96f9-7e2c964674ec"  #< removed 2024-11-24 (for capacity upgrade)
+      "device=/dev/disk/by-partuuid/409a147e-2282-49eb-87a7-c968032ede88"  #< added 2024-11-24
+      # "device=/dev/disk/by-partuuid/6b86cc10-c3cc-ec4d-b20d-b6688f0959a6"  #< removed 2025-06-04 (early drive failure; capacity upgrade)
+      # "device=/dev/disk/by-partuuid/7fd85cac-b6f3-8248-af4e-68e703d11020"  #< removed 2024-11-13 (early drive failure)
+      "device=/dev/disk/by-partuuid/92ebbbfb-022f-427d-84d5-39349d4bc02a"  #< added 2025-05-14
+      "device=/dev/disk/by-partuuid/9e6c06b0-4a39-4d69-813f-1f5992f62ed7"  #< added 2025-06-05
+      "device=/dev/disk/by-partuuid/d9ad5ebc-0fc4-4d89-9fd0-619ce5210f1b"  #< added 2024-11-13
+      # "device=/dev/disk/by-partuuid/ef0e5c7b-fccf-f444-bac4-534424326159"  #< removed 2025-05-14 (early drive failure)
+      "nofail"
+      # "x-systemd.before=local-fs.target"
+      "x-systemd.device-bound=false"  #< don't unmount when `device` disappears (i thought this was necessary, for drive replacement, but it might not be)
+      "x-systemd.device-timeout=60s"
+      "x-systemd.mount-timeout=60s"
     ];
   };
-  sane.fs."/mnt/usb-hdd".mount = {};
 
+  # TODO: move this elsewhere and automate the ACLs!
   # FIRST TIME SETUP FOR MEDIA DIRECTORY:
-  # - set the group stick bit: `sudo find /var/media -type d -exec chmod g+s {} +`
+  # - set the group sticky bit: `sudo find /var/media -type d -exec chmod g+s {} +`
   #   - this ensures new files/dirs inherit the group of their parent dir (instead of the user who creates them)
   # - ensure everything under /var/media is mounted with `-o acl`, to support acls
   # - ensure all files are rwx by group: `setfacl --recursive --modify d:g::rwx /var/media`
@@ -104,8 +69,9 @@
     mode = "0775";
   }];
   sane.fs."/var/media/archive".dir = {};
+  sane.fs."/var/media/archive/temp".dir = {};
   # this is file.text instead of symlink.text so that it may be read over a remote mount (where consumers might not have any /nix/store/.../README.md path)
-  sane.fs."/var/media/archive/README.md".file.text = ''
+  sane.fs."/var/media/archive/temp/README.md".file.text = ''
     this directory is for media i wish to remove from my library,
     but keep for a short time in case i reverse my decision.
     treat it like a system trash can.
@@ -115,7 +81,6 @@
   sane.fs."/var/media/Books/Books".dir = {};
   sane.fs."/var/media/Books/Visual".dir = {};
   sane.fs."/var/media/collections".dir = {};
-  # sane.fs."/var/media/datasets".dir = {};
   sane.fs."/var/media/freeleech".dir = {};
   sane.fs."/var/media/Music".dir = {};
   sane.fs."/var/media/Pictures".dir = {};
@@ -124,34 +89,18 @@
   sane.fs."/var/media/Videos/Shows".dir = {};
   sane.fs."/var/media/Videos/Talks".dir = {};
 
-  # this is file.text instead of symlink.text so that it may be read over a remote mount (where consumers might not have any /nix/store/.../README.md path)
-  sane.fs."/var/lib/uninsane/datasets/README.md".file.text = ''
-    this directory may seem redundant with ../media/datasets. it isn't.
-    this directory exists on SSD, allowing for speedy access to specific datasets when necessary.
-    the contents should be a subset of what's in ../media/datasets.
-  '';
-
-  # btrfs doesn't easily support swapfiles
-  # swapDevices = [
-  #   { device = "/nix/persist/swapfile"; size = 4096; }
-  # ];
-
-  # this can be a partition. create with:
-  #   fdisk <dev>
-  #     n
-  #     <default partno>
-  #     <start>
-  #     <end>
-  #     t
-  #     <partno>
-  #     19  # set part type to Linux swap
-  #     w   # write changes
-  #   mkswap -L swap <part>
-  # swapDevices = [
-  #   {
-  #     label = "swap";
-  #     # TODO: randomEncryption.enable = true;
-  #   }
-  # ];
+  systemd.services.dedupe-media = {
+    description = "transparently de-duplicate /var/media entries by using block-level hardlinks";
+    script = ''
+      ${lib.getExe' pkgs.util-linux "hardlink"} /var/media --reflink=always --ignore-time --verbose
+    '';
+  };
+  systemd.timers.dedupe-media = {
+    wantedBy = [ "multi-user.target" ];
+    timerConfig = {
+      OnStartupSec = "23min";
+      OnUnitActiveSec = "720min";
+    };
+  };
 }
 

hosts/by-name/servo/net.nix

@@ -1,184 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  portOpts = with lib; types.submodule {
-    options = {
-      visibleTo.ovpn = mkOption {
-        type = types.bool;
-        default = false;
-      };
-    };
-  };
-in
-{
-  options = with lib; {
-    sane.ports.ports = mkOption {
-      # add the `visibleTo.ovpn` option
-      type = types.attrsOf portOpts;
-    };
-  };
-
-  config = {
-    networking.domain = "uninsane.org";
-
-    sane.ports.openFirewall = true;
-    sane.ports.openUpnp = true;
-
-    # unless we add interface-specific settings for each VPN, we have to define nameservers globally.
-    # networking.nameservers = [
-    #   "1.1.1.1"
-    #   "9.9.9.9"
-    # ];
-
-    # services.resolved.extraConfig = ''
-    #   # docs: `man resolved.conf`
-    #   # DNS servers to use via the `wg-ovpns` interface.
-    #   #   i hope that from the root ns, these aren't visible.
-    #   DNS=46.227.67.134%wg-ovpns 192.165.9.158%wg-ovpns
-    #   FallbackDNS=1.1.1.1 9.9.9.9
-    # '';
-
-    # OVPN CONFIG (https://www.ovpn.com):
-    # DOCS: https://nixos.wiki/wiki/WireGuard
-    # if you `systemctl restart wireguard-wg-ovpns`, make sure to also restart any other services in `NetworkNamespacePath = .../ovpns`.
-    # TODO: why not create the namespace as a seperate operation (nix config for that?)
-    networking.wireguard.enable = true;
-    networking.wireguard.interfaces.wg-ovpns = let
-      ip = "${pkgs.iproute2}/bin/ip";
-      in-ns = "${ip} netns exec ovpns";
-      iptables = "${pkgs.iptables}/bin/iptables";
-      veth-host-ip = "10.0.1.5";
-      veth-local-ip = "10.0.1.6";
-      vpn-ip = "185.157.162.178";
-      # DNS = 46.227.67.134, 192.165.9.158, 2a07:a880:4601:10f0:cd45::1, 2001:67c:750:1:cafe:cd45::1
-      vpn-dns = "46.227.67.134";
-      bridgePort = port: proto: ''
-        ${in-ns} ${iptables} -A PREROUTING -t nat -p ${proto} --dport ${port} -m iprange --dst-range ${vpn-ip} \
-          -j DNAT --to-destination ${veth-host-ip}
-      '';
-      bridgeStatements = lib.foldlAttrs
-        (acc: port: portCfg: acc ++ (builtins.map (bridgePort port) portCfg.protocol))
-        []
-        config.sane.ports.ports;
-    in {
-      privateKeyFile = config.sops.secrets.wg_ovpns_privkey.path;
-      # wg is active only in this namespace.
-      # run e.g. ip netns exec ovpns <some command like ping/curl/etc, it'll go through wg>
-      #   sudo ip netns exec ovpns ping www.google.com
-      interfaceNamespace = "ovpns";
-      ips = [
-        "185.157.162.178/32"
-      ];
-      peers = [
-        {
-          publicKey = "SkkEZDCBde22KTs/Hc7FWvDBfdOCQA4YtBEuC3n5KGs=";
-          endpoint = "185.157.162.10:9930";
-          # alternatively: use hostname, but that presents bootstrapping issues (e.g. if host net flakes)
-          # endpoint = "vpn36.prd.amsterdam.ovpn.com:9930";
-          allowedIPs = [ "0.0.0.0/0" ];
-          # nixOS says this is important for keeping NATs active
-          persistentKeepalive = 25;
-          # re-executes wg this often. docs hint that this might help wg notice DNS/hostname changes.
-          # so, maybe that helps if we specify endpoint as a domain name
-          # dynamicEndpointRefreshSeconds = 30;
-          # when refresh fails, try it again after this period instead.
-          # TODO: not avail until nixpkgs upgrade
-          # dynamicEndpointRefreshRestartSeconds = 5;
-        }
-      ];
-      preSetup = ''
-        ${ip} netns add ovpns || (test -e /run/netns/ovpns && echo "ovpns already exists")
-      '';
-      postShutdown = ''
-        ${in-ns} ip link del ovpns-veth-b || echo "couldn't delete ovpns-veth-b"
-        ${ip} link del ovpns-veth-a || echo "couldn't delete ovpns-veth-a"
-        ${ip} netns delete ovpns || echo "couldn't delete ovpns"
-        # restore rules/routes
-        ${ip} rule del from ${veth-host-ip} lookup ovpns pref 50 || echo "couldn't delete init -> ovpns rule"
-        ${ip} route del default via ${veth-local-ip} dev ovpns-veth-a proto kernel src ${veth-host-ip} metric 1002 table ovpns || echo "couldn't delete init -> ovpns route"
-        ${ip} rule add from all lookup local pref 0
-        ${ip} rule del from all lookup local pref 100
-      '';
-      postSetup = ''
-        # DOCS:
-        # - some of this approach is described here: <https://josephmuia.ca/2018-05-16-net-namespaces-veth-nat/>
-        # - iptables primer: <https://danielmiessler.com/study/iptables/>
-        # create veth pair
-        ${ip} link add ovpns-veth-a type veth peer name ovpns-veth-b
-        ${ip} addr add ${veth-host-ip}/24 dev ovpns-veth-a
-        ${ip} link set ovpns-veth-a up
-
-        # mv veth-b into the ovpns namespace
-        ${ip} link set ovpns-veth-b netns ovpns
-        ${in-ns} ip addr add ${veth-local-ip}/24 dev ovpns-veth-b
-        ${in-ns} ip link set ovpns-veth-b up
-
-        # make it so traffic originating from the host side of the veth
-        # is sent over the veth no matter its destination.
-        ${ip} rule add from ${veth-host-ip} lookup ovpns pref 50
-        # for traffic originating at the host veth to the WAN, use the veth as our gateway
-        # not sure if the metric 1002 matters.
-        ${ip} route add default via ${veth-local-ip} dev ovpns-veth-a proto kernel src ${veth-host-ip} metric 1002 table ovpns
-        # give the default route lower priority
-        ${ip} rule add from all lookup local pref 100
-        ${ip} rule del from all lookup local pref 0
-
-        # in order to access DNS in this netns, we need to route it to the VPN's nameservers
-        # - alternatively, we could fix DNS servers like 1.1.1.1.
-        ${in-ns} ${iptables} -A OUTPUT -t nat -p udp --dport 53 -m iprange --dst-range 127.0.0.53 \
-          -j DNAT --to-destination ${vpn-dns}:53
-      '' + (lib.concatStringsSep "\n" bridgeStatements);
-    };
-
-    # create a new routing table that we can use to proxy traffic out of the root namespace
-    # through the ovpns namespace, and to the WAN via VPN.
-    networking.iproute2.rttablesExtraConfig = ''
-      5 ovpns
-    '';
-    networking.iproute2.enable = true;
-
-
-    # HURRICANE ELECTRIC CONFIG:
-    # networking.sits = {
-    #   hurricane = {
-    #     remote = "216.218.226.238";
-    #     local = "192.168.0.5";
-    #     # local = "10.0.0.5";
-    #     # remote = "10.0.0.1";
-    #     # local = "10.0.0.22";
-    #     dev = "eth0";
-    #     ttl = 255;
-    #   };
-    # };
-    # networking.interfaces."hurricane".ipv6 = {
-    #   addresses = [
-    #     # mx.uninsane.org (publically routed /64)
-    #     {
-    #       address = "2001:470:b:465::1";
-    #       prefixLength = 128;
-    #     }
-    #     # client addr
-    #     # {
-    #     #   address = "2001:470:a:466::2";
-    #     #   prefixLength = 64;
-    #     # }
-    #   ];
-    #   routes = [
-    #     {
-    #       address = "::";
-    #       prefixLength = 0;
-    #       # via = "2001:470:a:466::1";
-    #     }
-    #   ];
-    # };
-
-    # # after configuration, we want the hurricane device to look like this:
-    # # hurricane: flags=209<UP,POINTOPOINT,RUNNING,NOARP>  mtu 1480
-    # #            inet6 2001:470:a:450::2  prefixlen 64  scopeid 0x0<global>
-    # #            inet6 fe80::c0a8:16  prefixlen 64  scopeid 0x20<link>
-    # #            sit  txqueuelen 1000  (IPv6-in-IPv4)
-    # # test with:
-    # #   curl --interface hurricane http://[2607:f8b0:400a:80b::2004]
-    # #   ping 2607:f8b0:400a:80b::2004
-  };
-}

hosts/by-name/servo/net/default.nix

@@ -0,0 +1,60 @@
+# debugging:
+# - enable logs (shows handshake attempts)
+#   - `echo module wireguard +p | sane-sudo-redirect /sys/kernel/debug/dynamic_debug/control`
+#   - `sudo dmesg --follow`
+#     patterns: "Sending keepalive packet to peer NN (N.N.N.N:NNNNN)"
+#     patterns: "Sending handshake initiation to peer NN (N.N.N.N:NNNNN)"
+# - when wg-doof and wg-ovpns stop routing traffic, restart with:
+#   - `systemctl restart netns-doof-wg`
+# - handshaking:
+#   - `wg show` should *always* show "latest handshake: N", with N < 2 minutes ago.
+{ lib, ... }:
+let
+  portOpts = with lib; types.submodule {
+    options = {
+      visibleTo.ovpns = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          whether to forward inbound traffic on the OVPN vpn port to the corresponding localhost port.
+        '';
+      };
+      visibleTo.doof = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          whether to forward inbound traffic on the doofnet vpn port to the corresponding localhost port.
+        '';
+      };
+    };
+  };
+in
+{
+  options = with lib; {
+    sane.ports.ports = mkOption {
+      # add the `visibleTo.{doof,ovpns}` options
+      type = types.attrsOf portOpts;
+    };
+  };
+
+  imports = [
+    ./doof.nix
+    ./ovpn.nix
+    ./wg-home.nix
+  ];
+
+  config = {
+    networking.domain = "uninsane.org";
+    systemd.network.networks."50-eth0" = {
+      matchConfig.Name = "eth0";
+      networkConfig.Address = [
+        "205.201.63.12/32"
+        "10.78.79.51/22"
+      ];
+      networkConfig.DNS = [ "10.78.79.1" ];
+    };
+
+    sane.ports.openFirewall = true;
+    sane.ports.openUpnp = true;
+  };
+}

hosts/by-name/servo/net/doof.nix

@@ -0,0 +1,27 @@
+{ config, ... }:
+{
+  # tun-sea config
+  sane.dns.zones."uninsane.org".inet.A."doof.tunnel" = "205.201.63.12";
+  # sane.dns.zones."uninsane.org".inet.AAAA."doof.tunnel" = "2602:fce8:106::51";  #< TODO: enable IPv6 (i have /128)
+
+  # if the tunnel breaks, restart it manually:
+  # - `systemctl restart netns-doof.service`
+  sane.netns.doof = {
+    veth.initns.ipv4 = "10.0.2.5";
+    veth.netns.ipv4 = "10.0.2.6";
+    routeTable = 12;
+    # wg.port = 51821;
+    wg.privateKeyFile = config.sops.secrets.wg_doof_privkey.path;
+    wg.address.ipv4 = "205.201.63.12";
+    wg.peer.publicKey = "nuESyYEJ3YU0hTZZgAd7iHBz1ytWBVM5PjEL1VEoTkU=";
+    wg.peer.endpoint = "tun-sea.doof.net:53263";
+    # wg.peer.endpoint = "205.201.63.44:53263";
+  };
+
+  # inside doof, forward DNS requests back to the root machine
+  # this is fine: nothing inside the ns performs DNS except for wireguard,
+  # and we're not forwarding external DNS requests here
+  # XXX: ACTUALLY, CAN'T EASILY DO THAT BECAUSE HICKORY-DNS IS ALREADY USING PORT 53
+  # but that's ok, we don't really need DNS *inside* this namespace.
+  # sane.netns.doof.dns.ipv4 = config.sane.netns.doof.veth.netns.ipv4;
+}

hosts/by-name/servo/net/ovpn.nix

@@ -0,0 +1,20 @@
+{ config, ... }:
+{
+  sane.ovpn.addrV4 = "172.23.174.114";  #< this applies to the dynamic VPNs -- NOT the static VPN
+  # sane.ovpn.addrV6 = "fd00:0000:1337:cafe:1111:1111:8df3:14b0";
+
+  # OVPN CONFIG (https://www.ovpn.com):
+  # DOCS: https://nixos.wiki/wiki/WireGuard
+  sane.netns.ovpns = {
+    veth.initns.ipv4 = "10.0.1.5";
+    veth.netns.ipv4 = "10.0.1.6";
+    routeTable = 11;
+    dns.ipv4 = "46.227.67.134";  #< DNS requests inside the namespace are forwarded here
+    # wg.port = 51822;
+    wg.privateKeyFile = config.sops.secrets.wg_ovpns_privkey.path;
+    wg.address.ipv4 = "146.70.100.165";  #< IP address for my end of the VPN tunnel. for OVPN public IPv4, this is also the public IP address.
+    wg.peer.publicKey = "xc9p/lf2uLg6IGDh54E0Pbc6WI/J9caaByhwD4Uiu0Q=";  #< pubkey by which i can authenticate OVPN, varies per OVPN endpoint
+    wg.peer.endpoint = "vpn31.prd.losangeles.ovpn.com:9930";
+    # wg.peer.endpoint = "45.83.89.131:9930";
+  };
+}

hosts/by-name/servo/net/wg-home.nix

@@ -0,0 +1,14 @@
+{ config, ... }:
+{
+  sane.services.wg-home.enable = true;
+  sane.services.wg-home.visibleToWan = true;
+  sane.services.wg-home.forwardToWan = true;
+  sane.services.wg-home.routeThroughServo = false;
+  services.unbound.settings.server.interface = [
+    # provide DNS to my wireguard clients
+    config.sane.hosts.by-name."servo".wg-home.ip
+  ];
+  services.unbound.settings.server.access-control = [
+    "${config.sane.hosts.by-name."servo".wg-home.ip}/24 allow"
+  ];
+}

hosts/by-name/servo/services/bitmagnet.nix

@@ -0,0 +1,70 @@
+# bitmagnet is a DHT crawler. it discovers publicly reachable torrents and indexes:
+# - torrent's magnet URI
+# - torrent's name
+# - torrent's file list (the first 100 files, per torrent), including size and "type" (e.g. video)
+# - seeder/leecher counts
+# - torrent's size
+# it provides a web UI to query these, especially a search form.
+# data is stored in postgresql as `bitmagnet` db (`sudo -u bitmagnet psql`)
+# after 30 days of operation:
+# - 12m torrents discovered
+# - 77GB database size  => 6500B per torrent
+{ config, ... }:
+{
+  services.bitmagnet.enable = true;
+  sane.netns.ovpns.services = [ "bitmagnet" ];
+  sane.ports.ports."3334" = {
+    protocol = [ "tcp" "udp" ];
+    # visibleTo.ovpns = true;  #< not needed: it runs in the ovpns namespace
+    description = "colin-bitmagnet";
+  };
+
+  services.bitmagnet.settings = {
+    # dht_crawler.scaling_factor: how rapidly to crawl the DHT.
+    #   influences number of worker threads, buffer sizes, etc.
+    #   default: 10.
+    #   docs claim "diminishing returns" above 10, but seems weakly confident about that.
+    dht_crawler.scaling_factor = 64;
+    # http_server.local_address: `$addr:$port` to `listen` to.
+    # default is `:3333`, which listens on _all_ interfaces.
+    # the http server exposes unprotected admin endpoints though, so restrict to private interfaces:
+    http_server.local_address = "${config.sane.netns.ovpns.veth.netns.ipv4}:3333";
+    # tmdb.enabled: whether to query The Movie DataBase to resolve filename -> movie title.
+    #   default: true.
+    #   docs claim 1 query per second rate limit, unless you supply your own API key.
+    tmdb.enabled = false;
+  };
+
+  # bitmagnet web client
+  # protected by passwd because it exposes some mutation operations:
+  # - queuing "jobs"
+  # - deleting torrent infos (in bulk)
+  # it uses graphql for _everything_, so no easy way to disable just the mutations (and remove the password) AFAICT.
+  services.nginx.virtualHosts."bitmagnet.uninsane.org" = {
+    # basicAuth is cleartext user/pw, so FORCE this to happen over SSL
+    forceSSL = true;
+    enableACME = true;
+    locations."/" = {
+      proxyPass = "http://${config.sane.netns.ovpns.veth.netns.ipv4}:3333";
+      recommendedProxySettings = true;
+    };
+    basicAuthFile = config.sops.secrets.bitmagnet_passwd.path;
+  };
+  sops.secrets."bitmagnet_passwd" = {
+    owner = config.users.users.nginx.name;
+    mode = "0400";
+  };
+
+  sane.dns.zones."uninsane.org".inet.CNAME."bitmagnet" = "native";
+
+  systemd.services.bitmagnet = {
+    # hardening (systemd-analyze security bitmagnet). base nixos service is already partially hardened.
+    serviceConfig.CapabilityBoundingSet = "";
+    serviceConfig.SystemCallArchitectures = "native";
+    serviceConfig.PrivateDevices = true;
+    serviceConfig.PrivateUsers = true;
+    serviceConfig.ProtectProc = "invisible";
+    serviceConfig.ProcSubset = "pid";
+    serviceConfig.SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+  };
+}

hosts/by-name/servo/services/calibre.nix

@@ -1,34 +0,0 @@
-{ config, lib, ... }:
-
-let
-  cweb-cfg = config.services.calibre-web;
-  inherit (cweb-cfg) user group;
-  inherit (cweb-cfg.listen) ip port;
-  svc-dir = "/var/lib/${cweb-cfg.dataDir}";
-in
-# XXX: disabled because of runtime errors like:
-# > File "/nix/store/c7jqvx980nlg9xhxi065cba61r2ain9y-calibre-web-0.6.19/lib/python3.10/site-packages/calibreweb/cps/db.py", line 926, in speaking_language
-# > languages = self.session.query(Languages) \
-# > AttributeError: 'NoneType' object has no attribute 'query'
-lib.mkIf false
-{
-  sane.persist.sys.byStore.plaintext = [
-    { inherit user group; mode = "0700"; path = svc-dir; method = "bind"; }
-  ];
-
-  services.calibre-web.enable = true;
-  services.calibre-web.listen.ip = "127.0.0.1";
-  # XXX: externally populate `${svc-dir}/metadata.db` (once) from
-  #   <https://github.com/janeczku/calibre-web/blob/master/library/metadata.db>
-  # i don't know why you have to do this??
-  # services.calibre-web.options.calibreLibrary = svc-dir;
-
-  services.nginx.virtualHosts."calibre.uninsane.org" = {
-    forceSSL = true;
-    enableACME = true;
-    locations."/" = {
-      proxyPass = "http://${ip}:${builtins.toString port}";
-    };
-  };
-  sane.dns.zones."uninsane.org".inet.CNAME."calibre" = "native";
-}

hosts/by-name/servo/services/coturn.nix

@@ -29,7 +29,15 @@
 #   - bind the turn server to the veth connecting it to the VPN namespace (so it sends outgoing traffic to the right place).
 #   - NAT the turn port range from VPN into root namespace (so it receives incomming traffic).
 #   - this approach would fail the prosody conversations.im check, but i didn't notice *obvious* call routing errors.
-{ lib, ... }:
+#
+# debugging:
+# - log messages like 'usage: realm=<turn.uninsane.org>, username=<1715915193>, rp=14, rb=1516, sp=8, sb=684'
+#   - rp = received packets
+#   - rb = received bytes
+#   - sp = sent packets
+#   - sb = sent bytes
+
+{ config, lib, ... }:
 let
   # TURN port range (inclusive).
   # default coturn behavior is to use the upper quarter of all ports. i.e. 49152 - 65535.
@@ -48,7 +56,7 @@ in
   #       protocol = [ "tcp" "udp" ];
   #       # visibleTo.lan = true;
   #       # visibleTo.wan = true;
-  #       visibleTo.ovpn = true;  # forward traffic from the VPN to the root NS
+  #       visibleTo.ovpns = true;  # forward traffic from the VPN to the root NS
   #       description = "colin-stun-turn";
   #     };
   #     "5349" = {
@@ -56,7 +64,7 @@ in
   #       protocol = [ "tcp" ];
   #       # visibleTo.lan = true;
   #       # visibleTo.wan = true;
-  #       visibleTo.ovpn = true;
+  #       visibleTo.ovpns = true;
   #       description = "colin-stun-turn-over-tls";
   #     };
   #   }
@@ -69,7 +77,7 @@ in
   #       protocol = [ "tcp" "udp" ];
   #       # visibleTo.lan = true;
   #       # visibleTo.wan = true;
-  #       visibleTo.ovpn = true;
+  #       visibleTo.ovpns = true;
   #       description = "colin-turn-${builtins.toString count}-of-${builtins.toString numPorts}";
   #     };
   #   })
@@ -96,13 +104,6 @@ in
     SRV."_turns._tcp" =                       "5 50 5349 turn";
   };
 
-  sane.derived-secrets."/var/lib/coturn/shared_secret.bin" = {
-    encoding = "base64";
-    # TODO: make this not globally readable
-    acl.mode = "0644";
-  };
-  sane.fs."/var/lib/coturn/shared_secret.bin".wantedBeforeBy = [ "coturn.service" ];
-
   # provide access to certs
   users.users.turnserver.extraGroups = [ "nginx" ];
 
@@ -110,21 +111,29 @@ in
   services.coturn.realm = "turn.uninsane.org";
   services.coturn.cert = "/var/lib/acme/turn.uninsane.org/fullchain.pem";
   services.coturn.pkey = "/var/lib/acme/turn.uninsane.org/key.pem";
+
+  # N.B.: prosody needs to read this shared secret
+  sops.secrets."coturn_shared_secret".owner = "turnserver";
+  sops.secrets."coturn_shared_secret".group = "turnserver";
+  sops.secrets."coturn_shared_secret".mode = "0440";
+
+  #v disable to allow unauthenticated access (or set `services.coturn.no-auth = true`)
   services.coturn.use-auth-secret = true;
-  services.coturn.static-auth-secret-file = "/var/lib/coturn/shared_secret.bin";
-  services.coturn.lt-cred-mech = true;
+  services.coturn.static-auth-secret-file = "/run/secrets/coturn_shared_secret";
+  services.coturn.lt-cred-mech = true; #< XXX: use-auth-secret overrides lt-cred-mech
+
   services.coturn.min-port = turnPortLow;
   services.coturn.max-port = turnPortHigh;
   # services.coturn.secure-stun = true;
   services.coturn.extraConfig = lib.concatStringsSep "\n" [
     "verbose"
-    # "Verbose"  #< even MORE verbosity than "verbose"
-    # "no-multicast-peers"  # disables sending to IPv4 broadcast addresses (e.g. 224.0.0.0/3)
-    # "listening-ip=10.0.1.5" "external-ip=185.157.162.178"  #< 2024/04/25: works, if running in root namespace
-    "listening-ip=185.157.162.178" "external-ip=185.157.162.178"
+    # "Verbose"  #< even MORE verbosity than "verbose"  (it's TOO MUCH verbosity really)
+    "no-multicast-peers"  # disables sending to IPv4 broadcast addresses (e.g. 224.0.0.0/3)
+    # "listening-ip=${config.sane.netns.ovpns.veth.initns.ipv4}" "external-ip=${config.sane.netns.ovpns.wg.address.ipv4}"  #< 2024/04/25: works, if running in root namespace
+    "listening-ip=${config.sane.netns.ovpns.wg.address.ipv4}" "external-ip=${config.sane.netns.ovpns.wg.address.ipv4}"
 
     # old attempts:
-    # "external-ip=185.157.162.178/10.0.1.5"
+    # "external-ip=${config.sane.netns.ovpns.wg.address.ipv4}/${config.sane.netns.ovpns.veth.initns.ipv4}"
     # "listening-ip=10.78.79.51"  # can be specified multiple times; omit for *
     # "external-ip=97.113.128.229/10.78.79.51"
     # "external-ip=97.113.128.229"

hosts/by-name/servo/services/cryptocurrencies/bitcoin.nix

@@ -1,4 +1,5 @@
 # as of 2023/12/02: complete blockchain is 530 GiB (on-disk size may be larger)
+# as of 2025/08/06: on-disk blockchain as reported by `du` is 732 GiB
 #
 # ports:
 # - 8333: for node-to-node communications
@@ -16,14 +17,17 @@
 # - validate with `bitcoin-cli -netinfo`
 { config, lib, pkgs, sane-lib, ... }:
 let
+  # bitcoind = config.sane.programs.bitcoind.packageUnwrapped;
+  bitcoind = pkgs.bitcoind;
   # wrapper to run bitcoind with the tor onion address as externalip (computed at runtime)
-  _bitcoindWithExternalIp = with pkgs; writeShellScriptBin "bitcoind" ''
+  _bitcoindWithExternalIp = pkgs.writeShellScriptBin "bitcoind" ''
+    set -xeu
     externalip="$(cat /var/lib/tor/onion/bitcoind/hostname)"
-    exec ${bitcoind}/bin/bitcoind "-externalip=$externalip" "$@"
+    exec ${lib.getExe' bitcoind "bitcoind"} "-externalip=$externalip" "$@"
   '';
   # the package i provide to services.bitcoind ends up on system PATH, and used by other tools like clightning.
   # therefore, even though services.bitcoind only needs `bitcoind` binary, provide all the other bitcoin-related binaries (notably `bitcoin-cli`) as well:
-  bitcoindWithExternalIp = with pkgs; symlinkJoin {
+  bitcoindWithExternalIp = pkgs.symlinkJoin {
     name = "bitcoind-with-external-ip";
     paths = [ _bitcoindWithExternalIp bitcoind ];
   };
@@ -61,23 +65,67 @@ in
       passwordHMAC = "30002c05d82daa210550e17a182db3f3$6071444151281e1aa8a2729f75e3e2d224e9d7cac3974810dab60e7c28ffaae4";
     };
     extraConfig = ''
+      # checkblocks: default 6: how many blocks to verify on start
+      checkblocks=3
       # don't load the wallet, and disable wallet RPC calls
       disablewallet=1
       # proxy all outbound traffic through Tor
       proxy=127.0.0.1:9050
     '';
+    extraCmdlineOptions = [
+      # `man bitcoind` for options
+      # "-assumevalid=0"  # to perform script validation on all blocks, instead of just the latest checkpoint published by bitcoin-core
+      # "-debug"
+      # "-debug=estimatefee"
+      # "-debug=leveldb"
+      # "-debug=http"
+      # "-debug=net"
+      "-debug=proxy"
+      "-debug=rpc"
+      # "-debug=validation"
+      # "-reindex"  # wipe chainstate, block index, other indices; rebuild from blk*.dat (takes 2.5hrs)
+      # "-reindex-chainstate"  # wipe chainstate; rebuild from blk*.dat
+    ];
   };
 
   users.users.bitcoind-mainnet.extraGroups = [ "tor" ];
 
-  systemd.services.bitcoind-mainnet.serviceConfig.RestartSec = "30s";  #< default is 0
+  systemd.services.bitcoind-mainnet = {
+    after = [ "tor.service" ];
+    requires = [ "tor.service" ];
+    serviceConfig.RestartSec = "30s";  #< default is 0
+
+    # hardening (systemd-analyze security bitcoind-mainnet)
+    serviceConfig.StateDirectory = "bitcoind-mainnet";
+    serviceConfig.LockPersonality = true;
+    serviceConfig.MemoryDenyWriteExecute = "true";
+    serviceConfig.NoNewPrivileges = "true";
+    serviceConfig.PrivateDevices = "true";
+    serviceConfig.PrivateMounts = true;
+    serviceConfig.PrivateTmp = "true";
+    serviceConfig.PrivateUsers = true;
+    serviceConfig.ProcSubset = "pid";
+    serviceConfig.ProtectControlGroups = true;
+    serviceConfig.ProtectHome = true;
+    serviceConfig.ProtectHostname = true;
+    serviceConfig.ProtectKernelLogs = true;
+    serviceConfig.ProtectKernelModules = true;
+    serviceConfig.ProtectKernelTunables = true;
+    serviceConfig.ProtectProc = "invisible";
+    serviceConfig.ProtectSystem = lib.mkForce "strict";
+    serviceConfig.RemoveIPC = true;
+    serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+    serviceConfig.RestrictNamespaces = true;
+    serviceConfig.RestrictSUIDSGID = true;
+    serviceConfig.SystemCallArchitectures = "native";
+    serviceConfig.SystemCallFilter = [ "@system-service" ];
+  };
 
-  sane.users.colin.fs.".bitcoin/bitcoin.conf" = sane-lib.fs.wantedSymlinkTo config.sops.secrets."bitcoin.conf".path;
   sops.secrets."bitcoin.conf" = {
     mode = "0600";
     owner = "colin";
     group = "users";
   };
 
-  sane.programs.bitcoind.enableFor.user.colin = true;  # for debugging/administration: `bitcoin-cli`
+  sane.programs.bitcoin-cli.enableFor.user.colin = true;  # for debugging/administration: `bitcoin-cli`
 }

hosts/by-name/servo/services/cryptocurrencies/clightning.nix

@@ -72,13 +72,11 @@
 
 { config, pkgs, ... }:
 {
-  sane.persist.sys.byStore.ext = [
+  sane.persist.sys.byStore.private = [
+    # clightning takes up only a few MB. but then several hundred MB of crash logs that i should probably GC.
     { user = "clightning"; group = "clightning"; mode = "0710"; path = "/var/lib/clightning"; method = "bind"; }
   ];
 
-  # `lightning-cli` finds its RPC file via `~/.lightning/bitcoin/lightning-rpc`, to message the daemon
-  sane.user.fs.".lightning".symlink.target = "/var/lib/clightning";
-
   # see bitcoin.nix for how to generate this
   services.bitcoind.mainnet.rpc.users.clightning.passwordHMAC =
     "befcb82d9821049164db5217beb85439$2c31ac7db3124612e43893ae13b9527dbe464ab2d992e814602e7cb07dc28985";
@@ -105,6 +103,7 @@
   users.users.clightning.extraGroups = [ "tor" ];
 
   systemd.services.clightning.after = [ "tor.service" ];
+  systemd.services.clightning.requires = [ "tor.service" ];
 
   # lightning-config contains fields from here:
   # - <https://docs.corelightning.org/docs/configuration>
@@ -116,11 +115,24 @@
   # - fee-per-satoshi=<ppm>
   # - feature configs (i.e. experimental-xyz options)
   sane.services.clightning.extraConfig = ''
-    log-level=debug:lightningd
+    # log levels: "io", "trace", "debug", "info", "unusual", "broken"
+    # log-level=info
+    # log-level=info:lightningd
+    # log-level=debug:lightningd
+    log-level=debug
+    # log-level=io
+
+    disable-plugin=cln-xpay
+
+    # let me use `lightning-cli dev-*` subcommands, fucktards.
+    developer
+    # `developer` enables `dev-*` but *disables* the older commands. asshats.
+    allow-deprecated-apis=true
+
     # peerswap:
     # - config example: <https://github.com/fort-nix/nix-bitcoin/pull/462/files#diff-b357d832705b8ce8df1f41934d613f79adb77c4cd5cd9e9eb12a163fca3e16c6>
     # XXX: peerswap crashes clightning on launch. stacktrace is useless.
-    # plugin=${pkgs.peerswap}/bin/peerswap
+    # plugin={lib.getExe' pkgs.peerswap "peerswap"}
     # peerswap-db-path=/var/lib/clightning/peerswap/swaps
     # peerswap-policy-path=...
   '';
@@ -131,5 +143,5 @@
     group = "clightning";
   };
 
-  sane.programs.clightning.enableFor.user.colin = true;  # for debugging/admin: `lightning-cli`
+  sane.programs.lightning-cli.enableFor.user.colin = true;  # for debugging/admin:
 }

hosts/by-name/servo/services/cryptocurrencies/i2p.nix

@@ -1,4 +1,5 @@
-{ ... }:
+{ lib, ... }:
+lib.mkIf false  #< 2024/07/27: i don't use it, too much surface-area for me to run it pro-bono (`systemd-analyze security monero`)
 {
   services.i2p.enable = true;
 }

hosts/by-name/servo/services/cryptocurrencies/monero.nix

@@ -1,5 +1,6 @@
 # as of 2023/11/26: complete downloaded blockchain should be 200GiB on disk, give or take.
-{ ... }:
+{ lib, ... }:
+lib.mkIf false  #< 2024/07/27: i don't use it, too much surface-area for me to run it pro-bono (`systemd-analyze security monero`)
 {
   sane.persist.sys.byStore.ext = [
     # /var/lib/monero/lmdb is what consumes most of the space

hosts/by-name/servo/services/cryptocurrencies/tor.nix

@@ -1,10 +1,10 @@
 # tor settings: <https://2019.www.torproject.org/docs/tor-manual.html.en>
 { lib, ... }:
 {
-  # tor hidden service hostnames aren't deterministic, so persist.
-  # might be able to get away with just persisting /var/lib/tor/onion, not sure.
-  sane.persist.sys.byStore.plaintext = [
-    { user = "tor"; group = "tor"; mode = "0710"; path = "/var/lib/tor"; method = "bind"; }
+  sane.persist.sys.byStore.ephemeral = [
+    # N.B.: tor hidden service hostnames aren't deterministic, so if you need them
+    # to be preserved across reboots then persist /var/lib/tor/onion in "private" store.
+   { user = "tor"; group = "tor"; mode = "0710"; path = "/var/lib/tor"; method = "bind"; }
   ];
 
   # tor: `tor.enable` doesn't start a relay, exit node, proxy, etc. it's minimal.

hosts/by-name/servo/services/default.nix

@@ -1,34 +1,36 @@
 { ... }:
 {
   imports = [
-    ./calibre.nix
+    ./bitmagnet.nix
     ./coturn.nix
     ./cryptocurrencies
     ./email
     ./ejabberd.nix
     ./freshrss.nix
     ./export
+    ./hickory-dns.nix
+    ./gerbera.nix
     ./gitea.nix
     ./goaccess.nix
     ./ipfs.nix
-    ./jackett.nix
-    ./jellyfin.nix
+    ./jackett
+    ./jellyfin
     ./kiwix-serve.nix
     ./komga.nix
     ./lemmy.nix
     ./matrix
+    ./minidlna.nix
+    ./mumble.nix
     ./navidrome.nix
-    ./nginx.nix
+    ./nginx
     ./nixos-prebuild.nix
-    ./nixserve.nix
     ./ntfy
     ./pict-rs.nix
     ./pleroma.nix
-    ./postgres.nix
+    ./postgresql
     ./prosody
     ./slskd.nix
-    ./transmission.nix
-    ./trust-dns.nix
+    ./transmission
     ./wikipedia.nix
   ];
 }

hosts/by-name/servo/services/ejabberd.nix

@@ -44,61 +44,61 @@ in
 # everything configured below was fine: used ejabberd for several months.
 lib.mkIf false
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     { user = "ejabberd"; group = "ejabberd"; path = "/var/lib/ejabberd"; method = "bind"; }
   ];
   sane.ports.ports = lib.mkMerge ([
     {
       "3478" = {
         protocol = [ "tcp" "udp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-stun-turn";
       };
       "5222" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-client-to-server";
       };
       "5223" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpps-client-to-server";  # XMPP over TLS
       };
       "5269" = {
         protocol = [ "tcp" ];
-        visibleTo.wan = true;
+        visibleTo.doof = true;
         description = "colin-xmpp-server-to-server";
       };
       "5270" = {
         protocol = [ "tcp" ];
-        visibleTo.wan = true;
+        visibleTo.doof = true;
         description = "colin-xmpps-server-to-server";  # XMPP over TLS
       };
       "5280" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-bosh";
       };
       "5281" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-bosh-https";
       };
       "5349" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-stun-turn-over-tls";
       };
       "5443" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-web-services";  # file uploads, websockets, admin
       };
     }
@@ -109,8 +109,8 @@ lib.mkIf false
         numPorts = turnPortHigh - turnPortLow + 1;
       in {
         protocol = [ "tcp" "udp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-xmpp-turn-${builtins.toString count}-of-${builtins.toString numPorts}";
       };
     })
@@ -457,13 +457,12 @@ lib.mkIf false
         mod_version = {};
       };
     });
-    sed = "${pkgs.gnused}/bin/sed";
   in ''
     ip=$(cat '${config.sane.services.dyn-dns.ipPath}')
     # config is 444 (not 644), so we want to write out-of-place and then atomically move
     # TODO: factor this out into `sane-woop` helper?
     rm -f /var/lib/ejabberd/ejabberd.yaml.new
-    ${sed} "s/%ANATIVE%/$ip/g" ${config-in} > /var/lib/ejabberd/ejabberd.yaml.new
+    ${lib.getExe pkgs.gnused} "s/%ANATIVE%/$ip/g" ${config-in} > /var/lib/ejabberd/ejabberd.yaml.new
     mv /var/lib/ejabberd/ejabberd.yaml{.new,}
   '';
 

hosts/by-name/servo/services/email/default.nix

@@ -25,10 +25,10 @@
 #
 # debugging: general connectivity issues
 # - test that inbound port 25 is unblocked:
-#   - `curl https://canyouseeme.org/ --data 'port=25&IP=185.157.162.178' | grep 'see your service'`
+#   - `curl https://canyouseeme.org/ --data 'port=25&IP=$MX_IP' | grep 'see your service'`
 #     - and retry with port 465, 587
 #     - i think this API requires the queried IP match the source IP
-#   - if necessary, `systemctl stop postfix` and `sudo nc -l 185.157.162.178 25`, then try https://canyouseeme.org
+#   - if necessary, `systemctl stop postfix` and `sudo nc -l $MX_IP 25`, then try https://canyouseeme.org
 
 { ... }:
 {

hosts/by-name/servo/services/email/dovecot.nix

@@ -8,14 +8,14 @@
 {
   sane.ports.ports."143" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-imap-imap.uninsane.org";
   };
   sane.ports.ports."993" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-imaps-imap.uninsane.org";
   };
 
@@ -83,8 +83,8 @@
     #   sieve_plugins = sieve_imapsieve
     # }
 
-    mail_debug = yes
-    auth_debug = yes
+    # mail_debug = yes
+    # auth_debug = yes
     # verbose_ssl = yes
   '';
 
@@ -124,7 +124,9 @@
       # ];
     };
   };
-  services.dovecot2.modules = [
+  environment.systemPackages = [
+    # XXX(2025-03-16): dovecot loads modules from /run/current-system/sw/lib/dovecot/modules
+    # see: <https://github.com/NixOS/nixpkgs/pull/387642>
     pkgs.dovecot_pigeonhole  # enables sieve execution (?)
   ];
   services.dovecot2.sieve = {
@@ -141,5 +143,5 @@
     '';
   };
 
-  systemd.services.dovecot2.serviceConfig.RestartSec = lib.mkForce "15s";  # nixos defaults this to 1s
+  systemd.services.dovecot.serviceConfig.RestartSec = lib.mkForce "15s";  # nixos defaults this to 1s
 }

hosts/by-name/servo/services/email/postfix.nix

@@ -1,6 +1,13 @@
 # postfix config options: <https://www.postfix.org/postconf.5.html>
+# config files:
+# - /etc/postfix/main.cf
+# - /etc/postfix/master.cf
+#
+# logs:
+# - postfix logs directly to *syslog*,
+#   so check e.g. ~/.local/share/rsyslog
 
-{ lib, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
 let
   submissionOptions = {
@@ -18,14 +25,14 @@ let
   };
 in
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     # TODO: mode? could be more granular
-    { user = "opendkim"; group = "opendkim"; path = "/var/lib/opendkim"; method = "bind"; }
-    { user = "root"; group = "root"; path = "/var/lib/postfix"; method = "bind"; }
+    { user = "opendkim"; group = "opendkim"; path = "/var/lib/opendkim"; method = "bind"; }  #< TODO: migrate to secrets
     { user = "root"; group = "root"; path = "/var/spool/mail"; method = "bind"; }
     # *probably* don't need these dirs:
     # "/var/lib/dhparams"          # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/security/dhparams.nix
     # "/var/lib/dovecot"
+    # "/var/lib/postfix"
   ];
 
   # XXX(2023/10/20): opening these ports in the firewall has the OPPOSITE effect as intended.
@@ -56,8 +63,7 @@ in
 
   sane.dns.zones."uninsane.org".inet = {
     MX."@" = "10 mx.uninsane.org.";
-    # XXX: RFC's specify that the MX record CANNOT BE A CNAME
-    A."mx" = "185.157.162.178";
+    A."mx" = "%AOVPNS%"; #< XXX: RFC's specify that the MX record CANNOT BE A CNAME. TODO: use "%AOVPNS%?
 
     # Sender Policy Framework:
     #   +mx     => mail passes if it originated from the MX
@@ -93,9 +99,12 @@ in
   services.postfix.hostname = "mx.uninsane.org";
   services.postfix.origin = "uninsane.org";
   services.postfix.destination = [ "localhost" "uninsane.org" ];
-  services.postfix.sslCert = "/var/lib/acme/mx.uninsane.org/fullchain.pem";
-  services.postfix.sslKey = "/var/lib/acme/mx.uninsane.org/key.pem";
+  services.postfix.config.smtpd_tls_chain_files = [
+    "/var/lib/acme/mx.uninsane.org/key.pem"
+    "/var/lib/acme/mx.uninsane.org/fullchain.pem"
+  ];
 
+  # see: `man 5 virtual`
   services.postfix.virtual = ''
     notify.matrix@uninsane.org matrix-synapse
     @uninsane.org colin
@@ -105,7 +114,7 @@ in
     # smtpd_milters = local:/run/opendkim/opendkim.sock
     # milter docs: http://www.postfix.org/MILTER_README.html
     # mail filters for receiving email and from authorized SMTP clients (i.e. via submission)
-    # smtpd_milters = inet:185.157.162.190:8891
+    # smtpd_milters = inet:$IP:8891
     # opendkim.sock will add a Authentication-Results header, with `dkim=pass|fail|...` value to received messages
     smtpd_milters = "unix:/run/opendkim/opendkim.sock";
     # mail filters for sendmail
@@ -136,17 +145,32 @@ in
     # smtpd_sender_restrictions = reject_unknown_sender_domain
   };
 
+  # debugging options:
+  # services.postfix.masterConfig = {
+  #   "proxymap".args = [ "-v" ];
+  #   "proxywrite".args = [ "-v" ];
+  #   "relay".args = [ "-v" ];
+  #   "smtp".args = [ "-v" ];
+  #   "smtp_inet".args = [ "-v" ];
+  #   "submission".args = [ "-v" ];
+  #   "submissions".args = [ "-v" ];
+  #   "submissions".chroot = false;
+  #   "submissions".private = false;
+  #   "submissions".privileged = true;
+  # };
+
   services.postfix.enableSubmission = true;
   services.postfix.submissionOptions = submissionOptions;
   services.postfix.enableSubmissions = true;
   services.postfix.submissionsOptions = submissionOptions;
 
-  systemd.services.postfix.after = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.postfix.partOf = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.postfix.serviceConfig = {
-    # run this behind the OVPN static VPN
-    NetworkNamespacePath = "/run/netns/ovpns";
-  };
+  systemd.services.postfix.unitConfig.RequiresMountsFor = [
+    "/var/spool/mail"  # spooky errors when postfix is run w/o this: `warning: connect #1 to subsystem private/proxymap: Connection refused`
+    "/var/lib/opendkim"
+  ];
+
+  # run these behind the OVPN static VPN
+  sane.netns.ovpns.services = [ "opendkim" "postfix" ];
 
 
   #### OPENDKIM
@@ -165,34 +189,37 @@ in
   # keeping this the same as the hostname seems simplest
   services.opendkim.selector = "mx";
 
-  systemd.services.opendkim.after = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.opendkim.partOf = [ "wireguard-wg-ovpns.service" ];
   systemd.services.opendkim.serviceConfig = {
-    # run this behind the OVPN static VPN
-    NetworkNamespacePath = "/run/netns/ovpns";
     # /run/opendkim/opendkim.sock needs to be rw by postfix
     UMask = lib.mkForce "0011";
   };
 
 
   #### OUTGOING MESSAGE REWRITING:
-  services.postfix.enableHeaderChecks = true;
-  services.postfix.headerChecks = [
-    # intercept gitea registration confirmations and manually screen them
-    {
-      # headerChecks are somehow ignorant of alias rules: have to redirect to a real user
-      action = "REDIRECT colin@uninsane.org";
-      pattern = "/^Subject: Please activate your account/";
-    }
-    # intercept Matrix registration confirmations
-    {
-      action = "REDIRECT colin@uninsane.org";
-      pattern = "/^Subject:.*Validate your email/";
-    }
-    # XXX postfix only supports performing ONE action per header.
-    # {
-    #   action = "REPLACE Subject: git application: Please activate your account";
-    #   pattern = "/^Subject:.*activate your account/";
-    # }
-  ];
+  # - `man 5 header_checks`
+  #   - <https://www.postfix.org/header_checks.5.html>
+  # - populates `/var/lib/postfix/conf/header_checks`
+  # XXX(2024-08-06): registration gating via email matches is AWFUL:
+  # 1. bypassed if the service offers localization.
+  # 2. if i try to forward the registration request, it may match the filter again and get sent back to my inbox.
+  # 3. header checks are possibly under-used in the ecosystem, and may break postfix config.
+  # services.postfix.enableHeaderChecks = true;
+  # services.postfix.headerChecks = [
+  #   # intercept gitea registration confirmations and manually screen them
+  #   {
+  #     # headerChecks are somehow ignorant of alias rules: have to redirect to a real user
+  #     action = "REDIRECT colin@uninsane.org";
+  #     pattern = "/^Subject: Please activate your account/";
+  #   }
+  #   # intercept Matrix registration confirmations
+  #   {
+  #     action = "REDIRECT colin@uninsane.org";
+  #     pattern = "/^Subject:.*Validate your email/";
+  #   }
+  #   # XXX postfix only supports performing ONE action per header.
+  #   # {
+  #   #   action = "REPLACE Subject: git application: Please activate your account";
+  #   #   pattern = "/^Subject:.*activate your account/";
+  #   # }
+  # ];
 }

hosts/by-name/servo/services/export/default.nix

@@ -10,7 +10,7 @@
   fileSystems."/var/export/media" = {
     # everything in here could be considered publicly readable (based on the viewer's legal jurisdiction)
     device = "/var/media";
-    options = [ "rbind" ];
+    options = [ "rbind" "nofail" ];
   };
   # fileSystems."/var/export/playground" = {
   #   device = config.fileSystems."/mnt/persist/ext".device;
@@ -34,20 +34,25 @@
   ];
 
   sane.fs."/var/export/README.md" = {
-    wantedBy = [ "nfs.service" "sftpgo.service" ];
     file.text = ''
       - media/         read-only:  Videos, Music, Books, etc
-      - playground/    read-write: use it to share files with other users of this server
+      - playground/    read-write*: use it to share files with other users of this server, inaccessible from the www
+                                    *if you can't write to it, make sure you're connected to the WiFi and not mobile.
     '';
   };
 
   sane.fs."/var/export/playground/README.md" = {
-    wantedBy = [ "nfs.service" "sftpgo.service" ];
     file.text = ''
-      this directory is intentionally read+write by anyone with access (i.e. on the LAN).
+      this directory is intentionally read+write by anyone with access.
       - share files
       - write poetry
       - be a friendly troll
     '';
   };
+
+  sane.fs."/var/export/.public_for_test/test" = {
+    file.text = ''
+      automated tests read this file to probe connectivity
+    '';
+  };
 }

hosts/by-name/servo/services/export/nfs.nix

@@ -15,6 +15,7 @@
 # - could maybe be done with some mount option?
 
 { config, lib, ... }:
+lib.mkIf false  #< TODO: remove nfs altogether! it's not exactly the most secure
 {
   services.nfs.server.enable = true;
 

hosts/by-name/servo/services/export/sftpgo/default.nix

@@ -9,9 +9,10 @@
 
 { config, lib, pkgs, sane-lib, ... }:
 let
-  external_auth_hook = pkgs.static-nix-shell.mkPython3Bin {
+  external_auth_hook = pkgs.static-nix-shell.mkPython3 {
     pname = "external_auth_hook";
     srcRoot = ./.;
+    pkgs = [ "python3.pkgs.passlib" ];
   };
   # Client initiates a FTP "control connection" on port 21.
   # - this handles the client -> server commands, and the server -> client status, but not the actual data
@@ -26,13 +27,12 @@ in
     "21" = {
       protocol = [ "tcp" ];
       visibleTo.lan = true;
-      # visibleTo.wan = true;
       description = "colin-FTP server";
     };
     "990" = {
       protocol = [ "tcp" ];
+      visibleTo.doof = true;
       visibleTo.lan = true;
-      visibleTo.wan = true;
       description = "colin-FTPS server";
     };
   } // (sane-lib.mapToAttrs
@@ -40,8 +40,8 @@ in
       name = builtins.toString port;
       value = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-FTP server data port range";
       };
     })
@@ -59,18 +59,8 @@ in
     enable = true;
     group = "export";
 
-    package = lib.warnIf (lib.versionOlder "2.5.6" pkgs.sftpgo.version) "sftpgo update: safe to use nixpkgs' sftpgo but keep my own `patches`" pkgs.buildGoModule {
-      inherit (pkgs.sftpgo) name ldflags nativeBuildInputs doCheck subPackages postInstall passthru meta;
-      version = "2.5.6-unstable-2024-04-18";
-      src = pkgs.fetchFromGitHub {
-        # need to use > 2.5.6 for sftpgo_safe_fileinfo.patch to apply
-        owner = "drakkan";
-        repo = "sftpgo";
-        rev = "950cf67e4c03a12c7e439802cabbb0b42d4ee5f5";
-        hash = "sha256-UfiFd9NK3DdZ1J+FPGZrM7r2mo9xlKi0dsSlLEinYXM=";
-      };
-      vendorHash = "sha256-n1/9A2em3BCtFX+132ualh4NQwkwewMxYIMOphJEamg=";
-      patches = (pkgs.sftpgo.patches or []) ++ [
+    package = pkgs.sftpgo.overrideAttrs (upstream: {
+      patches = (upstream.patches or []) ++ [
         # fix for compatibility with kodi:
         # ftp LIST operation returns entries over-the-wire like:
         # - dgrwxrwxr-x 1 ftp ftp            9 Apr  9 15:05 Videos
@@ -79,7 +69,7 @@ in
         # the full set of bits, from which i filter, is found here: <https://pkg.go.dev/io/fs#FileMode>
         ./safe_fileinfo.patch
       ];
-    };
+    });
 
     settings = {
       ftpd = {
@@ -90,12 +80,6 @@ in
             port = 21;
             debug = true;
           }
-          {
-            # binding this means any LAN client can connect (also WAN traffic forwarded from the gateway)
-            address = "10.78.79.51";
-            port = 21;
-            debug = true;
-          }
           {
             # binding this means any wireguard client can connect
             address = "10.0.10.5";
@@ -106,6 +90,26 @@ in
           {
             # binding this means any LAN client can connect (also WAN traffic forwarded from the gateway)
             address = "10.78.79.51";
+            port = 21;
+            debug = true;
+          }
+          {
+            # binding this means any LAN client can connect (also WAN traffic forwarded from the gateway)
+            address = "10.78.79.51";
+            port = 990;
+            debug = true;
+            tls_mode = 2;  # 2 = "implicit FTPS": client negotiates TLS before any FTP command.
+          }
+          {
+            # binding this means any doof client can connect (TLS only)
+            address = config.sane.netns.doof.veth.initns.ipv4;
+            port = 990;
+            debug = true;
+            tls_mode = 2;  # 2 = "implicit FTPS": client negotiates TLS before any FTP command.
+          }
+          {
+            # binding this means any LAN client can connect via `ftp.uninsane.org` (TLS only)
+            address = config.sane.netns.doof.wg.address.ipv4;
             port = 990;
             debug = true;
             tls_mode = 2;  # 2 = "implicit FTPS": client negotiates TLS before any FTP command.
@@ -126,7 +130,7 @@ in
         banner = ''
           Welcome, friends, to Colin's FTP server! Also available via NFS on the same host, but LAN-only.
 
-          Read-only access (LAN-restricted):
+          Read-only access (LAN clients see everything; WAN clients can only see /pub):
           Username: "anonymous"
           Password: "anonymous"
 
@@ -137,7 +141,7 @@ in
       };
       data_provider = {
         driver = "memory";
-        external_auth_hook = "${external_auth_hook}/bin/external_auth_hook";
+        external_auth_hook = lib.getExe external_auth_hook;
         # track_quota:
         # - 0: disable quota tracking
         # - 1: quota is updated on every upload/delete, even if user has no quota restriction
@@ -154,14 +158,15 @@ in
   ];
 
   systemd.services.sftpgo = {
-    after = [ "network-online.target" ];
+    after = [ "network-online.target" ];  #< so that it reliably binds to all interfaces/netns's?
     wants = [ "network-online.target" ];
-    serviceConfig = {
-      ReadWritePaths = [ "/var/export" ];
-
-      Restart = "always";
-      RestartSec = "20s";
-      UMask = lib.mkForce "0002";
-    };
+    unitConfig.RequiresMountsFor = [
+      "/var/export/media"
+      "/var/export/playground"
+    ];
+    serviceConfig.ReadWritePaths = [ "/var/export" ];
+    serviceConfig.Restart = "always";
+    serviceConfig.RestartSec = "20s";
+    serviceConfig.UMask = lib.mkForce "0002";
   };
 }

hosts/by-name/servo/services/export/sftpgo/external_auth_hook

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i python3 -p "python3.withPackages (ps: [  ])"
+#!nix-shell -i python3 -p python3 -p python3.pkgs.passlib
 # vim: set filetype=python :
 #
 # available environment variables:
@@ -37,14 +37,16 @@
 # - it seems (empirically) that a user can't cd above their home directory.
 #   though i don't have a reference for that in the docs.
 
-import crypt
 import json
 import os
+import passlib.hosts
 
 from hmac import compare_digest
 
 authFail = dict(username="")
 
+PERM_DENY = []
+PERM_LIST = [ "list" ]
 PERM_RO = [ "list", "download" ]
 PERM_RW = [
   # read-only:
@@ -67,8 +69,12 @@ TRUSTED_CREDS = [
   # /etc/shadow style creds.
   # mkpasswd -m sha-512
   # $<method>$<salt>$<hash>
-  "$6$Zq3c2u4ghUH4S6EP$pOuRt13sEKfX31OqPbbd1LuhS21C9MICMc94iRdTAgdAcJ9h95gQH/6Jf6Ie4Obb0oxQtojRJ1Pd/9QHOlFMW."  #< m. rocket boy
+  "$6$Zq3c2u4ghUH4S6EP$pOuRt13sEKfX31OqPbbd1LuhS21C9MICMc94iRdTAgdAcJ9h95gQH/6Jf6Ie4Obb0oxQtojRJ1Pd/9QHOlFMW.",  #< m. rocket boy
+  "$6$B0NLGNdCL51PNse1$46G.aA1ATWIv5v.jUsKf4F3NS7emV2jB2gkZ3MytZtMvw2pjniHmRl0fywRjKW9TuXTeK9T50v.H0f2BaQ4PT1",  #< v. telephony
 ]
+TRUSTED_VIEWING_OR_PLAYGROUND_CREDS = [
+  # "$6$iikDajz5b.YH1.on$tfSzzBEtX8IeDiJJXCasOTxRTd7cFDKXU6dhlWYVhK6xDeJhV2fh6bmm1WIHItjIth9Eh9zNgUB8xibMIWCm/."  # fedi (2024-08-27); music appreciation
+];
 
 def mkAuthOk(username: str, permissions: dict[str, list[str]]) -> dict:
   return dict(
@@ -110,12 +116,10 @@ def isLan(ip: str) -> bool:
 def isWireguard(ip: str) -> bool:
   return ip.startswith("10.0.10.")
 
-def isTrustedCred(password: str) -> bool:
-  for cred in TRUSTED_CREDS:
-    _, method, salt, hash_ = cred.split("$")
-    # assert method == "6", f"unrecognized crypt entry: {cred}"
-    if crypt.crypt(password, f"${method}${salt}") == cred:
-        return True
+def isTrustedCred(password: str, credlist: list[str] = TRUSTED_CREDS) -> bool:
+  for cred in credlist:
+    if passlib.hosts.linux_context.verify(password, cred):
+      return True
 
   return False
 
@@ -129,12 +133,30 @@ def getAuthResponse(ip: str, username: str, password: str) -> dict:
     return mkAuthOk(username, permissions = {
       "/": PERM_RW,
       "/playground": PERM_RW,
+      "/.public_for_test": PERM_RO,
+      "/media/Music": PERM_RO,  #< i am too picky about Music organization
+    })
+  if isTrustedCred(password, TRUSTED_VIEWING_OR_PLAYGROUND_CREDS) and username != "colin":
+    return mkAuthOk(username, permissions = {
+      # error prone, but... not the worst if i miss something
+      "/": PERM_LIST,
+      "/media/archive": PERM_DENY,
+      "/media/Books": PERM_RO,
+      "/media/collections": PERM_DENY,
+      "/media/games": PERM_RO,
+      "/media/Music": PERM_RO,
+      "/media/Pictures": PERM_RO,
+      "/media/torrents": PERM_DENY,
+      "/media/Videos": PERM_RO,
+      "/playground": PERM_RW,
+      "/.public_for_test": PERM_RO,
     })
   if isWireguard(ip):
     # allow any user from wireguard
     return mkAuthOk(username, permissions = {
       "/": PERM_RW,
       "/playground": PERM_RW,
+      "/.public_for_test": PERM_RO,
     })
   if isLan(ip):
     if username == "anonymous":
@@ -142,7 +164,19 @@ def getAuthResponse(ip: str, username: str, password: str) -> dict:
       return mkAuthOk("anonymous", permissions = {
         "/": PERM_RO,
         "/playground": PERM_RW,
+        "/.public_for_test": PERM_RO,
       })
+  if username == "anonymous":
+    # anonymous users from the www can have even more limited access.
+    # mostly because i need an easy way to test WAN connectivity :-)
+    return mkAuthOk("anonymous", permissions = {
+      # "/": PERM_DENY,
+      "/": PERM_LIST,  #< REQUIRED, even for lftp to list a subdir
+      "/media": PERM_DENY,
+      "/playground": PERM_DENY,
+      "/.public_for_test": PERM_RO,
+      # "/README.md": PERM_RO,  #< does not work
+    })
 
   return authFail
 

hosts/by-name/servo/services/freshrss.nix

@@ -10,6 +10,7 @@
 # ```
 
 { config, lib, pkgs, sane-lib, ... }:
+lib.mkIf false  #< 2024/07/04: i haven't actively used this for months
 {
   sops.secrets."freshrss_passwd" = {
     owner = config.users.users.freshrss.name;

hosts/by-name/servo/services/gerbera.nix

@@ -0,0 +1,38 @@
+# gerbera UPNP/media server
+# accessible from TVs on the LAN
+# unauthenticated admin and playback UI at http://servo:49152/
+#
+# supposedly does transcoding, but i poked at it for 10 minutes and couldn't get that working
+#
+# compatibility:
+# - LG TV: music: all working
+# - LG TV: videos: mixed
+{ lib, ... }:
+lib.mkIf false  #< XXX(2024-11-17): WORKS, but no better than any other service; slow to index and transcoding doesn't work
+{
+  sane.ports.ports."1900" = {
+    protocol = [ "udp" ];
+    visibleTo.lan = true;
+    description = "colin-upnp-for-gerbera";
+  };
+  sane.ports.ports."49152" = {
+    protocol = [ "tcp" "udp" ];  # TODO: is udp required?
+    visibleTo.lan = true;
+    description = "colin-gerbera-http";
+  };
+
+  sane.persist.sys.byStore.plaintext = [
+    # persist the index database, since it takes a good 30 minutes to scan the media collection
+    { user = "mediatomb"; group = "mediatomb"; mode = "0700"; path = "/var/lib/gerbera"; method = "bind"; }
+  ];
+
+  services.mediatomb.enable = true;
+  services.mediatomb.serverName = "servo";
+  services.mediatomb.transcoding = true;
+  services.mediatomb.mediaDirectories = [
+    { path = "/var/media/Music"; recursive = true; hidden-files = false; }
+    { path = "/var/media/Videos/Film"; recursive = true; hidden-files = false; }
+    { path = "/var/media/Videos/Shows"; recursive = true; hidden-files = false; }
+  ];
+  users.users.mediatomb.extraGroups = [ "media" ];
+}

hosts/by-name/servo/services/gitea.nix

@@ -1,22 +1,33 @@
 # config options: <https://docs.gitea.io/en-us/administration/config-cheat-sheet/>
+# TODO: service shouldn't run as `git` user, but as `gitea`
 { config, pkgs, lib, ... }:
 
 {
-  sane.persist.sys.byStore.plaintext = [
-    # TODO: mode? could be more granular
-    { user = "git"; group = "gitea"; path = "/var/lib/gitea"; method = "bind"; }
+  sane.persist.sys.byStore.private = [
+    { user = "git"; group = "gitea"; mode = "0750"; path = "/var/lib/gitea"; method = "bind"; }
   ];
+
+  sane.programs.gitea.enableFor.user.colin = true;  # for admin, and monitoring
+
   services.gitea.enable = true;
   services.gitea.user = "git";  # default is 'gitea'
-  services.gitea.database.type = "postgres";
-  services.gitea.database.user = "git";
   services.gitea.appName = "Perfectly Sane Git";
   # services.gitea.disableRegistration = true;
 
-  services.gitea.database.createDatabase = false;  #< silence warning which wants db user and name to be equal
-  # TODO: remove this after merge: <https://github.com/NixOS/nixpkgs/pull/268849>
+  services.gitea.database.createDatabase = false;  # can only createDatabase if user ("git") == dbname ("gitea")
+  services.gitea.database.type = "postgres";
+  services.gitea.database.user = "git";
+  # createDatabase=false means manually specify the connection; see: <https://github.com/NixOS/nixpkgs/pull/268849>
+  services.gitea.database.name = "gitea";
   services.gitea.database.socket = "/run/postgresql";  #< would have been set if createDatabase = true
 
+  services.postgresql.enable = true;
+  services.postgresql.ensureDatabases = [ "gitea" ];
+  services.postgresql.ensureUsers = [{
+    name = "git";
+    # ensureDBOwnership = true;  # not possible if db name ("gitea") != db username ("git"); one-time manual setup required to grant user ownership of the relevant db
+  }];
+
   # gitea doesn't create the git user
   users.users.git = {
     description = "Gitea Service";
@@ -38,28 +49,41 @@
       ROOT_URL = "https://git.uninsane.org/";
     };
     service = {
-      # timeout for email approval. 5760 = 4 days
-      ACTIVE_CODE_LIVE_MINUTES = 5760;
+      # timeout for email approval. 5760 = 4 days. 10080 = 7 days
+      ACTIVE_CODE_LIVE_MINUTES = 10080;
       # REGISTER_EMAIL_CONFIRM = false;
-      # REGISTER_MANUAL_CONFIRM = true;
-      REGISTER_EMAIL_CONFIRM = true;
-      # not sure what this notified on?
+      # REGISTER_EMAIL_CONFIRM = true;  #< override REGISTER_MANUAL_CONFIRM
+      REGISTER_MANUAL_CONFIRM = true;
+      # not sure what this notifies *on*...
       ENABLE_NOTIFY_MAIL = true;
       # defaults to image-based captcha.
       # also supports recaptcha (with custom URLs) or hCaptcha.
       ENABLE_CAPTCHA = true;
       NOREPLY_ADDRESS = "noreply.anonymous.git@uninsane.org";
+      EMAIL_DOMAIN_BLOCKLIST = lib.concatStringsSep ", " [
+        "*.claychoen.top"
+        "*.gemmasmith.co.uk"
+        "*.jenniferlawrence.uk"
+        "*.sarahconnor.co.uk"
+        "*.marymarshall.co.uk"
+      ];
+    };
+    session = {
+      COOKIE_SECURE = true;
+      # keep me logged in for 30 days
+      SESSION_LIFE_TIME = 60 * 60 * 24 * 30;
     };
-    session.COOKIE_SECURE = true;
     repository = {
       DEFAULT_BRANCH = "master";
+      ENABLE_PUSH_CREATE_USER = true;
+      ENABLE_PUSH_CREATE_ORG = true;
     };
     other = {
       SHOW_FOOTER_TEMPLATE_LOAD_TIME = false;
     };
     ui = {
-      # options: "auto", "gitea", "arc-green"
-      DEFAULT_THEME = "arc-green";
+      # options: "gitea-auto" (adapt to system theme), "gitea-dark", "gitea-light"
+      # DEFAULT_THEME = "gitea-auto";
       # cache frontend assets if true
       # USE_SERVICE_WORKER = true;
     };
@@ -68,9 +92,10 @@
       # alternative is to use nixos-level config:
       # services.gitea.mailerPasswordFile = ...
       ENABLED = true;
-      MAILER_TYPE = "sendmail";
       FROM = "notify.git@uninsane.org";
-      SENDMAIL_PATH = "${pkgs.postfix}/bin/sendmail";
+      PROTOCOL = "sendmail";
+      SENDMAIL_PATH = lib.getExe' pkgs.postfix "sendmail";
+      SENDMAIL_ARGS = "--";  # most "sendmail" programs take options, "--" will prevent an email address being interpreted as an option.
     };
     time = {
       # options: ANSIC, UnixDate, RubyDate, RFC822, RFC822Z, RFC850, RFC1123, RFC1123Z, RFC3339, RFC3339Nano, Kitchen, Stamp, StampMilli, StampMicro, StampNano
@@ -79,31 +104,75 @@
     };
   };
 
+  systemd.services.gitea.wants = [ "postgresql.service" ];
   systemd.services.gitea.serviceConfig = {
     # nix default is AF_UNIX AF_INET AF_INET6.
     # we need more protos for sendmail to work. i thought it only needed +AF_LOCAL, but that didn't work.
     RestrictAddressFamilies = lib.mkForce "~";
     # add maildrop to allow sendmail to work
-    ReadWritePaths = lib.mkForce [
+    ReadWritePaths = [
       "/var/lib/postfix/queue/maildrop"
-      "/var/lib/gitea"
     ];
+    # rate limit the restarts to prevent systemd from disabling it
+    RestartSec = 5;
+    RestartMaxDelaySec = 30;
+    StartLimitBurst = 120;
+    RestartSteps = 5;
+  };
+
+  # services.openssh.settings.UsePAM = true;  #< required for `git` user to authenticate
+
+  services.anubis.instances."git.uninsane.org" = {
+    settings.TARGET = "http://127.0.0.1:3000";
+    # allow IM clients/etc to show embeds/previews, else they just show "please verify you aren't a bot..."
+    botPolicy.openGraph.enabled = true;
   };
 
   # hosted git (web view and for `git <cmd>` use
   # TODO: enable publog?
-  services.nginx.virtualHosts."git.uninsane.org" = {
+  services.nginx.virtualHosts."git.uninsane.org" = let
+    # XXX(2025-07-24): gitea's still being crawled, even with robots.txt.
+    # the load is less than when Anthropic first started, but it's still pretty high (like 600%).
+    # place behind anubis to prevent AI crawlers from hogging my CPU (gitea is slow to render pages).
+    proxyPassHeavy = "http://unix:${config.services.anubis.instances."git.uninsane.org".settings.BIND}";
+    # but anubis breaks embeds, so only protect the expensive repos.
+    proxyPassLight = "http://127.0.0.1:3000";
+    proxyTo = proxy: root: {
+      proxyPass = proxy;
+      recommendedProxySettings = true;
+    };
+  in {
     forceSSL = true;  # gitea complains if served over a different protocol than its config file says
     enableACME = true;
     # inherit kTLS;
+    extraConfig = ''
+      client_max_body_size 100m;
+    '';
 
     locations."/" = {
-      proxyPass = "http://127.0.0.1:3000";
+      proxyPass = proxyPassLight;
+      recommendedProxySettings = true;
     };
+    # selectively proxy the heavyweight items through anubis.
+    # a typical interaction is:
+    # nginx:/colin/linux -> anubis:/colin/linux -> browser is served a loading page
+    # -> nginx:.within.website/x/cmd/anubis/api/pass-challenge?response=... -> anubis:.within.website/x/cmd/anubis/api/pass-challenge?response=... -> browser is forwarded to /colin/linux
+    # -> nginx:/colin/linux -> anubis:/colin/linux -> gitea:/colin/linux -> browser is served the actual content
+    locations."/.within.website/" = proxyTo proxyPassHeavy;
+    locations."/colin/linux" = proxyTo proxyPassHeavy;
+    locations."/colin/nixpkgs" = proxyTo proxyPassHeavy;
+    locations."/colin/opencellid-mirror" = proxyTo proxyPassHeavy;
+    locations."/colin/podcastindex-db-mirror" = proxyTo proxyPassHeavy;
+
+    # fuck you @anthropic
+    # locations."= /robots.txt".extraConfig = ''
+    #   return 200 "User-agent: *\nDisallow: /\n";
+    # '';
     # gitea serves all `raw` files as content-type: plain, but i'd like to serve them as their actual content type.
     # or at least, enough to make specific pages viewable (serving unoriginal content as arbitrary content type is dangerous).
     locations."~ ^/colin/phone-case-cq/raw/.*.html" = {
-      proxyPass = "http://127.0.0.1:3000";
+      proxyPass = proxyPassLight;
+      recommendedProxySettings = true;
       extraConfig = ''
         proxy_hide_header Content-Type;
         default_type text/html;
@@ -111,7 +180,8 @@
       '';
     };
     locations."~ ^/colin/phone-case-cq/raw/.*.js" = {
-      proxyPass = "http://127.0.0.1:3000";
+      proxyPass = proxyPassLight;
+      recommendedProxySettings = true;
       extraConfig = ''
         proxy_hide_header Content-Type;
         default_type text/html;
@@ -125,7 +195,7 @@
   sane.ports.ports."22" = {
     protocol = [ "tcp" ];
     visibleTo.lan = true;
-    visibleTo.wan = true;
+    visibleTo.doof = true;
     description = "colin-git@git.uninsane.org";
   };
 }

hosts/by-name/servo/services/goaccess.nix

@@ -1,4 +1,5 @@
-{ pkgs, ... }:
+{ lib, pkgs, ... }:
+lib.mkIf false  #< 2024/09/30: disabled because i haven't used it in several months
 {
   # based on <https://bytes.fyi/real-time-goaccess-reports-with-nginx/>
   # log-format setting can be derived with this tool if custom:
@@ -10,7 +11,7 @@
     description = "GoAccess server monitoring";
     serviceConfig = {
       ExecStart = ''
-        ${pkgs.goaccess}/bin/goaccess \
+        ${lib.getExe pkgs.goaccess} \
           -f /var/log/nginx/public.log \
           --log-format=VCOMBINED \
           --real-time-html \
@@ -22,7 +23,7 @@
           --port=7890 \
           -o /var/lib/goaccess/index.html
       '';
-      ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      ExecReload = "${lib.getExe' pkgs.coreutils "kill"} -HUP $MAINPID";
       Type = "simple";
       Restart = "on-failure";
       RestartSec = "10s";
@@ -55,6 +56,7 @@
 
     locations."/ws" = {
       proxyPass = "http://127.0.0.1:7890";
+      recommendedProxySettings = true;
       # XXX not sure how much of this is necessary
       extraConfig = ''
         proxy_http_version 1.1;

hosts/by-name/servo/services/hickory-dns.nix

@@ -0,0 +1,149 @@
+# TODO: split this file apart into smaller files to make it easier to understand
+{ config, lib, ... }:
+
+let
+  dyn-dns = config.sane.services.dyn-dns;
+  nativeAddrs = lib.mapAttrs (_name: builtins.head) config.sane.dns.zones."uninsane.org".inet.A;
+in
+{
+  sane.ports.ports."53" = {
+    protocol = [ "udp" "tcp" ];
+    visibleTo.lan = true;
+    # visibleTo.wan = true;
+    visibleTo.ovpns = true;
+    visibleTo.doof = true;
+    description = "colin-dns-hosting";
+  };
+
+  sane.dns.zones."uninsane.org".TTL = 900;
+
+  # SOA record structure: <https://en.wikipedia.org/wiki/SOA_record#Structure>
+  # SOA MNAME RNAME (... rest)
+  # MNAME = Master name server for this zone. this is where update requests should be sent.
+  # RNAME = admin contact (encoded email address)
+  # Serial = YYYYMMDDNN, where N is incremented every time this file changes, to trigger secondary NS to re-fetch it.
+  # Refresh = how frequently secondary NS should query master
+  # Retry = how long secondary NS should wait until re-querying master after a failure (must be < Refresh)
+  # Expire = how long secondary NS should continue to reply to queries after master fails (> Refresh + Retry)
+  sane.dns.zones."uninsane.org".inet = {
+    SOA."@" = ''
+      ns1.uninsane.org. admin-dns.uninsane.org. (
+                                      2023092101 ; Serial
+                                      4h         ; Refresh
+                                      30m        ; Retry
+                                      7d         ; Expire
+                                      5m)        ; Negative response TTL
+    '';
+    TXT."rev" = "2023092101";
+
+    CNAME."native" = "%CNAMENATIVE%";
+    A."@" =      "%ANATIVE%";
+    A."servo.wan" = "%AWAN%";
+    A."servo.doof" = "%ADOOF%";
+    A."servo.lan" = config.sane.hosts.by-name."servo".lan-ip;
+    A."servo.hn" = config.sane.hosts.by-name."servo".wg-home.ip;
+
+    # XXX NS records must also not be CNAME
+    # it's best that we keep this identical, or a superset of, what org. lists as our NS.
+    # so, org. can specify ns2/ns3 as being to the VPN, with no mention of ns1. we provide ns1 here.
+    A."ns1" =    "%ANATIVE%";
+    A."ns2" =    "%ADOOF%";
+    A."ovpns" =  "%AOVPNS%";
+    NS."@" = [
+      "ns1.uninsane.org."
+      "ns2.uninsane.org."
+    ];
+  };
+
+  services.hickory-dns.settings.zones = builtins.attrNames config.sane.dns.zones;
+
+  networking.nat.enable = true;  #< TODO: try removing this?
+  # networking.nat.extraCommands = ''
+  #   # redirect incoming DNS requests from LAN addresses
+  #   #   to the LAN-specialized DNS service
+  #   # N.B.: use the `nixos-*` chains instead of e.g. PREROUTING
+  #   #   because they get cleanly reset across activations or `systemctl restart firewall`
+  #   #   instead of accumulating cruft
+  #   iptables -t nat -A nixos-nat-pre -p udp --dport 53 \
+  #     -m iprange --src-range 10.78.76.0-10.78.79.255 \
+  #     -j DNAT --to-destination :1053
+  #   iptables -t nat -A nixos-nat-pre -p tcp --dport 53 \
+  #     -m iprange --src-range 10.78.76.0-10.78.79.255 \
+  #     -j DNAT --to-destination :1053
+  # '';
+  # sane.ports.ports."1053" = {
+  #   # because the NAT above redirects in nixos-nat-pre, LAN requests behave as though they arrived on the external interface at the redirected port.
+  #   # TODO: try nixos-nat-post instead?
+  #   # TODO: or, don't NAT from port 53 -> port 1053, but rather nat from LAN addr to a loopback addr.
+  #   # - this is complicated in that loopback is a different interface than eth0, so rewriting the destination address would cause the packets to just be dropped by the interface
+  #   protocol = [ "udp" "tcp" ];
+  #   visibleTo.lan = true;
+  #   description = "colin-redirected-dns-for-lan-namespace";
+  # };
+
+
+  sane.services.hickory-dns.enable = true;
+  sane.services.hickory-dns.instances = let
+    mkSubstitutions = flavor: {
+      "%ADOOF%" = config.sane.netns.doof.wg.address.ipv4;
+      "%ANATIVE%" = nativeAddrs."servo.${flavor}";
+      "%AOVPNS%" = config.sane.netns.ovpns.wg.address.ipv4;
+      "%AWAN%" = "$(cat '${dyn-dns.ipPath}')";
+      "%CNAMENATIVE%" = "servo.${flavor}";
+    };
+  in
+  {
+    doof = {
+      substitutions = mkSubstitutions "doof";
+      listenAddrsIpv4 = [
+        config.sane.netns.doof.veth.initns.ipv4
+        config.sane.netns.doof.wg.address.ipv4
+        nativeAddrs."servo.lan"
+        # config.sane.netns.ovpns.veth.initns.ipv4
+      ];
+    };
+    # hn = {
+    #   substitutions = mkSubstitutions "hn";
+    #   listenAddrsIpv4 = [ nativeAddrs."servo.hn" ];
+    #   enableRecursiveResolver = true;  #< allow wireguard clients to use this as their DNS resolver
+    #   # extraConfig = {
+    #   #   zones = [
+    #   #     {
+    #   #       # forward the root zone to the local DNS resolver
+    #   #       # to allow wireguard clients to use this as their DNS resolver
+    #   #       zone = ".";
+    #   #       zone_type = "Forward";
+    #   #       stores = {
+    #   #         type = "forward";
+    #   #         name_servers = [
+    #   #           {
+    #   #             socket_addr = "127.0.0.53:53";
+    #   #             protocol = "udp";
+    #   #             trust_nx_responses = true;
+    #   #           }
+    #   #         ];
+    #   #       };
+    #   #     }
+    #   #   ];
+    #   # };
+    # };
+    # lan = {
+    #   substitutions = mkSubstitutions "lan";
+    #   listenAddrsIpv4 = [ nativeAddrs."servo.lan" ];
+    #   # port = 1053;
+    # };
+    # wan = {
+    #   substitutions = mkSubstitutions "wan";
+    #   listenAddrsIpv4 = [
+    #     nativeAddrs."servo.lan"
+    #   ];
+    # };
+  };
+
+  systemd.services.hickory-dns-doof.after = [
+    # service will fail to bind the veth, otherwise
+    "netns-doof-veth.service"
+  ];
+
+  sane.services.dyn-dns.restartOnChange = lib.map (c: "${c.service}.service") (builtins.attrValues config.sane.services.hickory-dns.instances);
+}

hosts/by-name/servo/services/ipfs.nix

@@ -10,7 +10,7 @@
 
 lib.mkIf false # i don't actively use ipfs anymore
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     # TODO: mode? could be more granular
     { user = "261"; group = "261"; path = "/var/lib/ipfs"; method = "bind"; }
   ];
@@ -27,6 +27,7 @@ lib.mkIf false # i don't actively use ipfs anymore
 
     locations."/" = {
       proxyPass = "http://127.0.0.1:8080";
+      recommendedProxySettings = true;
       extraConfig = ''
         proxy_set_header Host $host;
         proxy_set_header X-Ipfs-Gateway-Prefix "";

hosts/by-name/servo/services/jackett.nix

@@ -1,35 +0,0 @@
-{ lib, pkgs, ... }:
-
-{
-  sane.persist.sys.byStore.plaintext = [
-    # TODO: mode? we only need this to save Indexer creds ==> migrate to config?
-    { user = "root"; group = "root"; path = "/var/lib/jackett"; method = "bind"; }
-  ];
-  services.jackett.enable = true;
-
-  systemd.services.jackett.after = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.jackett.partOf = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.jackett.serviceConfig = {
-    # run this behind the OVPN static VPN
-    NetworkNamespacePath = "/run/netns/ovpns";
-    ExecStartPre = [ "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect 185.157.162.178" ];  # abort if public IP is not as expected
-
-    # patch jackett to listen on the public interfaces
-    # ExecStart = lib.mkForce "${pkgs.jackett}/bin/Jackett --NoUpdates --DataFolder /var/lib/jackett/.config/Jackett --ListenPublic";
-  };
-
-  # jackett torrent search
-  services.nginx.virtualHosts."jackett.uninsane.org" = {
-    forceSSL = true;
-    enableACME = true;
-    # inherit kTLS;
-    locations."/" = {
-      # proxyPass = "http://ovpns.uninsane.org:9117";
-      proxyPass = "http://10.0.1.6:9117";
-      recommendedProxySettings = true;
-    };
-  };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."jackett" = "native";
-}
-

hosts/by-name/servo/services/jackett/default.nix

@@ -0,0 +1,69 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.jackett;
+in
+{
+  sane.persist.sys.byStore.private = [
+    # TODO: mode? we only need this to save Indexer creds ==> migrate to config?
+    { user = "jackett"; group = "jackett"; path = "/var/lib/jackett"; method = "bind"; }
+  ];
+  services.jackett.enable = true;
+
+  # run this behind the OVPN static VPN
+  sane.netns.ovpns.services = [ "jackett" ];
+  systemd.services.jackett = {
+    serviceConfig.ExecStartPre = [
+      # abort if public IP is not as expected
+      "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect ${config.sane.netns.ovpns.wg.address.ipv4}"
+    ];
+    # patch in `--ListenPublic` so that it's reachable from the netns veth.
+    # this also makes it reachable from the VPN pub address. oh well.
+    serviceConfig.ExecStart = lib.mkForce "${lib.getExe' cfg.package "Jackett"} --ListenPublic --NoUpdates --DataFolder '${cfg.dataDir}'";
+    serviceConfig.RestartSec = "30s";
+
+    # hardening (systemd-analyze security jackett)
+    # TODO: upstream into nixpkgs
+    serviceConfig.StateDirectory = "jackett";
+    serviceConfig.LockPersonality = true;
+    serviceConfig.NoNewPrivileges = true;
+    # serviceConfig.MemoryDenyWriteExecute = true;  #< Failed to create CoreCLR, HRESULT: 0x80004005
+    serviceConfig.PrivateDevices = true;
+    serviceConfig.PrivateMounts = true;
+    serviceConfig.PrivateTmp = true;
+    serviceConfig.PrivateUsers = true;
+    serviceConfig.ProcSubset = "pid";
+    serviceConfig.ProtectClock = true;
+    serviceConfig.ProtectControlGroups = true;
+    serviceConfig.ProtectHome = true;
+    serviceConfig.ProtectHostname = true;
+    serviceConfig.ProtectKernelLogs = true;
+    serviceConfig.ProtectKernelModules = true;
+    serviceConfig.ProtectKernelTunables = true;
+    serviceConfig.ProtectProc = "invisible";
+    serviceConfig.ProtectSystem = "strict";
+    serviceConfig.RemoveIPC = true;
+    serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+    serviceConfig.RestrictNamespaces = true;
+    serviceConfig.RestrictSUIDSGID = true;
+    serviceConfig.SystemCallArchitectures = "native";
+    serviceConfig.SystemCallFilter = [ "@system-service" "~@privileged" ];
+  };
+
+  # jackett torrent search
+  services.nginx.virtualHosts."jackett.uninsane.org" = {
+    forceSSL = true;
+    enableACME = true;
+    # inherit kTLS;
+    locations."/" = {
+      proxyPass = "http://${config.sane.netns.ovpns.veth.netns.ipv4}:9117";
+      recommendedProxySettings = true;
+    };
+    locations."= /robots.txt".extraConfig = ''
+      return 200 "User-agent: *\nDisallow: /\n";
+    '';
+  };
+
+  sane.dns.zones."uninsane.org".inet.CNAME."jackett" = "native";
+}
+

hosts/by-name/servo/services/jellyfin.nix

@@ -1,127 +0,0 @@
-# configuration options (today i don't store my config in nix):
-#
-# - jellyfin-web can be statically configured (result/share/jellyfin-web/config.json)
-#   - <https://jellyfin.org/docs/general/clients/web-config>
-#   - configure server list, plugins, "menuLinks", colors
-#
-# - jellfyin server is configured in /var/lib/jellfin/
-#   - root/default/<LibraryType>/
-#     - <LibraryName>.mblink: contains the directory name where this library lives
-#     - options.xml: contains preferences which were defined in the web UI during import
-#       - e.g. `EnablePhotos`, `EnableChapterImageExtraction`, etc.
-#   - config/encoding.xml: transcoder settings
-#   - config/system.xml: misc preferences like log file duration, audiobook resume settings, etc.
-#   - data/jellyfin.db: maybe account definitions? internal state?
-
-{ config, lib, ... }:
-
-{
-  # https://jellyfin.org/docs/general/networking/index.html
-  sane.ports.ports."1900" = {
-    protocol = [ "udp" ];
-    visibleTo.lan = true;
-    description = "colin-upnp-for-jellyfin";
-  };
-  sane.ports.ports."7359" = {
-    protocol = [ "udp" ];
-    visibleTo.lan = true;
-    description = "colin-jellyfin-specific-client-discovery";
-    # ^ not sure if this is necessary: copied this port from nixos jellyfin.openFirewall
-  };
-  # not sure if 8096/8920 get used either:
-  sane.ports.ports."8096" = {
-    protocol = [ "tcp" ];
-    visibleTo.lan = true;
-    description = "colin-jellyfin-http-lan";
-  };
-  sane.ports.ports."8920" = {
-    protocol = [ "tcp" ];
-    visibleTo.lan = true;
-    description = "colin-jellyfin-https-lan";
-  };
-
-  sane.persist.sys.byStore.plaintext = [
-    { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin"; method = "bind"; }
-  ];
-  sane.fs."/var/lib/jellyfin/config/logging.json" = {
-    # "Emby.Dlna" logging: <https://jellyfin.org/docs/general/networking/dlna>
-    symlink.text = ''
-      {
-        "Serilog": {
-          "MinimumLevel": {
-            "Default": "Information",
-            "Override": {
-              "Microsoft": "Warning",
-              "System": "Warning",
-              "Emby.Dlna": "Debug",
-              "Emby.Dlna.Eventing": "Debug"
-            }
-          },
-          "WriteTo": [
-            {
-              "Name": "Console",
-              "Args": {
-                "outputTemplate": "[{Timestamp:HH:mm:ss}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message:lj}{NewLine}{Exception}"
-              }
-            }
-          ],
-          "Enrich": [ "FromLogContext", "WithThreadId" ]
-        }
-      }
-    '';
-    wantedBeforeBy = [ "jellyfin.service" ];
-  };
-
-  # Jellyfin multimedia server
-  # this is mostly taken from the official jellfin.org docs
-  services.nginx.virtualHosts."jelly.uninsane.org" = {
-    forceSSL = true;
-    enableACME = true;
-    # inherit kTLS;
-
-    locations."/" = {
-      proxyPass = "http://127.0.0.1:8096";
-      extraConfig = ''
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Protocol $scheme;
-        proxy_set_header X-Forwarded-Host $http_host;
-
-        # Disable buffering when the nginx proxy gets very resource heavy upon streaming
-        proxy_buffering off;
-      '';
-    };
-    # locations."/web/" = {
-    #   proxyPass = "http://127.0.0.1:8096/web/index.html";
-    #   extraConfig = ''
-    #     proxy_set_header Host $host;
-    #     proxy_set_header X-Real-IP $remote_addr;
-    #     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    #     proxy_set_header X-Forwarded-Proto $scheme;
-    #     proxy_set_header X-Forwarded-Protocol $scheme;
-    #     proxy_set_header X-Forwarded-Host $http_host;
-    #   '';
-    # };
-    locations."/socket" = {
-      proxyPass = "http://127.0.0.1:8096";
-      extraConfig = ''
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Protocol $scheme;
-        proxy_set_header X-Forwarded-Host $http_host;
-      '';
-    };
-  };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."jelly" = "native";
-
-  services.jellyfin.enable = true;
-}

hosts/by-name/servo/services/jellyfin/default.nix

@@ -0,0 +1,173 @@
+# configuration options (today only a *subset* of the config is done in nix)
+# - jellyfin-web can be statically configured (result/share/jellyfin-web/config.json)
+#   - <https://jellyfin.org/docs/general/clients/web-config>
+#   - configure server list, plugins, "menuLinks", colors
+#
+# - jellfyin server is configured in /var/lib/jellfin/
+#   - root/default/<LibraryType>/
+#     - <LibraryName>.mblink: contains the directory name where this library lives
+#     - options.xml: contains preferences which were defined in the web UI during import
+#       - e.g. `EnablePhotos`, `EnableChapterImageExtraction`, etc.
+#   - config/encoding.xml: transcoder settings
+#   - config/system.xml: misc preferences like log file duration, audiobook resume settings, etc.
+#   - data/jellyfin.db: maybe account definitions? internal state?
+#
+# N.B.: default install DOES NOT SUPPORT DLNA out of the box.
+#       one must install it as a "plugin", which can be done through the UI.
+{ config, lib, ... }:
+
+# lib.mkIf false  #< XXX(2024-11-17): disabled because it hasn't been working for months; web UI hangs on load, TVs see no files
+{
+  config = lib.mkIf (config.sane.maxBuildCost >= 2) {
+    # https://jellyfin.org/docs/general/networking/index.html
+    sane.ports.ports."1900" = {
+      protocol = [ "udp" ];
+      visibleTo.lan = true;
+      description = "colin-upnp-for-jellyfin";
+    };
+    sane.ports.ports."7359" = {
+      protocol = [ "udp" ];
+      visibleTo.lan = true;
+      description = "colin-jellyfin-specific-client-discovery";
+      # ^ not sure if this is necessary: copied this port from nixos jellyfin.openFirewall
+    };
+    # not sure if 8096/8920 get used either:
+    sane.ports.ports."8096" = {
+      protocol = [ "tcp" ];
+      visibleTo.lan = true;
+      description = "colin-jellyfin-http-lan";
+    };
+    sane.ports.ports."8920" = {
+      protocol = [ "tcp" ];
+      visibleTo.lan = true;
+      description = "colin-jellyfin-https-lan";
+    };
+
+    sane.persist.sys.byStore.plaintext = [
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/data"; method = "bind"; }
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/metadata"; method = "bind"; }
+      # TODO: ship plugins statically, via nix. that'll be less fragile
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/plugins/DLNA_5.0.0.0"; method = "bind"; }
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/root"; method = "bind"; }
+    ];
+    sane.persist.sys.byStore.ephemeral = [
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/log"; method = "bind"; }
+      { user = "jellyfin"; group = "jellyfin"; mode = "0700"; path = "/var/lib/jellyfin/transcodes"; method = "bind"; }
+    ];
+
+    services.jellyfin.enable = true;
+    users.users.jellyfin.extraGroups = [ "media" ];
+
+    sane.fs."/var/lib/jellyfin".dir.acl = {
+      user = "jellyfin";
+      group = "jellyfin";
+      mode = "0700";
+    };
+
+    # `"Jellyfin.Plugin.Dlna": "Debug"` logging: <https://jellyfin.org/docs/general/networking/dlna>
+    # TODO: switch Dlna back to 'Information' once satisfied with stability
+    sane.fs."/var/lib/jellyfin/config/logging.json".symlink.text = ''
+      {
+        "Serilog": {
+          "MinimumLevel": {
+            "Default": "Information",
+            "Override": {
+              "Microsoft": "Warning",
+              "System": "Warning",
+              "Jellyfin.Plugin.Dlna": "Debug"
+            }
+          },
+          "WriteTo": [
+            {
+              "Name": "Console",
+              "Args": {
+                "outputTemplate": "[{Timestamp:HH:mm:ss}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message:lj}{NewLine}{Exception}"
+              }
+            }
+          ],
+          "Enrich": [ "FromLogContext", "WithThreadId" ]
+        }
+      }
+    '';
+
+    sane.fs."/var/lib/jellyfin/config/network.xml".file.text = ''
+      <?xml version="1.0" encoding="utf-8"?>
+      <NetworkConfiguration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+        <BaseUrl />
+        <EnableHttps>false</EnableHttps>
+        <RequireHttps>false</RequireHttps>
+        <InternalHttpPort>8096</InternalHttpPort>
+        <InternalHttpsPort>8920</InternalHttpsPort>
+        <PublicHttpPort>8096</PublicHttpPort>
+        <PublicHttpsPort>8920</PublicHttpsPort>
+        <AutoDiscovery>true</AutoDiscovery>
+        <EnableUPnP>false</EnableUPnP>
+        <EnableIPv4>true</EnableIPv4>
+        <EnableIPv6>false</EnableIPv6>
+        <EnableRemoteAccess>true</EnableRemoteAccess>
+        <LocalNetworkSubnets>
+          <string>10.78.76.0/22</string>
+        </LocalNetworkSubnets>
+        <KnownProxies>
+          <string>127.0.0.1</string>
+          <string>localhost</string>
+          <string>10.78.79.1</string>
+        </KnownProxies>
+        <IgnoreVirtualInterfaces>false</IgnoreVirtualInterfaces>
+        <VirtualInterfaceNames />
+        <EnablePublishedServerUriByRequest>false</EnablePublishedServerUriByRequest>
+        <PublishedServerUriBySubnet />
+        <RemoteIPFilter />
+        <IsRemoteIPFilterBlacklist>false</IsRemoteIPFilterBlacklist>
+      </NetworkConfiguration>
+    '';
+
+    # guest user id is `5ad194d60dca41de84b332950ffc4308`
+    sane.fs."/var/lib/jellyfin/plugins/configurations/Jellyfin.Plugin.Dlna.xml".file.text = ''
+      <?xml version="1.0" encoding="utf-8"?>
+      <DlnaPluginConfiguration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+        <EnablePlayTo>true</EnablePlayTo>
+        <ClientDiscoveryIntervalSeconds>60</ClientDiscoveryIntervalSeconds>
+        <BlastAliveMessages>true</BlastAliveMessages>
+        <AliveMessageIntervalSeconds>180</AliveMessageIntervalSeconds>
+        <SendOnlyMatchedHost>true</SendOnlyMatchedHost>
+        <DefaultUserId>5ad194d6-0dca-41de-84b3-32950ffc4308</DefaultUserId>
+      </DlnaPluginConfiguration>
+    '';
+
+    # fix LG TV to play more files.
+    # there are certain files for which it only supports Direct Play (not even "Direct Stream" -- but "Direct Play").
+    # this isn't a 100% fix: patching the profile allows e.g. Azumanga Daioh to play,
+    # but A Place Further Than the Universe still fails as before.
+    #
+    # profile is based on upstream: <https://github.com/jellyfin/jellyfin-plugin-dlna>
+    sane.fs."/var/lib/jellyfin/plugins/DLNA_5.0.0.0/profiles/LG Smart TV.xml".symlink.target = ./dlna/user/LG_Smart_TV.xml;
+    # XXX(2024-11-17): old method, but the file referenced seems not to be used and setting just it causes failures:
+    # > [DBG] Jellyfin.Plugin.Dlna.ContentDirectory.ContentDirectoryService: Not eligible for DirectPlay due to unsupported subtitles
+    # sane.fs."/var/lib/jellyfin/plugins/configurations/dlna/user/LG Smart TV.xml".symlink.target = ./dlna/user/LG_Smart_TV.xml;
+
+    systemd.services.jellyfin.unitConfig.RequiresMountsFor = [
+      "/var/media"
+    ];
+
+    # Jellyfin multimedia server
+    # this is mostly taken from the official jellfin.org docs
+    services.nginx.virtualHosts."jelly.uninsane.org" = {
+      forceSSL = true;
+      enableACME = true;
+      # inherit kTLS;
+
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:8096";
+        proxyWebsockets = true;
+        recommendedProxySettings = true;
+        # extraConfig = ''
+        #   # Disable buffering when the nginx proxy gets very resource heavy upon streaming
+        #   proxy_buffering off;
+        # '';
+      };
+    };
+
+    sane.dns.zones."uninsane.org".inet.CNAME."jelly" = "native";
+  };
+}

hosts/by-name/servo/services/jellyfin/dlna/user/LG_Smart_TV.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<Profile xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+  <Name>LG Smart TV</Name>
+  <Identification>
+    <ModelName>LG TV</ModelName>
+    <Headers />
+  </Identification>
+  <Manufacturer>Jellyfin</Manufacturer>
+  <ManufacturerUrl>https://github.com/jellyfin/jellyfin</ManufacturerUrl>
+  <ModelName>Jellyfin Server</ModelName>
+  <ModelDescription>UPnP/AV 1.0 Compliant Media Server</ModelDescription>
+  <ModelNumber>01</ModelNumber>
+  <ModelUrl>https://github.com/jellyfin/jellyfin</ModelUrl>
+  <EnableAlbumArtInDidl>false</EnableAlbumArtInDidl>
+  <EnableSingleAlbumArtLimit>false</EnableSingleAlbumArtLimit>
+  <EnableSingleSubtitleLimit>false</EnableSingleSubtitleLimit>
+  <SupportedMediaTypes>Audio,Photo,Video</SupportedMediaTypes>
+  <AlbumArtPn>JPEG_SM</AlbumArtPn>
+  <MaxAlbumArtWidth>480</MaxAlbumArtWidth>
+  <MaxAlbumArtHeight>480</MaxAlbumArtHeight>
+  <MaxIconWidth>48</MaxIconWidth>
+  <MaxIconHeight>48</MaxIconHeight>
+  <MaxStreamingBitrate>140000000</MaxStreamingBitrate>
+  <MaxStaticBitrate>140000000</MaxStaticBitrate>
+  <MusicStreamingTranscodingBitrate>192000</MusicStreamingTranscodingBitrate>
+  <MaxStaticMusicBitrate xsi:nil="true" />
+  <ProtocolInfo>http-get:*:video/mpeg:*,http-get:*:video/mp4:*,http-get:*:video/vnd.dlna.mpeg-tts:*,http-get:*:video/avi:*,http-get:*:video/x-matroska:*,http-get:*:video/x-ms-wmv:*,http-get:*:video/wtv:*,http-get:*:audio/mpeg:*,http-get:*:audio/mp3:*,http-get:*:audio/mp4:*,http-get:*:audio/x-ms-wma:*,http-get:*:audio/wav:*,http-get:*:audio/L16:*,http-get:*:image/jpeg:*,http-get:*:image/png:*,http-get:*:image/gif:*,http-get:*:image/tiff:*</ProtocolInfo>
+  <TimelineOffsetSeconds>10</TimelineOffsetSeconds>
+  <RequiresPlainVideoItems>false</RequiresPlainVideoItems>
+  <RequiresPlainFolders>false</RequiresPlainFolders>
+  <EnableMSMediaReceiverRegistrar>false</EnableMSMediaReceiverRegistrar>
+  <IgnoreTranscodeByteRangeRequests>false</IgnoreTranscodeByteRangeRequests>
+  <XmlRootAttributes />
+  <DirectPlayProfiles>
+    <DirectPlayProfile container="ts,mpegts,avi,mkv,m2ts" audioCodec="aac,ac3,eac3,mp3,dca,dts" videoCodec="h264,hevc" type="Video" />
+    <DirectPlayProfile container="mp4,m4v" audioCodec="aac,ac3,eac3,mp3,dca,dts" videoCodec="h264,mpeg4,hevc" type="Video" />
+    <DirectPlayProfile container="mp3" type="Audio" />
+    <DirectPlayProfile container="jpeg" type="Photo" />
+    <DirectPlayProfile container="" audioCodec="" videoCodec="" type="Video" />
+  </DirectPlayProfiles>
+  <TranscodingProfiles>
+    <TranscodingProfile container="mp3" type="Audio" audioCodec="mp3" estimateContentLength="false" enableMpegtsM2TsMode="false" transcodeSeekInfo="Auto" copyTimestamps="false" context="Streaming" enableSubtitlesInManifest="false" minSegments="0" segmentLength="0" breakOnNonKeyFrames="false" />
+    <TranscodingProfile container="ts" type="Video" videoCodec="h264" audioCodec="ac3,aac,mp3" estimateContentLength="false" enableMpegtsM2TsMode="false" transcodeSeekInfo="Auto" copyTimestamps="false" context="Streaming" enableSubtitlesInManifest="false" minSegments="0" segmentLength="0" breakOnNonKeyFrames="false" />
+    <TranscodingProfile container="jpeg" type="Photo" estimateContentLength="false" enableMpegtsM2TsMode="false" transcodeSeekInfo="Auto" copyTimestamps="false" context="Streaming" enableSubtitlesInManifest="false" minSegments="0" segmentLength="0" breakOnNonKeyFrames="false" />
+  </TranscodingProfiles>
+  <ContainerProfiles>
+    <ContainerProfile type="Photo">
+      <Conditions>
+        <ProfileCondition condition="LessThanEqual" property="Width" value="1920" isRequired="true" />
+        <ProfileCondition condition="LessThanEqual" property="Height" value="1080" isRequired="true" />
+      </Conditions>
+    </ContainerProfile>
+  </ContainerProfiles>
+  <CodecProfiles>
+    <CodecProfile type="Video" codec="mpeg4">
+      <Conditions>
+        <ProfileCondition condition="LessThanEqual" property="Width" value="1920" isRequired="true" />
+        <ProfileCondition condition="LessThanEqual" property="Height" value="1080" isRequired="true" />
+        <ProfileCondition condition="LessThanEqual" property="VideoFramerate" value="30" isRequired="true" />
+      </Conditions>
+      <ApplyConditions />
+    </CodecProfile>
+    <CodecProfile type="Video" codec="h264">
+      <Conditions>
+        <ProfileCondition condition="LessThanEqual" property="Width" value="1920" isRequired="true" />
+        <ProfileCondition condition="LessThanEqual" property="Height" value="1080" isRequired="true" />
+        <ProfileCondition condition="LessThanEqual" property="VideoLevel" value="41" isRequired="true" />
+      </Conditions>
+      <ApplyConditions />
+    </CodecProfile>
+    <CodecProfile type="VideoAudio" codec="ac3,eac3,aac,mp3">
+      <Conditions>
+        <ProfileCondition condition="LessThanEqual" property="AudioChannels" value="6" isRequired="true" />
+      </Conditions>
+      <ApplyConditions />
+    </CodecProfile>
+  </CodecProfiles>
+  <ResponseProfiles>
+    <ResponseProfile container="m4v" type="Video" mimeType="video/mp4">
+      <Conditions />
+    </ResponseProfile>
+    <ResponseProfile container="ts,mpegts" type="Video" mimeType="video/mpeg">
+      <Conditions />
+    </ResponseProfile>
+  </ResponseProfiles>
+  <SubtitleProfiles>
+    <SubtitleProfile format="srt" method="Embed" />
+    <SubtitleProfile format="srt" method="External" />
+  </SubtitleProfiles>
+</Profile>

hosts/by-name/servo/services/kiwix-serve.nix

@@ -1,27 +1,42 @@
-# how to update wikipedia snapshot:
-# - browse for later snapshots:
-#   - <https://mirror.accum.se/mirror/wikimedia.org/other/kiwix/zim/wikipedia>
-# - DL directly, or via rsync (resumable):
-#   - `rsync --progress --append-verify rsync://mirror.accum.se/mirror/wikimedia.org/other/kiwix/zim/wikipedia/wikipedia_en_all_maxi_2022-05.zim .`
-
-{ ... }:
+{ config, lib, pkgs, ... }:
 {
-  sane.persist.sys.byStore.ext = [
-    { user = "colin"; group = "users"; path = "/var/lib/kiwix"; method = "bind"; }
-  ];
+  config = lib.mkIf (config.sane.maxBuildCost >= 3) {
+    sane.services.kiwix-serve = {
+      enable = true;
+      port = 8013;
+      zimPaths = with pkgs.zimPackages; [
+        alpinelinux_en_all_maxi.zimPath
+        archlinux_en_all_maxi.zimPath
+        bitcoin_en_all_maxi.zimPath
+        devdocs_en_nix.zimPath
+        gentoo_en_all_maxi.zimPath
+        # khanacademy_en_all.zimPath  #< TODO: enable
+        openstreetmap-wiki_en_all_maxi.zimPath
+        psychonautwiki_en_all_maxi.zimPath
+        rationalwiki_en_all_maxi.zimPath
+        # wikipedia_en_100.zimPath
+        wikipedia_en_all_maxi.zimPath
+        # wikipedia_en_all_mini.zimPath
+        zimgit-food-preparation_en.zimPath
+        zimgit-medicine_en.zimPath
+        zimgit-post-disaster_en.zimPath
+        zimgit-water_en.zimPath
+      ];
+    };
 
-  sane.services.kiwix-serve = {
-    enable = true;
-    port = 8013;
-    zimPaths = [ "/var/lib/kiwix/wikipedia_en_all_maxi_2023-11.zim" ];
+    services.nginx.virtualHosts."w.uninsane.org" = {
+      forceSSL = true;
+      enableACME = true;
+      # inherit kTLS;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:8013";
+        recommendedProxySettings = true;
+      };
+      locations."= /robots.txt".extraConfig = ''
+        return 200 "User-agent: *\nDisallow: /\n";
+      '';
+    };
+
+    sane.dns.zones."uninsane.org".inet.CNAME."w" = "native";
   };
-
-  services.nginx.virtualHosts."w.uninsane.org" = {
-    forceSSL = true;
-    enableACME = true;
-    # inherit kTLS;
-    locations."/".proxyPass = "http://127.0.0.1:8013";
-  };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."w" = "native";
 }

hosts/by-name/servo/services/komga.nix

@@ -1,8 +1,9 @@
-{ config, ... }:
+{ config, lib, ... }:
 let
   svc-cfg = config.services.komga;
   inherit (svc-cfg) user group port stateDir;
 in
+lib.mkIf false  #< 2024/09/30: disabled because i haven't used this for several months
 {
   sane.persist.sys.byStore.plaintext = [
     { inherit user group; mode = "0700"; path = stateDir; method = "bind"; }
@@ -16,7 +17,11 @@ in
     enableACME = true;
     locations."/" = {
       proxyPass = "http://127.0.0.1:${builtins.toString port}";
+      recommendedProxySettings = true;
     };
+    locations."= /robots.txt".extraConfig = ''
+      return 200 "User-agent: *\nDisallow: /\n";
+    '';
   };
   sane.dns.zones."uninsane.org".inet.CNAME."komga" = "native";
 }

hosts/by-name/servo/services/lemmy.nix

@@ -5,86 +5,175 @@
 
 { config, lib, pkgs, ... }:
 let
-  inherit (builtins) toString;
-  inherit (lib) mkForce;
   uiPort = 1234;  # default ui port is 1234
   backendPort = 8536; # default backend port is 8536
   #^ i guess the "backend" port is used for federation?
   pict-rs = pkgs.pict-rs;
-  # pict-rs = pkgs.pict-rs.overrideAttrs (upstream: {
-  #   # as of v0.4.2, all non-GIF video is forcibly transcoded.
-  #   # that breaks lemmy, because of the request latency.
-  #   # and it eats up hella CPU.
-  #   # pict-rs is iffy around video altogether: mp4 seems the best supported.
-  #   # XXX: this patch no longer applies after 0.5.10 -> 0.5.11 update.
-  #   #      git log is hard to parse, but *suggests* that video is natively supported
-  #   #      better than in the 0.4.2 days, e.g. 5fd59fc5b42d31559120dc28bfef4e5002fb509e
-  #   #      "Change commandline flag to allow disabling video, since it is enabled by default"
-  #   postPatch = (upstream.postPatch or "") + ''
-  #     substituteInPlace src/validate.rs \
-  #       --replace 'if transcode_options.needs_reencode() {' 'if false {'
-  #   '';
-  # });
-in {
-  services.lemmy = {
-    enable = true;
-    settings.hostname = "lemmy.uninsane.org";
-    # federation.debug forces outbound federation queries to be run synchronously
-    # N.B.: this option might not be read for 0.17.0+? <https://github.com/LemmyNet/lemmy/blob/c32585b03429f0f76d1e4ff738786321a0a9df98/RELEASES.md#upgrade-instructions>
-    # settings.federation.debug = true;
-    settings.port = backendPort;
-    ui.port = uiPort;
-    database.createLocally = true;
-    nginx.enable = true;
-  };
-
-  systemd.services.lemmy.serviceConfig = {
-    # fix to use a normal user so we can configure perms correctly
-    DynamicUser = mkForce false;
-    User = "lemmy";
-    Group = "lemmy";
-  };
-  systemd.services.lemmy.environment = {
-    RUST_BACKTRACE = "full";
-    # RUST_LOG = "debug";
-    # RUST_LOG = "trace";
-    # upstream defaults LEMMY_DATABASE_URL = "postgres:///lemmy?host=/run/postgresql";
-    # - Postgres complains that we didn't specify a user
-    # lemmy formats the url as:
-    # - postgres://{user}:{password}@{host}:{port}/{database}
-    # SO suggests (https://stackoverflow.com/questions/3582552/what-is-the-format-for-the-postgresql-connection-string-url):
-    # - postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]
-    # LEMMY_DATABASE_URL = "postgres://lemmy@/run/postgresql";  # connection to server on socket "/run/postgresql/.s.PGSQL.5432" failed: FATAL:  database "run/postgresql" does not exist
-    # LEMMY_DATABASE_URL = "postgres://lemmy?host=/run/postgresql";  # no PostgreSQL user name specified in startup packet
-    # LEMMY_DATABASE_URL = mkForce "postgres://lemmy@?host=/run/postgresql";  # WORKS
-    LEMMY_DATABASE_URL = mkForce "postgres://lemmy@/lemmy?host=/run/postgresql";
-  };
-  users.groups.lemmy = {};
-  users.users.lemmy = {
-    group = "lemmy";
-    isSystemUser = true;
-  };
-
-  services.nginx.virtualHosts."lemmy.uninsane.org" = {
-    forceSSL = true;
-    enableACME = true;
-  };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."lemmy" = "native";
-
-  #v DO NOT REMOVE: defaults to 0.3, instead of latest, so always need to explicitly set this.
-  services.pict-rs.package = pict-rs;
-
   # pict-rs configuration is applied in this order:
   # - via toml
   # - via env vars (overrides everything above)
   # - via CLI flags (overrides everything above)
   # some of the CLI flags have defaults, making it the only actual way to configure certain things even when docs claim otherwise.
   # CLI args: <https://git.asonix.dog/asonix/pict-rs#user-content-running>
-  systemd.services.pict-rs.serviceConfig.ExecStart = lib.mkForce (lib.concatStringsSep " " [
-    "${lib.getBin pict-rs}/bin/pict-rs run"
-    "--media-video-max-frame-count" (builtins.toString (30*60*60))
-    "--media-process-timeout 120"
-    "--media-video-allow-audio"  # allow audio
-  ]);
+  # TOML args: <https://git.asonix.dog/asonix/pict-rs/src/branch/main/pict-rs.toml>
+  toml = pkgs.formats.toml { };
+  tomlConfig = toml.generate "pict-rs.toml" pictrsConfig;
+  pictrsConfig = {
+    media.process_timeout = 120;
+    media.video.allow_audio = true;
+    media.video.max_frame_count = 30 * 60 * 60;
+  };
+in {
+  config = lib.mkIf (config.sane.maxBuildCost >= 2) {
+    services.lemmy = {
+      enable = true;
+      settings.hostname = "lemmy.uninsane.org";
+      # federation.debug forces outbound federation queries to be run synchronously
+      # N.B.: this option might not be read for 0.17.0+? <https://github.com/LemmyNet/lemmy/blob/c32585b03429f0f76d1e4ff738786321a0a9df98/RELEASES.md#upgrade-instructions>
+      # settings.federation.debug = true;
+      settings.port = backendPort;
+      ui.port = uiPort;
+      database.createLocally = true;
+      nginx.enable = true;
+    };
+
+    systemd.services.lemmy.environment = {
+      RUST_BACKTRACE = "full";
+      RUST_LOG = "error";
+      # RUST_LOG = "warn";
+      # RUST_LOG = "debug";
+      # RUST_LOG = "trace";
+      # upstream defaults LEMMY_DATABASE_URL = "postgres:///lemmy?host=/run/postgresql";
+      # - Postgres complains that we didn't specify a user
+      # lemmy formats the url as:
+      # - postgres://{user}:{password}@{host}:{port}/{database}
+      # SO suggests (https://stackoverflow.com/questions/3582552/what-is-the-format-for-the-postgresql-connection-string-url):
+      # - postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]
+      # LEMMY_DATABASE_URL = "postgres://lemmy@/run/postgresql";  # connection to server on socket "/run/postgresql/.s.PGSQL.5432" failed: FATAL:  database "run/postgresql" does not exist
+      # LEMMY_DATABASE_URL = "postgres://lemmy?host=/run/postgresql";  # no PostgreSQL user name specified in startup packet
+      # LEMMY_DATABASE_URL = lib.mkForce "postgres://lemmy@?host=/run/postgresql";  # WORKS
+      LEMMY_DATABASE_URL = lib.mkForce "postgres://lemmy@/lemmy?host=/run/postgresql";
+    };
+    users.groups.lemmy = {};
+    users.users.lemmy = {
+      group = "lemmy";
+      isSystemUser = true;
+    };
+
+    services.nginx.virtualHosts."lemmy.uninsane.org" = {
+      forceSSL = true;
+      enableACME = true;
+    };
+
+    sane.dns.zones."uninsane.org".inet.CNAME."lemmy" = "native";
+
+    systemd.services.lemmy = {
+      # fix to use a normal user so we can configure perms correctly
+      # XXX(2024-07-28): this hasn't been rigorously tested:
+      # possible that i've set something too strict and won't notice right away
+      serviceConfig.DynamicUser = lib.mkForce false;
+      serviceConfig.User = "lemmy";
+      serviceConfig.Group = "lemmy";
+
+      # switch postgres from Requires -> Wants, so that postgres may restart without taking lemmy down with it.
+      requires = lib.mkForce [];
+      wants = [ "postgresql.service" ];
+
+      # hardening (systemd-analyze security lemmy)
+      # a handful of these are specified in upstream nixpkgs, but mostly not
+      serviceConfig.LockPersonality = true;
+      serviceConfig.NoNewPrivileges = true;
+      serviceConfig.MemoryDenyWriteExecute = true;
+      serviceConfig.PrivateDevices = true;
+      serviceConfig.PrivateMounts = true;
+      serviceConfig.PrivateTmp = true;
+      serviceConfig.PrivateUsers = true;
+      serviceConfig.ProcSubset = "pid";
+
+      serviceConfig.ProtectClock = true;
+      serviceConfig.ProtectControlGroups = true;
+      serviceConfig.ProtectHome = true;
+      serviceConfig.ProtectHostname = true;
+      serviceConfig.ProtectKernelLogs = true;
+      serviceConfig.ProtectKernelModules = true;
+      serviceConfig.ProtectKernelTunables = true;
+      serviceConfig.ProtectProc = "invisible";
+      serviceConfig.ProtectSystem = "strict";
+      serviceConfig.RemoveIPC = true;
+      serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+
+      serviceConfig.RestrictNamespaces = true;
+      serviceConfig.RestrictSUIDSGID = true;
+      serviceConfig.SystemCallArchitectures = "native";
+      serviceConfig.SystemCallFilter = [ "@system-service" ];
+    };
+
+    systemd.services.lemmy-ui = {
+      # hardening (systemd-analyze security lemmy-ui)
+      # TODO: upstream into nixpkgs
+      serviceConfig.LockPersonality = true;
+      serviceConfig.NoNewPrivileges = true;
+      # serviceConfig.MemoryDenyWriteExecute = true;  #< it uses v8, JIT
+      serviceConfig.PrivateDevices = true;
+      serviceConfig.PrivateMounts = true;
+      serviceConfig.PrivateTmp = true;
+      serviceConfig.PrivateUsers = true;
+      serviceConfig.ProcSubset = "pid";
+
+      serviceConfig.ProtectClock = true;
+      serviceConfig.ProtectControlGroups = true;
+      serviceConfig.ProtectHome = true;
+      serviceConfig.ProtectHostname = true;
+      serviceConfig.ProtectKernelLogs = true;
+      serviceConfig.ProtectKernelModules = true;
+      serviceConfig.ProtectKernelTunables = true;
+      serviceConfig.ProtectProc = "invisible";
+      serviceConfig.ProtectSystem = "strict";
+      serviceConfig.RemoveIPC = true;
+      serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+
+      serviceConfig.RestrictNamespaces = true;
+      serviceConfig.RestrictSUIDSGID = true;
+      serviceConfig.SystemCallArchitectures = "native";
+      serviceConfig.SystemCallFilter = [ "@system-service" "@pkey" "@sandbox" ];
+    };
+
+    #v DO NOT REMOVE: defaults to 0.3, instead of latest, so always need to explicitly set this.
+    services.pict-rs.package = pict-rs;
+
+    systemd.services.pict-rs = {
+      serviceConfig.ExecStart = lib.mkForce (lib.concatStringsSep " " [
+        (lib.getExe pict-rs)
+        "--config-file"
+        tomlConfig
+        "run"
+      ]);
+
+      # hardening (systemd-analyze security pict-rs)
+      # TODO: upstream into nixpkgs
+      serviceConfig.LockPersonality = true;
+      serviceConfig.NoNewPrivileges = true;
+      serviceConfig.MemoryDenyWriteExecute = true;
+      serviceConfig.PrivateDevices = true;
+      serviceConfig.PrivateMounts = true;
+      serviceConfig.PrivateTmp = true;
+      serviceConfig.PrivateUsers = true;
+      serviceConfig.ProcSubset = "pid";
+      serviceConfig.ProtectClock = true;
+      serviceConfig.ProtectControlGroups = true;
+      serviceConfig.ProtectHome = true;
+      serviceConfig.ProtectHostname = true;
+      serviceConfig.ProtectKernelLogs = true;
+      serviceConfig.ProtectKernelModules = true;
+      serviceConfig.ProtectKernelTunables = true;
+      serviceConfig.ProtectProc = "invisible";
+      serviceConfig.ProtectSystem = "strict";
+      serviceConfig.RemoveIPC = true;
+      serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+      serviceConfig.RestrictNamespaces = true;
+      serviceConfig.RestrictSUIDSGID = true;
+      serviceConfig.SystemCallArchitectures = "native";
+      serviceConfig.SystemCallFilter = [ "@system-service" ];
+    };
+  };
 }

hosts/by-name/servo/services/matrix/default.nix

@@ -1,6 +1,6 @@
 # docs: <https://nixos.wiki/wiki/Matrix>
 # docs: <https://nixos.org/manual/nixos/stable/index.html#module-services-matrix-synapse>
-# example config: <https://github.com/matrix-org/synapse/blob/develop/docs/sample_config.yaml>
+# example config: <https://github.com/element-hq/synapse/blob/develop/docs/sample_config.yaml>
 #
 # ENABLING PUSH NOTIFICATIONS (with UnifiedPush/ntfy):
 # - Matrix "pushers" API spec: <https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3pushersset>
@@ -12,7 +12,9 @@
 # - delete a notification destination by setting `kind` to `null` (otherwise, request is identical to above)
 #
 { config, lib, pkgs, ... }:
-
+let
+  ntfy = config.services.ntfy-sh.enable;
+in
 {
   imports = [
     ./discord-puppet.nix
@@ -20,19 +22,17 @@
     ./signal.nix
   ];
 
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     { user = "matrix-synapse"; group = "matrix-synapse"; path = "/var/lib/matrix-synapse"; method = "bind"; }
   ];
   services.matrix-synapse.enable = true;
+  services.matrix-synapse.log.root.level = "ERROR";  # accepts "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" (?)
   services.matrix-synapse.settings = {
-    # this changes the default log level from INFO to WARN.
-    # maybe there's an easier way?
-    log_config = ./synapse-log_level.yaml;
     server_name = "uninsane.org";
 
     # services.matrix-synapse.enable_registration_captcha = true;
     # services.matrix-synapse.enable_registration_without_verification = true;
-    enable_registration = true;
+    # enable_registration = true;
     # services.matrix-synapse.registration_shared_secret = "<shared key goes here>";
 
     # default for listeners is port = 8448, tls = true, x_forwarded = false.
@@ -70,21 +70,30 @@
     config.sops.secrets."matrix_synapse_secrets.yaml".path
   ];
 
-  systemd.services.matrix-synapse.postStart = ''
-    ACCESS_TOKEN=$(${pkgs.coreutils}/bin/cat ${config.sops.secrets.matrix_access_token.path})
-    TOPIC=$(${pkgs.coreutils}/bin/cat ${config.sops.secrets.ntfy-sh-topic.path})
+  # tune restart settings to ensure systemd doesn't disable it, and we don't overwhelm postgres
+  systemd.services.matrix-synapse.serviceConfig.RestartSec = 5;
+  systemd.services.matrix-synapse.serviceConfig.RestartMaxDelaySec = 20;
+  systemd.services.matrix-synapse.serviceConfig.StartLimitBurst = 120;
+  systemd.services.matrix-synapse.serviceConfig.RestartSteps = 3;
+  # switch postgres from Requires -> Wants, so that postgres may restart without taking matrix down with it.
+  systemd.services.matrix-synapse.requires = lib.mkForce [];
+  systemd.services.matrix-synapse.wants = [ "postgresql.service" ];
+
+  systemd.services.matrix-synapse.postStart = lib.optionalString ntfy ''
+    ACCESS_TOKEN=$(${lib.getExe' pkgs.coreutils "cat"} ${config.sops.secrets.matrix_access_token.path})
+    TOPIC=$(${lib.getExe' pkgs.coreutils "cat"} ${config.sops.secrets.ntfy-sh-topic.path})
 
     echo "ensuring ntfy push gateway"
-    ${pkgs.curl}/bin/curl \
+    ${lib.getExe pkgs.curl} \
       --header "Authorization: Bearer $ACCESS_TOKEN" \
       --data "{ \"app_display_name\": \"ntfy-adapter\", \"app_id\": \"ntfy.uninsane.org\", \"data\": { \"url\": \"https://ntfy.uninsane.org/_matrix/push/v1/notify\", \"format\": \"event_id_only\" }, \"device_display_name\": \"ntfy-adapter\", \"kind\": \"http\", \"lang\": \"en-US\", \"profile_tag\": \"\", \"pushkey\": \"$TOPIC\" }" \
       localhost:8008/_matrix/client/v3/pushers/set
 
     echo "registered push gateways:"
-    ${pkgs.curl}/bin/curl \
+    ${lib.getExe pkgs.curl} \
       --header "Authorization: Bearer $ACCESS_TOKEN" \
       localhost:8008/_matrix/client/v3/pushers \
-      | ${pkgs.jq}/bin/jq .
+      | ${lib.getExe pkgs.jq} .
   '';
 
 
@@ -114,6 +123,7 @@
 
     locations."/" = {
       proxyPass = "http://127.0.0.1:8008";
+      recommendedProxySettings = true;
       extraConfig = ''
         # allow uploading large files (matrix enforces a separate limit, downstream)
         client_max_body_size  512m;
@@ -161,5 +171,5 @@
     owner = config.users.users.matrix-synapse.name;
   };
   # provide access to ntfy-sh-topic secret
-  users.users.matrix-synapse.extraGroups = [ "ntfy-sh" ];
+  users.users.matrix-synapse.extraGroups = lib.optionals ntfy [ "ntfy-sh" ];
 }

hosts/by-name/servo/services/matrix/discord-puppet.nix

@@ -5,7 +5,7 @@
 # - recommended to use mautrix-discord: <https://github.com/NixOS/nixpkgs/pull/200462>
 lib.mkIf false
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     { user = "matrix-synapse"; group = "matrix-synapse"; path = "/var/lib/mx-puppet-discord"; method = "bind"; }
   ];
 

hosts/by-name/servo/services/matrix/irc.nix

@@ -1,15 +1,13 @@
 # config docs:
 # - <https://github.com/matrix-org/matrix-appservice-irc/blob/develop/config.sample.yaml>
-#       probably want to remove that.
-{ config, lib, ... }:
+{ lib, ... }:
 
 let
-  ircServer = { name, additionalAddresses ? [], sasl ? true, port ? 6697 }: let
+  ircServer = { name, additionalAddresses ? [], ssl ? true, sasl ? true, port ? if ssl then 6697 else 6667 }: let
     lowerName = lib.toLower name;
   in {
     # XXX sasl: appservice doesn't support NickServ identification (only SASL, or PASS if sasl = false)
-    inherit name additionalAddresses sasl port;
-    ssl = true;
+    inherit additionalAddresses name port sasl ssl;
     botConfig = {
       # bot has no presence in IRC channel; only real Matrix users
       enabled = false;
@@ -101,7 +99,7 @@ in
     })
   ];
 
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     # TODO: mode?
     { user = "matrix-appservice-irc"; group = "matrix-appservice-irc"; path = "/var/lib/matrix-appservice-irc"; method = "bind"; }
   ];
@@ -129,6 +127,8 @@ in
     };
 
     ircService = {
+      logging.level = "warn";  # "error", "warn", "info", "debug"
+      mediaProxy.publicUrl = "https://irc.matrix.uninsane.org/media";
       servers = {
         "irc.esper.net" = ircServer {
           name = "esper";
@@ -154,8 +154,21 @@ in
           # notable channels:
           # - #sxmo
           # - #sxmo-offtopic
+          # supposedly also available at <irc://37lnq2veifl4kar7.onion:6667/> (unofficial)
         };
         "irc.rizon.net" = ircServer { name = "Rizon"; };
+        # "irc.sdf.org" = ircServer {
+        #   # XXX(2024-11-06): seems it can't connect. "matrix-appservice-irc: WARN:Provisioner Provisioner only handles text 'yes'/'y' (from BASHy2-EU on irc.sdf.org)"
+        #   # use instead? <https://lemmy.sdf.org/c/sdfpubnix>
+        #   name = "sdf";
+        #   # sasl = false;
+        #   # notable channels (see: <https://sdf.org/?tutorials/irc-channels>)
+        #   # - #sdf
+        # };
+        "wigle.net" = ircServer {
+          name = "WiGLE";
+          ssl = false;
+        };
       };
     };
   };
@@ -165,4 +178,17 @@ in
     # the service actively uses at least one of these, and both of them are fairly innocuous
     SystemCallFilter = lib.mkForce "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @setuid @swap";
   };
+
+  services.nginx.virtualHosts."irc.matrix.uninsane.org" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/media" = {
+      proxyPass = "http://127.0.0.1:11111";
+      recommendedProxySettings = true;
+    };
+  };
+
+  sane.dns.zones."uninsane.org".inet = {
+    CNAME."irc.matrix" = "native";
+  };
 }

hosts/by-name/servo/services/matrix/signal.nix

@@ -4,7 +4,7 @@
 
 lib.mkIf false  # disabled 2024/01/11: i don't use it, and pkgs.mautrix-signal had some API changes
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     { user = "mautrix-signal"; group = "mautrix-signal"; path = "/var/lib/mautrix-signal"; method = "bind"; }
     { user = "signald"; group = "signald"; path = "/var/lib/signald"; method = "bind"; }
   ];

hosts/by-name/servo/services/matrix/synapse-log_level.yaml

@@ -1,27 +0,0 @@
-version: 1
-
-# In systemd's journal, loglevel is implicitly stored, so let's omit it
-# from the message text.
-formatters:
-    journal_fmt:
-        format: '%(name)s: [%(request)s] %(message)s'
-
-filters:
-    context:
-        (): synapse.util.logcontext.LoggingContextFilter
-        request: ""
-
-handlers:
-    journal:
-        class: systemd.journal.JournalHandler
-        formatter: journal_fmt
-        filters: [context]
-        SYSLOG_IDENTIFIER: synapse
-
-# default log level: INFO
-root:
-    level: WARN
-    handlers: [journal]
-
-disable_existing_loggers: False
-

hosts/by-name/servo/services/minidlna.nix

@@ -0,0 +1,39 @@
+# - `man 5 minidlna.conf`
+# - `man 8 minidlnad`
+#
+# this is an extremely simple (but limited) DLNA server:
+# - no web UI
+# - no runtime configuration -- just statically configure media directories instead
+# - no transcoding
+# compatibility:
+# - LG TV: music: all working
+# - LG TV: videos: mixed. i can't see the pattern; HEVC works; H.264 sometimes works.
+{ lib, ... }:
+lib.mkIf false  #< XXX(2024-11-17): WORKS, but i'm trying gerbera instead for hopefully better transcoding
+{
+  sane.ports.ports."1900" = {
+    protocol = [ "udp" ];
+    visibleTo.lan = true;
+    description = "colin-upnp-for-minidlna";
+  };
+  sane.ports.ports."8200" = {
+    protocol = [ "tcp" ];
+    visibleTo.lan = true;
+    description = "colin-minidlna-http";
+  };
+
+  services.minidlna.enable = true;
+
+  services.minidlna.settings = {
+    media_dir = [
+      # A/V/P to restrict a directory to audio/video/pictures
+      "A,/var/media/Music"
+      "V,/var/media/Videos/Film"
+      # "V,/var/media/Videos/Milkbags"
+      "V,/var/media/Videos/Shows"
+    ];
+    notify_interval = 60;
+  };
+
+  users.users.minidlna.extraGroups = [ "media" ];
+}

hosts/by-name/servo/services/mumble.nix

@@ -0,0 +1,66 @@
+# murmur is the server component of mumble.
+# - docs: <https://www.mumble.info/documentation/>
+# - config docs: <https://www.mumble.info/documentation/administration/config-file/>
+#
+# default port is 64738 (UDP and TCP)
+#
+# FIRST-RUN:
+# - login from mumble client as `SuperUser`, password taken from `journalctl -u murmur`.
+# - login from another machine and right click on self -> 'Register'
+# - as SuperUser, right click on server root -> edit
+#   - Groups tab: select "admin", then add the other registered user to the group.
+# - log out as SuperUser and manage the server using that other user now.
+#
+# USAGE:
+# - 'auth' group = any user who has registered a cert with the server.
+{ ... }:
+{
+  sane.persist.sys.byStore.private = [
+    { user = "murmur"; group = "murmur"; mode = "0700"; path = "/var/lib/murmur"; method = "bind"; }
+  ];
+
+  services.murmur.enable = true;
+  services.murmur.welcometext = "welcome to Colin's mumble voice chat server";
+  # max bandwidth (bps) **per user**. i believe this affects both voice and uploads?
+  # mumble defaults to 558000, but nixos service defaults to 72000.
+  services.murmur.bandwidth = 558000;
+  services.murmur.imgMsgLength = 8 * 1024 * 1024;
+
+  services.murmur.sslCert = "/var/lib/acme/mumble.uninsane.org/fullchain.pem";
+  services.murmur.sslKey = "/var/lib/acme/mumble.uninsane.org/key.pem";
+  services.murmur.sslCa = "/etc/ssl/certs/ca-bundle.crt";
+
+  # allow clients on the LAN to discover this server
+  services.murmur.bonjour = true;
+
+  # mumble has a public server listing.
+  # my server doesn't associate with that registry (unless i specify registerPassword).
+  # however these settings appear to affect how the server presents itself to clients, regardless of registration.
+  services.murmur.registerName = "mumble.uninsane.org";
+  services.murmur.registerUrl = "https://mumble.uninsane.org";
+  services.murmur.registerHostname = "mumble.uninsane.org";
+
+  # defaultchannel=ID makes it so that unauthenticated users are placed in some specific channel when they join
+  services.murmur.extraConfig = ''
+    defaultchannel=2
+  '';
+
+  users.users.murmur.extraGroups = [
+    "nginx"  # provide access to certs
+  ];
+  services.nginx.virtualHosts."mumble.uninsane.org" = {
+    # allow ACME to procure a cert via nginx for this domain
+    enableACME = true;
+  };
+
+  sane.dns.zones."uninsane.org".inet = {
+    CNAME."mumble" = "native";
+  };
+
+  sane.ports.ports."64738" = {
+    protocol = [ "tcp" "udp" ];
+    visibleTo.lan = true;
+    visibleTo.doof = true;
+    description = "colin-mumble";
+  };
+}

hosts/by-name/servo/services/navidrome.nix

@@ -34,7 +34,10 @@ lib.mkIf false  #< i don't actively use navidrome
     forceSSL = true;
     enableACME = true;
     # inherit kTLS;
-    locations."/".proxyPass = "http://127.0.0.1:4533";
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:4533";
+      recommendedProxySettings = true;
+    };
   };
 
   sane.dns.zones."uninsane.org".inet.CNAME."music" = "native";

hosts/by-name/servo/services/nginx.nix

@@ -1,223 +0,0 @@
-# docs: <https://nixos.wiki/wiki/Nginx>
-# docs: <https://nginx.org/en/docs/>
-{ config, lib, pkgs, ... }:
-
-let
-  # make the logs for this host "public" so that they show up in e.g. metrics
-  publog = vhost: lib.attrsets.unionOfDisjoint vhost {
-    extraConfig = (vhost.extraConfig or "") + ''
-      access_log /var/log/nginx/public.log vcombined;
-    '';
-  };
-
-  # kTLS = true;  # in-kernel TLS for better perf
-in
-{
-
-  sane.ports.ports."80" = {
-    protocol = [ "tcp" ];
-    visibleTo.lan = true;
-    visibleTo.wan = true;
-    visibleTo.ovpn = true;  # so that letsencrypt can procure a cert for the mx record
-    description = "colin-http-uninsane.org";
-  };
-  sane.ports.ports."443" = {
-    protocol = [ "tcp" ];
-    visibleTo.lan = true;
-    visibleTo.wan = true;
-    description = "colin-https-uninsane.org";
-  };
-
-  services.nginx.enable = true;
-  services.nginx.appendConfig = ''
-    # use 1 process per core.
-    # may want to increase worker_connections too, but `ulimit -n` must be increased first.
-    worker_processes auto;
-  '';
-
-  # this is the standard `combined` log format, with the addition of $host
-  # so that we have the virtualHost in the log.
-  # KEEP IN SYNC WITH GOACCESS
-  # goaccess calls this VCOMBINED:
-  # - <https://gist.github.com/jyap808/10570005>
-  services.nginx.commonHttpConfig = ''
-    log_format vcombined '$host:$server_port $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referrer" "$http_user_agent"';
-    access_log /var/log/nginx/private.log vcombined;
-  '';
-  # sets gzip_comp_level = 5
-  services.nginx.recommendedGzipSettings = true;
-  # enables OCSP stapling (so clients don't need contact the OCSP server -- i do instead)
-  # - doesn't seem to, actually: <https://www.ssllabs.com/ssltest/analyze.html?d=uninsane.org>
-  # caches TLS sessions for 10m
-  services.nginx.recommendedTlsSettings = true;
-  # enables sendfile, tcp_nopush, tcp_nodelay, keepalive_timeout 65
-  services.nginx.recommendedOptimisation = true;
-
-  # web blog/personal site
-  # alternative way to link stuff into the share:
-  # sane.fs."/var/www/sites/uninsane.org/share/Ubunchu".mount.bind = "/var/media/Books/Visual/HiroshiSeo/Ubunchu";
-  # sane.fs."/var/media/Books/Visual/HiroshiSeo/Ubunchu".dir = {};
-  services.nginx.virtualHosts."uninsane.org" = publog {
-    # a lot of places hardcode https://uninsane.org,
-    # and then when we mix http + non-https, we get CORS violations
-    # and things don't look right. so force SSL.
-    forceSSL = true;
-    enableACME = true;
-    # inherit kTLS;
-    # for OCSP stapling
-    sslTrustedCertificate = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
-
-    locations."/" = {
-      root = "${pkgs.uninsane-dot-org}/share/uninsane-dot-org";
-      tryFiles = "$uri $uri/ @fallback";
-    };
-
-    # unversioned files
-    locations."@fallback" = {
-      root = "/var/www/sites/uninsane.org";
-    };
-
-    # uninsane.org/share/foo => /var/www/sites/uninsane.org/share/foo.
-    # special-cased to enable directory listings
-    locations."/share" = {
-      root = "/var/www/sites/uninsane.org";
-      extraConfig = ''
-        # autoindex => render directory listings
-        autoindex on;
-        # don't follow any symlinks when serving files
-        # otherwise it allows a directory escape
-        disable_symlinks on;
-      '';
-    };
-    locations."/share/Milkbags/" = {
-      alias = "/var/media/Videos/Milkbags/";
-      extraConfig = ''
-        # autoindex => render directory listings
-        autoindex on;
-        # don't follow any symlinks when serving files
-        # otherwise it allows a directory escape
-        disable_symlinks on;
-      '';
-    };
-
-    # allow matrix users to discover that @user:uninsane.org is reachable via matrix.uninsane.org
-    locations."= /.well-known/matrix/server".extraConfig =
-      let
-        # use 443 instead of the default 8448 port to unite
-        # the client-server and server-server port for simplicity
-        server = { "m.server" = "matrix.uninsane.org:443"; };
-      in ''
-        add_header Content-Type application/json;
-        return 200 '${builtins.toJSON server}';
-      '';
-    locations."= /.well-known/matrix/client".extraConfig =
-      let
-        client = {
-          "m.homeserver" =  { "base_url" = "https://matrix.uninsane.org"; };
-          "m.identity_server" =  { "base_url" = "https://vector.im"; };
-        };
-      # ACAO required to allow element-web on any URL to request this json file
-      in ''
-        add_header Content-Type application/json;
-        add_header Access-Control-Allow-Origin *;
-        return 200 '${builtins.toJSON client}';
-      '';
-
-    # static URLs might not be aware of .well-known (e.g. registration confirmation URLs),
-    # so hack around that.
-    locations."/_matrix" = {
-      proxyPass = "http://127.0.0.1:8008";
-    };
-    locations."/_synapse" = {
-      proxyPass = "http://127.0.0.1:8008";
-    };
-
-    # allow ActivityPub clients to discover how to reach @user@uninsane.org
-    # see: https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3361/
-    # not sure this makes sense while i run multiple AP services (pleroma, lemmy)
-    # locations."/.well-known/nodeinfo" = {
-    #   proxyPass = "http://127.0.0.1:4000";
-    #   extraConfig = pleromaExtraConfig;
-    # };
-
-    # redirect common feed URIs to the canonical feed
-    locations."= /atom".extraConfig = "return 301 /atom.xml;";
-    locations."= /feed".extraConfig = "return 301 /atom.xml;";
-    locations."= /feed.xml".extraConfig = "return 301 /atom.xml;";
-    locations."= /rss".extraConfig = "return 301 /atom.xml;";
-    locations."= /rss.xml".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/atom".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/atom.xml".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/feed".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/feed.xml".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/rss".extraConfig = "return 301 /atom.xml;";
-    locations."= /blog/rss.xml".extraConfig = "return 301 /atom.xml;";
-  };
-
-
-  # serve any site not listed above, if it's static.
-  # because we define it dynamically, SSL isn't trivial. support only http
-  # documented <https://nginx.org/en/docs/http/ngx_http_core_module.html#server_name>
-  services.nginx.virtualHosts."~^(?<domain>.+)$" = {
-    default = true;
-    addSSL = true;
-    enableACME = false;
-    sslCertificate = "/var/www/certs/wildcard/cert.pem";
-    sslCertificateKey = "/var/www/certs/wildcard/key.pem";
-    # sslCertificate = "/var/lib/acme/.minica/cert.pem";
-    # sslCertificateKey = "/var/lib/acme/.minica/key.pem";
-    # serverName = null;
-    locations."/" = {
-      # somehow this doesn't escape -- i get error 400 if i:
-      # curl 'http://..' --resolve '..:80:127.0.0.1'
-      root = "/var/www/sites/$domain";
-      # tryFiles = "$domain/$uri $domain/$uri/ =404";
-    };
-  };
-
-  security.acme.acceptTerms = true;
-  security.acme.defaults.email = "admin.acme@uninsane.org";
-
-  sane.persist.sys.byStore.plaintext = [
-    { user = "acme"; group = "acme"; path = "/var/lib/acme"; method = "bind"; }
-    { user = "colin"; group = "users"; path = "/var/www/sites"; method = "bind"; }
-  ];
-
-  # let's encrypt default chain looks like:
-  # - End-entity certificate ← R3 ← ISRG Root X1 ← DST Root CA X3
-  # - <https://community.letsencrypt.org/t/production-chain-changes/150739>
-  # DST Root CA X3 expired in 2021 (?)
-  # the alternative chain is:
-  # - End-entity certificate ← R3 ← ISRG Root X1 (self-signed)
-  # using this alternative chain grants more compatibility for services like ejabberd
-  # but might decrease compatibility with very old clients that don't get updates (e.g. old android, iphone <= 4).
-  # security.acme.defaults.extraLegoFlags = [
-  security.acme.certs."uninsane.org" = rec {
-    # ISRG Root X1 results in lets encrypt sending the same chain as default,
-    # just without the final ISRG Root X1 ← DST Root CA X3 link.
-    # i.e. we could alternative clip the last item and achieve the exact same thing.
-    extraLegoRunFlags = [
-      "--preferred-chain" "ISRG Root X1"
-    ];
-    extraLegoRenewFlags = extraLegoRunFlags;
-  };
-  # TODO: alternatively, we could clip the last cert IF it's expired,
-  # optionally outputting that to a new cert file.
-  # security.acme.defaults.postRun = "";
-
-  # create a self-signed SSL certificate for use with literally any domain.
-  # browsers will reject this, but proxies and local testing tools can be configured
-  # to accept it.
-  system.activationScripts.generate-x509-self-signed.text = ''
-    mkdir -p /var/www/certs/wildcard
-    test -f /var/www/certs/wildcard/key.pem || ${pkgs.openssl}/bin/openssl \
-      req -x509 -newkey rsa:4096 \
-      -keyout /var/www/certs/wildcard/key.pem \
-      -out /var/www/certs/wildcard/cert.pem \
-      -sha256 -nodes -days 3650 \
-      -addext 'subjectAltName=DNS:*' \
-      -subj '/CN=self-signed'
-    chmod 640 /var/www/certs/wildcard/{key,cert}.pem
-    chown root:nginx /var/www/certs/wildcard /var/www/certs/wildcard/{key,cert}.pem
-  '';
-}

hosts/by-name/servo/services/nginx/default.nix

@@ -0,0 +1,111 @@
+# docs: <https://nixos.wiki/wiki/Nginx>
+# docs: <https://nginx.org/en/docs/>
+{ lib, pkgs, ... }:
+{
+  imports = [
+    ./uninsane.org.nix
+    ./waka.laka.osaka
+  ];
+
+  sane.ports.ports."80" = {
+    protocol = [ "tcp" ];
+    visibleTo.lan = true;
+    visibleTo.ovpns = true;  # so that letsencrypt can procure a cert for the mx record
+    visibleTo.doof = true;
+    description = "colin-http-uninsane.org";
+  };
+  sane.ports.ports."443" = {
+    protocol = [ "tcp" ];
+    visibleTo.lan = true;
+    visibleTo.doof = true;
+    description = "colin-https-uninsane.org";
+  };
+
+  services.nginx.enable = true;
+
+  users.users.nginx.extraGroups = [ "anubis" ];
+  # nginxStable is one release behind nginxMainline.
+  # nginx itself recommends running mainline; nixos defaults to stable.
+  # services.nginx.package = pkgs.nginxMainline;
+  # XXX(2024-07-31): nixos defaults to zlib-ng -- supposedly more performant, but spams log with
+  # "gzip filter failed to use preallocated memory: ..."
+  # XXX(2025-07-24): "gzip filter" spam is gone => use default nginx package
+  # services.nginx.package = pkgs.nginxMainline.override { zlib = pkgs.zlib; };
+  services.nginx.appendConfig = ''
+    # use 1 process per core.
+    # may want to increase worker_connections too, but `ulimit -n` must be increased first.
+    worker_processes auto;
+  '';
+
+  # this is the standard `combined` log format, with the addition of $host
+  # so that we have the virtualHost in the log.
+  # KEEP IN SYNC WITH GOACCESS
+  # goaccess calls this VCOMBINED:
+  # - <https://gist.github.com/jyap808/10570005>
+  services.nginx.commonHttpConfig = ''
+    log_format vcombined '$host:$server_port $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referrer" "$http_user_agent"';
+    access_log /var/log/nginx/private.log vcombined;
+  '';
+  # enables gzip and sets gzip_comp_level = 5
+  services.nginx.recommendedGzipSettings = true;
+  # enables zstd and sets zstd_comp_level = 9
+  # services.nginx.recommendedZstdSettings = true;  #< XXX(2025-07-18): nginx zstd integration is unmaintained in NixOS
+  # enables OCSP stapling (so clients don't need contact the OCSP server -- i do instead)
+  # - doesn't seem to, actually: <https://www.ssllabs.com/ssltest/analyze.html?d=uninsane.org>
+  # caches TLS sessions for 10m
+  services.nginx.recommendedTlsSettings = true;
+  # enables sendfile, tcp_nopush, tcp_nodelay, keepalive_timeout 65
+  services.nginx.recommendedOptimisation = true;
+
+
+  # serve any site not otherwise declared, if it's static.
+  # because we define it dynamically, SSL isn't trivial. support only http
+  # documented <https://nginx.org/en/docs/http/ngx_http_core_module.html#server_name>
+  services.nginx.virtualHosts."~^(?<domain>.+)$" = {
+    default = true;
+    addSSL = true;
+    enableACME = false;
+    sslCertificate = "/var/www/certs/wildcard/cert.pem";
+    sslCertificateKey = "/var/www/certs/wildcard/key.pem";
+    # sslCertificate = "/var/lib/acme/.minica/cert.pem";
+    # sslCertificateKey = "/var/lib/acme/.minica/key.pem";
+    # serverName = null;
+    locations."/" = {
+      # somehow this doesn't escape -- i get error 400 if i:
+      # curl 'http://..' --resolve '..:80:127.0.0.1'
+      root = "/var/www/sites/$domain";
+      # tryFiles = "$domain/$uri $domain/$uri/ =404";
+    };
+  };
+
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "admin.acme@uninsane.org";
+
+  sane.persist.sys.byStore.plaintext = [
+    { user = "acme"; group = "acme"; path = "/var/lib/acme"; method = "bind"; }
+  ];
+  sane.persist.sys.byStore.private = [
+    { user = "colin"; group = "users"; path = "/var/www/sites"; method = "bind"; }
+  ];
+  sane.persist.sys.byStore.ephemeral = [
+    # logs *could* be persisted to private storage, but then there's the issue of
+    # "what if servo boots, isn't unlocked, and the whole / tmpfs is consumed by logs"
+    { user = "nginx"; group = "nginx"; path = "/var/log/nginx"; method = "bind"; }
+  ];
+
+  # create a self-signed SSL certificate for use with literally any domain.
+  # browsers will reject this, but proxies and local testing tools can be configured
+  # to accept it.
+  system.activationScripts.generate-x509-self-signed.text = ''
+    mkdir -p /var/www/certs/wildcard
+    test -f /var/www/certs/wildcard/key.pem || ${lib.getExe pkgs.openssl} \
+      req -x509 -newkey rsa:4096 \
+      -keyout /var/www/certs/wildcard/key.pem \
+      -out /var/www/certs/wildcard/cert.pem \
+      -sha256 -nodes -days 3650 \
+      -addext 'subjectAltName=DNS:*' \
+      -subj '/CN=self-signed'
+    chmod 640 /var/www/certs/wildcard/{key,cert}.pem
+    chown root:nginx /var/www/certs/wildcard /var/www/certs/wildcard/{key,cert}.pem
+  '';
+}

hosts/by-name/servo/services/nginx/uninsane.org.nix

@@ -0,0 +1,132 @@
+{ pkgs, ... }:
+{
+  # alternative way to link stuff into the share:
+  # sane.fs."/var/www/sites/uninsane.org/share/Ubunchu".mount.bind = "/var/media/Books/Visual/HiroshiSeo/Ubunchu";
+  # sane.fs."/var/media/Books/Visual/HiroshiSeo/Ubunchu".dir = {};
+  services.nginx.virtualHosts."uninsane.org" = {
+    # a lot of places hardcode https://uninsane.org,
+    # and then when we mix http + non-https, we get CORS violations
+    # and things don't look right. so force SSL.
+    forceSSL = true;
+    enableACME = true;
+
+    # extraConfig = ''
+    #   # "public" log so requests show up in goaccess metrics
+    #   access_log /var/log/nginx/public.log vcombined;
+    # '';
+
+    locations."/" = {
+      root = "${pkgs.uninsane-dot-org}/share/uninsane-dot-org";
+      tryFiles = "$uri $uri/ @fallback";
+    };
+
+    # unversioned files
+    locations."@fallback" = {
+      root = "/var/www/sites/uninsane.org";
+      extraConfig = ''
+        # instruct Google to not index these pages.
+        # see: <https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#xrobotstag>
+        add_header X-Robots-Tag 'none, noindex, nofollow';
+
+        # best-effort attempt to block archive.org from archiving these pages.
+        # reply with 403: Forbidden
+        # User Agent is *probably* "archive.org_bot"; maybe used to be "ia_archiver"
+        # source: <https://archive.org/details/archive.org_bot>
+        # additional UAs: <https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker>
+        #
+        # validate with: `curl -H 'User-Agent: "bot;archive.org_bot;like: something else"' -v https://uninsane.org/dne`
+        if ($http_user_agent ~* "(?:\b)archive.org_bot(?:\b)") {
+          return 403;
+        }
+        if ($http_user_agent ~* "(?:\b)archive.org(?:\b)") {
+          return 403;
+        }
+        if ($http_user_agent ~* "(?:\b)ia_archiver(?:\b)") {
+          return 403;
+        }
+      '';
+    };
+
+    # uninsane.org/share/foo => /var/www/sites/uninsane.org/share/foo.
+    # special-cased to enable directory listings
+    locations."/share" = {
+      root = "/var/www/sites/uninsane.org";
+      extraConfig = ''
+        # autoindex => render directory listings
+        autoindex on;
+        # don't follow any symlinks when serving files
+        # otherwise it allows a directory escape
+        disable_symlinks on;
+      '';
+    };
+    locations."/share/Milkbags/" = {
+      alias = "/var/media/Videos/Milkbags/";
+      extraConfig = ''
+        # autoindex => render directory listings
+        autoindex on;
+        # don't follow any symlinks when serving files
+        # otherwise it allows a directory escape
+        disable_symlinks on;
+      '';
+    };
+    locations."/share/Ubunchu/" = {
+      alias = "/var/media/Books/Visual/HiroshiSeo/Ubunchu/";
+      extraConfig = ''
+        # autoindex => render directory listings
+        autoindex on;
+        # don't follow any symlinks when serving files
+        # otherwise it allows a directory escape
+        disable_symlinks on;
+      '';
+    };
+
+    # allow matrix users to discover that @user:uninsane.org is reachable via matrix.uninsane.org
+    locations."= /.well-known/matrix/server".extraConfig =
+      let
+        # use 443 instead of the default 8448 port to unite
+        # the client-server and server-server port for simplicity
+        server = { "m.server" = "matrix.uninsane.org:443"; };
+      in ''
+        add_header Content-Type application/json;
+        return 200 '${builtins.toJSON server}';
+      '';
+    locations."= /.well-known/matrix/client".extraConfig =
+      let
+        client = {
+          "m.homeserver" =  { "base_url" = "https://matrix.uninsane.org"; };
+          "m.identity_server" =  { "base_url" = "https://vector.im"; };
+        };
+      # ACAO required to allow element-web on any URL to request this json file
+      in ''
+        add_header Content-Type application/json;
+        add_header Access-Control-Allow-Origin *;
+        return 200 '${builtins.toJSON client}';
+      '';
+
+    # static URLs might not be aware of .well-known (e.g. registration confirmation URLs),
+    # so hack around that.
+    locations."/_matrix".extraConfig = "return 301 https://matrix.uninsane.org$request_uri;";
+    locations."/_synapse".extraConfig = "return 301 https://matrix.uninsane.org$request_uri;";
+
+    # allow ActivityPub clients to discover how to reach @user@uninsane.org
+    # see: https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3361/
+    # not sure this makes sense while i run multiple AP services (pleroma, lemmy)
+    # locations."/.well-known/nodeinfo" = {
+    #   proxyPass = "http://127.0.0.1:4000";
+    #   extraConfig = pleromaExtraConfig;
+    # };
+
+    # redirect common feed URIs to the canonical feed
+    locations."= /atom".extraConfig = "return 301 /atom.xml;";
+    locations."= /feed".extraConfig = "return 301 /atom.xml;";
+    locations."= /feed.xml".extraConfig = "return 301 /atom.xml;";
+    locations."= /rss".extraConfig = "return 301 /atom.xml;";
+    locations."= /rss.xml".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/atom".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/atom.xml".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/feed".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/feed.xml".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/rss".extraConfig = "return 301 /atom.xml;";
+    locations."= /blog/rss.xml".extraConfig = "return 301 /atom.xml;";
+  };
+}

hosts/by-name/servo/services/nginx/waka.laka.osaka/default.nix

@@ -0,0 +1,35 @@
+{ config, pkgs, ... }:
+let
+  wakaLakaOsaka = pkgs.linkFarm "waka-laka-osaka" {
+    "index.html" = ./index.html;
+    "waka.laka.for.osaka.mp4" = pkgs.fetchurl {
+      # saved from: <https://www.youtube.com/watch?v=ehB_7bBKprY>
+      url = "https://uninsane.org/share/Milkbags/PG_Plays_Video_Games-Waka_Laka_For_Osaka_4K.mp4";
+      hash = "sha256-UW0qR4btX4pZ1bJp4Oxk20m3mvQGj9HweLKO27JBTFs=";
+    };
+  };
+in
+{
+  services.nginx.virtualHosts."laka.osaka" = {
+    addSSL = true;
+    enableACME = true;
+    locations."/" = {
+      # redirect everything to waka.laka.osaka
+      return = "301 https://waka.laka.osaka$request_uri";
+    };
+  };
+  services.nginx.virtualHosts."waka.laka.osaka" = {
+    addSSL = true;
+    enableACME = true;
+    locations."/" = {
+      root = wakaLakaOsaka;
+    };
+  };
+
+  sane.dns.zones."laka.osaka".inet = {
+    SOA."@" = config.sane.dns.zones."uninsane.org".inet.SOA."@";
+    A."@" = config.sane.dns.zones."uninsane.org".inet.A."@";
+    NS."@" = config.sane.dns.zones."uninsane.org".inet.NS."@";
+    CNAME."waka" = "native.uninsane.org.";
+  };
+}

hosts/by-name/servo/services/nginx/waka.laka.osaka/index.html

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width initial-scale=1" />
+  <meta name="description" content="Waka Laka (for Osaka)" />
+  <title>Waka Laka (for Osaka)</title>
+  <style>
+    html,body {
+      width: 100%;
+      height: 100%;
+      max-width: 100%;
+      max-height: 100%;
+    }
+    * {
+      margin: 0px;
+      padding: 0px;
+      border: 0px;
+    }
+    .bg-image {
+      width: 100%;
+      height: 100%;
+      min-width: 100%;
+      min-height: 100%;
+      position: fixed;
+      background-repeat: no-repeat;
+      background-position: 50% 50%;
+      background-size: contain;
+    }
+    body {
+      background-color: #000000;
+    }
+  </style>
+</head>
+<body>
+  <!-- TODO: how to autoplay video _without_ it being muted? -->
+  <video class="bg-image" id="waka-video" width="1440" height="1080"
+         autoplay loop muted
+         onclick="document.getElementById('waka-video').muted = !document.getElementById('waka-video').muted;"
+  >
+    <!-- from https://www.youtube.com/watch?v=ehB_7bBKprY -->
+    <!-- original and more info at https://www.aquilinestudios.org/wakalaka.html -->
+    <source src="waka.laka.for.osaka.mp4" type="video/mp4">
+  </video>
+</body>
+</html>

hosts/by-name/servo/services/nixos-prebuild.nix

@@ -6,7 +6,7 @@ lib.optionalAttrs false  # disabled until i can be sure it's not gonna OOM my se
     description = "build a nixos image with all updated deps";
     path = with pkgs; [ coreutils git nix ];
     script = ''
-      working=$(mktemp -d /tmp/nixos-prebuild.XXXXXX)
+      working=$(mktemp -d nixos-prebuild.XXXXXX --tmpdir)
       pushd "$working"
       git clone https://git.uninsane.org/colin/nix-files.git \
         && cd nix-files \

hosts/by-name/servo/services/nixserve.nix

@@ -1,21 +0,0 @@
-{ config, ... }:
-
-{
-  services.nginx.virtualHosts."nixcache.uninsane.org" = {
-    addSSL = true;
-    enableACME = true;
-    # inherit kTLS;
-    # serverAliases = [ "nixcache" ];
-    locations."/".extraConfig = ''
-      proxy_pass http://localhost:${toString config.services.nix-serve.port};
-      proxy_set_header Host $host;
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    '';
-  };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."nixcache" = "native";
-
-  sane.services.nixserve.enable = true;
-  sane.services.nixserve.secretKeyFile = config.sops.secrets.nix_serve_privkey.path;
-}

hosts/by-name/servo/services/ntfy/default.nix

@@ -1,12 +1,12 @@
 # ntfy: UnifiedPush notification delivery system
 # - used to get push notifications out of Matrix and onto a Phone (iOS, Android, or a custom client)
-{ config, ... }:
+{ config, lib, ... }:
 {
   imports = [
     ./ntfy-waiter.nix
     ./ntfy-sh.nix
   ];
-  sops.secrets."ntfy-sh-topic" = {
+  sops.secrets."ntfy-sh-topic" = lib.mkIf config.services.ntfy-sh.enable {
     mode = "0440";
     owner = config.users.users.ntfy-sh.name;
     group = config.users.users.ntfy-sh.name;

hosts/by-name/servo/services/ntfy/ntfy-sh.nix

@@ -29,8 +29,9 @@ let
   # at the IP layer, to enable e.g. wake-on-lan.
   altPort = 2587;
 in
+lib.mkIf false  #< 2024/09/30: disabled because i haven't used it in several months
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     # not 100% necessary to persist this, but ntfy does keep a 12hr (by default) cache
     # for pushing notifications to users who become offline.
     # ACLs also live here.
@@ -46,7 +47,7 @@ in
     # defaults to 45s.
     # note that the client may still do its own TCP-level keepalives, typically every 30s
     keepalive-interval = "15m";
-    log-level = "trace";  # trace, debug, info (default), warn, error
+    log-level = "info";  # trace, debug, info (default), warn, error
     auth-default-access = "deny-all";
   };
   systemd.services.ntfy-sh.serviceConfig.DynamicUser = lib.mkForce false;
@@ -58,7 +59,7 @@ in
     # note that this will fail upon first run, i.e. before ntfy has created its db.
     # just restart the service.
     topic=$(cat ${config.sops.secrets.ntfy-sh-topic.path})
-    ${pkgs.ntfy-sh}/bin/ntfy access everyone "$topic" read-write
+    ${lib.getExe' pkgs.ntfy-sh "ntfy"} access everyone "$topic" read-write
   '';
 
 
@@ -86,7 +87,7 @@ in
   sane.ports.ports."${builtins.toString altPort}" = {
     protocol = [ "tcp" ];
     visibleTo.lan = true;
-    visibleTo.wan = true;
+    visibleTo.doof = true;
     description = "colin-ntfy.uninsane.org";
   };
 }

hosts/by-name/servo/services/ntfy/ntfy-waiter

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i python3 -p "python3.withPackages (ps: [  ])" -p ntfy-sh
+#!nix-shell -i python3 -p ntfy-sh -p python3
 
 import argparse
 import logging

hosts/by-name/servo/services/ntfy/ntfy-waiter.nix

@@ -14,7 +14,7 @@ let
     silence = port - portLow;
     flags = lib.optional cfg.verbose "--verbose";
     cli = [
-      "${cfg.package}/bin/ntfy-waiter"
+      (lib.getExe cfg.package)
       "--port"
       "${builtins.toString port}"
       "--silence"
@@ -31,7 +31,7 @@ let
         ExecStart = lib.concatStringsSep " " cli;
       };
       after = [ "network.target" ];
-      wantedBy = [ "default.target" ];
+      wantedBy = [ "ntfy-sh.service" ];
     };
   };
 in
@@ -39,7 +39,7 @@ in
   options = with lib; {
     sane.ntfy-waiter.enable = mkOption {
       type = types.bool;
-      default = true;
+      default = config.services.ntfy-sh.enable;
     };
     sane.ntfy-waiter.verbose = mkOption {
       type = types.bool;
@@ -47,7 +47,7 @@ in
     };
     sane.ntfy-waiter.package = mkOption {
       type = types.package;
-      default = pkgs.static-nix-shell.mkPython3Bin {
+      default = pkgs.static-nix-shell.mkPython3 {
         pname = "ntfy-waiter";
         srcRoot = ./.;
         pkgs = [ "ntfy-sh" ];
@@ -62,8 +62,8 @@ in
     sane.ports.ports = lib.mkMerge (lib.forEach portRange (port: {
       "${builtins.toString port}" = {
         protocol = [ "tcp" ];
+        visibleTo.doof = true;
         visibleTo.lan = true;
-        visibleTo.wan = true;
         description = "colin-notification-waiter-${builtins.toString (port - portLow + 1)}-of-${builtins.toString numPorts}";
       };
     }));

hosts/by-name/servo/services/pleroma.nix

@@ -7,206 +7,216 @@
 # to run it in a oci-container: <https://github.com/barrucadu/nixfiles/blob/master/services/pleroma.nix>
 #
 # admin frontend: <https://fed.uninsane.org/pleroma/admin>
-{ config, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
 let
-  logLevel = "warn";
+  logLevel = "warning";
   # logLevel = "debug";
 in
 {
-  sane.persist.sys.byStore.plaintext = [
-    { user = "pleroma"; group = "pleroma"; path = "/var/lib/pleroma"; method = "bind"; }
-  ];
-  services.pleroma.enable = true;
-  services.pleroma.secretConfigFile = config.sops.secrets.pleroma_secrets.path;
-  services.pleroma.configs = [
-    ''
-    import Config
+  config = lib.mkIf (config.sane.maxBuildCost >= 2) {
+    sane.persist.sys.byStore.private = [
+      # contains media i've uploaded to the server
+      { user = "pleroma"; group = "pleroma"; path = "/var/lib/pleroma"; method = "bind"; }
+    ];
+    services.pleroma.enable = true;
+    services.pleroma.secretConfigFile = config.sops.secrets.pleroma_secrets.path;
+    services.pleroma.configs = [
+      ''
+      import Config
 
-    config :pleroma, Pleroma.Web.Endpoint,
-      url: [host: "fed.uninsane.org", scheme: "https", port: 443],
-      http: [ip: {127, 0, 0, 1}, port: 4040]
-    #   secret_key_base: "{secrets.pleroma.secret_key_base}",
-    #   signing_salt: "{secrets.pleroma.signing_salt}"
+      config :pleroma, Pleroma.Web.Endpoint,
+        url: [host: "fed.uninsane.org", scheme: "https", port: 443],
+        http: [ip: {127, 0, 0, 1}, port: 4040]
+      #   secret_key_base: "{secrets.pleroma.secret_key_base}",
+      #   signing_salt: "{secrets.pleroma.signing_salt}"
 
-    config :pleroma, :instance,
-      name: "Perfectly Sane",
-      description: "Single-user Pleroma instance",
-      email: "admin.pleroma@uninsane.org",
-      notify_email: "notify.pleroma@uninsane.org",
-      limit: 5000,
-      registrations_open: true,
-      account_approval_required: true,
-      max_pinned_statuses: 5,
-      external_user_synchronization: true
+      config :pleroma, :instance,
+        name: "Perfectly Sane",
+        description: "Single-user Pleroma instance",
+        email: "admin.pleroma@uninsane.org",
+        notify_email: "notify.pleroma@uninsane.org",
+        limit: 5000,
+        registrations_open: true,
+        account_approval_required: true,
+        max_pinned_statuses: 5,
+        external_user_synchronization: true
 
-    # docs: https://hexdocs.pm/swoosh/Swoosh.Adapters.Sendmail.html
-    # test mail config with sudo -u pleroma ./bin/pleroma_ctl email test --to someone@somewhere.net
-    config :pleroma, Pleroma.Emails.Mailer,
-      enabled: true,
-      adapter: Swoosh.Adapters.Sendmail,
-      cmd_path: "${pkgs.postfix}/bin/sendmail"
+      # docs: https://hexdocs.pm/swoosh/Swoosh.Adapters.Sendmail.html
+      # test mail config with sudo -u pleroma ./bin/pleroma_ctl email test --to someone@somewhere.net
+      config :pleroma, Pleroma.Emails.Mailer,
+        enabled: true,
+        adapter: Swoosh.Adapters.Sendmail,
+        cmd_path: "${lib.getExe' pkgs.postfix "sendmail"}"
 
-    config :pleroma, Pleroma.User,
-      restricted_nicknames: [ "admin", "uninsane", "root" ]
+      config :pleroma, Pleroma.User,
+        restricted_nicknames: [ "admin", "uninsane", "root" ]
 
-    config :pleroma, :media_proxy,
-      enabled: false,
-      redirect_on_failure: true
-      #base_url: "https://cache.pleroma.social"
+      config :pleroma, :media_proxy,
+        enabled: false,
+        redirect_on_failure: true
+        #base_url: "https://cache.pleroma.social"
 
-    # see for reference:
-    # - `force_custom_plan`: <https://docs.pleroma.social/backend/configuration/postgresql/#disable-generic-query-plans>
-    config :pleroma, Pleroma.Repo,
-      adapter: Ecto.Adapters.Postgres,
-      username: "pleroma",
-      database: "pleroma",
-      hostname: "localhost",
-      pool_size: 10,
-      prepare: :named,
-      parameters: [
-          plan_cache_mode: "force_custom_plan"
-      ]
-    # XXX: prepare: :named is needed only for PG <= 12
-    #   prepare: :named,
-    #   password: "{secrets.pleroma.db_password}",
+      # see for reference:
+      # - `force_custom_plan`: <https://docs.pleroma.social/backend/configuration/postgresql/#disable-generic-query-plans>
+      config :pleroma, Pleroma.Repo,
+        adapter: Ecto.Adapters.Postgres,
+        username: "pleroma",
+        database: "pleroma",
+        hostname: "localhost",
+        pool_size: 10,
+        prepare: :named,
+        parameters: [
+            plan_cache_mode: "force_custom_plan"
+        ]
+      # XXX: prepare: :named is needed only for PG <= 12
+      #   prepare: :named,
+      #   password: "{secrets.pleroma.db_password}",
 
-    # Configure web push notifications
-    config :web_push_encryption, :vapid_details,
-      subject: "mailto:notify.pleroma@uninsane.org"
-    #   public_key: "{secrets.pleroma.vapid_public_key}",
-    #   private_key: "{secrets.pleroma.vapid_private_key}"
+      # Configure web push notifications
+      config :web_push_encryption, :vapid_details,
+        subject: "mailto:notify.pleroma@uninsane.org"
+      #   public_key: "{secrets.pleroma.vapid_public_key}",
+      #   private_key: "{secrets.pleroma.vapid_private_key}"
 
-    # config :joken, default_signer: "{secrets.pleroma.joken_default_signer}"
+      # config :joken, default_signer: "{secrets.pleroma.joken_default_signer}"
 
-    config :pleroma, :database, rum_enabled: false
-    config :pleroma, :instance, static_dir: "/var/lib/pleroma/instance/static"
-    config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
-    config :pleroma, configurable_from_database: false
+      config :pleroma, :database, rum_enabled: false
+      config :pleroma, :instance, static_dir: "/var/lib/pleroma/instance/static"
+      config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+      config :pleroma, configurable_from_database: false
 
-    # strip metadata from uploaded images
-    config :pleroma, Pleroma.Upload, filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
+      # strip metadata from uploaded images
+      config :pleroma, Pleroma.Upload, filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
 
-    # TODO: GET /api/pleroma/captcha is broken
-    # there was a nixpkgs PR to fix this around 2022/10 though.
-    config :pleroma, Pleroma.Captcha,
-      enabled: false,
-      method: Pleroma.Captcha.Native
+      # fix log spam: <https://git.pleroma.social/pleroma/pleroma/-/issues/1659>
+      # specifically, remove LAN addresses from `reserved`
+      config :pleroma, Pleroma.Web.Plugs.RemoteIp,
+        enabled: true,
+        reserved: ["127.0.0.0/8", "::1/128", "fc00::/7", "172.16.0.0/12"]
+
+      # TODO: GET /api/pleroma/captcha is broken
+      # there was a nixpkgs PR to fix this around 2022/10 though.
+      config :pleroma, Pleroma.Captcha,
+        enabled: false,
+        method: Pleroma.Captcha.Native
 
 
-    # (enabled by colin)
-    # Enable Strict-Transport-Security once SSL is working:
-    config :pleroma, :http_security,
-      sts: true
+      # (enabled by colin)
+      # Enable Strict-Transport-Security once SSL is working:
+      config :pleroma, :http_security,
+        sts: true
 
-    # docs: https://docs.pleroma.social/backend/configuration/cheatsheet/#logger
-    config :logger,
-      backends: [{ExSyslogger, :ex_syslogger}]
+      # docs: https://docs.pleroma.social/backend/configuration/cheatsheet/#logger
+      config :logger,
+        backends: [{ExSyslogger, :ex_syslogger}]
 
-    config :logger, :ex_syslogger,
-      level: :${logLevel}
+      config :logger, :ex_syslogger,
+        level: :${logLevel}
 
-    # policies => list of message rewriting facilities to be enabled
-    # transparence => whether to publish these rules in node_info (and /about)
-    config :pleroma, :mrf,
-      policies: [Pleroma.Web.ActivityPub.MRF.SimplePolicy],
-      transparency: true
+      # policies => list of message rewriting facilities to be enabled
+      # transparence => whether to publish these rules in node_info (and /about)
+      config :pleroma, :mrf,
+        policies: [Pleroma.Web.ActivityPub.MRF.SimplePolicy],
+        transparency: true
 
-    # reject => { host, reason }
-    config :pleroma, :mrf_simple,
-      reject: [ {"threads.net", "megacorp"}, {"*.threads.net", "megacorp"} ]
-      # reject: [ [host: "threads.net", reason: "megacorp"], [host: "*.threads.net", reason: "megacorp"] ]
+      # reject => { host, reason }
+      config :pleroma, :mrf_simple,
+        reject: [ {"threads.net", "megacorp"}, {"*.threads.net", "megacorp"} ]
+        # reject: [ [host: "threads.net", reason: "megacorp"], [host: "*.threads.net", reason: "megacorp"] ]
 
-    # XXX colin: not sure if this actually _does_ anything
-    # better to steal emoji from other instances?
-    # - <https://docs.pleroma.social/backend/configuration/cheatsheet/#mrf_steal_emoji>
-    config :pleroma, :emoji,
-      shortcode_globs: ["/emoji/**/*.png"],
-      groups: [
-        "Cirno": "/emoji/cirno/*.png",
-        "Kirby": "/emoji/kirby/*.png",
-        "Bun": "/emoji/bun/*.png",
-        "Yuru Camp": "/emoji/yuru_camp/*.png",
-      ]
-    ''
-  ];
+      # XXX colin: not sure if this actually _does_ anything
+      # better to steal emoji from other instances?
+      # - <https://docs.pleroma.social/backend/configuration/cheatsheet/#mrf_steal_emoji>
+      config :pleroma, :emoji,
+        shortcode_globs: ["/emoji/**/*.png"],
+        groups: [
+          "Cirno": "/emoji/cirno/*.png",
+          "Kirby": "/emoji/kirby/*.png",
+          "Bun": "/emoji/bun/*.png",
+          "Yuru Camp": "/emoji/yuru_camp/*.png",
+        ]
+      ''
+    ];
 
-  systemd.services.pleroma.path = [
-    # something inside pleroma invokes `sh` w/o specifying it by path, so this is needed to allow pleroma to start
-    pkgs.bash
-    # used by Pleroma to strip geo tags from uploads
-    pkgs.exiftool
-    # i saw some errors when pleroma was shutting down about it not being able to find `awk`. probably not critical
-    pkgs.gawk
-    # needed for email operations like password reset
-    pkgs.postfix
-  ];
+    systemd.services.pleroma.path = [
+      # something inside pleroma invokes `sh` w/o specifying it by path, so this is needed to allow pleroma to start
+      pkgs.bash
+      # used by Pleroma to strip geo tags from uploads
+      pkgs.exiftool
+      # config.sane.programs.exiftool.package  #< XXX(2024-10-20): breaks image uploading
+      # i saw some errors when pleroma was shutting down about it not being able to find `awk`. probably not critical
+      # config.sane.programs.gawk.package
+      # needed for email operations like password reset
+      pkgs.postfix
+    ];
 
-  systemd.services.pleroma.serviceConfig = {
-    # postgres can be slow to service early requests, preventing pleroma from starting on the first try
-    Restart = "on-failure";
-    RestartSec = "10s";
-  };
+    systemd.services.pleroma = {
+      # postgres can be slow to service early requests, preventing pleroma from starting on the first try
+      serviceConfig.Restart = "on-failure";
+      serviceConfig.RestartSec = "10s";
 
-  # systemd.services.pleroma.serviceConfig = {
-  #   # required for sendmail. see https://git.pleroma.social/pleroma/pleroma/-/issues/2259
-  #   NoNewPrivileges = lib.mkForce false;
-  #   PrivateTmp = lib.mkForce false;
-  #   CapabilityBoundingSet = lib.mkForce "~";
-  # };
+      # hardening (systemd-analyze security pleroma)
+      # XXX(2024-07-28): this hasn't been rigorously tested:
+      # possible that i've set something too strict and won't notice right away
+      # make sure to test:
+      # - image/media uploading
+      serviceConfig.CapabilityBoundingSet = lib.mkForce [ "" "" ];  # nixos default is `~CAP_SYS_ADMIN`
+      serviceConfig.LockPersonality = true;
+      serviceConfig.NoNewPrivileges = true;
+      serviceConfig.MemoryDenyWriteExecute = true;
+      serviceConfig.PrivateDevices = lib.mkForce true;  #< dunno why nixpkgs has this set false; it seems to work as true
+      serviceConfig.PrivateMounts = true;
+      serviceConfig.PrivateTmp = true;
+      serviceConfig.PrivateUsers = true;
 
-  # this is required to allow pleroma to send email.
-  # raw `sendmail` works, but i think pleroma's passing it some funny flags or something, idk.
-  # hack to fix that.
-  users.users.pleroma.extraGroups = [ "postdrop" ];
+      serviceConfig.ProtectProc = "invisible";
+      serviceConfig.ProcSubset = "all";  #< needs /proc/sys/kernel/overflowuid for bwrap
 
-  # Pleroma server and web interface
-  # TODO: enable publog?
-  services.nginx.virtualHosts."fed.uninsane.org" = {
-    forceSSL = true;  # pleroma redirects to https anyway
-    enableACME = true;
-    # inherit kTLS;
-    locations."/" = {
-      proxyPass = "http://127.0.0.1:4040";
-      recommendedProxySettings = true;
-      # documented: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/installation/pleroma.nginx
-      extraConfig = ''
-        # XXX colin: this block is in the nixos examples: i don't understand all of it
-        add_header 'Access-Control-Allow-Origin' '*' always;
-        add_header 'Access-Control-Allow-Methods' 'POST, PUT, DELETE, GET, PATCH, OPTIONS' always;
-        add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, Idempotency-Key' always;
-        add_header 'Access-Control-Expose-Headers' 'Link, X-RateLimit-Reset, X-RateLimit-Limit, X-RateLimit-Remaining, X-Request-Id' always;
-        if ($request_method = OPTIONS) {
-            return 204;
-        }
+      serviceConfig.ProtectClock = true;
+      serviceConfig.ProtectControlGroups = true;
+      serviceConfig.ProtectHome = true;
+      serviceConfig.ProtectKernelModules = true;
+      serviceConfig.ProtectSystem = lib.mkForce "strict";
+      serviceConfig.RemoveIPC = true;
+      serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
 
-        add_header X-XSS-Protection "1; mode=block";
-        add_header X-Permitted-Cross-Domain-Policies none;
-        add_header X-Frame-Options DENY;
-        add_header X-Content-Type-Options nosniff;
-        add_header Referrer-Policy same-origin;
-        add_header X-Download-Options noopen;
+      serviceConfig.RestrictSUIDSGID = true;
+      serviceConfig.SystemCallArchitectures = "native";
+      serviceConfig.SystemCallFilter = [ "@system-service" "@mount" "@sandbox" ];  #< "sandbox" might not actually be necessary
 
-        # proxy_http_version 1.1;
-        # proxy_set_header Upgrade $http_upgrade;
-        # proxy_set_header Connection "upgrade";
-        # # proxy_set_header Host $http_host;
-        # proxy_set_header Host $host;
-        # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      serviceConfig.ProtectHostname = false;  #< else brap can't mount /proc
+      serviceConfig.ProtectKernelLogs = false;  #< else breaks exiftool  ("bwrap: Can't mount proc on /newroot/proc: Operation not permitted")
+      serviceConfig.ProtectKernelTunables = false;  #< else breaks exiftool
+      serviceConfig.RestrictNamespaces = false;  # media uploads require bwrap
+    };
 
-        # colin: added this due to Pleroma complaining in its logs
-        # proxy_set_header X-Real-IP $remote_addr;
-        # proxy_set_header X-Forwarded-Proto $scheme;
+    # this is required to allow pleroma to send email.
+    # raw `sendmail` works, but i think pleroma's passing it some funny flags or something, idk.
+    # hack to fix that.
+    users.users.pleroma.extraGroups = [ "postdrop" ];
 
-        # NB: this defines the maximum upload size
-        client_max_body_size 16m;
-      '';
+    # Pleroma server and web interface
+    # TODO: enable publog?
+    services.nginx.virtualHosts."fed.uninsane.org" = {
+      forceSSL = true;  # pleroma redirects to https anyway
+      enableACME = true;
+      # inherit kTLS;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:4040";
+        recommendedProxySettings = true;
+        # documented: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/installation/pleroma.nginx
+        extraConfig = ''
+          # client_max_body_size defines the maximum upload size
+          client_max_body_size 16m;
+        '';
+      };
+    };
+
+    sane.dns.zones."uninsane.org".inet.CNAME."fed" = "native";
+
+    sops.secrets."pleroma_secrets" = {
+      owner = config.users.users.pleroma.name;
     };
   };
-
-  sane.dns.zones."uninsane.org".inet.CNAME."fed" = "native";
-
-  sops.secrets."pleroma_secrets" = {
-    owner = config.users.users.pleroma.name;
-  };
 }

hosts/by-name/servo/services/postgresql/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ lib, pkgs, ... }:
 
 let
   GiB = n: MiB 1024*n;
@@ -6,9 +6,9 @@ let
   KiB = n: 1024*n;
 in
 {
-  sane.persist.sys.byStore.plaintext = [
-    # TODO: mode?
-    { user = "postgres"; group = "postgres"; path = "/var/lib/postgresql"; method = "bind"; }
+  sane.persist.sys.byStore.private = [
+    { user = "postgres"; group = "postgres"; mode = "0750"; path = "/var/lib/postgresql"; method = "bind"; }
+    { user = "postgres"; group = "postgres"; mode = "0750"; path = "/var/backup/postgresql"; method = "bind"; }
   ];
   services.postgresql.enable = true;
 
@@ -29,9 +29,10 @@ in
   # - as `sudo su postgres`:
   #   - `cd /var/lib/postgreql`
   #   - `psql -f state.sql`
+  #     (for a compressed dump: `gunzip --stdout state.sql.gz | psql`)
   # - restart dependent services (maybe test one at a time)
 
-  services.postgresql.package = pkgs.postgresql_15;
+  services.postgresql.package = pkgs.postgresql_16;
 
 
   # XXX colin: for a proper deploy, we'd want to include something for Pleroma here too.
@@ -44,34 +45,46 @@ in
   #     LC_CTYPE = "C";
   # '';
 
-  # perf tuning
-  # - for recommended values see: <https://pgtune.leopard.in.ua/>
-  # - for official docs (sparse), see: <https://www.postgresql.org/docs/11/config-setting.html#CONFIG-SETTING-CONFIGURATION-FILE>
   services.postgresql.settings = {
-    # DB Version: 15
+    # perf tuning
+    # - for recommended values see: <https://pgtune.leopard.in.ua/>
+    # - for official docs (sparse), see: <https://www.postgresql.org/docs/11/config-setting.html#CONFIG-SETTING-CONFIGURATION-FILE>
+    # DB Version: 16
     # OS Type: linux
     # DB Type: web
-    # Total Memory (RAM): 32 GB
+    # vvv artificially constrained because the server's resources are shared across maaany services
+    # Total Memory (RAM): 12 GB
     # CPUs num: 12
     # Data Storage: ssd
     max_connections = 200;
-    shared_buffers = "8GB";
-    effective_cache_size = "24GB";
-    maintenance_work_mem = "2GB";
+    shared_buffers = "3GB";
+    effective_cache_size = "9GB";
+    maintenance_work_mem = "768MB";
     checkpoint_completion_target = 0.9;
     wal_buffers = "16MB";
     default_statistics_target = 100;
     random_page_cost = 1.1;
     effective_io_concurrency = 200;
-    work_mem = "10485kB";
+    work_mem = "3932kB";
     min_wal_size = "1GB";
     max_wal_size = "4GB";
     max_worker_processes = 12;
     max_parallel_workers_per_gather = 4;
     max_parallel_workers = 12;
     max_parallel_maintenance_workers = 4;
+
+    # DEBUG OPTIONS:
+    log_min_messages = "DEBUG1";
   };
 
+  # regulate the restarts, so that systemd never disables it
+  systemd.services.postgresql.serviceConfig.Restart = lib.mkForce "on-failure";
+  systemd.services.postgresql.serviceConfig.RestartSec = 2;
+  systemd.services.postgresql.serviceConfig.RestartMaxDelaySec = 10;
+  systemd.services.postgresql.serviceConfig.RestartSteps = 4;
+  systemd.services.postgresql.serviceConfig.StartLimitBurst = 120;
+  # systemd.services.postgresql.serviceConfig.TimeoutStartSec = "14400s";  #< 14400 = 4 hours; recoveries are long
+
   # daily backups to /var/backup
   services.postgresqlBackup.enable = true;
 

hosts/by-name/servo/services/postgresql/recollate.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+# source: <https://gist.githubusercontent.com/troykelly/616df024050dd50744dde4a9579e152e/raw/fe84e53cedf0caa6903604894454629a15867439/reindex_and_refresh_collation.sh>
+#
+# run this whenever postgres complains like:
+# > WARNING:  database "gitea" has a collation version mismatch
+# > DETAIL:  The database was created using collation version 2.39, but the operating system provides version 2.40.
+# > HINT:  Rebuild all objects in this database that use the default collation and run ALTER DATABASE gitea REFRESH COLLATION VERSION, or build PostgreSQL with the right library version.
+#
+# this script checks which databases are in need of a collation update,
+# and re-collates them as appropriate.
+# invoking this script should have low perf impact in the non-upgrade case,
+# so safe to do this as a cron job.
+#
+# invoke as postgres user
+
+log_info() {
+  >&2 echo "$@"
+}
+
+list_databases() {
+  log_info "Retrieving list of databases from the PostgreSQL server..."
+  psql --dbname="postgres" -Atc \
+    "SELECT datname FROM pg_database WHERE datistemplate = false"
+}
+
+refresh_collation_version() {
+  local db=$1
+  log_info "Refreshing collation version for database: $db..."
+  psql --dbname="$db" -c \
+    "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION;"
+}
+
+check_collation_mismatches() {
+  local error=
+  log_info "Checking for collation mismatches in all databases..."
+  # Loop through each database and check for mismatching collations in table columns.
+  while IFS= read -r db; do
+    if [ -n "$db" ]; then
+      log_info "Checking database: $db for collation mismatches..."
+      local mismatches=$(psql --dbname="$db" -Atc \
+        "SELECT 'Mismatch in table ' || table_name || ' column ' || column_name || ' with collation ' || collation_name
+         FROM information_schema.columns
+         WHERE collation_name IS NOT NULL AND collation_name <> 'default' AND table_schema = 'public'
+         EXCEPT
+         SELECT 'No mismatch - default collation of ' || datcollate || ' used.'
+         FROM pg_database WHERE datname = '$db';"
+      )
+      if [ -z "$mismatches" ]; then
+        log_info "No collation mismatches found in database: $db"
+      else
+        # Print an informational message to stderr.
+        log_info "Collation mismatches found in database: $db:"
+        log_info "$mismatches"
+        error=1
+      fi
+    fi
+  done
+
+  if [ -n "$error" ]; then
+    exit 1
+  fi
+}
+
+log_info "Starting the reindexing and collation refresh process for all databases..."
+
+databases=$(list_databases)
+
+if [ -z "$databases" ]; then
+  log_info "No databases found for reindexing or collation refresh. Please check connection details to PostgreSQL server."
+  exit 1
+fi
+
+for db in $databases; do
+  refresh_collation_version "$db"
+done
+
+# Checking for collation mismatches after reindexing and collation refresh.
+# Pass the list of databases to the check_collation_mismatches function through stdin.
+echo "$databases" | check_collation_mismatches
+
+log_info "Reindexing and collation refresh process completed."

hosts/by-name/servo/services/prosody/default.nix

@@ -49,60 +49,62 @@
 # - disable or fix bosh (jabber over http):
 #   - "certmanager: No certificate/key found for client_https port 0"
 
-{ lib, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
 let
   # enables very verbose logging
   enableDebug = false;
 in
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
+    # TODO: mode?
     { user = "prosody"; group = "prosody"; path = "/var/lib/prosody"; method = "bind"; }
   ];
   sane.ports.ports."5000" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-xmpp-prosody-fileshare-proxy65";
   };
   sane.ports.ports."5222" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-xmpp-client-to-server";
   };
   sane.ports.ports."5223" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-xmpps-client-to-server";  # XMPP over TLS
   };
   sane.ports.ports."5269" = {
     protocol = [ "tcp" ];
-    visibleTo.wan = true;
+    visibleTo.doof = true;
     description = "colin-xmpp-server-to-server";
   };
   sane.ports.ports."5270" = {
     protocol = [ "tcp" ];
-    visibleTo.wan = true;
+    visibleTo.doof = true;
     description = "colin-xmpps-server-to-server";  # XMPP over TLS
   };
   sane.ports.ports."5280" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-xmpp-bosh";
   };
   sane.ports.ports."5281" = {
     protocol = [ "tcp" ];
+    visibleTo.doof = true;
     visibleTo.lan = true;
-    visibleTo.wan = true;
     description = "colin-xmpp-prosody-https";  # necessary?
   };
 
   users.users.prosody.extraGroups = [
     "nginx"  # provide access to certs
     "ntfy-sh"  # access to secret ntfy topic
+    "turnserver"  # to access the coturn shared secret
   ];
 
   security.acme.certs."uninsane.org".extraDomainNames = [
@@ -148,14 +150,8 @@ in
   # pointing it to /var/lib/acme doesn't quite work because it expects the private key
   # to be named `privkey.pem` instead of acme's `key.pem`
   # <https://prosody.im/doc/certificates#automatic_location>
-  sane.fs."/etc/prosody/certs/uninsane.org/fullchain.pem" = {
-    symlink.target = "/var/lib/acme/uninsane.org/fullchain.pem";
-    wantedBeforeBy = [ "prosody.service" ];
-  };
-  sane.fs."/etc/prosody/certs/uninsane.org/privkey.pem" = {
-    symlink.target = "/var/lib/acme/uninsane.org/key.pem";
-    wantedBeforeBy = [ "prosody.service" ];
-  };
+  environment.etc."prosody/certs/uninsane.org/fullchain.pem".source = "/var/lib/acme/uninsane.org/fullchain.pem";
+  environment.etc."prosody/certs/uninsane.org/privkey.pem".source = "/var/lib/acme/uninsane.org/key.pem";
 
   services.prosody = {
     enable = true;
@@ -177,7 +173,7 @@ in
         domain = "conference.xmpp.uninsane.org";
       }
     ];
-    uploadHttp.domain = "upload.xmpp.uninsane.org";
+    httpFileShare.domain = "upload.xmpp.uninsane.org";
 
     virtualHosts = {
       # "Prosody requires at least one enabled VirtualHost to function. You can
@@ -241,6 +237,7 @@ in
       # legacy coturn integration
       # see: <https://modules.prosody.im/mod_turncredentials.html>
       # "turncredentials"
+    ] ++ lib.optionals config.services.ntfy-sh.enable [
       "sane_ntfy"
     ] ++ lib.optionals enableDebug [
       "stanza_debug"  #< logs EVERY stanza as debug: <https://prosody.im/doc/modules/mod_stanza_debug>
@@ -272,18 +269,35 @@ in
       s2s_direct_tls_ports = { 5270 }
 
       turn_external_host = "turn.uninsane.org"
-      turn_external_secret = readAll("/var/lib/coturn/shared_secret.bin")
+      turn_external_secret = readAll("/run/secrets/coturn_shared_secret")
       -- turn_external_user = "prosody"
 
       -- legacy mod_turncredentials integration
       -- turncredentials_host = "turn.uninsane.org"
-      -- turncredentials_secret = readAll("/var/lib/coturn/shared_secret.bin")
-
-      ntfy_binary = "${pkgs.ntfy-sh}/bin/ntfy"
-      ntfy_topic = readAll("/run/secrets/ntfy-sh-topic")
+      -- turncredentials_secret = readAll("/run/secrets/coturn_shared_secret")
 
       -- s2s_require_encryption = true
       -- c2s_require_encryption = true
+    '' + lib.optionalString config.services.ntfy-sh.enable ''
+      ntfy_binary = "${lib.getExe' pkgs.ntfy-sh "ntfy"}"
+      ntfy_topic = readAll("/run/secrets/ntfy-sh-topic")
     '';
+    checkConfig = false;  # secrets aren't available at build time
+  };
+
+  systemd.services.prosody = {
+    # hardening (systemd-analyze security prosody)
+    serviceConfig.LockPersonality = true;
+    serviceConfig.NoNewPrivileges = true;
+    serviceConfig.PrivateUsers = true;
+    serviceConfig.ProcSubset = "pid";
+    serviceConfig.ProtectClock = true;
+    serviceConfig.ProtectKernelLogs = true;
+    serviceConfig.ProtectProc = "invisible";
+    serviceConfig.ProtectSystem = "strict";
+    serviceConfig.RemoveIPC = true;
+    serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+    serviceConfig.SystemCallArchitectures = "native";
+    serviceConfig.SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
   };
 }

hosts/by-name/servo/services/slskd.nix

@@ -9,10 +9,10 @@
 #   - "Soulseek.AddressException: Failed to resolve address 'vps.slsknet.org': Resource temporarily unavailable"
 { config, lib, pkgs, ... }:
 
-# TODO: re-enable once i'm satisfied this isn't escaping the net sandbox
-lib.mkIf false
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.ephemeral = [
+    # {data,downloads,incomplete,logs}: contains logs, search history, and downloads
+    # so, move the downloaded data to persistent storage regularly, or configure the downloads/incomplete dirs to point to persisted storage (in nixpkgs slskd config)
     { user = "slskd"; group = "media"; path = "/var/lib/slskd"; method = "bind"; }
   ];
   sops.secrets."slskd_env" = {
@@ -24,8 +24,7 @@ lib.mkIf false
 
   sane.ports.ports."50300" = {
     protocol = [ "tcp" ];
-    # not visible to WAN: i run this in a separate netns
-    visibleTo.ovpn = true;
+    # visibleTo.ovpns = true;  #< not needed: it runs in the ovpns namespace
     description = "colin-soulseek";
   };
 
@@ -35,8 +34,9 @@ lib.mkIf false
     forceSSL = true;
     enableACME = true;
     locations."/" = {
-      proxyPass = "http://10.0.1.6:5030";
+      proxyPass = "http://${config.sane.netns.ovpns.veth.netns.ipv4}:5030";
       proxyWebsockets = true;
+      recommendedProxySettings = true;
     };
   };
 
@@ -71,12 +71,23 @@ lib.mkIf false
     # flags.volatile = true;  # store searches and active transfers in RAM (completed transfers still go to disk). rec for btrfs/zfs
   };
 
-  systemd.services.slskd.serviceConfig = {
+  systemd.services.slskd = {
     # run this behind the OVPN static VPN
-    NetworkNamespacePath = "/run/netns/ovpns";
-    ExecStartPre = [ "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect 185.157.162.178" ];  # abort if public IP is not as expected
+    serviceConfig.NetworkNamespacePath = "/run/netns/ovpns";
+    serviceConfig.ExecStartPre = [
+      # abort if public IP is not as expected
+      "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect ${config.sane.netns.ovpns.wg.address.ipv4}"
+    ];
 
-    Restart = lib.mkForce "always";  # exits "success" when it fails to connect to soulseek server
-    RestartSec = "60s";
+    serviceConfig.Restart = lib.mkForce "always";  # exits "success" when it fails to connect to soulseek server
+    serviceConfig.RestartSec = "60s";
+
+    # hardening (systemd-analyze security slskd)
+    # upstream nixpkgs specifies moderate defaults; these are supplementary
+    # serviceConfig.MemoryDenyWriteExecute = true;
+    # serviceConfig.ProcSubset = "pid";
+    # serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+    # serviceConfig.SystemCallArchitectures = "native";
+    # serviceConfig.SystemCallFilter = [ "@system-service" ];
   };
 }

hosts/by-name/servo/services/transmission/default.nix

@@ -22,70 +22,23 @@ let
         --replace-fail 'set(TR_USER_AGENT_PREFIX "''${TR_SEMVER}")' 'set(TR_USER_AGENT_PREFIX "3.00")'
     '';
   });
-  download-dir = "/var/media/torrents";
-  torrent-done = pkgs.writeShellApplication {
-    name = "torrent-done";
-    runtimeInputs = with pkgs; [
-      acl
-      coreutils
-      findutils
-      rsync
-      util-linux
+  download-dir = "/var/media/torrents";  #< keep in sync with consts embedded in `torrent-done`
+  torrent-done = pkgs.static-nix-shell.mkBash {
+    pname = "torrent-done";
+    srcRoot = ./.;
+    pkgs = [
+      "acl"
+      "coreutils"
+      "findutils"
+      "rsync"
     ];
-    text = ''
-      destructive() {
-        if [ -n "''${TR_DRY_RUN-}" ]; then
-          echo "$*"
-        else
-          "$@"
-        fi
-      }
-      if [[ "$TR_TORRENT_DIR" =~ ^.*freeleech.*$ ]]; then
-        # freeleech torrents have no place in my permanent library
-        echo "freeleech: nothing to do"
-        exit 0
-      fi
-      if ! [[ "$TR_TORRENT_DIR" =~ ^${download-dir}/.*$ ]]; then
-        echo "unexpected torrent dir, aborting: $TR_TORRENT_DIR"
-        exit 0
-      fi
-
-      REL_DIR="''${TR_TORRENT_DIR#${download-dir}/}"
-      MEDIA_DIR="/var/media/$REL_DIR"
-
-      destructive mkdir -p "$(dirname "$MEDIA_DIR")"
-      destructive rsync -arv "$TR_TORRENT_DIR/" "$MEDIA_DIR/"
-      # make the media rwx by anyone in the group
-      destructive find "$MEDIA_DIR" -type d -exec setfacl --recursive --modify d:g::rwx,o::rx {} \;
-      destructive find "$MEDIA_DIR" -type d -exec chmod g+rw,a+rx {} \;
-
-      # if there's a single directory inside the media dir, then inline that
-      subdirs=("$MEDIA_DIR"/*)
-      if [ ''${#subdirs} -eq 1 ]; then
-        dirname="''${subdirs[0]}"
-        if [ -d "$dirname" ]; then
-          mv "$dirname"/* "$MEDIA_DIR/" && rmdir "$dirname"
-        fi
-      fi
-
-      # remove noisy files:
-      find "$MEDIA_DIR/" -type f \(\
-           -iname 'www.YTS.*.jpg' \
-        -o -iname 'WWW.YIFY*.COM.jpg' \
-        -o -iname 'YIFY*.com.txt' \
-        -o -iname 'YTS*.com.txt' \
-        \) -exec rm {} \;
-
-      # dedupe the whole media library.
-      # yeah, a bit excessive: move this to a cron job if that's problematic.
-      destructive hardlink /var/media --reflink=always --ignore-time --verbose
-    '';
   };
 in
 {
-  sane.persist.sys.byStore.plaintext = [
+  sane.persist.sys.byStore.private = [
     # TODO: mode? we need this specifically for the stats tracking in .config/
     { user = "transmission"; group = config.users.users.transmission.group; path = "/var/lib/transmission"; method = "bind"; }
+    { user = "transmission"; group = config.users.users.transmission.group; path = "/var/backup/torrents"; method = "bind"; }
   ];
   users.users.transmission.extraGroups = [ "media" ];
 
@@ -105,8 +58,8 @@ in
     # DOCUMENTATION/options list: <https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md#options>
 
     # message-level = 3;  #< enable for debug logging. 0-3, default is 2.
-    # 10.0.1.6 => allow rpc only from the root servo ns. it'll tunnel things to the net, if need be.
-    rpc-bind-address = "10.0.1.6";
+    # ovpns.veth.netns.ipv4 => allow rpc only from the root servo ns. it'll tunnel things to the net, if need be.
+    rpc-bind-address = config.sane.netns.ovpns.veth.netns.ipv4;
     #rpc-host-whitelist = "bt.uninsane.org";
     #rpc-whitelist = "*.*.*.*";
     rpc-authentication-required = true;
@@ -117,7 +70,7 @@ in
     rpc-whitelist-enabled = false;
 
     # force behind ovpns in case the NetworkNamespace fails somehow
-    bind-address-ipv4 = "185.157.162.178";
+    bind-address-ipv4 = config.sane.netns.ovpns.wg.address.ipv4;
     port-forwarding-enabled = false;
 
     # hopefully, make the downloads world-readable
@@ -151,26 +104,42 @@ in
     # - TR_TORRENT_NAME - Name of torrent (not filename)
     # - TR_TORRENT_TRACKERS - A comma-delimited list of the torrent's trackers' announce URLs
     script-torrent-done-enabled = true;
-    script-torrent-done-filename = "${torrent-done}/bin/torrent-done";
+    script-torrent-done-filename = lib.getExe torrent-done;
   };
 
-  systemd.services.transmission.after = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.transmission.partOf = [ "wireguard-wg-ovpns.service" ];
-  systemd.services.transmission.serviceConfig = {
-    # run this behind the OVPN static VPN
-    NetworkNamespacePath = "/run/netns/ovpns";
-    ExecStartPre = [ "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect 185.157.162.178" ];  # abort if public IP is not as expected
+  # run this behind the OVPN static VPN
+  sane.netns.ovpns.services = [ "transmission" ];
+  systemd.services.transmission = {
+    environment.TR_DEBUG = "1";
+    serviceConfig.ExecStartPre = [
+      # abort if public IP is not as expected
+      "${lib.getExe pkgs.sane-scripts.ip-check} --no-upnp --expect ${config.sane.netns.ovpns.wg.address.ipv4}"
+    ];
 
-    Restart = "on-failure";
-    RestartSec = "30s";
-    BindPaths = [ "/var/media" ];  #< so it can move completed torrents into the media library
+    serviceConfig.Restart = "on-failure";
+    serviceConfig.RestartSec = "30s";
+    serviceConfig.BindPaths = [ "/var/media" ];  #< so it can move completed torrents into the media library
+    serviceConfig.SystemCallFilter = lib.mkForce [
+      # the torrent-done script does stuff which fails the nixos default syscall filter.
+      # allow a bunch of stuff, speculatively, to hopefully fix that:
+      "@aio"
+      "@basic-io"
+      "@chown"
+      "@file-system"
+      "@io-event"
+      "@process"
+      "@sandbox"
+      "@sync"
+      "@system-service"
+      "quotactl"
+    ];
   };
 
   # service to automatically backup torrents i add to transmission
   systemd.services.backup-torrents = {
     description = "archive torrents to storage not owned by transmission";
     script = ''
-      ${pkgs.rsync}/bin/rsync -arv /var/lib/transmission/.config/transmission-daemon/torrents/ /var/backup/torrents/
+      ${lib.getExe pkgs.rsync} -arv /var/lib/transmission/.config/transmission-daemon/torrents/ /var/backup/torrents/
     '';
   };
   systemd.timers.backup-torrents = {
@@ -189,14 +158,15 @@ in
     # inherit kTLS;
     locations."/" = {
       # proxyPass = "http://ovpns.uninsane.org:9091";
-      proxyPass = "http://10.0.1.6:9091";
+      proxyPass = "http://${config.sane.netns.ovpns.veth.netns.ipv4}:9091";
+      recommendedProxySettings = true;
     };
   };
 
   sane.dns.zones."uninsane.org".inet.CNAME."bt" = "native";
   sane.ports.ports."51413" = {
     protocol = [ "tcp" "udp" ];
-    visibleTo.ovpn = true;
+    # visibleTo.ovpns = true;  #< not needed: it runs in the ovpns namespace
     description = "colin-bittorrent";
   };
 }

hosts/by-name/servo/services/transmission/torrent-done

@@ -0,0 +1,111 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p acl -p bash -p coreutils -p findutils -p rsync
+
+# transmission invokes this with no args, and the following env vars:
+# - TR_TORRENT_DIR: full path to the folder i told transmission to download it to.
+#     e.g. "/var/media/torrents/Videos/Film/Jason.Bourne-2016"
+# - TR_APP_VERSION
+# - TR_TIME_LOCALTIME
+# - TR_TORRENT_BYTES_DOWNLOADED
+# - TR_TORRENT_HASH
+# - TR_TORRENT_ID: local number to uniquely identify this torrent, used by e.g. transmission-remote.
+#     e.g. "67"
+# - TR_TORRENT_LABELS
+# - TR_TORRENT_NAME: file/folder name of the toplevel torrent item
+#     e.g. "Jason Bourne (2016) [2160p] [4K] [BluRay] [5.1] [YTS.MX]"
+# - TR_TORRENT_PRIORITY
+# - TR_TORRENT_TRACKERS
+
+# optionally, set these variables for debugging (these are specific to my script and not used upstream):
+# - TR_DRY_RUN=1
+# - TR_DEBUG=1
+
+DOWNLOAD_DIR=/var/media/torrents
+
+destructive() {
+  if [ -n "${TR_DRY_RUN-}" ]; then
+    echo "[dry-run] $*"
+  else
+    debug "$@"
+    "$@"
+  fi
+}
+debug() {
+  if [ -n "${TR_DEBUG-}" ]; then
+    echo "$@"
+  fi
+}
+
+echo "TR_TORRENT_DIR=$TR_TORRENT_DIR TR_TORRENT_NAME=$TR_TORRENT_NAME torrent-done $*"
+
+if [[ "$TR_TORRENT_DIR" =~ ^.*freeleech.*$ ]]; then
+  # freeleech torrents have no place in my permanent library
+  echo "freeleech: nothing to do"
+  exit 0
+fi
+if ! [[ "$TR_TORRENT_DIR" =~ ^$DOWNLOAD_DIR/.*$ ]]; then
+  echo "unexpected torrent dir, aborting: $TR_TORRENT_DIR"
+  exit 1
+fi
+
+TORRENT_PATH="$TR_TORRENT_DIR/$TR_TORRENT_NAME"
+if [[ ! -e "$TORRENT_PATH" ]]; then
+  echo "torrent unexpectedly doesn't exist at $TORRENT_PATH. will try fallback"
+  TORRENT_PATH="$TR_TORRENT_DIR"
+fi
+
+if [[ -d "$TORRENT_PATH" ]]; then
+  # trailing slash so that rsync copies the directory contents, without creating an extra toplevel dir.
+  TORRENT_PATH="$TORRENT_PATH/"
+elif [[ ! -e "$TORRENT_PATH" ]]; then
+  echo "torrent unexpectedly doesn't exist at TR_TORRENT_DIR=$TORRENT_PATH: bailing"
+  exit 1
+fi
+
+REL_DIR="${TR_TORRENT_DIR#$DOWNLOAD_DIR/}"
+MEDIA_DIR="/var/media/$REL_DIR"
+
+destructive mkdir -p "$(dirname "$MEDIA_DIR")"
+destructive rsync -rlv "$TORRENT_PATH" "$MEDIA_DIR/"
+# make the media rwx by anyone in the group
+destructive find "$MEDIA_DIR" -type d -exec setfacl --recursive --modify d:g::rwx,o::rx {} \;
+destructive find "$MEDIA_DIR" -type d -exec chmod g+rw,a+rx {} \;
+destructive find "$MEDIA_DIR" -type f -exec chmod g+rw,a+r {} \;
+
+# if there's a single directory inside the media dir, then inline that.
+# TODO: this is probably obsolete now that i process TR_TORRENT_NAME
+subdirs=("$MEDIA_DIR"/*)
+debug "top-level items in torrent dir:" "${subdirs[@]}"
+if [ ${#subdirs[@]} -eq 1 ]; then
+  dirname="${subdirs[0]}"
+  debug "exactly one top-level item, checking if directory: $dirname"
+  if [ -d "$dirname" ]; then
+    destructive mv "$dirname"/* "$MEDIA_DIR/" && destructive rmdir "$dirname"
+  fi
+fi
+
+# remove noisy files:
+# -iname means "insensitive", but the syntax is NOT regex -- more similar to shell matching
+destructive find "$MEDIA_DIR/" -type f \(\
+     -iname '*downloaded?from*' \
+  -o -iname '(xxxpav69).txt' \
+  -o -iname '*upcoming?releases*' \
+  -o -iname 'ETRG.mp4' \
+  -o -iname 'Encoded by*.txt' \
+  -o -iname 'PSArips.com.txt' \
+  -o -iname 'RARBG.com*' \
+  -o -iname 'RARBG.txt' \
+  -o -iname 'RARBG_DO_NOT_MIRROR.exe' \
+  -o -iname 'Tellytorrent.net.txt' \
+  -o -iname 'WWW.VPPV.LA.txt' \
+  -o -iname 'WWW.YIFY*.COM.jpg' \
+  -o -iname 'YIFY*.com.txt' \
+  -o -iname 'YTS*.com.txt' \
+  -o -iname 'YTSYify*.txt' \
+  -o -iname 'www.YTS*.jpg' \
+  \) -exec rm {} \;
+
+# might want to keep, might want to remove:
+# -o -iname 'info.txt'
+# -o -iname 'source.txt'
+# -o -iname 'sample.mkv'

hosts/by-name/servo/services/trust-dns.nix

@@ -1,159 +0,0 @@
-# TODO: split this file apart into smaller files to make it easier to understand
-{ config, lib, pkgs, ... }:
-
-let
-  dyn-dns = config.sane.services.dyn-dns;
-  nativeAddrs = lib.mapAttrs (_name: builtins.head) config.sane.dns.zones."uninsane.org".inet.A;
-  bindOvpn = "10.0.1.5";
-in
-{
-  sane.ports.ports."53" = {
-    protocol = [ "udp" "tcp" ];
-    visibleTo.lan = true;
-    visibleTo.wan = true;
-    visibleTo.ovpn = true;
-    description = "colin-dns-hosting";
-  };
-
-  sane.dns.zones."uninsane.org".TTL = 900;
-
-  # SOA record structure: <https://en.wikipedia.org/wiki/SOA_record#Structure>
-  # SOA MNAME RNAME (... rest)
-  # MNAME = Master name server for this zone. this is where update requests should be sent.
-  # RNAME = admin contact (encoded email address)
-  # Serial = YYYYMMDDNN, where N is incremented every time this file changes, to trigger secondary NS to re-fetch it.
-  # Refresh = how frequently secondary NS should query master
-  # Retry = how long secondary NS should wait until re-querying master after a failure (must be < Refresh)
-  # Expire = how long secondary NS should continue to reply to queries after master fails (> Refresh + Retry)
-  sane.dns.zones."uninsane.org".inet = {
-    SOA."@" = ''
-      ns1.uninsane.org. admin-dns.uninsane.org. (
-                                      2023092101 ; Serial
-                                      4h         ; Refresh
-                                      30m        ; Retry
-                                      7d         ; Expire
-                                      5m)        ; Negative response TTL
-    '';
-    TXT."rev" = "2023092101";
-
-    CNAME."native" = "%CNAMENATIVE%";
-    A."@" =      "%ANATIVE%";
-    A."servo.wan" = "%AWAN%";
-    A."servo.lan" = config.sane.hosts.by-name."servo".lan-ip;
-    A."servo.hn" = config.sane.hosts.by-name."servo".wg-home.ip;
-
-    # XXX NS records must also not be CNAME
-    # it's best that we keep this identical, or a superset of, what org. lists as our NS.
-    # so, org. can specify ns2/ns3 as being to the VPN, with no mention of ns1. we provide ns1 here.
-    A."ns1" =    "%ANATIVE%";
-    A."ns2" =    "185.157.162.178";
-    A."ns3" =    "185.157.162.178";
-    A."ovpns" =  "185.157.162.178";
-    NS."@" = [
-      "ns1.uninsane.org."
-      "ns2.uninsane.org."
-      "ns3.uninsane.org."
-    ];
-  };
-
-  services.trust-dns.settings.zones = [ "uninsane.org" ];
-
-
-  networking.nat.enable = true;
-  networking.nat.extraCommands = ''
-    # redirect incoming DNS requests from LAN addresses
-    #   to the LAN-specialized DNS service
-    # N.B.: use the `nixos-*` chains instead of e.g. PREROUTING
-    #   because they get cleanly reset across activations or `systemctl restart firewall`
-    #   instead of accumulating cruft
-    iptables -t nat -A nixos-nat-pre -p udp --dport 53 \
-      -m iprange --src-range 10.78.76.0-10.78.79.255 \
-      -j DNAT --to-destination :1053
-    iptables -t nat -A nixos-nat-pre -p tcp --dport 53 \
-      -m iprange --src-range 10.78.76.0-10.78.79.255 \
-      -j DNAT --to-destination :1053
-  '';
-  sane.ports.ports."1053" = {
-    # because the NAT above redirects in nixos-nat-pre, LAN requests behave as though they arrived on the external interface at the redirected port.
-    # TODO: try nixos-nat-post instead?
-    # TODO: or, don't NAT from port 53 -> port 1053, but rather nat from LAN addr to a loopback addr.
-    # - this is complicated in that loopback is a different interface than eth0, so rewriting the destination address would cause the packets to just be dropped by the interface
-    protocol = [ "udp" "tcp" ];
-    visibleTo.lan = true;
-    description = "colin-redirected-dns-for-lan-namespace";
-  };
-
-
-  sane.services.trust-dns.enable = true;
-  sane.services.trust-dns.instances = let
-    mkSubstitutions = flavor: {
-      "%AWAN%" = "$(cat '${dyn-dns.ipPath}')";
-      "%CNAMENATIVE%" = "servo.${flavor}";
-      "%ANATIVE%" = nativeAddrs."servo.${flavor}";
-      "%AOVPNS%" = "185.157.162.178";
-    };
-  in
-  {
-    wan = {
-      substitutions = mkSubstitutions "wan";
-      listenAddrs = [
-        nativeAddrs."servo.lan"
-        bindOvpn
-      ];
-    };
-    lan = {
-      substitutions = mkSubstitutions "lan";
-      listenAddrs = [ nativeAddrs."servo.lan" ];
-      port = 1053;
-    };
-    hn = {
-      substitutions = mkSubstitutions "hn";
-      listenAddrs = [ nativeAddrs."servo.hn" ];
-      port = 1053;
-    };
-    hn-resolver = {
-      # don't need %AWAN% here because we forward to the hn instance.
-      listenAddrs = [ nativeAddrs."servo.hn" ];
-      extraConfig = {
-        zones = [
-          {
-            zone = "uninsane.org";
-            zone_type = "Forward";
-            stores = {
-              type = "forward";
-              name_servers = [
-                {
-                  socket_addr = "${nativeAddrs."servo.hn"}:1053";
-                  protocol = "udp";
-                  trust_nx_responses = true;
-                }
-              ];
-            };
-          }
-          {
-            # forward the root zone to the local DNS resolver
-            zone = ".";
-            zone_type = "Forward";
-            stores = {
-              type = "forward";
-              name_servers = [
-                {
-                  socket_addr = "127.0.0.53:53";
-                  protocol = "udp";
-                  trust_nx_responses = true;
-                }
-              ];
-            };
-          }
-        ];
-      };
-    };
-  };
-
-  sane.services.dyn-dns.restartOnChange = [
-    "trust-dns-wan.service"
-    "trust-dns-lan.service"
-    "trust-dns-hn.service"
-    # "trust-dns-hn-resolver.service"  # doesn't need restart because it doesn't know about WAN IP
-  ];
-}

hosts/by-name/servo/users/default.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  imports = [
+    ./shelvacu.nix
+  ];
+}

hosts/by-name/servo/users/shelvacu.nix

@@ -0,0 +1,65 @@
+{ lib, pkgs, ... }:
+{
+  users.users.shelvacu = {
+    isNormalUser = true;
+    home = "/home/shelvacu";
+    subUidRanges = [
+      { startUid=300000; count=1; }
+    ];
+    group = "users";
+    initialPassword = lib.mkDefault "";
+    shell = pkgs.bash;
+
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKoy1TrmfhBGWtVedgOM1FB1oD2UdodN3LkBnnLx6Tug compute-deck"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAxAFFxQMXAgi+0cmGaNE/eAkVfEl91wafUqFIuAkI5I compute-deck-root"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQ2c0GzlVMjV06CS7bWbCaAbzG2+7g5FCg/vClJPe0C fw"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGHLPOxRd68+DJ/bYmqn0wsgwwIcMSMyuU1Ya16hCb/m fw-root"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOre0FnYDm3arsFj9c/l5H2Q8mdmv7kmvq683pL4heru legtop"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINznGot+L8kYoVQqdLV/R17XCd1ILMoDCILOg+I3s5wC pixel9pro-nod"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDcRDekd8ZOYfQS5X95/yNof3wFYIbHqWeq4jY0+ywQX pro1x-nod"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJNFbzt0NHVTaptBI38YtwLG+AsmeNYy0Nr5yX2zZEPE root@vacuInstaller toptop-root"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICVeSzDkGTueZijB0xUa08e06ovAEwwZK/D+Cc7bo91g triple-dezert"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOtwtao/TXbiuQOYJbousRPVesVcb/2nP0PCFUec0Nv8 triple-dezert-root"
+    ];
+  };
+
+  security.sudo.extraRules = [
+    {
+      users = [ "shelvacu" ];
+      runAs = "postgres";
+      commands = [
+        {
+          command = "ALL";
+          options = [ "NOPASSWD" ];
+        }
+      ];
+    }
+  ];
+
+  security.polkit.extraConfig = ''
+    // allow:
+    // - systemctl restart|start|stop SERVICE
+    polkit.addRule(function(action, subject) {
+      if (subject.user == "shelvacu" && action.id == "org.freedesktop.systemd1.manage-units") {
+        switch (action.lookup("verb")) {
+          // case "cancel":
+          // case "reenable":
+          case "restart":
+          // case "reload":
+          // case "reload-or-restart":
+          case "start":
+          case "stop":
+          // case "try-reload-or-restart":
+          // case "try-restart":
+            return polkit.Result.YES;
+          default:
+        }
+      }
+    })
+  '';
+
+  sane.persist.sys.byStore.private = [
+    { path = "/home/shelvacu/persist"; user = "shelvacu"; group = "users"; mode = "0700"; }
+  ];
+}

hosts/common/boot.nix

@@ -0,0 +1,57 @@
+{ lib, pkgs, ... }:
+{
+  boot.initrd.supportedFilesystems = [ "ext4" "btrfs" "ext2" "ext3" "vfat" ];
+  # useful emergency utils
+  boot.initrd.extraUtilsCommands = ''
+    copy_bin_and_libs ${lib.getExe' pkgs.btrfs-progs "btrfstune"}
+    copy_bin_and_libs ${lib.getExe' pkgs.e2fsprogs "resize2fs"}
+    copy_bin_and_libs ${lib.getExe' pkgs.gptfdisk "{cgdisk,gdisk}"}
+    copy_bin_and_libs ${lib.getExe' pkgs.mtools "mlabel"}
+    copy_bin_and_libs ${lib.getExe pkgs.nvme-cli}
+    copy_bin_and_libs ${lib.getExe' pkgs.smartmontools "smartctl"}
+    copy_bin_and_libs ${lib.getExe' pkgs.util-linux "{cfdisk,lsblk,lscpu}"}
+  '';
+  boot.kernelParams = [
+    "boot.shell_on_fail"
+    #v experimental full pre-emption for hopefully better call/audio latency on moby.
+    # also toggleable at runtime via /sys/kernel/debug/sched/preempt
+    # defaults to preempt=voluntary
+    # "preempt=full"
+  ];
+  # other kernelParams:
+  #   "boot.trace"
+  #   "systemd.log_level=debug"
+  #   "systemd.log_target=console"
+
+  # moby has to run recent kernels (defined elsewhere).
+  # meanwhile, kernel variation plays some minor role in things like sandboxing (landlock) and capabilities.
+  # - as of 2024/08/xx, my boot fails on 6.6, but works on 6.9 and (probably; recently) 6.8.
+  # simpler to keep near the latest kernel on all devices,
+  # and also makes certain that any weird system-level bugs i see aren't likely to be stale kernel bugs.
+  boot.kernelPackages = lib.mkDefault pkgs.linuxPackages_latest;
+  # boot.kernelPackages = lib.mkDefault pkgs.linuxPackages_testing;
+
+  # hack in the `boot.shell_on_fail` arg since that doesn't always seem to work.
+  boot.initrd.preFailCommands = "allowShell=1";
+
+  # default: 4 (warn). 7 is debug
+  boot.consoleLogLevel = 7;
+
+  boot.loader.grub.enable = lib.mkDefault false;
+  # boot.loader.generic-extlinux-compatible.enable = lib.mkDefault true;
+  boot.loader.systemd-boot.enable = lib.mkDefault true;
+  boot.loader.systemd-boot.configurationLimit = lib.mkDefault 20;
+  boot.loader.systemd-boot.edk2-uefi-shell.enable = lib.mkDefault true;
+  boot.loader.systemd-boot.memtest86.enable = lib.mkDefault
+    (lib.meta.availableOn pkgs.stdenv.hostPlatform pkgs.memtest86plus);
+
+  hardware.enableAllFirmware = true;  # firmware with licenses that don't allow for redistribution. fuck lawyers, fuck IP, give me the goddamn firmware.
+  # hardware.enableRedistributableFirmware = true;  # proprietary but free-to-distribute firmware (extraneous to `enableAllFirmware` option)
+
+  # default is 252274, which is too low particularly for servo.
+  # manifests as spurious "No space left on device" when trying to install watches,
+  # e.g. in dyn-dns by `systemctl start dyn-dns-watcher.path`.
+  # see: <https://askubuntu.com/questions/828779/failed-to-add-run-systemd-ask-password-to-directory-watch-no-space-left-on-dev>
+  boot.kernel.sysctl."fs.inotify.max_user_watches" = 4194304;
+  boot.kernel.sysctl."fs.inotify.max_user_instances" = 4194304;
+}

hosts/common/default.nix

@@ -1,33 +1,46 @@
-{ config, lib, pkgs, ... }:
+{ lib, pkgs, ... }:
 {
   imports = [
+    ./boot.nix
     ./feeds.nix
-    ./fs.nix
-    ./hardware
+    ./fs
     ./home
     ./hosts.nix
     ./ids.nix
     ./machine-id.nix
     ./net
-    ./nix
-    ./persist.nix
+    ./nix.nix
     ./polyunfill.nix
     ./programs
+    ./quirks.nix
     ./secrets.nix
+    ./snapper.nix
     ./ssh.nix
     ./systemd.nix
     ./users
   ];
 
+
+  # docs: https://nixos.org/manual/nixos/stable/options.html#opt-system.stateVersion
+  # this affects where nixos modules look for stateful data which might have been migrated across releases.
+  system.stateVersion = "21.11";
+
   sane.nixcache.enable-trusted-keys = true;
   sane.nixcache.enable = lib.mkDefault true;
   sane.persist.enable = lib.mkDefault true;
   sane.root-on-tmpfs = lib.mkDefault true;
   sane.programs.sysadminUtils.enableFor.system = lib.mkDefault true;
+  sane.programs.sysadminExtraUtils.enableFor.system = lib.mkDefault true;
   sane.programs.consoleUtils.enableFor.user.colin = lib.mkDefault true;
 
-  nixpkgs.config.allowUnfree = true;  # NIXPKGS_ALLOW_UNFREE=1
-  nixpkgs.config.allowBroken = true;  # NIXPKGS_ALLOW_BROKEN=1
+  services.buffyboard.enable = true;
+  services.buffyboard.settings.theme.default = "pmos-light";
+  # services.buffyboard.settings.quirks.fbdev_force_refresh = true;
+  services.buffyboard.extraFlags = [ "--verbose" ];
+
+  # irqbalance monitors interrupt count (as a daemon) and assigns high-frequency interrupts to different CPUs.
+  # that reduces contention between simultaneously-fired interrupts.
+  services.irqbalance.enable = true;
 
   # time.timeZone = "America/Los_Angeles";
   time.timeZone = "Etc/UTC";  # DST is too confusing for me => use a stable timezone
@@ -39,7 +52,7 @@
       # source: <https://github.com/luishfonseca/dotfiles/blob/32c10e775d9ec7cc55e44592a060c1c9aadf113e/modules/upgrade-diff.nix>
       # modified to not error on boot (when /run/current-system doesn't exist)
       if [ -d /run/current-system ]; then
-        ${pkgs.nvd}/bin/nvd --nix-bin-dir=${pkgs.nix}/bin diff /run/current-system "$systemConfig"
+        ${lib.getExe pkgs.nvd} --nix-bin-dir=${pkgs.nix}/bin diff /run/current-system "$systemConfig"
       fi
     '';
   };

hosts/common/feeds.nix

@@ -1,14 +1,15 @@
 # where to find good stuff?
 # - universal search/directory: <https://podcastindex.org>
+#   - the full database is downloadable
+# - find adjacent podcasts: <https://rephonic.com/graph>
+# - charts: <https://rephonic.com/charts/apple/united-states/technology>
+# - list of lists: <https://en.wikipedia.org/wiki/Category:Lists_of_podcasts>
 # - podcasts w/ a community: <https://lemmyverse.net/communities?query=podcast>
-# - podcast rec thread: <https://lemmy.ml/post/1565858>
+# - podcast recs:
+#   - active lemmy: <https://slrpnk.net/c/podcasts>
+#   - old thread: <https://lemmy.ml/post/1565858>
 #
-# candidates:
-# - The Nonlinear Library (podcast): <https://forum.effectivealtruism.org/posts/JTZTBienqWEAjGDRv/listen-to-more-ea-content-with-the-nonlinear-library>
-#   - has ~10 posts per day, text-to-speech; i would need better tagging before adding this
-# - <https://www.metaculus.com/questions/11102/introducing-the-metaculus-journal-podcast/>
-#   - dead since 2022/10 - 2023/03
-
+# - paywall bypass / bootlegs: <https://jumble.top/>
 { lib, sane-data, ... }:
 let
   hourly = { freq = "hourly"; };
@@ -60,65 +61,104 @@ let
     };
 
   podcasts = [
+    (fromDb "404media.co/the-404-media-podcast" // tech)
     (fromDb "acquiredlpbonussecretsecret.libsyn.com" // tech)  # ACQ2 - more "Acquired" episodes
-    (fromDb "allinchamathjason.libsyn.com" // pol)
+    (fromDb "adventofcomputing.com" // tech)  # computing history
+    (fromDb "api.oyez.org/podcasts/oral-arguments/2015" // pol)  # Supreme Court Oral Arguments ("2015" in URL means nothing -- it's still updated)
     (fromDb "anchor.fm/s/34c7232c/podcast/rss" // tech)  # Civboot -- https://anchor.fm/civboot
     (fromDb "anchor.fm/s/2da69154/podcast/rss" // tech)  # POD OF JAKE -- https://podofjake.com/
+    (fromDb "bluecityblues.org.podcastpage.io" // pol)  # hosts overlap with Seattle Nice
+    (fromDb "buzzsprout.com/2126417" // tech)  # Mystery AI Hype Theater 3000
     (fromDb "cast.postmarketos.org" // tech)
     (fromDb "congressionaldish.libsyn.com" // pol)  # Jennifer Briney
     (fromDb "craphound.com" // pol)  # Cory Doctorow -- both podcast & text entries
     (fromDb "darknetdiaries.com" // tech)
-    (fromDb "feed.podbean.com/matrixlive/feed.xml" // tech)  # Matrix (chat) Live
+    (fromDb "dwarkeshpatel.com" // tech)
     (fromDb "feeds.99percentinvisible.org/99percentinvisible" // pol)  # 99% Invisible -- also available here: <https://feeds.simplecast.com/BqbsxVfO>
+    (fromDb "feeds.acast.com/public/shows/lawfare" // pol)  # <https://www.lawfaremedia.org/podcasts-multimedia/podcast/the-lawfare-podcast>
+    (fromDb "feeds.buzzsprout.com/2412334.rss")  # Matt Stoller's _Organized Money_ <https://www.organizedmoney.fm/>
+    (fromDb "feeds.eff.org/howtofixtheinternet" // pol)
     (fromDb "feeds.feedburner.com/80000HoursPodcast" // rat)
     (fromDb "feeds.feedburner.com/dancarlin/history" // rat)
     (fromDb "feeds.feedburner.com/radiolab" // pol)  # Radiolab -- also available here, but ONLY OVER HTTP: <http://feeds.wnyc.org/radiolab>
+    (fromDb "feeds.megaphone.fm/CHTAL4990341033" // pol)  # ChinaTalk: https://www.chinatalk.media/podcast
+    (fromDb "feeds.megaphone.fm/GLT1412515089" // pol)  # JRE: Joe Rogan Experience
     (fromDb "feeds.megaphone.fm/behindthebastards" // pol)  # also Maggie Killjoy
+    (fromDb "feeds.megaphone.fm/cspantheweekly" // pol)
+    (fromDb "feeds.megaphone.fm/econ102")  # Noah Smith + Erik Torenberg <https://www.podpage.com/econ102/>
+    (fromDb "feeds.megaphone.fm/history102")  # <https://www.podpage.com/history-102-with-whatifalthist/>
     (fromDb "feeds.megaphone.fm/recodedecode" // tech)  # The Verge - Decoder
-    (fromDb "feeds.simplecast.com/54nAGcIl" // pol)  # The Daily
-    (fromDb "feeds.simplecast.com/82FI35Px" // pol)  # Ezra Klein Show
+    (fromDb "feeds.megaphone.fm/thiswontlast" // tech)  # <https://www.podpage.com/thiswontlast/>
+    (fromDb "feeds.megaphone.fm/unexplainable")
     (fromDb "feeds.simplecast.com/wgl4xEgL" // rat)  # Econ Talk
-    (fromDb "feeds.simplecast.com/xKJ93w_w" // uncat)  # Atlas Obscura
     (fromDb "feeds.transistor.fm/acquired" // tech)
+    (fromDb "feeds.transistor.fm/complex-systems-with-patrick-mckenzie-patio11" // tech)  # Patrick Mackenzie (from Bits About Money)
+    (fromDb "feeds.twit.tv/floss.xml" // tech)
     (fromDb "fulltimenix.com" // tech)
     (fromDb "futureofcoding.org/episodes" // tech)
     (fromDb "hackerpublicradio.org" // tech)
     (fromDb "lexfridman.com/podcast" // rat)
+    (fromDb "linktr.ee/betteroffline" // pol)
+    (fromDb "linuxdevtime.com" // tech)
+    (fromDb "malicious.life" // tech)
     (fromDb "mapspodcast.libsyn.com" // uncat)  # Multidisciplinary Association for Psychedelic Studies
+    (fromDb "motherearthnewsandfriends.libsyn.com" // uncat)  # off-grid living
     (fromDb "microarch.club" // tech)
+    (fromDb "nocturnepodcast.org")
     (fromDb "omegataupodcast.net" // tech)  # 3/4 German; 1/4 eps are English
     (fromDb "omny.fm/shows/cool-people-who-did-cool-stuff" // pol)  # Maggie Killjoy -- referenced by Cory Doctorow
     (fromDb "omny.fm/shows/money-stuff-the-podcast")  # Matt Levine
+    (fromDb "omny.fm/shows/stuff-you-should-know-1")
     (fromDb "omny.fm/shows/the-dollop-with-dave-anthony-and-gareth-reynolds")  # The Dollop history/comedy
+    (fromDb "omny.fm/shows/weird-little-guys")  # Cool Zone Media
     (fromDb "originstories.libsyn.com" // uncat)
-    (fromDb "podcast.posttv.com/itunes/post-reports.xml" // pol)
+    (fromDb "podcast.ergaster.org/@flintandsilicon" // tech)  # Thib's podcast: public interest tech, gnome, etc: <https://fed.uninsane.org/users/$ALLO9MZ5g5CsQTCBH6>
+    (fromDb "pods.media/api/rss/feed/channel/unchained" // tech)  # cryptocurrency happenings; rec via patio11
     (fromDb "politicalorphanage.libsyn.com" // pol)
     (fromDb "reverseengineering.libsyn.com/rss" // tech)  # UnNamed Reverse Engineering Podcast
-    (fromDb "rss.acast.com/deconstructed")  # The Intercept - Deconstructed
-    (fromDb "rss.acast.com/ft-tech-tonic" // tech)
-    (fromDb "rss.acast.com/intercepted-with-jeremy-scahill")  # The Intercept - Intercepted
-    (fromDb "rss.art19.com/60-minutes" // pol)
+    (fromDb "rss.acast.com/ft-tech-tonic" // tech)  # Financial Time's: Tech Tonic
     (fromDb "rss.art19.com/the-portal" // rat)  # Eric Weinstein
-    (fromDb "seattlenice.buzzsprout.com" // pol)
+    (fromDb "seattlenice.buzzsprout.com" // pol)  # Seattle Nice
+    (fromDb "speedboatdope.com" // pol)  # Chapo Trap House (premium feed)
     (fromDb "srslywrong.com" // pol)
     (fromDb "sharkbytes.transistor.fm" // tech)  # Wireshark Podcast o_0
-    (fromDb "sscpodcast.libsyn.com" // rat)  # Astral Codex Ten
+    (fromDb "sharptech.fm/feed/podcast" // tech)  # Ben Thompson
+    (fromDb "sscpodcast.libsyn.com" // rat)  # Astral Codex Ten; Scott Alexander
     (fromDb "talesfromthebridge.buzzsprout.com" // tech)  # Sci-Fi? has Peter Watts; author of No Moods, Ads or Cutesy Fucking Icons (rifters.com)
-    (fromDb "theamphour.com" // tech)
     (fromDb "techtalesshow.com" // tech)  # Corbin Davenport
-    (fromDb "techwontsave.us" // pol)  # rec by Cory Doctorow
-    (fromDb "wakingup.libsyn.com" // pol)  # Sam Harris
-    (fromDb "werenotwrong.fireside.fm" // pol)
+    (fromDb "theamphour.com" // tech)  # The Amp Hour
+    (fromDb "the-ben-marc-show.simplecast.com" // tech // pol)  # Ben Horowitz + Marc Andreessen; love to hate em
+    (fromDb "timclicks.dev/compose-podcast" // tech)  # Rust-heavy dev interviews
+    (fromDb "werenotwrong.fireside.fm" // pol)  # We're Not Wrong
+    (fromDb "whycast.podcast.audio/@whycast" // tech)  # What Hackers Yearn [for]: <https://why2025.org/>
     (mkPod "https://sfconservancy.org/casts/the-corresponding-source/feeds/ogg/" // tech)
 
+    # (fromDb "allinchamathjason.libsyn.com" // pol)
+    # (fromDb "feed.podbean.com/matrixlive/feed.xml" // tech)  # Matrix (chat) Live
     # (fromDb "feeds.libsyn.com/421877" // rat)  # Less Wrong Curated
     # (fromDb "feeds.megaphone.fm/hubermanlab" // uncat)  # Daniel Huberman on sleep
+    # (fromDb "feeds.simplecast.com/54nAGcIl" // pol)  # The Daily
+    # (fromDb "feeds.simplecast.com/82FI35Px" // pol)  # Ezra Klein Show
     # (fromDb "feeds.simplecast.com/l2i9YnTd" // tech // pol)  # Hard Fork (NYtimes tech)
+    # (fromDb "feeds.simplecast.com/whlwDbyc" // tech)  # Tech Lounge: <https://chrischinchilla.com/podcast/techlounge/>
+    # (fromDb "feeds.simplecast.com/xKJ93w_w" // uncat)  # Atlas Obscura
+    # (fromDb "iheart.com/podcast/1119-away-days-podcast-reporti-275359753" // pol)  # Away Days (Cool Zone Media)
+    # (fromDb "lastweekinai.com" // tech)  # Last Week in AI
+    # (fromDb "mintcast.org" // tech)
+    # (fromDb "podcast.posttv.com/itunes/post-reports.xml" // pol)
+    # (fromDb "podcast.sustainoss.org" // tech)  # "Sustainable tech", only... it somehow manages to avoid any tech which is actually sustainable, and most of the time doesn't even talk about Open Source Software (!). normie/surface-level/"feel good"
     # (fromDb "podcast.thelinuxexp.com" // tech)  # low-brow linux/foss PR announcements
+    # (fromDb "politicspoliticspolitics.com" // pol)  # don't judge me. Justin Robert Young.
+    # (fromDb "rss.acast.com/deconstructed")  # The Intercept - Deconstructed
+    # (fromDb "rss.acast.com/intercepted-with-jeremy-scahill")  # The Intercept - Intercepted
+    # (fromDb "rss.art19.com/60-minutes" // pol)
     # (fromDb "rss.art19.com/your-welcome" // pol)  # Michael Malice - Your Welcome -- also available here: <https://origin.podcastone.com/podcast?categoryID2=2232>
     # (fromDb "rss.prod.firstlook.media/deconstructed/podcast.rss" // pol)  #< possible URL rot
     # (fromDb "rss.prod.firstlook.media/intercepted/podcast.rss" // pol)  #< possible URL rot
+    # (fromDb "sites.libsyn.com/438684" // humor)  # Quorators - digging up *weird* Quota questions
+    # (fromDb "techwontsave.us" // pol)  # rec by Cory Doctorow, but way too info-sparse
     # (fromDb "trashfuturepodcast.podbean.com" // pol)  # rec by Cory Doctorow, but way rambly
+    # (fromDb "wakingup.libsyn.com" // pol)  # Sam Harris, but he just repeats himself now
     # (mkPod "https://anchor.fm/s/21bc734/podcast/rss" // pol // infrequent)  # Emerge: making sense of what's next -- <https://www.whatisemerging.com/emergepodcast>
     # (mkPod "https://audioboom.com/channels/5097784.rss" // tech)  # Lateral with Tom Scott
     # (mkPod "https://feeds.megaphone.fm/RUNMED9919162779" // pol // infrequent)  # The Witch Trials of J.K. Rowling: <https://www.thefp.com/witchtrials>
@@ -126,14 +166,17 @@ let
   ];
 
   texts = [
+    (fromDb "ergaster.org/blog" // tech)  # Thib's blog: public interest tech, gnome, etc: <https://fed.uninsane.org/users/$ALLO9MZ5g5CsQTCBH6>
     (fromDb "acoup.blog/feed")  # history, states. author: <https://historians.social/@bretdevereaux/following>
     (fromDb "amosbbatto.wordpress.com" // tech)
     (fromDb "anish.lakhwara.com" // tech)
+    (fromDb "antipope.org")  # Charles Stross
     (fromDb "apenwarr.ca/log/rss.php" // tech)  # CEO of tailscale
     (fromDb "applieddivinitystudies.com" // rat)
     (fromDb "artemis.sh" // tech)
     (fromDb "ascii.textfiles.com" // tech)  # Jason Scott
     (fromDb "austinvernon.site" // tech)
+    (fromDb "buttondown.email" // tech)
     (fromDb "ben-evans.com/benedictevans" // pol)
     (fromDb "bitbashing.io" // tech)
     (fromDb "bitsaboutmoney.com" // uncat)
@@ -142,6 +185,7 @@ let
     (fromDb "blog.jmp.chat" // tech)
     (fromDb "blog.rust-lang.org" // tech)
     (fromDb "blog.thalheim.io" // tech)  # Mic92
+    (fromDb "blog.brixit.nl" // tech)  # Martijn Braam
     (fromDb "bunniestudios.com" // tech)  # Bunnie Juang
     (fromDb "capitolhillseattle.com" // pol)
     (fromDb "edwardsnowden.substack.com" // pol // text)
@@ -154,6 +198,7 @@ let
     (fromDb "interconnected.org/home/feed" // rat)  # Matt Webb -- engineering-ish, but dreamy
     (fromDb "jeffgeerling.com" // tech)
     (fromDb "jefftk.com" // tech)
+    (fromDb "justine.lol" // tech)
     (fromDb "jwz.org/blog" // tech // pol)  # DNA lounge guy, loooong-time blogger
     (fromDb "kill-the-newsletter.com/feeds/joh91bv7am2pnznv.xml" // pol)  # Matt Levine - Money Stuff
     (fromDb "kosmosghost.github.io/index.xml" // tech)
@@ -163,6 +208,7 @@ let
     (fromDb "mako.cc/copyrighteous" // tech // pol)  # rec by Cory Doctorow
     (fromDb "mg.lol" // tech)
     (fromDb "mindingourway.com" // rat)
+    (fromDb "momi.ca" // tech)  # Anjan, pmOS
     (fromDb "morningbrew.com/feed" // pol)
     (fromDb "nixpkgs.news" // tech)
     (fromDb "overcomingbias.com" // rat)  # Robin Hanson
@@ -185,21 +231,25 @@ let
     (fromDb "slimemoldtimemold.com" // rat)
     (fromDb "spectrum.ieee.org" // tech)
     (fromDb "stpeter.im/atom.xml" // pol)
-    (fromDb "thediff.co" // pol)  # Byrne Hobart
     (fromDb "thisweek.gnome.org" // tech)
     (fromDb "tuxphones.com" // tech)
     (fromDb "uninsane.org" // tech)
     (fromDb "unintendedconsequenc.es" // rat)
     (fromDb "vitalik.eth.limo" // tech)  # Vitalik Buterin
+    (fromDb "weekinethereumnews.com" // tech)
     (fromDb "willow.phantoma.online")  # wizard@xyzzy.link
     (fromDb "xn--gckvb8fzb.com" // tech)
+    (fromDb "xorvoid.com" // tech)
+    (fromDb "www.thebignewsletter.com" // pol)
     (mkSubstack "astralcodexten" // rat // daily)  # Scott Alexander
+    (mkSubstack "chlamchowder" // tech)  # details CPU advancements
     (mkSubstack "eliqian" // rat // weekly)
     (mkSubstack "oversharing" // pol // daily)
     (mkSubstack "samkriss" // humor // infrequent)
     (mkText "http://benjaminrosshoffman.com/feed" // pol // weekly)
     (mkText "http://boginjr.com/feed" // tech // infrequent)
     (mkText "https://forum.merveilles.town/rss.xml" // pol // infrequent)  #quality RSS list here: <https://forum.merveilles.town/thread/57/share-your-rss-feeds%21-6/>
+    (mkText "https://icm.museum/rss20.xml" // tech // infrequent)  # Interim Computer Museum
     (mkText "https://jvns.ca/atom.xml" // tech // weekly)  # Julia Evans
     (mkText "https://linuxphoneapps.org/blog/atom.xml" // tech // infrequent)
     (mkText "https://nixos.org/blog/announcements-rss.xml" // tech // infrequent)  # more nixos stuff here, but unclear how to subscribe: <https://nixos.org/blog/categories.html>
@@ -212,6 +262,7 @@ let
     # (fromDb "econlib.org" // pol)
     # (fromDb "lesswrong.com" // rat)
     # (fromDb "profectusmag.com" // pol)  # some conservative/libertarian think tank
+    # (fromDb "thediff.co" // pol)  # Byrne Hobart; 80% is subscriber-only
     # (fromDb "thesideview.co" // uncat)  # spiritual journal; RSS items are stubs
     # (fromDb "theregister.com" // tech)
     # (fromDb "vitalik.ca" // tech)  # moved to vitalik.eth.limo
@@ -224,26 +275,38 @@ let
 
   videos = [
     (fromDb "youtube.com/@Channel5YouTube" // pol)
-    (fromDb "youtube.com/@ColdFusion")
     (fromDb "youtube.com/@ContraPoints" // pol)
     (fromDb "youtube.com/@Exurb1a")
     (fromDb "youtube.com/@hbomberguy")
     (fromDb "youtube.com/@JackStauber")
+    (fromDb "youtube.com/@jaketran")
+    (fromDb "youtube.com/@kurzgesagt")
+    (fromDb "youtube.com/@mii_beta" // tech)  # Baby Wogue / gnome reviewer
+    (fromDb "youtube.com/@Matrixdotorg" // tech)  # Matrix Live
     (fromDb "youtube.com/@NativLang")
     (fromDb "youtube.com/@PolyMatter")
+    (fromDb "youtube.com/@scenesbyben" // pol)  # video essays
     (fromDb "youtube.com/@TechnologyConnections" // tech)
-    (fromDb "youtube.com/@TheB1M")
+    (fromDb "youtube.com/@theodd1sout")
     (fromDb "youtube.com/@TomScottGo")
+    (fromDb "youtube.com/@TVW_Washington" // pol)  # interviews with WA public officials
+    (fromDb "youtube.com/@veritasium")
     (fromDb "youtube.com/@Vihart")
-    (fromDb "youtube.com/@Vox")
-    (fromDb "youtube.com/@Vsauce")
+    (fromDb "youtube.com/@InnuendoStudios" // pol)  # breaks down the nastier political strategies, from a "politics is power" angle
 
+    # (fromDb "youtube.com/@CasuallyExplained" // pol)
+    # (fromDb "youtube.com/@ColdFusion")
     # (fromDb "youtube.com/@rossmanngroup" // pol // tech)  # Louis Rossmann
+    # (fromDb "youtube.com/@TheB1M")
+    # (fromDb "youtube.com/@tested" // tech)  # Adam Savage  (uploads too frequently)
+    # (fromDb "youtube.com/@Vox")
+    # (fromDb "youtube.com/@Vsauce")  # they're all like 1-minute long videos now? what happened @Vsauce?
   ];
 
   images = [
     (fromDb "catandgirl.com" // img // humor)
     (fromDb "davidrevoy.com" // img // art)
+    (fromDb "grumpy.website" // img // humor)
     (fromDb "miniature-calendar.com" // img // art // daily)
     (fromDb "pbfcomics.com" // img // humor)
     (fromDb "poorlydrawnlines.com/feed" // img // humor)

hosts/common/fs.nix

@@ -1,236 +0,0 @@
-# docs
-# - x-systemd options: <https://www.freedesktop.org/software/systemd/man/systemd.mount.html>
-# - fuse options: `man mount.fuse`
-
-{ config, lib, pkgs, sane-lib, utils, ... }:
-
-let
-  fsOpts = rec {
-    common = [
-      "_netdev"
-      "noatime"
-      # user: allow any user with access to the device to mount the fs.
-      #       note that this requires a suid `mount` binary; see: <https://zameermanji.com/blog/2022/8/5/using-fuse-without-root-on-linux/>
-      "user"
-      "x-systemd.requires=network-online.target"
-      "x-systemd.after=network-online.target"
-      "x-systemd.mount-timeout=10s"  # how long to wait for mount **and** how long to wait for unmount
-    ];
-    # x-systemd.automount: mount the fs automatically *on first access*.
-    # creates a `path-to-mount.automount` systemd unit.
-    automount = [ "x-systemd.automount" ];
-    # noauto: don't mount as part of remote-fs.target.
-    # N.B.: `remote-fs.target` is a dependency of multi-user.target, itself of graphical.target.
-    # hence, omitting `noauto` can slow down boots.
-    noauto = [ "noauto" ];
-    # lazyMount: defer mounting until first access from userspace.
-    # see: `man systemd.automount`, `man automount`, `man autofs`
-    lazyMount = noauto ++ automount;
-    wg = [
-      "x-systemd.requires=wireguard-wg-home.service"
-      "x-systemd.after=wireguard-wg-home.service"
-    ];
-
-    fuse = [
-      "allow_other"  # allow users other than the one who mounts it to access it. needed, if systemd is the one mounting this fs (as root)
-      # allow_root: allow root to access files on this fs (if mounted by non-root, else it can always access them).
-      #             N.B.: if both allow_root and allow_other are specified, then only allow_root takes effect.
-      # "allow_root"
-      # default_permissions: enforce local permissions check. CRUCIAL if using `allow_other`.
-      # w/o this, permissions mode of sshfs is like:
-      # - sshfs runs all remote commands as the remote user.
-      # - if a local user has local permissions to the sshfs mount, then their file ops are sent blindly across the tunnel.
-      # - `allow_other` allows *any* local user to access the mount, and hence any local user can now freely become the remote mapped user.
-      # with default_permissions, sshfs doesn't tunnel file ops from users until checking that said user could perform said op on an equivalent local fs.
-      "default_permissions"
-    ];
-    fuseColin = fuse ++ [
-      "uid=1000"
-      "gid=100"
-    ];
-
-    ssh = common ++ fuse ++ [
-      "identityfile=/home/colin/.ssh/id_ed25519"
-      # i *think* idmap=user means that `colin` on `localhost` and `colin` on the remote are actually treated as the same user, even if their uid/gid differs?
-      # i.e., local colin's id is translated to/from remote colin's id on every operation?
-      "idmap=user"
-    ];
-    sshColin = ssh ++ fuseColin ++ [
-      # follow_symlinks: remote files which are symlinks are presented to the local system as ordinary files (as the target of the symlink).
-      #                  if the symlink target does not exist, the presentation is unspecified.
-      #                  symlinks which point outside the mount ARE followed. so this is more capable than `transform_symlinks`
-      "follow_symlinks"
-      # symlinks on the remote fs which are absolute paths are presented to the local system as relative symlinks pointing to the expected data on the remote fs.
-      # only symlinks which would point inside the mountpoint are translated.
-      "transform_symlinks"
-    ];
-    # sshRoot = ssh ++ [
-    #   # we don't transform_symlinks because that breaks the validity of remote /nix stores
-    #   "sftp_server=/run/wrappers/bin/sudo\\040/run/current-system/sw/libexec/sftp-server"
-    # ];
-    # in the event of hunt NFS mounts, consider:
-    # - <https://unix.stackexchange.com/questions/31979/stop-broken-nfs-mounts-from-locking-a-directory>
-
-    # NFS options: <https://linux.die.net/man/5/nfs>
-    # actimeo=n  = how long (in seconds) to cache file/dir attributes (default: 3-60s)
-    # bg         = retry failed mounts in the background
-    # retry=n    = for how many minutes `mount` will retry NFS mount operation
-    # intr       = allow Ctrl+C to abort I/O (it will error with `EINTR`)
-    # soft       = on "major timeout", report I/O error to userspace
-    # softreval  = on "major timeout", service the request using known-stale cache results instead of erroring -- if such cache data exists
-    # retrans=n  = how many times to retry a NFS request before giving userspace a "server not responding" error (default: 3)
-    # timeo=n    = number of *deciseconds* to wait for a response before retrying it (default: 600)
-    #              note: client uses a linear backup, so the second request will have double this timeout, then triple, etc.
-    # proto=udp  = encapsulate protocol ops inside UDP packets instead of a TCP session.
-    #              requires `nfsvers=3` and a kernel compiled with `NFS_DISABLE_UDP_SUPPORT=n`.
-    #              UDP might be preferable to TCP because the latter is liable to hang for ~100s (kernel TCP timeout) after a link drop.
-    #              however, even UDP has issues with `umount` hanging.
-    #
-    # N.B.: don't change these without first testing the behavior of sandboxed apps on a flaky network.
-    nfs = common ++ [
-      # "actimeo=5"
-      # "bg"
-      "retrans=1"
-      "retry=0"
-      # "intr"
-      "soft"
-      "softreval"
-      "timeo=30"
-      "nofail"  # don't fail remote-fs.target when this mount fails  (not an option for sshfs else would be common)
-      # "proto=udp"  # default kernel config doesn't support NFS over UDP: <https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1964093> (see comment 11).
-      # "nfsvers=3"  # NFSv4+ doesn't support UDP at *all*. it's ok to omit nfsvers -- server + client will negotiate v3 based on udp requirement. but omitting causes confusing mount errors when the server is *offline*, because the client defaults to v4 and thinks the udp option is a config error.
-      # "x-systemd.idle-timeout=10"  # auto-unmount after this much inactivity
-    ];
-
-    # manually perform a ftp mount via e.g.
-    #   curlftpfs -o ftpfs_debug=2,user=anonymous:anonymous,connect_timeout=10 -f -s ftp://servo-hn /mnt/my-ftp
-    ftp = common ++ fuseColin ++ [
-      # "ftpfs_debug=2"
-      "user=colin:ipauth"
-      # connect_timeout=10: casting shows to T.V. fails partway through about half the time
-      "connect_timeout=20"
-    ];
-  };
-  remoteHome = host: {
-    sane.programs.sshfs-fuse.enableFor.system = true;
-    fileSystems."/mnt/${host}/home" = {
-      device = "colin@${host}:/home/colin";
-      fsType = "fuse.sshfs";
-      options = fsOpts.sshColin ++ fsOpts.lazyMount;
-      noCheck = true;
-    };
-    sane.fs."/mnt/${host}/home" = sane-lib.fs.wanted {
-      dir.acl.user = "colin";
-      dir.acl.group = "users";
-      dir.acl.mode = "0700";
-    };
-  };
-  remoteServo = subdir: {
-    sane.programs.curlftpfs.enableFor.system = true;
-    sane.fs."/mnt/servo/${subdir}" = sane-lib.fs.wanted {
-      dir.acl.user = "colin";
-      dir.acl.group = "users";
-      dir.acl.mode = "0750";
-    };
-    fileSystems."/mnt/servo/${subdir}" = {
-      device = "ftp://servo-hn:/${subdir}";
-      noCheck = true;
-      fsType = "fuse.curlftpfs";
-      options = fsOpts.ftp ++ fsOpts.noauto ++ fsOpts.wg;
-      # fsType = "nfs";
-      # options = fsOpts.nfs ++ fsOpts.lazyMount ++ fsOpts.wg;
-    };
-    systemd.services."automount-servo-${utils.escapeSystemdPath subdir}" = let
-      fs = config.fileSystems."/mnt/servo/${subdir}";
-    in {
-      # this is a *flaky* network mount, especially on moby.
-      # if done as a normal autofs mount, access will eternally block when network is dropped.
-      # notably, this would block *any* sandboxed app which allows media access, whether they actually try to use that media or not.
-      # a practical solution is this: mount as a service -- instead of autofs -- and unmount on timeout error, in a restart loop.
-      # until the ftp handshake succeeds, nothing is actually mounted to the vfs, so this doesn't slow down any I/O when network is down.
-      description = "automount /mnt/servo/${subdir} in a fault-tolerant and non-blocking manner";
-      after = [ "network-online.target" ];
-      requires = [ "network-online.target" ];
-      wantedBy = [ "default.target" ];
-
-      serviceConfig.Type = "simple";
-      serviceConfig.ExecStart = lib.escapeShellArgs [
-        "/usr/bin/env"
-        "PATH=/run/current-system/sw/bin"
-        "mount.${fs.fsType}"
-        "-f"  # foreground (i.e. don't daemonize)
-        "-s"  # single-threaded (TODO: it's probably ok to disable this?)
-        "-o"
-        (lib.concatStringsSep "," (lib.filter (o: !lib.hasPrefix "x-systemd." o) fs.options))
-        fs.device
-        "/mnt/servo/${subdir}"
-      ];
-      # not sure if this configures a linear, or exponential backoff.
-      # but the first restart will be after `RestartSec`, and the n'th restart (n = RestartSteps) will be RestartMaxDelaySec after the n-1'th exit.
-      serviceConfig.Restart = "always";
-      serviceConfig.RestartSec = "10s";
-      serviceConfig.RestartMaxDelaySec = "120s";
-      serviceConfig.RestartSteps = "5";
-    };
-  };
-in
-lib.mkMerge [
-  {
-    # some services which use private directories error if the parent (/var/lib/private) isn't 700.
-    sane.fs."/var/lib/private".dir.acl.mode = "0700";
-
-    # in-memory compressed RAM
-    # defaults to compressing at most 50% size of RAM
-    # claimed compression ratio is about 2:1
-    # - but on moby w/ zstd default i see 4-7:1  (ratio lowers as it fills)
-    # note that idle overhead is about 0.05% of capacity (e.g. 2B per 4kB page)
-    # docs: <https://www.kernel.org/doc/Documentation/blockdev/zram.txt>
-    #
-    # to query effectiveness:
-    # `cat /sys/block/zram0/mm_stat`. whitespace separated fields:
-    # - *orig_data_size*  (bytes)
-    # - *compr_data_size* (bytes)
-    # - mem_used_total    (bytes)
-    # - mem_limit         (bytes)
-    # - mem_used_max      (bytes)
-    # - *same_pages*  (pages which are e.g. all zeros (consumes no additional mem))
-    # - *pages_compacted*  (pages which have been freed thanks to compression)
-    # - huge_pages  (incompressible)
-    #
-    # see also:
-    # - `man zramctl`
-    zramSwap.enable = true;
-    # how much ram can be swapped into the zram device.
-    # this shouldn't be higher than the observed compression ratio.
-    # the default is 50% (why?)
-    # 100% should be "guaranteed" safe so long as the data is even *slightly* compressible.
-    # but it decreases working memory under the heaviest of loads by however much space the compressed memory occupies (e.g. 50% if 2:1; 25% if 4:1)
-    zramSwap.memoryPercent = 100;
-
-    # environment.pathsToLink = [
-    #   # needed to achieve superuser access for user-mounted filesystems (see sshRoot above)
-    #   # we can only link whole directories here, even though we're only interested in pkgs.openssh
-    #   "/libexec"
-    # ];
-
-    programs.fuse.userAllowOther = true;  #< necessary for `allow_other` or `allow_root` options.
-  }
-
-  (remoteHome "desko")
-  (remoteHome "lappy")
-  (remoteHome "moby")
-  # this granularity of servo media mounts is necessary to support sandboxing:
-  # for flaky mounts, we can only bind the mountpoint itself into the sandbox,
-  # so it's either this or unconditionally bind all of media/.
-  (remoteServo "media/archive")
-  (remoteServo "media/Books")
-  (remoteServo "media/collections")
-  # (remoteServo "media/datasets")
-  (remoteServo "media/games")
-  (remoteServo "media/Music")
-  (remoteServo "media/Pictures/macros")
-  (remoteServo "media/torrents")
-  (remoteServo "media/Videos")
-  (remoteServo "playground")
-]
-

hosts/common/fs/default.nix

@@ -0,0 +1,51 @@
+{ ... }:
+{
+  imports = [
+    ./remote-home.nix
+    ./remote-servo.nix
+  ];
+  # some services which use private directories error if the parent (/var/lib/private) isn't 700.
+  sane.fs."/var/lib/private".dir.acl.mode = "0700";
+
+
+  # allocate a proper /tmp fs, else its capacity will be limited as per impermanence defaults (i.e. 1 GB).
+  fileSystems."/tmp" = {
+    device = "none";
+    fsType = "tmpfs";
+    options = [
+      "mode=777"
+      "defaults"
+    ];
+  };
+
+  # in-memory compressed RAM
+  # defaults to compressing at most 50% size of RAM
+  # claimed compression ratio is about 2:1
+  # - but on moby w/ zstd default i see 4-7:1  (ratio lowers as it fills)
+  # note that idle overhead is about 0.05% of capacity (e.g. 2B per 4kB page)
+  # docs: <https://www.kernel.org/doc/Documentation/blockdev/zram.txt>
+  #
+  # to query effectiveness:
+  # `cat /sys/block/zram0/mm_stat`. whitespace separated fields:
+  # - *orig_data_size*  (bytes)
+  # - *compr_data_size* (bytes)
+  # - mem_used_total    (bytes)
+  # - mem_limit         (bytes)
+  # - mem_used_max      (bytes)
+  # - *same_pages*  (pages which are e.g. all zeros (consumes no additional mem))
+  # - *pages_compacted*  (pages which have been freed thanks to compression)
+  # - huge_pages  (incompressible)
+  #
+  # see also:
+  # - `man zramctl`
+  zramSwap.enable = true;
+  # how much ram can be swapped into the zram device.
+  # this shouldn't be higher than the observed compression ratio.
+  # the default is 50% (why?)
+  # 100% should be "guaranteed" safe so long as the data is even *slightly* compressible.
+  # but it decreases working memory under the heaviest of loads by however much space the compressed memory occupies (e.g. 50% if 2:1; 25% if 4:1)
+  zramSwap.memoryPercent = 100;
+
+  programs.fuse.userAllowOther = true;  #< necessary for `allow_other` or `allow_root` options.
+}
+

hosts/common/fs/fs-opts.nix

@@ -0,0 +1,76 @@
+# docs
+# - x-systemd options: <https://www.freedesktop.org/software/systemd/man/systemd.mount.html>
+# - fuse options: `man mount.fuse`
+rec {
+  common = [
+    "_netdev"
+    "noatime"
+    # user: allow any user with access to the device to mount the fs.
+    #       note that this requires a suid `mount` binary; see: <https://zameermanji.com/blog/2022/8/5/using-fuse-without-root-on-linux/>
+    "user"
+    "x-systemd.requires=network-online.target"
+    "x-systemd.after=network-online.target"
+    "x-systemd.mount-timeout=10s"  # how long to wait for mount **and** how long to wait for unmount
+    # disable defaults: don't fail local-fs.target if this mount fails
+    "nofail"
+  ];
+  # x-systemd.automount: mount the fs automatically *on first access*.
+  # creates a `path-to-mount.automount` systemd unit.
+  automount = [ "x-systemd.automount" ];
+  # noauto: don't mount as part of remote-fs.target.
+  # N.B.: `remote-fs.target` is a dependency of multi-user.target, itself of graphical.target.
+  # hence, omitting `noauto` can slow down boots.
+  noauto = [ "noauto" ];
+  # lazyMount: defer mounting until first access from userspace.
+  # see: `man systemd.automount`, `man automount`, `man autofs`
+  lazyMount = noauto ++ automount;
+
+  fuse = [
+    "allow_other"  # allow users other than the one who mounts it to access it. needed, if systemd is the one mounting this fs (as root)
+    # allow_root: allow root to access files on this fs (if mounted by non-root, else it can always access them).
+    #             N.B.: if both allow_root and allow_other are specified, then only allow_root takes effect.
+    # "allow_root"
+    # default_permissions: enforce local permissions check. CRUCIAL if using `allow_other`.
+    # w/o this, permissions mode of sshfs is like:
+    # - sshfs runs all remote commands as the remote user.
+    # - if a local user has local permissions to the sshfs mount, then their file ops are sent blindly across the tunnel.
+    # - `allow_other` allows *any* local user to access the mount, and hence any local user can now freely become the remote mapped user.
+    # with default_permissions, sshfs doesn't tunnel file ops from users until checking that said user could perform said op on an equivalent local fs.
+    "default_permissions"
+    "drop_privileges"
+    "auto_unmount"  #< ensures that when the fs exits, it releases its mountpoint. then systemd can recognize it as failed.
+  ];
+  fuseColin = fuse ++ [
+    "uid=1000"
+    "gid=100"
+  ];
+
+  ssh = common ++ fuseColin ++ [
+    "identityfile=/home/colin/.ssh/id_ed25519"
+    # i *think* idmap=user means that `colin` on `localhost` and `colin` on the remote are actually treated as the same user, even if their uid/gid differs?
+    # i.e., local colin's id is translated to/from remote colin's id on every operation?
+    "idmap=user"
+  ];
+  sshColin = ssh ++ fuseColin ++ [
+    # follow_symlinks: remote files which are symlinks are presented to the local system as ordinary files (as the target of the symlink).
+    #                  if the symlink target does not exist, the presentation is unspecified.
+    #                  symlinks which point outside the mount ARE followed. so this is more capable than `transform_symlinks`
+    "follow_symlinks"
+    # symlinks on the remote fs which are absolute paths are presented to the local system as relative symlinks pointing to the expected data on the remote fs.
+    # only symlinks which would point inside the mountpoint are translated.
+    "transform_symlinks"
+  ];
+  # sshRoot = ssh ++ [
+  #   # we don't transform_symlinks because that breaks the validity of remote /nix stores
+  #   "sftp_server=/run/wrappers/bin/sudo\\040/run/current-system/sw/libexec/sftp-server"
+  # ];
+
+  # manually perform a ftp mount via e.g.
+  #   curlftpfs -o ftpfs_debug=2,user=anonymous:anonymous,connect_timeout=10 -f -s ftp://servo-hn /mnt/my-ftp
+  ftp = common ++ fuseColin ++ [
+    # "ftpfs_debug=2"
+    "user=colin:ipauth"
+    # connect_timeout=10: casting shows to T.V. fails partway through about half the time
+    "connect_timeout=20"
+  ];
+}

hosts/common/fs/remote-home.nix

@@ -0,0 +1,85 @@
+{ config, lib, ... }:
+let
+  fsOpts = import ./fs-opts.nix;
+  ifSshAuthorized = lib.mkIf (((config.sane.hosts.by-name."${config.networking.hostName}" or {}).ssh or {}).authorized or false);
+
+  remoteHome = name: { host ? name }: let
+    mountpoint = "/mnt/${name}/home";
+    device = "sshfs#colin@${host}:/home/colin";
+    fsType = "fuse3";
+    options = fsOpts.sshColin ++ fsOpts.lazyMount;
+  in {
+    sane.programs.sshfs-fuse.enableFor.system = true;
+    system.fsPackages = [
+      config.sane.programs.sshfs-fuse.package
+    ];
+    fileSystems."${mountpoint}" = {
+      inherit device fsType options;
+      noCheck = true;
+    };
+    # tell systemd about the mount so that i can sandbox it
+    systemd.mounts = [{
+      where = mountpoint;
+      what = device;
+      type = fsType;
+      options = lib.concatStringsSep "," options;
+      wantedBy = [ "default.target" ];
+      after = [
+        "emergency.service"
+        "network-online.target"
+      ];
+      requires = [ "network-online.target" ];
+
+      unitConfig.Conflicts = [
+        # emergency.service drops the user into a root shell;
+        # only accessible via physical TTY, but unmount sensitive data before that as a precaution.
+        "emergency.service"
+      ];
+
+      # mountConfig.LazyUnmount = true;  #< else it _ocassionally_ fails "target is busy"
+
+      mountConfig.ExecSearchPath = [ "/run/current-system/sw/bin" ];
+      mountConfig.User = "colin";
+      mountConfig.AmbientCapabilities = "CAP_SETPCAP CAP_SYS_ADMIN";
+      # hardening (systemd-analyze security mnt-desko-home.mount):
+      # TODO: i can't use ProtectSystem=full here, because i can't create a new mount space; but...
+      # with drop_privileges, i *could* sandbox the actual `sshfs` program using e.g. bwrap
+      mountConfig.CapabilityBoundingSet = "CAP_SETPCAP CAP_SYS_ADMIN";
+      mountConfig.LockPersonality = true;
+      mountConfig.MemoryDenyWriteExecute = true;
+      mountConfig.NoNewPrivileges = true;
+      mountConfig.ProtectClock = true;
+      mountConfig.ProtectHostname = true;
+      mountConfig.RemoveIPC = true;
+      mountConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+      #VVV this includes anything it reads from, e.g. /bin/sh; /nix/store/...
+      # see `systemd-analyze filesystems` for a full list
+      mountConfig.RestrictFileSystems = "@common-block @basic-api fuse";
+      mountConfig.RestrictRealtime = true;
+      mountConfig.RestrictSUIDSGID = true;
+      mountConfig.SystemCallArchitectures = "native";
+      mountConfig.SystemCallFilter = [
+        "@system-service"
+        "@mount"
+        "~@chown"
+        "~@cpu-emulation"
+        "~@keyring"
+        # could remove almost all io calls, however one has to keep `open`, and `write`, to communicate with the fuse device.
+        # so that's pretty useless as a way to prevent write access
+      ];
+      mountConfig.IPAddressDeny = "any";
+      mountConfig.IPAddressAllow = "10.0.0.0/8";
+      mountConfig.DevicePolicy = "closed";  # only allow /dev/{null,zero,full,random,urandom}
+      mountConfig.DeviceAllow = "/dev/fuse";
+      # mount.mountConfig.RestrictNamespaces = true;  #< my sshfs sandboxing uses bwrap
+    }];
+  };
+in
+lib.mkMerge [
+  (ifSshAuthorized (remoteHome "crappy" {}))
+  (ifSshAuthorized (remoteHome "desko" {}))
+  (ifSshAuthorized (remoteHome "flowy" {}))
+  # (ifSshAuthorized (remoteHome "lappy" {}))
+  (ifSshAuthorized (remoteHome "moby" { host = "moby-hn"; }))
+  (ifSshAuthorized (remoteHome "servo" {}))
+]