persist ~/.cache/nix

```
• Updated input 'nixpkgs-stable':
    'github:NixOS/nixpkgs/6a3f9996408c970b99b8b992b11bb249d1455b62' (2023-01-12)
  → 'github:NixOS/nixpkgs/e285dd0ca97c264003867c7329f0d1f4f028739c' (2023-01-13)
• Updated input 'nixpkgs-unpatched':
    'github:NixOS/nixpkgs/6c8644fc37b6e141cbfa6c7dc8d98846c4ff0c2e' (2023-01-11)
  → 'github:NixOS/nixpkgs/0f213d0fee84280d8c3a97f7469b988d6fe5fcdf' (2023-01-12)
```

flake.lock

@@ -60,7 +60,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-RM4ZprjMlUTMD0huOPQceF0ujXG2xk7LE1fGXfcg71g=",
+        "narHash": "sha256-FaeqnQPm3MIXkQsNsGTB4A9lN82YKpLkmVHgdJyJNGg=",
         "path": "nixpatches",
         "type": "path"
       },
@@ -71,11 +71,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1673527292,
-        "narHash": "sha256-903EpRSDCfUvic7Hsiqwy+h7zlMTLAUbCXkEGGriCfM=",
+        "lastModified": 1673612960,
+        "narHash": "sha256-DWR7hrbecJKmUJCswk9MXZta710mq+3jZwTvHU/UfyY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6a3f9996408c970b99b8b992b11bb249d1455b62",
+        "rev": "e285dd0ca97c264003867c7329f0d1f4f028739c",
         "type": "github"
       },
       "original": {
@@ -102,11 +102,11 @@
     },
     "nixpkgs-unpatched": {
       "locked": {
-        "lastModified": 1673450908,
-        "narHash": "sha256-b8em+kwrNtnB7gR8SyVf6WuTyQ+6tHS6dzt9D9wgKF0=",
+        "lastModified": 1673540789,
+        "narHash": "sha256-xqnxBOK3qctIeUVxecydrEDbEXjsvHCPGPbvsl63M/U=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6c8644fc37b6e141cbfa6c7dc8d98846c4ff0c2e",
+        "rev": "0f213d0fee84280d8c3a97f7469b988d6fe5fcdf",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,4 +1,13 @@
-# docs:
+# FLAKE FEEDBACK:
+# - flake input url schemes (like `github:nixos/nixpkgs`, or `nixpkgs/nixos-22.11`) are confusing and opaque.
+#   - use established URI formats like https://github.com/<owner>/<repo>.
+#   - or reuse `fetchGit*` idioms from nixpkgs.
+#   - but please don't just invent something new.
+# - need a way to apply local patches to inputs.
+#   - this could be part of the `inputs` proper, or something like `builtins.applyPatches`.
+#
+#
+# DEVELOPMENT DOCS:
 # - <https://nixos.wiki/wiki/Flakes>
 # - <https://serokell.io/blog/practical-nix-flakes>
 
@@ -98,7 +107,7 @@
 
       overlays = rec {
         default = pkgs;
-        pkgs = import ./pkgs/overlay.nix;
+        pkgs = import ./overlays/pkgs.nix;
         passthru =
           let
             stable = next: prev: {

hosts/common/default.nix

@@ -39,6 +39,11 @@
   nix.extraOptions = ''
     experimental-features = nix-command flakes
   '';
+  # allow `nix-shell` (and probably nix-index?) to locate our patched and custom packages
+  nix.nixPath = [
+    "nixpkgs=${pkgs.path}"
+    "nixpkgs-overlays=${../..}/overlays"
+  ];
 
   # TODO: move this into home-manager?
   fonts = {

hosts/common/users.nix

@@ -86,6 +86,7 @@ in
       "Pictures"
       "Videos"
 
+      ".cache/nix"
       ".cargo"
       ".rustup"
     ];

modules/data/keys.nix

@@ -21,4 +21,10 @@
       servo.root = root;
     };
   };
+
+  com.github = rec {
+    # documented here: <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints>
+    # Github actually uses multiple keys -- one per format
+    root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl";
+  };
 }

modules/gui/snippets.txt

@@ -1,6 +1,7 @@
 https://search.nixos.org/options?channel=unstable&query=
 https://search.nixos.org/packages?channel=unstable&query=
 https://nixos.wiki/index.php?go=Go&search=
+https://nixos.org/manual/nix/stable/language/builtins.html
 https://github.com/nixos/nixpkgs/pulls?q=
 https://nur.nix-community.org/
 https://nix-community.github.io/home-manager/options.html

modules/packages.nix

@@ -102,7 +102,9 @@ let
     gnome.gnome-terminal  # works on phosh
     gnome.gnome-weather
 
-    { pkg = gpodder-configured; dir = [ "gPodder/Downloads" ]; }
+    # XXX: we preserve the whole thing because if we only preserve gPodder/Downloads
+    #   then startup is SLOW during feed import, and we might end up with zombie eps in the dl dir.
+    { pkg = gpodder-configured; dir = [ "gPodder" ]; }
 
     gthumb
     handbrake
@@ -151,7 +153,9 @@ let
     # vlc remembers play position in ~/.config/vlc/vlc-qt-interface.conf
     { pkg = vlc; dir = [ ".config/vlc" ]; }
 
-    whalebird # pleroma client. input is broken on phosh
+    # pleroma client (Electron). input is broken on phosh.
+    { pkg = whalebird; private = [ ".config/Whalebird" ]; }
+
     xdg-utils  # for xdg-open
     xterm  # broken on phosh
   ]

nixpatches/list.nix

@@ -19,6 +19,13 @@
     sha256 = "sha256-Cendm7a23qj/YRBj5qVM8jhl9oTpztaDEkXcSG4dJys=";
   })
 
+  # whalebird: 4.6.5 -> 4.7.4
+  (fetchpatch {
+    # url = "https://git.uninsane.org/colin/nixpkgs/compare/master...pr.whalebird-4.7.4.diff";
+    url = "https://git.uninsane.org/colin/nixpkgs/commit/f5c7c70dde720e990fa7e0748d1dc4764d6e4406.diff";
+    sha256 = "sha256-L9Ie80loaP6yl5ZFnJ1b5WMDpvO1QFE8tbrW5HBauko=";
+  })
+
   ./2022-12-19-i2p-aarch64.patch
 
   # # kaiteki: init at 2022-09-03

overlays/pkgs.nix

@@ -0,0 +1,67 @@
+(next: prev:
+  let
+    sane = rec {
+      #### my own, non-upstreamable packages:
+      sane-scripts = prev.callPackage ../pkgs/sane-scripts { };
+      feeds = prev.callPackage ../pkgs/feeds { };
+      tow-boot-pinephone = prev.callPackage ../pkgs/tow-boot-pinephone { };
+      tow-boot-rpi4 = prev.callPackage ../pkgs/tow-boot-rpi4 { };
+      bootpart-uefi-x86_64 = prev.callPackage ../pkgs/bootpart-uefi-x86_64 { };
+      bootpart-tow-boot-rpi-aarch64 = prev.callPackage ../pkgs/bootpart-tow-boot-rpi-aarch64 {
+        # not sure why i can't just do `next.callPackage` instead
+        inherit tow-boot-rpi4;
+      };
+      bootpart-u-boot-rpi-aarch64 = prev.callPackage ../pkgs/bootpart-u-boot-rpi-aarch64 {
+        # not sure why i can't just do `next.callPackage` instead
+        inherit ubootRaspberryPi4_64bit;
+      };
+      rtl8723cs-firmware = prev.callPackage ../pkgs/rtl8723cs-firmware { };
+      linux-megous = prev.callPackage ../pkgs/linux-megous {
+        kernelPatches = [
+          prev.kernelPatches.bridge_stp_helper
+          prev.kernelPatches.request_key_helper
+        ];
+      };
+
+      sublime-music-mobile = prev.callPackage ../pkgs/sublime-music-mobile { };
+
+      #### customized packages
+      fluffychat-moby = prev.callPackage ../pkgs/fluffychat-moby { };
+      gpodder-configured = prev.callPackage ../pkgs/gpodder-configured { };
+      # nixos-unstable pleroma is too far out-of-date for our db
+      pleroma = prev.callPackage ../pkgs/pleroma { };
+      # jackett doesn't allow customization of the bind address: this will probably always be here.
+      jackett = prev.callPackage ../pkgs/jackett { inherit (prev) jackett; };
+      # mozilla keeps nerfing itself and removing configuration options
+      firefox-unwrapped = prev.callPackage ../pkgs/firefox-unwrapped { };
+
+      # patch rpi uboot with something that fixes USB HDD boot
+      ubootRaspberryPi4_64bit = prev.callPackage ../pkgs/ubootRaspberryPi4_64bit { };
+
+      gocryptfs = prev.callPackage ../pkgs/gocryptfs { inherit (prev) gocryptfs; };
+
+      browserpass = prev.callPackage ../pkgs/browserpass { inherit (prev) browserpass; inherit sane-scripts; };
+
+      fractal-latest = prev.callPackage ../pkgs/fractal-latest { };
+
+      #### TEMPORARY: PACKAGES WAITING TO BE UPSTREAMED
+
+      pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+        (py-final: py-prev: {
+          feedsearch-crawler = py-final.callPackage ../pkgs/feedsearch-crawler { };
+        })
+      ];
+
+      kaiteki = prev.callPackage ../pkgs/kaiteki { };
+      lightdm-mobile-greeter = prev.callPackage ../pkgs/lightdm-mobile-greeter { };
+      browserpass-extension = prev.callPackage ../pkgs/browserpass-extension { };
+      gopass-native-messaging-host = prev.callPackage ../pkgs/gopass-native-messaging-host { };
+      tokodon = prev.libsForQt5.callPackage ../pkgs/tokodon { };
+      signaldctl = prev.callPackage ../pkgs/signaldctl { };
+      splatmoji = prev.callPackage ../pkgs/splatmoji { };
+      # trust-dns = prev.callPackage ../pkgs/trust-dns { };
+      # kaiteki = prev.kaiteki;
+    };
+  in sane // { inherit sane; }
+)
+

pkgs/feeds/default.nix

@@ -35,7 +35,7 @@
           # but in a way where the least could go wrong.
           pushd "$sources_dir"; mkdir -p "$name"; popd
 
-          ${./update.sh} "$url" "$json_path"
+          ${./update.py} "$url" "$json_path"
           cat "$json_path"
         '';
     }

pkgs/feeds/template.nix

@@ -14,10 +14,7 @@ stdenv.mkDerivation {
   src = fetchurl {
     inherit url;
   };
-  passthru.updateScript = [ ./update.sh url jsonPath ];
-  # passthru.updateScript = callPackage ./update.nix {
-  #   inherit url jsonPath;
-  # };
+  passthru.updateScript = [ ./update.py url jsonPath ];
   meta = {
     description = "metadata about any feeds available at ${feedName}";
     homepage = feedName;

pkgs/feeds/update.nix

@@ -1,18 +0,0 @@
-{ lib
-, curl
-, jq
-, runtimeShell
-, writeScript
-# feed-specific args
-, jsonPath
-, url
-}:
-
-let
-  apiQuery = "https://feedsearch.dev/api/v1/search?url=${url}";
-in
-writeScript "update-feed" ''
-  #!${runtimeShell}
-  PATH=${lib.makeBinPath [ curl jq  ]}
-  curl -X GET '${apiQuery}' | jq '.[-1]' > '${jsonPath}'
-''

pkgs/feeds/update.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: [ ps.feedsearch-crawler ])"
+
+from feedsearch_crawler import search, sort_urls
+from feedsearch_crawler.crawler import coerce_url
+
+import json
+import sys
+url, jsonPath = sys.argv[1:]
+
+url = coerce_url(url, default_scheme="https")
+items = search(url)
+items = sort_urls(items)
+
+# print all results
+serialized = [item.serialize() for item in items]
+for item in serialized:
+        print(json.dumps(item, sort_keys=True, indent=2))
+
+# save the first result to disk
+keep = serialized[0] if serialized else {}
+results = json.dumps(keep, sort_keys=True, indent=2)
+with open(jsonPath, "w") as out:
+        out.write(results)

pkgs/feeds/update.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p curl jq
-
-set -xeu -o pipefail
-
-url="$1"
-jsonPath="$2"
-
-apiQuery="https://feedsearch.dev/api/v1/search?url=$url"
-curl -X GET "$apiQuery" | jq '.[0]' > "$jsonPath"

pkgs/feedsearch-crawler/default.nix

@@ -0,0 +1,65 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+# nativeBuildInputs python packages
+, poetry-core
+# propagatedBuildInputs python packages
+, aiodns
+, aiohttp
+, beautifulsoup4
+, brotlipy
+, cchardet
+, feedparser
+, python-dateutil
+, uvloop
+, w3lib
+, yarl
+}:
+
+buildPythonPackage rec {
+  pname = "feedsearch-crawler";
+  version = "2022-05-28";
+  format = "pyproject";
+
+  src = fetchFromGitHub {
+    owner = "DBeath";
+    repo = "feedsearch-crawler";
+    rev = "f49a6f5a07e796e359c4482fd29305b1a019f71f";
+    hash = "sha256-pzvyeXzqdi8pRjk2+QjKhJfgtxbgVT6C08K9fhVFVmY=";
+  };
+
+  nativeBuildInputs = [
+    poetry-core
+  ];
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace 'w3lib = "^1.22.0"' 'w3lib = "*"' \
+      --replace 'aiodns = "^2.0.0"' 'aiodns = "*"' \
+      --replace 'uvloop = "^0.15.2"' 'uvloop = "*"'
+  '';
+
+  propagatedBuildInputs = [
+    aiodns
+    aiohttp
+    beautifulsoup4
+    brotlipy
+    cchardet
+    feedparser
+    python-dateutil
+    uvloop
+    w3lib
+    yarl
+  ];
+
+  pythonImportsCheck = [
+    "feedsearch_crawler"
+  ];
+
+  meta = with lib; {
+    homepage = "https://feedsearch.dev";
+    description = "Crawl sites for RSS, Atom, and JSON feeds";
+    license = licenses.mit;
+    maintainers = with maintainers; [ colinsane ];
+  };
+}

pkgs/gpodder-configured/default.nix

@@ -1,22 +1,28 @@
 { makeWrapper
 , gpodder
+, linkFarm
 , symlinkJoin
-, writeShellScript
-, config
 }:
 
+let
+  remove-extra = linkFarm "gpodder-remove-extra" [
+    { name = "bin/gpodder-remove-extra"; path = ./remove_extra.py; }
+  ];
+in
+# we use a symlinkJoin so that we can inherit the .desktop and icon files from the original gPodder
 (symlinkJoin {
   name = "gpodder-configured";
-  paths = [ gpodder ];
+  paths = [ gpodder remove-extra ];
   buildInputs = [ makeWrapper ];
 
   # gpodder keeps all its feeds in a sqlite3 database.
   # we can configure the feeds externally by wrapping gpodder and just instructing it to import
   # a feedlist every time we run it.
-  # repeat imports are deduplicated -- assuming network access (not sure how it behaves when disconnected).
+  # repeat imports are deduplicated by url, even when offline.
   postBuild = ''
     makeWrapper $out/bin/gpodder $out/bin/gpodder-configured \
-      --run "$out/bin/gpo import ~/.config/gpodderFeeds.opml"
+      --run "$out/bin/gpodder-remove-extra ~/.config/gpodderFeeds.opml" \
+      --run "$out/bin/gpo import ~/.config/gpodderFeeds.opml" \
 
     # fix up the .desktop file to invoke our wrapped application
     orig_desktop=$(readlink $out/share/applications/gpodder.desktop)

pkgs/gpodder-configured/remove_extra.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: [gnome-feeds.listparser])" -p gpodder
+
+from dataclasses import dataclass, field
+import listparser
+import subprocess
+import sys
+
+@dataclass(repr=True)
+class Feed:
+    url: str
+    title: str  # Optional
+    def __init__(self, url: str, title: str):
+        self.url = url
+        self.title = title if title else None
+
+    def __eq__(self, other: 'Feed') -> bool:
+        return self.url == other.url and \
+            (self.title == other.title or None in [self.title, other.title])
+
+@dataclass(init=True)
+class Partitioned:
+    has_not_wanted: list[Feed] = field(default_factory=list)
+    wanted_not_has: list[Feed] = field(default_factory=list)
+    intersection:   list[Feed] = field(default_factory=list)
+
+
+def wanted_feeds(opml_file: str):
+    parsed = listparser.parse(open(opml_file).read())
+    return [Feed(url=p['url'], title=p['title']) for p in parsed.feeds]
+
+def has_feeds():
+    listing = subprocess.check_output(["gpo", "list"]).decode()
+    feeds = []
+
+    title = None
+    for line in listing.split("\n"):
+        if line.startswith("# "):  # title
+            title = line[2:].strip()
+        elif line.startswith("http"):  # feed URL:
+            feeds.append(Feed(url=line, title=title))
+            title = None
+
+    return feeds
+
+def partition_feeds(wanted: list[Feed], has: list[Feed]) -> Partitioned:
+    p = Partitioned()
+    for f in wanted + has:
+        w, h = f in wanted, f in has
+        if h and not w:
+            p.has_not_wanted.append(f)
+        elif w and not h:
+            p.wanted_not_has.append(f)
+        else:
+            assert w and h
+            p.intersection.append(f)
+
+    return p
+
+def remove_feed(feed: Feed):
+    subprocess.check_output(['gpo', 'unsubscribe', feed.url])
+
+def rationalize_feeds(opml_file: str):
+    wanted = wanted_feeds(opml_file)
+    has = has_feeds()
+    partitioned = partition_feeds(wanted, has)
+
+    print("extra feeds:", "" if partitioned.has_not_wanted else "(none)")
+    for f in partitioned.has_not_wanted:
+        print("  ", f)
+    print()
+
+    for f in partitioned.has_not_wanted:
+        remove_feed(f)
+
+if __name__ == "__main__":
+    wanted_opml, = sys.argv[1:]
+    rationalize_feeds(wanted_opml)
+

pkgs/overlay.nix

@@ -1,60 +0,0 @@
-(next: prev:
-  let
-    sane = rec {
-      #### my own, non-upstreamable packages:
-      sane-scripts = prev.callPackage ./sane-scripts { };
-      feeds = prev.callPackage ./feeds { };
-      tow-boot-pinephone = prev.callPackage ./tow-boot-pinephone { };
-      tow-boot-rpi4 = prev.callPackage ./tow-boot-rpi4 { };
-      bootpart-uefi-x86_64 = prev.callPackage ./bootpart-uefi-x86_64 { };
-      bootpart-tow-boot-rpi-aarch64 = prev.callPackage ./bootpart-tow-boot-rpi-aarch64 {
-        # not sure why i can't just do `next.callPackage` instead
-        inherit tow-boot-rpi4;
-      };
-      bootpart-u-boot-rpi-aarch64 = prev.callPackage ./bootpart-u-boot-rpi-aarch64 {
-        # not sure why i can't just do `next.callPackage` instead
-        inherit ubootRaspberryPi4_64bit;
-      };
-      rtl8723cs-firmware = prev.callPackage ./rtl8723cs-firmware { };
-      linux-megous = prev.callPackage ./linux-megous {
-        kernelPatches = [
-          prev.kernelPatches.bridge_stp_helper
-          prev.kernelPatches.request_key_helper
-        ];
-      };
-
-      sublime-music-mobile = prev.callPackage ./sublime-music-mobile { };
-
-      #### customized packages
-      fluffychat-moby = prev.callPackage ./fluffychat-moby { };
-      gpodder-configured = prev.callPackage ./gpodder-configured { };
-      # nixos-unstable pleroma is too far out-of-date for our db
-      pleroma = prev.callPackage ./pleroma { };
-      # jackett doesn't allow customization of the bind address: this will probably always be here.
-      jackett = prev.callPackage ./jackett { inherit (prev) jackett; };
-      # mozilla keeps nerfing itself and removing configuration options
-      firefox-unwrapped = prev.callPackage ./firefox-unwrapped { };
-
-      # patch rpi uboot with something that fixes USB HDD boot
-      ubootRaspberryPi4_64bit = prev.callPackage ./ubootRaspberryPi4_64bit { };
-
-      gocryptfs = prev.callPackage ./gocryptfs { inherit (prev) gocryptfs; };
-
-      browserpass = prev.callPackage ./browserpass { inherit (prev) browserpass; inherit sane-scripts; };
-
-      fractal-latest = prev.callPackage ./fractal-latest { };
-
-      #### TEMPORARY: PACKAGES WAITING TO BE UPSTREAMED
-      kaiteki = prev.callPackage ./kaiteki { };
-      lightdm-mobile-greeter = prev.callPackage ./lightdm-mobile-greeter { };
-      browserpass-extension = prev.callPackage ./browserpass-extension { };
-      gopass-native-messaging-host = prev.callPackage ./gopass-native-messaging-host { };
-      tokodon = prev.libsForQt5.callPackage ./tokodon { };
-      signaldctl = prev.callPackage ./signaldctl { };
-      splatmoji = prev.callPackage ./splatmoji { };
-      # trust-dns = prev.callPackage ./trust-dns { };
-      # kaiteki = prev.kaiteki;
-    };
-  in sane // { inherit sane; }
-)
-

pkgs/sane-scripts/src/sane-bt-search

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: [ ps.natsort ps.requests ])"
 """
 usage: sane-bt-search <query_string>
 
@@ -67,5 +68,6 @@ q = " ".join(sys.argv[1:])
 
 client = Client()
 res = client.query(q)
+print(f"found {len(res)} result(s)")
 for r in res[:5]:
     print(r)