feeds: write the basis for a module which reads feed metadata from disk and can (in the future) update it

hosts/servo/services/nginx.nix

@@ -1,5 +1,5 @@
 # docs: https://nixos.wiki/wiki/Nginx
-{ config, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
 let
   # make the logs for this host "public" so that they show up in e.g. metrics

modules/data/default.nix

@@ -4,6 +4,9 @@
 # don't put things like fully-specific ~/.config files in here,
 # even if they're "relatively unopinionated".
 
+moduleArgs:
+
 {
+  feeds = import ./feeds moduleArgs;
   keys = import ./keys.nix;
 }

modules/data/feeds/default.nix

@@ -0,0 +1,58 @@
+{ lib, ... }:
+
+let
+  inherit (builtins) concatLists concatStringsSep foldl' fromJSON map readDir readFile;
+  inherit (lib) init mapAttrsToList removePrefix removeSuffix splitString;
+  inherit (lib.attrsets) recursiveUpdate setAttrByPath;
+  inherit (lib.filesystem) listFilesRecursive;
+
+  # given a path to a .json file relative to sources, construct the best feed object we can.
+  # the .json file could be empty, in which case we make assumptions about the feed based
+  # on its fs path.
+  # Type: feedFromSourcePath :: String -> { path = [String]; value = feed; }
+  feedFromSourcePath = json-path:
+    let
+      canonical-name = removeSuffix "/default" (lib.removeSuffix ".json" json-path);
+      default-url = "https://${canonical-name}";
+      attr-path = splitString "/" canonical-name;
+      feed-details = { url = default-url; } // (tryImportJson (./sources/${json-path}));
+    in { path = attr-path; value = mkFeed feed-details; };
+
+  # TODO: for now, feeds are just ordinary Attrs.
+  # in the future, we'd like to set them up with an update script.
+  mkFeed = { url, ... }@details: details;
+
+  # return an AttrSet representing the json at the provided path,
+  # or {} if the path is empty.
+  tryImportJson = path:
+    let
+      as-str = readFile path;
+    in
+      if as-str == "" then
+        {}
+      else
+        fromJSON as-str;
+
+  sources = enumerateFilePaths ./sources;
+
+  # like `lib.listFilesRecursive` but does not mangle paths.
+  # Type: enumerateFilePaths :: path -> [String]
+  enumerateFilePaths = base:
+    concatLists (
+      mapAttrsToList
+        (name: type:
+          if type == "directory" then
+            # enumerate this directory and then prefix each result with the directory's name
+            map (e: "${name}/${e}") (enumerateFilePaths (base + "/${name}"))
+          else
+            [ name ]
+        )
+        (readDir base)
+    );
+
+  # like listToAttrs, except takes { path, value } pairs instead of { name, value } pairs.
+  # Type: listToAttrsByPath :: [{ path = [String]; value = Any; }] -> Attrs
+  listToAttrsByPath = items:
+    foldl' (acc: { path, value }: recursiveUpdate acc (setAttrByPath path value)) {} items;
+in
+  listToAttrsByPath (map feedFromSourcePath sources)

modules/default.nix

@@ -18,6 +18,6 @@
 
   _module.args =  {
     sane-lib = import ./lib { inherit lib utils; };
-    sane-data = import ./data;
+    sane-data = import ./data { inherit lib; };
   };
 }

modules/feeds.nix

@@ -14,6 +14,7 @@ let
       };
       format = mkOption {
         type = types.enum [ "text" "image" "podcast" ];
+        default = "text";
       };
       url = mkOption {
         type = types.str;

modules/lib/merge.nix

@@ -31,11 +31,11 @@ rec {
     let
       # define the current path, but nothing more.
       curLevel = lib.setAttrByPath path {};
-      # `take` will either set:
-      # - { $path = path }  => { $path = {} };
-      # - { $path.next = path.next }  => { $path = { next = ?; } }
+      # `take curLevel` will act one of two ways here:
+      # - { $path = f.$path; }  => { $path = {}; };
+      # - { $path.subAttr = f.$path.subAttr; }  => { $path = { subAttr = ?; }; }
       # so, index $path into the output of `take`,
-      # and if it has any attrs that means we're interested in those too.
+      # and if it has any attrs (like `subAttr`) that means we're interested in those too.
       nextLevel = lib.getAttrFromPath path (take curLevel);
     in
       builtins.attrNames nextLevel;
@@ -49,10 +49,7 @@ rec {
     in if subNames == [] then
       [ path ]
     else
-      let
-        terminalsPerChild = builtins.map (name: findTerminalPaths take (path ++ [name])) subNames;
-      in
-        lib.concatLists terminalsPerChild;
+      lib.concatMap (name: findTerminalPaths take (path ++ [name])) subNames;
 
   # ensures that all nodes in the attrset from the root to and including the given path
   # are ordinary attrs -- if they exist.
@@ -64,19 +61,15 @@ rec {
       items = lib.pushDownProperties i;
       # now items is a list where every element is undecorated at the toplevel.
       # e.g. each item is an ordinary attrset or primitive.
-    in
-      if path == [] then
-        items
+      # we still need to discharge the *rest* of the path though, for every item.
+      name = lib.head path;
+      downstream = lib.tail path;
+      dischargeDownstream = it: if path != [] && it ? name then
+        builtins.map (v: it // { "${name}" = v; }) (dischargeToPath downstream it."${name}")
       else
-        let
-          name = lib.head path;
-          downstream = lib.tail path;
-          dischargeItem = it: if it ? name then
-            builtins.map (v: it // { "${name}" = v; }) (dischargeToPath downstream it."${name}")
-          else
-            [ it ];
-        in
-          lib.concatMap dischargeItem items;
+        [ it ];
+    in
+      lib.concatMap dischargeDownstream items;
 
   # discharge many items but only over one path.
   # Type: dischargeItemsToPaths :: [Attrs] -> String -> [Attrs]
@@ -96,9 +89,12 @@ rec {
   # check that attrset `i` contains no terminals other than those specified in (or direct ancestors of) paths
   assertNoExtraPaths = paths: i:
     let
-      clearPath = acc: path: lib.recursiveUpdate acc (lib.setAttrByPath path null);
-      remainder = builtins.foldl' clearPath i paths;
-      expected-remainder = builtins.foldl' clearPath {} paths;
+      # since the act of discharging should have forced all the relevant data out to the leaves,
+      # we just set each expected terminal to null (initializing the parents when necessary)
+      # and that gives a standard value for any fully-consumed items that we can do equality comparisons with.
+      wipePath = acc: path: lib.recursiveUpdate acc (lib.setAttrByPath path null);
+      remainder = builtins.foldl' wipePath i paths;
+      expected-remainder = builtins.foldl' wipePath {} paths;
     in
       assert remainder == expected-remainder; true;
 }