pkgsCCache: init (if only to prevent me from wasting time with ccache again: its really not that great)

hosts/modules/roles/build-machine.nix

@@ -54,6 +54,17 @@ in
 
       nix.settings.system-features = [ "big-parallel" ];
 
+      programs.ccache.enable = true;
+      nix.settings.extra-sandbox-paths = [ "/var/cache/ccache" ];
+      sane.persist.sys.byStore.plaintext = [
+        { group = "nixbld"; mode = "0775"; path = "/var/cache/ccache"; method = "bind"; }
+      ];
+      # `sloppiness = random_seed`: see <https://wiki.nixos.org/wiki/CCache#Sloppiness>
+      sane.fs."/var/cache/ccache/ccache.conf".file.text = ''
+        max_size = 50G
+        sloppiness = random_seed
+      '';
+
       # corresponds to env var: NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM=1
       # nixpkgs.config.allowUnsupportedSystem = true;
     })

overlays/all.nix

@@ -7,6 +7,7 @@ let
   pkgs = import ./pkgs.nix;
   preferences = import ./preferences.nix;
   cross = import ./cross.nix;
+  pkgs-ccache = import ./pkgs-ccache.nix;
   pkgs-debug = import ./pkgs-debug.nix;
 
   isCross = prev.stdenv.hostPlatform != prev.stdenv.buildPlatform;
@@ -20,5 +21,6 @@ in
     pkgs
     preferences
     (ifCross cross)
+    pkgs-ccache
     pkgs-debug
   ]

overlays/pkgs-ccache.nix

@@ -0,0 +1,58 @@
+# `pkgsCCcache.foo`: builds `foo`, using the `ccache` compiler so that incremental builds are cheaper.
+# requires `programs.ccache.enable = true`.
+#
+# common operations:
+# - `nix-ccache --show-stats`
+# - `du -h /var/cache/ccache`
+#
+# example:
+# - `nix-build --builders "" -A pkgsCross.aarch64-multiplatform.pkgsCCache.mesa`  (it's stackable)
+#
+# cache efficacy is fairly disappointing. a cross-compiled kernel might build in 35min the first time then 34min the second time (thanks to the cache).
+(final: prev:
+let
+  mkCCacheStdenv = pkgs: let
+    # the clunky `buildPackages.ccacheWrapper.override { inherit (stdenv) cc; }`
+    # is to overcome some cross-compilation issues, where the `ccache.links` wrapper
+    # script isn't spliced, and so gets built for the host architecture.
+    ccacheWrapper' = pkgs.buildPackages.ccacheWrapper.override {
+        # inherit (pkgs) stdenv;
+      inherit (pkgs.stdenv) cc;
+      extraConfig = ''
+        # export CCACHE_COMPRESS=1
+        export CCACHE_DIR=/var/cache/ccache
+        export CCACHE_UMASK=007
+      '';
+    };
+  in pkgs.overrideCC pkgs.stdenv ccacheWrapper';
+
+  tryOverride = pkg: attrName: attrValue: let
+    pkg' = pkg.override { "${attrName}" = attrValue; };
+  in if ((pkg.override or {}).__functionArgs or {})."${attrName}" or true then
+    pkg
+  else
+    pkg';
+
+  replaceCcFlags = pkgs: pkg: pkg.overrideAttrs (base: {
+    makeFlags = builtins.map (final.lib.replaceStrings [ "${pkgs.stdenv.cc}" ] [ "${(mkCCacheStdenv pkgs).cc}" ]) base.makeFlags;
+  });
+
+  tryEnableCCache = pkg: let
+    pkgFixStdenv = tryOverride pkg "stdenv" (mkCCacheStdenv final);
+    pkgFixBuildPackages = tryOverride pkgFixStdenv "buildPackages" (final.buildPackages // { stdenv = mkCCacheStdenv final.buildPackages; });
+    pkgFixFlags = replaceCcFlags final pkgFixBuildPackages;
+    pkgFixFlagsBuild = replaceCcFlags final.buildPackages pkgFixFlags;
+  in
+    pkgFixFlagsBuild;
+
+in
+{
+  pkgsCCache = final.lib.mapAttrs (_pname: tryEnableCCache) final;
+
+  # this should work, but fails due to infinite recursion?
+  # pkgsCCache = final.extend (self: super: {
+  #   stdenv = super.overrideCC firstPkgs.stdenv firstPkgs.buildPackages.ccacheWrapper;
+  #   # or:
+  #   # callPackage = path: overrides: super.callPackage path ({ stdenv = self.ccacheStdenv; } // overrides);
+  # });
+})