cross: fix emulateBuilder to use build bash in the vmRunCommand

flake.nix

@@ -94,7 +94,17 @@
       evalHost = { name, local, target }: nixpkgs.lib.nixosSystem {
         system = target;
         modules = [
-          (import ./hosts/instantiate.nix { localSystem = local; hostName = name; })
+          {
+            nixpkgs = (if (local != null) then {
+              buildPlatform = local;
+            } else {}) // {
+              # TODO: does the earlier `system` arg to nixosSystem make its way here?
+              hostPlatform.system = target;
+            };
+            # nixpkgs.buildPlatform = local;  # set by instantiate.nix instead
+            # nixpkgs.config.replaceStdenv = { pkgs }: pkgs.ccacheStdenv;
+          }
+          (import ./hosts/instantiate.nix { hostName = name; })
           self.nixosModules.default
           self.nixosModules.passthru
           {
@@ -103,12 +113,6 @@
               self.overlays.sane-all
             ];
           }
-          ({ lib, ... }: {
-            # TODO: does the earlier `system` arg to nixosSystem make its way here?
-            nixpkgs.hostPlatform.system = target;
-            # nixpkgs.buildPlatform = local;  # set by instantiate.nix instead
-            # nixpkgs.config.replaceStdenv = { pkgs }: pkgs.ccacheStdenv;
-          })
         ];
       };
     in {

hosts/instantiate.nix

@@ -1,10 +1,10 @@
 # trampoline from flake.nix into the specific host definition, while doing a tiny bit of common setup
 
 # args from flake-level `import`
-{ hostName, localSystem }:
+{ hostName }:
 
 # module args
-{ lib, ... }:
+{ ... }:
 
 {
   imports = [
@@ -14,5 +14,4 @@
   ];
 
   networking.hostName = hostName;
-  nixpkgs.buildPlatform = lib.mkIf (localSystem != null) localSystem;
 }

nixpatches/flake.nix

@@ -15,9 +15,15 @@
       patchedFlakeFor = system: import "${patchedPkgsFor system}/flake.nix";
       patchedFlakeOutputsFor = system:
         (patchedFlakeFor system).outputs { inherit self; };
+
+      extractBuildPlatform = nixosSystemArgs:
+        let
+          firstMod = builtins.head nixosSystemArgs.modules;
+        in
+          firstMod.nixpkgs.buildPlatform or nixosSystemArgs.system;
     in
     {
-      lib.nixosSystem = args: (patchedFlakeOutputsFor args.system).lib.nixosSystem args;
+      lib.nixosSystem = args: (patchedFlakeOutputsFor (extractBuildPlatform args)).lib.nixosSystem args;
 
       legacyPackages = builtins.mapAttrs
         (system: _:

overlays/cross.nix

@@ -66,6 +66,44 @@ let
     inherit (emulated) stdenv;
   };
   emulated = mkEmulated final prev;
+
+  emulateBuilder = pkg: let
+    # create a derivation would could be realized by the host system -- only.
+    binfmtDeriv = pkg.override {
+      inherit (emulated) stdenv;
+    };
+    # fix up the nixpkgs command that runs a Linux OS inside QEMU:
+    # qemu_kvm doesn't support x86_64 -> aarch64; but full qemu package does.
+    qemuCommandLinux = lib.replaceStrings
+      [ "${final.buildPackages.qemu_kvm}" ]
+      [ "${final.buildPackages.qemu}"]
+      final.vmTools.qemuCommandLinux;
+  in
+    # to use binfmt emulation, just return the derivation with emulated stdenv as usual:
+    # binfmtDeriv
+    #
+    # without binfmt emulation, leverage the `vmTools.runInLinuxVM` infrastructure:
+    # final.vmTools.runInLinuxVM pkg
+    #
+    # except `runInLinuxVM` doesn't seem to support cross compilation (what's its purpose, then?)
+    # so hack its components into something which *does* handle cross compilation
+    lib.overrideDerivation binfmtDeriv ({ builder, args, ... }: {
+      builder = "${final.buildPackages.bash}/bin/sh";
+      args = ["-e" (final.buildPackages.vmTools.vmRunCommand qemuCommandLinux)];
+      # orig{Builder,Args} gets used by the vmRunCommand script:
+      origBuilder = builder;
+      origArgs = args;
+
+      QEMU_OPTS = "-m 4096";  # MiB of RAM
+      enableParallelBuilding = true;
+
+      # finally, let nix know that this package should be built by the build system
+      system = final.stdenv.buildPlatform.system;
+    })
+    # alternatively, `proot` could let us get per-package binfmt:
+    # - <https://proot-me.github.io/>
+    # - i.e., execute host programs *and* build programs, mixed
+  ;
 in {
   inherit emulated;
 
@@ -164,18 +202,14 @@ in {
   #   # configure: error: ifconfig or ip not found, install net-tools or iproute2
   #   nativeBuildInputs = orig.nativeBuildInputs ++ [ final.iproute2 ];
   # });
-  bonsai = prev.bonsai.override {
-    inherit (emulated) stdenv;
-    hare = final.hare.override {
-      inherit (emulated) stdenv;
-      # inherit (emulated) stdenv harePackages qbe;
-      qbe = useEmulatedStdenv final.qbe;
-      harePackages.harec = final.harePackages.harec.override {
-        inherit (emulated) stdenv;
-        qbe = useEmulatedStdenv final.qbe;
-      };
-    };
-  };
+  bonsai = emulateBuilder (prev.bonsai.override {
+    hare = emulateBuilder (final.hare.override {
+      qbe = emulateBuilder final.qbe;
+      harePackages.harec = emulateBuilder (final.harePackages.harec.override {
+        qbe = emulateBuilder final.qbe;
+      });
+    });
+  });
   # bonsai = prev.bonsai.override {
   #   inherit (emulated) stdenv hare;
   # };