cross: do emulation with a smaller linux image (experimental)

overlays/cross.nix

@@ -67,15 +67,62 @@ let
   };
   emulated = mkEmulated final prev;
 
+  linuxMinimal = final.linux.override {
+    # customize stock linux to compile using less RAM
+    # default config is in:
+    # - <pkgs/os-specific/linux/kernel/common-config.nix>
+    structuredExtraConfig = with lib.kernel; {
+      # recommended by: <https://nixos.wiki/wiki/Linux_kernel#Too_high_ram_usage>
+      DEBUG_INFO_BTF = lib.mkForce no;
+
+      # other debug-related things i can probably disable
+      CC_OPTIMIZE_FOR_SIZE = lib.mkForce yes;
+      DEBUG_INFO = lib.mkForce no;
+      DEBUG_KERNEL = lib.mkForce no;
+      GDB_SCRIPTS = lib.mkForce no;
+      SCHED_DEBUG = lib.mkForce no;
+      SUNRPC_DEBUG = lib.mkForce no;
+
+      # disable un-needed features
+      BT = no;
+      CAN = no;
+      DRM = no;  # uses a lot of space when compiling
+      FPGA = no;
+      GNSS = no;
+      IIO = no;  # 500 MB
+      INPUT_TOUCHSCREEN = no;
+      MEDIA_SDR_SUPPORT = no;
+      NFC = no;
+      SND = no;  # also uses a lot of disk space when compiling
+      SOUND = no;
+      WWAN = no;  # 1.4 GB (drivers/net/wireless)
+
+      # we could try disabling these, but i wonder if anything relies on them (e.g. autoconf)
+      # FONTS = lib.mkForce no;
+      # FB = lib.mkForce no;
+      # WAN = lib.mkForce no;
+      # INET = no;
+      # MEMTEST = lib.mkForce no;
+      # # NET = lib.mkForce no;  # we need net (9pnet_virtio; unix) for sharing fs with the build machine
+      MEDIA_ANALOG_TV_SUPPORT = lib.mkForce no;
+      MEDIA_CAMERA_SUPPORT = lib.mkForce no;
+      MEDIA_DIGITAL_TV_SUPPORT = lib.mkForce no;  # 150 MB disk space when compiling
+      MICROCODE = lib.mkForce no;
+      STAGING = lib.mkForce no;  # 450 MB disk space when compiling
+    };
+  };
   # given a package that's defined for build == host,
   # build it from the native build machine by emulating the builder.
-  emulateBuilder = pkg: let
+  emulateBuilderQemu = pkg: let
+    vmTools = final.buildPackages.vmTools.override {
+      kernel = linuxMinimal;
+    };
     # fix up the nixpkgs command that runs a Linux OS inside QEMU:
     # qemu_kvm doesn't support x86_64 -> aarch64; but full qemu package does.
     qemuCommandLinux = lib.replaceStrings
       [ "${final.buildPackages.qemu_kvm}" ]
-      [ "${final.buildPackages.qemu}"]
+      [ "${final.buildPackages.qemu}" ]
-      final.vmTools.qemuCommandLinux;
+      vmTools.qemuCommandLinux;
   in
     # without binfmt emulation, leverage the `vmTools.runInLinuxVM` infrastructure:
     # final.vmTools.runInLinuxVM pkg
@@ -84,19 +131,19 @@ let
     # so hack its components into something which *does* handle cross compilation
     lib.overrideDerivation pkg ({ builder, args, ... }: {
       builder = "${final.buildPackages.bash}/bin/sh";
-      args = [ "-e" (final.buildPackages.vmTools.vmRunCommand qemuCommandLinux) ];
+      args = [ "-e" (vmTools.vmRunCommand qemuCommandLinux) ];
       # orig{Builder,Args} gets used by the vmRunCommand script:
       origBuilder = builder;
       origArgs = args;
 
-      QEMU_OPTS = "-m 16386";  # MiB of RAM
+      QEMU_OPTS = "-m 16384";  # MiB of RAM
       enableParallelBuilding = true;
 
       # finally, let nix know that this package should be built by the build system
       system = final.stdenv.buildPlatform.system;
     }) // {
-      override = attrs: emulateBuilder (pkg.override attrs);
+      override = attrs: emulateBuilderQemu (pkg.override attrs);
-      overrideAttrs = mergeFn: emulateBuilder (pkg.overrideAttrs mergeFn);
+      overrideAttrs = mergeFn: emulateBuilderQemu (pkg.overrideAttrs mergeFn);
     }
   ;
 
@@ -122,19 +169,46 @@ let
       overrideAttrs = mergeFn: emulateBuilderProot (pkg.overrideAttrs mergeFn);
     };
 
-    prootBuilder = let
+  prootBuilder = let
-      proot = "${final.buildPackages.proot}/bin/proot";
+    proot = "${final.buildPackages.proot}/bin/proot";
-      # prootFlags = "-r / -b /:/";
+    # prootFlags = "-r / -b /:/";
-      prootFlags = "-b /nix:/nix -b /tmp:/tmp";
+    prootFlags = "-b /nix:/nix -b /tmp:/tmp";
-      # prootFlags = "-b /:/ -b ${final.bash}/bin/sh:/bin/sh";  # --mixed-mode false
+    # prootFlags = "-b /:/ -b ${final.bash}/bin/sh:/bin/sh";  # --mixed-mode false
-      qemu = "${final.buildPackages.qemu}/bin/qemu-aarch64";
+    qemu = "${final.buildPackages.qemu}/bin/qemu-aarch64";
-    in
+  in
-      final.pkgs.writeText "proot-run" ''
+    final.pkgs.writeText "proot-run" ''
-        echo "proot: ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs"
+      echo "proot: ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs"
-        ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs
+      ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs
-        echo "exited proot"
+      echo "exited proot"
-      '';
+    '';
 
+  emulateBuilderBinfmt = pkg:
+    lib.overrideDerivation pkg ({ builder, args, ...}: {
+      builder = "${final.buildPackages.bash}/bin/sh";
+      args = [ "-e" binfmtBuilder ];
+      origBuilder = builder;
+      origArgs = args;
+
+      # finally, let nix know that this package should be built by the build system
+      system = final.stdenv.buildPlatform.system;
+    }) // {
+      override = attrs: emulateBuilderBinfmt (pkg.override attrs);
+      overrideAttrs = mergeFn: emulateBuilderBinfmt (pkg.overrideAttrs mergeFn);
+    };
+
+  binfmtBuilder = let
+    sudo = "${final.buildPackages.sudo}/bin/sudo";
+    mount = "${final.buildPackages.util-linux.mount}/bin/mount";
+  in
+    final.pkgs.writeText "binfmt-run" ''
+      echo "binfmtBuilder: mounting binfmt_misc"
+      ${sudo} ${mount} binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
+      echo "binfmtBuilder: running $origBuilder $origArgs"
+      $origBuilder $origArgs
+    '';
+
+  # given a package defined for build != host, transform it to build on the host.
+  # i.e. build using the host's stdenv.
   buildOnHost =
     let
       # patch packages which can't ordinarily exist in buildPackages
@@ -175,10 +249,11 @@ let
         nativeInstallCheckInputs = unsplicePkgs (upstream.nativeInstallCheckInputs or []);
       });
 
-  emulateBuildMachine = pkg: emulateBuilder (buildOnHost pkg);
+  buildInQemu = pkg: emulateBuilderQemu (buildOnHost pkg);
   buildInProot = pkg: emulateBuilderProot (buildOnHost pkg);
+  buildInBinfmt = pkg: emulateBuilderBinfmt (buildOnHost pkg);
 in {
-  inherit emulated;
+  inherit emulated linuxMinimal;
 
   # pkgsi686Linux = prev.pkgsi686Linux.extend (i686Self: i686Super: {
   #   # fixes eval-time error: "Unsupported cross architecture"
@@ -387,7 +462,7 @@ in {
     unwrapped = super.unwrapped // {
       browserpass-extension = super.unwrapped.browserpass-extension.override {
         # this overlay is optional for binfmt machines, but non-binfmt can't cross-compile the modules (for use at runtime)
-        mkYarnModules = args: buildInProot {
+        mkYarnModules = args: buildInQemu {
           override = { stdenv }: (
             (final.yarn2nix-moretea.override {
               pkgs = final.pkgs.__splicedPackages // { inherit stdenv; };
@@ -826,7 +901,7 @@ in {
   koreader = (prev.koreader.override {
     # fixes runtime error: luajit: ./ffi/util.lua:757: attempt to call field 'pack' (a nil value)
     # inherit (emulated) luajit;
-    luajit = buildInProot (final.luajit.override {
+    luajit = buildInQemu (final.luajit.override {
       buildPackages.stdenv = emulated.stdenv;  # it uses buildPackages.stdenv for HOST_CC
     });
   }).overrideAttrs (upstream: {
@@ -837,7 +912,7 @@ in {
   koreader-from-src = prev.koreader-from-src.override {
     # fixes runtime error: luajit: ./ffi/util.lua:757: attempt to call field 'pack' (a nil value)
     # inherit (emulated) luajit;
-    luajit = buildInProot (final.luajit.override {
+    luajit = buildInQemu (final.luajit.override {
       buildPackages.stdenv = emulated.stdenv;  # it uses buildPackages.stdenv for HOST_CC
     });
   };
@@ -1686,7 +1761,7 @@ in {
   tangram = (prev.tangram.override {
     # N.B. blueprint-compiler is in nativeBuildInputs.
     # the trick here is to force the aarch64 versions to be used during build (via emulation),
-    blueprint-compiler = buildInProot (final.blueprint-compiler.overrideAttrs (upstream: {
+    blueprint-compiler = buildInQemu (final.blueprint-compiler.overrideAttrs (upstream: {
       # default is to propagate gobject-introspection *as a buildInput*, when it's supposed to be native.
       propagatedBuildInputs = [];
       # "Namespace Gtk not available"