From 98463ddeb4e09db9213ed94970d39fa357eda5d5 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 11 Aug 2023 06:54:16 +0000 Subject: [PATCH] cross: do emulation with a smaller linux image (experimental) --- overlays/cross.nix | 125 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 100 insertions(+), 25 deletions(-) diff --git a/overlays/cross.nix b/overlays/cross.nix index 8dbce127..2b67be17 100644 --- a/overlays/cross.nix +++ b/overlays/cross.nix @@ -67,15 +67,62 @@ let }; emulated = mkEmulated final prev; + linuxMinimal = final.linux.override { + # customize stock linux to compile using less RAM + # default config is in: + # - + structuredExtraConfig = with lib.kernel; { + # recommended by: + DEBUG_INFO_BTF = lib.mkForce no; + + # other debug-related things i can probably disable + CC_OPTIMIZE_FOR_SIZE = lib.mkForce yes; + DEBUG_INFO = lib.mkForce no; + DEBUG_KERNEL = lib.mkForce no; + GDB_SCRIPTS = lib.mkForce no; + SCHED_DEBUG = lib.mkForce no; + SUNRPC_DEBUG = lib.mkForce no; + + # disable un-needed features + BT = no; + CAN = no; + DRM = no; # uses a lot of space when compiling + FPGA = no; + GNSS = no; + IIO = no; # 500 MB + INPUT_TOUCHSCREEN = no; + MEDIA_SDR_SUPPORT = no; + NFC = no; + SND = no; # also uses a lot of disk space when compiling + SOUND = no; + WWAN = no; # 1.4 GB (drivers/net/wireless) + + # we could try disabling these, but i wonder if anything relies on them (e.g. autoconf) + # FONTS = lib.mkForce no; + # FB = lib.mkForce no; + # WAN = lib.mkForce no; + # INET = no; + # MEMTEST = lib.mkForce no; + # # NET = lib.mkForce no; # we need net (9pnet_virtio; unix) for sharing fs with the build machine + MEDIA_ANALOG_TV_SUPPORT = lib.mkForce no; + MEDIA_CAMERA_SUPPORT = lib.mkForce no; + MEDIA_DIGITAL_TV_SUPPORT = lib.mkForce no; # 150 MB disk space when compiling + MICROCODE = lib.mkForce no; + STAGING = lib.mkForce no; # 450 MB disk space when compiling + }; + }; # given a package that's defined for build == host, # build it from the native build machine by emulating the builder. - emulateBuilder = pkg: let + emulateBuilderQemu = pkg: let + vmTools = final.buildPackages.vmTools.override { + kernel = linuxMinimal; + }; # fix up the nixpkgs command that runs a Linux OS inside QEMU: # qemu_kvm doesn't support x86_64 -> aarch64; but full qemu package does. qemuCommandLinux = lib.replaceStrings [ "${final.buildPackages.qemu_kvm}" ] - [ "${final.buildPackages.qemu}"] - final.vmTools.qemuCommandLinux; + [ "${final.buildPackages.qemu}" ] + vmTools.qemuCommandLinux; in # without binfmt emulation, leverage the `vmTools.runInLinuxVM` infrastructure: # final.vmTools.runInLinuxVM pkg @@ -84,19 +131,19 @@ let # so hack its components into something which *does* handle cross compilation lib.overrideDerivation pkg ({ builder, args, ... }: { builder = "${final.buildPackages.bash}/bin/sh"; - args = [ "-e" (final.buildPackages.vmTools.vmRunCommand qemuCommandLinux) ]; + args = [ "-e" (vmTools.vmRunCommand qemuCommandLinux) ]; # orig{Builder,Args} gets used by the vmRunCommand script: origBuilder = builder; origArgs = args; - QEMU_OPTS = "-m 16386"; # MiB of RAM + QEMU_OPTS = "-m 16384"; # MiB of RAM enableParallelBuilding = true; # finally, let nix know that this package should be built by the build system system = final.stdenv.buildPlatform.system; }) // { - override = attrs: emulateBuilder (pkg.override attrs); - overrideAttrs = mergeFn: emulateBuilder (pkg.overrideAttrs mergeFn); + override = attrs: emulateBuilderQemu (pkg.override attrs); + overrideAttrs = mergeFn: emulateBuilderQemu (pkg.overrideAttrs mergeFn); } ; @@ -122,19 +169,46 @@ let overrideAttrs = mergeFn: emulateBuilderProot (pkg.overrideAttrs mergeFn); }; - prootBuilder = let - proot = "${final.buildPackages.proot}/bin/proot"; - # prootFlags = "-r / -b /:/"; - prootFlags = "-b /nix:/nix -b /tmp:/tmp"; - # prootFlags = "-b /:/ -b ${final.bash}/bin/sh:/bin/sh"; # --mixed-mode false - qemu = "${final.buildPackages.qemu}/bin/qemu-aarch64"; - in - final.pkgs.writeText "proot-run" '' - echo "proot: ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs" - ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs - echo "exited proot" - ''; + prootBuilder = let + proot = "${final.buildPackages.proot}/bin/proot"; + # prootFlags = "-r / -b /:/"; + prootFlags = "-b /nix:/nix -b /tmp:/tmp"; + # prootFlags = "-b /:/ -b ${final.bash}/bin/sh:/bin/sh"; # --mixed-mode false + qemu = "${final.buildPackages.qemu}/bin/qemu-aarch64"; + in + final.pkgs.writeText "proot-run" '' + echo "proot: ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs" + ${proot} -q ${qemu} ${prootFlags} $origBuilder $origArgs + echo "exited proot" + ''; + emulateBuilderBinfmt = pkg: + lib.overrideDerivation pkg ({ builder, args, ...}: { + builder = "${final.buildPackages.bash}/bin/sh"; + args = [ "-e" binfmtBuilder ]; + origBuilder = builder; + origArgs = args; + + # finally, let nix know that this package should be built by the build system + system = final.stdenv.buildPlatform.system; + }) // { + override = attrs: emulateBuilderBinfmt (pkg.override attrs); + overrideAttrs = mergeFn: emulateBuilderBinfmt (pkg.overrideAttrs mergeFn); + }; + + binfmtBuilder = let + sudo = "${final.buildPackages.sudo}/bin/sudo"; + mount = "${final.buildPackages.util-linux.mount}/bin/mount"; + in + final.pkgs.writeText "binfmt-run" '' + echo "binfmtBuilder: mounting binfmt_misc" + ${sudo} ${mount} binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc + echo "binfmtBuilder: running $origBuilder $origArgs" + $origBuilder $origArgs + ''; + + # given a package defined for build != host, transform it to build on the host. + # i.e. build using the host's stdenv. buildOnHost = let # patch packages which can't ordinarily exist in buildPackages @@ -175,10 +249,11 @@ let nativeInstallCheckInputs = unsplicePkgs (upstream.nativeInstallCheckInputs or []); }); - emulateBuildMachine = pkg: emulateBuilder (buildOnHost pkg); + buildInQemu = pkg: emulateBuilderQemu (buildOnHost pkg); buildInProot = pkg: emulateBuilderProot (buildOnHost pkg); + buildInBinfmt = pkg: emulateBuilderBinfmt (buildOnHost pkg); in { - inherit emulated; + inherit emulated linuxMinimal; # pkgsi686Linux = prev.pkgsi686Linux.extend (i686Self: i686Super: { # # fixes eval-time error: "Unsupported cross architecture" @@ -387,7 +462,7 @@ in { unwrapped = super.unwrapped // { browserpass-extension = super.unwrapped.browserpass-extension.override { # this overlay is optional for binfmt machines, but non-binfmt can't cross-compile the modules (for use at runtime) - mkYarnModules = args: buildInProot { + mkYarnModules = args: buildInQemu { override = { stdenv }: ( (final.yarn2nix-moretea.override { pkgs = final.pkgs.__splicedPackages // { inherit stdenv; }; @@ -826,7 +901,7 @@ in { koreader = (prev.koreader.override { # fixes runtime error: luajit: ./ffi/util.lua:757: attempt to call field 'pack' (a nil value) # inherit (emulated) luajit; - luajit = buildInProot (final.luajit.override { + luajit = buildInQemu (final.luajit.override { buildPackages.stdenv = emulated.stdenv; # it uses buildPackages.stdenv for HOST_CC }); }).overrideAttrs (upstream: { @@ -837,7 +912,7 @@ in { koreader-from-src = prev.koreader-from-src.override { # fixes runtime error: luajit: ./ffi/util.lua:757: attempt to call field 'pack' (a nil value) # inherit (emulated) luajit; - luajit = buildInProot (final.luajit.override { + luajit = buildInQemu (final.luajit.override { buildPackages.stdenv = emulated.stdenv; # it uses buildPackages.stdenv for HOST_CC }); }; @@ -1686,7 +1761,7 @@ in { tangram = (prev.tangram.override { # N.B. blueprint-compiler is in nativeBuildInputs. # the trick here is to force the aarch64 versions to be used during build (via emulation), - blueprint-compiler = buildInProot (final.blueprint-compiler.overrideAttrs (upstream: { + blueprint-compiler = buildInQemu (final.blueprint-compiler.overrideAttrs (upstream: { # default is to propagate gobject-introspection *as a buildInput*, when it's supposed to be native. propagatedBuildInputs = []; # "Namespace Gtk not available"