Merge pull request #185036 from K900/goblinization

make-initrd-ng: parse ELFs ourselves instead of shelling out to patchelf and friends

nixos/modules/system/boot/systemd/initrd.nix

@@ -129,6 +129,7 @@ let
   initialRamdisk = pkgs.makeInitrdNG {
     name = "initrd-${kernel-name}";
     inherit (config.boot.initrd) compressor compressorArgs prepend;
+    inherit (cfg) strip;
 
     contents = map (path: { object = path; symlink = ""; }) (subtractLists cfg.suppressedStorePaths cfg.storePaths)
       ++ mapAttrsToList (_: v: { object = v.source; symlink = v.target; }) (filterAttrs (_: v: v.enable) cfg.contents);
@@ -169,6 +170,19 @@ in {
       default = [];
     };
 
+    strip = mkOption {
+      description = lib.mdDoc ''
+        Whether to completely strip executables and libraries copied to the initramfs.
+
+        Setting this to false may save on the order of 30MiB on the
+        machine building the system (by avoiding a binutils
+        reference), at the cost of ~1MiB of initramfs size. This puts
+        this option firmly in the territory of micro-optimisation.
+      '';
+      type = types.bool;
+      default = true;
+    };
+
     extraBin = mkOption {
       description = lib.mdDoc ''
         Tools to add to /bin

pkgs/build-support/kernel/make-initrd-ng-tool.nix

@@ -7,10 +7,11 @@ rustPlatform.buildRustPackage {
   src = ./make-initrd-ng;
   cargoLock.lockFile = ./make-initrd-ng/Cargo.lock;
 
-  nativeBuildInputs = [ makeWrapper ];
+  passthru.updateScript = ./make-initrd-ng/update.sh;
 
-  postInstall = ''
-    wrapProgram $out/bin/make-initrd-ng \
-      --prefix PATH : ${lib.makeBinPath [ patchelf glibc binutils ]}
-  '';
+  meta = {
+    description = "Tool for copying binaries and their dependencies";
+    maintainers = with lib.maintainers; [ das_j elvishjerricco k900 lheckemann ];
+    license = lib.licenses.mit;
+  };
 }

pkgs/build-support/kernel/make-initrd-ng.nix

@@ -8,10 +8,12 @@ let
   # compression type and filename extension.
   compressorName = fullCommand: builtins.elemAt (builtins.match "([^ ]*/)?([^ ]+).*" fullCommand) 1;
 in
-{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost, makeInitrdNGTool, patchelf, runCommand
+{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost, makeInitrdNGTool, binutils, runCommand
 # Name of the derivation (not of the resulting file!)
 , name ? "initrd"
 
+, strip ? true
+
 # Program used to compress the cpio archive; use "cat" for no compression.
 # This can also be a function which takes a package set and returns the path to the compressor,
 # such as `pkgs: "${pkgs.lzop}/bin/lzop"`.
@@ -59,7 +61,7 @@ in
 # If this isn't guessed, you may want to complete the metadata above and send a PR :)
 , uInitrdCompression ? _compressorMeta.ubootName or
     (throw "Unrecognised compressor ${_compressorName}, please specify uInitrdCompression")
-}: runCommand name {
+}: runCommand name ({
   compress = "${_compressorExecutable} ${lib.escapeShellArgs _compressorArgsReal}";
   passthru = {
     compressorExecutableFunction = _compressorFunction;
@@ -72,8 +74,10 @@ in
   passAsFile = ["contents"];
   contents = lib.concatMapStringsSep "\n" ({ object, symlink, ... }: "${object}\n${if symlink == null then "" else symlink}") contents + "\n";
 
-  nativeBuildInputs = [makeInitrdNGTool patchelf cpio] ++ lib.optional makeUInitrd ubootTools;
-} ''
+  nativeBuildInputs = [makeInitrdNGTool cpio] ++ lib.optional makeUInitrd ubootTools ++ lib.optional strip binutils;
+
+  STRIP = if strip then "${(binutils.nativeDrv or binutils).targetPrefix}strip" else null;
+}) ''
   mkdir ./root
   make-initrd-ng "$contentsPath" ./root
   mkdir "$out"

pkgs/build-support/kernel/make-initrd-ng/Cargo.lock

@@ -1,5 +1,97 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "goblin"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91766b1121940d622933a13e20665857648681816089c9bc2075c4b75a6e4f6b"
+dependencies = [
+ "log",
+ "plain",
+ "scroll",
+]
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "make-initrd-ng"
 version = "0.1.0"
+dependencies = [
+ "goblin",
+]
+
+[[package]]
+name = "plain"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "scroll"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da"
+dependencies = [
+ "scroll_derive",
+]
+
+[[package]]
+name = "scroll_derive"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7"

pkgs/build-support/kernel/make-initrd-ng/Cargo.toml

@@ -7,3 +7,4 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+goblin = "0.5.0"

pkgs/build-support/kernel/make-initrd-ng/src/main.rs

@@ -3,11 +3,13 @@ use std::env;
 use std::ffi::OsStr;
 use std::fs;
 use std::hash::Hash;
-use std::io::{BufReader, BufRead, Error, ErrorKind};
+use std::io::{BufRead, BufReader, Error};
 use std::os::unix;
 use std::path::{Component, Path, PathBuf};
 use std::process::Command;
 
+use goblin::{elf::Elf, Object};
+
 struct NonRepeatingQueue<T> {
     queue: VecDeque<T>,
     seen: HashSet<T>,
@@ -38,42 +40,32 @@ impl<T: Clone + Eq + Hash> NonRepeatingQueue<T> {
     }
 }
 
-fn patch_elf<S: AsRef<OsStr>, P: AsRef<OsStr>>(mode: S, path: P) -> Result<String, Error> {
-    let output = Command::new("patchelf")
-        .arg(&mode)
-        .arg(&path)
-        .output()?;
-    if output.status.success() {
-        Ok(String::from_utf8(output.stdout).expect("Failed to parse output"))
-    } else {
-        Err(Error::new(ErrorKind::Other, format!("failed: patchelf {:?} {:?}", OsStr::new(&mode), OsStr::new(&path))))
-    }
-}
-
-fn copy_file<P: AsRef<Path> + AsRef<OsStr>, S: AsRef<Path> + AsRef<OsStr>>(
+fn add_dependencies<P: AsRef<Path> + AsRef<OsStr>>(
     source: P,
-    target: S,
+    elf: Elf,
     queue: &mut NonRepeatingQueue<Box<Path>>,
-) -> Result<(), Error> {
-    fs::copy(&source, &target)?;
-
-    if !Command::new("ldd").arg(&source).output()?.status.success() {
-        // Not dynamically linked - no need to recurse
-        return Ok(());
+) {
+    if let Some(interp) = elf.interpreter {
+        queue.push_back(Box::from(Path::new(interp)));
     }
 
-    let rpath_string = patch_elf("--print-rpath", &source)?;
-    let needed_string = patch_elf("--print-needed", &source)?;
-    // Shared libraries don't have an interpreter
-    if let Ok(interpreter_string) = patch_elf("--print-interpreter", &source) {
-        queue.push_back(Box::from(Path::new(&interpreter_string.trim())));
-    }
+    let rpaths = if elf.runpaths.len() > 0 {
+        elf.runpaths
+    } else if elf.rpaths.len() > 0 {
+        elf.rpaths
+    } else {
+        vec![]
+    };
 
-    let rpath = rpath_string.trim().split(":").map(|p| Box::<Path>::from(Path::new(p))).collect::<Vec<_>>();
+    let rpaths_as_path = rpaths
+        .into_iter()
+        .flat_map(|p| p.split(":"))
+        .map(|p| Box::<Path>::from(Path::new(p)))
+        .collect::<Vec<_>>();
 
-    for line in needed_string.lines() {
+    for line in elf.libraries {
         let mut found = false;
-        for path in &rpath {
+        for path in &rpaths_as_path {
             let lib = path.join(line);
             if lib.exists() {
                 // No need to recurse. The queue will bring it back round.
@@ -85,20 +77,45 @@ fn copy_file<P: AsRef<Path> + AsRef<OsStr>, S: AsRef<Path> + AsRef<OsStr>>(
         if !found {
             // glibc makes it tricky to make this an error because
             // none of the files have a useful rpath.
-            println!("Warning: Couldn't satisfy dependency {} for {:?}", line, OsStr::new(&source));
+            println!(
+                "Warning: Couldn't satisfy dependency {} for {:?}",
+                line,
+                OsStr::new(&source)
+            );
         }
     }
+}
 
-    // Make file writable to strip it
-    let mut permissions = fs::metadata(&target)?.permissions();
-    permissions.set_readonly(false);
-    fs::set_permissions(&target, permissions)?;
+fn copy_file<P: AsRef<Path> + AsRef<OsStr>, S: AsRef<Path> + AsRef<OsStr>>(
+    source: P,
+    target: S,
+    queue: &mut NonRepeatingQueue<Box<Path>>,
+) -> Result<(), Error> {
+    fs::copy(&source, &target)?;
 
-    // Strip further than normal
-    if !Command::new("strip").arg("--strip-all").arg(OsStr::new(&target)).output()?.status.success() {
-        println!("{:?} was not successfully stripped.", OsStr::new(&target));
-    }
+    let contents = fs::read(&source)?;
 
+    if let Ok(Object::Elf(e)) = Object::parse(&contents) {
+        add_dependencies(source, e, queue);
+
+        // Make file writable to strip it
+        let mut permissions = fs::metadata(&target)?.permissions();
+        permissions.set_readonly(false);
+        fs::set_permissions(&target, permissions)?;
+
+        // Strip further than normal
+        if let Ok(strip) = env::var("STRIP") {
+            if !Command::new(strip)
+                .arg("--strip-all")
+                .arg(OsStr::new(&target))
+                .output()?
+                .status
+                .success()
+            {
+                println!("{:?} was not successfully stripped.", OsStr::new(&target));
+            }
+        }
+    };
 
     Ok(())
 }

pkgs/build-support/kernel/make-initrd-ng/update.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p cargo -i bash
+cd "$(dirname "$0")"
+cargo update