From 88f192b24721bb872c573d524751018de92f4e9e Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:21:34 +0000 Subject: [PATCH 1/4] stdenv.darwin.make-boostrap-tools: refactor to use getBin, getDev and getLib --- pkgs/stdenv/darwin/make-bootstrap-tools.nix | 121 +++++++++++--------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index 27f138e75bfb..d00e6d939259 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -41,106 +41,113 @@ rec { nativeBuildInputs = [ nukeReferences dumpnar ]; - buildCommand = '' + buildCommand = let + inherit (lib) + getBin + getDev + getLib + ; + in + '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin ${lib.optionalString stdenv.targetPlatform.isx86_64 '' # Copy libSystem's .o files for various low-level boot stuff. - cp -d ${lib.getLib darwin.Libsystem}/lib/*.o $out/lib + cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib # Resolv is actually a link to another package, so let's copy it properly - cp -L ${lib.getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib + cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib ''} - cp -rL ${darwin.Libsystem}/include $out + cp -rL ${getDev darwin.Libsystem}/include $out chmod -R u+w $out/include - cp -rL ${darwin.ICU}/include* $out/include - cp -rL ${libiconv}/include/* $out/include - cp -rL ${lib.getDev gnugrep.pcre2}/include/* $out/include + cp -rL ${getDev darwin.ICU}/include* $out/include + cp -rL ${getDev libiconv}/include/* $out/include + cp -rL ${getDev gnugrep.pcre2}/include/* $out/include mv $out/include $out/include-Libsystem # Copy coreutils, bash, etc. - cp ${coreutils_}/bin/* $out/bin + cp ${getBin coreutils_}/bin/* $out/bin (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) - cp ${bash}/bin/bash $out/bin + cp ${getBin bash}/bin/bash $out/bin ln -s bash $out/bin/sh - cp ${findutils}/bin/find $out/bin - cp ${findutils}/bin/xargs $out/bin - cp -d ${diffutils}/bin/* $out/bin - cp -d ${gnused}/bin/* $out/bin - cp -d ${gnugrep}/bin/grep $out/bin - cp ${gawk}/bin/gawk $out/bin - cp -d ${gawk}/bin/awk $out/bin - cp ${gnutar}/bin/tar $out/bin - cp ${gzip}/bin/.gzip-wrapped $out/bin/gzip - cp ${bzip2_.bin}/bin/bzip2 $out/bin + cp ${getBin findutils}/bin/find $out/bin + cp ${getBin findutils}/bin/xargs $out/bin + cp -d ${getBin diffutils}/bin/* $out/bin + cp -d ${getBin gnused}/bin/* $out/bin + cp -d ${getBin gnugrep}/bin/grep $out/bin + cp ${getBin gawk}/bin/gawk $out/bin + cp -d ${getBin gawk}/bin/awk $out/bin + cp ${getBin gnutar}/bin/tar $out/bin + cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip + cp ${getBin bzip2_}/bin/bzip2 $out/bin ln -s bzip2 $out/bin/bunzip2 - cp -d ${gnumake}/bin/* $out/bin - cp -d ${patch}/bin/* $out/bin - cp -d ${xz.bin}/bin/xz $out/bin - cp ${cpio}/bin/cpio $out/bin + cp -d ${getBin gnumake}/bin/* $out/bin + cp -d ${getBin patch}/bin/* $out/bin + cp -d ${getBin xz}/bin/xz $out/bin + cp ${getBin cpio}/bin/cpio $out/bin # This used to be in-nixpkgs, but now is in the bundle # because I can't be bothered to make it partially static - cp ${curl_.bin}/bin/curl $out/bin - cp -d ${curl_.out}/lib/libcurl*.dylib $out/lib - cp -d ${libssh2.out}/lib/libssh*.dylib $out/lib - cp -d ${lib.getLib openssl}/lib/*.dylib $out/lib + cp ${getBin curl_}/bin/curl $out/bin + cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib + cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib + cp -d ${getLib openssl}/lib/*.dylib $out/lib - cp -d ${gnugrep.pcre2.out}/lib/libpcre2*.dylib $out/lib - cp -d ${lib.getLib libiconv}/lib/lib*.dylib $out/lib - cp -d ${lib.getLib gettext}/lib/libintl*.dylib $out/lib + cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib + cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib + cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib chmod +x $out/lib/libintl*.dylib - cp -d ${ncurses.out}/lib/libncurses*.dylib $out/lib - cp -d ${libxml2.out}/lib/libxml2*.dylib $out/lib + cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib + cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib # Copy what we need of clang - cp -d ${llvmPackages.clang-unwrapped}/bin/clang* $out/bin - cp -rd ${lib.getLib llvmPackages.clang-unwrapped}/lib/* $out/lib + cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin + cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib - cp -d ${lib.getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib + cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib '' # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11 + lib.optionalString (llvmPackages ? libcxxabi) '' - cp -d ${lib.getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib + cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib '' + '' - cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin - cp -d ${lib.getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib - cp -d ${lib.getLib llvmPackages.llvm.lib}/lib/libLLVM.dylib $out/lib - cp -d ${lib.getLib libffi}/lib/libffi*.dylib $out/lib + cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib + cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib + cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib mkdir $out/include - cp -rd ${llvmPackages.libcxx.dev}/include/c++ $out/include + cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include # copy .tbd assembly utils - cp -d ${pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin - cp -d ${lib.getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib + cp -d ${getBin pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin + cp -d ${getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib # copy package extraction tools - cp -d ${pkgs.pbzx}/bin/pbzx $out/bin - cp -d ${lib.getLib pkgs.xar}/lib/libxar*.dylib $out/lib - cp -d ${pkgs.bzip2.out}/lib/libbz2*.dylib $out/lib + cp -d ${getBin pkgs.pbzx}/bin/pbzx $out/bin + cp -d ${getLib pkgs.xar}/lib/libxar*.dylib $out/lib + cp -d ${getLib pkgs.bzip2}/lib/libbz2*.dylib $out/lib # copy sigtool - cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin - cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin + cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin + cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin - cp -d ${lib.getLib darwin.ICU}/lib/libicu*.dylib $out/lib - cp -d ${zlib.out}/lib/libz.* $out/lib - cp -d ${gmpxx.out}/lib/libgmp*.* $out/lib - cp -d ${xz.out}/lib/liblzma*.* $out/lib + cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib + cp -d ${getLib zlib}/lib/libz.* $out/lib + cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib + cp -d ${getLib xz}/lib/liblzma*.* $out/lib # Copy binutils. for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do - cp ${cctools_}/bin/$i $out/bin + cp ${getBin cctools_}/bin/$i $out/bin done - cp -d ${lib.getLib darwin.libtapi}/lib/libtapi* $out/lib + cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib - cp -rd ${pkgs.darwin.CF}/Library $out + cp -rd ${getLib pkgs.darwin.CF}/Library $out ${lib.optionalString stdenv.targetPlatform.isAarch64 '' - cp -rd ${pkgs.darwin.libobjc}/lib/* $out/lib/ + cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/ ''} chmod -R u+w $out @@ -194,7 +201,7 @@ rec { mv $out/.pack $out/pack mkdir $out/on-server - dumpnar $out/pack | ${xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz + dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz ''; allowedReferences = []; From dcc63e54a4e95eda404e30e9161b2707662dbdb3 Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:24:59 +0000 Subject: [PATCH 2/4] maintainers/scripts/bootstrap-files: add darwin targets; nar downloads - allow for fetching and expanding nar archives - add targets for aarch64 and x86_64 darwin --- .../bootstrap-files/refresh-tarballs.bash | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/maintainers/scripts/bootstrap-files/refresh-tarballs.bash b/maintainers/scripts/bootstrap-files/refresh-tarballs.bash index 775d7ef1379d..4b0380f09745 100755 --- a/maintainers/scripts/bootstrap-files/refresh-tarballs.bash +++ b/maintainers/scripts/bootstrap-files/refresh-tarballs.bash @@ -67,14 +67,8 @@ NATIVE_TARGETS=( i686-unknown-linux-gnu x86_64-unknown-linux-gnu x86_64-unknown-linux-musl - - # TODO: add darwin here once a few prerequisites are satisfied: - # - bootstrap-files are factored out into a separate file - # - the build artifacts are factored out into an `on-server` - # directory. Right onw if does not match `linux` layout. - # - #aarch64-apple-darwin - #x86_64-apple-darwin + aarch64-apple-darwin + x86_64-apple-darwin ) is_native() { @@ -106,6 +100,18 @@ is_cross() { return 1 } +nar_sri_get() { + local ouput sri + output=$(nix-build --expr \ + 'import { + url = "'"$1"'"; + unpack = true; + }' 2>&1 || true) + sri=$(echo "$output" | awk '/^\s+got:\s+/{ print $2 }') + [[ -z "$sri" ]] && die "$output" + echo "$sri" +} + # collect passed options targets=() @@ -222,6 +228,7 @@ EOF case "$fname" in bootstrap-tools.tar.xz) attr=bootstrapTools ;; busybox) attr=$fname ;; + unpack.nar.xz) attr=unpack ;; *) die "Don't know how to map '$fname' to attribute name. Please update me." esac @@ -229,18 +236,27 @@ EOF executable_nix= if [[ -x "$p" ]]; then executable_arg="--executable" - executable_nix=" executable = true;" + executable_nix="executable = true;" + fi + unpack_nix= + if [[ $fname = *.nar.* ]]; then + unpack_nix="unpack = true;" + sri=$(nar_sri_get "file://$p") + else + sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p") + [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'" + sri=$(nix-hash --to-sri "sha256:$sha256") + [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form" fi - sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p") - [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'" - sri=$(nix-hash --to-sri "sha256:$sha256") - [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form" # individual file entries cat < { url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname"; - hash = "${sri}";$(printf "\n%s" "${executable_nix}") + hash = "${sri}";$( + [[ -n ${executable_nix} ]] && printf "\n %s" "${executable_nix}" + [[ -n ${unpack_nix} ]] && printf "\n %s" "${unpack_nix}" +) }; EOF done From b31083035992f8090f20b6f321420c1caa08c1b6 Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Mon, 11 Mar 2024 23:02:27 +0000 Subject: [PATCH 3/4] stdenv.darwin.make-boostrap-tools: update tools and unpin LLVM11 - unpin LLVM11. fix discrepancy with freshBootstrapTools and the tools built on hydra. pinning the stdenv for the hydra build doesn't pin the tools as the included packages are able to change. - remove unused LLVM tools & libs which reduces the uncompressed and compressed file sizes by more than 1/2. compressed tarball is now 40M and uncompressed is around 200M - add @loader_path/. to dylibs that reference other libs in the archive. this is needed for libraries with re-exports. - validate shared objects with @rpath references contain the reference in lib - add a test to verify that the @loader_path/ works for libc++ as it re-exports libc++abi --- pkgs/stdenv/darwin/make-bootstrap-tools.nix | 251 ++++++++++-------- ...tools-next.sh => patch-bootstrap-tools.sh} | 7 +- 2 files changed, 149 insertions(+), 109 deletions(-) rename pkgs/stdenv/darwin/{patch-bootstrap-tools-next.sh => patch-bootstrap-tools.sh} (91%) diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index d00e6d939259..7725665ae436 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -16,30 +16,10 @@ let cross = if crossSystem != null in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); rec { - coreutils_ = (coreutils.override (args: { - # We want coreutils without ACL support. - aclSupport = false; - # Cannot use a single binary build, or it gets dynamically linked against gmp. - singleBinary = false; - })).overrideAttrs (oa: { - # Increase header size to be able to inject extra RPATHs. Otherwise - # x86_64-darwin build fails as: - # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv - NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; - }); - - cctools_ = darwin.cctools; - - # Avoid debugging larger changes for now. - bzip2_ = bzip2.override (args: { enableStatic = true; enableShared = false; }); - - # Avoid messing with libkrb5 and libnghttp2. - curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; }); - build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; - nativeBuildInputs = [ nukeReferences dumpnar ]; + nativeBuildInputs = [ dumpnar nukeReferences ]; buildCommand = let inherit (lib) @@ -47,6 +27,35 @@ rec { getDev getLib ; + + coreutils_ = (coreutils.override (args: { + # We want coreutils without ACL support. + aclSupport = false; + # Cannot use a single binary build, or it gets dynamically linked against gmp. + singleBinary = false; + })).overrideAttrs (oa: { + # Increase header size to be able to inject extra RPATHs. Otherwise + # x86_64-darwin build fails as: + # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv + NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; + }); + + cctools_ = darwin.cctools; + + # Avoid messing with libkrb5 and libnghttp2. + curl_ = curlMinimal.override (args: { + gssSupport = false; + http2Support = false; + scpSupport = false; + }); + + gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: { + configureFlags = [ + "--disable-nls" + ] ++ old.configureFlags or []; + }); + + xz_ = xz.override { enableStatic = true; }; in '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin @@ -61,7 +70,6 @@ rec { cp -rL ${getDev darwin.Libsystem}/include $out chmod -R u+w $out/include - cp -rL ${getDev darwin.ICU}/include* $out/include cp -rL ${getDev libiconv}/include/* $out/include cp -rL ${getDev gnugrep.pcre2}/include/* $out/include mv $out/include $out/include-Libsystem @@ -81,7 +89,7 @@ rec { cp -d ${getBin gawk}/bin/awk $out/bin cp ${getBin gnutar}/bin/tar $out/bin cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip - cp ${getBin bzip2_}/bin/bzip2 $out/bin + cp ${getBin bzip2}/bin/bzip2 $out/bin ln -s bzip2 $out/bin/bunzip2 cp -d ${getBin gnumake}/bin/* $out/bin cp -d ${getBin patch}/bin/* $out/bin @@ -92,7 +100,6 @@ rec { # because I can't be bothered to make it partially static cp ${getBin curl_}/bin/curl $out/bin cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib - cp -d ${getLib libssh2}/lib/libssh*.dylib $out/lib cp -d ${getLib openssl}/lib/*.dylib $out/lib cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib @@ -103,17 +110,14 @@ rec { cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib # Copy what we need of clang - cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang* $out/bin - cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/* $out/lib + cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin + cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib + cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib - '' - # libc++abi is contained in libcxx for LLVM12+. Remove once unpinned from LLVM11 - + lib.optionalString (llvmPackages ? libcxxabi) '' - cp -d ${getLib llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib - '' + '' - cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin - cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib + mkdir -p $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib @@ -133,7 +137,6 @@ rec { cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin - cp -d ${getLib darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${getLib zlib}/lib/libz.* $out/lib cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib cp -d ${getLib xz}/lib/liblzma*.* $out/lib @@ -145,63 +148,103 @@ rec { cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib - cp -rd ${getLib pkgs.darwin.CF}/Library $out - ${lib.optionalString stdenv.targetPlatform.isAarch64 '' - cp -rd ${getLib pkgs.darwin.libobjc}/lib/* $out/lib/ - ''} + # tools needed to unpack bootstrap archive. they should not contain any + # external references. we will process them like the other tools but + # perform some additional checks and will not pack them into the archive. + mkdir -p unpack/bin + cp ${getBin bash}/bin/bash unpack/bin + ln -s bash unpack/bin/sh + cp ${getBin coreutils_}/bin/mkdir unpack/bin + cp ${getBin gnutar_}/bin/tar unpack/bin + cp ${getBin xz_}/bin/xz unpack/bin - chmod -R u+w $out + # + # All files copied. Perform processing to update references to point into + # the archive + # - nuke-refs $out/bin/* + chmod -R u+w $out unpack + # - change nix store library paths to use @rpath/library + # - if needed add an rpath containing lib/ + # - strip executable rpathify() { - local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true - local newlib + local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) + local lib rpath for lib in $libs; do ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" done + + case "$(dirname "$1")" in + */bin) + # Strip executables even further + ${stdenv.cc.targetPrefix}strip "$i" + rpath='@executable_path/../lib' + ;; + */lib) + # the '/.' suffix is required + rpath='@loader_path/.' + ;; + */lib/darwin) + rpath='@loader_path/..' + ;; + *) + echo unkown executable $1 >&2 + exit 1 + ;; + esac + + # if shared object contains references add an rpath to lib/ + if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then + ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" + fi } - # Strip executables even further - for i in $out/bin/*; do + # check that linked library paths exist in $out/lib + # must be run after rpathify is performed + checkDeps() { + local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) + local lib + for lib in $deps; do + if [[ ! -e $out/''${lib/@rpath/lib} ]]; then + echo "error: $1 missing lib for $lib" >&2 + exit 1 + fi + done + } + + for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do if [[ ! -L $i ]] && isMachO "$i"; then - chmod +w $i - ${stdenv.cc.targetPrefix}strip $i || true + rpathify "$i" + checkDeps "$i" fi done - for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib; do - if [[ ! -L "$i" ]]; then - rpathify $i - fi - done - - for i in $out/bin/*; do - if [[ ! -L "$i" ]] && isMachO "$i"; then - ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i - fi - done - - ${if stdenv.targetPlatform.isx86_64 then '' - rpathify $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation - '' else '' - sed -i -e 's|/nix/store/.*/libobjc.A.dylib|@executable_path/../libobjc.A.dylib|g' \ - $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation.tbd - ''} - + nuke-refs $out/bin/* nuke-refs $out/lib/* - nuke-refs $out/lib/system/* nuke-refs $out/lib/darwin/* - ${lib.optionalString stdenv.targetPlatform.isx86_64 '' - nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation - ''} + nuke-refs $out/lib/system/* + nuke-refs unpack/bin/* mkdir $out/.pack mv $out/* $out/.pack mv $out/.pack $out/pack + # validate that tools contain no references into the archive + for tool in unpack/bin/*; do + deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true) + if [[ -n "$deps" ]]; then + printf "error: $tool is not self contained\n$deps\n" >&2 + exit 1 + fi + done + mkdir $out/on-server - dumpnar $out/pack | ${getBin xz}/bin/xz > $out/on-server/bootstrap-tools.nar.xz + cp -r unpack $out + + XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ + --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . + dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz ''; allowedReferences = []; @@ -211,44 +254,42 @@ rec { }; }; - dist = stdenv.mkDerivation { - name = "stdenv-bootstrap-tools"; - - buildCommand = '' - mkdir -p $out/nix-support - echo "file tools ${build}/on-server/bootstrap-tools.nar.xz" >> $out/nix-support/hydra-build-products - ''; - }; + dist = runCommand "stdenv-bootstrap-tools" {} '' + mkdir -p $out/nix-support + echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products + echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products + ''; bootstrapFiles = { - tools = "${build}/pack"; + bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; + unpack = runCommand "unpack" { allowedReferences = []; } '' + cp -r ${build}/unpack $out + ''; }; bootstrapTools = derivation { inherit (stdenv.hostPlatform) system; name = "bootstrap-tools"; - builder = "${bootstrapFiles.tools}/bin/bash"; + builder = "${bootstrapFiles.unpack}/bin/bash"; - # This is by necessity a near-duplicate of patch-bootstrap-tools.sh. If we refer to it directly, - # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the - # patch-bootstrap-tools.sh for the next round of bootstrap tools. - args = [ ./patch-bootstrap-tools-next.sh ]; - - inherit (bootstrapFiles) tools; + args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ]; + PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ]; allowedReferences = [ "out" ]; }; - test = stdenv.mkDerivation { - name = "test"; - - realBuilder = "${bootstrapTools}/bin/bash"; - + test = derivation { + name = "test-bootstrap-tools"; + inherit (stdenv.hostPlatform) system; + builder = "${bootstrapTools}/bin/bash"; + args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ]; + PATH = lib.makeBinPath [ bootstrapTools ]; tools = bootstrapTools; + "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion; + + # Create a pure environment where we use just what's in the bootstrap tools. buildCommand = '' - # Create a pure environment where we use just what's in the bootstrap tools. - export PATH=$tools/bin ls -l mkdir $out @@ -275,16 +316,15 @@ rec { ${stdenv.cc.libc_dev}/lib/system \ libSystem-boot - substituteInPlace libSystem-boot/libSystem.B.tbd \ - --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/" + sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd # End of bootstrap libSystem export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" - export CC="clang $flags -rpath $tools/lib" - export CXX="clang++ $flags --stdlib=libc++ -lc++abi -isystem$tools/include/c++/v1 -rpath $tools/lib" + export CC="clang $flags" + export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1" # NOTE: These tests do a separate 'install' step (using cp), because # having clang write directly to the final location apparently will make @@ -301,22 +341,23 @@ rec { cp hello1 $out/bin/ $out/bin/hello1 - echo '#include ' >> hello2.c - echo 'int main() { CFShow(CFSTR("Hullo")); return 0; }' >> hello2.c - $CC -F$tools/Library/Frameworks -framework CoreFoundation -o hello2 hello2.c - cp hello2 $out/bin/ - $out/bin/hello2 - echo '#include ' >> hello3.cc echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc $CXX -v -o hello3 hello3.cc cp hello3 $out/bin/ $out/bin/hello3 + # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked. + # using -Wl,-flat_namespace is required to generate an error + mkdir libtest/ + ln -s $tools/lib/libc++.dylib libtest/ + clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \ + --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc + tar xvf ${hello.src} cd hello-* - # stdenv bootstrap tools ship a broken libiconv.dylib https://github.com/NixOS/nixpkgs/issues/158331 - am_cv_func_iconv=no ./configure --prefix=$out + # hello configure detects -liconv is needed but doesn't add to the link step + LDFLAGS=-liconv ./configure --prefix=$out make make install $out/bin/hello diff --git a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh similarity index 91% rename from pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh rename to pkgs/stdenv/darwin/patch-bootstrap-tools.sh index a5b9edff7cd5..555c340a7e75 100644 --- a/pkgs/stdenv/darwin/patch-bootstrap-tools-next.sh +++ b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh @@ -1,9 +1,8 @@ set -euo pipefail -export PATH=$tools/bin - -cp -R $tools $out -chmod -R u+w $out +echo Unpacking the bootstrap tools... >&2 +mkdir $out +tar xf "$1" -C $out updateInstallName() { local path="$1" From 3cdd8d05f0fc175358e720b6af929ba1a528d803 Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:56:20 +0000 Subject: [PATCH 4/4] stdenv.darwin.make-boostrap-tools: include unpack script in the archive --- pkgs/stdenv/darwin/make-bootstrap-tools.nix | 53 ++++++++++++++++++++- pkgs/stdenv/darwin/patch-bootstrap-tools.sh | 37 -------------- 2 files changed, 51 insertions(+), 39 deletions(-) delete mode 100644 pkgs/stdenv/darwin/patch-bootstrap-tools.sh diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index 7725665ae436..836a938b84ae 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -56,6 +56,47 @@ rec { }); xz_ = xz.override { enableStatic = true; }; + + unpackScript = writeText "bootstrap-tools-unpack.sh" '' + set -euo pipefail + + echo Unpacking the bootstrap tools... >&2 + mkdir $out + tar xf "$1" -C $out + + updateInstallName() { + local path="$1" + + cp "$path" "$path.new" + install_name_tool -id "$path" "$path.new" + codesign -f -i "$(basename "$path")" -s - "$path.new" + mv -f "$path.new" "$path" + } + + find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do + updateInstallName "$lib" + done + + # Provide a gunzip script. + cat > $out/bin/gunzip < $out/bin/egrep + echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep + echo "#! $out/bin/sh" > $out/bin/fgrep + echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + + cat >$out/bin/dsymutil << EOF + #!$out/bin/sh + EOF + + chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil + ''; + in '' mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin @@ -157,6 +198,7 @@ rec { cp ${getBin coreutils_}/bin/mkdir unpack/bin cp ${getBin gnutar_}/bin/tar unpack/bin cp ${getBin xz_}/bin/xz unpack/bin + cp ${unpackScript} unpack/bootstrap-tools-unpack.sh # # All files copied. Perform processing to update references to point into @@ -273,8 +315,15 @@ rec { name = "bootstrap-tools"; builder = "${bootstrapFiles.unpack}/bin/bash"; - args = [ ./patch-bootstrap-tools.sh bootstrapFiles.bootstrapTools ]; - PATH = lib.makeBinPath [ (placeholder "out") bootstrapFiles.unpack ]; + args = [ + "${bootstrapFiles.unpack}/bootstrap-tools-unpack.sh" + bootstrapFiles.bootstrapTools + ]; + + PATH = lib.makeBinPath [ + (placeholder "out") + bootstrapFiles.unpack + ]; allowedReferences = [ "out" ]; }; diff --git a/pkgs/stdenv/darwin/patch-bootstrap-tools.sh b/pkgs/stdenv/darwin/patch-bootstrap-tools.sh deleted file mode 100644 index 555c340a7e75..000000000000 --- a/pkgs/stdenv/darwin/patch-bootstrap-tools.sh +++ /dev/null @@ -1,37 +0,0 @@ -set -euo pipefail - -echo Unpacking the bootstrap tools... >&2 -mkdir $out -tar xf "$1" -C $out - -updateInstallName() { - local path="$1" - - cp "$path" "$path.new" - install_name_tool -id "$path" "$path.new" - codesign -f -i "$(basename "$path")" -s - "$path.new" - mv -f "$path.new" "$path" -} - -find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do - updateInstallName "$lib" -done - -# Provide a gunzip script. -cat > $out/bin/gunzip < $out/bin/egrep -echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep -echo "#! $out/bin/sh" > $out/bin/fgrep -echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep - -cat >$out/bin/dsymutil << EOF -#!$out/bin/sh -EOF - -chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil