From 5106caae709ca065781ed697b8675f09b5b03f62 Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 22 Apr 2024 22:43:29 +0300 Subject: [PATCH 1/6] maintainers/scripts/eval-release: don't special case xbursttools --- maintainers/scripts/eval-release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/scripts/eval-release.nix b/maintainers/scripts/eval-release.nix index 10acfe328447..fb830981683f 100644 --- a/maintainers/scripts/eval-release.nix +++ b/maintainers/scripts/eval-release.nix @@ -5,7 +5,7 @@ let trace = if builtins.getEnv "VERBOSE" == "1" then builtins.trace else (x: y: y); - rel = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" "xbursttools" ]; + rel = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ]; # Add the ‘recurseForDerivations’ attribute to ensure that # nix-instantiate recurses into nested attribute sets. From 513c54588f8662dc08737ae2244a12147d1505ba Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 22 Apr 2024 22:43:53 +0300 Subject: [PATCH 2/6] top-level/make-tarball.nix: drop unused argument --- pkgs/top-level/make-tarball.nix | 1 - pkgs/top-level/release-small.nix | 2 +- pkgs/top-level/release.nix | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix index 17df40fa0fb1..586c65e1b9ab 100644 --- a/pkgs/top-level/make-tarball.nix +++ b/pkgs/top-level/make-tarball.nix @@ -4,7 +4,6 @@ { nixpkgs , officialRelease -, supportedSystems , pkgs ? import nixpkgs.outPath {} , nix ? pkgs.nix , lib-tests ? import ../../lib/tests/release.nix { inherit pkgs; } diff --git a/pkgs/top-level/release-small.nix b/pkgs/top-level/release-small.nix index 2b143248948f..0212464acb62 100644 --- a/pkgs/top-level/release-small.nix +++ b/pkgs/top-level/release-small.nix @@ -18,7 +18,7 @@ in { tarball = import ./make-tarball.nix { - inherit nixpkgs supportedSystems; + inherit nixpkgs; officialRelease = false; }; diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index 2da2d7d64675..d11d1d1dbd43 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -78,7 +78,7 @@ let ] (arch: elem "${arch}-darwin" supportedSystems); nonPackageJobs = - { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease supportedSystems; }; + { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease; }; release-checks = import ./nixpkgs-basic-release-checks.nix { inherit pkgs nixpkgs supportedSystems; }; From 4a0c2fa837936dbe87c8bb693559ffd32c8e5e85 Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 22 Apr 2024 22:59:43 +0300 Subject: [PATCH 3/6] pkgs/top-level/make-tarball.nix: just don't do, like, most of it - Making sure everything evals? That's covered by release-checks! - Building a list of tarballs? We throw it out anyway! - Copying files around to produce the right paths in the tarball? Just tell tar to do it! Also, multithread xz compression for additional fast. Tarball job is under two minutes now. I hate computers. --- pkgs/top-level/make-tarball.nix | 56 +++++++++------------------------ 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix index 586c65e1b9ab..7352d3d90940 100644 --- a/pkgs/top-level/make-tarball.nix +++ b/pkgs/top-level/make-tarball.nix @@ -1,7 +1,3 @@ -/* Hydra job to build a tarball for Nixpkgs from a Git checkout. It - also builds the documentation and tests whether the Nix expressions - evaluate correctly. */ - { nixpkgs , officialRelease , pkgs ? import nixpkgs.outPath {} @@ -31,54 +27,32 @@ pkgs.releaseTools.sourceTarball { echo "git-revision is $(cat .git-revision)" ''; - requiredSystemFeatures = [ "big-parallel" ]; # 1 thread but ~36G RAM (!) see #227945 - dontBuild = false; doCheck = true; checkPhase = '' - set -o pipefail - - export NIX_STATE_DIR=$TMPDIR - export NIX_PATH=nixpkgs=$TMPDIR/barf.nix - opts=(--option build-users-group "") - nix-store --init - - echo "checking eval-release.nix" - nix-instantiate --eval --strict --show-trace ./maintainers/scripts/eval-release.nix > /dev/null - - echo "checking find-tarballs.nix" - nix-instantiate --readonly-mode --eval --strict --show-trace --json \ - ./maintainers/scripts/find-tarballs.nix \ - --arg expr 'import ./maintainers/scripts/all-tarballs.nix' > $TMPDIR/tarballs.json - nrUrls=$(jq -r '.[].url' < $TMPDIR/tarballs.json | wc -l) - echo "found $nrUrls URLs" - if [ "$nrUrls" -lt 10000 ]; then - echo "suspiciously low number of URLs" - exit 1 - fi - echo "generating packages.json" - mkdir -p $out/nix-support - echo -n '{"version":2,"packages":' > tmp - nix-env -f . -I nixpkgs=$src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}' "''${opts[@]}" >> tmp - echo -n '}' >> tmp - packages=$out/packages.json.br - < tmp sed "s|$(pwd)/||g" | jq -c | brotli -9 > $packages - rm tmp + packages=$out/packages.json.br + + ( + echo -n '{"version":2,"packages":' + NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}' + echo -n '}' + ) | sed "s|$src/||g" | jq -c | brotli -9 > $packages + + mkdir -p $out/nix-support echo "file json-br $packages" >> $out/nix-support/hydra-build-products ''; distPhase = '' mkdir -p $out/tarballs - mkdir ../$releaseName - cp -prd . ../$releaseName - (cd .. && tar cfa $out/tarballs/$releaseName.tar.xz $releaseName) || false + XZ_OPT="-T0" tar \ + --transform="s/^[.]/$releaseName/" \ + --create \ + --xz \ + --file=$out/tarballs/$releaseName.tar.xz \ + . ''; - - meta = { - maintainers = [ ]; - }; } From ee6b0bdaa0962b6f660489f8ae91bb8542238b6a Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 22 Apr 2024 23:39:40 +0300 Subject: [PATCH 4/6] pkgs/top-level/make-tarball.nix: add back the number of packages sanity check But in a way that's fast. --- pkgs/top-level/make-tarball.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix index 7352d3d90940..ccc1d1d519cb 100644 --- a/pkgs/top-level/make-tarball.nix +++ b/pkgs/top-level/make-tarball.nix @@ -34,13 +34,20 @@ pkgs.releaseTools.sourceTarball { checkPhase = '' echo "generating packages.json" - packages=$out/packages.json.br - ( echo -n '{"version":2,"packages":' NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}' echo -n '}' - ) | sed "s|$src/||g" | jq -c | brotli -9 > $packages + ) | sed "s|$src/||g" | jq -c > packages.json + + # Arbitrary number. The index has ~115k packages as of April 2024. + if [ $(jq -r '.packages | length' < packages.json) -lt 100000 ]; then + echo "ERROR: not enough packages in the search index, bailing out!" + exit 1 + fi + + packages=$out/packages.json.br + brotli -9 < packages.json > $packages mkdir -p $out/nix-support echo "file json-br $packages" >> $out/nix-support/hydra-build-products From e3096615863499aa8d9a415985056a8eedc27254 Mon Sep 17 00:00:00 2001 From: K900 Date: Tue, 23 Apr 2024 08:54:07 +0300 Subject: [PATCH 5/6] pkgs/top-level/make-tarball.nix: save another copy --- pkgs/top-level/make-tarball.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix index ccc1d1d519cb..0e17ebef074a 100644 --- a/pkgs/top-level/make-tarball.nix +++ b/pkgs/top-level/make-tarball.nix @@ -27,6 +27,8 @@ pkgs.releaseTools.sourceTarball { echo "git-revision is $(cat .git-revision)" ''; + dontUnpack = true; + dontBuild = false; doCheck = true; @@ -47,19 +49,21 @@ pkgs.releaseTools.sourceTarball { fi packages=$out/packages.json.br - brotli -9 < packages.json > $packages mkdir -p $out/nix-support + brotli -9 < packages.json > $packages echo "file json-br $packages" >> $out/nix-support/hydra-build-products ''; distPhase = '' mkdir -p $out/tarballs XZ_OPT="-T0" tar \ - --transform="s/^[.]/$releaseName/" \ + --absolute-names \ + --transform="s|^$src|$releaseName|g" \ + --transform="s|^$(pwd)|$releaseName|g" \ --create \ --xz \ --file=$out/tarballs/$releaseName.tar.xz \ - . + $src $(pwd)/{.version-suffix,.git-revision} ''; } From f7672530de22a62b86401e59ac55e246dce78539 Mon Sep 17 00:00:00 2001 From: K900 Date: Tue, 23 Apr 2024 09:13:59 +0300 Subject: [PATCH 6/6] pkgs/top-level/make-tarball.nix: make reproducible --- pkgs/top-level/make-tarball.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix index 0e17ebef074a..78b90082547b 100644 --- a/pkgs/top-level/make-tarball.nix +++ b/pkgs/top-level/make-tarball.nix @@ -58,12 +58,19 @@ pkgs.releaseTools.sourceTarball { distPhase = '' mkdir -p $out/tarballs XZ_OPT="-T0" tar \ - --absolute-names \ - --transform="s|^$src|$releaseName|g" \ - --transform="s|^$(pwd)|$releaseName|g" \ --create \ --xz \ --file=$out/tarballs/$releaseName.tar.xz \ + --absolute-names \ + --transform="s|^$src|$releaseName|g" \ + --transform="s|^$(pwd)|$releaseName|g" \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + --format=gnu \ + --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --mode=ug+w \ $src $(pwd)/{.version-suffix,.git-revision} ''; }