diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e7e57f381246..b2584681b84d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -289,3 +289,8 @@ # Dotnet /pkgs/build-support/dotnet @IvarWithoutBones /pkgs/development/compilers/dotnet @IvarWithoutBones + +# Node.js +/pkgs/build-support/node/build-npm-package @winterqt +/pkgs/build-support/node/fetch-npm-deps @winterqt +/doc/languages-frameworks/javascript.section.md @winterqt diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index 9d16b951e8dd..490daf991588 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -157,6 +157,61 @@ git config --global url."https://github.com/".insteadOf git://github.com/ ## Tool specific instructions {#javascript-tool-specific} +### buildNpmPackage {#javascript-buildNpmPackage} + +`buildNpmPackage` allows you to package npm-based projects in Nixpkgs without the use of an auto-generated dependencies file (as used in [node2nix](#javascript-node2nix)). It works by utilizing npm's cache functionality -- creating a reproducible cache that contains the dependencies of a project, and pointing npm to it. + +```nix +{ lib, buildNpmPackage, fetchFromGitHub }: + +buildNpmPackage rec { + pname = "flood"; + version = "4.7.0"; + + src = fetchFromGitHub { + owner = "jesec"; + repo = pname; + rev = "v${version}"; + hash = "sha256-BR+ZGkBBfd0dSQqAvujsbgsEPFYw/ThrylxUbOksYxM="; + }; + + patches = [ ./remove-prepack-script.patch ]; + + npmDepsHash = "sha256-s8SpZY/1tKZVd3vt7sA9vsqHvEaNORQBMrSyhWpj048="; + + NODE_OPTIONS = "--openssl-legacy-provider"; + + meta = with lib; { + description = "A modern web UI for various torrent clients with a Node.js backend and React frontend"; + homepage = "https://flood.js.org"; + license = licenses.gpl3Only; + maintainers = with maintainers; [ winter ]; + }; +} +``` + +#### Arguments {#javascript-buildNpmPackage-arguments} + +* `npmDepsHash`: The output hash of the dependencies for this project. Can be calculated in advance with [`prefetch-npm-deps`](#javascript-buildNpmPackage-prefetch-npm-deps). +* `makeCacheWritable`: Whether to make the cache writable prior to installing dependencies. Don't set this unless npm tries to write to the cache directory, as it can slow down the build. +* `npmBuildScript`: The script to run to build the project. Defaults to `"build"`. +* `npmFlags`: Flags to pass to all npm commands. +* `npmInstallFlags`: Flags to pass to `npm ci`. +* `npmBuildFlags`: Flags to pass to `npm run ${npmBuildScript}`. +* `npmPackFlags`: Flags to pass to `npm pack`. + +#### prefetch-npm-deps {#javascript-buildNpmPackage-prefetch-npm-deps} + +`prefetch-npm-deps` can calculate the hash of the dependencies of an npm project ahead of time. + +```console +$ ls +package.json package-lock.json index.js +$ prefetch-npm-deps package-lock.json +... +sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +``` + ### node2nix {#javascript-node2nix} #### Preparation {#javascript-node2nix-preparation} diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix new file mode 100644 index 000000000000..1038bb2abcb4 --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -0,0 +1,54 @@ +{ lib, stdenv, fetchNpmDeps, npmHooks, nodejs }: + +{ name ? "${args.pname}-${args.version}" +, src ? null +, srcs ? null +, sourceRoot ? null +, patches ? [ ] +, nativeBuildInputs ? [ ] +, buildInputs ? [ ] + # The output hash of the dependencies for this project. + # Can be calculated in advance with prefetch-npm-deps. +, npmDepsHash ? "" + # Whether to make the cache writable prior to installing dependencies. + # Don't set this unless npm tries to write to the cache directory, as it can slow down the build. +, makeCacheWritable ? false + # The script to run to build the project. +, npmBuildScript ? "build" + # Flags to pass to all npm commands. +, npmFlags ? [ ] + # Flags to pass to `npm ci`. +, npmInstallFlags ? [ ] + # Flags to pass to `npm rebuild`. +, npmRebuildFlags ? [ ] + # Flags to pass to `npm run ${npmBuildScript}`. +, npmBuildFlags ? [ ] + # Flags to pass to `npm pack`. +, npmPackFlags ? [ ] +, ... +} @ args: + +let + npmDeps = fetchNpmDeps { + inherit src srcs sourceRoot patches; + name = "${name}-npm-deps"; + hash = npmDepsHash; + }; + + inherit (npmHooks.override { inherit nodejs; }) npmConfigHook npmBuildHook npmInstallHook; +in +stdenv.mkDerivation (args // { + inherit npmDeps npmBuildScript; + + nativeBuildInputs = nativeBuildInputs ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook ]; + buildInputs = buildInputs ++ [ nodejs ]; + + strictDeps = true; + + # Stripping takes way too long with the amount of files required by a typical Node.js project. + dontStrip = args.dontStrip or true; + + passthru = { inherit npmDeps; } // (args.passthru or { }); + + meta = (args.meta or { }) // { platforms = args.meta.platforms or nodejs.meta.platforms; }; +}) diff --git a/pkgs/build-support/node/build-npm-package/hooks/default.nix b/pkgs/build-support/node/build-npm-package/hooks/default.nix new file mode 100644 index 000000000000..d2293ed42f79 --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/hooks/default.nix @@ -0,0 +1,35 @@ +{ lib, makeSetupHook, nodejs, srcOnly, diffutils, jq, makeWrapper }: + +{ + npmConfigHook = makeSetupHook + { + name = "npm-config-hook"; + substitutions = { + nodeSrc = srcOnly nodejs; + + # Specify the stdenv's `diff` and `jq` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong binaries. + # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. + diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; + jq = "${jq.nativeDrv or jq}/bin/jq"; + + nodeVersion = nodejs.version; + nodeVersionMajor = lib.versions.major nodejs.version; + }; + } ./npm-config-hook.sh; + + npmBuildHook = makeSetupHook + { + name = "npm-build-hook"; + } ./npm-build-hook.sh; + + npmInstallHook = makeSetupHook + { + name = "npm-install-hook"; + deps = [ makeWrapper ]; + substitutions = { + hostNode = "${nodejs}/bin/node"; + jq = "${jq.nativeDrv or jq}/bin/jq"; + }; + } ./npm-install-hook.sh; +} diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-build-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-build-hook.sh new file mode 100644 index 000000000000..b99c9d94faff --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-build-hook.sh @@ -0,0 +1,37 @@ +# shellcheck shell=bash + +npmBuildHook() { + echo "Executing npmBuildHook" + + runHook preBuild + + if [ -z "${npmBuildScript-}" ]; then + echo + echo "ERROR: no build script was specified" + echo 'Hint: set `npmBuildScript`, override `buildPhase`, or set `dontNpmBuild = true`.' + echo + + exit 1 + fi + + if ! npm run "$npmBuildScript" $npmBuildFlags "${npmBuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then + echo + echo 'ERROR: `npm build` failed' + echo + echo "Here are a few things you can try, depending on the error:" + echo "1. Make sure your build script ($npmBuildScript) exists" + echo '2. If the error being thrown is something similar to "error:0308010C:digital envelope routines::unsupported", add `NODE_OPTIONS = "--openssl-legacy-provider"` to your derivation' + echo " See https://github.com/webpack/webpack/issues/14532 for more information." + echo + + exit 1 + fi + + runHook postBuild + + echo "Finished npmBuildHook" +} + +if [ -z "${dontNpmBuild-}" ] && [ -z "${buildPhase-}" ]; then + buildPhase=npmBuildHook +fi diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh new file mode 100644 index 000000000000..723d8c1a4643 --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh @@ -0,0 +1,102 @@ +# shellcheck shell=bash + +npmConfigHook() { + echo "Executing npmConfigHook" + + echo "Configuring npm" + + export HOME=$TMPDIR + export npm_config_nodedir="@nodeSrc@" + + local -r cacheLockfile="$npmDeps/package-lock.json" + local -r srcLockfile="$PWD/package-lock.json" + + echo "Validating consistency between $srcLockfile and $cacheLockfile" + + if ! @diff@ "$srcLockfile" "$cacheLockfile"; then + # If the diff failed, first double-check that the file exists, so we can + # give a friendlier error msg. + if ! [ -e "$srcLockfile" ]; then + echo + echo "ERROR: Missing package-lock.json from src. Expected to find it at: $srcLockfile" + echo "Hint: You can use the patches attribute to add a package-lock.json manually to the build." + echo + + exit 1 + fi + + if ! [ -e "$cacheLockfile" ]; then + echo + echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile" + echo + + exit 1 + fi + + echo + echo "ERROR: npmDepsHash is out of date" + echo + echo "The package-lock.json in src is not the same as the in $npmDeps." + echo + echo "To fix the issue:" + echo '1. Use `lib.fakeHash` as the npmDepsHash value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256-' value back into the npmDepsHash field" + echo + + exit 1 + fi + + local cachePath + + if [ -z "${makeCacheWritable-}" ]; then + cachePath=$npmDeps + else + echo "Making cache writable" + cp -r "$npmDeps" "$TMPDIR/cache" + chmod -R 700 "$TMPDIR/cache" + cachePath=$TMPDIR/cache + fi + + npm config set cache "$cachePath" + npm config set offline true + npm config set progress false + + echo "Installing dependencies" + + if ! npm ci --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then + echo + echo "ERROR: npm failed to install dependencies" + echo + echo "Here are a few things you can try, depending on the error:" + echo '1. Set `makeCacheWritable = true`' + echo " Note that this won't help if npm is complaining about not being able to write to the logs directory -- look above that for the actual error." + echo '2. Set `npmInstallFlags = [ "--legacy-peer-deps" ]`' + echo + + exit 1 + fi + + patchShebangs node_modules + + local -r lockfileVersion="$(@jq@ .lockfileVersion package-lock.json)" + + if (( lockfileVersion < 2 )); then + # This is required because npm consults a hidden lockfile in node_modules to figure out + # what to create bin links for. When using an old lockfile offline, this hidden lockfile + # contains insufficent data, making npm silently fail to create links. The hidden lockfile + # is bypassed when any file in node_modules is newer than it. Thus, we create a file when + # using an old lockfile, so bin links work as expected without having to downgrade Node or npm. + touch node_modules/.meow + fi + + npm rebuild "${npmRebuildFlags[@]}" "${npmFlags[@]}" + + if (( lockfileVersion < 2 )); then + rm node_modules/.meow + fi + + echo "Finished npmConfigHook" +} + +postPatchHooks+=(npmConfigHook) diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh new file mode 100644 index 000000000000..4a222de26bbf --- /dev/null +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh @@ -0,0 +1,43 @@ +# shellcheck shell=bash + +npmInstallHook() { + echo "Executing npmInstallHook" + + runHook preInstall + + # `npm pack` writes to cache + npm config delete cache + + local -r packageOut="$out/lib/node_modules/$(@jq@ --raw-output '.name' package.json)" + + while IFS= read -r file; do + local dest="$packageOut/$(dirname "$file")" + mkdir -p "$dest" + cp "$file" "$dest" + done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm pack --json --dry-run $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + + while IFS=" " read -ra bin; do + mkdir -p "$out/bin" + makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}" + done < <(@jq@ --raw-output '(.bin | type) as $typ | if $typ == "string" then + .name + " " + .bin + elif $typ == "object" then .bin | to_entries | map(.key + " " + .value) | join("\n") + else "invalid type " + $typ | halt_error end' package.json) + + local -r nodeModulesPath="$packageOut/node_modules" + + if [ ! -d "$nodeModulesPath" ]; then + npm prune --omit dev + find node_modules -maxdepth 1 -type d -empty -delete + + cp -r node_modules "$nodeModulesPath" + fi + + runHook postInstall + + echo "Finished npmInstallHook" +} + +if [ -z "${dontNpmInstall-}" ] && [ -z "${installPhase-}" ]; then + installPhase=npmInstallHook +fi diff --git a/pkgs/build-support/node/fetch-npm-deps/.gitignore b/pkgs/build-support/node/fetch-npm-deps/.gitignore new file mode 100644 index 000000000000..ea8c4bf7f35f --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/.gitignore @@ -0,0 +1 @@ +/target diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock new file mode 100644 index 000000000000..ba832d115e6e --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock @@ -0,0 +1,689 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "cpufeatures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc948ebb96241bb40ab73effeb80d9f93afaad49359d159a5e61be51619fe813" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "once_cell", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" + +[[package]] +name = "js-sys" +version = "0.3.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "prefetch-npm-deps" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "digest", + "rayon", + "serde", + "serde_json", + "sha1", + "sha2", + "tempfile", + "ureq", + "url", +] + +[[package]] +name = "proc-macro2" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "syn" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" + +[[package]] +name = "unicode-normalization" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" + +[[package]] +name = "web-sys" +version = "0.3.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +dependencies = [ + "webpki", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml new file mode 100644 index 000000000000..bebdaad29525 --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "prefetch-npm-deps" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.65" +base64 = "0.13.0" +digest = "0.10.5" +rayon = "1.5.3" +serde = { version = "1.0.145", features = ["derive"] } +serde_json = "1.0.85" +sha1 = "0.10.5" +sha2 = "0.10.6" +tempfile = "3.3.0" +ureq = { version = "2.5.0" } +url = { version = "2.3.1", features = ["serde"] } diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix new file mode 100644 index 000000000000..d6ee0124d285 --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/default.nix @@ -0,0 +1,137 @@ +{ lib, stdenvNoCC, rustPlatform, Security, testers, fetchurl, prefetch-npm-deps, fetchNpmDeps }: + +{ + prefetch-npm-deps = rustPlatform.buildRustPackage { + pname = "prefetch-npm-deps"; + version = (lib.importTOML ./Cargo.toml).package.version; + + src = lib.cleanSourceWith { + src = ./.; + filter = name: type: + let + name' = builtins.baseNameOf name; + in + name' != "default.nix" && name' != "target"; + }; + + cargoLock.lockFile = ./Cargo.lock; + + buildInputs = lib.optional stdenvNoCC.isDarwin Security; + + passthru.tests = + let + makeTestSrc = { name, src }: stdenvNoCC.mkDerivation { + name = "${name}-src"; + + inherit src; + + buildCommand = '' + mkdir -p $out + cp $src $out/package-lock.json + ''; + }; + + makeTest = { name, src, hash }: testers.invalidateFetcherByDrvHash fetchNpmDeps { + inherit name hash; + + src = makeTestSrc { inherit name src; }; + }; + in + { + lockfileV1 = makeTest { + name = "lockfile-v1"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/jellyfin/jellyfin-web/v10.8.4/package-lock.json"; + hash = "sha256-uQmc+S+V1co1Rfc4d82PpeXjmd1UqdsG492ADQFcZGA="; + }; + + hash = "sha256-fk7L9vn8EHJsGJNMAjYZg9h0PT6dAwiahdiEeXVrMB8="; + }; + + lockfileV2 = makeTest { + name = "lockfile-v2"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/jesec/flood/v4.7.0/package-lock.json"; + hash = "sha256-qS29tq5QPnGxV+PU40VgMAtdwVLtLyyhG2z9GMeYtC4="; + }; + + hash = "sha256-s8SpZY/1tKZVd3vt7sA9vsqHvEaNORQBMrSyhWpj048="; + }; + + hashPrecedence = makeTest { + name = "hash-precedence"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/matrix-org/matrix-appservice-irc/0.34.0/package-lock.json"; + hash = "sha256-1+0AQw9EmbHiMPA/H8OP8XenhrkhLRYBRhmd1cNPFjk="; + }; + + hash = "sha256-KRxwrEij3bpZ5hbQhX67KYpnY2cRS7u2EVZIWO1FBPM="; + }; + + hostedGitDeps = makeTest { + name = "hosted-git-deps"; + + src = fetchurl { + url = "https://cyberchaos.dev/yuka/trainsearch/-/raw/e3cba6427e8ecfd843d0f697251ddaf5e53c2327/package-lock.json"; + hash = "sha256-X9mCwPqV5yP0S2GonNvpYnLSLJMd/SUIked+hMRxDpA="; + }; + + hash = "sha256-oIM05TGHstX1D4k2K4TJ+SHB7H/tNKzxzssqf0GJwvY="; + }; + }; + + meta = with lib; { + description = "Prefetch dependencies from npm (for use with `fetchNpmDeps`)"; + maintainers = with maintainers; [ winter ]; + license = licenses.mit; + }; + }; + + fetchNpmDeps = + { name ? "npm-deps" + , hash ? "" + , ... + } @ args: + let + hash_ = + if hash != "" then { + outputHash = hash; + } else { + outputHash = ""; + outputHashAlgo = "sha256"; + }; + in + stdenvNoCC.mkDerivation (args // { + inherit name; + + nativeBuildInputs = [ prefetch-npm-deps ]; + + buildPhase = '' + runHook preBuild + + if [[ ! -f package-lock.json ]]; then + echo + echo "ERROR: The package-lock.json file does not exist!" + echo + echo "package-lock.json is required to make sure that npmDepsHash doesn't change" + echo "when packages are updated on npm." + echo + echo "Hint: You can use the patches attribute to add a package-lock.json manually to the build." + echo + + exit 1 + fi + + prefetch-npm-deps package-lock.json $out + + runHook postBuild + ''; + + dontInstall = true; + + outputHashMode = "recursive"; + } // hash_); +} diff --git a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs new file mode 100644 index 000000000000..865a320954b5 --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs @@ -0,0 +1,116 @@ +use digest::{Digest, Update}; +use serde::Serialize; +use sha1::Sha1; +use sha2::{Sha256, Sha512}; +use std::{ + fs::{self, File}, + io::Write, + path::PathBuf, +}; +use url::Url; + +#[derive(Serialize)] +struct Key { + key: String, + integrity: String, + time: u8, + size: usize, + metadata: Metadata, +} + +#[derive(Serialize)] +struct Metadata { + url: Url, + options: Options, +} + +#[derive(Serialize)] +struct Options { + compress: bool, +} + +pub struct Cache(PathBuf); + +fn push_hash_segments(path: &mut PathBuf, hash: &str) { + path.push(&hash[0..2]); + path.push(&hash[2..4]); + path.push(&hash[4..]); +} + +impl Cache { + pub fn new(path: PathBuf) -> Cache { + Cache(path) + } + + pub fn put( + &self, + key: String, + url: Url, + data: &[u8], + integrity: Option, + ) -> anyhow::Result<()> { + let (algo, hash, integrity) = if let Some(integrity) = integrity { + let (algo, hash) = integrity.split_once('-').unwrap(); + + (algo.to_string(), base64::decode(hash)?, integrity) + } else { + let hash = Sha512::new().chain(data).finalize(); + + ( + String::from("sha512"), + hash.to_vec(), + format!("sha512-{}", base64::encode(hash)), + ) + }; + + let content_path = { + let mut p = self.0.join("content-v2"); + + p.push(algo); + + push_hash_segments( + &mut p, + &hash + .into_iter() + .map(|x| format!("{:02x}", x)) + .collect::(), + ); + + p + }; + + fs::create_dir_all(content_path.parent().unwrap())?; + + fs::write(content_path, data)?; + + let index_path = { + let mut p = self.0.join("index-v5"); + + push_hash_segments( + &mut p, + &format!("{:x}", Sha256::new().chain(&key).finalize()), + ); + + p + }; + + fs::create_dir_all(index_path.parent().unwrap())?; + + let data = serde_json::to_string(&Key { + key, + integrity, + time: 0, + size: data.len(), + metadata: Metadata { + url, + options: Options { compress: true }, + }, + })?; + + let mut file = File::options().append(true).create(true).open(index_path)?; + + write!(file, "\n{:x}\t{data}", Sha1::new().chain(&data).finalize())?; + + Ok(()) + } +} diff --git a/pkgs/build-support/node/fetch-npm-deps/src/main.rs b/pkgs/build-support/node/fetch-npm-deps/src/main.rs new file mode 100644 index 000000000000..097148fef82a --- /dev/null +++ b/pkgs/build-support/node/fetch-npm-deps/src/main.rs @@ -0,0 +1,334 @@ +#![warn(clippy::pedantic)] + +use crate::cacache::Cache; +use anyhow::anyhow; +use rayon::prelude::*; +use serde::Deserialize; +use std::{ + collections::HashMap, + env, fs, + path::Path, + process::{self, Command}, +}; +use tempfile::tempdir; +use url::Url; + +mod cacache; + +#[derive(Deserialize)] +struct PackageLock { + #[serde(rename = "lockfileVersion")] + version: u8, + dependencies: Option>, + packages: Option>, +} + +#[derive(Deserialize)] +struct OldPackage { + version: String, + resolved: Option, + integrity: Option, + dependencies: Option>, +} + +#[derive(Deserialize)] +struct Package { + resolved: Option, + integrity: Option, +} + +fn to_new_packages( + old_packages: HashMap, +) -> anyhow::Result> { + let mut new = HashMap::new(); + + for (name, package) in old_packages { + new.insert( + format!("{name}-{}", package.version), + Package { + resolved: if let Ok(url) = Url::parse(&package.version) { + Some(url) + } else { + package.resolved.as_deref().map(Url::parse).transpose()? + }, + integrity: package.integrity, + }, + ); + + if let Some(dependencies) = package.dependencies { + new.extend(to_new_packages(dependencies)?); + } + } + + Ok(new) +} + +#[allow(clippy::case_sensitive_file_extension_comparisons)] +fn get_hosted_git_url(url: &Url) -> Option { + if ["git", "http", "git+ssh", "git+https", "ssh", "https"].contains(&url.scheme()) { + let mut s = url.path_segments()?; + + match url.host_str()? { + "github.com" => { + let user = s.next()?; + let mut project = s.next()?; + let typ = s.next(); + let mut commit = s.next(); + + if typ.is_none() { + commit = url.fragment(); + } else if typ.is_some() && typ != Some("tree") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = commit.unwrap(); + + Some( + Url::parse(&format!( + "https://codeload.github.com/{user}/{project}/tar.gz/{commit}" + )) + .ok()?, + ) + } + "bitbucket.org" => { + let user = s.next()?; + let mut project = s.next()?; + let aux = s.next(); + + if aux == Some("get") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://bitbucket.org/{user}/{project}/get/{commit}.tar.gz" + )) + .ok()?, + ) + } + "gitlab.com" => { + let path = &url.path()[1..]; + + if path.contains("/~/") || path.contains("/archive.tar.gz") { + return None; + } + + let user = s.next()?; + let mut project = s.next()?; + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://gitlab.com/{user}/{project}/repository/archive.tar.gz?ref={commit}" + )) + .ok()?, + ) + } + "git.sr.ht" => { + let user = s.next()?; + let mut project = s.next()?; + let aux = s.next(); + + if aux == Some("archive") { + return None; + } + + if project.ends_with(".git") { + project = project.strip_suffix(".git")?; + } + + let commit = url.fragment()?; + + Some( + Url::parse(&format!( + "https://git.sr.ht/{user}/{project}/archive/{commit}.tar.gz" + )) + .ok()?, + ) + } + _ => None, + } + } else { + None + } +} + +fn get_ideal_hash(integrity: &str) -> anyhow::Result<&str> { + let split: Vec<_> = integrity.split_ascii_whitespace().collect(); + + if split.len() == 1 { + Ok(split[0]) + } else { + for hash in ["sha512-", "sha1-"] { + if let Some(h) = split.iter().find(|s| s.starts_with(hash)) { + return Ok(h); + } + } + + Err(anyhow!("not sure which hash to select out of {split:?}")) + } +} + +fn main() -> anyhow::Result<()> { + let args = env::args().collect::>(); + + if args.len() < 2 { + println!("usage: {} ", args[0]); + println!(); + println!("Prefetches npm dependencies for usage by fetchNpmDeps."); + + process::exit(1); + } + + let lock_content = fs::read_to_string(&args[1])?; + let lock: PackageLock = serde_json::from_str(&lock_content)?; + + let out_tempdir; + + let (out, print_hash) = if let Some(path) = args.get(2) { + (Path::new(path), false) + } else { + out_tempdir = tempdir()?; + + (out_tempdir.path(), true) + }; + + let agent = ureq::agent(); + + eprintln!("lockfile version: {}", lock.version); + + let packages = match lock.version { + 1 => lock.dependencies.map(to_new_packages).transpose()?, + 2 | 3 => lock.packages, + _ => panic!( + "We don't support lockfile version {}, please file an issue.", + lock.version + ), + }; + + if packages.is_none() { + return Ok(()); + } + + let cache = Cache::new(out.join("_cacache")); + + packages + .unwrap() + .into_par_iter() + .try_for_each(|(dep, package)| { + if dep.is_empty() || package.resolved.is_none() { + return Ok::<_, anyhow::Error>(()); + } + + eprintln!("{dep}"); + + let mut resolved = package.resolved.unwrap(); + + if let Some(hosted_git_url) = get_hosted_git_url(&resolved) { + resolved = hosted_git_url; + } + + let mut data = Vec::new(); + + agent + .get(resolved.as_str()) + .call()? + .into_reader() + .read_to_end(&mut data)?; + + cache + .put( + format!("make-fetch-happen:request-cache:{resolved}"), + resolved, + &data, + package + .integrity + .map(|i| Ok::(get_ideal_hash(&i)?.to_string())) + .transpose()?, + ) + .map_err(|e| anyhow!("couldn't insert cache entry for {dep}: {e:?}"))?; + + Ok(()) + })?; + + fs::write(out.join("package-lock.json"), lock_content)?; + + if print_hash { + Command::new("nix") + .args(["--experimental-features", "nix-command", "hash", "path"]) + .arg(out.as_os_str()) + .status()?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{get_hosted_git_url, get_ideal_hash}; + use url::Url; + + #[test] + fn hosted_git_urls() { + for (input, expected) in [ + ( + "git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525", + Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"), + ), + ( + "https://user@github.com/foo/bar#fix/bug", + Some("https://codeload.github.com/foo/bar/tar.gz/fix/bug") + ), + ( + "https://github.com/eligrey/classList.js/archive/1.2.20180112.tar.gz", + None + ), + ( + "git+ssh://bitbucket.org/foo/bar#branch", + Some("https://bitbucket.org/foo/bar/get/branch.tar.gz") + ), + ( + "ssh://git@gitlab.com/foo/bar.git#fix/bug", + Some("https://gitlab.com/foo/bar/repository/archive.tar.gz?ref=fix/bug") + ), + ( + "git+ssh://git.sr.ht/~foo/bar#branch", + Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz") + ), + ] { + assert_eq!( + get_hosted_git_url(&Url::parse(input).unwrap()), + expected.map(|u| Url::parse(u).unwrap()) + ); + } + } + + #[test] + fn ideal_hashes() { + for (input, expected) in [ + ("sha512-foo sha1-bar", Some("sha512-foo")), + ("sha1-bar md5-foo", Some("sha1-bar")), + ("sha1-bar", Some("sha1-bar")), + ("sha512-foo", Some("sha512-foo")), + ("foo-bar sha1-bar", Some("sha1-bar")), + ("foo-bar baz-foo", None), + ] { + assert_eq!(get_ideal_hash(input).ok(), expected); + } + } +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7e50142f8e3e..b0aa2ab4f9f3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -8788,6 +8788,14 @@ with pkgs; nodejs_latest = nodejs-19_x; nodejs-slim_latest = nodejs-slim-19_x; + buildNpmPackage = callPackage ../build-support/node/build-npm-package { }; + + npmHooks = callPackage ../build-support/node/build-npm-package/hooks { }; + + inherit (callPackage ../build-support/node/fetch-npm-deps { + inherit (darwin.apple_sdk.frameworks) Security; + }) fetchNpmDeps prefetch-npm-deps; + nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs; nodePackages = dontRecurseIntoAttrs nodejs.pkgs;