Merge pull request #289231 from adisbladis/fetchnpmlock

importNpmLock: init

doc/languages-frameworks/javascript.section.md

@@ -233,6 +233,37 @@ sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 
 It returns a derivation with all `package-lock.json` dependencies downloaded into `$out/`, usable as an npm cache.
 
+#### importNpmLock {#javascript-buildNpmPackage-importNpmLock}
+
+`importNpmLock` is a Nix function that requires the following optional arguments:
+
+- `npmRoot`: Path to package directory containing the source tree
+- `package`: Parsed contents of `package.json`
+- `packageLock`: Parsed contents of `package-lock.json`
+- `pname`: Package name
+- `version`: Package version
+
+It returns a derivation with a patched `package.json` & `package-lock.json` with all dependencies resolved to Nix store paths.
+
+This function is analogous to using `fetchNpmDeps`, but instead of specifying `hash` it uses metadata from `package.json` & `package-lock.json`.
+
+Note that `npmHooks.npmConfigHook` cannot be used with `importNpmLock`. You will instead need to use `importNpmLock.npmConfigHook`:
+
+```nix
+{ buildNpmPackage, importNpmLock }:
+
+buildNpmPackage {
+  pname = "hello";
+  version = "0.1.0";
+
+  npmDeps = importNpmLock {
+    npmRoot = ./.;
+  };
+
+  npmConfigHook = importNpmLock.npmConfigHook;
+}
+```
+
 ### corepack {#javascript-corepack}
 
 This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`.

pkgs/build-support/node/build-npm-package/default.nix

@@ -49,6 +49,12 @@
   name = "${name}-npm-deps";
   hash = npmDepsHash;
 }
+  # Custom npmConfigHook
+, npmConfigHook ? null
+  # Custom npmBuildHook
+, npmBuildHook ? null
+  # Custom npmInstallHook
+, npmInstallHook ? null
 , ...
 } @ args:
 
@@ -57,14 +63,19 @@ let
   npmHooks = buildPackages.npmHooks.override {
     inherit nodejs;
   };
-
-  inherit (npmHooks) npmConfigHook npmBuildHook npmInstallHook;
 in
 stdenv.mkDerivation (args // {
   inherit npmDeps npmBuildScript;
 
   nativeBuildInputs = nativeBuildInputs
-    ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ]
+    ++ [
+      nodejs
+      # Prefer passed hooks
+      (if npmConfigHook != null then npmConfigHook else npmHooks.npmConfigHook)
+      (if npmBuildHook != null then npmBuildHook else npmHooks.npmBuildHook)
+      (if npmInstallHook != null then npmInstallHook else npmHooks.npmInstallHook)
+      nodejs.python
+    ]
     ++ lib.optionals stdenv.isDarwin [ darwin.cctools ];
   buildInputs = buildInputs ++ [ nodejs ];
 

pkgs/build-support/node/import-npm-lock/default.nix

@@ -0,0 +1,134 @@
+{ lib
+, fetchurl
+, stdenv
+, callPackages
+, runCommand
+}:
+
+let
+  inherit (builtins) match elemAt toJSON removeAttrs;
+  inherit (lib) importJSON mapAttrs;
+
+  matchGitHubReference = match "github(.com)?:.+";
+  getName = package: package.name or "unknown";
+  getVersion = package: package.version or "0.0.0";
+
+  # Fetch a module from package-lock.json -> packages
+  fetchModule =
+    { module
+    , npmRoot ? null
+    }: (
+      if module ? "resolved" then
+        (
+          let
+            # Parse scheme from URL
+            mUrl = match "(.+)://(.+)" module.resolved;
+            scheme = elemAt mUrl 0;
+          in
+          (
+            if mUrl == null then
+              (
+                assert npmRoot != null; {
+                  outPath = npmRoot + "/${module.resolved}";
+                }
+              )
+            else if (scheme == "http" || scheme == "https") then
+              (
+                fetchurl {
+                  url = module.resolved;
+                  hash = module.integrity;
+                }
+              )
+            else if lib.hasPrefix "git" module.resolved then
+              (
+                builtins.fetchGit {
+                  url = module.resolved;
+                }
+              )
+            else throw "Unsupported URL scheme: ${scheme}"
+          )
+        )
+      else null
+    );
+
+  # Manage node_modules outside of the store with hooks
+  hooks = callPackages ./hooks { };
+
+in
+{
+  importNpmLock =
+    { npmRoot ? null
+    , package ? importJSON (npmRoot + "/package.json")
+    , packageLock ? importJSON (npmRoot + "/package-lock.json")
+    , pname ? getName package
+    , version ? getVersion package
+    }:
+    let
+      mapLockDependencies =
+        mapAttrs
+          (name: version: (
+            # Substitute the constraint with the version of the dependency from the top-level of package-lock.
+            if (
+              # if the version is `latest`
+              version == "latest"
+              ||
+              # Or if it's a github reference
+              matchGitHubReference version != null
+            ) then packageLock'.packages.${"node_modules/${name}"}.version
+            # But not a regular version constraint
+            else version
+          ));
+
+      packageLock' = packageLock // {
+        packages =
+          mapAttrs
+            (_: module:
+              let
+                src = fetchModule {
+                  inherit module npmRoot;
+                };
+              in
+              (removeAttrs module [
+                "link"
+                "funding"
+              ]) // lib.optionalAttrs (src != null) {
+                resolved = "file:${src}";
+              } // lib.optionalAttrs (module ? dependencies) {
+                dependencies = mapLockDependencies module.dependencies;
+              } // lib.optionalAttrs (module ? optionalDependencies) {
+                optionalDependencies = mapLockDependencies module.optionalDependencies;
+              })
+            packageLock.packages;
+      };
+
+      mapPackageDependencies = mapAttrs (name: _: packageLock'.packages.${"node_modules/${name}"}.resolved);
+
+      # Substitute dependency references in package.json with Nix store paths
+      packageJSON' = package // lib.optionalAttrs (package ? dependencies) {
+        dependencies = mapPackageDependencies package.dependencies;
+      } // lib.optionalAttrs (package ? devDependencies) {
+        devDependencies = mapPackageDependencies package.devDependencies;
+      };
+
+      pname = package.name or "unknown";
+
+    in
+    runCommand "${pname}-${version}-sources"
+      {
+        inherit pname version;
+
+        passAsFile = [ "package" "packageLock" ];
+
+        package = toJSON packageJSON';
+        packageLock = toJSON packageLock';
+      } ''
+      mkdir $out
+      cp "$packagePath" $out/package.json
+      cp "$packageLockPath" $out/package-lock.json
+    '';
+
+  inherit hooks;
+  inherit (hooks) npmConfigHook;
+
+  __functor = self: self.importNpmLock;
+}

pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+// When installing files rewritten to the Nix store with npm
+// npm writes the symlinks relative to the build directory.
+//
+// This makes relocating node_modules tricky when refering to the store.
+// This script walks node_modules and canonicalizes symlinks.
+
+async function canonicalize(storePrefix, root) {
+  console.log(storePrefix, root)
+  const entries = await fs.promises.readdir(root);
+  const paths = entries.map((entry) => path.join(root, entry));
+
+  const stats = await Promise.all(
+    paths.map(async (path) => {
+      return {
+        path: path,
+        stat: await fs.promises.lstat(path),
+      };
+    })
+  );
+
+  const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink());
+  const dirs = stats.filter((stat) => stat.stat.isDirectory());
+
+  // Canonicalize symlinks to their real path
+  await Promise.all(
+    symlinks.map(async (stat) => {
+      const target = await fs.promises.realpath(stat.path);
+      if (target.startsWith(storePrefix)) {
+        await fs.promises.unlink(stat.path);
+        await fs.promises.symlink(target, stat.path);
+      }
+    })
+  );
+
+  // Recurse into directories
+  await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path)));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+
+  if (fs.existsSync("node_modules")) {
+    await canonicalize(storePrefix, "node_modules");
+  }
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/default.nix

@@ -0,0 +1,13 @@
+{ callPackage, lib, makeSetupHook, srcOnly, nodejs }:
+{
+  npmConfigHook = makeSetupHook
+    {
+      name = "npm-config-hook";
+      substitutions = {
+        nodeSrc = srcOnly nodejs;
+        nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+        canonicalizeSymlinksScript = ./canonicalize-symlinks.js;
+        storePrefix = builtins.storeDir;
+      };
+    } ./npm-config-hook.sh;
+}

pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh

@@ -0,0 +1,70 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    if [ -z "${npmDeps-}" ]; then
+        echo "Error: 'npmDeps' should be set when using npmConfigHook."
+        exit 1
+    fi
+
+    echo "Configuring npm"
+
+    export HOME="$TMPDIR"
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+    npm config set offline true
+    npm config set progress false
+    npm config set fund false
+
+    echo "Installing patched package.json/package-lock.json"
+
+    # Save original package.json/package-lock.json for closure size reductions.
+    # The patched one contains store paths we don't want at runtime.
+    mv package.json .package.json.orig
+    if test -f package-lock.json; then # Not all packages have package-lock.json.
+        mv package-lock.json .package-lock.json.orig
+    fi
+    cp --no-preserve=mode "${npmDeps}/package.json" package.json
+    cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json
+
+    echo "Installing dependencies"
+
+    if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
+
+    patchShebangs node_modules
+
+    # Canonicalize symlinks from relative paths to the Nix store.
+    node @canonicalizeSymlinksScript@ @storePrefix@
+
+    # Revert to pre-patched package.json/package-lock.json for closure size reductions
+    mv .package.json.orig package.json
+    if test -f ".package-lock.json.orig"; then
+        mv .package-lock.json.orig package-lock.json
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postConfigureHooks+=(npmConfigHook)

pkgs/top-level/all-packages.nix

@@ -10262,6 +10262,8 @@ with pkgs;
   inherit (callPackages ../build-support/node/fetch-npm-deps { })
     fetchNpmDeps prefetch-npm-deps;
 
+  importNpmLock = callPackages ../build-support/node/import-npm-lock { };
+
   nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs // { __attrsFailEvaluation = true; };
 
   nodePackages = dontRecurseIntoAttrs nodejs.pkgs // { __attrsFailEvaluation = true; };