prefetch-yarn-deps, fetchYarnDeps: init

This commit is contained in:
Yureka 2021-10-06 04:52:35 +02:00 committed by Yuka
parent 7141eb9c57
commit d6e0195ccd
7 changed files with 284 additions and 0 deletions

View File

@ -0,0 +1,74 @@
{ stdenv, lib, makeWrapper, coreutils, nix-prefetch-git, fetchurl, nodejs-slim, prefetch-yarn-deps, cacert, callPackage }:
let
yarnpkg-lockfile-tar = fetchurl {
url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz";
sha512 = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==";
};
in {
prefetch-yarn-deps = stdenv.mkDerivation {
name = "prefetch-yarn-deps";
dontUnpack = true;
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ coreutils nix-prefetch-git nodejs-slim ];
buildPhase = ''
runHook preBuild
mkdir libexec
tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js
mv index.js libexec/yarnpkg-lockfile.js
cp ${./index.js} libexec/index.js
patchShebangs libexec/index.js
runHook postBuild
'';
installPhase = ''
runHook preInstall
mkdir -p $out/bin
cp -r libexec $out
makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \
--prefix PATH : ${lib.makeBinPath [ coreutils nix-prefetch-git ]}
runHook postInstall
'';
};
fetchYarnDeps = let
f = {
name ? "offline",
yarnLock,
hash ? "",
sha256 ? "",
}: let
hash_ =
if hash != "" then { outputHashAlgo = null; outputHash = hash; }
else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; }
else throw "fetchYarnDeps requires a hash";
in stdenv.mkDerivation {
inherit name;
dontUnpack = true;
dontInstall = true;
nativeBuildInputs = [ prefetch-yarn-deps ];
GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
buildPhase = ''
mkdir -p $out
(cd $out; prefetch-yarn-deps --verbose --builder ${yarnLock})
'';
outputHashMode = "recursive";
inherit (hash_) outputHashAlgo outputHash;
};
in lib.setFunctionArgs f (lib.functionArgs f) // {
tests = callPackage ./tests {};
};
}

View File

@ -0,0 +1,168 @@
#!/usr/bin/env node
'use strict'
const fs = require('fs')
const crypto = require('crypto')
const process = require('process')
const https = require('https')
const child_process = require('child_process')
const path = require('path')
const lockfile = require('./yarnpkg-lockfile.js')
const { promisify } = require('util')
const execFile = promisify(child_process.execFile)
const exec = async (...args) => {
const res = await execFile(...args)
if (res.error) throw new Error(res.stderr)
return res
}
const downloadFileHttps = (fileName, url, expectedHash) => {
return new Promise((resolve, reject) => {
https.get(url, (res) => {
const file = fs.createWriteStream(fileName)
const hash = crypto.createHash('sha1')
res.pipe(file)
res.pipe(hash).setEncoding('hex')
res.on('end', () => {
file.close()
const h = hash.read()
if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`))
resolve()
})
res.on('error', e => reject(e))
})
})
}
const downloadGit = async (fileName, url, rev) => {
await exec('nix-prefetch-git', [
'--out', fileName + '.tmp',
'--url', url,
'--rev', rev,
'--builder'
])
await exec('tar', [
// hopefully make it reproducible across runs and systems
'--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1',
// Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79
'--mode', 'u+w',
'-C', fileName + '.tmp',
'-cf', fileName, '.'
])
await exec('rm', [ '-rf', fileName + '.tmp', ])
}
const downloadPkg = (pkg, verbose) => {
const [ url, hash ] = pkg.resolved.split('#')
if (verbose) console.log('downloading ' + url)
if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) {
const fileName = path.basename(url)
const s = url.split('/')
downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[6])
} else if (url.startsWith('https://')) {
const fileName = url
.replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
.replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
return downloadFileHttps(fileName, url, hash)
} else if (url.startsWith('git+')) {
const fileName = path.basename(url)
return downloadGit(fileName, url.replace(/^git\+/, ''), hash)
} else {
throw new Error('don\'t know how to download "' + url + '"')
}
}
const performParallel = tasks => {
const worker = async () => {
while (tasks.length > 0) await tasks.shift()()
}
const workers = []
for (let i = 0; i < 4; i++) {
workers.push(worker())
}
return Promise.all(workers)
}
const prefetchYarnDeps = async (lockData, verbose) => {
const tasks = Object.values(
Object.entries(lockData.object)
.map(([key, value]) => {
return { key, ...value }
})
.reduce((out, pkg) => {
out[pkg.resolved] = pkg
return out
}, {})
)
.map(pkg => () => downloadPkg(pkg, verbose))
await performParallel(tasks)
if (verbose) console.log('Done')
}
const showUsage = async () => {
process.stderr.write(`
syntax: prefetch-yarn-deps [path to yarn.lock] [options]
Options:
-h --help Show this help
-v --verbose Verbose output
--builder Only perform the download to current directory, then exit
`)
process.exit(1)
}
const main = async () => {
const args = process.argv.slice(2)
let next, lockFile, verbose, isBuilder
while (next = args.shift()) {
if (next == '--builder') {
isBuilder = true
} else if (next == '--verbose' || next == '-v') {
verbose = true
} else if (next == '--help' || next == '-h') {
showUsage()
} else if (!lockFile) {
lockFile = next
} else {
showUsage()
}
}
let lockContents
try {
lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
} catch(e) {
showUsage()
}
const lockData = lockfile.parse(lockContents)
if (isBuilder) {
await prefetchYarnDeps(lockData, verbose)
} else {
const { stdout: tmpDir } = await exec('mktemp', [ '-d' ])
try {
process.chdir(tmpDir.trim())
await prefetchYarnDeps(lockData, verbose)
const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ])
console.log(hash)
} finally {
await exec('rm', [ '-rf', tmpDir.trim() ])
}
}
}
main()
.catch(e => {
console.error(e)
process.exit(1)
})

View File

@ -0,0 +1,16 @@
{ invalidateFetcherByDrvHash, fetchYarnDeps, ... }:
{
simple = invalidateFetcherByDrvHash fetchYarnDeps {
yarnLock = ./simple.lock;
sha256 = "sha256-Erdkw2E8wWT09jFNLXGkrdwKl0HuSZWnUDJUrV95vSE=";
};
gitDep = invalidateFetcherByDrvHash fetchYarnDeps {
yarnLock = ./git.lock;
sha256 = "sha256-lAqN4LpoE+jgsQO1nDtuORwcVEO7ogEV53jCu2jFJUI=";
};
githubDep = invalidateFetcherByDrvHash fetchYarnDeps {
yarnLock = ./github.lock;
sha256 = "sha256-Tsfgyjxz8x6gNmfN0xR7G/NQNoEs4svxRN/N+26vfJU=";
};
}

View File

@ -0,0 +1,7 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
"async@git+https://github.com/caolan/async":
version "3.2.1"
resolved "git+https://github.com/caolan/async#fc9ba651341af5ab974aade6b1640e345912be83"

View File

@ -0,0 +1,7 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
"async@github:caolan/async":
version "3.2.1"
resolved "https://codeload.github.com/caolan/async/tar.gz/fc9ba651341af5ab974aade6b1640e345912be83"

View File

@ -0,0 +1,8 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
lit-html@1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/lit-html/-/lit-html-1.4.1.tgz#0c6f3ee4ad4eb610a49831787f0478ad8e9ae5e0"
integrity sha512-B9btcSgPYb1q4oSOb/PrOT6Z/H+r6xuNzfH4lFli/AWhYwdtrgQkQWBbIc6mdnf6E2IL3gDXdkkqNktpU0OZQA==

View File

@ -468,6 +468,10 @@ with pkgs;
fetchMavenArtifact = callPackage ../build-support/fetchmavenartifact { };
inherit (callPackage ../build-support/node/fetch-yarn-deps { })
prefetch-yarn-deps
fetchYarnDeps;
find-cursor = callPackage ../tools/X11/find-cursor { };
flare-floss = callPackage ../tools/security/flare-floss { };