curl-impersonate: init at 0.5.4

pkgs/tools/networking/curl-impersonate/curl-impersonate-0.5.2-fix-shebangs.patch

@@ -0,0 +1,13 @@
+diff --git a/Makefile.in b/Makefile.in
+index 877c54f..3e39ed1 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz:
+ 
+ $(nss_static_libs): $(NSS_VERSION).tar.gz
+ 	tar xf $(NSS_VERSION).tar.gz
++	sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh
++	sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi
+ 
+ ifeq ($(host),$(build))
+ 	# Native build, use NSS' build script.

pkgs/tools/networking/curl-impersonate/default.nix

@@ -1,27 +1,183 @@
-#TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries.
-{ lib, stdenv, fetchzip, zlib, autoPatchelfHook }:
-stdenv.mkDerivation rec {
-  pname = "curl-impersonate-bin";
-  version = "v0.5.3";
+{ lib
+, stdenv
+, fetchFromGitHub
+, fetchpatch
+, callPackage
+, buildGoModule
+, installShellFiles
+, symlinkJoin
+, zlib
+, sqlite
+, cmake
+, python3
+, ninja
+, perl
+, autoconf
+, automake
+, libtool
+, darwin
+, cacert
+, unzip
+, go
+, p11-kit
+}:
 
-  src = fetchzip {
-    url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz";
-    sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g=";
-    stripRoot = false;
+let
+  makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec {
+    pname = "curl-impersonate-${name}";
+    version = "0.5.4";
+
+    src = fetchFromGitHub {
+      owner = "lwthiker";
+      repo = "curl-impersonate";
+      rev = "v${version}";
+      hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A=";
+    };
+
+    patches = [
+      # Fix shebangs in the NSS build script
+      # (can't just patchShebangs since makefile unpacks it)
+      ./curl-impersonate-0.5.2-fix-shebangs.patch
+    ];
+
+    strictDeps = true;
+
+    nativeBuildInputs = lib.optionals stdenv.isDarwin [
+      # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize'
+      darwin.cctools
+    ] ++ [
+      installShellFiles
+      cmake
+      python3
+      python3.pkgs.gyp
+      ninja
+      perl
+      autoconf
+      automake
+      libtool
+      unzip
+      go
+    ];
+
+    buildInputs = [
+      zlib
+      sqlite
+    ];
+
+    configureFlags = [
+      "--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}"
+      "--with-ca-path=${cacert}/etc/ssl/certs"
+    ];
+
+    buildFlags = [ "${target}-build" ];
+    checkTarget = "${target}-checkbuild";
+    installTargets = [ "${target}-install" ];
+
+    doCheck = true;
+
+    dontUseCmakeConfigure = true;
+    dontUseNinjaBuild = true;
+    dontUseNinjaInstall = true;
+    dontUseNinjaCheck = true;
+
+    postUnpack = lib.concatStringsSep "\n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps));
+
+    preConfigure = ''
+      export GOCACHE=$TMPDIR/go-cache
+      export GOPATH=$TMPDIR/go
+      export GOPROXY=file://${passthru.boringssl-go-modules}
+      export GOSUMDB=off
+
+      # Need to get value of $out for this flag
+      configureFlagsArray+=("--with-libnssckbi=$out/lib")
+    '';
+
+    postInstall = ''
+      # Remove vestigial *-config script
+      rm $out/bin/curl-impersonate-${name}-config
+
+      # Patch all shebangs of installed scripts
+      patchShebangs $out/bin
+
+      # Build and install completions for each curl binary
+
+      # Patch in correct binary name and alias it to all scripts
+      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh
+      substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \
+        --replace \
+          '#compdef curl' \
+          "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')"
+
+      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish
+      substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \
+        --replace \
+          '--command curl' \
+          "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')"
+
+      # Install zsh and fish completions
+      installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish}
+    '';
+
+    preFixup = let
+      libext = stdenv.hostPlatform.extensions.sharedLibrary;
+    in ''
+      # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure
+      if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then
+        # NOTE: "p11-kit-trust" always ends in ".so" even when on darwin
+        ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext}
+        ${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"}
+      fi
+    '';
+
+    disallowedReferences = [ go ];
+
+    passthru = {
+      deps = callPackage ./deps.nix {};
+
+      boringssl-go-modules = (buildGoModule {
+        inherit (passthru.deps."boringssl.zip") name;
+
+        src = passthru.deps."boringssl.zip";
+        vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q=";
+
+        nativeBuildInputs = [ unzip ];
+
+        proxyVendor = true;
+      }).go-modules;
+    };
+
+    meta = with lib; {
+      description = "A special build of curl that can impersonate Chrome & Firefox";
+      homepage = "https://github.com/lwthiker/curl-impersonate";
+      license = with licenses; [ curl mit ];
+      maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ];
+      platforms = platforms.unix;
+      knownVulnerabilities = [
+        "CVE-2023-32001"  # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html
+        "CVE-2022-43551"  # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html
+        "CVE-2022-42916"  # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html
+      ];
+    };
   };
+in
 
-  nativeBuildInputs = [ autoPatchelfHook zlib ];
+symlinkJoin rec {
+  pname = "curl-impersonate";
+  inherit (passthru.curl-impersonate-ff) version meta;
 
-  installPhase = ''
-    mkdir -p $out/bin
-    cp * $out/bin
-  '';
+  name = "${pname}-${version}";
 
-  meta = with lib; {
-    description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox ";
-    homepage = "https://github.com/lwthiker/curl-impersonate";
-    license = with licenses; [ curl mit ];
-    maintainers = with maintainers; [ deliciouslytyped ];
-    platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere.
+  paths = [
+    passthru.curl-impersonate-ff
+    passthru.curl-impersonate-chrome
+  ];
+
+  passthru = {
+    curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; };
+    curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; };
+
+    updateScript = ./update.sh;
+
+    inherit (passthru.curl-impersonate-ff) src;
   };
 }

pkgs/tools/networking/curl-impersonate/deps.nix

@@ -0,0 +1,29 @@
+# Generated by update.sh
+{ fetchurl }:
+
+{
+  "curl-7.84.0.tar.xz" = fetchurl {
+    url = "https://curl.se/download/curl-7.84.0.tar.xz";
+    hash = "sha256-LRGLQ/VHv+W66AbY1HtOWW6lslpsHwgK70n7zYF8Xbg=";
+  };
+
+  "brotli-1.0.9.tar.gz" = fetchurl {
+    url = "https://github.com/google/brotli/archive/refs/tags/v1.0.9.tar.gz";
+    hash = "sha256-+ejYHQQFumbRgVKa9CozVPg4yTkJX/mZMNpqqc32/kY=";
+  };
+
+  "nss-3.87.tar.gz" = fetchurl {
+    url = "https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_87_RTM/src/nss-3.87-with-nspr-4.35.tar.gz";
+    hash = "sha256-63DqC1jc5pqkkOnp/s0TKn1kTh2j1jHhYzdqDcwRoCI=";
+  };
+
+  "boringssl.zip" = fetchurl {
+    url = "https://github.com/google/boringssl/archive/3a667d10e94186fd503966f5638e134fe9fb4080.zip";
+    hash = "sha256-HsDIkd1x5IH49fUF07dJaabMIMsQygW+NI7GneULpA8=";
+  };
+
+  "nghttp2-1.46.0.tar.bz2" = fetchurl {
+    url = "https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2";
+    hash = "sha256-moKXjIcAcbdp8n0riBkct3/clFpRwdaFx/YafhP8Ryk=";
+  };
+}

pkgs/tools/networking/curl-impersonate/update.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts
+set -euo pipefail
+
+nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs repo\nAre we running from within the nixpkgs git repo?\n' >&2; exit 1))"
+
+stripwhitespace() {
+    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
+}
+
+narhash() {
+    nix --extra-experimental-features nix-command store prefetch-file --json "$1" | jq -r .hash
+}
+
+nixeval() {
+    nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1" | jq -r .
+}
+
+vendorhash() {
+    (nix --extra-experimental-features nix-command build --no-link -f "$nixpkgs" --no-link "$1" 2>&1 >/dev/null | tail -n3 | grep -F got: | cut -d: -f2- | stripwhitespace) 2>/dev/null || true
+}
+
+findpath() {
+    path="$(nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1.meta.position" | jq -r . | cut -d: -f1)"
+    outpath="$(nix --extra-experimental-features nix-command eval --json --impure --expr "builtins.fetchGit \"$nixpkgs\"")"
+
+    if [ -n "$outpath" ]; then
+        path="${path/$(echo "$outpath" | jq -r .)/$nixpkgs}"
+    fi
+
+    echo "$path"
+}
+
+getvar() {
+    echo "$2" | grep -F "$1" | sed -e 's/:=/:/g' | cut -d: -f2- | stripwhitespace
+}
+
+attr="${UPDATE_NIX_ATTR_PATH:-curl-impersonate}"
+version="$(curl -sSL "https://api.github.com/repos/lwthiker/curl-impersonate/releases/latest" | jq -r .tag_name | sed -e 's/^v//')"
+
+pkgpath="$(findpath "$attr")"
+
+updated="$(cd "$nixpkgs" && update-source-version "$attr" "$version" --file="$pkgpath" --print-changes | jq -r length)"
+
+if [ "$updated" -eq 0 ]; then
+    echo 'update.sh: Package version not updated, nothing to do.'
+    exit 0
+fi
+
+vars="$(curl -sSL "https://github.com/lwthiker/curl-impersonate/raw/v$version/Makefile.in" | grep '^ *[^ ]*_\(VERSION\|URL\|COMMIT\) *:=')"
+
+cat >"$(dirname "$pkgpath")"/deps.nix <<EOF
+# Generated by update.sh
+{ fetchurl }:
+
+{
+  "$(getvar CURL_VERSION "$vars").tar.xz" = fetchurl {
+    url = "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz";
+    hash = "$(narhash "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz")";
+  };
+
+  "brotli-$(getvar BROTLI_VERSION "$vars").tar.gz" = fetchurl {
+    url = "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz";
+    hash = "$(narhash "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz")";
+  };
+
+  "$(getvar NSS_VERSION "$vars").tar.gz" = fetchurl {
+    url = "$(getvar NSS_URL "$vars")";
+    hash = "$(narhash "$(getvar NSS_URL "$vars")")";
+  };
+
+  "boringssl.zip" = fetchurl {
+    url = "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip";
+    hash = "$(narhash "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip")";
+  };
+
+  "$(getvar NGHTTP2_VERSION "$vars").tar.bz2" = fetchurl {
+    url = "$(getvar NGHTTP2_URL "$vars")";
+    hash = "$(narhash "$(getvar NGHTTP2_URL "$vars")")";
+  };
+}
+EOF
+
+curhash="$(nixeval "$attr.curl-impersonate-chrome.boringssl-go-modules.outputHash")"
+newhash="$(vendorhash "$attr.curl-impersonate-chrome.boringssl-go-modules")"
+
+if [ -n "$newhash" ] && [ "$curhash" != "$newhash" ]; then
+    sed -i -e "s|\"$curhash\"|\"$newhash\"|" "$pkgpath"
+else
+    echo 'update.sh: New vendorHash same as old vendorHash, nothing to do.'
+fi

pkgs/top-level/aliases.nix

@@ -314,6 +314,7 @@ mapAliases ({
   cups-kyodialog3 = cups-kyodialog; # Added 2022-11-12
   cupsBjnp = throw "'cupsBjnp' has been renamed to/replaced by 'cups-bjnp'"; # Converted to throw 2022-02-22
   cups_filters = throw "'cups_filters' has been renamed to/replaced by 'cups-filters'"; # Converted to throw 2022-02-22
+  curl-impersonate-bin = throw "'curl-impersonate-bin' has been replaced by 'curl-impersonate'"; # Added 2022-10-08
   curlcpp = throw "curlcpp has been removed, no active maintainers and no usage within nixpkgs"; # Added 2022-05-10
   curaByDagoma = throw "curaByDagoma has been removed from nixpkgs, because it was unmaintained and dependent on python2 packages"; # Added 2022-01-12
   curaLulzbot = throw "curaLulzbot has been removed due to insufficient upstream support for a modern dependency chain"; # Added 2021-10-23

pkgs/top-level/all-packages.nix

@@ -6215,7 +6215,8 @@ with pkgs;
 
   curlWithGnuTls = curl.override { gnutlsSupport = true; opensslSupport = false; };
 
-  curl-impersonate-bin = callPackage ../tools/networking/curl-impersonate { };
+  curl-impersonate = darwin.apple_sdk_11_0.callPackage ../tools/networking/curl-impersonate { };
+  inherit (curl-impersonate) curl-impersonate-ff curl-impersonate-chrome;
 
   curlie = callPackage ../tools/networking/curlie { };