require fewer network interactions during build

flake.nix

@@ -47,6 +47,11 @@
 # `./node_modules/.bin/electron .` is reading `./package.json`.
 #  - dependent on:   "main": "app/main.js"
 #    app/main.js is generated (from app/main.ts, checked into git)
+# - yarn install --ignore-scripts:
+#   - ringrtc uses a script, to DL https://build-artifacts.signal.org/libraries/ringrtc-desktop-build-v2.33.0.tar.gz
+#   - it's in package.scripts.install
+#   - node scripts/fetch-prebuild.js
+#   - wants it to exist as `prebuild.tar.gz`  (i could provide that for it)
 {
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
@@ -56,6 +61,14 @@
     with flake-utils.lib; eachSystem allSystems (system:
       let
         pkgs = import nixpkgs { inherit system; };
+        ringrtcPrebuild = builtins.fetchurl {
+          url = "https://build-artifacts.signal.org/libraries/ringrtc-desktop-build-v2.33.0.tar.gz";
+          sha256 = "sha256:0zmgjax0ycjkvhqz1hybfi799lzss52s1cd8hzbqnm4ka3b1lhsf";
+        };
+        sqlcipher = builtins.fetchurl {
+          url = "https://build-artifacts.signal.org/desktop/sqlcipher-4.5.5-fts5-fix--3.0.7--0.2.1-ef53ea45ed92b928ecfd33c552d8d405263e86e63dec38e1ec63e1b0193b630b.tar.gz";
+          sha256 = "sha256:02v37ccv1qb3xkhkiv1xws33w9h5skc55i9kzpn2ifcjxm2yllzg";
+        };
         nodejs_18_15_0 = pkgs.nodejs.overrideAttrs (upstream:
         let
           # 18.15.0 matches the version in package.json
@@ -68,6 +81,21 @@
             inherit sha256;
           };
         });
+        yarn_18_15_0 = (pkgs.yarn.override { nodejs = nodejs_18_15_0; }).overrideAttrs (upstream: {
+          # patches = (upstream.patches or []) ++ [
+          #   ./yarn-allow-extraneous.patch
+          # ];
+          # force yarn to never delete files; that way i can install the ringrtc prebuild w/o yarn deleting it.
+          # postPatch = (upstream.postPatch or "") + ''
+          #   substituteInPlace lib/cli.js \
+          #     --replace 'yield (_fs || _load_fs()).unlink' '// yield (_fs || _load_fs()).unlink'
+          # '';
+          preFixup = (upstream.preFixup or "") + ''
+            substituteInPlace $out/libexec/yarn/lib/cli.js --replace \
+              "this.reporter.verbose(this.reporter.lang('verboseFileRemoveExtraneous', _loc6));" \
+              "if (false)"
+          '';
+        });
         electronDeps = with pkgs; [
           # packages which electron dynamically links against
           alsa-lib
@@ -174,12 +202,273 @@
 
           echo "launch signal with: ./release/linux-unpacked/signal-desktop"
         '';
-      in {
+        reproducible2 = with pkgs; writeShellScriptBin "reproducible2" ''
+          clean
+          link-caches
+          yarn install --frozen-lockfile
+          yarn generate
+          yarn build:esbuild:prod
+          # this fails during the "fuse" process, which is ok to skip
+          yarn build:release \
+            -c.electronDist=${electron}/libexec/electron \
+            -c.electronVersion=${electron.version} \
+            --dir \
+            || true
+
+          patch-interpreter release/linux-unpacked/signal-desktop
+
+          # nixpkgs electron doesn't seem to support zygote/GPU.
+          # not electron (27); not electron_25
+          echo "launch signal with: ./release/linux-unpacked/signal-desktop --no-sandbox --no-zygote"
+        '';
+        reproducible3 = with pkgs; writeShellScriptBin "reproducible3" ''
+          clean
+          link-caches
+          yarn install --frozen-lockfile
+          yarn generate
+          yarn build:esbuild:prod
+          # this fails during the "fuse" process, which is ok to skip
+          yarn build:release \
+            -c.electronDist=${electron-bin}/libexec/electron \
+            -c.electronVersion=${electron-bin.version} \
+            --dir \
+            || true
+
+          # patch-interpreter release/linux-unpacked/signal-desktop
+
+          # electron-bin (26.3.0) DOES support default flags (i.e. zygote)
+          # echo "launch signal with: ${electron-bin}/libexec/electron/electron ./release/linux-unpacked/resources/app.asar"
+          echo "launch signal with: ${electron-bin}/bin/electron ./release/linux-unpacked/resources/app.asar"  # same as libexec
+        '';
+        reproducible4 = with pkgs; writeShellScriptBin "reproducible4" ''
+          clean
+          link-caches
+
+          yarn install --frozen-lockfile
+          # yarn generate --offline --frozen-lockfile
+          # apparently this isn't equivalent to `yarn generate --offline --frozen-lockfile`!
+          yarn build-module-protobuf --offline --frozen-lockfile
+          yarn build:esbuild --offline --frozen-lockfile
+          yarn sass
+          yarn get-expire-time
+          yarn copy-components
+
+          yarn build:esbuild:prod --offline --frozen-lockfile
+          # this fails during the "fuse" process, which is ok to skip
+          yarn build:release \
+            -c.electronDist=${electron-bin}/libexec/electron \
+            -c.electronVersion=${electron-bin.version} \
+            --dir \
+            || true
+
+          # electron-bin (26.3.0) DOES support default flags (i.e. zygote)
+          # echo "launch signal with: ${electron-bin}/libexec/electron/electron ./release/linux-unpacked/resources/app.asar"
+          echo "launch signal with: ${electron-bin}/bin/electron ./release/linux-unpacked/resources/app.asar"  # same as libexec
+        '';
+        reproducible5 = with pkgs; writeShellScriptBin "reproducible5" ''
+          set -x
+          clean
+          link-caches
+
+          # mkdir -p "$HOME/.node-gyp/${nodejs_18_15_0.version}"
+          # echo 9 > "$HOME/.node-gyp/${nodejs_18_15_0.version}/installVersion"
+          # ln -sfv "$(pwd)/include" "$HOME/.node-gyp/${nodejs_18_15_0.version}"
+          # export npm_config_nodedir=${nodejs_18_15_0}
+
+          yarn install --frozen-lockfile --ignore-scripts
+          cp ${ringrtcPrebuild} node_modules/@signalapp/ringrtc/scripts/prebuild.tar.gz
+          cp ${sqlcipher} node_modules/@signalapp/better-sqlite3/deps/sqlcipher.tar.gz
+          # N.B.: without patched yarn, this deletes the vendored dependencies i just installed
+          yarn install --frozen-lockfile
+
+          # N.B.: `yarn install` ships node_modules/@signalapp/better-sqlite3/build/Release/better_sqlite3.node
+          # also build/Release/obj.target/better_sqlite3.noe
+          # also libcrypto.a, libsignal_tokenizer.a
+          # - both of these for x64 and aarch64
+          # also sqlite3.a
+          # remove build/Release to remove allof those prebuilts
+          rm -rf node_modules/@signalapp/better-sqlite3/build/Release
+
+          npm rebuild @signalapp/better-sqlite3  --offline
+
+          # yarn generate --offline --frozen-lockfile
+          # apparently this isn't equivalent to `yarn generate --offline --frozen-lockfile`!
+          yarn build-module-protobuf --offline --frozen-lockfile
+          yarn build:esbuild --offline --frozen-lockfile
+          yarn sass
+          yarn get-expire-time
+          yarn copy-components
+
+          yarn build:esbuild:prod --offline --frozen-lockfile
+          # this fails during the "fuse" process, which is ok to skip
+          yarn build:release \
+            -c.electronDist=${electron-bin}/libexec/electron \
+            -c.electronVersion=${electron-bin.version} \
+            --dir \
+            || true
+
+          # patch-interpreter release/linux-unpacked/signal-desktop
+
+          # electron-bin (26.3.0) DOES support default flags (i.e. zygote)
+          # echo "launch signal with: ${electron-bin}/libexec/electron/electron ./release/linux-unpacked/resources/app.asar"
+          echo "launch signal with: ${electron-bin}/bin/electron ./release/linux-unpacked/resources/app.asar"  # same as libexec
+        '';
+
+        reproducible6 = with pkgs; writeShellScriptBin "reproducible6" ''
+          set -x
+          clean
+          link-caches
+
+          yarn install --frozen-lockfile --ignore-scripts
+          # these packages have build scripts:
+          # - [x] node_modules/@signalapp/ringrtc/package.json
+          #   - 22:    "install": "node scripts/fetch-prebuild.js",
+          # - [x] node_modules/@signalapp/mock-server/node_modules/@signalapp/libsignal-client/package.json
+          #   - 19:    "install": "node-gyp-build",
+          # - [x] node_modules/@signalapp/better-sqlite3/package.json
+          #   - 37:    "install": "npm run download && npm run build-release",
+          # - [x] node_modules/@signalapp/libsignal-client/package.json
+          #   - 19:    "install": "node-gyp-build",
+          #
+          # - [x] node_modules/bufferutil/package.json
+          #   - 10:    "install": "node-gyp-build",
+          # - [x] node_modules/mac-screen-capture-permissions/package.json
+          #   - 18:    "install": "prebuild-install || node install.js"
+          # - [x] node_modules/playwright/package.json
+          #   - 28:    "install": "node install.js"
+          # - [x] node_modules/utf-8-validate/package.json
+          #   - 10:   "install": "node-gyp-build",
+          #
+          # several packages also have a postinstall in their package.json
+          # - [x] node_modules/@babel/runtime-corejs3/node_modules/core-js-pure/package.json
+          #   - 55:    "postinstall": "node -e \"try{require('./postinstall')}catch(e){}\""
+          # - [x] node_modules/@storybook/builder-manager/node_modules/esbuild/package.json
+          #   - 7:    "postinstall": "node install.js"
+          # - [x] node_modules/@storybook/core-common/node_modules/esbuild/package.json
+          #   - 7:    "postinstall": "node install.js"
+          # - [x] node_modules/@swc/core/package.json
+          #   - 51:    "postinstall": "node postinstall.js",
+          # - [x] node_modules/core-js-pure/package.json
+          #   - 71:    "postinstall": "node -e \"try{require('./postinstall')}catch(e){}\""
+          # - [x] node_modules/core-js/package.json
+          #   - 41:    "postinstall": "node scripts/postinstall || echo \"ignore\""
+          # - [x] node_modules/danger/node_modules/core-js/package.json
+          #   - 55:    "postinstall": "node -e \"try{require('./postinstall')}catch(e){}\""
+          # - [x] node_modules/electron/package.json
+          #   - 8:    "postinstall": "node install.js"
+          # - [x] node_modules/esbuild/package.json
+          #   - 7:    "postinstall": "node install.js"
+          # - [x] node_modules/protobufjs/package.json
+          #   - 43:    "postinstall": "node scripts/postinstall",
+          # - [x] node_modules/websocket/node_modules/es5-ext/package.json
+          #   - 115:   "postinstall": " node -e \"try{require('./_postinstall')}catch(e){}\" || exit 0",
+
+          cp ${ringrtcPrebuild} node_modules/@signalapp/ringrtc/scripts/prebuild.tar.gz
+          pushd node_modules/@signalapp/ringrtc/
+            tar -xzf ./scripts/prebuild.tar.gz
+          popd
+          cp ${sqlcipher} node_modules/@signalapp/better-sqlite3/deps/sqlcipher.tar.gz
+          pushd node_modules/@signalapp/libsignal-client
+            yarn node-gyp-build
+          popd
+          pushd node_modules/@signalapp/mock-server/node_modules/@signalapp/libsignal-client
+            yarn node-gyp-build
+          popd
+          pushd node_modules/bufferutil
+            yarn node-gyp-build
+          popd
+          pushd node_modules/bufferutil
+            yarn node-gyp-build
+          popd
+          pushd node_modules/mac-screen-capture-permissions
+            yarn prebuild-install || node install.js
+          popd
+          pushd node_modules/playwright
+            node install.js
+          popd
+          pushd node_modules/utf-8-validate
+            yarn node-gyp-build
+          popd
+
+          pushd node_modules/@babel/runtime-corejs3/node_modules/core-js-pure
+            node -e "try{require('./postinstall')}catch(e){}"
+          popd
+          pushd node_modules/@swc/core
+            node postinstall.js
+          popd
+          pushd node_modules/@storybook/builder-manager/node_modules/esbuild
+            node install.js
+          popd
+          pushd node_modules/@storybook/core-common/node_modules/esbuild
+            node install.js
+          popd
+          pushd node_modules/core-js-pure
+            node -e "try{require('./postinstall')}catch(e){}"
+          popd
+          pushd node_modules/core-js
+            node scripts/postinstall || echo "ignore"
+          popd
+          pushd node_modules/danger/node_modules/core-js
+            node -e "try{require('./postinstall')}catch(e){}"
+          popd
+          pushd node_modules/electron
+            node install.js
+          popd
+          pushd node_modules/esbuild
+            node install.js
+          popd
+          pushd node_modules/protobufjs
+            node scripts/postinstall
+          popd
+          pushd node_modules/websocket/node_modules/es5-ext
+            node -e "try{require('./_postinstall')}catch(e){}"
+          popd
+
+          # run signal's own `postinstall`:
+          yarn build:acknowledgments
+          yarn patch-package
+          yarn electron:install-app-deps
+
+          # N.B.: `yarn install` ships node_modules/@signalapp/better-sqlite3/build/Release/better_sqlite3.node
+          # also build/Release/obj.target/better_sqlite3.noe
+          # also libcrypto.a, libsignal_tokenizer.a
+          # - both of these for x64 and aarch64
+          # also sqlite3.a
+          # remove build/Release to remove allof those prebuilts
+          rm -rf node_modules/@signalapp/better-sqlite3/build/Release
+
+          npm rebuild @signalapp/better-sqlite3  --offline
+
+          # yarn generate --offline --frozen-lockfile
+          # apparently this isn't equivalent to `yarn generate --offline --frozen-lockfile`!
+          yarn build-module-protobuf --offline --frozen-lockfile
+          yarn build:esbuild --offline --frozen-lockfile
+          yarn sass
+          yarn get-expire-time
+          yarn copy-components
+
+          yarn build:esbuild:prod --offline --frozen-lockfile
+          # this fails during the "fuse" process, which is ok to skip
+          yarn build:release \
+            -c.electronDist=${electron-bin}/libexec/electron \
+            -c.electronVersion=${electron-bin.version} \
+            --dir \
+            || true
+
+          # patch-interpreter release/linux-unpacked/signal-desktop
+
+          # electron-bin (26.3.0) DOES support default flags (i.e. zygote)
+          # echo "launch signal with: ${electron-bin}/libexec/electron/electron ./release/linux-unpacked/resources/app.asar"
+          echo "launch signal with: ${electron-bin}/bin/electron ./release/linux-unpacked/resources/app.asar"  # same as libexec
+        '';
+
+        LD_LIBRARY_PATH = builtins.concatStringsSep ":" (builtins.map
+          (pkg: "${pkgs.lib.getLib pkg}/lib")
+          (electronDeps ++ _7zaDeps ++ fpmDeps)
+        );
+      in rec {
         devShells.default = with pkgs; mkShell {
-          env.LD_LIBRARY_PATH = builtins.concatStringsSep ":" (builtins.map
+          env = { inherit LD_LIBRARY_PATH; };
-            (pkg: "${lib.getLib pkg}/lib")
-            (electronDeps ++ _7zaDeps ++ fpmDeps)
-          );
           buildInputs = [
             clean
             link-caches
@@ -187,11 +476,21 @@
             patch-signal
             reproducible0
             reproducible1
+            reproducible2
+            reproducible3
+            reproducible4
+            reproducible5
+            reproducible6
 
             # yarn
-            (yarn.override { nodejs = nodejs_18_15_0; })
+            # nodejs
+            # nodejs 18.18.2 causes signal to hang on launch.
+            # 18.15.0 (matching package.json) works!
+            yarn_18_15_0
             nodejs_18_15_0  # needed for `yarn install` bettersqlite step
 
+            # electron
+
             # ruby  # needed by fpm (?)
 
             # this seems to all not be needed, thanks to LD_LIBRARY_PATH above
@@ -213,6 +512,91 @@
             # pango
           ];
         };
+
+        packages.yarn_18_15_0 = yarn_18_15_0;
+        packages.signal-desktop-from-src = with pkgs; stdenv.mkDerivation {
+          pname = "signal-desktop-from-src";
+          version = "6.36.0";
+          src = ./.;
+
+          env = {
+            inherit LD_LIBRARY_PATH;
+            yarnOfflineCache = fetchYarnDeps {
+              # this might be IFD: if `nix run '.#check.nur'` fails then inline the lock: `yarnLock = ./yarn.lock`
+              yarnLock = ./yarn.lock;
+              hash = "sha256-AXT6p5lgF0M9ckoxiAvT1HaJhUWVtwEOadY4otdeB0Q=";
+            };
+            # ELECTRON_SKIP_BINARY_DOWNLOAD = "1";
+          };
+          nativeBuildInputs = with pkgs; [
+            clean
+            reproducible4
+
+            fixup_yarn_lock
+            nodejs_18_15_0  # needed for `yarn install` bettersqlite step
+            python3
+            yarn_18_15_0
+          ];
+
+          configurePhase = ''
+            runHook preConfigure
+            set -x
+
+            export HOME=$NIX_BUILD_TOP
+            yarn config --offline set yarn-offline-mirror $yarnOfflineCache
+            fixup_yarn_lock yarn.lock
+
+            # prevent node-gyp from trying to download nodejs headers
+            mkdir -p "$HOME/.node-gyp/${nodejs_18_15_0.version}"
+            echo 9 > "$HOME/.node-gyp/${nodejs_18_15_0.version}/installVersion"
+            ln -sfv "$(pwd)/include" "$HOME/.node-gyp/${nodejs_18_15_0.version}"
+            export npm_config_nodedir=${nodejs_18_15_0}
+
+            # optional flags:  --no-progress --non-interactive
+            # yarn install creates the node_modules/ directory
+            yarn install --offline --frozen-lockfile --ignore-scripts
+            cp ${ringrtcPrebuild} node_modules/@signalapp/ringrtc/scripts/prebuild.tar.gz
+            cp ${sqlcipher} node_modules/@signalapp/better-sqlite3/deps/sqlcipher.tar.gz
+            patchShebangs node_modules/
+            # N.B.: without patched yarn, this deletes the prebuild.tar.gz i just installed
+            yarn install --offline --frozen-lockfile --verbose
+            patchShebangs node_modules/
+
+            runHook postConfigure
+          '';
+
+          buildPhase = ''
+            runHook preBuild
+
+            set -x
+
+            # yarn generate
+            yarn build-module-protobuf --offline --frozen-lockfile
+            yarn build:esbuild --offline --frozen-lockfile
+            yarn sass
+            yarn get-expire-time
+            yarn copy-components
+
+            yarn build:esbuild:prod --offline --frozen-lockfile
+            # this fails during the "fuse" process, which is ok to skip
+            yarn build:release \
+              -c.electronDist=${electron-bin}/libexec/electron \
+              -c.electronVersion=${electron-bin.version} \
+              --dir \
+              || true
+
+            runHook postBuild
+          '';
+
+          installPhase = ''
+            runHook preInstall
+
+            cp -R ./release $out
+
+            runHook postInstall
+          '';
+        };
+        defaultPackage = packages.signal-desktop-from-src;
       }
     )
   ;

ts/sql/Server.ts

@@ -511,7 +511,6 @@ function openAndMigrateDatabase(
   let db: Database | undefined;
 
   // First, we try to open the database without any cipher changes
-  try {
     db = new SQL(filePath, {
       readonly,
     });
@@ -520,12 +519,6 @@ function openAndMigrateDatabase(
     migrateSchemaVersion(db);
 
     return db;
-  } catch (error) {
-    if (db) {
-      db.close();
-    }
-    logger.info('migrateDatabase: Migration without cipher change failed');
-  }
 
   // If that fails, we try to open the database with 3.x compatibility to extract the
   //   user_version (previously stored in schema_version, blown away by cipher_migrate).

yarn-allow-extraneous.patch

@@ -0,0 +1,16 @@
+--- src/package-linker.js
++++ src/package-linker.js
+@@ -472,12 +472,6 @@ export default class PackageLinker {
+       },
+     });
+
+-    // remove all extraneous files that weren't in the tree
+-    for (const loc of possibleExtraneous) {
+-      this.reporter.verbose(this.reporter.lang('verboseFileRemoveExtraneous', loc));
+-      await fs.unlink(loc);
+-    }
+-
+     // remove any empty scoped directories
+     for (const scopedPath of scopedPaths) {
+       const files = await fs.readdir(scopedPath);
+