anki: 2.1.15 -> 2.1.60

This redoes all the packaging for their new build-system. It feels a bit fragile, but in practice it works. Basically, we build most of it in nix, write some wrapper scripts to mock out stuff we just did in nix, and then call thier build system to make a wheel
2023-03-14 21:15:18 +09:00 · 2023-03-14 21:15:18 +09:00 · 903f78ebde
commit 903f78ebde
parent 5451b89b28
9 changed files with 5227 additions and 174 deletions
--- a/pkgs/games/anki/Cargo.lock
+++ b/pkgs/games/anki/Cargo.lock
--- a/pkgs/games/anki/default.nix
+++ b/pkgs/games/anki/default.nix
@ -1,188 +1,259 @@
-{ stdenv
-, buildPythonApplication
-, lib
-, python
-, fetchurl
+{ lib
+, stdenv
+, buildEnv
 , fetchFromGitHub
-, fetchpatch
+, fetchYarnDeps
+, fixup_yarn_lock
+, installShellFiles
 , lame
 , mpv-unwrapped
-, libpulseaudio
-, pyqtwebengine
-, decorator
-, beautifulsoup4
-, sqlalchemy
-, pyaudio
-, requests
-, markdown
-, matplotlib
-, mock
-, pytest
-, glibcLocales
-, nose
-, jsonschema
-, setuptools
-, send2trash
+, ninja
+, nodejs
+, nodejs-slim
+, protobuf
+, python3
+, qt6
+, rsync
+, rustPlatform
+, writeShellScriptBin
+, yarn
 , CoreAudio
-  # This little flag adds a huge number of dependencies, but we assume that
-  # everyone wants Anki to draw plots with statistics by default.
-, plotsSupport ? true
-  # manual
-, asciidoc
 }:

 let
-  # when updating, also update rev-manual to a recent version of
-  # https://github.com/ankitects/anki-docs
-  # The manual is distributed independently of the software.
-  version = "2.1.15";
-  sha256-pkg = "12dvyf3j9df4nrhhnqbzd9b21rpzkh4i6yhhangn2zf7ch0pclss";
-  rev-manual = "8f6387867ac37ef3fe9d0b986e70f898d1a49139";
-  sha256-manual = "0pm5slxn78r44ggvbksz7rv9hmlnsvn9z811r6f63dsc8vm6mfml";
+  pname = "anki";
+  version = "2.1.60";
+  rev = "76d8807315fcc2675e7fa44d9ddf3d4608efc487";

-  manual = stdenv.mkDerivation {
-    pname = "anki-manual";
-    inherit version;
-    src = fetchFromGitHub {
-      owner = "ankitects";
-      repo = "anki-docs";
-      rev = rev-manual;
-      sha256 = sha256-manual;
+  src = fetchFromGitHub {
+    owner = "ankitects";
+    repo = "anki";
+    rev = version;
+    hash = "sha256-hNrf6asxF7r7QK2XO150yiRjyHAYKN8OFCFYX0SAiwA=";
+    fetchSubmodules = true;
+  };
+
+
+  cargoDeps = rustPlatform.importCargoLock {
+    lockFile = ./Cargo.lock;
+    outputHashes = {
+      "csv-1.1.6" = "sha256-w728ffOVkI+IfK6FbmkGhr0CjuyqgJnPB1kutMJIUYg=";
+      "linkcheck-0.4.1-alpha.0" = "sha256-Fiom8oHW9y7vV2RLXW0ClzHOdIlBq3Z9jLP+p6Sk4GI=";
    };
-    dontInstall = true;
-    nativeBuildInputs = [ asciidoc ];
-    patchPhase = ''
-      # rsync isnt needed
-      # WEB is the PREFIX
-      # We remove any special ankiweb output generation
-      # and rename every .mako to .html
-      sed -e 's/rsync -a/cp -a/g' \
-          -e "s|\$(WEB)/docs|$out/share/doc/anki/html|" \
-          -e '/echo asciidoc/,/mv $@.tmp $@/c \\tasciidoc -b html5 -o $@ $<' \
-          -e 's/\.mako/.html/g' \
-          -i Makefile
-      # patch absolute links to the other language manuals
-      sed -e 's|https://apps.ankiweb.net/docs/|link:./|g' \
-          -i {manual.txt,manual.*.txt}
-      # there’s an artifact in most input files
-      sed -e '/<%def.*title.*/d' \
-          -i *.txt
-      mkdir -p $out/share/doc/anki/html
+  };
+
+  anki-build-python = python3.withPackages (ps: with ps; [
+    pip
+    mypy-protobuf
+  ]);
+
+  # anki shells out to git to check its revision, and also to update submodules
+  # We don't actually need the submodules, so we stub that out
+  fakeGit = writeShellScriptBin "git" ''
+    case "$*" in
+      "rev-parse --short=8 HEAD")
+        echo ${builtins.substring 0 8 rev}
+      ;;
+      *"submodule update "*)
+        exit 0
+      ;;
+      *)
+        echo "Unrecognized git: $@"
+        exit 1
+      ;;
+    esac
+  '';
+
+  # We don't want to run pip-sync, it does network-io
+  fakePipSync = writeShellScriptBin "pip-sync" ''
+    exit 0
+  '';
+
+  offlineYarn = writeShellScriptBin "yarn" ''
+    [[ "$1" == "install" ]] && exit 0
+    exec ${yarn}/bin/yarn --offline "$@"
+  '';
+
+  pyEnv = buildEnv {
+    name = "anki-pyenv-${version}";
+    paths = with python3.pkgs; [
+      pip
+      fakePipSync
+      anki-build-python
+    ];
+    pathsToLink = [ "/bin" ];
+  };
+
+  yarnOfflineCache = fetchYarnDeps {
+    yarnLock = "${src}/yarn.lock";
+    hash = "sha256-bAtmMGWi5ETIidFFnG3jzJg2mSBnH5ONO2/Lr9A3PpQ=";
+  };
+
+  # https://discourse.nixos.org/t/mkyarnpackage-lockfile-has-incorrect-entry/21586/3
+  anki-nodemodules = stdenv.mkDerivation {
+    pname = "anki-nodemodules";
+    inherit version src yarnOfflineCache;
+
+    nativeBuildInputs = [
+      fixup_yarn_lock
+      yarn
+      nodejs-slim
+    ];
+
+    configurePhase = ''
+      export HOME=$NIX_BUILD_TOP
+      yarn config --offline set yarn-offline-mirror $yarnOfflineCache
+      fixup_yarn_lock yarn.lock
+      yarn install --offline --frozen-lockfile --ignore-scripts --no-progress --non-interactive
+      patchShebangs node_modules/
+      yarn run postinstall --offline
+    '';
+
+    installPhase = ''
+      mv node_modules $out
    '';
  };
-
 in
-buildPythonApplication rec {
-  pname = "anki";
-  inherit version;
-  format = "other";
-
-  src = fetchurl {
-    urls = [
-      "https://apps.ankiweb.net/downloads/current/${pname}-${version}-source.tgz"
-      # "https://apps.ankiweb.net/downloads/current/${name}-source.tgz"
-      # "http://ankisrs.net/download/mirror/${name}.tgz"
-      # "http://ankisrs.net/download/mirror/archive/${name}.tgz"
-    ];
-    sha256 = sha256-pkg;
-  };
+python3.pkgs.buildPythonApplication {
+  inherit pname version src;

  outputs = [ "out" "doc" "man" ];

-  propagatedBuildInputs = [
-    pyqtwebengine
-    sqlalchemy
-    beautifulsoup4
-    send2trash
-    pyaudio
-    requests
-    decorator
-    markdown
-    jsonschema
-    setuptools
-  ]
-  ++ lib.optional plotsSupport matplotlib
-  ++ lib.optionals stdenv.isDarwin [ CoreAudio ]
-  ;
-
-  nativeCheckInputs = [ pytest glibcLocales mock nose ];
-
-  nativeBuildInputs = [ pyqtwebengine.wrapQtAppsHook ];
-  buildInputs = [ lame mpv-unwrapped libpulseaudio ];
-
  patches = [
-    # Disable updated version check.
-    ./no-version-check.patch
-    (fetchpatch {
-      name = "fix-mpv-args.patch";
-      url = "https://sources.debian.org/data/main/a/anki/2.1.15+dfsg-3/debian/patches/fix-mpv-args.patch";
-      sha256 = "1dimnnawk64m5bbdbjrxw5k08q95l728n94cgkrrwxwavmmywaj2";
-    })
-    (fetchpatch {
-      name = "anki-2.1.15-unescape.patch";
-      url = "https://795309.bugs.gentoo.org/attachment.cgi?id=715200";
-      sha256 = "14rz864kdaba4fd1marwkyz9n1jiqnbjy4al8bvwlhpvp0rm1qk6";
-    })
+    ./patches/gl-fixup.patch
+    ./patches/no-update-check.patch
+    # Upstreamed in https://github.com/ankitects/anki/pull/2446
+    # We can drop these once we update to an anki version that includes them
+    # already
+    ./patches/0001-Don-t-download-nodejs-if-NODE_BINARY-is-set.patch
+    ./patches/0002-Allow-setting-YARN_BINARY-for-the-build-system.patch
+    # Not upstreamed
+    ./patches/0003-Skip-formatting-python-code.patch
  ];

-  # Anki does not use setup.py
-  dontBuild = true;
+  inherit cargoDeps;

-  postPatch = ''
-    # Remove QT translation files. We'll use the standard QT ones.
-    rm "locale/"*.qm
+  nativeBuildInputs = [
+    fakeGit
+    fixup_yarn_lock
+    offlineYarn

-    # hitting F1 should open the local manual
-    substituteInPlace anki/consts.py \
-      --replace 'HELP_SITE="http://ankisrs.net/docs/manual.html"' \
-                'HELP_SITE="${manual}/share/doc/anki/html/manual.html"'
+    installShellFiles
+    rustPlatform.rust.cargo
+    rustPlatform.cargoSetupHook
+    ninja
+    qt6.wrapQtAppsHook
+    rsync
+  ];
+  nativeCheckInputs = with python3.pkgs; [ pytest mock astroid  ];
+
+  buildInputs = [
+    qt6.qtbase
+  ];
+  propagatedBuildInputs = with python3.pkgs; [
+    # This rather long list came from running:
+    #    grep --no-filename -oE "^[^ =]*" python/{requirements.base.txt,requirements.bundle.txt,requirements.qt6_4.txt} | \
+    #      sort | uniq | grep -v "^#$"
+    # in their repo at the git tag for this version
+    # There's probably a more elegant way, but the above extracted all the
+    # names, without version numbers, of their python dependencies. The hope is
+    # that nixpkgs versions are "close enough"
+    # I then removed the ones the check phase failed on (pythonCatchConflictsPhase)
+    beautifulsoup4
+    certifi
+    charset-normalizer
+    click
+    colorama
+    decorator
+    distro
+    flask
+    flask-cors
+    idna
+    importlib-metadata
+    itsdangerous
+    jinja2
+    jsonschema
+    markdown
+    markupsafe
+    orjson
+    pep517
+    python3.pkgs.protobuf
+    pyparsing
+    pyqt6
+    pyqt6-sip
+    pyqt6-webengine
+    pyrsistent
+    pysocks
+    requests
+    send2trash
+    six
+    soupsieve
+    urllib3
+    waitress
+    werkzeug
+    zipp
+  ] ++ lib.optionals stdenv.isDarwin [ CoreAudio ];
+
+  # Activate optimizations
+  RELEASE = true;
+
+  PROTOC_BINARY = lib.getExe protobuf;
+  NODE_BINARY = lib.getExe nodejs;
+  YARN_BINARY = lib.getExe offlineYarn;
+  PYTHON_BINARY = lib.getExe python3;
+
+  inherit yarnOfflineCache;
+  dontUseNinjaInstall = false;
+
+  buildPhase = ''
+    export RUST_BACKTRACE=1
+    export RUST_LOG=debug
+
+    mkdir -p out/pylib/anki \
+             .git
+
+    echo ${builtins.substring 0 8 rev} > out/buildhash
+    touch out/env
+    touch .git/HEAD
+
+    ln -vsf ${pyEnv} ./out/pyenv
+    ln -vsf ${pyEnv} ./out/pyenv-qt5
+    rsync --chmod +w -avP ${anki-nodemodules}/ out/node_modules/
+    ln -vsf out/node_modules node_modules
+
+    export HOME=$NIX_BUILD_TOP
+    yarn config --offline set yarn-offline-mirror $yarnOfflineCache
+    fixup_yarn_lock yarn.lock
+
+    patchShebangs ./ninja
+    PIP_USER=1 ./ninja build wheels
  '';

-  # UTF-8 locale needed for testing
-  LC_ALL = "en_US.UTF-8";
-
  # tests fail with to many open files
+  # TODO: verify if this is still true (I can't, no mac)
  doCheck = !stdenv.isDarwin;
-
-  # - Anki writes some files to $HOME during tests
-  # - Skip tests using network
+  # mimic https://github.com/ankitects/anki/blob/76d8807315fcc2675e7fa44d9ddf3d4608efc487/build/ninja_gen/src/python.rs#L232-L250
  checkPhase = ''
-    HOME=$TMP pytest --ignore tests/test_sync.py
+    HOME=$TMP ANKI_TEST_MODE=1 PYTHONPATH=$PYTHONPATH:$PWD/out/pylib \
+      pytest -p no:cacheprovider pylib/tests
+    HOME=$TMP ANKI_TEST_MODE=1 PYTHONPATH=$PYTHONPATH:$PWD/out/pylib:$PWD/pylib:$PWD/out/qt \
+      pytest -p no:cacheprovider qt/tests
  '';

-  installPhase = ''
-    pp=$out/lib/${python.libPrefix}/site-packages
-
-    mkdir -p $out/bin
-    mkdir -p $out/share/applications
-    mkdir -p $doc/share/doc/anki
-    mkdir -p $man/share/man/man1
-    mkdir -p $out/share/mime/packages
-    mkdir -p $out/share/pixmaps
-    mkdir -p $pp
-
-    cat > $out/bin/anki <<EOF
-    #!${python}/bin/python
-    import aqt
-    aqt.run()
-    EOF
-    chmod 755 $out/bin/anki
-
-    cp -v anki.desktop $out/share/applications/
-    cp -v README* LICENSE* $doc/share/doc/anki/
-    cp -v anki.1 $man/share/man/man1/
-    cp -v anki.xml $out/share/mime/packages/
-    cp -v anki.{png,xpm} $out/share/pixmaps/
-    cp -rv locale $out/share/
-    cp -rv anki aqt web $pp/
-
-    # copy the manual into $doc
-    cp -r ${manual}/share/doc/anki/html $doc/share/doc/anki
+  preInstall = ''
+    mkdir dist
+    mv out/wheels/* dist
+  '';
+
+  postInstall = ''
+    install -D -t $out/share/applications qt/bundle/lin/anki.desktop
+    install -D -t $doc/share/doc/anki README* LICENSE*
+    install -D -t $out/share/mime/packages qt/bundle/lin/anki.xml
+    install -D -t $out/share/pixmaps qt/bundle/lin/anki.{png,xpm}
+    installManPage qt/bundle/lin/anki.1
  '';

-  # now wrapPythonPrograms from postFixup will add both python and qt env variables
  dontWrapQtApps = true;
-
  preFixup = ''
    makeWrapperArgs+=(
      "''${qtWrapperArgs[@]}"
@ -190,10 +261,6 @@ buildPythonApplication rec {
    )
  '';

-  passthru = {
-    inherit manual;
-  };
-
  meta = with lib; {
    homepage = "https://apps.ankiweb.net/";
    description = "Spaced repetition flashcard program";
@ -211,6 +278,6 @@ buildPythonApplication rec {
    '';
    license = licenses.agpl3Plus;
    platforms = platforms.mesaPlatforms;
-    maintainers = with maintainers; [ oxij Profpatsch ];
+    maintainers = with maintainers; [ oxij Profpatsch euank ];
  };
 }
--- a/pkgs/games/anki/no-version-check.patch
+++ b/pkgs/games/anki/no-version-check.patch
@ -1,13 +0,0 @@
-diff -Nurp anki-2.0.33.orig/aqt/main.py anki-2.0.33/aqt/main.py
--- anki-2.0.33.orig/aqt/main.py	2016-01-05 21:37:53.904533750 +0100
-+++ anki-2.0.33/aqt/main.py	2016-01-05 21:39:11.469175976 +0100
-@@ -820,6 +820,9 @@ title="%s">%s</button>''' % (
-     ##########################################################################
- 
-     def setupAutoUpdate(self):
-+        # Don't check for latest version since the versions are
-+        # managed in Nixpkgs.
-+        return
-         import aqt.update
-         self.autoUpdate = aqt.update.LatestVersionFinder(self)
-         self.connect(self.autoUpdate, SIGNAL("newVerAvail"), self.newVerAvail)
--- a/pkgs/games/anki/patches/0001-Don-t-download-nodejs-if-NODE_BINARY-is-set.patch
+++ b/pkgs/games/anki/patches/0001-Don-t-download-nodejs-if-NODE_BINARY-is-set.patch
@ -0,0 +1,53 @@
+From 53740ca75d167fab5c403a462e21ecd717b1dafa Mon Sep 17 00:00:00 2001
+From: Euan Kemp <euank@euank.com>
+Date: Fri, 17 Mar 2023 22:38:04 +0900
+Subject: [PATCH 1/2] Don't download nodejs if NODE_BINARY is set
+
+Some build environments, such as nixpkgs, restrict network access and
+thus would prefer to not download anything at all. Setting PROTOC_BINARY
+and friends makes the build system not download stuff, and the same
+should be true for nodejs
+---
+ build/ninja_gen/src/node.rs | 19 +++++++++----------
+ 1 file changed, 9 insertions(+), 10 deletions(-)
+
+diff --git a/build/ninja_gen/src/node.rs b/build/ninja_gen/src/node.rs
+index df05e149d..d08c7011e 100644
+--- a/build/ninja_gen/src/node.rs
+++ b/build/ninja_gen/src/node.rs
+@@ -105,16 +105,6 @@ pub fn setup_node(
+     binary_exports: &[&'static str],
+     mut data_exports: HashMap<&str, Vec<Cow<str>>>,
+ ) -> Result<()> {
+-    download_and_extract(
+-        build,
+-        "node",
+-        archive,
+-        hashmap! {
+-            "bin" => vec![if cfg!(windows) { "node.exe" } else { "bin/node" }],
+-            "npm" => vec![if cfg!(windows) { "npm.cmd " } else { "bin/npm" }]
+-        },
+-    )?;
+-
+     let node_binary = match std::env::var("NODE_BINARY") {
+         Ok(path) => {
+             assert!(
+@@ -124,6 +114,15 @@ pub fn setup_node(
+             path.into()
+         }
+         Err(_) => {
+            download_and_extract(
+                build,
+                "node",
+                archive,
+                hashmap! {
+                    "bin" => vec![if cfg!(windows) { "node.exe" } else { "bin/node" }],
+                    "npm" => vec![if cfg!(windows) { "npm.cmd " } else { "bin/npm" }]
+                },
+            )?;
+             inputs![":extract:node:bin"]
+         }
+     };
+-- 
+2.39.2
+
--- a/pkgs/games/anki/patches/0002-Allow-setting-YARN_BINARY-for-the-build-system.patch
+++ b/pkgs/games/anki/patches/0002-Allow-setting-YARN_BINARY-for-the-build-system.patch
@ -0,0 +1,36 @@
+From 16af7d4cabcf10797bd110c905a9d7694bde0fb4 Mon Sep 17 00:00:00 2001
+From: Euan Kemp <euank@euank.com>
+Date: Fri, 17 Mar 2023 23:07:05 +0900
+Subject: [PATCH 2/2] Allow setting YARN_BINARY for the build system
+
+---
+ build/ninja_gen/src/node.rs | 13 ++++++++++++-
+ 1 file changed, 12 insertions(+), 1 deletion(-)
+
+diff --git a/build/ninja_gen/src/node.rs b/build/ninja_gen/src/node.rs
+index d08c7011e..c1e2ce1b3 100644
+--- a/build/ninja_gen/src/node.rs
+++ b/build/ninja_gen/src/node.rs
+@@ -129,7 +129,18 @@ pub fn setup_node(
+     let node_binary = build.expand_inputs(node_binary);
+     build.variable("node_binary", &node_binary[0]);
+ 
+-    build.add("yarn", YarnSetup {})?;
+    match std::env::var("YARN_BINARY") {
+        Ok(path) => {
+            assert!(
+                Utf8Path::new(&path).is_absolute(),
+                "YARN_BINARY must be absolute"
+            );
+            build.add_resolved_files_to_group("yarn:bin", &vec![path]);
+        },
+        Err(_) => {
+            build.add("yarn", YarnSetup {})?;
+        },
+    };
+ 
+     for binary in binary_exports {
+         data_exports.insert(
+-- 
+2.39.2
+
--- a/pkgs/games/anki/patches/0003-Skip-formatting-python-code.patch
+++ b/pkgs/games/anki/patches/0003-Skip-formatting-python-code.patch
@ -0,0 +1,31 @@
+From ed5090b237bca768dbf7dfc3b4414b955879f15e Mon Sep 17 00:00:00 2001
+From: Euan Kemp <euank@euank.com>
+Date: Fri, 7 Apr 2023 20:22:34 +0900
+Subject: [PATCH 3/3] Skip formatting python code
+
+---
+ pylib/tools/hookslib.py | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/pylib/tools/hookslib.py b/pylib/tools/hookslib.py
+index 6361c633e..95ecb64a2 100644
+--- a/pylib/tools/hookslib.py
+++ b/pylib/tools/hookslib.py
+@@ -82,7 +82,7 @@ class Hook:
+         code = f"""\
+ class {self.classname()}:
+ {classdoc}{self.list_code()}
+-    
+
+     def append(self, callback: {self.callable()}) -> None:
+         '''{appenddoc}'''
+         self._hooks.append(callback)
+@@ -208,4 +208,4 @@ def write_file(path: str, hooks: list[Hook], prefix: str, suffix: str):
+         os.environ["USERPROFILE"] = os.environ["HOME"]
+     with open(path, "wb") as file:
+         file.write(code.encode("utf8"))
+-    subprocess.run([sys.executable, "-m", "black", "-q", path], check=True)
+    # subprocess.run([sys.executable, "-m", "black", "-q", path], check=True)
+-- 
+2.39.2
+
--- a/pkgs/games/anki/patches/gl-fixup.patch
+++ b/pkgs/games/anki/patches/gl-fixup.patch
@ -0,0 +1,17 @@
+diff --git a/qt/aqt/__init__.py b/qt/aqt/__init__.py
+index 352848cfd..3fd5d0769 100644
+--- a/qt/aqt/__init__.py
+++ b/qt/aqt/__init__.py
+@@ -402,12 +402,6 @@ def parseArgs(argv: list[str]) -> tuple[argparse.Namespace, list[str]]:
+ def setupGL(pm: aqt.profiles.ProfileManager) -> None:
+     driver = pm.video_driver()
+
+-    # work around pyqt loading wrong GL library
+-    if is_lin:
+-        import ctypes
+-
+-        ctypes.CDLL("libGL.so.1", ctypes.RTLD_GLOBAL)
+-
+     # catch opengl errors
+     def msgHandler(category: Any, ctx: Any, msg: Any) -> None:
+         if category == QtMsgType.QtDebugMsg:
--- a/pkgs/games/anki/patches/no-update-check.patch
+++ b/pkgs/games/anki/patches/no-update-check.patch
@ -0,0 +1,13 @@
+diff --git a/qt/aqt/main.py b/qt/aqt/main.py
+index 0f2764f66..c42a88402 100644
+--- a/qt/aqt/main.py
+++ b/qt/aqt/main.py
+@@ -1395,6 +1395,8 @@ title="{}" {}>{}</button>""".format(
+     ##########################################################################
+
+     def setupAutoUpdate(self) -> None:
+        # nixpkgs patch; updates are managed by nix
+        return
+         import aqt.update
+
+         self.autoUpdate = aqt.update.LatestVersionFinder(self)
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -35572,7 +35572,7 @@ with pkgs;

  angband = callPackage ../games/angband { };

-  anki = python39Packages.callPackage ../games/anki {
+  anki = callPackage ../games/anki {
    inherit (darwin.apple_sdk.frameworks) CoreAudio;
  };
  anki-bin = callPackage ../games/anki/bin.nix { buildFHSUserEnv = buildFHSUserEnvBubblewrap; };