koreader-from-src: remove a bunch of old cruft

this is beginning to look upstream-ready :)
This commit is contained in:
Colin 2024-03-22 20:16:33 +00:00
parent 58341b75f2
commit d129ae2c03
2 changed files with 16 additions and 413 deletions

View File

@ -1,3 +1,11 @@
# to update:
# - first, figure the rev for `koreader-base`:
# - inside `koreader` repo:
# - `git submodule status base`
# - or `git log base`
# - inside `koreader-base` repo:
# - `git diff old-rev..new-rev thirdparty`
#
# koreader's native build process
# 1. git clone each dependency lib into base/thirdparty/$lib
# 2. git checkout a specific rev into base/thirdparty/$lib/build/$platform
@ -15,40 +23,25 @@
#
# TODO:
# - don't vendor fonts
# - package enough of KOReader's deps to remove `sources.nix`
# - SDL2 (only used by macos??)
# - FBINK
# - NANOSVG (slightly complicated; koreader needs access to its source code)
# - SRELL
# - build crengine dep via nixpkgs `coolreader` pkg (with source patched to <https://github.com/koreader/crengine>)?
{ lib
, autoPatchelfHook
, autoconf
, automake
, buildPackages
, callPackage
, cmake
, dpkg
, fetchFromGitHub
, fetchFromGitLab
, fetchgit
, fetchurl
, gettext
, git
, libtool
, luajit
, makeWrapper
, perl
, pkg-config
, pkgs
, python3
, ragel
, stdenv
, substituteAll
, which
, symlinkJoin
# third-party dependencies which KOReader would ordinarily vendor
, symlinkJoin
, curl
, czmq
, djvulibre
@ -78,7 +71,6 @@
, zsync
}:
let
sources = callPackage ./sources.nix { luajit = luajit52; };
version = "2024.03";
src = fetchFromGitHub {
owner = "koreader";
@ -327,7 +319,6 @@ let
# values to provide to koreader/base/Makefile.defs.
# should be ok to put this in `makeFlags` array, but i can't get that to work!
# LUAROCKS_BINARY substitution is to support the cross-compilation case (i.e. emulate it during the build process)
makefileDefs = ''
CURL_LIB="${lib.getLib curl}/lib/libcurl.so" \
CURL_DIR="${lib.getDev curl}" \
@ -468,18 +459,7 @@ stdenv.mkDerivation rec {
popen-noshell-src-ko
tesseract-src-ko
turbo-src-ko
] ++ (lib.mapAttrsToList
(name: src: fetchgit (
{
inherit name;
} // src.source // {
# koreader sometimes specifies the rev as `tags/FOO`.
# we need to remember that to place the repo where it expects, but we have to strip it here for fetchgit to succeed.
rev = lib.removePrefix "tags/" src.source.rev;
}
))
sources.thirdparty
);
];
patches = [
# ./debug.patch #< not needed to build, just helps debug packaging issues
@ -519,8 +499,13 @@ stdenv.mkDerivation rec {
substituteInPlace base/Makefile.third \
--replace-fail ' -rm ' ' # -rm'
# make some sources writable (only the `sourceRoot` is writable by default)
# make some sources writable, particularly so koreader can apply its patches (by default only the `sourceRoot` is writable)
chmod -R u+w "$NIX_BUILD_TOP"/{fbink,kobo-usbms,leptonica,libk2pdfopt,lodepng,lunasvg,minizip,mupdf,popen-noshell,tesseract,turbo}
# koreader builds these deps itself: we mock out the download stage, and it does the rest
for dep in fbink kobo-usbms libk2pdfopt lodepng lunasvg minizip mupdf popen-noshell turbo; do
# sed -i 's/DOWNLOAD_COMMAND .*/DOWNLOAD_COMMAND ""/' "base/thirdparty/$dep/CMakeLists.txt"
sed -i "s:DOWNLOAD_COMMAND .*:DOWNLOAD_COMMAND rm -fd $dep $dep-build \\&\\& ln -s $NIX_BUILD_TOP/$dep $dep \\&\\& ln -s $NIX_BUILD_TOP/$dep $dep-build :" "base/thirdparty/$dep/CMakeLists.txt"
done
# lots of places in Makefile.third (incorrectly) assume lib paths are relative to CURDIR,
# so link /nix into CURDIR to allow them to work anyway
@ -529,79 +514,12 @@ stdenv.mkDerivation rec {
dontConfigure = true;
buildPhase = ''
link_lib_into_build_dir() {
lib="$1"
rev="$2"
lib_src="../$lib"
cmake_lists="base/thirdparty/$lib/CMakeLists.txt"
build_dir="base/thirdparty/$lib/build/${stdenv.hostPlatform.config}"
# link the nix clone into the directory koreader would use for checkout
# ref="base/thirdparty/$l/build/git_checkout"
# echo "linking thirdparty library $l $ref -> $deref"
# mkdir -p "$ref"
# ln -s "$deref" "$ref/$l"
# mv "$deref" "$ref/$l"
# cp -R "$deref" "$ref/$l"
# needs to be writable for koreader to checkout it specific revision
# chmod u+w -R "$ref/$l/.git"
# koreader wants to clone each library into this git_checkout dir,
# then checkout a specific revision,
# and then copy that checkout into the build/working directory further down.
# instead, we replicate that effect here, and by creating these "stamp" files
# koreader will know to skip the `git clone` and `git checkout` calls.
# the logic we're spoofing lives in koreader/base/thirdparty/cmake_modules/koreader_thirdparty_git.cmake
stamp_dir="$build_dir/git_checkout/stamp"
stamp_info="$stamp_dir/$lib-gitinfo-$rev.txt"
stamp_clone="$stamp_dir/$lib-gitclone-lastrun.txt"
echo "creating stamps for $lib: $stamp_clone > $stamp_info"
# mkdir $(dirname ..) to handle the case where `$rev` contains slashes
mkdir -p $(dirname "$stamp_info")
# koreader-base decides whether to redo the git checkout based on a timestamp compare of these two stamp files
touch -d "last week" $(dirname "$stamp_info") #< XXX: necessary?
touch -d "last week" "$stamp_info"
touch -d "next week" "$stamp_clone"
# koreader would copy the checkout into this build/working directory,
# but because we spoof the stamps to work around other git errors,
# copy it there on koreader's behalf
prefix="$build_dir/$lib-prefix"
mkdir -p "$prefix/src"
cp -R "$lib_src" "$prefix/src/$lib"
# src dir needs to be writable for koreader to apply its own patches
chmod u+w -R "$prefix/src/$lib"
}
skip_download() {
# sed -i 's/DOWNLOAD_COMMAND .*/DOWNLOAD_COMMAND ""/' "base/thirdparty/$1/CMakeLists.txt"
sed -i "s:DOWNLOAD_COMMAND .*:DOWNLOAD_COMMAND rm -fd $1 $1-build \\&\\& ln -s $NIX_BUILD_TOP/$1 $1 \\&\\& ln -s $NIX_BUILD_TOP/$1 $1-build :" "base/thirdparty/$1/CMakeLists.txt"
}
${builtins.concatStringsSep "\n" (
lib.mapAttrsToList
(name: src: lib.escapeShellArgs [ "link_lib_into_build_dir" name src.source.rev ])
sources.thirdparty
)}
skip_download fbink
skip_download kobo-usbms
skip_download libk2pdfopt
skip_download lodepng
skip_download lunasvg
skip_download minizip
skip_download mupdf
skip_download popen-noshell
skip_download turbo
# outDir should match OUTPUT_DIR in koreader-base
outDir="$NIX_BUILD_TOP/koreader/base/build/${stdenv.hostPlatform.config}"
mkdir -p "$outDir"
${symlinkThirdpartyBins "$outDir"}
make ${makeFlags}
'';
env = lib.optionalAttrs (stdenv.buildPlatform != stdenv.hostPlatform) {

View File

@ -1,315 +0,0 @@
# to update:
# - first, figure the rev for `koreader-base`:
# - inside `koreader` repo:
# - `git submodule status base`
# - or `git log base`
# - inside `koreader-base` repo:
# - `git diff old-rev..new-rev thirdparty`
# - update `source.rev` everywhere here that changed upstream
# - zero the hashes here and correct them based on build errors
# - tweak ./vendor-external-projects.patch until it applies
# - usually just upstream changed a URL or something minor
#
# a full rebuild takes approximately 10 minutes on a mid-range desktop
#
# the following build output may look like an error, but is safe to ignore:
# - "awk: fatal: cannot open file `3.9' for reading: No such file or directory"
# - this number comes from the luarocks version
#
# how to automate koreader updates?
# - it may be that koreader-base is more strongly decoupled from `koreader` than first appears:
# - most `koreader` commits which update base simply bump its rev and nothing more.
# - then, `koreader-base` could be its own package, updated independently from the main koreader.
{ lib
, symlinkJoin
, curl
, czmq
, djvulibre
, freetype
, fribidi
, giflib
, glib
, harfbuzz
, k2pdfopt
, leptonica
, libjpeg_turbo
, libpng
, libunibreak
, libwebp
, luajit
, minizip
, mupdf
, mupdf_1_17
, nanosvg
, openssh
, openssl_1_1
, sdcv
, tesseract
, turbo
, utf8proc
, zeromq4
, zstd
, zsync
}:
let
libAndDev = pkg: symlinkJoin {
inherit (pkg) name;
paths = [
(lib.getLib pkg)
(lib.getDev pkg)
];
};
in
{
thirdparty = {
# providing `package` is just a way to optimize builds, by getting KOReader to use the built nixpkg instead of building it itself from source.
# if it fails during an update, it should always be safe to delete the package key.
# curl = {
# source.url = "https://github.com/curl/curl.git";
# source.rev = "tags/curl-7_80_0";
# source.hash = "sha256-kzozc0Io+1f4UMivSV2IhzJDQXmad4wNhXN/Y2Lsg3Q=";
# package = curl;
# };
# czmq = {
# source.url = "https://github.com/zeromq/czmq.git";
# source.rev = "2a0ddbc4b2dde623220d7f4980ddd60e910cfa78";
# source.hash = "sha256-p4Cl2PLVgRQ0S4qr3VClJXjvAd2LUBU9oRUvOCfVnyw=";
# # package = czmq; # koreader wants v1, nixpkgs has v4
# };
# djvulibre = {
# source.url = "https://gitlab.com/koreader/djvulibre.git";
# source.rev = "6a1e5ba1c9ef81c205a4b270c3f121a1e106f4fc";
# source.hash = "sha256-OWSbxdr93FH3ed0D+NSFWIah7VDTcL3LIGOciY+f4dk=";
# # package = djvulibre; # "cp -fL /build/koreader/base/thirdparty/djvulibre/build/aarch64-unknown-linux-gnu/djvulibre-prefix/src/djvulibre/libdjvu/.libs/libdjvulibre.so.21 ..."
# };
# fbink = {
# source.url = "https://github.com/NiLuJe/FBInk.git";
# source.rev = "1a989b30a195ca240a3cf37f9de61b4b3c7e891c";
# source.hash = "sha256-O3bZzvuj/BRVV+UoutaaZZgGZws2J/i5ArfBHbz6omI=";
# # package: not packaged for nix
# };
# freetype2 = {
# source.url = "https://gitlab.com/koreader/freetype2.git";
# source.rev = "VER-2-13-2";
# source.hash = "sha256-yylSmVM3D5xnbFx9qEEHFIP/K0x/WDXZr0MA4C7ng7k=";
# package = libAndDev freetype;
# };
# fribidi = {
# source.url = "https://github.com/fribidi/fribidi.git";
# source.rev = "tags/v1.0.12";
# source.hash = "sha256-L4m/F9rs8fiv9rSf8oy7P6cthhupc6R/lCv30PLiQ4M=";
# package = libAndDev fribidi;
# };
# giflib = {
# source.url = "https://gitlab.com/koreader/giflib.git";
# source.rev = "5.1.4";
# source.hash = "sha256-znbY4tliXHXVLBd8sTKrbglOdCUb7xhcCQsDDWcQfhw=";
# package = giflib;
# };
# glib = {
# source.url = "https://github.com/GNOME/glib.git";
# source.rev = "2.58.3";
# source.hash = "sha256-KmJXCJ6h2QhPyK1axk+Y9+yJzO0wnCczcogopxGShJc=";
# # package = libAndDev glib; # breaks sdcv build
# };
# harfbuzz = {
# source.url = "https://github.com/harfbuzz/harfbuzz.git";
# source.rev = "8.3.0";
# source.hash = "sha256-sO0Kd2wAbMm+Auf7tXsDNal7hqND8iwkb0M/9WWt9sI=";
# # package = harfbuzz;
# package = libAndDev harfbuzz;
# };
# kobo-usbms = {
# source.url = "https://github.com/koreader/KoboUSBMS.git";
# source.rev = "v1.3.9";
# source.hash = "sha256-/yYpagekWlfTrXu/1DNTmBmdd3IkCDjRtslRv13mtCg=";
# # package: not in nixpkgs
# };
# leptonica = {
# source.url = "https://github.com/DanBloomberg/leptonica.git";
# source.rev = "1.74.1";
# source.hash = "sha256-SDXKam768xvZZvTbXe3sssvZyeLEEiY97Vrzx8hoc6g=";
# # k2pdf needs leptonica src, because it actually patches it and builds it itself
# # `cp -f $(LEPTONICA_MOD)/dewarp2.c $(LEPTONICA_DIR)/src/dewarp2.c`
# # i.e. cp -f /build/koreader/base/thirdparty/libk2pdfopt/build/aarch64-unknown-linux-gnu/libk2pdfopt-prefix/src/libk2pdfopt/leptonica_mod/dewarp2.c ...
# # k2pdf uses an old leptonica -- like 2015-2017-ish.
# # seems it can be at least partially updated, by replacing `numaGetMedianVariation` with `numaGetMedianDevFromMedian` (drop-in replacement)
# # and replacing references to `liblept.so` with `libleptonica.so`,
# # but eventually this requires patching the tesseract Makefiles. could get intense, idk.
# # package = leptonica;
# };
# libjpeg-turbo = {
# source.url = "https://github.com/libjpeg-turbo/libjpeg-turbo.git";
# source.rev = "3.0.1";
# source.hash = "sha256-ofdecix4m0FA9gdyQh7zYn99SYBbH2+a7jfoZlsadoA=";
# # package = libAndDev libjpeg_turbo;
# };
# libk2pdfopt = {
# source.url = "https://github.com/koreader/libk2pdfopt.git";
# source.rev = "09f1e011a618c8ec06b4caa67079682119d2aaa7";
# source.hash = "sha256-37sZ46dG6Z1Wk7NrhKAKl5j9r1bN6g01cd5Iyt/2coM=";
# # package = k2pdfopt; # nixpkgs k2pdfopt does not compile (broken deps). also, uses old insecure mupdf 1.17 (oh well, koreader is even older)
# };
# libpng = {
# source.url = "https://github.com/glennrp/libpng.git";
# source.rev = "v1.6.40";
# source.hash = "sha256-Rad7Y5Z9PUCipBTQcB7LEP8fIVTG3JsnMeknUkZ/rRg=";
# # package = libAndDev libpng; # "/build/koreader/base/thirdparty/libpng/build/aarch64-unknown-linux-gnu/libpng-prefix/src/libpng-build/.libs/libpng16.so.16"
# };
# libunibreak = {
# source.url = "https://github.com/adah1972/libunibreak.git";
# source.rev = "tags/libunibreak_5_1";
# source.hash = "sha256-hjgT5DCQ6KFXKlxk9LLzxGHz6B71X/3Ot7ipK3KY85A=";
# # package = libAndDev libunibreak; # nixpkgs version is incompatible (kpvcrlib/crengine #includes libunibreak and then fails, calling into undefined functions)
# };
# libwebp = {
# source.url = "https://github.com/webmproject/libwebp.git";
# source.rev = "v1.3.2";
# source.hash = "sha256-gfwUlJ44biO1lB/3SKfMkM/YBiYcz6RqeMOw+0o6Z/Q=";
# package = libAndDev libwebp;
# };
# libzmq = {
# source.url = "https://github.com/zeromq/libzmq";
# source.rev = "883e95b22e0bffffa72312ea1fec76199afbe458";
# source.hash = "sha256-R76EREtHsqcoKxKrgT8gfEf9pIWdLTBXvF9cDvjEf3E=";
# # package = zeromq4; # despite the name, it's libzmq.so.5 instead of libzmq.so.4
# };
# lj-wpaclient = {
# # only needed for select targets (e.g. kobo)
# source.url = "https://github.com/koreader/lj-wpaclient.git";
# source.rev = "2f93beb3071e6ebb57c783bd5b92f83aa5ebb757";
# source.hash = "sha256-ilJviGZTvL2i1TN5lHQ4eA9pFiM7NlXD+v9ofv520b8=";
# machineAgnostic = true;
# # package: not in nixpkgs; not even a non-luajit `wpaclient`
# };
# lodepng = {
# source.url = "https://github.com/lvandeve/lodepng.git";
# source.rev = "d398e0f10d152a5d17fa30463474dc9f56523f9c";
# source.hash = "sha256-ApOHUgU6X1rHwyjAHA/0Nt+buDFqY2ttXEnEvdrRl3A=";
# # package: not in nixpkgs, except in source-only form (mujoco.pin.lodepng)
# };
# lua-htmlparser = {
# source.url = "https://github.com/msva/lua-htmlparser";
# source.rev = "5ce9a775a345cf458c0388d7288e246bb1b82bff";
# source.hash = "sha256-aSTLSfqz/MIDFVRwtBlDNBUhPb7KqOl32/Y62Hdec1s=";
# # package: not in nixpkgs
# };
# luajit = {
# source.url = "https://github.com/LuaJIT/LuaJIT";
# source.rev = "29b0b282f59ac533313199f4f7be79490b7eee51";
# source.hash = "sha256-S57/NR+0hF1KTdn+cbVkJh3MTfklSwtZua1CYKduVlk=";
# # package = luajit; #< could be fixed; follows a different install structure
# };
# lua-rapidjson = {
# source.url = "https://github.com/xpol/lua-rapidjson";
# source.rev = "242b40c8eaceb0cc43bcab88309736461cac1234";
# source.hash = "sha256-y/czEVPtCt4uN1n49Qi7BrgZmkG+SDXlM5D2GvvO2qg=";
# # package: TODO: packaged in nix as a luarocks package
# };
# luasec = {
# source.url = "https://github.com/brunoos/luasec";
# source.rev = "tags/v1.3.1";
# source.hash = "sha256-3iYRNQoVk5HFjDSqRRmg1taSqeT2cHFil36vxjrEofo=";
# # package: TODO: packaged in nix as a luarocks package
# };
# luasocket = {
# source.url = "https://github.com/lunarmodules/luasocket";
# source.rev = "8c2ff7217e2a205eb107a6f48b04ff1b2b3090a1";
# source.hash = "sha256-Y35QYNLznQmErr6rIjxLzw0/6Y7y8TbzD4yaEdgEljA=";
# # package: TODO: packaged in nix as a luarocks package
# };
# lua-Spore = {
# # Complete... ish?
# # this originally failed like so:
# # Missing dependencies for lua-spore 0.3.1-1:
# # luajson >= 1.3 (not installed)
# # it passes now only because we patch out its build-time check for luajson (which we DO provide at runtime)
# source.url = "https://framagit.org/fperrad/lua-Spore";
# source.rev = "tags/0.3.3";
# source.hash = "sha256-wb7ykJsndoq0DazHpfXieUcBBptowYqD/eTTN/EK/6g=";
# # package: not in nixpkgs
# };
# lunasvg = {
# source.url = "https://github.com/sammycage/lunasvg.git";
# source.rev = "59d6f6ba835c1b7c7a0f9d4ea540ec3981777885";
# source.hash = "sha256-gW2ikakS6Omz5upmy26nAo/jkGHYO2kjlB3UmKJBh1k=";
# # package: not in nixpkgs
# };
# minizip = {
# source.url = "https://github.com/nmoinvaz/minizip";
# source.rev = "0b46a2b4ca317b80bc53594688883f7188ac4d08";
# source.hash = "sha256-P/3MMMGYDqD9NmkYvw/thKpUNa3wNOSlBBjANHSonAg=";
# # package = libAndDev minizip; # weird #include incompatibilities... maybe resolvable (looks like this is actually minizip-ng! but from 2015 and slightly patched; used as a lib for mupdf).
# };
# mupdf = {
# source.url = "https://github.com/ArtifexSoftware/mupdf.git";
# source.rev = "tags/1.13.0";
# source.hash = "sha256-pQejRon9fO9A1mhz3oLjBr1j4HveDLcQIWjR1/Rpy5Q=";
# # package = libAndDev mupdf; # nixpkgs' mupdf is incompatible with koreader's `libwrap-mupdf`
# # package = libAndDev mupdf_1_17; # does not compile
# };
# nanosvg = {
# source.url = "https://github.com/memononen/nanosvg.git";
# source.rev = "93ce879dc4c04a3ef1758428ec80083c38610b1f";
# source.hash = "sha256-ZtenaXJqMZr2+BxYENG1zUoQ+Qoxlxy0m/1YfJBKAFk=";
# machineAgnostic = true;
# package = nanosvg.src; # KOReader only wants the .h files, but decides to do that without even building it.
# };
# openssh = {
# source.url = "https://github.com/openssh/openssh-portable.git";
# source.rev = "V_8_6_P1";
# source.hash = "sha256-yjIpSbe5pt9sEV2MZYGztxejg/aBFfKO8ieRvoLN2KA=";
# package = openssh;
# };
# openssl = {
# source.url = "https://github.com/openssl/openssl.git";
# source.rev = "OpenSSL_1_1_1u";
# source.hash = "sha256-JOcUj4ovA6621+1k2HUsvhGX1B9BjvaMbCaSx680nSs=";
# # TODO: i think we can use nixpkgs openssl, just lift lib/* up to the root of the package directory
# # package = lib.getLib openssl_1_1; # N.B.: requires building with `NIXPKGS_ALLOW_INSECURE=1 nix build --impure ...`
# };
# popen-noshell = {
# source.url = "https://github.com/famzah/popen-noshell.git";
# source.rev = "e715396a4951ee91c40a98d2824a130f158268bb";
# source.hash = "sha256-JeBZMsg6ZUGSnyZ4eds4w63gM/L73EsAnLaHOPpL6iM=";
# # package: not in nixpkgs
# };
# sdcv = {
# # upstream is (temporarily?) acquiring this via `download_project` machinery
# source.url = "https://github.com/Dushistov/sdcv.git";
# source.rev = "v0.5.5";
# source.hash = "sha256-EyvljVXhOsdxIYOGTzD+T16nvW7/RNx3DuQ2OdhjXJ4=";
# package = sdcv;
# };
# tesseract = {
# source.url = "https://github.com/tesseract-ocr/tesseract.git";
# source.rev = "60176fc5ae5e7f6bdef60c926a4b5ea03de2bfa7";
# source.hash = "sha256-FQvlrJ+Uy7+wtUxBuS5NdoToUwNRhYw2ju8Ya8MLyQw=";
# # package = tesseract; # i guess koreader's k2pdf also builds tessearct??
# };
# turbo = {
# source.url = "https://github.com/kernelsauce/turbo";
# source.rev = "tags/v2.1.3";
# source.hash = "sha256-vBRkFdc5a0FIt15HBz3TnqMZ+GGsqjEefnfJEpuVTBs=";
# # package = turbo; # nixpkgs' turbo is a totally different thing
# };
# utf8proc = {
# source.url = "https://github.com/JuliaStrings/utf8proc.git";
# source.rev = "v2.9.0";
# source.hash = "sha256-Sgh8vTbclUV+lFZdR29PtNUy8F+9L/OAXk647B+l2mg=";
# # package = libAndDev utf8proc; # nixpkgs is v3, not v2; incompatible .so name. /build/koreader/base/thirdparty/utf8proc/build/aarch64-unknown-linux-gnu/utf8proc-prefix/src/utf8proc/libutf8proc.so.2
# };
# zstd = {
# source.url = "https://github.com/facebook/zstd.git";
# source.rev = "tags/v1.5.5";
# source.hash = "sha256-tHHHIsQU7vJySrVhJuMKUSq11MzkmC+Pcsj00uFJdnQ=";
# package = libAndDev zstd;
# };
# zsync2 = {
# source.url = "https://github.com/NiLuJe/zsync2.git";
# source.rev = "e618d18f6a7cbf350cededa17ddfe8f76bdf0b5c";
# source.hash = "sha256-S0vxCON1l6S+NWlnRPfm7R07DVkvkG+6QW5LNvXBlA8=";
# package = zsync; # possibly a different thing than koreader's
# };
};
}