Merge pull request #297147 from a-n-n-a-l-e-e/duckdb-update

duckdb,python311Packages.duckdb: 0.10.0 -> 0.10.1, add update script

pkgs/development/libraries/duckdb/default.nix

@@ -15,32 +15,22 @@
 
 let
   enableFeature = yes: if yes then "ON" else "OFF";
+  versions = lib.importJSON ./versions.json;
 in
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "duckdb";
-  version = "0.10.0";
+  inherit (versions) rev version;
 
   src = fetchFromGitHub {
-    owner = pname;
-    repo = pname;
-    rev = "refs/tags/v${version}";
-    hash = "sha256-qGUq0iYTaLNHKqbXNLRmvqHMqunvIlP991IKb4qdSt4=";
+    # to update run:
+    # nix-shell maintainers/scripts/update.nix --argstr path duckdb
+    inherit (versions) hash;
+    owner = "duckdb";
+    repo = "duckdb";
+    rev = "refs/tags/v${finalAttrs.version}";
   };
 
-  patches = [
-    # remove calls to git and set DUCKDB_VERSION to version
-    (substituteAll {
-      src = ./version.patch;
-      version = "v${version}";
-    })
-    # add missing file needed for httpfs compile
-    # remove on next update
-    (fetchpatch {
-      name = "missing-httpfs-file.patch";
-      url = "https://github.com/duckdb/duckdb/commit/3d7aa3ed46ecf5f18122559e385b75f1f5e9aba8.patch";
-      hash = "sha256-Q4IHCpMpxn86OquUZdEF7P0nHEPOcWS0TQijTkvBYbQ=";
-    })
-  ];
+  outputs = [ "out" "lib" "dev" ];
 
   nativeBuildInputs = [ cmake ninja python3 ];
   buildInputs = [ openssl ]
@@ -48,22 +38,22 @@ stdenv.mkDerivation rec {
     ++ lib.optionals withOdbc [ unixODBC ];
 
   cmakeFlags = [
-    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/in_tree_extensions.cmake"
+    "-DDUCKDB_EXTENSION_CONFIGS=${finalAttrs.src}/.github/config/in_tree_extensions.cmake"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
-  ] ++ lib.optionals doInstallCheck [
+    "-DOVERRIDE_GIT_DESCRIBE=v${finalAttrs.version}-0-g${finalAttrs.rev}"
+  ] ++ lib.optionals finalAttrs.doInstallCheck [
     # development settings
     "-DBUILD_UNITTESTS=ON"
   ];
 
-  doInstallCheck = true;
-
-  preInstallCheck = ''
-    export HOME="$(mktemp -d)"
-  '' + lib.optionalString stdenv.isDarwin ''
-    export DYLD_LIBRARY_PATH="$out/lib''${DYLD_LIBRARY_PATH:+:}''${DYLD_LIBRARY_PATH}"
+  postInstall = ''
+    mkdir -p $lib
+    mv $out/lib $lib
   '';
 
+  doInstallCheck = true;
+
   installCheckPhase =
     let
       excludes = map (pattern: "exclude:'${pattern}'") ([
@@ -105,26 +95,29 @@ stdenv.mkDerivation rec {
         "test/sql/copy/csv/auto/test_csv_auto.test"
         # test expects installed file timestamp to be > 2024
         "test/sql/table_function/read_text_and_blob.test"
-        # can re-enable next update (broken for 0.10.0)
-        "test/sql/secrets/create_secret_non_writable_persistent_dir.test"
-        # https://github.com/duckdb/duckdb/issues/10722
-        "test/sql/types/nested/list/list_aggregate_dict.test"
+        # fails with Out of Memory Error
+        "test/sql/copy/parquet/batched_write/batch_memory_usage.test"
+        # wants http connection
+        "test/sql/copy/csv/test_mixed_lines.test"
       ] ++ lib.optionals stdenv.isAarch64 [
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"
         "test/sql/function/list/aggregates/skewness.test"
       ]);
+      LD_LIBRARY_PATH = lib.optionalString stdenv.isDarwin "DY" + "LD_LIBRARY_PATH";
     in
     ''
       runHook preInstallCheck
 
-      ./test/unittest ${toString excludes}
+      HOME="$(mktemp -d)" ${LD_LIBRARY_PATH}="$lib/lib" ./test/unittest ${toString excludes}
 
       runHook postInstallCheck
     '';
 
+  passthru.updateScript = ./update.sh;
+
   meta = with lib; {
-    changelog = "https://github.com/duckdb/duckdb/releases/tag/v${version}";
+    changelog = "https://github.com/duckdb/duckdb/releases/tag/v${finalAttrs.version}";
     description = "Embeddable SQL OLAP Database Management System";
     homepage = "https://duckdb.org/";
     license = licenses.mit;
@@ -132,4 +125,4 @@ stdenv.mkDerivation rec {
     maintainers = with maintainers; [ costrouc cpcloud ];
     platforms = platforms.all;
   };
-}
+})

pkgs/development/libraries/duckdb/update.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --pure -i bash -p cacert curl jq moreutils nix-prefetch
+# shellcheck shell=bash
+
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+nixpkgs=$(while [[ ! -e .git ]]; do [[ ${PWD} != / ]] || exit 1; cd ..; done; echo "${PWD}")
+
+repo=duckdb
+owner=duckdb
+
+msg() {
+    echo "$*" >&2
+}
+
+json_get() {
+    jq -r "$1" < 'versions.json'
+}
+
+json_set() {
+    jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json'
+}
+
+get_latest() {
+    curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s \
+        "https://api.github.com/repos/${owner}/${repo}/releases/latest" | jq -r .tag_name
+}
+
+get_sha() {
+    curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s \
+        "https://api.github.com/repos/${owner}/${repo}/git/ref/tags/$1" | jq -r .object.sha
+}
+
+tag=$(get_latest)
+version=${tag/v/}
+
+[[ ${version} = $(json_get .version) ]] && { msg "${version} is up to date"; exit 0; }
+
+sha=$(get_sha "${tag}")
+sri=$(nix-prefetch -I nixpkgs="${nixpkgs}" -E "duckdb.overrideAttrs { version = \"${version}\"; }")
+
+json_set ".version" "${version}"
+json_set ".rev" "${sha}"
+json_set ".hash" "${sri}"

pkgs/development/libraries/duckdb/version.patch

@@ -1,58 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 2b49e11288..0a4a69b9a0 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -244,52 +244,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
-   set(SUN TRUE)
- endif()
- 
--find_package(Git)
--if(Git_FOUND)
--  if (NOT DEFINED GIT_COMMIT_HASH)
--    execute_process(
--            COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
--            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--            RESULT_VARIABLE GIT_RESULT
--            OUTPUT_VARIABLE GIT_COMMIT_HASH
--            OUTPUT_STRIP_TRAILING_WHITESPACE)
--  endif()
--  execute_process(
--          COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
--          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--          OUTPUT_VARIABLE GIT_LAST_TAG
--          OUTPUT_STRIP_TRAILING_WHITESPACE)
--  execute_process(
--          COMMAND ${GIT_EXECUTABLE} describe --tags --long
--          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
--          OUTPUT_VARIABLE GIT_ITERATION
--          OUTPUT_STRIP_TRAILING_WHITESPACE)
--else()
--  message("Git NOT FOUND")
--endif()
--
--if(GIT_RESULT EQUAL "0")
--  string(REGEX REPLACE "v([0-9]+).[0-9]+.[0-9]+" "\\1" DUCKDB_MAJOR_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE "v[0-9]+.([0-9]+).[0-9]+" "\\1" DUCKDB_MINOR_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE "v[0-9]+.[0-9]+.([0-9]+)" "\\1" DUCKDB_PATCH_VERSION "${GIT_LAST_TAG}")
--  string(REGEX REPLACE ".*-([0-9]+)-.*" "\\1" DUCKDB_DEV_ITERATION "${GIT_ITERATION}")
--
--  if(DUCKDB_DEV_ITERATION EQUAL 0)
--    # on a tag; directly use the version
--    set(DUCKDB_VERSION "${GIT_LAST_TAG}")
--  else()
--    # not on a tag, increment the patch version by one and add a -devX suffix
--    math(EXPR DUCKDB_PATCH_VERSION "${DUCKDB_PATCH_VERSION}+1")
--    set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
--  endif()
--else()
--  # fallback for when building from tarball
--  set(DUCKDB_MAJOR_VERSION 0)
--  set(DUCKDB_MINOR_VERSION 0)
--  set(DUCKDB_PATCH_VERSION 1)
--  set(DUCKDB_DEV_ITERATION 0)
--  set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
--endif()
-+set(DUCKDB_VERSION "@version@")
- 
- message(STATUS "git hash ${GIT_COMMIT_HASH}, version ${DUCKDB_VERSION}")
- 

pkgs/development/libraries/duckdb/versions.json

@@ -0,0 +1,5 @@
+{
+  "version": "0.10.1",
+  "rev": "4a89d97db8a5a23a15f3025c8d2d2885337c2637",
+  "hash": "sha256-/j/DaUzsfACI5Izr4lblkYmIEmKsOXr760UTwC0l/qg="
+}

pkgs/development/python-modules/duckdb/default.nix

@@ -14,7 +14,7 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) patches pname src version;
+  inherit (duckdb) patches pname rev src version;
   pyproject = true;
 
   postPatch = (duckdb.postPatch or "") + ''
@@ -31,8 +31,11 @@ buildPythonPackage rec {
   env = {
     BUILD_HTTPFS = 1;
     DUCKDB_BUILD_UNITY = 1;
+    OVERRIDE_GIT_DESCRIBE="v${version}-0-g${rev}";
   };
 
+  dontPretendSetuptoolsSCMVersion = true;
+
   nativeBuildInputs = [
     pybind11
     setuptools-scm