duckdb,python311Packages.duckdb: 0.9.2 -> 0.10.0

pkgs/development/libraries/duckdb/default.nix

@@ -1,10 +1,13 @@
 { lib
 , stdenv
 , fetchFromGitHub
+, fetchpatch
+, substituteAll
 , cmake
 , ninja
 , openssl
 , openjdk11
+, python3
 , unixODBC
 , withJdbc ? false
 , withOdbc ? false
@@ -15,22 +18,31 @@ let
 in
 stdenv.mkDerivation rec {
   pname = "duckdb";
-  version = "0.9.2";
+  version = "0.10.0";
 
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
-    rev = "v${version}";
-    hash = "sha256-QFK8mEMcqQwALFNvAdD8yWixwMYHSbeo6xqx86PvejU=";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-qGUq0iYTaLNHKqbXNLRmvqHMqunvIlP991IKb4qdSt4=";
   };
 
-  patches = [ ./version.patch ];
+  patches = [
+    # remove calls to git and set DUCKDB_VERSION to version
+    (substituteAll {
+      src = ./version.patch;
+      version = "v${version}";
+    })
+    # add missing file needed for httpfs compile
+    # remove on next update
+    (fetchpatch {
+      name = "missing-httpfs-file.patch";
+      url = "https://github.com/duckdb/duckdb/commit/3d7aa3ed46ecf5f18122559e385b75f1f5e9aba8.patch";
+      hash = "sha256-Q4IHCpMpxn86OquUZdEF7P0nHEPOcWS0TQijTkvBYbQ=";
+    })
+  ];
 
-  postPatch = ''
-    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
-  '';
-
-  nativeBuildInputs = [ cmake ninja ];
+  nativeBuildInputs = [ cmake ninja python3 ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
     ++ lib.optionals withOdbc [ unixODBC ];
@@ -54,7 +66,7 @@ stdenv.mkDerivation rec {
 
   installCheckPhase =
     let
-      excludes = map (pattern: "exclude:'${pattern}'") [
+      excludes = map (pattern: "exclude:'${pattern}'") ([
         "[s3]"
         "Test closing database during long running query"
         "Test using a remote optimizer pass in case thats important to someone"
@@ -91,16 +103,22 @@ stdenv.mkDerivation rec {
         "[!hide]"
         # this test apparently never terminates
         "test/sql/copy/csv/auto/test_csv_auto.test"
+        # test expects installed file timestamp to be > 2024
+        "test/sql/table_function/read_text_and_blob.test"
+        # can re-enable next update (broken for 0.10.0)
+        "test/sql/secrets/create_secret_non_writable_persistent_dir.test"
+        # https://github.com/duckdb/duckdb/issues/10722
+        "test/sql/types/nested/list/list_aggregate_dict.test"
       ] ++ lib.optionals stdenv.isAarch64 [
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"
         "test/sql/function/list/aggregates/skewness.test"
-      ];
+      ]);
     in
     ''
       runHook preInstallCheck
 
-      $PWD/test/unittest ${lib.concatStringsSep " " excludes}
+      ./test/unittest ${toString excludes}
 
       runHook postInstallCheck
     '';

pkgs/development/libraries/duckdb/version.patch

@@ -52,7 +52,7 @@ index 2b49e11288..0a4a69b9a0 100644
 -  set(DUCKDB_DEV_ITERATION 0)
 -  set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")
 -endif()
-+set(DUCKDB_VERSION "@DUCKDB_VERSION@")
++set(DUCKDB_VERSION "@version@")
  
  message(STATUS "git hash ${GIT_COMMIT_HASH}, version ${DUCKDB_VERSION}")
  

pkgs/development/python-modules/duckdb/default.nix

@@ -1,4 +1,5 @@
 { lib
+, stdenv
 , buildPythonPackage
 , duckdb
 , fsspec
@@ -13,25 +14,24 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) pname version src;
-  format = "setuptools";
-
-  # 1. let nix control build cores
-  # 2. default to extension autoload & autoinstall disabled
-  # 3. unconstrain setuptools_scm version
-  patches = (duckdb.patches or []) ++ [ ./setup.patch ];
+  inherit (duckdb) patches pname src version;
+  pyproject = true;
 
   postPatch = (duckdb.postPatch or "") + ''
     # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library
     cd tools/pythonpkg
 
-    substituteInPlace setup.py --subst-var NIX_BUILD_CORES
-
-    # avoid dependency on mypy
-    rm tests/stubs/test_stubs.py
+    # 1. let nix control build cores
+    # 2. default to extension autoload & autoinstall disabled
+    substituteInPlace setup.py \
+      --replace-fail "ParallelCompile()" 'ParallelCompile("NIX_BUILD_CORES")' \
+      --replace-fail "define_macros.extend([('DUCKDB_EXTENSION_AUTOLOAD_DEFAULT', '1'), ('DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT', '1')])" ""
   '';
 
-  BUILD_HTTPFS = 1;
+  env = {
+    BUILD_HTTPFS = 1;
+    DUCKDB_BUILD_UNITY = 1;
+  };
 
   nativeBuildInputs = [
     pybind11
@@ -52,21 +52,35 @@ buildPythonPackage rec {
     pytestCheckHook
   ];
 
+  # test flags from .github/workflows/Python.yml
+  pytestFlagsArray = [
+    "--verbose"
+  ] ++ lib.optionals stdenv.isDarwin [
+    "tests/fast"
+  ];
+
+  disabledTestPaths = [
+    # avoid dependency on mypy
+    "tests/stubs/test_stubs.py"
+  ];
+
   disabledTests = [
     # tries to make http request
     "test_install_non_existent_extension"
-    # test is racy and interrupt can be delivered before or after target point
+
+    # https://github.com/duckdb/duckdb/issues/10702
+    # tests are racy and interrupt can be delivered before or after target point
+    # causing a later test to fail with a spurious KeyboardInterrupt
     "test_connection_interrupt"
+    "test_query_interruption"
   ];
 
+  # remove duckdb dir to prevent import confusion by pytest
   preCheck = ''
     export HOME="$(mktemp -d)"
+    rm -rf duckdb
   '';
 
-  setupPyBuildFlags = [
-    "--inplace"
-  ];
-
   pythonImportsCheck = [
     "duckdb"
   ];

pkgs/development/python-modules/duckdb/setup.patch

@@ -1,30 +0,0 @@
-diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
-index 30f1e1ccdd..6784169fcb 100644
---- a/tools/pythonpkg/setup.py
-+++ b/tools/pythonpkg/setup.py
-@@ -96,7 +96,7 @@ def parallel_cpp_compile(
-             return
-         self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
- 
--    list(multiprocessing.pool.ThreadPool(multiprocessing.cpu_count()).imap(_single_compile, objects))
-+    list(multiprocessing.pool.ThreadPool(@NIX_BUILD_CORES@).imap(_single_compile, objects))
-     return objects
- 
- 
-@@ -163,7 +163,6 @@ if 'BUILD_HTTPFS' in os.environ:
- for ext in extensions:
-     toolchain_args.extend(['-DDUCKDB_EXTENSION_{}_LINKED'.format(ext.upper())])
- 
--toolchain_args.extend(['-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1', '-DDUCKDB_EXTENSION_AUTOINSTALL_DEFAULT=1'])
- 
- 
- class get_pybind_include(object):
-@@ -348,7 +347,7 @@ setup(
-     packages=packages,
-     include_package_data=True,
-     python_requires='>=3.7.0',
--    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
-+    setup_requires=setup_requires + ["setuptools_scm", 'pybind11>=2.6.0'],
-     use_scm_version=setuptools_scm_conf,
-     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
-     classifiers=[