Merge pull request #303043 from GaetanLepage/coffea

python311Packages.coffea: 2024.2.2 -> 2024.4.1

pkgs/development/python-modules/awkward/default.nix

@@ -75,6 +75,9 @@ buildPythonPackage rec {
   # The following tests have been disabled because they need to be run on a GPU platform.
   disabledTestPaths = [
     "tests-cuda"
+  # Disable tests dependending on jax on darwin
+  ] ++ lib.optionals stdenv.isDarwin [
+    "tests/test_2603_custom_behaviors_with_jax.py"
   ];
 
   meta = with lib; {

pkgs/development/python-modules/coffea/default.nix

@@ -32,7 +32,7 @@
 
 buildPythonPackage rec {
   pname = "coffea";
-  version = "2024.2.2";
+  version = "2024.4.1";
   pyproject = true;
 
   disabled = pythonOlder "3.8";
@@ -41,7 +41,7 @@ buildPythonPackage rec {
     owner = "CoffeaTeam";
     repo = "coffea";
     rev = "refs/tags/v${version}";
-    hash = "sha256-GdoVb9YtlUlrSx7TWWrdHOqOJJ4M+kJspOllv6HgFXk=";
+    hash = "sha256-Iu1GHnLUqdhYO7hoHaf+O/S6KO0P+dvl0wgfRA5vtGI=";
   };
 
   postPatch = ''

pkgs/development/python-modules/dask-expr/default.nix

@@ -0,0 +1,60 @@
+{
+  lib,
+  buildPythonPackage,
+  pythonOlder,
+  fetchFromGitHub,
+  setuptools,
+  versioneer,
+  wheel,
+  dask,
+  pandas,
+  pyarrow,
+  distributed,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "dask-expr";
+  version = "1.0.12";
+  pyproject = true;
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchFromGitHub {
+    owner = "dask";
+    repo = "dask-expr";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-B/BkLOZhvUyjinaFKp0ecUfzvLb5S90q+YHmJwS6WSQ=";
+  };
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace-fail "versioneer[toml]==0.28" "versioneer[toml]"
+  '';
+
+  nativeBuildInputs = [
+    setuptools
+    versioneer
+    wheel
+  ];
+
+  propagatedBuildInputs = [
+    dask
+    pandas
+    pyarrow
+  ];
+
+  pythonImportsCheck = [ "dask_expr" ];
+
+  nativeCheckInputs = [
+    distributed
+    pytestCheckHook
+  ];
+
+  meta = with lib; {
+    description = "";
+    homepage = "https://github.com/dask/dask-expr";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ GaetanLepage ];
+  };
+}

pkgs/development/python-modules/dask/default.nix

@@ -2,7 +2,6 @@
 , stdenv
 , buildPythonPackage
 , fetchFromGitHub
-, fetchpatch
 
 # build-system
 , setuptools
@@ -29,6 +28,7 @@
 
 # tests
 , arrow-cpp
+, dask-expr
 , hypothesis
 , pytest-asyncio
 , pytest-rerunfailures
@@ -37,9 +37,9 @@
 , pythonOlder
 }:
 
-buildPythonPackage rec {
+let self = buildPythonPackage rec {
   pname = "dask";
-  version = "2024.2.1";
+  version = "2024.4.2";
   pyproject = true;
 
   disabled = pythonOlder "3.9";
@@ -48,22 +48,9 @@ buildPythonPackage rec {
     owner = "dask";
     repo = "dask";
     rev = "refs/tags/${version}";
-    hash = "sha256-8VFtKPaF0PqCjqFB+plFe1GjUno5j7j86+wxKhzByyw=";
+    hash = "sha256-iD+diwctXaQlOpL0fjOiFoWVONtlMq7AonbC0vCmXc0=";
   };
 
-  patches = [
-    # A pair of fixes with python 3.11.9, merged upstream;
-    # see https://github.com/dask/dask/issues/11038
-    (fetchpatch {
-      url = "https://github.com/dask/dask/pull/11035.diff";
-      hash = "sha256-aQTzas8gn7pCyp7L6VV3NpSYgqC1Ov7YN7YGnX0Vwmo=";
-    })
-    (fetchpatch {
-      url = "https://github.com/dask/dask/pull/11039.diff";
-      hash = "sha256-gvEEvnyhFlhiFvVaB6jwMy4auUOvECf49FbFJyjqQm4=";
-    })
-  ];
-
   nativeBuildInputs = [
     setuptools
     wheel
@@ -93,6 +80,7 @@ buildPythonPackage rec {
     ++ self.distributed
     ++ self.diagnostics;
     dataframe = [
+      # dask-expr -> circular dependency with dask-expr
       numpy
       pandas
     ];
@@ -106,6 +94,7 @@ buildPythonPackage rec {
   });
 
   nativeCheckInputs = [
+    dask-expr
     pytestCheckHook
     pytest-rerunfailures
     pytest-xdist
@@ -113,6 +102,7 @@ buildPythonPackage rec {
     hypothesis
     pytest-asyncio
   ]
+  ++ passthru.optional-dependencies.array
   ++ passthru.optional-dependencies.dataframe
   ++ lib.optionals (!arrow-cpp.meta.broken) [ # support is sparse on aarch64
     pyarrow
@@ -172,15 +162,29 @@ buildPythonPackage rec {
 
   pythonImportsCheck = [
     "dask"
-    "dask.array"
     "dask.bag"
     "dask.bytes"
-    "dask.dataframe"
-    "dask.dataframe.io"
-    "dask.dataframe.tseries"
     "dask.diagnostics"
   ];
 
+  doCheck = false;
+
+  # Enable tests via passthru to avoid cyclic dependency with dask-expr.
+  passthru.tests = {
+    check = self.overridePythonAttrs (old: {
+      doCheck = true;
+      pythonImportsCheck = [
+        # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs`
+        "dask.array"
+        # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs`
+        "dask.dataframe"
+        "dask.dataframe.io"
+        "dask.dataframe.tseries"
+      ] ++ old.pythonImportsCheck;
+    });
+  };
+
+
   meta = with lib; {
     description = "Minimal task scheduling abstraction";
     mainProgram = "dask";
@@ -189,4 +193,4 @@ buildPythonPackage rec {
     license = licenses.bsd3;
     maintainers = with maintainers; [ fridh ];
   };
-}
+}; in self

pkgs/development/python-modules/distributed/default.nix

@@ -25,7 +25,7 @@
 
 buildPythonPackage rec {
   pname = "distributed";
-  version = "2023.12.0";
+  version = "2024.4.2";
   pyproject = true;
 
   disabled = pythonOlder "3.9";
@@ -34,7 +34,7 @@ buildPythonPackage rec {
     owner = "dask";
     repo = "distributed";
     rev = "refs/tags/${version}";
-    hash = "sha256-Zv31BTzY31eXkU7wqa+h33qGrH+OTzKEj6L7Ei/aizk=";
+    hash = "sha256-xoQ+b7qzstZl9gRNs4jssNOsGQHDdvTXU7pTjBSuyWs=";
   };
 
   postPatch = ''

pkgs/top-level/python-packages.nix

@@ -2700,6 +2700,8 @@ self: super: with self; {
 
   dask-awkward = callPackage ../development/python-modules/dask-awkward { };
 
+  dask-expr = callPackage ../development/python-modules/dask-expr { };
+
   dask-gateway = callPackage ../development/python-modules/dask-gateway { };
 
   dask-gateway-server = callPackage ../development/python-modules/dask-gateway-server { };