diff --git a/pkgs/by-name/gf/gfal2-util/package.nix b/pkgs/by-name/gf/gfal2-util/package.nix new file mode 100644 index 000000000000..8af9899455af --- /dev/null +++ b/pkgs/by-name/gf/gfal2-util/package.nix @@ -0,0 +1,2 @@ +{ python3Packages }: +with python3Packages; toPythonApplication gfal2-util diff --git a/pkgs/development/python-modules/gfal2-util/default.nix b/pkgs/development/python-modules/gfal2-util/default.nix new file mode 100644 index 000000000000..2aabf3b43e88 --- /dev/null +++ b/pkgs/development/python-modules/gfal2-util/default.nix @@ -0,0 +1,103 @@ +{ lib +, buildPythonPackage +, callPackage +, fetchFromGitHub +, runCommandLocal + # Build inputs +, gfal2-python + # For tests +, xrootd # pkgs.xrootd +}: +(buildPythonPackage rec { + pname = "gfal2-util"; + version = "1.8.1"; + src = fetchFromGitHub { + owner = "cern-fts"; + repo = "gfal2-util"; + rev = "v${version}"; + hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk="; + }; + + # Replace the ad-hoc python executable finding + # and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python` + # for fixup phase to work correctly. + postPatch = '' + for script in src/gfal-*; do + patch "$script" ${./gfal-util-script.patch} + done + ''; + + propagatedBuildInputs = [ + gfal2-python + ]; + + pythonImportsCheck = [ + "gfal2_util" + ]; + + meta = with lib; { + description = "CLI for gfal2"; + homepage = "https://github.com/cern-fts/gfal2-utils"; + license = licenses.asl20; + maintainers = with maintainers; [ ShamrockLee ]; + }; +}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs { + passthru = { + inherit (gfal2-python) gfal2; + + fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; }); + + # With these functionality tests, it should be safe to merge version bumps once all the tests are passed. + tests = + let + # Use the the bin output hash of gfal2-util as version to ensure that + # the test gets rebuild everytime gfal2-util gets rebuild + versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}"; + self = finalAttrs.finalPackage; + in + lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd ( + let + # Test against a real-world dataset from CERN Open Data + # borrowed from `xrootd.tests`. + urlTestFile = xrootd.tests.test-xrdcp.url; + hashTestFile = xrootd.tests.test-xrdcp.outputHash; + urlTestDir = dirOf urlTestFile; + in + { + test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 { + url = urlTestFile; + hash = hashTestFile; + extraGfalCopyFlags = [ "--verbose" ]; + pname = "gfal2-util-test-copy-file-xrootd"; + version = versionFODTests; + allowSubstitutes = false; + }; + + test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 { + url = urlTestDir; + hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw="; + recursive = true; + extraGfalCopyFlags = [ "--verbose" ]; + pname = "gfal2-util-test-copy-dir-xrootd"; + version = versionFODTests; + allowSubstitutes = false; + }; + + test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } '' + set -eu -o pipefail + gfal-ls "$url" | grep "$baseNameExpected" | tee "$out" + '').overrideAttrs (finalAttrs: previousAttrs: { + pname = previousAttrs.name; + version = versionFODTests; + name = "${finalAttrs.pname}-${finalAttrs.version}"; + nativeBuildInputs = [ self ]; + url = urlTestDir; + baseNameExpected = baseNameOf urlTestFile; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n"); + }); + } + ); + }; +}) diff --git a/pkgs/development/python-modules/gfal2-util/fetchgfal2.nix b/pkgs/development/python-modules/gfal2-util/fetchgfal2.nix new file mode 100644 index 000000000000..ab5ae45ef160 --- /dev/null +++ b/pkgs/development/python-modules/gfal2-util/fetchgfal2.nix @@ -0,0 +1,48 @@ +{ lib +, runCommandLocal +, gfal2-util +}: + +# `url` and `urls` should only be overriden via `.override`, but not `.overrideAttrs`. +{ name ? "" +, pname ? "" +, version ? "" +, urls ? [ ] +, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls +, hash ? lib.fakeHash +, recursive ? false +, intermediateDestUrls ? [ ] +, extraGfalCopyFlags ? [ ] +, allowSubstitutes ? true +}: + +(runCommandLocal name { } '' + for u in "''${urls[@]}"; do + gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out" + ret="$?" + (( ret )) && break + done + if (( ret )); then + echo "gfal-copy failed trying to download from any of the urls" >&2 + exit "$ret" + fi +'').overrideAttrs (finalAttrs: previousAttrs: +{ + __structuredAttrs = true; + inherit allowSubstitutes; + nativeBuildInputs = [ gfal2-util ]; + outputHashAlgo = null; + outputHashMode = if finalAttrs.recursive then "recursive" else "flat"; + outputHash = hash; + inherit url; + urls = if urls == [ ] then lib.singleton url else urls; + gfalCopyFlags = extraGfalCopyFlags + ++ lib.optional finalAttrs.recursive "--recursive" + ; + inherit recursive intermediateDestUrls; +} // (if (pname != "" && version != "") then { + inherit pname version; + name = "${finalAttrs.pname}-${finalAttrs.version}"; +} else { + name = if (name != "") then name else (baseNameOf url); +})) diff --git a/pkgs/development/python-modules/gfal2-util/gfal-util-script.patch b/pkgs/development/python-modules/gfal2-util/gfal-util-script.patch new file mode 100644 index 000000000000..89cd82d850e1 --- /dev/null +++ b/pkgs/development/python-modules/gfal2-util/gfal-util-script.patch @@ -0,0 +1,19 @@ +--- a/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000 ++++ b/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000 +@@ -1,4 +1,4 @@ +-#!/bin/sh ++#!/usr/bin/env python3 + # -*- coding: utf-8 -*- + # + # Copyright (c) 2022 CERN +@@ -15,10 +15,2 @@ + # See the License for the specific language governing permissions and + # limitations under the License. +- +-# Execute script content with first python interpreter found: +-# * GFAL_PYTHONBIN environment variable +-# * python on the PATH if import gfal2, gfal2_util succeeds +-# * python3 on the PATH if import gfal2, gfal2_util succeeds +-# * python2 on the PATH if import gfal2, gfal2_util succeeds +-# * /usr/bin/python +-"exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index dfa74365675b..ad8c81faf0a9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1834,6 +1834,12 @@ with pkgs; github-copilot-cli = callPackage ../tools/misc/github-copilot-cli { }; + # This is to workaround gfal2-python broken against Python 3.12 or later. + # TODO: Remove these lines after solving the breakage. + gfal2-util = callPackage ../by-name/gf/gfal2-util/package.nix (lib.optionalAttrs python3Packages.gfal2-python.meta.broken { + python3Packages = python311Packages; + }); + gfshare = callPackage ../tools/security/gfshare { }; gh-actions-cache = callPackage ../tools/misc/gh-actions-cache { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 104a8f5ef2b5..49433cce66de 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4687,6 +4687,10 @@ self: super: with self; { gfal2-python = callPackage ../development/python-modules/gfal2-python { }; + gfal2-util = callPackage ../development/python-modules/gfal2-util { + inherit (pkgs) xrootd; + }; + gflags = callPackage ../development/python-modules/gflags { }; gflanguages = callPackage ../development/python-modules/gflanguages { };