From b3b12c2d08e2f5aab1c6f6ae0021ea11a6825b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sun, 7 Apr 2024 18:54:44 +0200 Subject: [PATCH] Revert "Fix venv creation in Python environments" This commit reverts all python changes from 234bb31f611f43f8b744b305ab82035de637aaca. --- .../python/cpython/2.7/default.nix | 2 + .../interpreters/python/cpython/default.nix | 3 +- .../interpreters/python/pypy/default.nix | 3 + .../interpreters/python/pypy/prebuilt.nix | 3 + .../interpreters/python/pypy/prebuilt_2_7.nix | 3 + .../interpreters/python/sitecustomize.py | 39 ++++++++ .../development/interpreters/python/tests.nix | 91 ++++--------------- .../tests/test_environments/test_python.py | 2 +- 8 files changed, 71 insertions(+), 75 deletions(-) create mode 100644 pkgs/development/interpreters/python/sitecustomize.py diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index 86b09fa87768..dda254fca389 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -318,6 +318,8 @@ in with passthru; stdenv.mkDerivation ({ inherit passthru; postFixup = '' + # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7. + cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py '' + lib.optionalString strip2to3 '' rm -R $out/bin/2to3 $out/lib/python*/lib2to3 '' + lib.optionalString stripConfig '' diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index 96dcb6c25a13..301af7a29c9e 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -537,7 +537,8 @@ in with passthru; stdenv.mkDerivation (finalAttrs: { # Strip tests rm -R $out/lib/python*/test $out/lib/python*/**/test{,s} '' + optionalString includeSiteCustomize '' - + # Include a sitecustomize.py file + cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py '' + optionalString stripBytecode '' # Determinism: deterministic bytecode # First we delete all old bytecode. diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 5724f4944d9c..9b414944bba5 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -126,6 +126,9 @@ in with passthru; stdenv.mkDerivation rec { ln -s $out/${executable}-c/include $out/include/${libPrefix} ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix} + # Include a sitecustomize.py file + cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py + runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/pypy/prebuilt.nix b/pkgs/development/interpreters/python/pypy/prebuilt.nix index 70f8711ef086..4b47c642eca4 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt.nix @@ -95,6 +95,9 @@ in with passthru; stdenv.mkDerivation { echo "Removing bytecode" find . -name "__pycache__" -type d -depth -delete + # Include a sitecustomize.py file + cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py + runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix index f0b60c2333f5..37a06f9f61ed 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix @@ -96,6 +96,9 @@ in with passthru; stdenv.mkDerivation { echo "Removing bytecode" find . -name "__pycache__" -type d -depth -delete + # Include a sitecustomize.py file + cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py + runHook postInstall ''; diff --git a/pkgs/development/interpreters/python/sitecustomize.py b/pkgs/development/interpreters/python/sitecustomize.py new file mode 100644 index 000000000000..c6924a8e93f0 --- /dev/null +++ b/pkgs/development/interpreters/python/sitecustomize.py @@ -0,0 +1,39 @@ +""" +This is a Nix-specific module for discovering modules built with Nix. + +The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In +order to process possible `.pth` files `site.addsitedir` is used. + +The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they +will be added before the entries we add here and thus take precedence. + +Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage. + +Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE` +and sets `sys.executable` to its value. +""" +import site +import sys +import os +import functools + +paths = os.environ.pop('NIX_PYTHONPATH', None) +if paths: + functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo()) + +# Check whether we are in a venv or virtualenv. +# For Python 3 we check whether our `base_prefix` is different from our current `prefix`. +# For Python 2 we check whether the non-standard `real_prefix` is set. +# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv +in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix")) + +if not in_venv: + executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None) + prefix = os.environ.pop('NIX_PYTHONPREFIX', None) + + if 'PYTHONEXECUTABLE' not in os.environ and executable is not None: + sys.executable = executable + if prefix is not None: + # Sysconfig does not like it when sys.prefix is set to None + sys.prefix = sys.exec_prefix = prefix + site.PREFIXES.insert(0, prefix) diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index e989c92d5110..2cd29ca99032 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -39,25 +39,13 @@ let is_virtualenv = "False"; }; } // lib.optionalAttrs (!python.isPyPy && !stdenv.isDarwin) { - # Use virtualenv with symlinks from a Nix env. + # Use virtualenv from a Nix env. # Fails on darwin with # virtualenv: error: argument dest: the destination . is not write-able at /nix/store - nixenv-virtualenv-links = rec { - env = runCommand "${python.name}-virtualenv-links" {} '' - ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out - ''; - interpreter = "${env}/bin/${python.executable}"; - is_venv = "False"; - is_nixenv = "True"; - is_virtualenv = "True"; - }; - } // lib.optionalAttrs (!python.isPyPy && !stdenv.isDarwin) { - # Use virtualenv with copies from a Nix env. - # Fails on darwin with - # virtualenv: error: argument dest: the destination . is not write-able at /nix/store - nixenv-virtualenv-copies = rec { - env = runCommand "${python.name}-virtualenv-copies" {} '' - ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out + nixenv-virtualenv = rec { + env = runCommand "${python.name}-virtualenv" {} '' + ${pythonVirtualEnv.interpreter} -m virtualenv venv + mv venv $out ''; interpreter = "${env}/bin/${python.executable}"; is_venv = "False"; @@ -73,48 +61,27 @@ let is_nixenv = "True"; is_virtualenv = "False"; }; - } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) { - # Venv built using links to plain Python + } // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) { + # Venv built using plain Python # Python 2 does not support venv # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3. - plain-venv-links = rec { - env = runCommand "${python.name}-venv-links" {} '' - ${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out - ''; - interpreter = "${env}/bin/${python.executable}"; - is_venv = "True"; - is_nixenv = "False"; - is_virtualenv = "False"; - }; - } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) { - # Venv built using copies from plain Python - # Python 2 does not support venv - # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3. - plain-venv-copies = rec { - env = runCommand "${python.name}-venv-copies" {} '' - ${python.interpreter} -m venv --system-site-packages --copies --without-pip $out + plain-venv = rec { + env = runCommand "${python.name}-venv" {} '' + ${python.interpreter} -m venv $out ''; interpreter = "${env}/bin/${python.executable}"; is_venv = "True"; is_nixenv = "False"; is_virtualenv = "False"; }; + } // lib.optionalAttrs (python.pythonAtLeast "3.8") { # Venv built using Python Nix environment (python.buildEnv) - nixenv-venv-links = rec { - env = runCommand "${python.name}-venv-links" {} '' - ${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out - ''; - interpreter = "${env}/bin/${pythonEnv.executable}"; - is_venv = "True"; - is_nixenv = "True"; - is_virtualenv = "False"; - }; - } // lib.optionalAttrs (python.pythonAtLeast "3.8") { - # Venv built using Python Nix environment (python.buildEnv) - nixenv-venv-copies = rec { - env = runCommand "${python.name}-venv-copies" {} '' - ${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out + # TODO: Cannot create venv from a nix env + # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1. + nixenv-venv = rec { + env = runCommand "${python.name}-venv" {} '' + ${pythonEnv.interpreter} -m venv $out ''; interpreter = "${env}/bin/${pythonEnv.executable}"; is_venv = "True"; @@ -126,33 +93,11 @@ let testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({ inherit (python) pythonVersion; } // attrs) '' - mkdir $out - - # set up the test files cp -r ${./tests/test_environments} tests chmod -R +w tests substituteAllInPlace tests/test_python.py - - # run the tests by invoking the interpreter via full path - echo "absolute path: ${attrs.interpreter}" - ${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt" - - # run the tests by invoking the interpreter via $PATH - export PATH="$(dirname ${attrs.interpreter}):$PATH" - echo "PATH: $(basename ${attrs.interpreter})" - "$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt" - - # make sure we get the right path when invoking through a result link - ln -s "${attrs.env}" result - relative="result/bin/$(basename ${attrs.interpreter})" - expected="$PWD/$relative" - actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")" - if [ "$actual" != "$expected" ]; then - echo "expected $expected, got $actual" - exit 1 - fi - - # if we got this far, the tests passed + ${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py + mkdir $out touch $out/success ''; diff --git a/pkgs/development/interpreters/python/tests/test_environments/test_python.py b/pkgs/development/interpreters/python/tests/test_environments/test_python.py index 538273f65dbc..0fc4b8a9e91c 100644 --- a/pkgs/development/interpreters/python/tests/test_environments/test_python.py +++ b/pkgs/development/interpreters/python/tests/test_environments/test_python.py @@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase): @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix") def test_base_prefix(self): - if IS_VENV or IS_VIRTUALENV: + if IS_VENV or IS_NIXENV or IS_VIRTUALENV: self.assertNotEqual(sys.prefix, sys.base_prefix) else: self.assertEqual(sys.prefix, sys.base_prefix)