Merge pull request #297628 from cwp/python-env-venv
Fix venv creation in Python environments
This commit is contained in:
Domen Kožar
GitHub
commit
fb884172ab
|
|
@ -19,6 +19,7 @@ assertExecutable() {
|
|||
# (if unset or empty, defaults to EXECUTABLE)
|
||||
# --inherit-argv0 : the executable inherits argv0 from the wrapper.
|
||||
# (use instead of --argv0 '$0')
|
||||
# --resolve-argv0 : if argv0 doesn't include a / character, resolve it against PATH
|
||||
# --set VAR VAL : add VAR with value VAL to the executable's environment
|
||||
# --set-default VAR VAL : like --set, but only adds VAR if not already set in
|
||||
# the environment
|
||||
|
|
@ -87,6 +88,7 @@ makeDocumentedCWrapper() {
|
|||
makeCWrapper() {
|
||||
local argv0 inherit_argv0 n params cmd main flagsBefore flagsAfter flags executable length
|
||||
local uses_prefix uses_suffix uses_assert uses_assert_success uses_stdio uses_asprintf
|
||||
local resolve_path
|
||||
executable=$(escapeStringLiteral "$1")
|
||||
params=("$@")
|
||||
length=${#params[*]}
|
||||
|
|
@ -169,6 +171,12 @@ makeCWrapper() {
|
|||
# Whichever comes last of --argv0 and --inherit-argv0 wins
|
||||
inherit_argv0=1
|
||||
;;
|
||||
--resolve-argv0)
|
||||
# this gets processed after other argv0 flags
|
||||
uses_stdio=1
|
||||
uses_string=1
|
||||
resolve_argv0=1
|
||||
;;
|
||||
*) # Using an error macro, we will make sure the compiler gives an understandable error message
|
||||
main="$main#error makeCWrapper: Unknown argument ${p}"$'\n'
|
||||
;;
|
||||
|
|
@ -176,6 +184,7 @@ makeCWrapper() {
|
|||
done
|
||||
[[ -z "$flagsBefore" && -z "$flagsAfter" ]] || main="$main"${main:+$'\n'}$(addFlags "$flagsBefore" "$flagsAfter")$'\n'$'\n'
|
||||
[ -z "$inherit_argv0" ] && main="${main}argv[0] = \"${argv0:-${executable}}\";"$'\n'
|
||||
[ -z "$resolve_argv0" ] || main="${main}argv[0] = resolve_argv0(argv[0]);"$'\n'
|
||||
main="${main}return execv(\"${executable}\", argv);"$'\n'
|
||||
|
||||
[ -z "$uses_asprintf" ] || printf '%s\n' "#define _GNU_SOURCE /* See feature_test_macros(7) */"
|
||||
|
|
@ -183,9 +192,11 @@ makeCWrapper() {
|
|||
printf '%s\n' "#include <stdlib.h>"
|
||||
[ -z "$uses_assert" ] || printf '%s\n' "#include <assert.h>"
|
||||
[ -z "$uses_stdio" ] || printf '%s\n' "#include <stdio.h>"
|
||||
[ -z "$uses_string" ] || printf '%s\n' "#include <string.h>"
|
||||
[ -z "$uses_assert_success" ] || printf '\n%s\n' "#define assert_success(e) do { if ((e) < 0) { perror(#e); abort(); } } while (0)"
|
||||
[ -z "$uses_prefix" ] || printf '\n%s\n' "$(setEnvPrefixFn)"
|
||||
[ -z "$uses_suffix" ] || printf '\n%s\n' "$(setEnvSuffixFn)"
|
||||
[ -z "$resolve_argv0" ] || printf '\n%s\n' "$(resolveArgv0Fn)"
|
||||
printf '\n%s' "int main(int argc, char **argv) {"
|
||||
printf '\n%s' "$(indent4 "$main")"
|
||||
printf '\n%s\n' "}"
|
||||
|
|
@ -338,6 +349,41 @@ void set_env_suffix(char *env, char *sep, char *suffix) {
|
|||
"
|
||||
}
|
||||
|
||||
resolveArgv0Fn() {
|
||||
printf '%s' "\
|
||||
char *resolve_argv0(char *argv0) {
|
||||
if (strchr(argv0, '/') != NULL) {
|
||||
return argv0;
|
||||
}
|
||||
char *path = getenv(\"PATH\");
|
||||
if (path == NULL) {
|
||||
return argv0;
|
||||
}
|
||||
char *path_copy = strdup(path);
|
||||
if (path_copy == NULL) {
|
||||
return argv0;
|
||||
}
|
||||
char *dir = strtok(path_copy, \":\");
|
||||
while (dir != NULL) {
|
||||
char *candidate = malloc(strlen(dir) + strlen(argv0) + 2);
|
||||
if (candidate == NULL) {
|
||||
free(path_copy);
|
||||
return argv0;
|
||||
}
|
||||
sprintf(candidate, \"%s/%s\", dir, argv0);
|
||||
if (access(candidate, X_OK) == 0) {
|
||||
free(path_copy);
|
||||
return candidate;
|
||||
}
|
||||
free(candidate);
|
||||
dir = strtok(NULL, \":\");
|
||||
}
|
||||
free(path_copy);
|
||||
return argv0;
|
||||
}
|
||||
"
|
||||
}
|
||||
|
||||
# Embed a C string which shows up as readable text in the compiled binary wrapper,
|
||||
# giving instructions for recreating the wrapper.
|
||||
# Keep in sync with makeBinaryWrapper.extractCmd
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ assertExecutable() {
|
|||
# (if unset or empty, defaults to EXECUTABLE)
|
||||
# --inherit-argv0 : the executable inherits argv0 from the wrapper.
|
||||
# (use instead of --argv0 '$0')
|
||||
# --resolve-argv0 : if argv0 doesn't include a / character, resolve it against PATH
|
||||
# --set VAR VAL : add VAR with value VAL to the executable's environment
|
||||
# --set-default VAR VAL : like --set, but only adds VAR if not already set in
|
||||
# the environment
|
||||
|
|
@ -177,6 +178,9 @@ makeShellWrapper() {
|
|||
elif [[ "$p" == "--inherit-argv0" ]]; then
|
||||
# Whichever comes last of --argv0 and --inherit-argv0 wins
|
||||
argv0='$0'
|
||||
elif [[ "$p" == "--resolve-argv0" ]]; then
|
||||
# this is noop in shell wrappers, since bash will always resolve $0
|
||||
resolve_argv0=1
|
||||
else
|
||||
die "makeWrapper doesn't understand the arg $p"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -318,8 +318,6 @@ in with passthru; stdenv.mkDerivation ({
|
|||
inherit passthru;
|
||||
|
||||
postFixup = ''
|
||||
# Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
|
||||
cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
'' + lib.optionalString strip2to3 ''
|
||||
rm -R $out/bin/2to3 $out/lib/python*/lib2to3
|
||||
'' + lib.optionalString stripConfig ''
|
||||
|
|
|
|||
|
|
@ -537,8 +537,7 @@ in with passthru; stdenv.mkDerivation (finalAttrs: {
|
|||
# Strip tests
|
||||
rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
|
||||
'' + optionalString includeSiteCustomize ''
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
'' + optionalString stripBytecode ''
|
||||
# Determinism: deterministic bytecode
|
||||
# First we delete all old bytecode.
|
||||
|
|
|
|||
|
|
@ -126,9 +126,6 @@ in with passthru; stdenv.mkDerivation rec {
|
|||
ln -s $out/${executable}-c/include $out/include/${libPrefix}
|
||||
ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
|
|
|||
|
|
@ -95,9 +95,6 @@ in with passthru; stdenv.mkDerivation {
|
|||
echo "Removing bytecode"
|
||||
find . -name "__pycache__" -type d -depth -delete
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
|
|
|||
|
|
@ -96,9 +96,6 @@ in with passthru; stdenv.mkDerivation {
|
|||
echo "Removing bytecode"
|
||||
find . -name "__pycache__" -type d -depth -delete
|
||||
|
||||
# Include a sitecustomize.py file
|
||||
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
"""
|
||||
This is a Nix-specific module for discovering modules built with Nix.
|
||||
|
||||
The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In
|
||||
order to process possible `.pth` files `site.addsitedir` is used.
|
||||
|
||||
The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they
|
||||
will be added before the entries we add here and thus take precedence.
|
||||
|
||||
Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage.
|
||||
|
||||
Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE`
|
||||
and sets `sys.executable` to its value.
|
||||
"""
|
||||
import site
|
||||
import sys
|
||||
import os
|
||||
import functools
|
||||
|
||||
paths = os.environ.pop('NIX_PYTHONPATH', None)
|
||||
if paths:
|
||||
functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
|
||||
|
||||
# Check whether we are in a venv or virtualenv.
|
||||
# For Python 3 we check whether our `base_prefix` is different from our current `prefix`.
|
||||
# For Python 2 we check whether the non-standard `real_prefix` is set.
|
||||
# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv
|
||||
in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix"))
|
||||
|
||||
if not in_venv:
|
||||
executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
|
||||
prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
|
||||
|
||||
if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
|
||||
sys.executable = executable
|
||||
if prefix is not None:
|
||||
# Sysconfig does not like it when sys.prefix is set to None
|
||||
sys.prefix = sys.exec_prefix = prefix
|
||||
site.PREFIXES.insert(0, prefix)
|
||||
|
|
@ -39,11 +39,21 @@ let
|
|||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (!python.isPyPy) {
|
||||
# Use virtualenv from a Nix env.
|
||||
nixenv-virtualenv = rec {
|
||||
env = runCommand "${python.name}-virtualenv" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv venv
|
||||
mv venv $out
|
||||
# Use virtualenv with symlinks from a Nix env.
|
||||
nixenv-virtualenv-links = rec {
|
||||
env = runCommand "${python.name}-virtualenv-links" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "False";
|
||||
is_nixenv = "True";
|
||||
is_virtualenv = "True";
|
||||
};
|
||||
} // lib.optionalAttrs (!python.isPyPy) {
|
||||
# Use virtualenv with copies from a Nix env.
|
||||
nixenv-virtualenv-copies = rec {
|
||||
env = runCommand "${python.name}-virtualenv-copies" {} ''
|
||||
${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "False";
|
||||
|
|
@ -59,27 +69,48 @@ let
|
|||
is_nixenv = "True";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) {
|
||||
# Venv built using plain Python
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
|
||||
# Venv built using links to plain Python
|
||||
# Python 2 does not support venv
|
||||
# TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
|
||||
plain-venv = rec {
|
||||
env = runCommand "${python.name}-venv" {} ''
|
||||
${python.interpreter} -m venv $out
|
||||
plain-venv-links = rec {
|
||||
env = runCommand "${python.name}-venv-links" {} ''
|
||||
${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "False";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
|
||||
# Venv built using copies from plain Python
|
||||
# Python 2 does not support venv
|
||||
# TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
|
||||
plain-venv-copies = rec {
|
||||
env = runCommand "${python.name}-venv-copies" {} ''
|
||||
${python.interpreter} -m venv --system-site-packages --copies --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${python.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "False";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8") {
|
||||
# Venv built using Python Nix environment (python.buildEnv)
|
||||
# TODO: Cannot create venv from a nix env
|
||||
# Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
|
||||
nixenv-venv = rec {
|
||||
env = runCommand "${python.name}-venv" {} ''
|
||||
${pythonEnv.interpreter} -m venv $out
|
||||
nixenv-venv-links = rec {
|
||||
env = runCommand "${python.name}-venv-links" {} ''
|
||||
${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${pythonEnv.executable}";
|
||||
is_venv = "True";
|
||||
is_nixenv = "True";
|
||||
is_virtualenv = "False";
|
||||
};
|
||||
} // lib.optionalAttrs (python.pythonAtLeast "3.8") {
|
||||
# Venv built using Python Nix environment (python.buildEnv)
|
||||
nixenv-venv-copies = rec {
|
||||
env = runCommand "${python.name}-venv-copies" {} ''
|
||||
${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out
|
||||
'';
|
||||
interpreter = "${env}/bin/${pythonEnv.executable}";
|
||||
is_venv = "True";
|
||||
|
|
@ -91,11 +122,33 @@ let
|
|||
testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({
|
||||
inherit (python) pythonVersion;
|
||||
} // attrs) ''
|
||||
mkdir $out
|
||||
|
||||
# set up the test files
|
||||
cp -r ${./tests/test_environments} tests
|
||||
chmod -R +w tests
|
||||
substituteAllInPlace tests/test_python.py
|
||||
${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py
|
||||
mkdir $out
|
||||
|
||||
# run the tests by invoking the interpreter via full path
|
||||
echo "absolute path: ${attrs.interpreter}"
|
||||
${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt"
|
||||
|
||||
# run the tests by invoking the interpreter via $PATH
|
||||
export PATH="$(dirname ${attrs.interpreter}):$PATH"
|
||||
echo "PATH: $(basename ${attrs.interpreter})"
|
||||
"$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt"
|
||||
|
||||
# make sure we get the right path when invoking through a result link
|
||||
ln -s "${attrs.env}" result
|
||||
relative="result/bin/$(basename ${attrs.interpreter})"
|
||||
expected="$PWD/$relative"
|
||||
actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")"
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "expected $expected, got $actual"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# if we got this far, the tests passed
|
||||
touch $out/success
|
||||
'';
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase):
|
|||
|
||||
@unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
|
||||
def test_base_prefix(self):
|
||||
if IS_VENV or IS_NIXENV or IS_VIRTUALENV:
|
||||
if IS_VENV or IS_VIRTUALENV:
|
||||
self.assertNotEqual(sys.prefix, sys.base_prefix)
|
||||
else:
|
||||
self.assertEqual(sys.prefix, sys.base_prefix)
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ let
|
|||
fi
|
||||
mkdir -p "$out/bin"
|
||||
|
||||
rm -f $out/bin/.*-wrapped
|
||||
|
||||
for path in ${lib.concatStringsSep " " paths}; do
|
||||
if [ -d "$path/bin" ]; then
|
||||
cd "$path/bin"
|
||||
|
|
@ -42,7 +44,13 @@ let
|
|||
if [ -f "$prg" ]; then
|
||||
rm -f "$out/bin/$prg"
|
||||
if [ -x "$prg" ]; then
|
||||
makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
|
||||
if [ -f ".$prg-wrapped" ]; then
|
||||
echo "#!${pythonExecutable}" > "$out/bin/$prg"
|
||||
sed -e '1d' -e '3d' ".$prg-wrapped" >> "$out/bin/$prg"
|
||||
chmod +x "$out/bin/$prg"
|
||||
else
|
||||
makeWrapper "$path/bin/$prg" "$out/bin/$prg" --inherit-argv0 --resolve-argv0 ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user