From a404380caf76cc80b67f6f68b4d15022bd9f07b5 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Thu, 22 Oct 2026 14:35:18 -0700 Subject: [PATCH] gh-106045: Keep searching from venv home when the resolved base executable has no stdlib (GH-160123) GH-156483 made `getpath` locate a venv's base installation relative to the *resolved* base executable instead of the `home` key in `pyvenv.cfg`. When the resolved executable lives outside the installation tree (e.g. a launcher binary symlinked into it), nothing is found there and discovery falls back to the compiled-in `PREFIX`. The venv then either fails to start or silently runs with another installation's standard library. This PR: - Keeps GH-156483's behavior when the resolved executable's directory locates a standard library (`pybuilddir.txt`, the stdlib zip, or `os.py`). Otherwise it keeps searching from `home`, as before GH-156483. - In that fallback case, still searches the resolved executable's directory for `lib-dynload` when determining `exec_prefix`: after `prefix`, before `home` and the compiled-in `EXEC_PREFIX`. Layouts with a separate exec prefix (e.g. `./configure --exec-prefix`) then keep loading extension modules from the right place, and a stale `lib-dynload` near the `home` symlink can't take precedence over the one beside the resolved executable. The tests added by GH-156483 pass unchanged. Three new `test_getpath` cases cover the launcher-outside-the-installation and separate-exec-prefix layouts, and a misleading `lib-dynload` near `home`; each fails without this change.
Reproducer On `main` (0a86a34), with a plain `./configure --prefix=$HOME/py-under-test && make && make install`: ```sh #!/bin/sh # Usage: venv-home-repro.sh set -eu PREFIX=$1; VER=$2 W=$(mktemp -d) mkdir -p "$W/runtime/bin" "$W/runtime/lib" "$W/launcher" cp -R "$PREFIX/lib/python$VER" "$W/runtime/lib/" cp "$PREFIX/bin/python$VER" "$W/launcher/python3" ln -s "$W/launcher/python3" "$W/runtime/bin/python3" PYTHONHOME="$W/runtime" "$W/runtime/bin/python3" -m venv --without-pip "$W/venv" grep '^home' "$W/venv/pyvenv.cfg" echo "expected base_prefix: $W/runtime" "$W/venv/bin/python" -c 'import os, sys; print("actual base_prefix:", sys.base_prefix); print("os.py loaded from:", os.__file__)' ``` Before: ``` home = /tmp/tmp.NFug9wEr4h/runtime/bin expected base_prefix: /tmp/tmp.NFug9wEr4h/runtime actual base_prefix: /home/user/py-under-test os.py loaded from: /home/user/py-under-test/lib/python3.16/os.py ``` (With `$HOME/py-under-test/lib/python3.16` removed, the venv instead fails with `ModuleNotFoundError: No module named 'encodings'`.) After: ``` home = /tmp/tmp.8Zne2otFrw/runtime/bin expected base_prefix: /tmp/tmp.8Zne2otFrw/runtime actual base_prefix: /tmp/tmp.8Zne2otFrw/runtime os.py loaded from: /tmp/tmp.8Zne2otFrw/runtime/lib/python3.16/os.py ```
(cherry picked from commit 5b55b502221b67660a4d652cdd3dd2203920e58a) Co-authored-by: Itamar Oren --- Lib/test/test_getpath.py | 103 ++++++++++++++++++ ...-10-22-12-30-11.gh-issue-106045.fkMW34.rst | 5 + Modules/getpath.py | 18 ++- 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 2907abe4b4cee67..abf8913cc950e88 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -475,6 +475,109 @@ def test_venv_symlink_with_misleading_landmarks_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_symlink_to_launcher_outside_runtime_posix(self): + "Test a venv whose base executable resolves away from its stdlib." + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ) + ns.add_known_xfile("/out/launcher") + ns.add_known_xfile("/runtime/bin/python") + ns.add_known_link("/runtime/bin/python", "/out/launcher") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", "/runtime/bin/python") + ns.add_known_file("/runtime/lib/python9.8/os.py") + ns.add_known_dir("/runtime/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + "home = /runtime/bin", + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/venv", + exec_prefix="/venv", + base_executable="/out/launcher", + base_prefix="/runtime", + base_exec_prefix="/runtime", + module_search_paths_set=1, + module_search_paths=[ + "/runtime/lib/python98.zip", + "/runtime/lib/python9.8", + "/runtime/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_symlink_to_separate_exec_prefix_posix(self): + "Test a venv whose base executable lives under a separate exec_prefix." + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + EXEC_PREFIX="/usr", + ) + ns.add_known_xfile("/exec/bin/python") + ns.add_known_xfile("/runtime/bin/python") + ns.add_known_link("/runtime/bin/python", "/exec/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", "/runtime/bin/python") + ns.add_known_file("/runtime/lib/python9.8/os.py") + ns.add_known_dir("/exec/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + "home = /runtime/bin", + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/venv", + exec_prefix="/venv", + base_executable="/exec/bin/python", + base_prefix="/runtime", + base_exec_prefix="/exec", + module_search_paths_set=1, + module_search_paths=[ + "/runtime/lib/python98.zip", + "/runtime/lib/python9.8", + "/exec/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_separate_exec_prefix_with_misleading_home_posix(self): + "Test that a stale lib-dynload near home does not override the base executable." + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/stdlib", + EXEC_PREFIX="/build/exec", + ) + ns.add_known_xfile("/exec/bin/python") + ns.add_known_xfile("/public/bin/python") + ns.add_known_link("/public/bin/python", "/exec/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", "/public/bin/python") + ns.add_known_file("/stdlib/lib/python9.8/os.py") + ns.add_known_dir("/exec/lib/python9.8/lib-dynload") + # An empty directory near the symlink must not win. + ns.add_known_dir("/public/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + "home = /public/bin", + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/venv", + exec_prefix="/venv", + base_executable="/exec/bin/python", + base_prefix="/stdlib", + base_exec_prefix="/exec", + module_search_paths_set=1, + module_search_paths=[ + "/stdlib/lib/python98.zip", + "/stdlib/lib/python9.8", + "/exec/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_non_installed_zip_path_posix(self): "Test a venv created from non-installed python has correct zip path.""" ns = MockPosixNamespace( diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst new file mode 100644 index 000000000000000..98abf1debe08d6d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst @@ -0,0 +1,5 @@ +Fix :mod:`venv` path discovery when the base executable resolves to a +location outside the installation pointed to by the ``home`` key in +``pyvenv.cfg``. The standard library is now found from ``home``, and +extension modules from the resolved executable's installation, instead of +falling back to the compiled-in prefix. diff --git a/Modules/getpath.py b/Modules/getpath.py index f5f928f1795de53..2ffb1d964c0fb63 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -232,6 +232,7 @@ def search_up(prefix, *landmarks, test=isfile): stdlib_dir_was_set_in_config = bool(stdlib_dir) real_executable_dir = None +base_executable_dir = None platstdlib_dir = None stdlib_zip = None @@ -412,7 +413,17 @@ def search_up(prefix, *landmarks, test=isfile): if base_executable and isfile(base_executable): # Search relative to the resolved base executable rather # than the potentially external symlink in pyvenv.cfg. - executable_dir = real_executable_dir = dirname(base_executable) + # If nothing there locates a standard library (e.g. a + # launcher binary stored outside the installation that + # 'home' points into), keep searching from 'home'. + # base_executable_dir is still used to find platform + # dependent libraries (exec_prefix) in that case. + base_executable_dir = dirname(base_executable) + if ( + isfile(joinpath(base_executable_dir, BUILDDIR_TXT)) + or search_up(base_executable_dir, ZIP_LANDMARK, *STDLIB_LANDMARKS) + ): + executable_dir = real_executable_dir = base_executable_dir # home key found; stop iterating over lines break @@ -622,6 +633,11 @@ def search_up(prefix, *landmarks, test=isfile): exec_prefix = prefix if not exec_prefix and prefix and isdir(joinpath(prefix, PLATSTDLIB_LANDMARK)): exec_prefix = prefix + if not exec_prefix and base_executable_dir and base_executable_dir != executable_dir: + # The venv's resolved base executable may live under a separate + # exec_prefix (e.g. ./configure --exec-prefix) even when the + # stdlib was found from 'home'. + exec_prefix = search_up(base_executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: