GH-126789: fix some sysconfig data on late site initializations

Author: FFY00

Lib/sysconfig/__init__.py

@@ -173,9 +173,7 @@ def joinuser(*args):
 _PY_VERSION = sys.version.split()[0]
 _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}'
 _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}'
-_PREFIX = os.path.normpath(sys.prefix)
 _BASE_PREFIX = os.path.normpath(sys.base_prefix)
-_EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
 _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
 # Mutex guarding initialization of _CONFIG_VARS.
 _CONFIG_VARS_LOCK = threading.RLock()
@@ -466,8 +464,10 @@ def _init_config_vars():
     # Normalized versions of prefix and exec_prefix are handy to have;
     # in fact, these are the standard versions used most places in the
     # Distutils.
-    _CONFIG_VARS['prefix'] = _PREFIX
-    _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX
+    _PREFIX = os.path.normpath(sys.prefix)
+    _EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+    _CONFIG_VARS['prefix'] = _PREFIX  # FIXME: This gets overwriten by _init_posix.
+    _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX  # FIXME: This gets overwriten by _init_posix.
     _CONFIG_VARS['py_version'] = _PY_VERSION
     _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
     _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT
@@ -540,6 +540,7 @@ def get_config_vars(*args):
     With arguments, return a list of values that result from looking up
     each argument in the configuration variable dictionary.
     """
+    global _CONFIG_VARS_INITIALIZED
 
     # Avoid claiming the lock once initialization is complete.
     if not _CONFIG_VARS_INITIALIZED:
@@ -550,6 +551,15 @@ def get_config_vars(*args):
             # don't re-enter init_config_vars().
             if _CONFIG_VARS is None:
                 _init_config_vars()
+    else:
+        # If the site module initialization happened after _CONFIG_VARS was
+        # initialized, a virtual environment might have been activated, resulting in
+        # variables like sys.prefix changing their value, so we need to re-init the
+        # config vars (see GH-126789).
+        if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix):
+            with _CONFIG_VARS_LOCK:
+                _CONFIG_VARS_INITIALIZED = False
+                _init_config_vars()
 
     if args:
         vals = []

Lib/test/support/venv.py

@@ -0,0 +1,70 @@
+import contextlib
+import logging
+import os
+import subprocess
+import shlex
+import sys
+import sysconfig
+import tempfile
+import venv
+
+
+class VirtualEnvironment:
+    def __init__(self, prefix, **venv_create_args):
+        self._logger = logging.getLogger(self.__class__.__name__)
+        venv.create(prefix, **venv_create_args)
+        self._prefix = prefix
+        self._paths = sysconfig.get_paths(
+            scheme='venv',
+            vars={'base': self.prefix},
+            expand=True,
+        )
+
+    @classmethod
+    @contextlib.contextmanager
+    def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args):
+        delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV'))
+        with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir:
+            yield cls(tmpdir, **venv_create_args)
+
+    @property
+    def prefix(self):
+        return self._prefix
+
+    @property
+    def paths(self):
+        return self._paths
+
+    @property
+    def interpreter(self):
+        return os.path.join(self.paths['scripts'], os.path.basename(sys.executable))
+
+    def _format_output(self, name, data, indent='\t'):
+        if not data:
+            return indent + f'{name}: (none)'
+        if len(data.splitlines()) == 1:
+            return indent + f'{name}: {data}'
+        else:
+            prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines())
+            return indent + f'{name}:\n' + prefixed_lines
+
+    def run(self, *args, **subprocess_args):
+        if subprocess_args.get('shell'):
+            raise ValueError('Running the subprocess in shell mode is not supported.')
+        default_args = {
+            'capture_output': True,
+            'check': True,
+        }
+        try:
+            result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args)
+        except subprocess.CalledProcessError as e:
+            if e.returncode != 0:
+                self._logger.error(
+                    f'Interpreter returned non-zero exit status {e.returncode}.\n'
+                    + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n'
+                    + self._format_output('STDOUT', e.stdout.decode()) + '\n'
+                    + self._format_output('STDERR', e.stderr.decode()) + '\n'
+                )
+            raise
+        else:
+            return result

Lib/test/test_sysconfig.py

@@ -5,6 +5,8 @@
 import os
 import subprocess
 import shutil
+import json
+import textwrap
 from copy import copy
 
 from test.support import (
@@ -17,6 +19,7 @@
 from test.support.import_helper import import_module
 from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink,
                                     change_cwd)
+from test.support.venv import VirtualEnvironment
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -101,6 +104,12 @@ def _cleanup_testfn(self):
         elif os.path.isdir(path):
             shutil.rmtree(path)
 
+    def venv(self, **venv_create_args):
+        return VirtualEnvironment.from_tmpdir(
+            prefix=f'{self.id()}-venv-',
+            **venv_create_args,
+        )
+
     def test_get_path_names(self):
         self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS)
 
@@ -582,6 +591,72 @@ def test_osx_ext_suffix(self):
         suffix = sysconfig.get_config_var('EXT_SUFFIX')
         self.assertTrue(suffix.endswith('-darwin.so'), suffix)
 
+    @unittest.skipIf(sys.platform == 'wasi', 'venv is unsupported on WASI')
+    def test_config_vars_depend_on_site_initialization(self):
+        script = textwrap.dedent("""
+            import sysconfig
+
+            config_vars = sysconfig.get_config_vars()
+
+            import json
+            print(json.dumps(config_vars, indent=2))
+        """)
+
+        with self.venv() as venv:
+            site_config_vars = json.loads(venv.run('-c', script).stdout)
+            no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout)
+
+        self.assertNotEqual(site_config_vars, no_site_config_vars)
+        # With the site initialization, the virtual environment should be enabled.
+        self.assertEqual(site_config_vars['base'], venv.prefix)
+        self.assertEqual(site_config_vars['platbase'], venv.prefix)
+        #self.assertEqual(site_config_vars['prefix'], venv.prefix)  # # FIXME: prefix gets overwriten by _init_posix
+        # Without the site initialization, the virtual environment should be disabled.
+        self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base'])
+        self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase'])
+
+    @unittest.skipIf(sys.platform == 'wasi', 'venv is unsupported on WASI')
+    def test_config_vars_recalculation_after_site_initialization(self):
+        script = textwrap.dedent("""
+            import sysconfig
+
+            before = sysconfig.get_config_vars()
+
+            import site
+            site.main()
+
+            after = sysconfig.get_config_vars()
+
+            import json
+            print(json.dumps({'before': before, 'after': after}, indent=2))
+        """)
+
+        with self.venv() as venv:
+            config_vars = json.loads(venv.run('-S', '-c', script).stdout)
+
+        self.assertNotEqual(config_vars['before'], config_vars['after'])
+        self.assertEqual(config_vars['after']['base'], venv.prefix)
+        #self.assertEqual(config_vars['after']['prefix'], venv.prefix)  # FIXME: prefix gets overwriten by _init_posix
+        #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix)  # FIXME: exec_prefix gets overwriten by _init_posix
+
+    @unittest.skipIf(sys.platform == 'wasi', 'venv is unsupported on WASI')
+    def test_paths_depend_on_site_initialization(self):
+        script = textwrap.dedent("""
+            import sysconfig
+
+            paths = sysconfig.get_paths()
+
+            import json
+            print(json.dumps(paths, indent=2))
+        """)
+
+        with self.venv() as venv:
+            site_paths = json.loads(venv.run('-c', script).stdout)
+            no_site_paths = json.loads(venv.run('-S', '-c', script).stdout)
+
+        self.assertNotEqual(site_paths, no_site_paths)
+
+
 class MakefileTests(unittest.TestCase):
 
     @unittest.skipIf(sys.platform.startswith('win'),

Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst

@@ -0,0 +1,4 @@
+Fixed the values of :py:func:`sysconfig.get_config_vars`,
+:py:func:`sysconfig.get_paths`, and their siblings when the :py:mod:`site`
+initialization happens after :py:mod:`sysconfig` has built a cache for
+:py:func:`sysconfig.get_config_vars`.