Move classes used by scripts to their own module

Those classes are used by wrapper scripts and we should not have to
import the rest of mesonlib, build.py, and all their dependencies for
that.

This renames mesonlib/ directory to utils/ and add a mesonlib.py module
that imports everything from utils/ to not have to change `import
mesonlib` everywhere. It allows to import utils.core without importing
the rest of mesonlib.

Author: xclaesse

.flake8

@@ -29,5 +29,5 @@ extend-ignore =
     # A003: builtin class attribute
     A003
 per-file-ignores =
-    mesonbuild/mesonlib/__init__.py:F401,F403
+    mesonbuild/mesonlib.py:F401,F403
 max-line-length = 120

meson.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This file is an entry point for all commands, including scripts. Include the
+# strict minimum python modules for performance reasons.
 import sys
 
 # Check python version before importing anything else, we might have an older

mesonbuild/backend/backends.py

@@ -34,7 +34,8 @@
 from ..compilers import LANGUAGES_USING_LDFLAGS, detect
 from ..mesonlib import (
     File, MachineChoice, MesonException, OrderedSet,
-    classify_unity_sources, OptionKey, join_args
+    classify_unity_sources, OptionKey, join_args,
+    ExecutableSerialisation
 )
 
 if T.TYPE_CHECKING:
@@ -185,27 +186,6 @@ def __init__(self, path: str, install_path: str, install_path_name: str,
         super().__init__(path, install_path, install_path_name, install_mode, subproject, tag, data_type)
         self.exclude = exclude
 
-@dataclass(eq=False)
-class ExecutableSerialisation:
-
-    # XXX: should capture and feed default to False, instead of None?
-
-    cmd_args: T.List[str]
-    env: T.Optional[build.EnvironmentVariables] = None
-    exe_wrapper: T.Optional['programs.ExternalProgram'] = None
-    workdir: T.Optional[str] = None
-    extra_paths: T.Optional[T.List] = None
-    capture: T.Optional[bool] = None
-    feed: T.Optional[bool] = None
-    tag: T.Optional[str] = None
-    verbose: bool = False
-
-    def __post_init__(self) -> None:
-        if self.exe_wrapper is not None:
-            assert isinstance(self.exe_wrapper, programs.ExternalProgram)
-        self.pickled = False
-        self.skip_if_destdir = False
-        self.subproject = ''
 
 @dataclass(eq=False)
 class TestSerialisation:

mesonbuild/build.py

@@ -36,7 +36,7 @@
     extract_as_list, typeslistify, stringlistify, classify_unity_sources,
     get_filenames_templates_dict, substitute_values, has_path_sep,
     OptionKey, PerMachineDefaultable, OptionOverrideProxy,
-    MesonBugException
+    MesonBugException, EnvironmentVariables
 )
 from .compilers import (
     is_object, clink_langs, sort_clink, all_languages,
@@ -502,71 +502,6 @@ def needs_copy(self) -> bool:
         return False
 
 
-EnvInitValueType = T.Dict[str, T.Union[str, T.List[str]]]
-
-
-class EnvironmentVariables(HoldableObject):
-    def __init__(self, values: T.Optional[EnvInitValueType] = None,
-                 init_method: Literal['set', 'prepend', 'append'] = 'set', separator: str = os.pathsep) -> None:
-        self.envvars: T.List[T.Tuple[T.Callable[[T.Dict[str, str], str, T.List[str], str], str], str, T.List[str], str]] = []
-        # The set of all env vars we have operations for. Only used for self.has_name()
-        self.varnames: T.Set[str] = set()
-
-        if values:
-            init_func = getattr(self, init_method)
-            for name, value in values.items():
-                init_func(name, listify(value), separator)
-
-    def __repr__(self) -> str:
-        repr_str = "<{0}: {1}>"
-        return repr_str.format(self.__class__.__name__, self.envvars)
-
-    def hash(self, hasher: T.Any):
-        myenv = self.get_env({})
-        for key in sorted(myenv.keys()):
-            hasher.update(bytes(key, encoding='utf-8'))
-            hasher.update(b',')
-            hasher.update(bytes(myenv[key], encoding='utf-8'))
-            hasher.update(b';')
-
-    def has_name(self, name: str) -> bool:
-        return name in self.varnames
-
-    def get_names(self) -> T.Set[str]:
-        return self.varnames
-
-    def set(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._set, name, values, separator))
-
-    def append(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._append, name, values, separator))
-
-    def prepend(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
-        self.varnames.add(name)
-        self.envvars.append((self._prepend, name, values, separator))
-
-    @staticmethod
-    def _set(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        return separator.join(values)
-
-    @staticmethod
-    def _append(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        curr = env.get(name)
-        return separator.join(values if curr is None else [curr] + values)
-
-    @staticmethod
-    def _prepend(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
-        curr = env.get(name)
-        return separator.join(values if curr is None else values + [curr])
-
-    def get_env(self, full_env: T.MutableMapping[str, str]) -> T.Dict[str, str]:
-        env = full_env.copy()
-        for method, name, values, separator in self.envvars:
-            env[name] = method(env, name, values, separator)
-        return env
-
 @dataclass(eq=False)
 class Target(HoldableObject):
 

mesonbuild/dependencies/dub.py

@@ -14,8 +14,7 @@
 
 from .base import ExternalDependency, DependencyException, DependencyTypeName
 from .pkgconfig import PkgConfigDependency
-from ..mesonlib import (Popen_safe, OptionKey)
-from ..mesonlib.universal import join_args
+from ..mesonlib import (Popen_safe, OptionKey, join_args)
 from ..programs import ExternalProgram
 from .. import mlog
 import re

mesonbuild/interpreter/type_checking.py

@@ -8,13 +8,15 @@
 import typing as T
 
 from .. import compilers
-from ..build import (EnvironmentVariables, EnvInitValueType, CustomTarget, BuildTarget,
+from ..build import (CustomTarget, BuildTarget,
                      CustomTargetIndex, ExtractedObjects, GeneratedList, IncludeDirs,
                      BothLibraries, SharedLibrary, StaticLibrary, Jar, Executable)
 from ..coredata import UserFeatureOption
 from ..dependencies import Dependency, InternalDependency
 from ..interpreterbase.decorators import KwargInfo, ContainerTypeInfo
-from ..mesonlib import File, FileMode, MachineChoice, listify, has_path_sep, OptionKey
+from ..mesonlib import (
+    File, FileMode, MachineChoice, listify, has_path_sep, OptionKey,
+    EnvInitValueType, EnvironmentVariables)
 from ..programs import ExternalProgram
 
 # Helper definition for type checks that are `Optional[T]`

mesonbuild/mesonlib/__init__.py mesonbuild/mesonlib.py

@@ -14,19 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: skip-file
 
 """Helper functions and classes."""
 
 import os
 
-from .universal import *
-from .vsenv import setup_vsenv
+from .utils.core import *
+from .utils.vsenv import *
+
+from .utils.universal import *
 
 # Here we import either the posix implementations, the windows implementations,
 # or a generic no-op implementation
 if os.name == 'posix':
-    from .posix import *
+    from .utils.posix import *
 elif os.name == 'nt':
-    from .win32 import *
+    from .utils.win32 import *
 else:
-    from .platform import *
+    from .utils.platform import *

mesonbuild/mesonmain.py

@@ -18,18 +18,18 @@
 import sys
 sys.modules['pathlib'] = _pathlib
 
+# This file is an entry point for all commands, including scripts. Include the
+# strict minimum python modules for performance reasons.
 import os.path
 import platform
 import importlib
-import traceback
 import argparse
-import shutil
 
-from . import mesonlib
+from .utils.core import MesonException, MesonBugException
 from . import mlog
-from .mesonlib import MesonException, MesonBugException
 
 def errorhandler(e, command):
+    import traceback
     if isinstance(e, MesonException):
         mlog.exception(e)
         logfile = mlog.shutdown()
@@ -72,6 +72,7 @@ def __init__(self):
         from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects, munstable_coredata, mcompile, mdevenv
         from .scripts import env2mfile
         from .wrap import wraptool
+        import shutil
 
         self.term_width = shutil.get_terminal_size().columns
         self.formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=int(self.term_width / 2), width=self.term_width)
@@ -176,6 +177,7 @@ def run(self, args):
             parser = self.parser
             command = None
 
+        from . import mesonlib
         args = mesonlib.expand_arguments(args)
         options = parser.parse_args(args)
 
@@ -228,6 +230,11 @@ def ensure_stdout_accepts_unicode():
     if sys.stdout.encoding and not sys.stdout.encoding.upper().startswith('UTF-'):
         sys.stdout.reconfigure(errors='surrogateescape')
 
+def set_meson_command(mainfile):
+    # Set the meson command that will be used to run scripts and so on
+    from . import mesonlib
+    mesonlib.set_meson_command(mainfile)
+
 def run(original_args, mainfile):
     if sys.version_info >= (3, 10) and os.environ.get('MESON_RUNNING_IN_PROJECT_TESTS'):
         # workaround for https://bugs.python.org/issue34624
@@ -245,15 +252,13 @@ def run(original_args, mainfile):
         mlog.error('Please install and use mingw-w64-x86_64-python3 and/or mingw-w64-x86_64-meson with Pacman')
         return 2
 
-    # Set the meson command that will be used to run scripts and so on
-    mesonlib.set_meson_command(mainfile)
-
     args = original_args[:]
 
     # Special handling of internal commands called from backends, they don't
     # need to go through argparse.
     if len(args) >= 2 and args[0] == '--internal':
         if args[1] == 'regenerate':
+            set_meson_command(mainfile)
             from . import msetup
             try:
                 return msetup.run(['--reconfigure'] + args[2:])
@@ -262,6 +267,7 @@ def run(original_args, mainfile):
         else:
             return run_script_command(args[1], args[2:])
 
+    set_meson_command(mainfile)
     return CommandLineParser().run(args)
 
 def main():

mesonbuild/scripts/meson_exe.py

@@ -20,8 +20,7 @@
 import typing as T
 import locale
 
-from .. import mesonlib
-from ..backend.backends import ExecutableSerialisation
+from ..utils.core import ExecutableSerialisation
 
 def buildparser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?')
@@ -46,7 +45,8 @@ def run_exe(exe: ExecutableSerialisation, extra_env: T.Optional[T.Dict[str, str]
     if exe.extra_paths:
         child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) +
                              child_env['PATH'])
-        if exe.exe_wrapper and mesonlib.substring_is_in_list('wine', exe.exe_wrapper.get_command()):
+        if exe.exe_wrapper and any('wine' in i for i in exe.exe_wrapper.get_command()):
+            from .. import mesonlib
             child_env['WINEPATH'] = mesonlib.get_wine_shortpath(
                 exe.exe_wrapper.get_command(),
                 ['Z:' + p for p in exe.extra_paths] + child_env.get('WINEPATH', '').split(';'),

mesonbuild/scripts/test_loaded_modules.py

@@ -0,0 +1,11 @@
+import sys
+import json
+import typing as T
+from . import meson_exe
+
+# This script is used by run_unittests.py to verify we don't load too many
+# modules when executing a wrapped command.
+def run(args: T.List[str]) -> int:
+    meson_exe.run(args)
+    print(json.dumps(list(sys.modules.keys())))
+    return 0