From 8cbf219eea6422349f3c5b74fc3a6d45920d589e Mon Sep 17 00:00:00 2001 From: Jean-Christophe Morin Date: Sat, 15 Feb 2025 18:00:42 -0500 Subject: [PATCH] Attempt at decuding the possibility of having import cycles when user configs import rez Signed-off-by: Jean-Christophe Morin --- src/rez/package_order.py | 2 +- src/rez/package_repository.py | 2 +- src/rez/packages.py | 20 +++++++++++++++-- src/rez/plugin_managers.py | 4 +++- src/rez/serialise.py | 41 ++++++++++++++++++++++++----------- src/rez/utils/memcached.py | 12 +++++++--- 6 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 706c5f0ca..89af4b74a 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -6,7 +6,6 @@ from hashlib import sha1 from typing import Dict, Iterable, List, Optional, Union -from rez.config import config from rez.utils.data_utils import cached_class_property from rez.version import Version, VersionRange from rez.version._version import _Comparable, _ReversedComparable, _LowerBound, _UpperBound, _Bound @@ -637,6 +636,7 @@ def from_pod(cls, data): @cached_class_property def singleton(cls): """Filter list as configured by rezconfig.package_filter.""" + from rez.config import config return cls.from_pod(config.package_orderers) @staticmethod diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index 2c7be5ab2..765f455a9 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -5,7 +5,6 @@ from rez.utils.resources import ResourcePool, ResourceHandle from rez.utils.data_utils import cached_property from rez.plugin_managers import plugin_manager -from rez.config import config from rez.exceptions import ResourceError from contextlib import contextmanager import threading @@ -525,6 +524,7 @@ def __init__(self, resource_pool=None): None, a default pool is created based on config settings. """ if resource_pool is None: + from rez.config import config cache_size = config.resource_caching_maxsize if cache_size < 0: # -1 == disable caching cache_size = None diff --git a/src/rez/packages.py b/src/rez/packages.py index dc816303b..d20897520 100644 --- a/src/rez/packages.py +++ b/src/rez/packages.py @@ -2,7 +2,6 @@ # Copyright Contributors to the Rez Project -from rez.package_repository import package_repository_manager from rez.package_resources import PackageFamilyResource, PackageResource, \ VariantResource, package_family_schema, package_schema, variant_schema, \ package_release_keys, late_requires_schema @@ -17,7 +16,6 @@ from rez.version import Version, VersionRange from rez.version import VersionedObject from rez.serialise import FileFormat -from rez.config import config import os import sys @@ -99,11 +97,13 @@ def config(self): Defaults to global config if this package did not provide a 'config' section. """ + from rez.config import config return self.resource.config or config @cached_property def is_local(self): """Returns True if the package is in the local package repository""" + from rez.package_repository import package_repository_manager local_repo = package_repository_manager.get_repository( self.config.local_packages_path) return (self.resource._repository.uid == local_repo.uid) @@ -262,6 +262,7 @@ def is_relocatable(self): if self.relocatable is not None: return self.relocatable + from rez.config import config if config.default_relocatable_per_repository: value = config.default_relocatable_per_repository.get( self.repository.location) @@ -282,6 +283,7 @@ def is_cachable(self): if self.cachable is not None: return self.cachable + from rez.config import config if config.default_cachable_per_repository: # TODO: The location of filesystem repository is canonical path, # so if the path in `default_cachable_per_repository` isn't @@ -451,6 +453,7 @@ def install(self, path, dry_run=False, overrides=None): `Variant` object - the (existing or newly created) variant in the specified repository. If `dry_run` is True, None may be returned. """ + from rez.package_repository import package_repository_manager repo = package_repository_manager.get_repository(path) resource = repo.install_variant(self.resource, dry_run=dry_run, @@ -507,6 +510,7 @@ def __contains__(self, package): @cached_property def _repository_uids(self): + from rez.package_repository import package_repository_manager uids = set() for path in self.paths: repo = package_repository_manager.get_repository(path) @@ -532,6 +536,8 @@ def iter_package_families(paths=None): Returns: `PackageFamily` iterator. """ + from rez.config import config + from rez.package_repository import package_repository_manager for path in (paths or config.packages_path): repo = package_repository_manager.get_repository(path) for resource in repo.iter_package_families(): @@ -607,6 +613,7 @@ def get_package_family_from_repository(name, path): Returns: `PackageFamily` object, or None if the family was not found. """ + from rez.package_repository import package_repository_manager repo = package_repository_manager.get_repository(path) family_resource = repo.get_package_family(name) @@ -626,6 +633,7 @@ def get_package_from_repository(name, version, path): Returns: `Package` object, or None if the package was not found. """ + from rez.package_repository import package_repository_manager repo = package_repository_manager.get_repository(path) if isinstance(version, str): @@ -649,6 +657,7 @@ def get_package_from_handle(package_handle): Returns: `Package`. """ + from rez.package_repository import package_repository_manager if isinstance(package_handle, dict): package_handle = ResourceHandle.from_dict(package_handle) package_resource = package_repository_manager.get_resource_from_handle(package_handle) @@ -713,6 +722,7 @@ def get_variant(variant_handle, context=None): Returns: `Variant`. """ + from rez.package_repository import package_repository_manager if isinstance(variant_handle, dict): variant_handle = ResourceHandle.from_dict(variant_handle) @@ -733,6 +743,7 @@ def get_package_from_uri(uri, paths=None): Returns: `Package`, or None if the package could not be found. """ + from rez.package_repository import package_repository_manager def _find_in_path(path): repo = package_repository_manager.get_repository(path) pkg_resource = repo.get_package_from_uri(uri) @@ -741,6 +752,7 @@ def _find_in_path(path): else: return None + from rez.config import config for path in (paths or config.packages_path): pkg = _find_in_path(path) if pkg is not None: @@ -780,6 +792,7 @@ def get_variant_from_uri(uri, paths=None): Returns: `Variant`, or None if the variant could not be found. """ + from rez.package_repository import package_repository_manager def _find_in_path(path): repo = package_repository_manager.get_repository(path) variant_resource = repo.get_variant_from_uri(uri) @@ -788,6 +801,7 @@ def _find_in_path(path): else: return None + from rez.config import config for path in (paths or config.packages_path): variant = _find_in_path(path) if variant is not None: @@ -951,6 +965,8 @@ def get_latest_package_from_string(txt, paths=None, error=False): def _get_families(name, paths=None): entries = [] + from rez.config import config + from rez.package_repository import package_repository_manager for path in (paths or config.packages_path): repo = package_repository_manager.get_repository(path) family_resource = repo.get_package_family(name) diff --git a/src/rez/plugin_managers.py b/src/rez/plugin_managers.py index 64b1d8f09..4b03dfed0 100644 --- a/src/rez/plugin_managers.py +++ b/src/rez/plugin_managers.py @@ -5,7 +5,6 @@ """ Manages loading of all types of Rez plugins. """ -from rez.config import config, expand_system_vars, _load_config_from_filepaths from rez.utils.formatting import columnise from rez.utils.schema import dict_to_schema from rez.utils.data_utils import LazySingleton, cached_property, deep_update @@ -54,6 +53,7 @@ def extend_path(path, name): init_py = "__init__" + os.extsep + "py" path = path[:] + from rez.config import config def append_if_valid(dir_): if os.path.isdir(dir_): subdir = os.path.normcase(os.path.join(dir_, pname)) @@ -126,6 +126,7 @@ def load_plugins(self): # be found before the builtin plugins (from /rezplugins). paths = reversed(paths) + from rez.config import config, _load_config_from_filepaths for path in paths: if config.debug("plugins"): print_debug("searching plugin path %s...", path) @@ -226,6 +227,7 @@ def config_schema(self): """Returns the merged configuration data schema for this plugin type.""" from rez.config import _plugin_config_dict + from rez.config import expand_system_vars d = _plugin_config_dict.get(self.type_name, {}) for name, plugin_class in self.plugin_classes.items(): diff --git a/src/rez/serialise.py b/src/rez/serialise.py index dac33a38d..e0f05e542 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -24,13 +24,10 @@ from rez.utils.memcached import memcached from rez.utils.execution import add_sys_paths from rez.util import get_function_arg_names -from rez.config import config from rez.vendor.atomicwrites import atomic_write from rez.vendor import yaml -tmpdir_manager = TempDirs(config.tmpdir, prefix="rez_write_") -debug_print = config.debug_printer("file_loads") file_cache = {} @@ -60,14 +57,19 @@ def open_file_for_write(filepath, mode=None): Yields: File-like object. """ + from rez.config import config + stream = StringIO() yield stream content = stream.getvalue() filepath = os.path.realpath(filepath) + tmpdir_manager = TempDirs(config.tmpdir, prefix="rez_write_") + tmpdir = tmpdir_manager.mkdtemp() cache_filepath = os.path.join(tmpdir, os.path.basename(filepath)) + debug_print = config.debug_printer("file_loads") debug_print("Writing to %s (local cache of %s)", cache_filepath, filepath) # Attempt to make file writable if it isn't already. Just fallthrough @@ -136,9 +138,11 @@ def load_from_file(filepath, format_=FileFormat.py, update_data_callback=None, format_=format_, update_data_callback=update_data_callback) else: - return _load_from_file(filepath=filepath, - format_=format_, - update_data_callback=update_data_callback) + return _load_from_file()( + filepath=filepath, + format_=format_, + update_data_callback=update_data_callback + ) def _load_from_file__key(filepath, format_, update_data_callback): @@ -152,17 +156,28 @@ def _load_from_file__key(filepath, format_, update_data_callback): int(st.st_ino), st.st_mtime)) -@memcached(servers=config.memcached_uri if config.cache_package_files else None, - min_compress_len=config.memcached_package_file_min_compress_len, - key=_load_from_file__key, - debug=config.debug_memcache) -def _load_from_file(filepath, format_, update_data_callback): - return _load_file(filepath, format_, update_data_callback) +def _load_from_file(): + from rez.config import config + + # A lot of gymnastics to avoid habing to import rez.config at the top level. + @memcached( + servers=config.memcached_uri if config.cache_package_files else None, + min_compress_len=config.memcached_package_file_min_compress_len, + key=_load_from_file__key, + debug=config.debug_memcache + ) + def _load_from_file_inner(filepath, format_, update_data_callback): + return _load_file(filepath, format_, update_data_callback) + + return _load_from_file def _load_file(filepath, format_, update_data_callback, original_filepath=None): + from rez.config import config + load_func = load_functions[format_] + debug_print = config.debug_printer("file_loads") if debug_print: if original_filepath: debug_print("Loading file: %s (local cache of %s)", @@ -441,7 +456,7 @@ def load_txt(stream, **kwargs): def clear_file_caches(): """Clear any cached files.""" - _load_from_file.forget() + _load_from_file().forget() load_functions = {FileFormat.py: load_py, diff --git a/src/rez/utils/memcached.py b/src/rez/utils/memcached.py index 000d4d262..432668ec5 100644 --- a/src/rez/utils/memcached.py +++ b/src/rez/utils/memcached.py @@ -2,7 +2,6 @@ # Copyright Contributors to the Rez Project -from rez.config import config from rez.vendor.memcache.memcache import Client as Client_, \ SERVER_MAX_KEY_LENGTH, __version__ as memcache_client_version from rez.util import get_function_arg_names @@ -32,7 +31,6 @@ def __bool__(self): miss = _Miss() - logger = config.debug_printer("memcache") def __init__(self, servers, debug=False): """Create a memcached client. @@ -48,6 +46,8 @@ def __init__(self, servers, debug=False): self._client = None self.debug = debug self.current = '' + from rez.config import config + self.logger = config.debug_printer("memcache") def __bool__(self): return bool(self.servers) @@ -223,7 +223,7 @@ def release(self, key): @contextmanager -def memcached_client(servers=config.memcached_uri, debug=config.debug_memcache): +def memcached_client(servers=None, debug=None): """Get a shared memcached instance. This function shares the same memcached instance across nested invocations. @@ -236,6 +236,12 @@ def memcached_client(servers=config.memcached_uri, debug=config.debug_memcache): Returns: `Client`: Memcached instance. """ + from rez.config import config + if servers is None: + servers = config.memcached_uri + if debug is None: + debug = config.debug_memcache + key = None try: client, key = scoped_instance_manager.acquire(servers, debug=debug)