From dd734a940af5ddd077ddb2d092117895993f72e0 Mon Sep 17 00:00:00 2001 From: Meagan Lang Date: Wed, 22 May 2024 12:55:42 -0400 Subject: [PATCH] Ignore installation requriements for now Allow for environment variables at the tool level --- tests/test_tools.py | 5 + utils/setup_test_env.py | 3 + yggdrasil/drivers/CModelDriver.py | 66 +---------- yggdrasil/drivers/CompiledModelDriver.py | 137 ++++++++++++++++++----- yggdrasil/tools.py | 38 +++---- 5 files changed, 136 insertions(+), 113 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index 08daa868d..c44d14a40 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -197,12 +197,17 @@ def test_locate_file(): r"""Test file location method.""" # Missing file assert not tools.locate_file('missing_file.fake') + assert not tools.locate_file('missing_file.fake', use_os=True) assert not tools.locate_file(['missing_file.fake']) # Single file sdir, spat, sans = make_temp_single() sout = tools.locate_file(spat, verification_func=os.path.isfile) assert isinstance(sout, (bytes, str)) assert sout == sans[0] + sout = tools.locate_file(spat, verification_func=os.path.isfile, + use_os=True) + assert isinstance(sout, (bytes, str)) + assert sout == sans[0] # Multiple files mdir, mpat, mans = make_temp_multiple() with pytest.warns(RuntimeWarning): diff --git a/utils/setup_test_env.py b/utils/setup_test_env.py index b02d77f30..663bf3ed4 100644 --- a/utils/setup_test_env.py +++ b/utils/setup_test_env.py @@ -1805,6 +1805,9 @@ def verify_pkg(install_opts=None): from yggdrasil.tools import is_lang_installed, is_comm_installed errors = [] for name in ['c', 'r', 'fortran', 'sbml', 'lpy', 'julia']: + # TODO: temp + if name in ['r', 'julia']: + continue flag = install_opts[name] if flag and (not is_lang_installed(name)): errors.append("Language '%s' should be installed, but is not." diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py index fbc2d73ed..d5df7f9cc 100755 --- a/yggdrasil/drivers/CModelDriver.py +++ b/yggdrasil/drivers/CModelDriver.py @@ -9,9 +9,9 @@ from collections import OrderedDict from yggdrasil import platform, tools, constants, rapidjson from yggdrasil.drivers.CompiledModelDriver import ( - CompiledModelDriver, CompilerBase, LinkerBase, ArchiverBase) + CompiledModelDriver, CompilerBase, LinkerBase, ArchiverBase, + _osx_sysroot) from yggdrasil.languages import get_language_dir -from yggdrasil.config import ygg_cfg logger = logging.getLogger(__name__) if platform._is_win: logger.setLevel(level=logging.DEBUG) @@ -21,48 +21,6 @@ _top_lang_dir = get_language_dir('c') -def get_OSX_SYSROOT(): - r"""Determin the path to the OSX SDK. - - Returns: - str: Full path to the SDK directory if one is located. None - otherwise. - - """ - fname = None - if platform._is_mac: - try: - xcode_dir = subprocess.check_output( - 'echo "$(xcode-select -p)"', shell=True).decode("utf-8").strip() - except BaseException: # pragma: debug - xcode_dir = None - fname_try = [] - cfg_sdkroot = ygg_cfg.get('c', 'macos_sdkroot', None) - if cfg_sdkroot: - fname_try.append(cfg_sdkroot) - if os.environ.get('SDKROOT', False): - fname_try.append(os.environ['SDKROOT']) - if xcode_dir is not None: - bases_try = [ - os.path.join(xcode_dir, 'SDKs', 'MacOSX%s.sdk'), - os.path.join(xcode_dir, 'Platforms', - 'MacOSX.platform', 'Developer', - 'SDKs', 'MacOSX%s.sdk')] - vers_try = ['11.0', ''] # 11.0 used by conda-forge - if os.environ.get('MACOSX_DEPLOYMENT_TARGET', False): - vers_try.insert(0, os.environ['MACOSX_DEPLOYMENT_TARGET']) - for v in vers_try: - fname_try += [x % v for x in bases_try] - for fcheck in fname_try: - if os.path.isdir(fcheck): - fname = fcheck - break - return fname - - -_osx_sysroot = get_OSX_SYSROOT() - - class CCompilerBase(CompilerBase): r"""Base class for C compilers.""" languages = ['c'] @@ -502,26 +460,6 @@ class CModelDriver(CompiledModelDriver): 'compiler_flags': ['-fPIC'], 'external_dependencies': ['m'], }, - 'MacOS': { - 'global_env': { - 'CONDA_BUILD_SYSROOT': { - 'value': _osx_sysroot, - 'overwrite': True, - }, - 'SDKROOT': { - 'value': _osx_sysroot, - 'overwrite': True, - }, - 'MACOSX_DEPLOYMENT_TARGET': { - 'value': ( - re.search( - r'MacOSX(?P[0-9]+\.[0-9]+)?', - _osx_sysroot).groupdict()['target'] - if _osx_sysroot else False), - 'overwrite': True, - } - } - } }}, 'regex_win32': {'name': 'regex', 'source': 'regex_win32.cpp', diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index 44d502a1e..8db6e816f 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -42,6 +42,49 @@ _library_types = ['include', 'static', 'shared', 'windows_import'] +def get_OSX_SYSROOT(): + r"""Determin the path to the OSX SDK. + + Returns: + str: Full path to the SDK directory if one is located. None + otherwise. + + """ + fname = None + if platform._is_mac: + from yggdrasil.config import ygg_cfg + try: + xcode_dir = subprocess.check_output( + 'echo "$(xcode-select -p)"', shell=True).decode("utf-8").strip() + except BaseException: # pragma: debug + xcode_dir = None + fname_try = [] + cfg_sdkroot = ygg_cfg.get('c', 'macos_sdkroot', None) + if cfg_sdkroot: + fname_try.append(cfg_sdkroot) + if os.environ.get('SDKROOT', False): + fname_try.append(os.environ['SDKROOT']) + if xcode_dir is not None: + bases_try = [ + os.path.join(xcode_dir, 'SDKs', 'MacOSX%s.sdk'), + os.path.join(xcode_dir, 'Platforms', + 'MacOSX.platform', 'Developer', + 'SDKs', 'MacOSX%s.sdk')] + vers_try = ['11.0', ''] # 11.0 used by conda-forge + if os.environ.get('MACOSX_DEPLOYMENT_TARGET', False): + vers_try.insert(0, os.environ['MACOSX_DEPLOYMENT_TARGET']) + for v in vers_try: + fname_try += [x % v for x in bases_try] + for fcheck in fname_try: + if os.path.isdir(fcheck): + fname = fcheck + break + return fname + + +_osx_sysroot = get_OSX_SYSROOT() + + class CompilationToolError(Exception): r"""Class for errors related to compilation tools""" pass @@ -3460,6 +3503,23 @@ def _shared_path_env(self, to_update=None, out=None, out[env_var] = os.pathsep.join(path_list) return out + @classmethod + def _update_env_val(cls, k, v, out, to_update=None): + if to_update is None: + to_update = out + append_char = None + overwrite = False + if isinstance(v, dict): + append_char = v.get('append', None) + overwrite = v.get('overwrite', False) + v = v['value'] + if not v: + return + if overwrite or k not in to_update: + out[k] = v + elif append_char: + out[k] = to_update[k] + append_char + v + def _update_env(self, vals, to_update=None, out=None, dont_update_return=False): if to_update is None: @@ -3472,18 +3532,7 @@ def _update_env(self, vals, to_update=None, out=None, else: to_update.update(out) for k, v in vals.items(): - append_char = None - overwrite = False - if isinstance(v, dict): - append_char = v.get('append', None) - overwrite = v.get('overwrite', False) - v = v['value'] - if not v: - continue - if overwrite or k not in to_update: - out[k] = v - elif append_char: - out[k] = to_update[k] + append_char + v + self._update_env_val(k, v, out, to_update=to_update) return out def _generic_env(self, key, to_update=None, for_build=False, @@ -3584,12 +3633,16 @@ def _search_linked(self, filetype, **kwargs): if not flags: if self.origin == 'standard': flags.append(f'-l{self.name}') - for lib in self.basetool.find_component( - self.name, cfg=self.cfg, flags=flags, - component_types='shared_libraries', **kwargs): - out = self._search_brute(lib, libtype='shared', **kwargs) - if out: - break + try: + for lib in self.basetool.find_component( + self.name, cfg=self.cfg, flags=flags, + component_types='shared_libraries', **kwargs): + out = self._search_brute(lib, libtype='shared', **kwargs) + if out: + break + except RuntimeError as e: + logger.debug(f"Error in using diassembly to locate " + f"\'{self.name}\': {e}") return out def _search_brute(self, fname, libtype=None, verbose=False, @@ -3635,7 +3688,7 @@ def _search_brute(self, fname, libtype=None, verbose=False, fname_ext = expected_ext[0] if os.path.isfile(fname): return fname - use_regex = True # (not platform._is_win) + # use_regex = (not platform._is_win) fname_try = [fname_base] if platform._is_win and libtype in self.library_files: if fname_base.startswith('lib'): @@ -3645,19 +3698,15 @@ def _search_brute(self, fname, libtype=None, verbose=False, search_list = self.tool(libtype).get_search_path( libtype=libtype, cfg=self.cfg, **kwargs) for fname_base in fname_try: - use_glob = fname_base + '*' + fname_ext - if use_regex: - fname = ( - r'[^a-zA-Z]' - + tools.escape_regex(fname_base) - + r'([^a-zA-Z].*)?' - + tools.escape_regex(fname_ext)) - else: - fname = use_glob + fname = fname_base + '*' + fname_ext + use_regex = ( + r'[^a-zA-Z]' + + tools.escape_regex(fname_base) + + r'([^a-zA-Z].*)?' + + tools.escape_regex(fname_ext)) out = tools.locate_file(fname, directory_list=search_list, environment_variable=None, use_regex=use_regex, - use_glob=use_glob, select_return='shortest') if out: break @@ -3715,6 +3764,11 @@ class CompilationToolBase(object): [REQUIRED] platforms (list): Platforms that the tool is available on. Defaults to ['Windows', 'MacOS', 'Linux']. + env (dict): Environment variables that should be updated when + calling the tool. + env_platform_specific (dict): Mapping of platform specific + environment variables that should be updated when calling the + tool. default_executable (str): The default tool executable command if different than the toolname. default_executable_env (str): Environment variable where the executable @@ -3824,6 +3878,27 @@ class CompilationToolBase(object): output_filetypes = [] languages = [] platforms = ['Windows', 'MacOS', 'Linux'] # all by default + env = {} + env_platform_specific = { + 'MacOS': { + 'CONDA_BUILD_SYSROOT': { + 'value': _osx_sysroot, + 'overwrite': True, + }, + 'SDKROOT': { + 'value': _osx_sysroot, + 'overwrite': True, + }, + 'MACOSX_DEPLOYMENT_TARGET': { + 'value': ( + re.search( + r'MacOSX(?P[0-9]+\.[0-9]+)?', + _osx_sysroot).groupdict()['target'] + if _osx_sysroot else False), + 'overwrite': True, + }, + } + } default_executable = None default_executable_env = None default_flags = [] @@ -4138,6 +4213,10 @@ def set_env(cls, existing=None, **kwargs): if existing is None: existing = {} existing.update(os.environ) + for k, v in dict(cls.env, **cls.env_platform_specific.get( + platform._platform, {})).items(): + CompilationDependency._update_env_val(k, v, existing, + to_update=existing) if not cls.env_matches_tool(): config_vars = {} cls.env_matches_tool(use_sysconfig=True, env=config_vars) diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py index 42efd3186..4d0415ae2 100644 --- a/yggdrasil/tools.py +++ b/yggdrasil/tools.py @@ -619,7 +619,7 @@ def convert_regex_to_find(name): def find_all(name, path, verification_func=None, use_regex=False, - use_glob=False): + use_os=False): r"""Find all instances of a file with a given name within the directory tree starting at a given path. @@ -631,11 +631,12 @@ def find_all(name, path, verification_func=None, use_regex=False, verification_func (function, optional): Function that returns True when a file is valid and should be returned and False otherwise. Defaults to None and is ignored. - use_regex (bool, optional): If True, use full regex to interpret - name and locate files. - use_glob (bool, str, optional): If True or string, use glob - to locate the file. If use_regex is True, a string must be - provided containing the glob search expression. + use_regex (bool, str, optional): If True or string, use full + regex to interpret name and locate files. If a string is + provided, it will be used as the regex pattern, otherwise + name will be used. A string is required unless use_os is True + use_os (bool, optional): If True, use a system tool to search + for the file (find on unix, where on windows). Returns: list: All instances of the specified file. @@ -643,17 +644,9 @@ def find_all(name, path, verification_func=None, use_regex=False, """ result = [] args = [] - if use_glob: - fglob = os.path.join(path, name) - if use_regex: - assert isinstance(use_glob, str) - fglob = os.path.join(path, use_glob) - result = glob.glob(fglob) - if use_regex: - result = [ - x for x in result - if re.fullmatch(r'.*' + name, x)] - else: + regex_pattern = use_regex if isinstance(use_regex, str) else name + regex_pattern = r'.*' + regex_pattern # to match the directory + if use_os: try: if platform._is_win: # pragma: windows assert not use_regex @@ -672,8 +665,8 @@ def find_all(name, path, verification_func=None, use_regex=False, shell = False args = ["find", "-L", path, "-type", "f"] if use_regex: - name = convert_regex_to_find(name) - args += ["-regex", r'.*' + name] + regex_pattern = convert_regex_to_find(regex_pattern) + args += ["-regex", regex_pattern] args.insert(1, "-E") args = ' '.join(args) shell = True @@ -697,6 +690,11 @@ def find_all(name, path, verification_func=None, use_regex=False, out = '' if not out.isspace(): result = sorted(out.splitlines()) + else: + result = glob.glob(os.path.join(path, name)) + if use_regex: + result = [ + x for x in result if re.fullmatch(regex_pattern, x)] result = [os.path.normcase(os.path.normpath(bytes2str(m))) for m in result] if verification_func is not None: @@ -752,7 +750,7 @@ def locate_file(fname, environment_variable='PATH', directory_list=None, out = [] if ((platform._is_win and (environment_variable == 'PATH') and (directory_list is None) - and not kwargs.get('use_glob', False))): # pragma: windows + and kwargs.get('use_os', False))): # pragma: windows out += find_all(fname, None, **kwargs) else: if directory_list is None: