From 9ac53e083d41673a8afbb58d50b23e36efa78f8e Mon Sep 17 00:00:00 2001 From: Meagan Lang Date: Thu, 23 May 2024 15:55:42 -0400 Subject: [PATCH] Change how tools are checked against the environment to use the version regexes --- tests/drivers/test_CompiledModelDriver.py | 13 +- yggdrasil/drivers/CModelDriver.py | 45 ++++--- yggdrasil/drivers/CPPModelDriver.py | 8 +- yggdrasil/drivers/CompiledModelDriver.py | 155 ++++++++++++++-------- yggdrasil/drivers/FortranModelDriver.py | 1 + yggdrasil/tools.py | 2 +- 6 files changed, 138 insertions(+), 86 deletions(-) diff --git a/tests/drivers/test_CompiledModelDriver.py b/tests/drivers/test_CompiledModelDriver.py index 0f23a20ef..cb91b1398 100644 --- a/tests/drivers/test_CompiledModelDriver.py +++ b/tests/drivers/test_CompiledModelDriver.py @@ -377,15 +377,10 @@ def test_executable_command(self, python_class): basetool.get_default_libtype(), None) if not next_tool: return - if basetool.no_separate_next_stage: - with pytest.raises(RuntimeError): - python_class.executable_command(['test'], - exec_type=next_tool) - else: - python_class.executable_command(['test'], - no_additional_stages=True) - python_class.executable_command(['test'], - exec_type=next_tool) + python_class.executable_command(['test'], + no_additional_stages=True) + python_class.executable_command(['test'], + exec_type=next_tool) def test_basetool_call(self, python_class): r"""Test basetool call.""" diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py index 14517cb9d..c41a949f7 100755 --- a/yggdrasil/drivers/CModelDriver.py +++ b/yggdrasil/drivers/CModelDriver.py @@ -91,8 +91,11 @@ class GCCCompiler(CCompilerBase): default_disassembler = 'objdump' toolset = 'gnu' aliases = ['gnu-cc', 'gnu-gcc'] - # standard_library = 'c' compatible_toolsets = ['llvm'] + next_stage_flag_flag = '-Xlinker' + version_regex = [ + r'(?P(?:.*gnu\-)?g?cc \(.+\) \d+\.\d+\.\d+)'] + # standard_library = 'c' libraries = { 'asan': {'dep_executable_flags': ['-fsanitize=address'], 'dep_shared_flags': ['-fsanitize=address'], @@ -111,13 +114,11 @@ def is_clang(cls): bool: True if gcc actually points to clang. """ - if platform._is_mac: - try: - ver = cls.tool_version() - return ('clang' in ver) - except InvalidCompilationTool: - pass - return False + try: + return (platform._is_mac + and 'clang' in cls.tool_version(skip_regex=True)) + except InvalidCompilationTool: + return False @classmethod def is_installed(cls): @@ -150,19 +151,18 @@ class ClangCompiler(CCompilerBase): default_linker = 'clang' default_archiver = 'libtool' default_disassembler = 'otool' + toolset = 'llvm' + compatible_toolsets = ['gnu'] + next_stage_flag_flag = '-Xlinker' + version_regex = [ + r'(?P(?:Apple )?clang version \d+\.\d+\.\d+)'] flag_options = OrderedDict(list(CCompilerBase.flag_options.items()) + [('sysroot', '--sysroot'), ('isysroot', {'key': '-isysroot', 'prepend': True}), ('mmacosx-version-min', '-mmacosx-version-min=%s')]) - version_regex = [ - r'(?P(?:Apple )?clang version \d+\.\d+\.\d+)'] product_exts = ['.dSYM'] - # Set to False since ClangLinker has its own class to handle - # conflict between versions of clang and ld. - toolset = 'llvm' - compatible_toolsets = ['gnu'] libraries = { 'asan': {'dep_executable_flags': ['-fsanitize=address'], 'dep_shared_flags': ['-fsanitize=address', @@ -225,7 +225,7 @@ class MSVCCompiler(CCompilerBase): search_path_envvar = ['INCLUDE'] search_path_flags = None version_flags = [] - version_regex = r'(?P.+)\s+Copyright' + version_regex = [r'(?P.+)\s+Copyright'] product_exts = ['.dir', '.ilk', '.pdb', '.sln', '.vcxproj', '.vcxproj.filters', '.exp', '.lib'] builtin_next_stage = 'linker' @@ -240,7 +240,7 @@ class LDLinker(LinkerBase): # Languages disabled for ld by default to prevent it being # selected instead of the default which seems to be happening # on the CI - languages = ['c'] # ['c', 'c++', 'fortran'] + languages = ['c', 'c++', 'fortran'] default_executable_env = 'LD' default_flags_env = 'LDFLAGS' version_flags = ['-v'] @@ -250,6 +250,10 @@ class LDLinker(LinkerBase): r'(?P\d+(?:\.\d+){0,2})') ] search_path_envvar = ['LIBRARY_PATH', 'LD_LIBRARY_PATH'] + compatible_toolsets = ['gnu', 'llvm'] + preload_envvar = ('DYLD_INSERT_LIBRARIES' if platform._is_mac + else ('LD_PRELOAD' if platform._is_linux + else None)) # @classmethod # def get_flags(cls, *args, **kwargs): @@ -269,12 +273,12 @@ class GCCLinker(LDLinker): default_executable = GCCCompiler.default_executable toolset = GCCCompiler.toolset compatible_toolsets = GCCCompiler.compatible_toolsets - version_regex = GCCCompiler.version_regex + version_flags = ['-Xlinker', '--verbose'] + version_regex = LDLinker.version_regex + GCCCompiler.version_regex search_path_flags = ['-Xlinker', '--verbose'] search_regex = [r'SEARCH_DIR\("=([^"]+)"\);'] flag_options = OrderedDict(LDLinker.flag_options, **{'library_rpath': '-Wl,-rpath'}) - preload_envvar = 'LD_PRELOAD' class ClangLinker(LDLinker): @@ -289,7 +293,9 @@ class ClangLinker(LDLinker): default_executable = ClangCompiler.default_executable toolset = ClangCompiler.toolset compatible_toolsets = ClangCompiler.compatible_toolsets - version_regex = ClangCompiler.version_regex + version_flags = ['-Xlinker', '-v'] + version_regex = LDLinker.version_regex + ClangCompiler.version_regex + # version_regex = ClangCompiler.version_regex search_path_flags = ['-Xlinker', '-v'] search_regex = [r'\t([^\t\n]+)\n'] search_regex_begin = 'Library search paths:' @@ -297,7 +303,6 @@ class ClangLinker(LDLinker): **{'linker-version': '-mlinker-version=%s', 'library_rpath': '-rpath', 'library_libs_nonstd': ''}) - preload_envvar = 'DYLD_INSERT_LIBRARIES' # libtype_flags = {'shared': '-dynamiclib'} @staticmethod diff --git a/yggdrasil/drivers/CPPModelDriver.py b/yggdrasil/drivers/CPPModelDriver.py index e4a158f8d..d76eaafcf 100644 --- a/yggdrasil/drivers/CPPModelDriver.py +++ b/yggdrasil/drivers/CPPModelDriver.py @@ -3,7 +3,7 @@ from yggdrasil import platform from yggdrasil.drivers.CModelDriver import ( CCompilerBase, CModelDriver, GCCCompiler, ClangCompiler, MSVCCompiler, - GCCLinker, ClangLinker, MSVCLinker) + LDLinker, GCCLinker, ClangLinker, MSVCLinker) logger = logging.getLogger(__name__) @@ -59,6 +59,8 @@ class GPPCompiler(CPPCompilerBase, GCCCompiler): toolname = 'g++' aliases = ['gnu-c++', 'gnu-g++'] default_linker = 'g++' + version_regex = [ + r'(?P(?:.*gnu\-)?(?:g|c)\+\+ \(.+\) \d+\.\d+\.\d+)'] standard_library = 'stdc++' libraries = {} @@ -97,7 +99,7 @@ def before_registration(cls): if platform._is_win: # pragma: windows cls.default_executable = 'clang' if GPPCompiler.is_clang() and 'g++' not in cls.aliases: - cls.aliases.append('g++') + cls.aliases = cls.aliases + ['g++'] CPPCompilerBase.before_registration(cls) @classmethod @@ -170,6 +172,7 @@ class GPPLinker(GCCLinker): languages = GPPCompiler.languages default_executable = GPPCompiler.default_executable toolset = GPPCompiler.toolset + version_regex = LDLinker.version_regex + GPPCompiler.version_regex standard_library = GPPCompiler.standard_library libraries = {} @@ -184,6 +187,7 @@ class ClangPPLinker(ClangLinker): languages = ClangPPCompiler.languages default_executable = ClangPPCompiler.default_executable toolset = ClangPPCompiler.toolset + version_regex = LDLinker.version_regex + ClangPPCompiler.version_regex class MSVCPPLinker(MSVCLinker): diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index cb8e92433..0d7104980 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -3895,6 +3895,8 @@ class CompilationToolBase(object): next_stage_switch (str): Flag to indicate beginning of flags that should be passed to the tool for the next stage. (e.g. /link for MSVC cl.exe). + next_stage_flag_flag (str): Flag to indicate that the next flag + should be passed to the next stage. create_next_stage_tool (dict): Parameters for a tool that should be created for the next stage based on this tool. local_kws (list): Keyword arguments that are unique to this tool. @@ -3968,11 +3970,11 @@ class CompilationToolBase(object): standard_library = None standard_library_type = None libraries = {} - no_separate_next_stage = False builtin_next_stage = None combine_with_next_stage = None no_additional_stages_flag = None next_stage_switch = None + next_stage_flag_flag = None create_next_stage_tool = None is_gnu = False toolset = None @@ -4056,7 +4058,7 @@ def before_registration(cls): assert cls.combine_with_next_stage copy_attr = ['toolname', 'aliases', 'languages', 'platforms', 'default_executable', 'default_executable_env', - 'toolset'] + 'toolset', 'version_flags', 'version_regex'] if cls.create_next_stage_tool is True: cls.create_next_stage_tool = {} stage_attr = copy.deepcopy( @@ -4473,44 +4475,32 @@ def env_matches_tool(cls, use_sysconfig=False, env=None, env.update(sysconfig.get_config_vars()) else: env.update(os.environ) - tool_base = cls.aliases.copy() - envi_base = '' envi_full = '' - if isinstance(cls.toolname, str): - tool_base.append(cls.toolname) - if isinstance(cls.default_executable, str): - tool_base.append(cls.default_executable) if isinstance(cls.default_executable_env, str): envi_full = env.get(cls.default_executable_env, '').split( 'ccache ')[-1] if envi_full: - envi_base = os.path.basename(envi_full.split(maxsplit=1)[0]) - if os.environ.get('PATHEXT', ''): - tool_base = [x.split(os.environ['PATHEXT'])[0] - for x in tool_base] - envi_base = envi_base.split(os.environ['PATHEXT'])[0] - out = None - regex_literal = '-+*$%#@!^&(){}[]<>,.;:' - regex_pathsep = r'(?:[\-\_\.0-9])' - if tool_base and envi_base: - for x in tool_base: - for k in regex_literal: - x = x.replace(k, '\\' + k) - regex = r'(?:(?:^)|%s)%s(?:(?:$)|%s)' % ( - regex_pathsep, x, regex_pathsep) - if re.search(regex, envi_base): - out = True - break - if out: - if not with_flags: - envi_full = envi_full.split(maxsplit=1)[0] - return envi_full - if tool_base and envi_base: - logger.info(f"{cls.tooltype.title()} {cls.toolname} does not " - f"match environment:" - f"\n\ttool_base = {tool_base}" - f"\n\tenvi_base = {envi_base}") - return out + this_executable = (cls.default_executable + if cls.default_executable + else cls.toolname) + envi_executable = envi_full.split(maxsplit=1)[0] + out = envi_full if with_flags else envi_full.split(maxsplit=1)[0] + this_version = CompilationToolBase.tool_version_static( + cls, this_executable, require_match=True) + envi_version = CompilationToolBase.tool_version_static( + cls, envi_executable, require_match=True) + if this_version and this_version and this_version == envi_version: + return out + if this_executable == cls.toolname and envi_version: + return out + if this_version and envi_version: + logger.info(f"{cls.tooltype.title()} {cls.toolname} " + f"does not match environment:" + f"\n\ttool_exe = {this_executable}" + f"\n\tenvi_exe = {envi_executable}" + f"\n\ttool_ver = {this_version}" + f"\n\tenvi_ver = {envi_version}") + return None @classmethod def get_env_flags(cls): @@ -4984,33 +4974,92 @@ def append_product(cls, products, new, sources=None, products.append_compilation_product(new, **kwargs) return products.last - @classmethod - def tool_version(cls, **kwargs): - r"""Get the version of the compilation tool. + @staticmethod + def extract_tool_version(cls, x, require_match=False): + r"""Extract the tool's version from the provided string. + + Args: + x (str): Raw version string. + require_match (bool, optional): If True, a match to + version_regex is required. Returns: - str: Version. + str: Extracted version string. """ - kwargs.setdefault('cache_key', True) - out = cls.call(cls.version_flags, for_version=True, **kwargs)[0] - if cls.version_regex: + if x and cls.version_regex: match = None regexes = ( cls.version_regex if isinstance(cls.version_regex, list) else [cls.version_regex]) for regex in regexes: - match = re.search(regex, out) + match = re.search(regex, x) if match is not None: - break - if match is None: # pragma: debug - warnings.warn( - f"Could not locate version in string: {out} with " - f"regex {cls.version_regex}") - else: - return match.group('version') - return out + return match.group('version') + if require_match: + return '' + warnings.warn( + f"Could not locate version in string: {x} with " + f"regex {cls.version_regex}") + if x and require_match: + raise Exception(f"{cls}: {cls.tooltype.title()} " + f"{cls.toolname} does not have a " + f"version regex") + return x + + @staticmethod + def tool_version_static(cls, executable=None, skip_regex=False, + **kwargs): + r"""Get the version of the compilation tool using only static + class properties. + + Args: + executable (str, optional): Executable that should be used + with the version flags for this class. If not provided + the default executable will be used if it is set and + toolname will be used if it is not set. + skip_regex (bool, optional): If True, don't call + extract_tool_version and return the raw version result. + **kwargs: Additional keyword arguments are pased to + extract_tool_version. + + Returns: + str: Version string associated with the provided executable. + + """ + if executable is None: + executable = ( + cls.default_executable + if cls.default_executable else cls.toolname) + try: + out = subprocess.check_output( + [executable] + cls.version_flags, + stderr=subprocess.STDOUT).decode('utf-8').strip() + except (subprocess.CalledProcessError, OSError): + out = '' + if skip_regex: + return out + return CompilationToolBase.extract_tool_version(cls, out, **kwargs) + + @classmethod + def tool_version(cls, skip_regex=False, **kwargs): + r"""Get the version of the compilation tool. + + Args: + skip_regex (bool, optional): If True, don't call + extract_tool_version and return the raw version result. + **kwargs: Additional keyword arguments are passed to call. + + Returns: + str: Version. + + """ + kwargs.setdefault('cache_key', True) + out = cls.call(cls.version_flags, for_version=True, **kwargs)[0] + if skip_regex: + return out + return CompilationToolBase.extract_tool_version(cls, out) @classmethod def run_executable_command(cls, args, skip_flags=False, @@ -5125,9 +5174,7 @@ def format_out(name, x, indent=2, wrap=100, tab=' ', if (not skip_flags) and ('env' not in unused_kwargs): unused_kwargs['env'] = cls.set_env() message_before = ( - format_out('Executable', - cls.get_executable(full_path=True)) - + format_out('Working Dir', working_dir) + format_out('Working Dir', working_dir) + format_out('Command', f"\"{' '.join(cmd)}\"")) if not for_version: try: diff --git a/yggdrasil/drivers/FortranModelDriver.py b/yggdrasil/drivers/FortranModelDriver.py index 9f6d93566..9c755e6e1 100644 --- a/yggdrasil/drivers/FortranModelDriver.py +++ b/yggdrasil/drivers/FortranModelDriver.py @@ -143,6 +143,7 @@ class GFortranCompiler(FortranCompilerBase): default_archiver = 'ar' default_disassembler = 'objdump' standard_library = 'gfortran' + version_regex = [r'(?PGNU Fortran \(.+\) \d+\.\d+\.\d+)'] # GNU ASAN not currently installed with gfortran on osx # libraries = { # 'asan': {'dep_executable_flags': ['-fsanitize=address'], diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py index 4d0415ae2..f5221b974 100644 --- a/yggdrasil/tools.py +++ b/yggdrasil/tools.py @@ -597,7 +597,7 @@ def escape_regex(name): """ out = name - for k in '\\.+*[]()^$?:': + for k in '\\?-+*$%#@!^&(){}[]<>,.;:': out = out.replace(k, '\\' + k) return out