Change how tools are checked against the environment to use the versi…

…on regexes

tests/drivers/test_CompiledModelDriver.py

@@ -377,15 +377,10 @@ def test_executable_command(self, python_class):
             basetool.get_default_libtype(), None)
         if not next_tool:
             return
-        if basetool.no_separate_next_stage:
-            with pytest.raises(RuntimeError):
-                python_class.executable_command(['test'],
-                                                exec_type=next_tool)
-        else:
-            python_class.executable_command(['test'],
-                                            no_additional_stages=True)
-            python_class.executable_command(['test'],
-                                            exec_type=next_tool)
+        python_class.executable_command(['test'],
+                                        no_additional_stages=True)
+        python_class.executable_command(['test'],
+                                        exec_type=next_tool)
 
     def test_basetool_call(self, python_class):
         r"""Test basetool call."""

yggdrasil/drivers/CModelDriver.py

@@ -91,8 +91,11 @@ class GCCCompiler(CCompilerBase):
     default_disassembler = 'objdump'
     toolset = 'gnu'
     aliases = ['gnu-cc', 'gnu-gcc']
-    # standard_library = 'c'
     compatible_toolsets = ['llvm']
+    next_stage_flag_flag = '-Xlinker'
+    version_regex = [
+        r'(?P<version>(?:.*gnu\-)?g?cc \(.+\) \d+\.\d+\.\d+)']
+    # standard_library = 'c'
     libraries = {
         'asan': {'dep_executable_flags': ['-fsanitize=address'],
                  'dep_shared_flags': ['-fsanitize=address'],
@@ -111,13 +114,11 @@ def is_clang(cls):
             bool: True if gcc actually points to clang.
         
         """
-        if platform._is_mac:
-            try:
-                ver = cls.tool_version()
-                return ('clang' in ver)
-            except InvalidCompilationTool:
-                pass
-        return False
+        try:
+            return (platform._is_mac
+                    and 'clang' in cls.tool_version(skip_regex=True))
+        except InvalidCompilationTool:
+            return False
 
     @classmethod
     def is_installed(cls):
@@ -150,19 +151,18 @@ class ClangCompiler(CCompilerBase):
     default_linker = 'clang'
     default_archiver = 'libtool'
     default_disassembler = 'otool'
+    toolset = 'llvm'
+    compatible_toolsets = ['gnu']
+    next_stage_flag_flag = '-Xlinker'
+    version_regex = [
+        r'(?P<version>(?:Apple )?clang version \d+\.\d+\.\d+)']
     flag_options = OrderedDict(list(CCompilerBase.flag_options.items())
                                + [('sysroot', '--sysroot'),
                                   ('isysroot', {'key': '-isysroot',
                                                 'prepend': True}),
                                   ('mmacosx-version-min',
                                    '-mmacosx-version-min=%s')])
-    version_regex = [
-        r'(?P<version>(?:Apple )?clang version \d+\.\d+\.\d+)']
     product_exts = ['.dSYM']
-    # Set to False since ClangLinker has its own class to handle
-    # conflict between versions of clang and ld.
-    toolset = 'llvm'
-    compatible_toolsets = ['gnu']
     libraries = {
         'asan': {'dep_executable_flags': ['-fsanitize=address'],
                  'dep_shared_flags': ['-fsanitize=address',
@@ -225,7 +225,7 @@ class MSVCCompiler(CCompilerBase):
     search_path_envvar = ['INCLUDE']
     search_path_flags = None
     version_flags = []
-    version_regex = r'(?P<version>.+)\s+Copyright'
+    version_regex = [r'(?P<version>.+)\s+Copyright']
     product_exts = ['.dir', '.ilk', '.pdb', '.sln', '.vcxproj',
                     '.vcxproj.filters', '.exp', '.lib']
     builtin_next_stage = 'linker'
@@ -240,7 +240,7 @@ class LDLinker(LinkerBase):
     # Languages disabled for ld by default to prevent it being
     # selected instead of the default which seems to be happening
     # on the CI
-    languages = ['c']  # ['c', 'c++', 'fortran']
+    languages = ['c', 'c++', 'fortran']
     default_executable_env = 'LD'
     default_flags_env = 'LDFLAGS'
     version_flags = ['-v']
@@ -250,6 +250,10 @@ class LDLinker(LinkerBase):
          r'(?P<version>\d+(?:\.\d+){0,2})')
     ]
     search_path_envvar = ['LIBRARY_PATH', 'LD_LIBRARY_PATH']
+    compatible_toolsets = ['gnu', 'llvm']
+    preload_envvar = ('DYLD_INSERT_LIBRARIES' if platform._is_mac
+                      else ('LD_PRELOAD' if platform._is_linux
+                            else None))
 
     # @classmethod
     # def get_flags(cls, *args, **kwargs):
@@ -269,12 +273,12 @@ class GCCLinker(LDLinker):
     default_executable = GCCCompiler.default_executable
     toolset = GCCCompiler.toolset
     compatible_toolsets = GCCCompiler.compatible_toolsets
-    version_regex = GCCCompiler.version_regex
+    version_flags = ['-Xlinker', '--verbose']
+    version_regex = LDLinker.version_regex + GCCCompiler.version_regex
     search_path_flags = ['-Xlinker', '--verbose']
     search_regex = [r'SEARCH_DIR\("=([^"]+)"\);']
     flag_options = OrderedDict(LDLinker.flag_options,
                                **{'library_rpath': '-Wl,-rpath'})
-    preload_envvar = 'LD_PRELOAD'
 
 
 class ClangLinker(LDLinker):
@@ -289,15 +293,16 @@ class ClangLinker(LDLinker):
     default_executable = ClangCompiler.default_executable
     toolset = ClangCompiler.toolset
     compatible_toolsets = ClangCompiler.compatible_toolsets
-    version_regex = ClangCompiler.version_regex
+    version_flags = ['-Xlinker', '-v']
+    version_regex = LDLinker.version_regex + ClangCompiler.version_regex
+    # version_regex = ClangCompiler.version_regex
     search_path_flags = ['-Xlinker', '-v']
     search_regex = [r'\t([^\t\n]+)\n']
     search_regex_begin = 'Library search paths:'
     flag_options = OrderedDict(LDLinker.flag_options,
                                **{'linker-version': '-mlinker-version=%s',
                                   'library_rpath': '-rpath',
                                   'library_libs_nonstd': ''})
-    preload_envvar = 'DYLD_INSERT_LIBRARIES'
     # libtype_flags = {'shared': '-dynamiclib'}
 
     @staticmethod

yggdrasil/drivers/CPPModelDriver.py

@@ -3,7 +3,7 @@
 from yggdrasil import platform
 from yggdrasil.drivers.CModelDriver import (
     CCompilerBase, CModelDriver, GCCCompiler, ClangCompiler, MSVCCompiler,
-    GCCLinker, ClangLinker, MSVCLinker)
+    LDLinker, GCCLinker, ClangLinker, MSVCLinker)
 logger = logging.getLogger(__name__)
 
 
@@ -59,6 +59,8 @@ class GPPCompiler(CPPCompilerBase, GCCCompiler):
     toolname = 'g++'
     aliases = ['gnu-c++', 'gnu-g++']
     default_linker = 'g++'
+    version_regex = [
+        r'(?P<version>(?:.*gnu\-)?(?:g|c)\+\+ \(.+\) \d+\.\d+\.\d+)']
     standard_library = 'stdc++'
     libraries = {}
 
@@ -97,7 +99,7 @@ def before_registration(cls):
         if platform._is_win:  # pragma: windows
             cls.default_executable = 'clang'
         if GPPCompiler.is_clang() and 'g++' not in cls.aliases:
-            cls.aliases.append('g++')
+            cls.aliases = cls.aliases + ['g++']
         CPPCompilerBase.before_registration(cls)
 
     @classmethod
@@ -170,6 +172,7 @@ class GPPLinker(GCCLinker):
     languages = GPPCompiler.languages
     default_executable = GPPCompiler.default_executable
     toolset = GPPCompiler.toolset
+    version_regex = LDLinker.version_regex + GPPCompiler.version_regex
     standard_library = GPPCompiler.standard_library
     libraries = {}
 
@@ -184,6 +187,7 @@ class ClangPPLinker(ClangLinker):
     languages = ClangPPCompiler.languages
     default_executable = ClangPPCompiler.default_executable
     toolset = ClangPPCompiler.toolset
+    version_regex = LDLinker.version_regex + ClangPPCompiler.version_regex
 
 
 class MSVCPPLinker(MSVCLinker):

yggdrasil/drivers/CompiledModelDriver.py

@@ -3895,6 +3895,8 @@ class CompilationToolBase(object):
         next_stage_switch (str): Flag to indicate beginning of flags that
             should be passed to the tool for the next stage. (e.g. /link
             for MSVC cl.exe).
+        next_stage_flag_flag (str): Flag to indicate that the next flag
+            should be passed to the next stage.
         create_next_stage_tool (dict): Parameters for a tool that should
             be created for the next stage based on this tool.
         local_kws (list): Keyword arguments that are unique to this tool.
@@ -3968,11 +3970,11 @@ class CompilationToolBase(object):
     standard_library = None
     standard_library_type = None
     libraries = {}
-    no_separate_next_stage = False
     builtin_next_stage = None
     combine_with_next_stage = None
     no_additional_stages_flag = None
     next_stage_switch = None
+    next_stage_flag_flag = None
     create_next_stage_tool = None
     is_gnu = False
     toolset = None
@@ -4056,7 +4058,7 @@ def before_registration(cls):
             assert cls.combine_with_next_stage
             copy_attr = ['toolname', 'aliases', 'languages', 'platforms',
                          'default_executable', 'default_executable_env',
-                         'toolset']
+                         'toolset', 'version_flags', 'version_regex']
             if cls.create_next_stage_tool is True:
                 cls.create_next_stage_tool = {}
             stage_attr = copy.deepcopy(
@@ -4473,44 +4475,32 @@ def env_matches_tool(cls, use_sysconfig=False, env=None,
             env.update(sysconfig.get_config_vars())
         else:
             env.update(os.environ)
-        tool_base = cls.aliases.copy()
-        envi_base = ''
         envi_full = ''
-        if isinstance(cls.toolname, str):
-            tool_base.append(cls.toolname)
-        if isinstance(cls.default_executable, str):
-            tool_base.append(cls.default_executable)
         if isinstance(cls.default_executable_env, str):
             envi_full = env.get(cls.default_executable_env, '').split(
                 'ccache ')[-1]
         if envi_full:
-            envi_base = os.path.basename(envi_full.split(maxsplit=1)[0])
-        if os.environ.get('PATHEXT', ''):
-            tool_base = [x.split(os.environ['PATHEXT'])[0]
-                         for x in tool_base]
-            envi_base = envi_base.split(os.environ['PATHEXT'])[0]
-        out = None
-        regex_literal = '-+*$%#@!^&(){}[]<>,.;:'
-        regex_pathsep = r'(?:[\-\_\.0-9])'
-        if tool_base and envi_base:
-            for x in tool_base:
-                for k in regex_literal:
-                    x = x.replace(k, '\\' + k)
-                regex = r'(?:(?:^)|%s)%s(?:(?:$)|%s)' % (
-                    regex_pathsep, x, regex_pathsep)
-                if re.search(regex, envi_base):
-                    out = True
-                    break
-        if out:
-            if not with_flags:
-                envi_full = envi_full.split(maxsplit=1)[0]
-            return envi_full
-        if tool_base and envi_base:
-            logger.info(f"{cls.tooltype.title()} {cls.toolname} does not "
-                        f"match environment:"
-                        f"\n\ttool_base = {tool_base}"
-                        f"\n\tenvi_base = {envi_base}")
-        return out
+            this_executable = (cls.default_executable
+                               if cls.default_executable
+                               else cls.toolname)
+            envi_executable = envi_full.split(maxsplit=1)[0]
+            out = envi_full if with_flags else envi_full.split(maxsplit=1)[0]
+            this_version = CompilationToolBase.tool_version_static(
+                cls, this_executable, require_match=True)
+            envi_version = CompilationToolBase.tool_version_static(
+                cls, envi_executable, require_match=True)
+            if this_version and this_version and this_version == envi_version:
+                return out
+            if this_executable == cls.toolname and envi_version:
+                return out
+            if this_version and envi_version:
+                logger.info(f"{cls.tooltype.title()} {cls.toolname} "
+                            f"does not match environment:"
+                            f"\n\ttool_exe = {this_executable}"
+                            f"\n\tenvi_exe = {envi_executable}"
+                            f"\n\ttool_ver = {this_version}"
+                            f"\n\tenvi_ver = {envi_version}")
+        return None
 
     @classmethod
     def get_env_flags(cls):
@@ -4984,33 +4974,92 @@ def append_product(cls, products, new, sources=None,
         products.append_compilation_product(new, **kwargs)
         return products.last
 
-    @classmethod
-    def tool_version(cls, **kwargs):
-        r"""Get the version of the compilation tool.
+    @staticmethod
+    def extract_tool_version(cls, x, require_match=False):
+        r"""Extract the tool's version from the provided string.
+
+        Args:
+            x (str): Raw version string.
+            require_match (bool, optional): If True, a match to
+                version_regex is required.
 
         Returns:
-            str: Version.
+            str: Extracted version string.
 
         """
-        kwargs.setdefault('cache_key', True)
-        out = cls.call(cls.version_flags, for_version=True, **kwargs)[0]
-        if cls.version_regex:
+        if x and cls.version_regex:
             match = None
             regexes = (
                 cls.version_regex
                 if isinstance(cls.version_regex, list)
                 else [cls.version_regex])
             for regex in regexes:
-                match = re.search(regex, out)
+                match = re.search(regex, x)
                 if match is not None:
-                    break
-            if match is None:  # pragma: debug
-                warnings.warn(
-                    f"Could not locate version in string: {out} with "
-                    f"regex {cls.version_regex}")
-            else:
-                return match.group('version')
-        return out
+                    return match.group('version')
+            if require_match:
+                return ''
+            warnings.warn(
+                f"Could not locate version in string: {x} with "
+                f"regex {cls.version_regex}")
+        if x and require_match:
+            raise Exception(f"{cls}: {cls.tooltype.title()} "
+                            f"{cls.toolname} does not have a "
+                            f"version regex")
+        return x
+
+    @staticmethod
+    def tool_version_static(cls, executable=None, skip_regex=False,
+                            **kwargs):
+        r"""Get the version of the compilation tool using only static
+        class properties.
+
+        Args:
+            executable (str, optional): Executable that should be used
+                with the version flags for this class. If not provided
+                the default executable will be used if it is set and
+                toolname will be used if it is not set.
+            skip_regex (bool, optional): If True, don't call
+                extract_tool_version and return the raw version result.
+            **kwargs: Additional keyword arguments are pased to
+                extract_tool_version.
+
+        Returns:
+            str: Version string associated with the provided executable.
+
+        """
+        if executable is None:
+            executable = (
+                cls.default_executable
+                if cls.default_executable else cls.toolname)
+        try:
+            out = subprocess.check_output(
+                [executable] + cls.version_flags,
+                stderr=subprocess.STDOUT).decode('utf-8').strip()
+        except (subprocess.CalledProcessError, OSError):
+            out = ''
+        if skip_regex:
+            return out
+        return CompilationToolBase.extract_tool_version(cls, out, **kwargs)
+
+    @classmethod
+    def tool_version(cls, skip_regex=False, **kwargs):
+        r"""Get the version of the compilation tool.
+
+        Args:
+            skip_regex (bool, optional): If True, don't call
+                extract_tool_version and return the raw version result.
+            **kwargs: Additional keyword arguments are passed to call.
+
+        Returns:
+            str: Version.
+
+        """
+        kwargs.setdefault('cache_key', True)
+        out = cls.call(cls.version_flags, for_version=True, **kwargs)[0]
+        if skip_regex:
+            return out
+        return CompilationToolBase.extract_tool_version(cls, out)
 
     @classmethod
     def run_executable_command(cls, args, skip_flags=False,
@@ -5125,9 +5174,7 @@ def format_out(name, x, indent=2, wrap=100, tab='  ',
             if (not skip_flags) and ('env' not in unused_kwargs):
                 unused_kwargs['env'] = cls.set_env()
             message_before = (
-                format_out('Executable',
-                           cls.get_executable(full_path=True))
-                + format_out('Working Dir', working_dir)
+                format_out('Working Dir', working_dir)
                 + format_out('Command', f"\"{' '.join(cmd)}\""))
             if not for_version:
                 try: