From f63d6a3e17d5651189696b0d56c3550ac6c733d4 Mon Sep 17 00:00:00 2001 From: Meagan Lang Date: Wed, 14 Feb 2024 10:35:27 -0500 Subject: [PATCH] Updated python-rapidjson submodule address to https Correct description in ygginfo Add more debug messages for setting compiler executable and try to prevent over eager setting Remove verbose flag from cmake call method Clean up log messages for calls to external executables and versions for compiled languages Make path adjustments permanent --- .gitmodules | 2 +- utils/setup_test_env.py | 14 +++- yggdrasil/command_line.py | 2 +- yggdrasil/drivers/CMakeModelDriver.py | 1 - yggdrasil/drivers/CompiledModelDriver.py | 81 +++++++++++++----------- yggdrasil/drivers/ModelDriver.py | 21 +++--- 6 files changed, 69 insertions(+), 52 deletions(-) diff --git a/.gitmodules b/.gitmodules index fdf0c73b8..306b15011 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,5 +11,5 @@ url = https://github.com/cropsinsilico/CiS2021-hackathon [submodule "python-rapidjson"] path = yggdrasil/python-rapidjson - url = git@github.com:cropsinsilico/python-rapidjson.git + url = https://github.com/cropsinsilico/python-rapidjson.git branch = yggdrasil diff --git a/utils/setup_test_env.py b/utils/setup_test_env.py index 7e85c1c2c..57a3dfc5d 100644 --- a/utils/setup_test_env.py +++ b/utils/setup_test_env.py @@ -521,7 +521,15 @@ def add_argument(*a_args, **a_kwargs): help=("Install %s" % k)) -def prune_windows_path(): +def prune_windows_path(permanent=False): + r"""Prune directories not required by yggdrasil from the windows path + on a GHA runner to make room to avoid the command limit. + + Args: + permanent (bool, optional): If True, the path will be permanently + modified via the GITHUB_ENV variable. + + """ to_remove = ( 'C:\\Program Files\\Microsoft SQL Server', 'C:\\Strawberry', @@ -549,6 +557,8 @@ def prune_windows_path(): print(f"PRUNED PATH ({len(new_path)}):\n" f"{pprint.pformat(new_path.split(os.pathsep))}\n") cmds += [f"set \"PATH={new_path}\""] + if permanent and _on_gha: + cmds.append("echo \"PATH=$PATH\" >> $GITHUB_ENV") return cmds @@ -1170,7 +1180,7 @@ def build_conda_recipe(recipe='recipe', param=None, assert conda_env == 'base' or param.dry_run assert conda_idx if _is_win and _on_gha: - cmds += prune_windows_path() + cmds += prune_windows_path(permanent=True) # Might invalidate cache cmds += [f"{CONDA_CMD} clean --all {param.conda_flags_general}"] cmds += setup_conda(param=param, conda_env=conda_env, diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py index b77190138..e07ea8ba3 100755 --- a/yggdrasil/command_line.py +++ b/yggdrasil/command_line.py @@ -906,7 +906,7 @@ def func(cls, args, return_str=False): env=env_reticulate).strip() vardict.append( (curr_prefix - + "R reticulate::py_versions_windows():", + + "R reticulate::conda_binary info --json:", "\n%s%s" % ( curr_prefix + prefix, ("\n" + curr_prefix + prefix).join( diff --git a/yggdrasil/drivers/CMakeModelDriver.py b/yggdrasil/drivers/CMakeModelDriver.py index 75763a991..1271e928d 100644 --- a/yggdrasil/drivers/CMakeModelDriver.py +++ b/yggdrasil/drivers/CMakeModelDriver.py @@ -211,7 +211,6 @@ def call(cls, args, **kwargs): True, produced file otherwise. """ - kwargs['verbose'] = True try: out = super(CMakeConfigure, cls).call(args, **kwargs) except RuntimeError as e: diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index 3bd8170b7..2d318c3c6 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -208,11 +208,13 @@ def get_compilation_tool(tooltype, name, default=False): break if out is None: if default is False: - raise ValueError("Could not locate a %s tool with name '%s'" - % (tooltype, name)) + raise ValueError(f"Could not locate a {tooltype} tool with " + f"name '{name}'") out = default - if ((isinstance(out, CompilationToolMeta) and (out.toolname != name) - and (os.path.isfile(name) or shutil.which(name)))): + elif ((isinstance(out, CompilationToolMeta) and (out.toolname != name) + and (os.path.isfile(name) or shutil.which(name)))): + logger.info(f"Setting executable for toolname = {out.toolname} to" + f" {name}") out.executable = name return out @@ -385,6 +387,9 @@ def __init__(self, **kwargs): setattr(self, k, v) if len(kwargs) > 0: raise RuntimeError("Unused keyword arguments: %s" % kwargs.keys()) + if getattr(self, 'executable', None): + logger.info(f"Creating {self.toolname} {self.tooltype}: " + f"{getattr(self, 'executable', None)}") super(CompilationToolBase, self).__init__(**kwargs) @staticmethod @@ -1297,26 +1302,30 @@ def call(cls, args, language=None, toolname=None, skip_flags=False, if (not skip_flags) and ('env' not in unused_kwargs): unused_kwargs['env'] = cls.set_env() if verbose: - logger.info('Command: "%s"' % ' '.join(cmd)) + logger.info(f"Command: \"{' '.join(cmd)}\"") else: - logger.debug('Command: "%s"' % ' '.join(cmd)) + logger.debug(f"Command: \"{' '.join(cmd)}\"") proc = tools.popen_nobuffer(cmd, **unused_kwargs) output, err = proc.communicate() output = output.decode("utf-8") + err = err.decode("utf-8") if err else '' + message = (f"Command '{' '.join(cmd)}' resulted in code " + f"{proc.returncode}:\n" + f"out = {output}\n" + f"err = {err}\n") if (proc.returncode != 0) and (not allow_error): - raise RuntimeError("Command '%s' failed with code %d:\n%s." - % (' '.join(cmd), proc.returncode, output)) + raise RuntimeError(message) try: if verbose: - logger.info(' '.join(cmd) + '\n' + output) + logger.info(message) else: - logger.debug(' '.join(cmd) + '\n' + output) + logger.debug(message) except UnicodeDecodeError: # pragma: debug - tools.print_encoded(output) + tools.print_encoded(message) except (subprocess.CalledProcessError, OSError) as e: if not allow_error: - raise RuntimeError("Could not call command '%s': %s" - % (' '.join(cmd), e)) + raise RuntimeError(f"Could not call command " + f"'{' '.join(cmd)}': {e}") except BaseException as e: try: print(f"Unexpected call error {type(e)}: {e}") @@ -1327,7 +1336,7 @@ def call(cls, args, language=None, toolname=None, skip_flags=False, if (not skip_flags): if (out != 'clean'): if not products.last.exists: # pragma: debug - logger.error('%s\n%s' % (' '.join(cmd), output)) + logger.error(f"{' '.join(cmd)}\n{output}") raise RuntimeError( f"{cls.tooltype.title()} tool, {cls.toolname}" f", failed to produce result \'{out}\'") @@ -2436,23 +2445,23 @@ def after_registration(cls, **kwargs): else: libtype = [libtype] for t in libtype: - libfile = cls.cfg.get(cls.language, - '%s_%s' % (k, t), None) + libfile = cls.cfg.get(cls.language, f'{k}_{t}', None) if libfile is not None: v[t] = libfile for k in ['compiler', 'linker', 'archiver']: # Set default linker/archiver based on compiler - default_tool_name = getattr(cls, 'default_%s' % k, None) + default_tool_name = getattr(cls, f'default_{k}', None) if default_tool_name: default_tool = get_compilation_tool(k, default_tool_name, default=None) if (((default_tool is None) or (not default_tool.is_installed()))): # pragma: debug if not tools.is_subprocess(): - logger.debug(('Default %s for %s (%s) not installed. ' - 'Attempting to locate an alternative .') - % (k, cls.language, default_tool_name)) - setattr(cls, 'default_%s' % k, None) + logger.debug(f'Default {k} for {cls.language} ' + f'({default_tool_name}) is not ' + f'installed. Attempting to locate ' + f'an alternative .') + setattr(cls, f'default_{k}', None) @classmethod def select_suffix_kwargs(cls, kwargs): @@ -2782,7 +2791,7 @@ def get_tool_static(cls, tooltype, toolname=None, kwargs = {'flags': tool_flags} kwargs['executable'] = cls.cfg.get(cls.language, f'{toolname}_executable', - toolname) + None) if tooltype == 'compiler': kwargs.update( linker=cls.get_tool( @@ -2807,10 +2816,10 @@ def get_tool_static(cls, tooltype, toolname=None, # Github Actions images now include GNU compilers by default if default is False: raise NotImplementedError( - "%s not set for language '%s' (toolname=%s)." - % (tooltype.title(), cls.language, toolname)) - logger.debug("%s not set for language '%s' (toolname=%s)." - % (tooltype.title(), cls.language, toolname)) + f"{tooltype.title()} not set for language " + f"'{cls.language}' (toolname={toolname}).") + logger.debug(f"{tooltype.title()} not set for language " + f"'{cls.language}' (toolname={toolname}).") out = default if isinstance(out, type): out = out(**kwargs) @@ -2822,7 +2831,7 @@ def get_tool_static(cls, tooltype, toolname=None, elif return_prop == 'flags': # pragma: no cover return out.flags else: - raise ValueError("Invalid return_prop: '%s'" % return_prop) + raise ValueError(f"Invalid return_prop: '{return_prop}'") def get_tool_instance(self, *args, **kwargs): r"""Get tool from a driver instance. @@ -3620,7 +3629,7 @@ def language_version(cls, toolname=None, **kwargs): """ compiler = cls.get_tool('compiler', toolname=toolname) - return compiler.tool_version(**kwargs).strip() + return compiler.tool_version(**kwargs).splitlines()[0].strip() def run_model(self, **kwargs): r"""Run the model. Unless overridden, the model will be run using @@ -3820,24 +3829,24 @@ def configure_executable_type(cls, cfg): for k in ['compiler', 'linker', 'archiver']: # Set default linker/archiver based on compiler default_tool_name = cfg.get( - cls.language, k, getattr(cls, 'default_%s' % k, None)) + cls.language, k, getattr(cls, f'default_{k}', None)) if (((default_tool_name is None) and (compiler is not None) and (k in ['linker', 'archiver']))): - default_tool_name = getattr(compiler, 'default_%s' % k, None) + default_tool_name = getattr(compiler, f'default_{k}', None) # Check default tool to make sure it is installed if default_tool_name: default_tool = get_compilation_tool(k, default_tool_name) if not default_tool.is_installed(): # pragma: debug - logger.debug(('Default %s for %s (%s) not installed. ' - 'Attempting to locate an alternative .') - % (k, cls.language, default_tool_name)) + logger.debug(f'Default {k} for {cls.language} ' + f'({default_tool_name}) not installed. ' + f'Attempting to locate an alternative.') default_tool_name = None # Determine compilation tools based on language/platform if default_tool_name is None: # pragma: no cover - default_tool_name = find_compilation_tool(k, cls.language, - allow_failure=True) + default_tool_name = find_compilation_tool( + k, cls.language, allow_failure=True) # Set default tool attribute & record compiler tool if set - setattr(cls, 'default_%s' % k, default_tool_name) + setattr(cls, f'default_{k}', default_tool_name) if default_tool_name: cfg.set(cls.language, k, default_tool_name) if k == 'compiler': diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py index 509c9980d..36fb75236 100755 --- a/yggdrasil/drivers/ModelDriver.py +++ b/yggdrasil/drivers/ModelDriver.py @@ -991,24 +991,23 @@ def run_executable(cls, args, return_process=False, debug_flags=None, unused_kwargs.setdefault('cwd', unused_kwargs.pop('working_dir')) unused_kwargs.setdefault('shell', platform._is_win) # Call command - logger.debug("Running '%s' from %s" - % (' '.join(cmd), unused_kwargs.get('cwd', os.getcwd()))) + logger.debug(f"Running '{' '.join(cmd)}' from " + f"{unused_kwargs.get('cwd', os.getcwd())}") logger.debug("Process keyword arguments:\n%s\n", ' ' + pformat(unused_kwargs).replace('\n', '\n ')) - print(' '.join(cmd)) proc = tools.popen_nobuffer(cmd, **unused_kwargs) if return_process: return proc out, err = proc.communicate() + out = out.decode("utf-8") if out else '' + err = err.decode("utf-8") if err else '' + message = (f"Command '{' '.join(cmd)}' returned " + f"{proc.returncode}:\n" + f"out = {out}\n" + f"err = {err}\n") if proc.returncode != 0: - if out: - logger.info('\n%s' % out.decode('utf-8')) - if err: # pragma: debug - logger.info('\n%s' % err.decode('utf-8')) - raise RuntimeError("Command '%s' failed with code %d." - % (' '.join(cmd), proc.returncode)) - out = out.decode("utf-8") - logger.debug('%s\n%s' % (' '.join(cmd), out)) + raise RuntimeError(message) + logger.debug(message) return out except (subprocess.CalledProcessError, OSError) as e: # pragma: debug raise RuntimeError("Could not call command '%s': %s"