Skip to content

Commit

Permalink
Updated python-rapidjson submodule address to https
Browse files Browse the repository at this point in the history 
Correct description in ygginfo
Add more debug messages for setting compiler executable and try to prevent over eager setting
Remove verbose flag from cmake call method
Clean up log messages for calls to external executables and versions for compiled languages
Make path adjustments permanent
  • Loading branch information
@langmm
langmm committed Feb 14, 2024
1 parent 1377872 commit f63d6a3
Show file tree
Hide file tree
Showing 6 changed files with 69 additions and 52 deletions.
2 changes: 1 addition & 1 deletion .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@
url = https://github.com/cropsinsilico/CiS2021-hackathon
[submodule "python-rapidjson"]
path = yggdrasil/python-rapidjson
url = git@github.com:cropsinsilico/python-rapidjson.git
url = https://github.com/cropsinsilico/python-rapidjson.git
branch = yggdrasil
14 changes: 12 additions & 2 deletions utils/setup_test_env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,15 @@ def add_argument(*a_args, **a_kwargs):
help=("Install %s" % k))


def prune_windows_path():
def prune_windows_path(permanent=False):
r"""Prune directories not required by yggdrasil from the windows path
on a GHA runner to make room to avoid the command limit.
Args:
permanent (bool, optional): If True, the path will be permanently
modified via the GITHUB_ENV variable.
"""
to_remove = (
'C:\\Program Files\\Microsoft SQL Server',
'C:\\Strawberry',
Expand Down Expand Up @@ -549,6 +557,8 @@ def prune_windows_path():
print(f"PRUNED PATH ({len(new_path)}):\n"
f"{pprint.pformat(new_path.split(os.pathsep))}\n")
cmds += [f"set \"PATH={new_path}\""]
if permanent and _on_gha:
cmds.append("echo \"PATH=$PATH\" >> $GITHUB_ENV")
return cmds


Expand Down Expand Up @@ -1170,7 +1180,7 @@ def build_conda_recipe(recipe='recipe', param=None,
assert conda_env == 'base' or param.dry_run
assert conda_idx
if _is_win and _on_gha:
cmds += prune_windows_path()
cmds += prune_windows_path(permanent=True)
# Might invalidate cache
cmds += [f"{CONDA_CMD} clean --all {param.conda_flags_general}"]
cmds += setup_conda(param=param, conda_env=conda_env,
Expand Down
2 changes: 1 addition & 1 deletion yggdrasil/command_line.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -906,7 +906,7 @@ def func(cls, args, return_str=False):
env=env_reticulate).strip()
vardict.append(
(curr_prefix
+ "R reticulate::py_versions_windows():",
+ "R reticulate::conda_binary info --json:",
"\n%s%s" % (
curr_prefix + prefix,
("\n" + curr_prefix + prefix).join(
Expand Down
1 change: 0 additions & 1 deletion yggdrasil/drivers/CMakeModelDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,6 @@ def call(cls, args, **kwargs):
True, produced file otherwise.
"""
kwargs['verbose'] = True
try:
out = super(CMakeConfigure, cls).call(args, **kwargs)
except RuntimeError as e:
Expand Down
81 changes: 45 additions & 36 deletions yggdrasil/drivers/CompiledModelDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,13 @@ def get_compilation_tool(tooltype, name, default=False):
break
if out is None:
if default is False:
raise ValueError("Could not locate a %s tool with name '%s'"
% (tooltype, name))
raise ValueError(f"Could not locate a {tooltype} tool with "
f"name '{name}'")
out = default
if ((isinstance(out, CompilationToolMeta) and (out.toolname != name)
and (os.path.isfile(name) or shutil.which(name)))):
elif ((isinstance(out, CompilationToolMeta) and (out.toolname != name)
and (os.path.isfile(name) or shutil.which(name)))):
logger.info(f"Setting executable for toolname = {out.toolname} to"
f" {name}")
out.executable = name
return out

Expand Down Expand Up @@ -385,6 +387,9 @@ def __init__(self, **kwargs):
setattr(self, k, v)
if len(kwargs) > 0:
raise RuntimeError("Unused keyword arguments: %s" % kwargs.keys())
if getattr(self, 'executable', None):
logger.info(f"Creating {self.toolname} {self.tooltype}: "
f"{getattr(self, 'executable', None)}")
super(CompilationToolBase, self).__init__(**kwargs)

@staticmethod
Expand Down Expand Up @@ -1297,26 +1302,30 @@ def call(cls, args, language=None, toolname=None, skip_flags=False,
if (not skip_flags) and ('env' not in unused_kwargs):
unused_kwargs['env'] = cls.set_env()
if verbose:
logger.info('Command: "%s"' % ' '.join(cmd))
logger.info(f"Command: \"{' '.join(cmd)}\"")
else:
logger.debug('Command: "%s"' % ' '.join(cmd))
logger.debug(f"Command: \"{' '.join(cmd)}\"")
proc = tools.popen_nobuffer(cmd, **unused_kwargs)
output, err = proc.communicate()
output = output.decode("utf-8")
err = err.decode("utf-8") if err else ''
message = (f"Command '{' '.join(cmd)}' resulted in code "
f"{proc.returncode}:\n"
f"out = {output}\n"
f"err = {err}\n")
if (proc.returncode != 0) and (not allow_error):
raise RuntimeError("Command '%s' failed with code %d:\n%s."
% (' '.join(cmd), proc.returncode, output))
raise RuntimeError(message)
try:
if verbose:
logger.info(' '.join(cmd) + '\n' + output)
logger.info(message)
else:
logger.debug(' '.join(cmd) + '\n' + output)
logger.debug(message)
except UnicodeDecodeError: # pragma: debug
tools.print_encoded(output)
tools.print_encoded(message)
except (subprocess.CalledProcessError, OSError) as e:
if not allow_error:
raise RuntimeError("Could not call command '%s': %s"
% (' '.join(cmd), e))
raise RuntimeError(f"Could not call command "
f"'{' '.join(cmd)}': {e}")
except BaseException as e:
try:
print(f"Unexpected call error {type(e)}: {e}")
Expand All @@ -1327,7 +1336,7 @@ def call(cls, args, language=None, toolname=None, skip_flags=False,
if (not skip_flags):
if (out != 'clean'):
if not products.last.exists: # pragma: debug
logger.error('%s\n%s' % (' '.join(cmd), output))
logger.error(f"{' '.join(cmd)}\n{output}")
raise RuntimeError(
f"{cls.tooltype.title()} tool, {cls.toolname}"
f", failed to produce result \'{out}\'")
Expand Down Expand Up @@ -2436,23 +2445,23 @@ def after_registration(cls, **kwargs):
else:
libtype = [libtype]
for t in libtype:
libfile = cls.cfg.get(cls.language,
'%s_%s' % (k, t), None)
libfile = cls.cfg.get(cls.language, f'{k}_{t}', None)
if libfile is not None:
v[t] = libfile
for k in ['compiler', 'linker', 'archiver']:
# Set default linker/archiver based on compiler
default_tool_name = getattr(cls, 'default_%s' % k, None)
default_tool_name = getattr(cls, f'default_{k}', None)
if default_tool_name:
default_tool = get_compilation_tool(k, default_tool_name,
default=None)
if (((default_tool is None)
or (not default_tool.is_installed()))): # pragma: debug
if not tools.is_subprocess():
logger.debug(('Default %s for %s (%s) not installed. '
'Attempting to locate an alternative .')
% (k, cls.language, default_tool_name))
setattr(cls, 'default_%s' % k, None)
logger.debug(f'Default {k} for {cls.language} '
f'({default_tool_name}) is not '
f'installed. Attempting to locate '
f'an alternative .')
setattr(cls, f'default_{k}', None)

@classmethod
def select_suffix_kwargs(cls, kwargs):
Expand Down Expand Up @@ -2782,7 +2791,7 @@ def get_tool_static(cls, tooltype, toolname=None,
kwargs = {'flags': tool_flags}
kwargs['executable'] = cls.cfg.get(cls.language,
f'{toolname}_executable',
toolname)
None)
if tooltype == 'compiler':
kwargs.update(
linker=cls.get_tool(
Expand All @@ -2807,10 +2816,10 @@ def get_tool_static(cls, tooltype, toolname=None,
# Github Actions images now include GNU compilers by default
if default is False:
raise NotImplementedError(
"%s not set for language '%s' (toolname=%s)."
% (tooltype.title(), cls.language, toolname))
logger.debug("%s not set for language '%s' (toolname=%s)."
% (tooltype.title(), cls.language, toolname))
f"{tooltype.title()} not set for language "
f"'{cls.language}' (toolname={toolname}).")
logger.debug(f"{tooltype.title()} not set for language "
f"'{cls.language}' (toolname={toolname}).")
out = default
if isinstance(out, type):
out = out(**kwargs)
Expand All @@ -2822,7 +2831,7 @@ def get_tool_static(cls, tooltype, toolname=None,
elif return_prop == 'flags': # pragma: no cover
return out.flags
else:
raise ValueError("Invalid return_prop: '%s'" % return_prop)
raise ValueError(f"Invalid return_prop: '{return_prop}'")

def get_tool_instance(self, *args, **kwargs):
r"""Get tool from a driver instance.
Expand Down Expand Up @@ -3620,7 +3629,7 @@ def language_version(cls, toolname=None, **kwargs):
"""
compiler = cls.get_tool('compiler', toolname=toolname)
return compiler.tool_version(**kwargs).strip()
return compiler.tool_version(**kwargs).splitlines()[0].strip()

def run_model(self, **kwargs):
r"""Run the model. Unless overridden, the model will be run using
Expand Down Expand Up @@ -3820,24 +3829,24 @@ def configure_executable_type(cls, cfg):
for k in ['compiler', 'linker', 'archiver']:
# Set default linker/archiver based on compiler
default_tool_name = cfg.get(
cls.language, k, getattr(cls, 'default_%s' % k, None))
cls.language, k, getattr(cls, f'default_{k}', None))
if (((default_tool_name is None) and (compiler is not None)
and (k in ['linker', 'archiver']))):
default_tool_name = getattr(compiler, 'default_%s' % k, None)
default_tool_name = getattr(compiler, f'default_{k}', None)
# Check default tool to make sure it is installed
if default_tool_name:
default_tool = get_compilation_tool(k, default_tool_name)
if not default_tool.is_installed(): # pragma: debug
logger.debug(('Default %s for %s (%s) not installed. '
'Attempting to locate an alternative .')
% (k, cls.language, default_tool_name))
logger.debug(f'Default {k} for {cls.language} '
f'({default_tool_name}) not installed. '
f'Attempting to locate an alternative.')
default_tool_name = None
# Determine compilation tools based on language/platform
if default_tool_name is None: # pragma: no cover
default_tool_name = find_compilation_tool(k, cls.language,
allow_failure=True)
default_tool_name = find_compilation_tool(
k, cls.language, allow_failure=True)
# Set default tool attribute & record compiler tool if set
setattr(cls, 'default_%s' % k, default_tool_name)
setattr(cls, f'default_{k}', default_tool_name)
if default_tool_name:
cfg.set(cls.language, k, default_tool_name)
if k == 'compiler':
Expand Down
21 changes: 10 additions & 11 deletions yggdrasil/drivers/ModelDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -991,24 +991,23 @@ def run_executable(cls, args, return_process=False, debug_flags=None,
unused_kwargs.setdefault('cwd', unused_kwargs.pop('working_dir'))
unused_kwargs.setdefault('shell', platform._is_win)
# Call command
logger.debug("Running '%s' from %s"
% (' '.join(cmd), unused_kwargs.get('cwd', os.getcwd())))
logger.debug(f"Running '{' '.join(cmd)}' from "
f"{unused_kwargs.get('cwd', os.getcwd())}")
logger.debug("Process keyword arguments:\n%s\n",
' ' + pformat(unused_kwargs).replace('\n', '\n '))
print(' '.join(cmd))
proc = tools.popen_nobuffer(cmd, **unused_kwargs)
if return_process:
return proc
out, err = proc.communicate()
out = out.decode("utf-8") if out else ''
err = err.decode("utf-8") if err else ''
message = (f"Command '{' '.join(cmd)}' returned "
f"{proc.returncode}:\n"
f"out = {out}\n"
f"err = {err}\n")
if proc.returncode != 0:
if out:
logger.info('\n%s' % out.decode('utf-8'))
if err: # pragma: debug
logger.info('\n%s' % err.decode('utf-8'))
raise RuntimeError("Command '%s' failed with code %d."
% (' '.join(cmd), proc.returncode))
out = out.decode("utf-8")
logger.debug('%s\n%s' % (' '.join(cmd), out))
raise RuntimeError(message)
logger.debug(message)
return out
except (subprocess.CalledProcessError, OSError) as e: # pragma: debug
raise RuntimeError("Could not call command '%s': %s"
Expand Down

0 comments on commit f63d6a3

Please sign in to comment.