Skip to content

Commit

Permalink
Ignore installation requriements for now
Browse files Browse the repository at this point in the history 
Allow for environment variables at the tool level
  • Loading branch information
@langmm
langmm committed May 22, 2024
1 parent 7681680 commit dd734a9
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 113 deletions.
5 changes: 5 additions & 0 deletions tests/test_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,17 @@ def test_locate_file():
r"""Test file location method."""
# Missing file
assert not tools.locate_file('missing_file.fake')
assert not tools.locate_file('missing_file.fake', use_os=True)
assert not tools.locate_file(['missing_file.fake'])
# Single file
sdir, spat, sans = make_temp_single()
sout = tools.locate_file(spat, verification_func=os.path.isfile)
assert isinstance(sout, (bytes, str))
assert sout == sans[0]
sout = tools.locate_file(spat, verification_func=os.path.isfile,
use_os=True)
assert isinstance(sout, (bytes, str))
assert sout == sans[0]
# Multiple files
mdir, mpat, mans = make_temp_multiple()
with pytest.warns(RuntimeWarning):
Expand Down
3 changes: 3 additions & 0 deletions utils/setup_test_env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1805,6 +1805,9 @@ def verify_pkg(install_opts=None):
from yggdrasil.tools import is_lang_installed, is_comm_installed
errors = []
for name in ['c', 'r', 'fortran', 'sbml', 'lpy', 'julia']:
# TODO: temp
if name in ['r', 'julia']:
continue
flag = install_opts[name]
if flag and (not is_lang_installed(name)):
errors.append("Language '%s' should be installed, but is not."
Expand Down
66 changes: 2 additions & 64 deletions yggdrasil/drivers/CModelDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
from collections import OrderedDict
from yggdrasil import platform, tools, constants, rapidjson
from yggdrasil.drivers.CompiledModelDriver import (
CompiledModelDriver, CompilerBase, LinkerBase, ArchiverBase)
CompiledModelDriver, CompilerBase, LinkerBase, ArchiverBase,
_osx_sysroot)
from yggdrasil.languages import get_language_dir
from yggdrasil.config import ygg_cfg
logger = logging.getLogger(__name__)
if platform._is_win:
logger.setLevel(level=logging.DEBUG)
Expand All @@ -21,48 +21,6 @@
_top_lang_dir = get_language_dir('c')


def get_OSX_SYSROOT():
r"""Determin the path to the OSX SDK.
Returns:
str: Full path to the SDK directory if one is located. None
otherwise.
"""
fname = None
if platform._is_mac:
try:
xcode_dir = subprocess.check_output(
'echo "$(xcode-select -p)"', shell=True).decode("utf-8").strip()
except BaseException: # pragma: debug
xcode_dir = None
fname_try = []
cfg_sdkroot = ygg_cfg.get('c', 'macos_sdkroot', None)
if cfg_sdkroot:
fname_try.append(cfg_sdkroot)
if os.environ.get('SDKROOT', False):
fname_try.append(os.environ['SDKROOT'])
if xcode_dir is not None:
bases_try = [
os.path.join(xcode_dir, 'SDKs', 'MacOSX%s.sdk'),
os.path.join(xcode_dir, 'Platforms',
'MacOSX.platform', 'Developer',
'SDKs', 'MacOSX%s.sdk')]
vers_try = ['11.0', ''] # 11.0 used by conda-forge
if os.environ.get('MACOSX_DEPLOYMENT_TARGET', False):
vers_try.insert(0, os.environ['MACOSX_DEPLOYMENT_TARGET'])
for v in vers_try:
fname_try += [x % v for x in bases_try]
for fcheck in fname_try:
if os.path.isdir(fcheck):
fname = fcheck
break
return fname


_osx_sysroot = get_OSX_SYSROOT()


class CCompilerBase(CompilerBase):
r"""Base class for C compilers."""
languages = ['c']
Expand Down Expand Up @@ -502,26 +460,6 @@ class CModelDriver(CompiledModelDriver):
'compiler_flags': ['-fPIC'],
'external_dependencies': ['m'],
},
'MacOS': {
'global_env': {
'CONDA_BUILD_SYSROOT': {
'value': _osx_sysroot,
'overwrite': True,
},
'SDKROOT': {
'value': _osx_sysroot,
'overwrite': True,
},
'MACOSX_DEPLOYMENT_TARGET': {
'value': (
re.search(
r'MacOSX(?P<target>[0-9]+\.[0-9]+)?',
_osx_sysroot).groupdict()['target']
if _osx_sysroot else False),
'overwrite': True,
}
}
}
}},
'regex_win32': {'name': 'regex',
'source': 'regex_win32.cpp',
Expand Down
137 changes: 108 additions & 29 deletions yggdrasil/drivers/CompiledModelDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,49 @@
_library_types = ['include', 'static', 'shared', 'windows_import']


def get_OSX_SYSROOT():
r"""Determin the path to the OSX SDK.
Returns:
str: Full path to the SDK directory if one is located. None
otherwise.
"""
fname = None
if platform._is_mac:
from yggdrasil.config import ygg_cfg
try:
xcode_dir = subprocess.check_output(
'echo "$(xcode-select -p)"', shell=True).decode("utf-8").strip()
except BaseException: # pragma: debug
xcode_dir = None
fname_try = []
cfg_sdkroot = ygg_cfg.get('c', 'macos_sdkroot', None)
if cfg_sdkroot:
fname_try.append(cfg_sdkroot)
if os.environ.get('SDKROOT', False):
fname_try.append(os.environ['SDKROOT'])
if xcode_dir is not None:
bases_try = [
os.path.join(xcode_dir, 'SDKs', 'MacOSX%s.sdk'),
os.path.join(xcode_dir, 'Platforms',
'MacOSX.platform', 'Developer',
'SDKs', 'MacOSX%s.sdk')]
vers_try = ['11.0', ''] # 11.0 used by conda-forge
if os.environ.get('MACOSX_DEPLOYMENT_TARGET', False):
vers_try.insert(0, os.environ['MACOSX_DEPLOYMENT_TARGET'])
for v in vers_try:
fname_try += [x % v for x in bases_try]
for fcheck in fname_try:
if os.path.isdir(fcheck):
fname = fcheck
break
return fname


_osx_sysroot = get_OSX_SYSROOT()


class CompilationToolError(Exception):
r"""Class for errors related to compilation tools"""
pass
Expand Down Expand Up @@ -3460,6 +3503,23 @@ def _shared_path_env(self, to_update=None, out=None,
out[env_var] = os.pathsep.join(path_list)
return out

@classmethod
def _update_env_val(cls, k, v, out, to_update=None):
if to_update is None:
to_update = out
append_char = None
overwrite = False
if isinstance(v, dict):
append_char = v.get('append', None)
overwrite = v.get('overwrite', False)
v = v['value']
if not v:
return
if overwrite or k not in to_update:
out[k] = v
elif append_char:
out[k] = to_update[k] + append_char + v

def _update_env(self, vals, to_update=None, out=None,
dont_update_return=False):
if to_update is None:
Expand All @@ -3472,18 +3532,7 @@ def _update_env(self, vals, to_update=None, out=None,
else:
to_update.update(out)
for k, v in vals.items():
append_char = None
overwrite = False
if isinstance(v, dict):
append_char = v.get('append', None)
overwrite = v.get('overwrite', False)
v = v['value']
if not v:
continue
if overwrite or k not in to_update:
out[k] = v
elif append_char:
out[k] = to_update[k] + append_char + v
self._update_env_val(k, v, out, to_update=to_update)
return out

def _generic_env(self, key, to_update=None, for_build=False,
Expand Down Expand Up @@ -3584,12 +3633,16 @@ def _search_linked(self, filetype, **kwargs):
if not flags:
if self.origin == 'standard':
flags.append(f'-l{self.name}')
for lib in self.basetool.find_component(
self.name, cfg=self.cfg, flags=flags,
component_types='shared_libraries', **kwargs):
out = self._search_brute(lib, libtype='shared', **kwargs)
if out:
break
try:
for lib in self.basetool.find_component(
self.name, cfg=self.cfg, flags=flags,
component_types='shared_libraries', **kwargs):
out = self._search_brute(lib, libtype='shared', **kwargs)
if out:
break
except RuntimeError as e:
logger.debug(f"Error in using diassembly to locate "
f"\'{self.name}\': {e}")
return out

def _search_brute(self, fname, libtype=None, verbose=False,
Expand Down Expand Up @@ -3635,7 +3688,7 @@ def _search_brute(self, fname, libtype=None, verbose=False,
fname_ext = expected_ext[0]
if os.path.isfile(fname):
return fname
use_regex = True # (not platform._is_win)
# use_regex = (not platform._is_win)
fname_try = [fname_base]
if platform._is_win and libtype in self.library_files:
if fname_base.startswith('lib'):
Expand All @@ -3645,19 +3698,15 @@ def _search_brute(self, fname, libtype=None, verbose=False,
search_list = self.tool(libtype).get_search_path(
libtype=libtype, cfg=self.cfg, **kwargs)
for fname_base in fname_try:
use_glob = fname_base + '*' + fname_ext
if use_regex:
fname = (
r'[^a-zA-Z]'
+ tools.escape_regex(fname_base)
+ r'([^a-zA-Z].*)?'
+ tools.escape_regex(fname_ext))
else:
fname = use_glob
fname = fname_base + '*' + fname_ext
use_regex = (
r'[^a-zA-Z]'
+ tools.escape_regex(fname_base)
+ r'([^a-zA-Z].*)?'
+ tools.escape_regex(fname_ext))
out = tools.locate_file(fname, directory_list=search_list,
environment_variable=None,
use_regex=use_regex,
use_glob=use_glob,
select_return='shortest')
if out:
break
Expand Down Expand Up @@ -3715,6 +3764,11 @@ class CompilationToolBase(object):
[REQUIRED]
platforms (list): Platforms that the tool is available on. Defaults to
['Windows', 'MacOS', 'Linux'].
env (dict): Environment variables that should be updated when
calling the tool.
env_platform_specific (dict): Mapping of platform specific
environment variables that should be updated when calling the
tool.
default_executable (str): The default tool executable command if
different than the toolname.
default_executable_env (str): Environment variable where the executable
Expand Down Expand Up @@ -3824,6 +3878,27 @@ class CompilationToolBase(object):
output_filetypes = []
languages = []
platforms = ['Windows', 'MacOS', 'Linux'] # all by default
env = {}
env_platform_specific = {
'MacOS': {
'CONDA_BUILD_SYSROOT': {
'value': _osx_sysroot,
'overwrite': True,
},
'SDKROOT': {
'value': _osx_sysroot,
'overwrite': True,
},
'MACOSX_DEPLOYMENT_TARGET': {
'value': (
re.search(
r'MacOSX(?P<target>[0-9]+\.[0-9]+)?',
_osx_sysroot).groupdict()['target']
if _osx_sysroot else False),
'overwrite': True,
},
}
}
default_executable = None
default_executable_env = None
default_flags = []
Expand Down Expand Up @@ -4138,6 +4213,10 @@ def set_env(cls, existing=None, **kwargs):
if existing is None:
existing = {}
existing.update(os.environ)
for k, v in dict(cls.env, **cls.env_platform_specific.get(
platform._platform, {})).items():
CompilationDependency._update_env_val(k, v, existing,
to_update=existing)
if not cls.env_matches_tool():
config_vars = {}
cls.env_matches_tool(use_sysconfig=True, env=config_vars)
Expand Down
38 changes: 18 additions & 20 deletions yggdrasil/tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,7 @@ def convert_regex_to_find(name):


def find_all(name, path, verification_func=None, use_regex=False,
use_glob=False):
use_os=False):
r"""Find all instances of a file with a given name within the directory
tree starting at a given path.
Expand All @@ -631,29 +631,22 @@ def find_all(name, path, verification_func=None, use_regex=False,
verification_func (function, optional): Function that returns
True when a file is valid and should be returned and False
otherwise. Defaults to None and is ignored.
use_regex (bool, optional): If True, use full regex to interpret
name and locate files.
use_glob (bool, str, optional): If True or string, use glob
to locate the file. If use_regex is True, a string must be
provided containing the glob search expression.
use_regex (bool, str, optional): If True or string, use full
regex to interpret name and locate files. If a string is
provided, it will be used as the regex pattern, otherwise
name will be used. A string is required unless use_os is True
use_os (bool, optional): If True, use a system tool to search
for the file (find on unix, where on windows).
Returns:
list: All instances of the specified file.
"""
result = []
args = []
if use_glob:
fglob = os.path.join(path, name)
if use_regex:
assert isinstance(use_glob, str)
fglob = os.path.join(path, use_glob)
result = glob.glob(fglob)
if use_regex:
result = [
x for x in result
if re.fullmatch(r'.*' + name, x)]
else:
regex_pattern = use_regex if isinstance(use_regex, str) else name
regex_pattern = r'.*' + regex_pattern # to match the directory
if use_os:
try:
if platform._is_win: # pragma: windows
assert not use_regex
Expand All @@ -672,8 +665,8 @@ def find_all(name, path, verification_func=None, use_regex=False,
shell = False
args = ["find", "-L", path, "-type", "f"]
if use_regex:
name = convert_regex_to_find(name)
args += ["-regex", r'.*' + name]
regex_pattern = convert_regex_to_find(regex_pattern)
args += ["-regex", regex_pattern]
args.insert(1, "-E")
args = ' '.join(args)
shell = True
Expand All @@ -697,6 +690,11 @@ def find_all(name, path, verification_func=None, use_regex=False,
out = ''
if not out.isspace():
result = sorted(out.splitlines())
else:
result = glob.glob(os.path.join(path, name))
if use_regex:
result = [
x for x in result if re.fullmatch(regex_pattern, x)]
result = [os.path.normcase(os.path.normpath(bytes2str(m)))
for m in result]
if verification_func is not None:
Expand Down Expand Up @@ -752,7 +750,7 @@ def locate_file(fname, environment_variable='PATH', directory_list=None,
out = []
if ((platform._is_win and (environment_variable == 'PATH')
and (directory_list is None)
and not kwargs.get('use_glob', False))): # pragma: windows
and kwargs.get('use_os', False))): # pragma: windows
out += find_all(fname, None, **kwargs)
else:
if directory_list is None:
Expand Down

0 comments on commit dd734a9

Please sign in to comment.