From 443680789163d9080b7e00b91e2e7ba2e1aae865 Mon Sep 17 00:00:00 2001 From: Meagan Lang Date: Tue, 28 May 2024 20:06:16 -0400 Subject: [PATCH] Fix test for invalid compilation tool on base class Try changing order that msvc compiler initialized before conda Remove CFLAGS from compilation arguments in example makefiles to prevent incorrect linking on mac arm64 to x86-64 Return results from find_all in sorted order Add -fPIC flag to fortran interface compilation arguments for GNU Allow for basetool to be None in case of missing compilation tools Handle case of clang on windows when parsing version --- .github/workflows/test-install.yml | 40 +++++++++---------- tests/drivers/test_ModelDriver.py | 5 ++- utils/test-install-base.yml | 14 ++++--- yggdrasil/drivers/CMakeModelDriver.py | 6 ++- yggdrasil/drivers/CModelDriver.py | 4 +- yggdrasil/drivers/CompiledModelDriver.py | 8 +++- yggdrasil/drivers/FortranModelDriver.py | 3 ++ .../examples/backwards/src/Makefile_linux | 4 +- .../examples/gs_lesson4/src/Makefile_linux | 4 +- .../model_error_with_io/src/Makefile_linux | 4 +- yggdrasil/tools.py | 5 ++- 11 files changed, 57 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 7aa07eb82..6e7294d94 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -447,6 +447,10 @@ jobs: ' shell: bash -l {0} + - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && + runner.os == 'Windows' && env.INSTALLC == 1 + name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 @@ -457,10 +461,6 @@ jobs: miniforge-variant: Mambaforge python-version: ${{ matrix.python-version }} use-mamba: true - - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && - runner.os == 'Windows' && env.INSTALLC == 1 - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - continue-on-error: true id: conda_build if: matrix.install-method == 'conda' || matrix.install-method == 'mamba' @@ -845,6 +845,10 @@ jobs: ' shell: bash -l {0} + - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && + runner.os == 'Windows' && env.INSTALLC == 1 + name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 @@ -855,10 +859,6 @@ jobs: miniforge-variant: Mambaforge python-version: ${{ matrix.python-version }} use-mamba: true - - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && - runner.os == 'Windows' && env.INSTALLC == 1 - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - continue-on-error: true id: conda_build if: matrix.install-method == 'conda' || matrix.install-method == 'mamba' @@ -1219,6 +1219,10 @@ jobs: ' shell: bash -l {0} + - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && + runner.os == 'Windows' && env.INSTALLC == 1 + name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 @@ -1229,10 +1233,6 @@ jobs: miniforge-variant: Mambaforge python-version: ${{ matrix.python-version }} use-mamba: true - - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && - runner.os == 'Windows' && env.INSTALLC == 1 - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - continue-on-error: true id: conda_build if: matrix.install-method == 'conda' || matrix.install-method == 'mamba' @@ -1574,6 +1574,10 @@ jobs: ' shell: bash -l {0} + - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && + runner.os == 'Windows' && env.INSTALLC == 1 + name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 @@ -1584,10 +1588,6 @@ jobs: miniforge-variant: Mambaforge python-version: ${{ matrix.python-version }} use-mamba: true - - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && - runner.os == 'Windows' && env.INSTALLC == 1 - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - continue-on-error: true id: conda_build if: matrix.install-method == 'conda' || matrix.install-method == 'mamba' @@ -1928,6 +1928,10 @@ jobs: ' shell: bash -l {0} + - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && + runner.os == 'Windows' && env.INSTALLC == 1 + name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 @@ -1938,10 +1942,6 @@ jobs: miniforge-variant: Mambaforge python-version: ${{ matrix.python-version }} use-mamba: true - - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && - runner.os == 'Windows' && env.INSTALLC == 1 - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - continue-on-error: true id: conda_build if: matrix.install-method == 'conda' || matrix.install-method == 'mamba' diff --git a/tests/drivers/test_ModelDriver.py b/tests/drivers/test_ModelDriver.py index 7fe09e77f..5a61b86c6 100644 --- a/tests/drivers/test_ModelDriver.py +++ b/tests/drivers/test_ModelDriver.py @@ -8,7 +8,7 @@ from yggdrasil import platform, constants from yggdrasil.drivers.ModelDriver import ModelDriver from yggdrasil.drivers.CompiledModelDriver import ( - CompiledModelDriver, InvalidCompilationTool) + CompiledModelDriver, InvalidCompilationTool, get_tool_registry) from yggdrasil.drivers.InterpretedModelDriver import InterpretedModelDriver @@ -20,8 +20,9 @@ def test_ModelDriver_implementation(): ModelDriver.executable_command(None) with pytest.raises(NotImplementedError): ModelDriver.is_library_installed(None) + get_tool_registry()._init_languages(['c++']) with pytest.raises(InvalidCompilationTool): - print(CompiledModelDriver.get_tool('compiler')) + CompiledModelDriver.get_tool('compiler') with pytest.raises(NotImplementedError): CompiledModelDriver.language_executable() with pytest.raises(NotImplementedError): diff --git a/utils/test-install-base.yml b/utils/test-install-base.yml index 4633128af..321a7f69f 100644 --- a/utils/test-install-base.yml +++ b/utils/test-install-base.yml @@ -231,6 +231,10 @@ jobs: run: | echo "PATH=$(python utils/setup_test_env.py prune-path)" >> $GITHUB_ENV shell: bash -l {0} + # Try initializing compiler first + - name: (WINDOWS,CONDA) Set up MSVC Compiler + uses: ilammy/msvc-dev-cmd@v1 + if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && runner.os == 'Windows' && env.INSTALLC == 1 - name: (CONDA) Set up miniconda base environment uses: conda-incubator/setup-miniconda@v3 if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') @@ -247,11 +251,11 @@ jobs: # initialization # This step might need to stay here so that dev prompt comes before # conda? - - name: (WINDOWS,CONDA) Set up MSVC Compiler - uses: ilammy/msvc-dev-cmd@v1 - if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && runner.os == 'Windows' && env.INSTALLC == 1 - # with: - # toolset: 14.0 + # - name: (WINDOWS,CONDA) Set up MSVC Compiler + # uses: ilammy/msvc-dev-cmd@v1 + # if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && runner.os == 'Windows' && env.INSTALLC == 1 + # # with: + # # toolset: 14.0 # - name: (UNIX,CONDA) Add miniconda bin to path # if: (matrix.install-method == 'conda' || matrix.install-method == 'mamba') && runner.os != 'Windows' # run: | diff --git a/yggdrasil/drivers/CMakeModelDriver.py b/yggdrasil/drivers/CMakeModelDriver.py index 2000510e8..f6c4f219d 100644 --- a/yggdrasil/drivers/CMakeModelDriver.py +++ b/yggdrasil/drivers/CMakeModelDriver.py @@ -825,8 +825,10 @@ def create_imports(cls, dep, imp, products=None, overwrite=False, logger.debug(f"CREATE_IMPORTS {dep} {imp}") buildfile = dep['buildfile'] if not os.path.isabs(buildfile): - buildfile = dep._relative_to_directory(buildfile) - logger.info(f"CREATE_IMPORTS {dep}: {buildfile}") + buildfile = dep._relative_to_directory( + buildfile, directory=dep.get('working_dir', None)) + logger.info(f"CREATE_IMPORTS {dep}: {buildfile}\n" + f"source_dir: {dep.get('source_dir', None)}") target = dep.get('target', '${PROJECT_NAME}') if products is None: products = tools.IntegrationPathSet(overwrite=overwrite) diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py index 71e176e59..bc14ea451 100755 --- a/yggdrasil/drivers/CModelDriver.py +++ b/yggdrasil/drivers/CModelDriver.py @@ -327,8 +327,8 @@ def get_flags(cls, *args, **kwargs): # https://bugs.llvm.org/show_bug.cgi?id=44813 # https://reviews.llvm.org/D71579 # https://reviews.llvm.org/D74784 - ver = cls.tool_version().split()[-1] - if int(ver.split('.')[0]) >= 10: + ver = cls.tool_version().split()[-1].split('.')[0] + if ver.isnumeric() and int(ver) >= 10: ld_version = LDLinker.tool_version() if float(ld_version.split('.')[0]) < 520: # pragma: version # No longer covered as the default conda diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index c89b32730..475df4df9 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -157,6 +157,10 @@ def _check_driver(self, language=None, driver=None, driver = None elif driver: language = driver.language + if not language: + raise InvalidCompilationTool(f"Driver {driver} does not " + f"does not have a language " + f"set") if language and (not skip_driver) and (not driver): driver = import_component('model', language) return (language, driver) @@ -1873,9 +1877,11 @@ def all_parameters(self, generalize=False, only_active=False): tooltypes = _tool_types elif only_active: tooltypes = self.active_tools() - else: + elif self.basetool: tooltypes = [self.basetool.tooltype] tooltypes += self.basetool.associated_tooltypes + else: + tooltypes = [] for k in tooltypes: out += self.tool_parameters(k, generalize=generalize) if generalize or only_active: diff --git a/yggdrasil/drivers/FortranModelDriver.py b/yggdrasil/drivers/FortranModelDriver.py index da111e252..83be92107 100644 --- a/yggdrasil/drivers/FortranModelDriver.py +++ b/yggdrasil/drivers/FortranModelDriver.py @@ -216,6 +216,9 @@ class FortranModelDriver(CompiledModelDriver): if tools.get_conda_prefix() else False } } + }, + 'Linux': { + 'compiler_flags': ['-fPIC'], } }}, 'c_wrappers': { diff --git a/yggdrasil/examples/backwards/src/Makefile_linux b/yggdrasil/examples/backwards/src/Makefile_linux index 372cfde5a..4b50115ff 100644 --- a/yggdrasil/examples/backwards/src/Makefile_linux +++ b/yggdrasil/examples/backwards/src/Makefile_linux @@ -13,10 +13,10 @@ $(ODIR)/%.o: %.c $(CC) -c $(CFLAGS) $< -o $@ backwards_modelA: $(ODIR)/backwards_modelA.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) backwards_modelB: $(ODIR)/backwards_modelB.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) .PHONY: clean clean: diff --git a/yggdrasil/examples/gs_lesson4/src/Makefile_linux b/yggdrasil/examples/gs_lesson4/src/Makefile_linux index 0bd78adb8..6c7518e9c 100644 --- a/yggdrasil/examples/gs_lesson4/src/Makefile_linux +++ b/yggdrasil/examples/gs_lesson4/src/Makefile_linux @@ -13,10 +13,10 @@ $(ODIR)/%.o: %.c $(CC) -c $(CFLAGS) $< -o $@ gs_lesson4_modelA: $(ODIR)/gs_lesson4_modelA.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) gs_lesson4_modelB: $(ODIR)/gs_lesson4_modelB.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) .PHONY: clean clean: diff --git a/yggdrasil/examples/model_error_with_io/src/Makefile_linux b/yggdrasil/examples/model_error_with_io/src/Makefile_linux index ebc2431dd..4df74dbbc 100644 --- a/yggdrasil/examples/model_error_with_io/src/Makefile_linux +++ b/yggdrasil/examples/model_error_with_io/src/Makefile_linux @@ -13,10 +13,10 @@ $(ODIR)/%.o: %.c $(CC) -c $(CFLAGS) $< -o $@ model_error_with_io_modelA: $(ODIR)/model_error_with_io_modelA.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) model_error_with_io_modelB: $(ODIR)/model_error_with_io_modelB.o - $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + $(CC) -o $@ $^ $(LIBS) .PHONY: clean clean: diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py index f5221b974..f68df2025 100644 --- a/yggdrasil/tools.py +++ b/yggdrasil/tools.py @@ -695,8 +695,9 @@ def find_all(name, path, verification_func=None, use_regex=False, if use_regex: result = [ x for x in result if re.fullmatch(regex_pattern, x)] - result = [os.path.normcase(os.path.normpath(bytes2str(m))) - for m in result] + result = sorted( + [os.path.normcase(os.path.normpath(bytes2str(m))) + for m in result]) if verification_func is not None: result = [x for x in result if verification_func(x)] return result