Merge pull request #4859 from lexming/modenvvar-expand-paths

move `EasyBlock.expand_module_search_path` into `ModuleEnvironmentVariable.expand_paths`

Author: boegel

easybuild/framework/easyblock.py

@@ -93,11 +93,10 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock
 from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files
-from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs
-from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum
-from easybuild.tools.filetools import weld_paths, write_file
+from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate
+from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path
+from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
+from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
     BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
     MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
@@ -1709,10 +1708,7 @@ def make_module_req(self):
                 mod_req_paths = search_paths
                 self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}")
             else:
-                mod_req_paths = [
-                    expanded_path for unexpanded_path in search_paths
-                    for expanded_path in self.expand_module_search_path(unexpanded_path, path_type=search_paths.type)
-                ]
+                mod_req_paths = search_paths.expand_paths(self.installdir)
 
             if mod_req_paths:
                 mod_req_paths = nub(mod_req_paths)  # remove duplicates
@@ -1786,64 +1782,10 @@ def inject_module_extra_paths(self):
 
     def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES):
         """
-        Expand given path glob and return list of suitable paths to be used as search paths:
-            - Paths must point to existing files/directories
-            - Relative paths are relative to installation prefix root and are kept relative after expansion
-            - Absolute paths are kept as absolute paths after expansion
-            - Follow symlinks and resolve their paths (avoids duplicate paths through symlinks)
-            - :path_type: ModEnvVarType that controls requirements for population of directories
-              - PATH: no requirements, can be empty
-              - PATH_WITH_FILES: must contain at least one file in them (default)
-              - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory
-        """
-        populated_path_types = (
-            ModEnvVarType.PATH_WITH_FILES,
-            ModEnvVarType.PATH_WITH_TOP_FILES,
-            ModEnvVarType.STRICT_PATH_WITH_FILES,
-        )
-
-        if os.path.isabs(search_path):
-            abs_glob = search_path
-        else:
-            real_installdir = os.path.realpath(self.installdir)
-            abs_glob = os.path.join(real_installdir, search_path)
-
-        exp_search_paths = glob.glob(abs_glob, recursive=True)
-
-        retained_search_paths = []
-        for abs_path in exp_search_paths:
-            # avoid going through symlink for strict path types
-            if path_type is ModEnvVarType.STRICT_PATH_WITH_FILES and abs_path != os.path.realpath(abs_path):
-                self.log.debug(
-                    f"Discarded strict search path '{search_path} of type '{path_type}' that does not correspond "
-                    f"to its real path: {abs_path}"
-                )
-                continue
-
-            if os.path.isdir(abs_path) and path_type in populated_path_types:
-                # only retain paths to directories that contain at least one file
-                recursive = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES)
-                if not dir_contains_files(abs_path, recursive=recursive):
-                    self.log.debug("Discarded search path to empty directory: %s", abs_path)
-                    continue
-
-            if os.path.isabs(search_path):
-                retain_path = abs_path
-            else:
-                # recover relative path
-                retain_path = os.path.relpath(os.path.realpath(abs_path), start=real_installdir)
-                if retain_path == '.':
-                    retain_path = ''  # use empty string to represent root of install dir
-
-            if retain_path.startswith('..' + os.path.sep):
-                raise EasyBuildError(
-                    f"Expansion of search path glob pattern '{search_path}' resulted in a relative path "
-                    f"pointing outside of install directory: {retain_path}"
-                )
-
-            retained_search_paths.append(retain_path)
-
-        return retained_search_paths
+        REMOVED in EasyBuild 5.1, use EasyBlock.module_load_environment.expand_paths instead
+        """
+        msg = "expand_module_search_path is replaced by EasyBlock.module_load_environment.expand_paths"
+        self.log.nosupport(msg, '5.1')
 
     def make_module_req_guess(self):
         """

easybuild/tools/modules.py

@@ -50,7 +50,8 @@
 from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET
 from easybuild.tools.config import build_option, get_modules_tool, install_path
 from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars
-from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
+from easybuild.tools.filetools import convert_name, dir_contains_files, mkdir, normalize_path, path_matches, read_file
+from easybuild.tools.filetools import which, write_file
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
@@ -237,6 +238,7 @@ def remove(self, *args):
 
     @property
     def is_path(self):
+        """Return True for any ModEnvVarType that is a path"""
         path_like_types = [
             ModEnvVarType.PATH,
             ModEnvVarType.PATH_WITH_FILES,
@@ -245,6 +247,81 @@ def is_path(self):
         ]
         return self.type in path_like_types
 
+    def expand_paths(self, parent):
+        """
+        Expand path glob into list of unique corresponding real paths.
+        General behaviour:
+        - Only expand path-like variables
+        - Paths must point to existing files/directories
+        - Resolve paths following symlinks into real paths to avoid duplicate
+          paths through symlinks
+        - Relative paths are expanded on given parent folder and are kept
+          relative after expansion
+        - Absolute paths are kept as absolute paths after expansion
+        Follow requirements based on current type (ModEnvVarType):
+          - PATH: no requirements, must exist but can be empty
+          - PATH_WITH_FILES: must contain at least one file anywhere in subtree
+          - PATH_WITH_TOP_FILES: must contain files in top level directory of path
+          - STRICT_PATH_WITH_FILES: given path must expand into its real path and
+            contain files anywhere in subtree
+        """
+        if not self.is_path:
+            return None
+
+        populated_path_types = (
+            ModEnvVarType.PATH_WITH_FILES,
+            ModEnvVarType.PATH_WITH_TOP_FILES,
+            ModEnvVarType.STRICT_PATH_WITH_FILES,
+        )
+
+        retained_expanded_paths = []
+        real_parent = os.path.realpath(parent)
+
+        for path_glob in self.contents:
+            abs_glob = path_glob
+            if not os.path.isabs(path_glob):
+                abs_glob = os.path.join(real_parent, path_glob)
+
+            expanded_paths = glob.glob(abs_glob, recursive=True)
+
+            for exp_path in expanded_paths:
+                real_path = os.path.realpath(exp_path)
+
+                if self.type is ModEnvVarType.STRICT_PATH_WITH_FILES and exp_path != real_path:
+                    # avoid going through symlink for strict path types
+                    self.log.debug(
+                        f"Discarded search path '{exp_path} of type '{self.type}' as it does not correspond "
+                        f"to its real path: {real_path}"
+                    )
+                    continue
+
+                if os.path.isdir(exp_path) and self.type in populated_path_types:
+                    # only retain paths to directories that contain at least one file
+                    recursive = self.type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES)
+                    if not dir_contains_files(exp_path, recursive=recursive):
+                        self.log.debug(f"Discarded search path '{exp_path}' of type '{self.type}' to empty directory.")
+                        continue
+
+                retain_path = exp_path  # no discards, we got a keeper
+
+                if not os.path.isabs(path_glob):
+                    # recover relative path
+                    retain_path = os.path.relpath(real_path, start=real_parent)
+                    # modules use empty string to represent root of install dir
+                    if retain_path == '.':
+                        retain_path = ''
+
+                if retain_path.startswith('..' + os.path.sep):
+                    raise EasyBuildError(
+                        f"Expansion of search path glob pattern '{path_glob}' resulted in a relative path "
+                        f"pointing outside of parent directory: {retain_path}"
+                    )
+
+                if retain_path not in retained_expanded_paths:
+                    retained_expanded_paths.append(retain_path)
+
+        return retained_expanded_paths
+
 
 class ModuleLoadEnvironment:
     """