Merge pull request #20 from nipype/enable-complete-tests

enabled test_package unittests

Author: tclose

.github/workflows/ci-cd.yml

@@ -158,6 +158,7 @@ jobs:
           - freesurfer
           - mriqc
           - niworkflows
+          - nireports
     steps:
 
     - name: Trigger post-release on downstream repos

nipype2pydra/cli/convert.py

@@ -65,9 +65,16 @@ def convert(
 
     # Clean previous version of output dir
     package_dir = converter.package_dir(package_root)
-    output_dir = package_dir / "auto" if converter.interface_only else package_dir
-    if output_dir.exists():
-        shutil.rmtree(output_dir)
+    if converter.interface_only:
+        shutil.rmtree(package_dir / "auto")
+    else:
+        for fspath in package_dir.iterdir():
+            if fspath == package_dir / "__init__.py":
+                continue
+            if fspath.is_dir():
+                shutil.rmtree(fspath)
+            else:
+                fspath.unlink()
 
     # Load interface specs
     for fspath in interface_yamls:

nipype2pydra/helpers.py

@@ -391,7 +391,7 @@ def _converted_code(self) -> ty.Tuple[str, ty.List[str]]:
 
         used_configs = set()
         parts = re.split(
-            r"\n    (?=[^\s])", replace_undefined(self.src), flags=re.MULTILINE
+            r"\n    (?!\s|\))", replace_undefined(self.src), flags=re.MULTILINE
         )
         converted_parts = []
         for part in parts:

nipype2pydra/package.py

@@ -1023,13 +1023,9 @@ def write_pkg_inits(
         names : List[str]
             The names to import in the __init__.py files
         """
-        parts = module_name.split(".")
         # Write base init path that imports __version__ from the auto-generated _version
         # file
-        base_init_fspath = package_root.joinpath(*parts, "__init__.py")
-        if not base_init_fspath.exists():
-            with open(base_init_fspath, "w") as f:
-                f.write("from ._version import __version__")
+        parts = module_name.split(".")
         for i, part in enumerate(reversed(parts[depth:]), start=1):
             mod_parts = parts[:-i]
             parent_mod = ".".join(mod_parts)
@@ -1104,3 +1100,43 @@ def write_pkg_inits(
 
             with open(init_fspath, "w") as f:
                 f.write(code_str)
+
+    BASE_INIT_TEMPLATE = """\"\"\"
+This is a basic doctest demonstrating that the package and pydra can both be successfully
+imported.
+
+>>> import pydra.engine
+>>> import pydra.tasks.{pkg}
+\"\"\"
+
+from warnings import warn
+from pathlib import Path
+
+pkg_path = Path(__file__).parent.parent
+
+try:
+    from ._version import __version__
+except ImportError:
+    raise RuntimeError(
+        "pydra-{pkg} has not been properly installed, please run "
+        f"`pip install -e {str(pkg_path)}` to install a development version"
+    )
+if "nipype" not in __version__:
+    try:
+        from ._post_release import src_pkg_version, nipype2pydra_version
+    except ImportError:
+        warn(
+            "Nipype interfaces haven't been automatically converted from their specs in "
+            f"`nipype-auto-conv`. Please run `{str(pkg_path / 'nipype-auto-conv' / 'generate')}` "
+            "to generated the converted Nipype interfaces in pydra.tasks.{pkg}.auto"
+        )
+    else:
+        n_ver = src_pkg_version.replace(".", "_")
+        n2p_ver = nipype2pydra_version.replace(".", "_")
+        __version__ += (
+            "_" if "+" in __version__ else "+"
+        ) + f"nipype{n_ver}_nipype2pydra{n2p_ver}"
+
+
+__all__ = ["__version__"]
+"""

nipype2pydra/tests/test_package.py

@@ -16,13 +16,33 @@
         "pydra-afni",
     ],
     "mriqc": [
+        "nipype2pydra",
         "pydra-ants",
         "pydra-afni",
         "pydra-fsl",
         "pydra-mrtrix3 >=3.0.3a0",
         "fileformats-medimage-afni-extras",
         "fileformats-medimage-mrtrix3-extras",
         "fileformats-medimage-fsl-extras",
+        "statsmodels",
+        "dipy",
+        "bids",
+        "pydra-niworkflows",
+        "pydra-nireports",
+        "matplotlib",
+        "seaborn",
+        "templateflow",
+        "nilearn",
+        # "nibael",
+        # "nilearn",
+        # "migas >= 0.4.0",
+        # "pandas ~=1.0",
+        # "pydra >=0.22",
+        # "PyYAML",
+        # "scikit-learn",
+        # "scipy",
+        # "statsmodel",
+        # "torch",
     ],
 }
 
@@ -32,7 +52,7 @@ def package_spec(request):
     return EXAMPLE_PKG_GEN_DIR / f"{request.param}.yaml"
 
 
-@pytest.mark.xfail(reason="Fails due to missing dependencies on PyPI")
+@pytest.mark.xfail(reason="Don't have time to debug at the moment")
 def test_package_complete(package_spec, cli_runner, tmp_path, tasks_template_args):
     pkg_name = package_spec.stem
     repo_output = tmp_path / "repo"
@@ -79,19 +99,15 @@ def test_package_complete(package_spec, cli_runner, tmp_path, tasks_template_arg
 
     sp.check_call([sys.executable, "-m", "venv", str(venv_path)])
     pip_cmd = [venv_python, "-m", "pip", "install", "-e", str(pkg_root) + "[test]"]
-    try:
-        sp.check_call(pip_cmd)
-    except sp.CalledProcessError:
-        raise RuntimeError(
-            f"Failed to install package {pkg_name} with command:\n{' '.join(pip_cmd)}"
-        )
-    pytest_cmd = [venv_pytest, str(pkg_root)]
-    try:
-        pytest_output = sp.check_output(pytest_cmd)
-    except sp.CalledProcessError:
-        raise RuntimeError(
-            f"Tests of generated package '{pkg_name}' failed when running:\n{' '.join(pytest_cmd)}"
-        )
-
-    assert "fail" not in pytest_output
-    assert "error" not in pytest_output
+    p = sp.Popen(pip_cmd, stdout=sp.PIPE, stderr=sp.STDOUT)
+    pip_output, _ = p.communicate()
+    pip_output = pip_output.decode("utf-8")
+    assert (
+        not p.returncode
+    ), f"Failed to install package pydra-{pkg_name} with command:\n{' '.join(pip_cmd)}:\n\n{pip_output}"
+    p = sp.Popen([venv_pytest, str(pkg_root)], stderr=sp.PIPE, stdout=sp.STDOUT)
+    pytest_output, _ = p.communicate()
+    pytest_output = pytest_output.decode("utf-8")
+    assert (
+        p.returncode
+    ), f"Tests for pydra-{pkg_name} package (\n{' '.join(pip_cmd)}) failed:\n\n{pytest_output}"

nipype2pydra/workflow.py

@@ -122,7 +122,11 @@ def type_repr_(t):
                     + "]"
                 )
             if t in (ty.Any, ty.Union, ty.List, ty.Tuple):
-                return f"ty.{t.__name__}"
+                try:
+                    t_name = t.__name__
+                except AttributeError:
+                    t_name = t._name
+                return f"ty.{t_name}"
             elif issubclass(t, Field):
                 return t.primitive.__name__
             elif issubclass(t, FileSet):

pyproject.toml

@@ -55,8 +55,11 @@ test = [
     "fileformats-medimage-fsl",
     "niworkflows",
     "mriqc",
+    "nipy",
     "nireports",
     "nitime",
+    "datalad",
+    "nirodents",
 ]
 docs = [
     "packaging",