debugged changes so that all packages build successfully again

Author: tclose

nipype2pydra/interface/base.py

@@ -42,6 +42,7 @@
 )
 from fileformats.generic import File
 import nipype2pydra.package
+from nipype2pydra.exceptions import UnmatchedParensException
 
 logger = logging.getLogger("nipype2pydra")
 
@@ -498,7 +499,10 @@ def _referenced_funcs_and_methods(self):
             method_returns[method_name] = []
             method_stacks[method_name] = ()
         for method_name in self.INCLUDED_METHODS:
-            if method_name not in self.nipype_interface.__dict__:
+            base = find_super_method(
+                self.nipype_interface, method_name, include_class=True
+            )[1]
+            if self.package.is_omitted(base):
                 continue  # Don't include base methods
             method = getattr(self.nipype_interface, method_name)
             referenced_methods.add(method)
@@ -1103,9 +1107,7 @@ def _get_referenced(
             )
         for match in re.findall(r"super\([^\)]*\)\.(\w+)\(", method_body):
             super_method, base = find_super_method(super_base, match)
-            if any(
-                base.__module__.startswith(m) for m in UsedSymbols.ALWAYS_OMIT_MODULES
-            ):
+            if self.package.is_omitted(super_method):
                 continue
             func_name = self._common_parent_pkg_prefix(base) + match
             if func_name not in referenced_supers:
@@ -1296,28 +1298,28 @@ def replace_supers(self, method_body, super_base=None):
         if super_base is None:
             super_base = self.nipype_interface
         name_map = self.method_supers[super_base]
-
-        def replace_super(match):
-            super_method, base = find_super_method(super_base, match.group(1))
+        splits = re.split(r"super\([^\)]*\)\.(\w+)(?=\()", method_body)
+        new_body = splits[0]
+        for name, block in zip(splits[1::2], splits[2::2]):
+            super_method, base = find_super_method(super_base, name)
+            _, args, post = extract_args(block)
+            arg_str = ", ".join(args)
             try:
-                return self.SPECIAL_SUPER_MAPPINGS[super_method].format(
-                    args=match.group(2)
+                new_body += self.SPECIAL_SUPER_MAPPINGS[super_method].format(
+                    args=arg_str
                 )
             except KeyError:
                 try:
-                    return name_map[match.group(1)] + "(" + match.group(2) + ")"
+                    new_body += name_map[name] + "(" + arg_str + ")"
                 except KeyError:
-                    if any(
-                        base.__module__.startswith(m)
-                        for m in UsedSymbols.ALWAYS_OMIT_MODULES
-                    ):
+                    if self.package.is_omitted(base):
                         raise KeyError(
-                            f"Require special mapping for '{match.group(1)}' in {base} class "
+                            f"Require special mapping for '{name}' in {base} class "
                             "as methods in that module are being omitted from the conversion"
                         ) from None
                     raise
-
-        return re.sub(r"super\([^\)]*\)\.(\w+)\(([^\)]*)\)", replace_super, method_body)
+            new_body += post[1:]
+        return new_body
 
     def unwrap_nested_methods(
         self, method_body, additional_args=(), inputs_as_dict: bool = False
@@ -1375,13 +1377,22 @@ def unwrap_nested_methods(
                     )
             # Insert additional arguments to the method call (which were previously
             # accessed via member attributes)
-            new_body += name + insert_args_in_signature(
-                args,
-                [
-                    f"{a}=inputs['{a}']" if inputs_as_dict else f"{a}={a}"
-                    for a in (list(self.method_args[name]) + list(additional_args))
-                ],
-            )
+            args_to_be_inserted = list(self.method_args[name]) + list(additional_args)
+            try:
+                new_body += name + insert_args_in_signature(
+                    args,
+                    [
+                        f"{a}=inputs['{a}']" if inputs_as_dict else f"{a}={a}"
+                        for a in args_to_be_inserted
+                    ],
+                )
+            except UnmatchedParensException:
+                logger.warning(
+                    f"Nested method call inside '{name}' in {self.full_address}, "
+                    "the following args will need to be manually inserted up after the "
+                    f"conversion: {args_to_be_inserted}"
+                )
+                new_body += name + args
         method_body = new_body
         # Convert assignment to self attributes into method-scoped variables (hopefully
         # there aren't any name clashes)
@@ -1395,6 +1406,7 @@ def unwrap_nested_methods(
         CommandLine._format_arg: "argstr.format(**inputs)",
         CommandLine._filename_from_source: "{args} + '_generated'",
         BaseInterface._check_version_requirements: "[]",
+        CommandLine._parse_inputs: "{{}}",
     }
 
     INPUT_KEYS = [

nipype2pydra/interface/shell_command.py

@@ -174,7 +174,8 @@ def types_to_names(spec_fields):
                 self.parse_inputs_code,
                 self.callables_code,
                 self.defaults_code,
-            ],
+            ]
+            + list(self.referenced_methods),
             omit_classes=self.package.omit_classes + [BaseInterface, TraitedSpec],
             omit_modules=self.package.omit_modules,
             omit_functions=self.package.omit_functions,
@@ -215,7 +216,7 @@ def input_fields(self):
         for field in input_fields:
             if field[0] in self.formatted_input_field_names:
                 field[-1]["formatter"] = f"{field[0]}_formatter"
-                self._format_argstrs[field[0]] = field[-1].pop("argstr")
+                self._format_argstrs[field[0]] = field[-1].pop("argstr", "")
         return input_fields
 
     @cached_property
@@ -284,12 +285,13 @@ def format_arg_code(self):
         # Strip out return value
         body = re.sub(
             (
-                r"\s*return super\((\w+,\s*self)?\)\._format_arg\("
+                r"^    return super\((\w+,\s*self)?\)\._format_arg\("
                 + ", ".join(existing_args)
                 + r"\)\n"
             ),
-            "",
+            "return argstr.format(**inputs)",
             body,
+            flags=re.MULTILINE,
         )
         if not body:
             return ""
@@ -299,10 +301,13 @@ def format_arg_code(self):
         code_str = f"""def _format_arg({name_arg}, {val_arg}, inputs, argstr):{self.parse_inputs_call}
     if {val_arg} is None:
         return ""
-{body}
-    return argstr.format(**inputs)
+{body}"""
+
+        if not code_str.rstrip().endswith("return argstr.format(**inputs)"):
+            code_str += "\n    return argstr.format(**inputs)"
+
+        code_str += "\n\n"
 
-"""
         for field_name in self.formatted_input_field_names:
             code_str += (
                 f"def {field_name}_formatter(field, inputs):\n"
@@ -328,6 +333,8 @@ def parse_inputs_code(self) -> str:
 
         # Strip out return value
         body = re.sub(r"\s*return .*\n", "", body)
+        if not body:
+            return ""
         body = self.unwrap_nested_methods(body, inputs_as_dict=True)
         body = self.replace_supers(body)
 

nipype2pydra/package.py

@@ -310,7 +310,7 @@ def all_import_translations(self) -> ty.List[ty.Tuple[str, str]]:
 
     @property
     def all_omit_modules(self) -> ty.List[str]:
-        return self.omit_modules + ["nipype.interfaces.utility"]
+        return self.omit_modules + UsedSymbols.ALWAYS_OMIT_MODULES
 
     @property
     def all_explicit(self):
@@ -344,6 +344,17 @@ def config_defaults(self) -> ty.Dict[str, ty.Dict[str, str]]:
             all_defaults[name] = defaults
         return all_defaults
 
+    def is_omitted(self, obj: ty.Any) -> bool:
+        if full_address(obj) in self.omit_classes + self.omit_functions:
+            return True
+        if inspect.ismodule(obj):
+            mod_name = obj.__name__
+        else:
+            mod_name = obj.__module__
+        if any(re.match(m + r"\b", mod_name) for m in self.all_omit_modules):
+            return True
+        return False
+
     def write(self, package_root: Path, to_include: ty.List[str] = None):
         """Writes the package to the specified package root"""
 
@@ -886,6 +897,7 @@ def write_to_module(
                 # Write to file for debugging
                 debug_file = "~/unparsable-nipype2pydra-output.py"
                 with open(Path(debug_file).expanduser(), "w") as f:
+                    f.write(f"# Attemping to convert {self.nipype_name}\n")
                     f.write(converted_code)
                 raise RuntimeError(
                     f"Black could not parse generated code (written to {debug_file}): "

nipype2pydra/utils/misc.py

@@ -542,9 +542,12 @@ def get_return_line(func: ty.Union[str, ty.Callable]) -> str:
 
 
 def find_super_method(
-    super_base: type, method_name: str
+    super_base: type, method_name: str, include_class: bool = False
 ) -> ty.Tuple[ty.Callable, type]:
-    for base in super_base.__mro__[1:]:
+    mro = super_base.__mro__
+    if not include_class:
+        mro = mro[1:]
+    for base in mro:
         if method_name in base.__dict__:  # Found the match
             return getattr(base, method_name), base
     raise RuntimeError(
@@ -554,4 +557,4 @@ def find_super_method(
 
 
 def strip_comments(src: str) -> str:
-    return re.sub(r"\s+#.*", "", src)
+    return re.sub(r"^\s+#.*", "", src, flags=re.MULTILINE)