input file list

src/diffpy/labpdfproc/labpdfprocapp.py

@@ -1,9 +1,14 @@
 import sys
 from argparse import ArgumentParser
-from pathlib import Path
 
 from diffpy.labpdfproc.functions import apply_corr, compute_cve
-from diffpy.labpdfproc.tools import known_sources, load_user_metadata, set_output_directory, set_wavelength
+from diffpy.labpdfproc.tools import (
+    known_sources,
+    load_user_metadata,
+    set_input_lists,
+    set_output_directory,
+    set_wavelength,
+)
 from diffpy.utils.parsers.loaddata import loadData
 from diffpy.utils.scattering_objects.diffraction_objects import XQUANTITIES, Diffraction_object
 
@@ -21,7 +26,7 @@ def get_args(override_cli_inputs=None):
         "data-files in that directory will be processed. Examples of valid "
         "inputs are 'file.xy', 'data/file.xy', 'file.xy, data/file.xy', "
         "'.' (load everything in the current directory), 'data' (load"
-        "everything in the folder ./data', 'data/file_list.txt' (load"
+        "everything in the folder ./data), 'data/file_list.txt' (load"
         " the list of files contained in the text-file called "
         "file_list.txt that can be found in the folder ./data).",
     )
@@ -89,45 +94,46 @@ def get_args(override_cli_inputs=None):
 
 def main():
     args = get_args()
+    args = set_input_lists(args)
     args.output_directory = set_output_directory(args)
     args.wavelength = set_wavelength(args)
     args = load_user_metadata(args)
 
-    filepath = Path(args.input_file)
-    outfilestem = filepath.stem + "_corrected"
-    corrfilestem = filepath.stem + "_cve"
-    outfile = args.output_directory / (outfilestem + ".chi")
-    corrfile = args.output_directory / (corrfilestem + ".chi")
+    for filepath in args.input_directory:
+        outfilestem = filepath.stem + "_corrected"
+        corrfilestem = filepath.stem + "_cve"
+        outfile = args.output_directory / (outfilestem + ".chi")
+        corrfile = args.output_directory / (corrfilestem + ".chi")
 
-    if outfile.exists() and not args.force_overwrite:
-        sys.exit(
-            f"Output file {str(outfile)} already exists. Please rerun "
-            f"specifying -f if you want to overwrite it."
-        )
-    if corrfile.exists() and args.output_correction and not args.force_overwrite:
-        sys.exit(
-            f"Corrections file {str(corrfile)} was requested and already "
-            f"exists. Please rerun specifying -f if you want to overwrite it."
-        )
+        if outfile.exists() and not args.force_overwrite:
+            sys.exit(
+                f"Output file {str(outfile)} already exists. Please rerun "
+                f"specifying -f if you want to overwrite it."
+            )
+        if corrfile.exists() and args.output_correction and not args.force_overwrite:
+            sys.exit(
+                f"Corrections file {str(corrfile)} was requested and already "
+                f"exists. Please rerun specifying -f if you want to overwrite it."
+            )
 
-    input_pattern = Diffraction_object(wavelength=args.wavelength)
-    xarray, yarray = loadData(args.input_file, unpack=True)
-    input_pattern.insert_scattering_quantity(
-        xarray,
-        yarray,
-        "tth",
-        scat_quantity="x-ray",
-        name=str(args.input_file),
-        metadata={"muD": args.mud, "anode_type": args.anode_type},
-    )
+        input_pattern = Diffraction_object(wavelength=args.wavelength)
+        xarray, yarray = loadData(args.input_file, unpack=True)
+        input_pattern.insert_scattering_quantity(
+            xarray,
+            yarray,
+            "tth",
+            scat_quantity="x-ray",
+            name=str(args.input_file),
+            metadata={"muD": args.mud, "anode_type": args.anode_type},
+        )
 
-    absorption_correction = compute_cve(input_pattern, args.mud, args.wavelength)
-    corrected_data = apply_corr(input_pattern, absorption_correction)
-    corrected_data.name = f"Absorption corrected input_data: {input_pattern.name}"
-    corrected_data.dump(f"{outfile}", xtype="tth")
+        absorption_correction = compute_cve(input_pattern, args.mud, args.wavelength)
+        corrected_data = apply_corr(input_pattern, absorption_correction)
+        corrected_data.name = f"Absorption corrected input_data: {input_pattern.name}"
+        corrected_data.dump(f"{outfile}", xtype="tth")
 
-    if args.output_correction:
-        absorption_correction.dump(f"{corrfile}", xtype="tth")
+        if args.output_correction:
+            absorption_correction.dump(f"{corrfile}", xtype="tth")
 
 
 if __name__ == "__main__":

src/diffpy/labpdfproc/tests/test_tools.py

@@ -69,7 +69,7 @@ def test_set_input_lists(inputs, expected, user_filesystem):
     cli_inputs = ["2.5"] + inputs
     actual_args = get_args(cli_inputs)
     actual_args = set_input_lists(actual_args)
-    assert list(actual_args.input_paths).sort() == expected_paths.sort()
+    assert sorted(actual_args.input_paths) == sorted(expected_paths)
 
 
 # This test covers non-existing single input file or directory, in this case we raise an error with message

src/diffpy/labpdfproc/tools.py

@@ -28,6 +28,22 @@ def set_output_directory(args):
     return output_dir
 
 
+def _parse_file_list_file(file_list_path):
+    with open(file_list_path, "r") as f:
+        # file_paths = [Path(file_path.strip()).resolve() for file_path in f.readlines()
+        # if Path(file_path.strip()).is_file()]
+        file_paths = [file_path.strip() for file_path in f.readlines()]
+        return file_paths
+
+
+def expand_list_file(input):
+    file_list_inputs = [input_name for input_name in input if "file_list" in str(input_name)]
+    for file_list_input in file_list_inputs:
+        input.remove(file_list_input)
+        input.extend(_parse_file_list_file(file_list_input))
+    return input
+
+
 def set_input_lists(args):
     """
     Set input directory and files.
@@ -47,20 +63,23 @@ def set_input_lists(args):
     """
 
     input_paths = []
-    for input in args.input:
+    expanded_input = expand_list_file(args.input)
+    for input in expanded_input:
         input_path = Path(input).resolve()
         if input_path.exists():
             if input_path.is_file():
                 input_paths.append(input_path)
             elif input_path.is_dir():
                 input_files = input_path.glob("*")
-                input_files = [file.resolve() for file in input_files if file.is_file()]
+                input_files = [
+                    file.resolve() for file in input_files if file.is_file() and "file_list" not in file.name
+                ]
                 input_paths.extend(input_files)
             else:
                 raise FileNotFoundError(f"Cannot find {input}. Please specify valid input file(s) or directories.")
         else:
             raise FileNotFoundError(f"Cannot find {input}")
-    setattr(args, "input_paths", input_paths)
+    setattr(args, "input_paths", list(set(input_paths)))
     return args