update to new lgdo

Author: ggmarshall

workflow/src/legenddataflow/scripts/complete_run.py

@@ -9,7 +9,7 @@
 
 from snakemake.script import snakemake
 
-from legenddataflow import FileKey, patterns
+from legenddataflow import patterns
 from legenddataflow import utils as ut
 from legenddataflow.execenv import _execenv2str, execenv_pyexe
 
@@ -88,82 +88,6 @@ def check_log_files(log_path, output_file, gen_output, warning_file=None):
             Path(path).rmdir()
 
 
-def add_spaces(n):
-    out_string = ""
-    for _i in range(n):
-        out_string += " "
-    return out_string
-
-
-def readable_json(dic, ncol=6, indent=4):
-    def reformat_dict(dic, out_string="", indent_level=0, ncol=ncol, indent=indent):
-        for key, item in dic.items():
-            if isinstance(item, list):
-                out_string += (
-                    f'{add_spaces(indent_level+indent)}"{key}"'
-                    + ": [\n"
-                    + f"{add_spaces(2*indent+indent_level)}"
-                )
-                for i, _item in enumerate(item):
-                    if i > 0 and (i) % ncol == 0:
-                        out_string += f"\n{add_spaces(2*indent+indent_level)}"
-                    out_string += f'"{_item}", '
-                out_string = out_string[:-2]
-                out_string += "\n" + f"{add_spaces(indent+indent_level)}" + "],\n"
-
-            elif isinstance(item, dict):
-                out_string += f'{add_spaces(indent+indent_level)}"{key}": ' + "{\n"
-                out_string = reformat_dict(
-                    item, out_string, indent_level=indent_level + indent, ncol=ncol
-                )
-                out_string += "\n" + f"{add_spaces(indent_level+indent)}" + "},\n"
-        return out_string[:-2]
-
-    out_string = "{\n"
-    out_string = reformat_dict(dic, out_string=out_string, ncol=6)
-    out_string += "\n}\n"
-
-    return out_string
-
-
-def get_keys(files):
-    def get_run(Filekey):
-        return f"{Filekey.experiment}-{Filekey.period}-{Filekey.run}-{Filekey.datatype}"
-
-    key_dict = {}
-    for file in files:
-        key = FileKey.get_filekey_from_filename(Path(file).name)
-        if get_run(key) in key_dict:
-            key_dict[get_run(key)].append(file)
-        else:
-            key_dict[get_run(key)] = [file]
-    return key_dict
-
-
-def build_valid_keys(input_files_regex, output_dir):
-    in_regex = Path(as_ro(input_files_regex))
-    infiles = in_regex.parent.glob(in_regex.name)
-    key_dict = get_keys(infiles)
-
-    for key in list(key_dict):
-        dtype = key.split("-")[-1]
-        out_file = (
-            Path(output_dir) / f'{key.replace(f"-{dtype}", "")}-valid_{dtype}.json'
-        )
-        out_file.parent.mkdir(parents=True, exist_ok=True)
-        if Path(out_file).is_file():
-            out_dict = Props.read_from([out_file] + key_dict[key])
-        else:
-            out_dict = Props.read_from(key_dict[key])
-        out_string = readable_json(out_dict)
-        with Path(out_file).open("w") as w:
-            w.write(out_string)
-
-    for input_file in infiles:
-        if Path(input_file).is_file():
-            Path(input_file).unlink()
-
-
 def find_gen_runs(gen_tier_path):
     # first look for non-concat tiers
     paths = gen_tier_path.glob("*/*/*/*")
@@ -285,26 +209,18 @@ def tdirs(tier):
         "table_format": snakemake.params.setup["table_format"],
     }
 
-if snakemake.wildcards.tier != "daq":
-    if snakemake.params.setup.get("build_file_dbs", True):
-        print(f"INFO: ...building FileDBs with {snakemake.threads} threads")
-
-        Path(snakemake.params.filedb_path).mkdir(parents=True, exist_ok=True)
+if (snakemake.wildcards.tier != "daq") and snakemake.params.setup.get(
+    "build_file_dbs", True
+):
+    print(f"INFO: ...building FileDBs with {snakemake.threads} threads")
 
-        with (Path(snakemake.params.filedb_path) / "file_db_config.json").open(
-            "w"
-        ) as f:
-            json.dump(file_db_config, f, indent=2)
+    Path(snakemake.params.filedb_path).mkdir(parents=True, exist_ok=True)
 
-        build_file_dbs(
-            ut.tier_path(snakemake.params.setup), snakemake.params.filedb_path
-        )
-        (Path(snakemake.params.filedb_path) / "file_db_config.json").unlink()
+    with (Path(snakemake.params.filedb_path) / "file_db_config.json").open("w") as f:
+        json.dump(file_db_config, f, indent=2)
 
-    build_valid_keys(
-        Path(ut.tmp_par_path(snakemake.params.setup)) / "*_db.json",
-        snakemake.params.valid_keys_path,
-    )
+    build_file_dbs(ut.tier_path(snakemake.params.setup), snakemake.params.filedb_path)
+    (Path(snakemake.params.filedb_path) / "file_db_config.json").unlink()
 
 if snakemake.params.setup.get("check_log_files", True):
     print("INFO: ...checking log files")

workflow/src/legenddataflow/scripts/filedb.py

@@ -62,7 +62,7 @@ def build_filedb() -> None:
                     fdb.table_format["raw"].format(ch=table) + "/timestamp",
                     row.raw_file.strip("/"),
                     n_rows=1,
-                )[0][0]
+                )[0]
                 found = True
             except KeyError:
                 pass