implement config option to disable filedb generation

Author: gipert

dataflow-config.yaml

@@ -1,5 +1,6 @@
 # legend_metadata_version: main
 allow_none_par: false
+build_file_dbs: true
 
 paths:
   sandbox_path: $_/sandbox

workflow/src/legenddataflow/scripts/complete_run.py

@@ -244,57 +244,61 @@ def build_file_dbs(gen_tier_path, outdir):
     print(f"INFO: ...took {dt}")
 
 
-file_db_config = {}
-
-if (
-    os.getenv("PRODENV") is not None
-    and os.getenv("PRODENV") in snakemake.params.filedb_path
-):
-    prodenv = as_ro(os.getenv("PRODENV"))
+def fformat(tier):
+    abs_path = patterns.get_pattern_tier(
+        snakemake.params.setup, tier, check_in_cycle=False
+    )
+    return str(abs_path).replace(ut.get_tier_path(snakemake.params.setup, tier), "")
 
-    def tdirs(tier):
-        return as_ro(ut.get_tier_path(snakemake.params.setup, tier)).replace(
-            prodenv, ""
-        )
 
-    file_db_config["data_dir"] = "$PRODENV"
+if snakemake.params.setup.get("build_file_dbs", True):
+    file_db_config = {}
 
-else:
-    print("WARNING: $PRODENV not set, the FileDB will not be relocatable")
+    if (
+        os.getenv("PRODENV") is not None
+        and os.getenv("PRODENV") in snakemake.params.filedb_path
+    ):
+        prodenv = as_ro(os.getenv("PRODENV"))
 
-    def tdirs(tier):
-        return as_ro(ut.get_tier_path(snakemake.params.setup, tier))
+        def tdirs(tier):
+            return as_ro(ut.get_tier_path(snakemake.params.setup, tier)).replace(
+                prodenv, ""
+            )
 
-    file_db_config["data_dir"] = "/"
+        file_db_config["data_dir"] = "$PRODENV"
 
+    else:
+        print("WARNING: $PRODENV not set, the FileDB will not be relocatable")
 
-file_db_config["tier_dirs"] = {
-    k: tdirs(k) for k in snakemake.params.setup["table_format"]
-}
+        def tdirs(tier):
+            return as_ro(ut.get_tier_path(snakemake.params.setup, tier))
 
+        file_db_config["data_dir"] = "/"
 
-def fformat(tier):
-    abs_path = patterns.get_pattern_tier(
-        snakemake.params.setup, tier, check_in_cycle=False
-    )
-    return str(abs_path).replace(ut.get_tier_path(snakemake.params.setup, tier), "")
-
+    file_db_config["tier_dirs"] = {
+        k: tdirs(k) for k in snakemake.params.setup["table_format"]
+    }
 
-file_db_config |= {
-    "file_format": {k: fformat(k) for k in snakemake.params.setup["table_format"]},
-    "table_format": snakemake.params.setup["table_format"],
-}
+    file_db_config |= {
+        "file_format": {k: fformat(k) for k in snakemake.params.setup["table_format"]},
+        "table_format": snakemake.params.setup["table_format"],
+    }
 
 if snakemake.wildcards.tier != "daq":
-    print(f"INFO: ...building FileDBs with {snakemake.threads} threads")
+    if snakemake.params.setup.get("build_file_dbs", True):
+        print(f"INFO: ...building FileDBs with {snakemake.threads} threads")
 
-    Path(snakemake.params.filedb_path).mkdir(parents=True, exist_ok=True)
+        Path(snakemake.params.filedb_path).mkdir(parents=True, exist_ok=True)
 
-    with (Path(snakemake.params.filedb_path) / "file_db_config.json").open("w") as f:
-        json.dump(file_db_config, f, indent=2)
+        with (Path(snakemake.params.filedb_path) / "file_db_config.json").open(
+            "w"
+        ) as f:
+            json.dump(file_db_config, f, indent=2)
 
-    build_file_dbs(ut.tier_path(snakemake.params.setup), snakemake.params.filedb_path)
-    (Path(snakemake.params.filedb_path) / "file_db_config.json").unlink()
+        build_file_dbs(
+            ut.tier_path(snakemake.params.setup), snakemake.params.filedb_path
+        )
+        (Path(snakemake.params.filedb_path) / "file_db_config.json").unlink()
 
     build_valid_keys(
         Path(ut.tmp_par_path(snakemake.params.setup)) / "*_db.json",