Snakefile-build-raw

"""
Snakefile for doing the first stages of data processing from the daq sandbox files
to the blinded raw data. It handles:
- moving the daq files from the sandbox to the sorted file system
- running build raw on this data (with trimming)
- blinding the physics data
"""

import os, sys
from pathlib import Path
from scripts.util import patterns as patt
from scripts.util.utils import (
    subst_vars_in_snakemake_config,
    runcmd,
    config_path,
    chan_map_path,
    filelist_path,
    pars_path,
    metadata_path,
    det_status_path,
)
from scripts.util.create_pars_keylist import ParsKeyResolve

check_in_cycle = True

# Set with `snakemake --configfile=/path/to/your/config.json`
# configfile: "have/to/specify/path/to/your/config.json"

subst_vars_in_snakemake_config(workflow, config)

setup = config["setups"]["l200"]
configs = config_path(setup)
chan_maps = chan_map_path(setup)
swenv = runcmd(setup)
meta = metadata_path(setup)
det_status = det_status_path(setup)

basedir = workflow.basedir


wildcard_constraints:
    experiment=r"\w+",
    period=r"p\d{2}",
    run=r"r\d{3}",
    datatype=r"\w{3}",
    timestamp=r"\d{8}T\d{6}Z",


localrules:
    gen_filelist,
    autogen_output,


include: "rules/common.smk"
include: "rules/filelist_gen.smk"
include: "rules/main.smk"
include: "rules/raw.smk"
include: "rules/blinding_check.smk"


onstart:
    print("INFO: starting workflow")

    # Make sure some packages are initialized before we begin to avoid race conditions
    shell('{swenv} python3 -B -c "import daq2lh5 "')


    raw_par_cat_file = Path(pars_path(setup)) / "raw" / "validity.yaml"
    if raw_par_cat_file.is_file():
        raw_par_cat_file.unlink()
    try:
        Path(raw_par_cat_file).parent.mkdir(parents=True, exist_ok=True)
        ParsKeyResolve.write_to_yaml(raw_par_catalog, raw_par_cat_file)
    except NameError:
        print("No raw parameter catalog found")


onsuccess:
    print("Workflow finished, no error")
    shell("rm *.gen || true")
    shell(f"rm {filelist_path(setup)}/* || true")


rule gen_filelist:
    input:
        lambda wildcards: get_filelist(
            wildcards,
            setup,
            get_search_pattern(wildcards.tier),
            ignore_keys_file=Path(det_status) / "ignored_daq_cycles.yaml",
            analysis_runs_file=Path(det_status) / "runlists.yaml",
        ),
    output:
        temp(Path(filelist_path(setup)) / "{label}-{tier}.filelist"),
    script:
        "scripts/write_filelist.py"


rule sort_data:
    """
    This rules moves the daq data from the unsorted sandbox dir
    to the sorted dirs under generated
    """
    input:
        patt.get_pattern_tier_daq_unsorted(setup, extension="fcio"),
    output:
        patt.get_pattern_tier_daq(setup, extension="fcio"),
    shell:
        "mv {input} {output}"