Skip to content

Commit d0bfb54

Browse files
ggmarshallggmarshall
authored
Merge pull request #29 from erin717/master
Add scripts for running LQ calibration
2 parents d8da22d + 57b7807 commit d0bfb54

File tree

4 files changed

+812
-10
lines changed

4 files changed

+812
-10
lines changed

‎rules/hit.smk

+53-3
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,13 @@ rule build_aoe_calibration:
8686
datatype="cal",
8787
channel="{channel}",
8888
output:
89-
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit")),
89+
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit", "aoe_cal")),
9090
aoe_results=temp(
91-
get_pattern_pars_tmp_channel(setup, "hit", "objects", extension="pkl")
91+
get_pattern_pars_tmp_channel(
92+
setup, "hit", "aoe_cal_objects", extension="pkl"
93+
)
9294
),
93-
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit")),
95+
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit", "aoe_cal")),
9496
log:
9597
get_pattern_log_channel(setup, "pars_hit_aoe_cal"),
9698
group:
@@ -115,6 +117,54 @@ rule build_aoe_calibration:
115117
"{input.files}"
116118

117119

120+
# This rule builds the lq calibration using the calibration dsp files
121+
rule build_lq_calibration:
122+
input:
123+
files=os.path.join(
124+
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
125+
),
126+
tcm_filelist=os.path.join(
127+
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-tcm.filelist"
128+
),
129+
ecal_file=get_pattern_pars_tmp_channel(setup, "hit", "aoe_cal"),
130+
eres_file=get_pattern_pars_tmp_channel(
131+
setup, "hit", "aoe_cal_objects", extension="pkl"
132+
),
133+
inplots=get_pattern_plts_tmp_channel(setup, "hit", "aoe_cal"),
134+
params:
135+
timestamp="{timestamp}",
136+
datatype="cal",
137+
channel="{channel}",
138+
output:
139+
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit")),
140+
lq_results=temp(
141+
get_pattern_pars_tmp_channel(setup, "hit", "objects", extension="pkl")
142+
),
143+
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit")),
144+
log:
145+
get_pattern_log_channel(setup, "pars_hit_lq_cal"),
146+
group:
147+
"par-hit"
148+
resources:
149+
runtime=300,
150+
shell:
151+
"{swenv} python3 -B "
152+
f"{workflow.source_path('../scripts/pars_hit_lq.py')} "
153+
"--log {log} "
154+
"--configs {configs} "
155+
"--datatype {params.datatype} "
156+
"--timestamp {params.timestamp} "
157+
"--inplots {input.inplots} "
158+
"--channel {params.channel} "
159+
"--lq_results {output.lq_results} "
160+
"--eres_file {input.eres_file} "
161+
"--hit_pars {output.hit_pars} "
162+
"--plot_file {output.plot_file} "
163+
"--tcm_filelist {input.tcm_filelist} "
164+
"--ecal_file {input.ecal_file} "
165+
"{input.files}"
166+
167+
118168
rule build_pars_hit:
119169
input:
120170
lambda wildcards: read_filelist_pars_cal_channel(wildcards, "hit"),

‎rules/pht.smk

+174-7
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ rule build_per_energy_calibration:
3838
timestamp="{timestamp}",
3939
datatype="cal",
4040
channel="{channel}",
41-
tier = "pht",
41+
tier="pht",
4242
output:
4343
ecal_file=temp(get_pattern_pars_tmp_channel(setup, "pht", "energy_cal")),
4444
results_file=temp(
@@ -351,6 +351,7 @@ for key, dataset in part.datasets.items():
351351
partition,
352352
key,
353353
tier="pht",
354+
name="aoecal",
354355
)
355356
],
356357
aoe_results=[
@@ -360,7 +361,7 @@ for key, dataset in part.datasets.items():
360361
partition,
361362
key,
362363
tier="pht",
363-
name="objects",
364+
name="aoecal_objects",
364365
extension="pkl",
365366
)
366367
],
@@ -371,6 +372,7 @@ for key, dataset in part.datasets.items():
371372
partition,
372373
key,
373374
tier="pht",
375+
name="aoecal",
374376
)
375377
],
376378
log:
@@ -379,7 +381,7 @@ for key, dataset in part.datasets.items():
379381
partition,
380382
key,
381383
"pht",
382-
name="par_pht",
384+
name="par_pht_aoe",
383385
),
384386
group:
385387
"par-pht"
@@ -413,7 +415,7 @@ for key, dataset in part.datasets.items():
413415

414416
# Merged energy and a/e supercalibrations to reduce number of rules as they have same inputs/outputs
415417
# This rule builds the a/e calibration using the calibration dsp files for the whole partition
416-
rule build_pht_super_calibrations:
418+
rule build_pht_aoe_calibrations:
417419
input:
418420
files=os.path.join(
419421
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
@@ -431,11 +433,13 @@ rule build_pht_super_calibrations:
431433
channel="{channel}",
432434
timestamp="{timestamp}",
433435
output:
434-
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht")),
436+
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht", "aoecal")),
435437
aoe_results=temp(
436-
get_pattern_pars_tmp_channel(setup, "pht", "objects", extension="pkl")
438+
get_pattern_pars_tmp_channel(
439+
setup, "pht", "objeaoecal_objectscts", extension="pkl"
440+
)
437441
),
438-
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht")),
442+
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht", "aoecal")),
439443
log:
440444
get_pattern_log_channel(setup, "pars_pht_aoe_cal"),
441445
group:
@@ -470,3 +474,166 @@ for key, items in ordered.items():
470474
rule_order_list += [item.name for item in items]
471475
rule_order_list.append(fallback_pht_rule.name)
472476
workflow._ruleorder.add(*rule_order_list) # [::-1]
477+
478+
part_pht_rules = {}
479+
for key, dataset in part.datasets.items():
480+
for partition in dataset.keys():
481+
482+
rule:
483+
input:
484+
files=part.get_filelists(partition, key, "dsp"),
485+
tcm_files=part.get_filelists(partition, key, "tcm"),
486+
ecal_file=part.get_par_files(
487+
f"{par_pht_path(setup)}/validity.jsonl",
488+
partition,
489+
key,
490+
tier="pht",
491+
name="aoecal",
492+
),
493+
eres_file=part.get_par_files(
494+
f"{par_pht_path(setup)}/validity.jsonl",
495+
partition,
496+
key,
497+
tier="pht",
498+
name="aoecal_objects",
499+
extension="pkl",
500+
),
501+
inplots=part.get_plt_files(
502+
f"{par_pht_path(setup)}/validity.jsonl",
503+
partition,
504+
key,
505+
tier="pht",
506+
name="aoecal",
507+
),
508+
params:
509+
datatype="cal",
510+
channel="{channel}",
511+
timestamp=part.get_timestamp(
512+
f"{par_pht_path(setup)}/validity.jsonl", partition, key, tier="pht"
513+
),
514+
output:
515+
hit_pars=[
516+
temp(file)
517+
for file in part.get_par_files(
518+
f"{par_pht_path(setup)}/validity.jsonl",
519+
partition,
520+
key,
521+
tier="pht",
522+
)
523+
],
524+
lq_results=[
525+
temp(file)
526+
for file in part.get_par_files(
527+
f"{par_pht_path(setup)}/validity.jsonl",
528+
partition,
529+
key,
530+
tier="pht",
531+
name="objects",
532+
extension="pkl",
533+
)
534+
],
535+
plot_file=[
536+
temp(file)
537+
for file in part.get_plt_files(
538+
f"{par_pht_path(setup)}/validity.jsonl",
539+
partition,
540+
key,
541+
tier="pht",
542+
)
543+
],
544+
log:
545+
part.get_log_file(
546+
f"{par_pht_path(setup)}/validity.jsonl",
547+
partition,
548+
key,
549+
"pht",
550+
name="par_pht_lq",
551+
),
552+
group:
553+
"par-pht"
554+
resources:
555+
mem_swap=75,
556+
runtime=300,
557+
shell:
558+
"{swenv} python3 -B "
559+
f"{basedir}/../scripts/pars_pht_lqcal.py "
560+
"--log {log} "
561+
"--configs {configs} "
562+
"--datatype {params.datatype} "
563+
"--timestamp {params.timestamp} "
564+
"--inplots {input.inplots} "
565+
"--channel {params.channel} "
566+
"--lq_results {output.lq_results} "
567+
"--eres_file {input.eres_file} "
568+
"--hit_pars {output.hit_pars} "
569+
"--plot_file {output.plot_file} "
570+
"--ecal_file {input.ecal_file} "
571+
"--tcm_filelist {input.tcm_files} "
572+
"--input_files {input.files}"
573+
574+
# fix_name(f"{key}-{partition}")
575+
576+
if key in part_pht_rules:
577+
part_pht_rules[key].append(list(workflow.rules)[-1])
578+
else:
579+
part_pht_rules[key] = [list(workflow.rules)[-1]]
580+
581+
582+
# This rule builds the lq calibration using the calibration dsp files for the whole partition
583+
rule build_pht_lq_calibration:
584+
input:
585+
files=os.path.join(
586+
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
587+
),
588+
tcm_filelist=os.path.join(
589+
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-tcm.filelist"
590+
),
591+
ecal_file=get_pattern_pars_tmp_channel(setup, "pht", "aoecal"),
592+
eres_file=get_pattern_pars_tmp_channel(
593+
setup, "pht", "aoecal_objects", extension="pkl"
594+
),
595+
inplots=get_pattern_plts_tmp_channel(setup, "pht", "aoecal"),
596+
params:
597+
datatype="cal",
598+
channel="{channel}",
599+
timestamp="{timestamp}",
600+
output:
601+
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht")),
602+
lq_results=temp(
603+
get_pattern_pars_tmp_channel(setup, "pht", "objects", extension="pkl")
604+
),
605+
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht")),
606+
log:
607+
get_pattern_log_channel(setup, "pars_pht_lq_cal"),
608+
group:
609+
"par-pht"
610+
resources:
611+
mem_swap=60,
612+
runtime=300,
613+
shell:
614+
"{swenv} python3 -B "
615+
f"{basedir}/./scripts/pars_pht_lqcal.py "
616+
"--log {log} "
617+
"--configs {configs} "
618+
"--datatype {params.datatype} "
619+
"--timestamp {params.timestamp} "
620+
"--inplots {input.inplots} "
621+
"--channel {params.channel} "
622+
"--lq_results {output.lq_results} "
623+
"--eres_file {input.eres_file} "
624+
"--hit_pars {output.hit_pars} "
625+
"--plot_file {output.plot_file} "
626+
"--ecal_file {input.ecal_file} "
627+
"--tcm_filelist {input.tcm_filelist} "
628+
"--input_files {input.files}"
629+
630+
631+
fallback_pht_rule = list(workflow.rules)[-1]
632+
633+
rule_order_list = []
634+
ordered = OrderedDict(part_pht_rules)
635+
ordered.move_to_end("default")
636+
for key, items in ordered.items():
637+
rule_order_list += [item.name for item in items]
638+
rule_order_list.append(fallback_pht_rule.name)
639+
workflow._ruleorder.add(*rule_order_list) # [::-1]

0 commit comments

Comments
 (0)