Skip to content

Commit beba0ca

Browse files
GosstyGossty
committed
Added columns to slurm_resource_allocation table
1 parent 022cd7c commit beba0ca

File tree

7 files changed

+557
-590
lines changed

7 files changed

+557
-590
lines changed

‎qiita_db/support_files/patches/92.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ CREATE TABLE qiita.slurm_resource_allocations (
1010
extra_info varchar DEFAULT NULL,
1111
memory_used bigint,
1212
walltime_used integer,
13+
start_stamp TIMESTAMP,
14+
node_name varchar DEFAULT NULL,
15+
node_model varchar DEFAULT NULL,
1316
CONSTRAINT pk_slurm_resource_allocations_processing_job_id PRIMARY KEY (
1417
processing_job_id )
1518
);

‎qiita_db/support_files/patches/test_db_sql/92.sql

Lines changed: 461 additions & 461 deletions
Large diffs are not rendered by default.

‎qiita_db/support_files/populate_test_db.sql

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -329,16 +329,16 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin
329329
-- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog
330330
--
331331

332-
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
333-
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
334-
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, NULL);
335-
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, NULL);
336-
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
337-
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', '[email protected]', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
338-
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
339-
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
340-
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
341-
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', '[email protected]', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, NULL);
332+
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757);
333+
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546);
334+
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377);
335+
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986);
336+
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1279011391);
337+
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', '[email protected]', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1286151876);
338+
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1283300404);
339+
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1275827198);
340+
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1266027);
341+
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', '[email protected]', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, 126652530);
342342
INSERT INTO qiita.processing_job VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 1, NULL, NULL, NULL, NULL, true, NULL);
343343
INSERT INTO qiita.processing_job VALUES ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', '[email protected]', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}', 2, NULL, '2015-11-22 21:00:00', 'demultiplexing', NULL, true, NULL);
344344
INSERT INTO qiita.processing_job VALUES ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 4, 1, '2015-11-22 21:30:00', 'generating demux file', NULL, true, NULL);

‎qiita_db/test/test_data/jobs_2024-02-21.tsv.gz

-7.17 MB
Binary file not shown.

‎qiita_db/test/test_data/slurm_data.txt.gz

658 KB
Binary file not shown.

‎qiita_db/test/test_util.py

Lines changed: 4 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1315,7 +1315,7 @@ def setUp(self):
13151315
self.columns = [
13161316
"sName", "sVersion", "cID", "cName", "processing_job_id",
13171317
"parameters", "samples", "columns", "input_size", "extra_info",
1318-
"MaxRSSRaw", "ElapsedRaw"]
1318+
"MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"]
13191319

13201320
# df is a dataframe that represents a table with columns specified in
13211321
# self.columns
@@ -1397,35 +1397,8 @@ def test_MaxRSS_helper(self):
13971397
self.assertEqual(qdb.util.MaxRSS_helper(x), y)
13981398

13991399
def test_db_update(self):
1400-
1401-
def read_slurm(id):
1402-
sacct_dummies = {
1403-
1005932: (
1404-
"JobID|ElapsedRaw|MaxRSS|Submit|Start|MaxRSS|CPUTimeRAW|"
1405-
"ReqMem|AllocCPUS|AveVMSize|\n"
1406-
"1005932|165||2023-02-23T14:55:07|2023-02-23T14:55:08|"
1407-
"|165|120Gn|1||\n"
1408-
"1005932.batch|165|328716K|2023-02-23T14:55:08"
1409-
"|2023-02-23T14:55:08|328716K|"
1410-
"165|120Gn|1|156284K|\n"
1411-
"1005932.extern|165|0|2023-02-23T14:55:08|"
1412-
"2023-02-23T14:55:08|0|165|120Gn|1|108052K|"
1413-
),
1414-
1001100: (
1415-
"JobID|ElapsedRaw|MaxRSS|Submit|Start|MaxRSS|"
1416-
"CPUTimeRAW|ReqMem|AllocCPUS|AveVMSize|\n"
1417-
"1001100|219||2023-02-22T11:05:26|"
1418-
"2023-02-22T11:05:27||219|120Gn|1||\n"
1419-
"1001100.batch|219|342204K|2023-02-22T11:05:27|"
1420-
"2023-02-22T11:05:27|342204K|219|120Gn|1|156284K|\n"
1421-
"1001100.extern|219|0|2023-02-22T11:05:27|"
1422-
"2023-02-22T11:05:27|0|219|120Gn|1|108052K|"
1423-
)
1424-
}
1425-
1426-
slurm_info = sacct_dummies.get(id)
1427-
return slurm_info
1428-
1400+
path_to_data = './qiita_db/test/test_data/slurm_data.txt.gz'
1401+
test_data = pd.read_csv(path_to_data, sep="|")
14291402
types = {
14301403
'Split libraries FASTQ': [
14311404
'6d368e16-2242-4cf8-87b4-a5dc40bb890b',
@@ -1444,7 +1417,7 @@ def read_slurm(id):
14441417
]
14451418
}
14461419

1447-
qdb.util.update_resource_allocation_table(test=read_slurm)
1420+
qdb.util.update_resource_allocation_table(test=test_data)
14481421

14491422
for curr_cname, ids in types.items():
14501423
updated_df = qdb.util._retrieve_resource_data(

0 commit comments

Comments
 (0)