Skip to content

Commit 43aa8bf

Browse files
GosstyGossty
committed
Merge branch 'dev' of https://github.com/qiita-spots/qiita into dev
2 parents 442e735 + 751e559 commit 43aa8bf

27 files changed

+1393
-540
lines changed

‎notebooks/resource-allocation/generate-allocation-summary-arrays.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from qiita_core.util import MaxRSS_helper
1+
from qiita_db.util import MaxRSS_helper
22
from qiita_db.software import Software
33
import datetime
44
from io import StringIO

‎notebooks/resource-allocation/generate-allocation-summary.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from json import loads
66
from os.path import join
77

8-
from qiita_core.util import MaxRSS_helper
8+
from qiita_db.util import MaxRSS_helper
99
from qiita_db.exceptions import QiitaDBUnknownIDError
1010
from qiita_db.processing_job import ProcessingJob
1111
from qiita_db.software import Software

‎qiita_core/tests/test_util.py

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from qiita_core.util import (
1212
qiita_test_checker, execute_as_transaction, get_qiita_version,
13-
is_test_environment, get_release_info, MaxRSS_helper)
13+
is_test_environment, get_release_info)
1414
from qiita_db.meta_util import (
1515
generate_biom_and_metadata_release, generate_plugin_releases)
1616
import qiita_db as qdb
@@ -82,20 +82,6 @@ def test_get_release_info(self):
8282
self.assertEqual(biom_metadata_release, ('', '', ''))
8383
self.assertNotEqual(archive_release, ('', '', ''))
8484

85-
def test_MaxRSS_helper(self):
86-
tests = [
87-
('6', 6.0),
88-
('6K', 6000),
89-
('6M', 6000000),
90-
('6G', 6000000000),
91-
('6.9', 6.9),
92-
('6.9K', 6900),
93-
('6.9M', 6900000),
94-
('6.9G', 6900000000),
95-
]
96-
for x, y in tests:
97-
self.assertEqual(MaxRSS_helper(x), y)
98-
9985

10086
if __name__ == '__main__':
10187
main()

‎qiita_core/util.py

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,3 @@ def get_release_info(study_status='public'):
151151
archive_release = ((md5sum, filepath, timestamp))
152152

153153
return (biom_metadata_release, archive_release)
154-
155-
156-
def MaxRSS_helper(x):
157-
if x[-1] == 'K':
158-
y = float(x[:-1]) * 1000
159-
elif x[-1] == 'M':
160-
y = float(x[:-1]) * 1000000
161-
elif x[-1] == 'G':
162-
y = float(x[:-1]) * 1000000000
163-
else:
164-
y = float(x)
165-
return y

‎qiita_db/handlers/tests/test_user.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,14 @@ def test_get(self):
5151

5252
obs = loads(obs.body)
5353
exp = {'data': [
54-
{'email': '[email protected]', 'name': 'Dude'},
5554
{'email': '[email protected]', 'name': 'Shared'},
5655
{'email': '[email protected]', 'name': 'Admin'},
57-
{'email': '[email protected]', 'name': 'Demo'}]}
56+
{'email': '[email protected]', 'name': 'Demo'},
57+
{'email': '[email protected]', 'name': 'Dude'},
58+
{'email': '[email protected]', 'name': 'JustNow'},
59+
{'email': '[email protected]', 'name': 'Oldie'},
60+
{'email': '[email protected]', 'name': 'TooLate'}
61+
]}
5862
self.assertEqual(obs, exp)
5963

6064

‎qiita_db/support_files/patches/92.sql

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ CREATE TABLE qiita.slurm_resource_allocations (
1010
extra_info varchar DEFAULT NULL,
1111
memory_used bigint,
1212
walltime_used integer,
13+
job_start TIMESTAMP,
14+
node_name varchar DEFAULT NULL,
15+
node_model varchar DEFAULT NULL,
1316
CONSTRAINT pk_slurm_resource_allocations_processing_job_id PRIMARY KEY (
1417
processing_job_id )
1518
);
@@ -27,3 +30,12 @@ ALTER TABLE qiita.prep_template ADD current_human_filtering boolean DEFAULT Fals
2730
-- Adding a new column: reprocess_job_id to qiita.prep_template to keep track of
2831
-- the job that reprocessed this prep
2932
ALTER TABLE qiita.prep_template ADD reprocess_job_id uuid DEFAULT NULL;
33+
34+
-- Jun 19, 2024
35+
-- Adding a new column to the user table that logs when this account was created
36+
-- Usefull e.g. to prune non-verified=inactive user or to plot user growth
37+
38+
ALTER TABLE qiita.qiita_user
39+
ADD creation_timestamp timestamp without time zone DEFAULT NOW();
40+
41+
COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user account was created';

‎qiita_db/support_files/patches/test_db_sql/92.sql

Lines changed: 472 additions & 461 deletions
Large diffs are not rendered by default.

‎qiita_db/support_files/populate_test_db.sql

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ INSERT INTO qiita.user_level VALUES (7, 'wet-lab admin', 'Can access the private
5050
-- Data for Name: qiita_user; Type: TABLE DATA; Schema: qiita; Owner: antoniog
5151
--
5252

53-
INSERT INTO qiita.qiita_user VALUES ('[email protected]', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false);
53+
INSERT INTO qiita.qiita_user VALUES ('[email protected]', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ');
5454
INSERT INTO qiita.qiita_user VALUES ('[email protected]', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false);
5555
INSERT INTO qiita.qiita_user VALUES ('[email protected]', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL, false);
5656
INSERT INTO qiita.qiita_user VALUES ('[email protected]', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL, false);
@@ -329,16 +329,16 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin
329329
-- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog
330330
--
331331

332-
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
333-
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
334-
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, NULL);
335-
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, NULL);
336-
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
337-
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', '[email protected]', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
338-
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
339-
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
340-
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
341-
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', '[email protected]', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, NULL);
332+
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757);
333+
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546);
334+
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377);
335+
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986);
336+
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1279011391);
337+
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', '[email protected]', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1286151876);
338+
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1283300404);
339+
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', '[email protected]', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1275827198);
340+
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', '[email protected]', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1266027);
341+
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', '[email protected]', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, 126652530);
342342
INSERT INTO qiita.processing_job VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', '[email protected]', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 1, NULL, NULL, NULL, NULL, true, NULL);
343343
INSERT INTO qiita.processing_job VALUES ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', '[email protected]', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}', 2, NULL, '2015-11-22 21:00:00', 'demultiplexing', NULL, true, NULL);
344344
INSERT INTO qiita.processing_job VALUES ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', '[email protected]', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 4, 1, '2015-11-22 21:30:00', 'generating demux file', NULL, true, NULL);

‎qiita_db/support_files/qiita-db-unpatched.sql

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1888,7 +1888,10 @@ CREATE TABLE qiita.qiita_user (
18881888
user_verify_code character varying,
18891889
pass_reset_code character varying,
18901890
pass_reset_timestamp timestamp without time zone,
1891-
receive_processing_job_emails boolean DEFAULT false
1891+
receive_processing_job_emails boolean DEFAULT false,
1892+
social_orcid character varying DEFAULT NULL,
1893+
social_researchgate character varying DEFAULT NULL,
1894+
social_googlescholar character varying DEFAULT NULL
18921895
);
18931896

18941897

‎qiita_db/test/test_data/jobs_2024-02-21.tsv.gz

-7.17 MB
Binary file not shown.

‎qiita_db/test/test_data/slurm_data.txt.gz

658 KB
Binary file not shown.

‎qiita_db/test/test_meta_util.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ def _get_daily_stats():
281281
self.assertDictEqual(f(redis_key), exp)
282282
# then the unique values
283283
vals = [
284-
('num_users', b'4', r_client.get),
284+
('num_users', b'7', r_client.get),
285285
('lat_longs', b'[]', r_client.get),
286286
('num_studies_ebi', b'1', r_client.get),
287287
('num_samples_ebi', b'27', r_client.get),

‎qiita_db/test/test_portal.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ def test_add_portal(self):
4747
qdb.sql_connection.TRN.add("SELECT * FROM qiita.analysis_portal")
4848
obs = qdb.sql_connection.TRN.execute_fetchindex()
4949
exp = [[1, 1], [2, 1], [3, 1], [4, 1], [5, 1], [6, 1], [7, 2], [8, 2],
50-
[9, 2], [10, 2], [11, 4], [12, 4], [13, 4], [14, 4]]
50+
[9, 2], [10, 2], [11, 4], [12, 4], [13, 4], [14, 4],
51+
[15, 4], [16, 4], [17, 4]]
5152
self.assertCountEqual(obs, exp)
5253

5354
with self.assertRaises(qdb.exceptions.QiitaDBDuplicateError):

‎qiita_db/test/test_processing_job.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -970,7 +970,7 @@ def test_notification_mail_generation(self):
970970
pj = qdb.processing_job.ProcessingJob(jid)
971971
obs = pj._generate_notification_message('running', 'Yahoo!')
972972
exp = {'subject': ('Single Rarefaction: running, 8a7a8461-e8a1-'
973-
'4b4e-a428-1bc2f4d3ebd0 [Not Available]'),
973+
'4b4e-a428-1bc2f4d3ebd0 [126652530]'),
974974
'message': 'Analysis Job Single Rarefaction\n'
975975
f'{qiita_config.base_url}/analysis/description/1/\n'
976976
'New status: running'}

‎qiita_db/test/test_setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class SetupTest(TestCase):
1515
"""Tests that the test database have been successfully populated"""
1616

1717
def test_qiita_user(self):
18-
self.assertEqual(get_count("qiita.qiita_user"), 4)
18+
self.assertEqual(get_count("qiita.qiita_user"), 7)
1919

2020
def test_study_person(self):
2121
self.assertEqual(get_count("qiita.study_person"), 3)

0 commit comments

Comments
 (0)