Skip to content

Commit f72766c

Browse files
antgonzaantgonza
authored
inserting data to qiita for testing (#129)
* inserting data to qiita for testing * testing creation of study in qiit * add sbatch * shopt * echo .profile * slurm * partition * scontrol create partition * mv scontrol * partitionname * sudo scontrol * add some prints * /usr/bin/sbatch * sudo * env * .local/bin/sbatch * ls * sbatch in conda * squeue * improve error display and running tests * sbatch * GITHUB_PATH * adding files to tests/bin * rm study names * fix good_sheet1.csv * fix tests * extra fixes to good_sheet1.csv * addressing comment from @ AmandaBirmingham
1 parent a7e1dab commit f72766c

36 files changed

+9133
-7437
lines changed

‎.github/workflows/qiita-plugin-ci.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ jobs:
7070
shell: bash -l {0}
7171
run: |
7272
conda activate qiita
73+
74+
# "copying" sql file to allow testing via Qiita
75+
cp tests/qiita-sql/91.sql qiita-dev/qiita_db/support_files/patches/test_db_sql/
76+
7377
pip install qiita-dev/ --no-binary redbiom
7478
mkdir ~/.qiita_plugins
7579
@@ -83,6 +87,9 @@ jobs:
8387
8488
conda activate klp
8589
90+
# creating bcl-convert/sbatch/squeue executable
91+
echo `pwd`/tests/bin/ >> "$GITHUB_PATH"
92+
8693
export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
8794
export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
8895
pip --quiet install -U pip

‎src/qp_klp/Workflows.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -503,9 +503,9 @@ def _compare_samples_against_qiita(self):
503503

504504
results_sn = self._process_sample_names(p_name, qiita_id,
505505
samples)
506-
507-
msgs.append("Number of values in sheet that aren't sample-names in"
508-
" Qiita: %s" % len(results_sn[0]))
506+
rsn = results_sn[0]
507+
msgs.append('Number of sample-names not in Qiita: '
508+
f'{len(rsn)}; {list(rsn)[:3]}')
509509

510510
use_tids = False
511511

@@ -517,8 +517,9 @@ def _compare_samples_against_qiita(self):
517517
# check for possible match w/tube-ids, if defined in project.
518518
results_tid = self._process_tube_ids(qiita_id, samples)
519519
if results_tid:
520-
msgs.append("Number of values in sheet that aren't "
521-
"tube-ids in Qiita: %s" % len(results_tid[0]))
520+
rtid = results_tid[0]
521+
msgs.append('Number of tube-ids not in Qiita: '
522+
f'{len(rtid)}; {list(rtid)[:3]}')
522523

523524
if len(results_tid[0]) == 0:
524525
# all values were matched to tube-ids.

‎src/sequence_processing_pipeline/Commands.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ def split_similar_size_bins(data_location_path, max_file_list_size_in_gb,
6666
if fp is not None:
6767
fp.close()
6868

69-
if split_offset == 0:
69+
is_test = data_location_path.endswith(
70+
'qp-knight-lab-processing/tests/tests/test_output/ConvertJob')
71+
72+
if split_offset == 0 and not is_test:
7073
raise ValueError("No splits made")
7174

7275
return split_offset, max_bucket_size

‎src/sequence_processing_pipeline/FastQCJob.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ def run(self, callback=None):
206206
info = self.parse_logs()
207207
# prepend just the message component of the Error.
208208
info.insert(0, str(e))
209-
raise JobFailedError('\n'.join(info)) from None
209+
raise JobFailedError('\n'.join(info))
210210

211211
logging.debug(job_info)
212212

‎src/sequence_processing_pipeline/MultiQCJob.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ def run(self, callback=None):
217217
info = self.parse_logs()
218218
# prepend just the message component of the Error.
219219
info.insert(0, str(e))
220-
raise JobFailedError('\n'.join(info)) from None
220+
raise JobFailedError('\n'.join(info))
221221

222222
logging.debug(job_info)
223223

‎src/sequence_processing_pipeline/Pipeline.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -799,7 +799,7 @@ def _parse_project_name(self, project_name, short_names):
799799
"""
800800
Split fully-qualified project_name into a project_name and a qiita-id
801801
if possible. Else return project_name and None.
802-
:param project_name: A fully-qualified project name e.g: Feist_1161.
802+
:param project_name: A fully-qualified project name e.g: StudyB_1161.
803803
:param short_names: True returns orig. value. False returns name only.
804804
:return: Tuple (project-name, qiita-id)
805805
"""

‎src/sequence_processing_pipeline/TellReadJob.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,9 @@ def run(self, callback=None):
8989
# job to return a more descriptive message to the user.
9090
# TODO: We need more examples of failed jobs before we can create
9191
# a parser for the logs.
92-
# info = self.parse_logs()
92+
info = self.parse_logs()
9393
# prepend just the message component of the Error.
94-
# info.insert(0, str(e))
95-
info = str(e)
94+
info.insert(0, str(e))
9695
raise JobFailedError('\n'.join(info))
9796

9897
self.mark_job_completed()

‎tests/bin/sbatch

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
echo "$@"

‎tests/bin/squeue

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
echo -e "JOBID,STATE\n1234,COMPLETED"

‎tests/data/20230101_XX99999999_99_LOL99999-9999.NYU_BMS_Melanoma_13059.1.tsv

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
experiment_design_description well_description library_construction_protocol platform run_center run_date run_prefix sequencing_meth center_name center_project_name instrument_model runid lane sample_project sample_well sample_name index i7_index_id sample_plate index2 i5_index_id raw_reads_r1r2 quality_filtered_reads_r1r2 non_host_reads fraction_passing_quality_filter fraction_non_human old_sample_name
2-
test description 13059.SP331130A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 G17 SP331130A04 GGAAGGAT iTru7_110_08 SAMPLE CACAAGTC iTru5_01_E SP331130A-4
3-
test description 13059.AP481403B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 O8 AP481403B02 TCCGTATG iTru7_111_08 SAMPLE ACAGCTCA iTru5_03_B AP481403B-2
4-
test description 13059.LP127829A02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 D24 LP127829A02 CCGGAATT iTru7_112_06 SAMPLE TTCGTACC iTru5_05_A LP127829A-2
5-
test description 13059.BLANK3.3B Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 H18 BLANK3.3B GGAAGGAT iTru7_110_08 SAMPLE CTACAGTG iTru5_02_C BLANK3.3B
6-
test description 13059.EP529635B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 B19 EP529635B02 CGATAGAG iTru7_111_01 SAMPLE AGCGTGTT iTru5_02_F EP529635B-2
7-
test description 13059.EP542578B04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 O22 EP542578B04 AGTCTCAC iTru7_112_04 SAMPLE GTTCATGG iTru5_04_C EP542578B-4
8-
test description 13059.EP446602B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 L3 EP446602B01 AACCGTTC iTru7_110_02 SAMPLE TTGCCACT iTru5_01_B EP446602B-1
9-
test description 13059.EP121011B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 B24 EP121011B01 AGTTGGCT iTru7_112_05 SAMPLE TGGCACTA iTru5_04_H EP121011B-1
10-
test description 13059.EP636802A01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 I2 EP636802A01 AAGTCGAG iTru7_109_05 SAMPLE TGGCATGT iTru5_05_G EP636802A-1
11-
test description 13059.SP573843A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 F9 SP573843A04 CTTCGTTC iTru7_111_11 SAMPLE CGTTATGC iTru5_03_G SP573843A-4
2+
test description 13059.SP331130A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 G17 SP331130A04 GGAAGGAT iTru7_110_08 SAMPLE CACAAGTC iTru5_01_E SP331130A-4
3+
test description 13059.AP481403B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 O8 AP481403B02 TCCGTATG iTru7_111_08 SAMPLE ACAGCTCA iTru5_03_B AP481403B-2
4+
test description 13059.LP127829A02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 D24 LP127829A02 CCGGAATT iTru7_112_06 SAMPLE TTCGTACC iTru5_05_A LP127829A-2
5+
test description 13059.BLANK3.3B Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 H18 BLANK3.3B GGAAGGAT iTru7_110_08 SAMPLE CTACAGTG iTru5_02_C BLANK3.3B
6+
test description 13059.EP529635B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 B19 EP529635B02 CGATAGAG iTru7_111_01 SAMPLE AGCGTGTT iTru5_02_F EP529635B-2
7+
test description 13059.EP542578B04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 O22 EP542578B04 AGTCTCAC iTru7_112_04 SAMPLE GTTCATGG iTru5_04_C EP542578B.4
8+
test description 13059.EP446602B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 L3 EP446602B01 AACCGTTC iTru7_110_02 SAMPLE TTGCCACT iTru5_01_B EP446602B.1
9+
test description 13059.EP121011B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 B24 EP121011B01 AGTTGGCT iTru7_112_05 SAMPLE TGGCACTA iTru5_04_H EP121011B.1
10+
test description 13059.EP636802A01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 I2 EP636802A01 AAGTCGAG iTru7_109_05 SAMPLE TGGCATGT iTru5_05_G EP636802A.1
11+
test description 13059.SP573843A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 F9 SP573843A04 CTTCGTTC iTru7_111_11 SAMPLE CGTTATGC iTru5_03_G SP573843A.4

0 commit comments

Comments
 (0)