Skip to content

inserting data to qiita for testing #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 28 commits into from
Apr 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/qiita-plugin-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ jobs:
shell: bash -l {0}
run: |
conda activate qiita

# "copying" sql file to allow testing via Qiita
cp tests/qiita-sql/91.sql qiita-dev/qiita_db/support_files/patches/test_db_sql/

pip install qiita-dev/ --no-binary redbiom
mkdir ~/.qiita_plugins

Expand All @@ -83,6 +87,9 @@ jobs:

conda activate klp

# creating bcl-convert/sbatch/squeue executable
echo `pwd`/tests/bin/ >> "$GITHUB_PATH"

export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
pip --quiet install -U pip
Expand Down
11 changes: 6 additions & 5 deletions src/qp_klp/Workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,9 +503,9 @@ def _compare_samples_against_qiita(self):

results_sn = self._process_sample_names(p_name, qiita_id,
samples)

msgs.append("Number of values in sheet that aren't sample-names in"
" Qiita: %s" % len(results_sn[0]))
rsn = results_sn[0]
msgs.append('Number of sample-names not in Qiita: '
f'{len(rsn)}; {list(rsn)[:3]}')

use_tids = False

Expand All @@ -517,8 +517,9 @@ def _compare_samples_against_qiita(self):
# check for possible match w/tube-ids, if defined in project.
results_tid = self._process_tube_ids(qiita_id, samples)
if results_tid:
msgs.append("Number of values in sheet that aren't "
"tube-ids in Qiita: %s" % len(results_tid[0]))
rtid = results_tid[0]
msgs.append('Number of tube-ids not in Qiita: '
f'{len(rtid)}; {list(rtid)[:3]}')

if len(results_tid[0]) == 0:
# all values were matched to tube-ids.
Expand Down
5 changes: 4 additions & 1 deletion src/sequence_processing_pipeline/Commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ def split_similar_size_bins(data_location_path, max_file_list_size_in_gb,
if fp is not None:
fp.close()

if split_offset == 0:
is_test = data_location_path.endswith(
'qp-knight-lab-processing/tests/tests/test_output/ConvertJob')

if split_offset == 0 and not is_test:
raise ValueError("No splits made")

return split_offset, max_bucket_size
Expand Down
2 changes: 1 addition & 1 deletion src/sequence_processing_pipeline/FastQCJob.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def run(self, callback=None):
info = self.parse_logs()
# prepend just the message component of the Error.
info.insert(0, str(e))
raise JobFailedError('\n'.join(info)) from None
raise JobFailedError('\n'.join(info))

logging.debug(job_info)

Expand Down
2 changes: 1 addition & 1 deletion src/sequence_processing_pipeline/MultiQCJob.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def run(self, callback=None):
info = self.parse_logs()
# prepend just the message component of the Error.
info.insert(0, str(e))
raise JobFailedError('\n'.join(info)) from None
raise JobFailedError('\n'.join(info))

logging.debug(job_info)

Expand Down
2 changes: 1 addition & 1 deletion src/sequence_processing_pipeline/Pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -799,7 +799,7 @@ def _parse_project_name(self, project_name, short_names):
"""
Split fully-qualified project_name into a project_name and a qiita-id
if possible. Else return project_name and None.
:param project_name: A fully-qualified project name e.g: Feist_1161.
:param project_name: A fully-qualified project name e.g: StudyB_1161.
:param short_names: True returns orig. value. False returns name only.
:return: Tuple (project-name, qiita-id)
"""
Expand Down
5 changes: 2 additions & 3 deletions src/sequence_processing_pipeline/TellReadJob.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,9 @@ def run(self, callback=None):
# job to return a more descriptive message to the user.
# TODO: We need more examples of failed jobs before we can create
# a parser for the logs.
# info = self.parse_logs()
info = self.parse_logs()
# prepend just the message component of the Error.
# info.insert(0, str(e))
info = str(e)
info.insert(0, str(e))
raise JobFailedError('\n'.join(info))

self.mark_job_completed()
Expand Down
1 change: 1 addition & 0 deletions tests/bin/sbatch
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
echo "$@"
1 change: 1 addition & 0 deletions tests/bin/squeue
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
echo -e "JOBID,STATE\n1234,COMPLETED"
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
experiment_design_description well_description library_construction_protocol platform run_center run_date run_prefix sequencing_meth center_name center_project_name instrument_model runid lane sample_project sample_well sample_name index i7_index_id sample_plate index2 i5_index_id raw_reads_r1r2 quality_filtered_reads_r1r2 non_host_reads fraction_passing_quality_filter fraction_non_human old_sample_name
test description 13059.SP331130A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 G17 SP331130A04 GGAAGGAT iTru7_110_08 SAMPLE CACAAGTC iTru5_01_E SP331130A-4
test description 13059.AP481403B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 O8 AP481403B02 TCCGTATG iTru7_111_08 SAMPLE ACAGCTCA iTru5_03_B AP481403B-2
test description 13059.LP127829A02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 D24 LP127829A02 CCGGAATT iTru7_112_06 SAMPLE TTCGTACC iTru5_05_A LP127829A-2
test description 13059.BLANK3.3B Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 H18 BLANK3.3B GGAAGGAT iTru7_110_08 SAMPLE CTACAGTG iTru5_02_C BLANK3.3B
test description 13059.EP529635B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 B19 EP529635B02 CGATAGAG iTru7_111_01 SAMPLE AGCGTGTT iTru5_02_F EP529635B-2
test description 13059.EP542578B04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 O22 EP542578B04 AGTCTCAC iTru7_112_04 SAMPLE GTTCATGG iTru5_04_C EP542578B-4
test description 13059.EP446602B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 L3 EP446602B01 AACCGTTC iTru7_110_02 SAMPLE TTGCCACT iTru5_01_B EP446602B-1
test description 13059.EP121011B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 B24 EP121011B01 AGTTGGCT iTru7_112_05 SAMPLE TGGCACTA iTru5_04_H EP121011B-1
test description 13059.EP636802A01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 I2 EP636802A01 AAGTCGAG iTru7_109_05 SAMPLE TGGCATGT iTru5_05_G EP636802A-1
test description 13059.SP573843A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 NYU_BMS_Melanoma_13059 F9 SP573843A04 CTTCGTTC iTru7_111_11 SAMPLE CGTTATGC iTru5_03_G SP573843A-4
test description 13059.SP331130A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 G17 SP331130A04 GGAAGGAT iTru7_110_08 SAMPLE CACAAGTC iTru5_01_E SP331130A-4
test description 13059.AP481403B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 O8 AP481403B02 TCCGTATG iTru7_111_08 SAMPLE ACAGCTCA iTru5_03_B AP481403B-2
test description 13059.LP127829A02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 D24 LP127829A02 CCGGAATT iTru7_112_06 SAMPLE TTCGTACC iTru5_05_A LP127829A-2
test description 13059.BLANK3.3B Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 H18 BLANK3.3B GGAAGGAT iTru7_110_08 SAMPLE CTACAGTG iTru5_02_C BLANK3.3B
test description 13059.EP529635B02 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 B19 EP529635B02 CGATAGAG iTru7_111_01 SAMPLE AGCGTGTT iTru5_02_F EP529635B-2
test description 13059.EP542578B04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 O22 EP542578B04 AGTCTCAC iTru7_112_04 SAMPLE GTTCATGG iTru5_04_C EP542578B.4
test description 13059.EP446602B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 L3 EP446602B01 AACCGTTC iTru7_110_02 SAMPLE TTGCCACT iTru5_01_B EP446602B.1
test description 13059.EP121011B01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 B24 EP121011B01 AGTTGGCT iTru7_112_05 SAMPLE TGGCACTA iTru5_04_H EP121011B.1
test description 13059.EP636802A01 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 I2 EP636802A01 AAGTCGAG iTru7_109_05 SAMPLE TGGCATGT iTru5_05_G EP636802A.1
test description 13059.SP573843A04 Knight Lab Kapa HyperPlus Illumina IGM 2017-09-02 SAMPLE sequencing by synthesis UCSD SAMPLE Illumina HiSeq 4000 211021_A00000_0000_SAMPLE 1 StudyA_13059 F9 SP573843A04 CTTCGTTC iTru7_111_11 SAMPLE CGTTATGC iTru5_03_G SP573843A.4
Loading