Skip to content

Commit 748ad9b

Browse files
kubranarcikubranarci
committed
add nf-test files
1 parent 0ca66e3 commit 748ad9b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+5307
-261
lines changed

‎.github/workflows/ci.yml

+79-21
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,117 @@
11
name: nf-core CI
22
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
33
on:
4-
push:
4+
pull_request:
55
branches:
66
- dev
7-
pull_request:
7+
- master
88
release:
99
types: [published]
1010
workflow_dispatch:
1111

1212
env:
13+
NFT_DIFF: "pdiff"
14+
NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
15+
NFT_VER: "0.9.2"
16+
NFT_WORKDIR: "~"
1317
NXF_ANSI_LOG: false
18+
NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
19+
NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
1420

1521
concurrency:
1622
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
1723
cancel-in-progress: true
1824

1925
jobs:
2026
test:
21-
name: Run pipeline with test data
27+
name: "${{ matrix.NXF_VER }} | ${{ matrix.profile }} | ${{ matrix.shard }}/6"
2228
# Only run on push if this is the nf-core dev branch (merged PRs)
23-
if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/variantbenchmarking') }}"
29+
if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/demultiplex') }}"
2430
runs-on: ubuntu-latest
2531
strategy:
32+
fail-fast: false
2633
matrix:
2734
NXF_VER:
2835
- "24.04.2"
29-
- "latest-everything"
36+
- "latest-stable"
37+
profile:
38+
- "docker"
39+
- "singularity"
40+
isMaster:
41+
- ${{ github.base_ref == 'master' }}
42+
# Exclude conda and singularity on dev
43+
exclude:
44+
- isMaster: false
45+
profile: "conda"
46+
- isMaster: false
47+
profile: "singularity"
48+
shard: [1, 2, 3, 4, 5, 6]
3049
steps:
3150
- name: Check out pipeline code
3251
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
52+
with:
53+
fetch-depth: 0
3354

34-
- name: Install Nextflow
55+
- name: Set up Nextflow
3556
uses: nf-core/setup-nextflow@v2
3657
with:
3758
version: "${{ matrix.NXF_VER }}"
3859

39-
- name: Disk space cleanup
40-
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
60+
- name: Set up Apptainer
61+
if: matrix.profile == 'singularity'
62+
uses: eWaterCycle/setup-apptainer@main
63+
64+
- name: Set up Singularity
65+
if: matrix.profile == 'singularity'
66+
run: |
67+
mkdir -p $NXF_SINGULARITY_CACHEDIR
68+
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
69+
70+
- name: Set up Miniconda
71+
if: matrix.profile == 'conda'
72+
uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
73+
with:
74+
miniconda-version: "latest"
75+
auto-update-conda: true
76+
conda-solver: libmamba
77+
channels: conda-forge,bioconda
4178

42-
- name: Run pipeline with test data for germline analysis (docker)
43-
# TODO nf-core: You can customise CI pipeline run tests as required
44-
# For example: adding multiple test runs with different parameters
45-
# Remember that you can parallelise this by using strategy.matrix
79+
- name: Set up Conda
80+
if: matrix.profile == 'conda'
4681
run: |
47-
nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
82+
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
83+
echo $(realpath python) >> $GITHUB_PATH
4884
49-
- name: Run pipeline with test data (singularity)
50-
# TODO nf-core: You can customise CI pipeline run tests as required
85+
- name: Clean up Disk space
86+
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
87+
88+
- uses: actions/setup-python@v4
89+
with:
90+
python-version: "3.11"
91+
architecture: "x64"
92+
93+
- name: Install pdiff to see diff between nf-test snapshots
5194
run: |
52-
nextflow run ${GITHUB_WORKSPACE} -profile test,singularity --outdir ./results
53-
if: "${{ github.base_ref == 'master' }}"
95+
python -m pip install --upgrade pip
96+
pip install pdiff
5497
55-
- name: Run pipeline with test data (conda)
56-
# TODO nf-core: You can customise CI pipeline run tests as required
98+
- uses: nf-core/setup-nf-test@v1
99+
with:
100+
version: ${{ env.NFT_VER }}
101+
102+
- name: Run Tests (Shard ${{ matrix.shard }}/6)
57103
run: |
58-
nextflow run ${GITHUB_WORKSPACE} -profile test,conda --outdir ./results
59-
if: "${{ github.base_ref == 'master' }}"
104+
nf-test test \
105+
--ci \
106+
--shard ${{ matrix.shard }}/6 \
107+
--changed-since HEAD^ \
108+
--profile "+${{ matrix.profile }}" \
109+
--filter pipeline \
110+
--junitxml=test.xml
111+
112+
- name: Publish Test Report
113+
uses: mikepenz/action-junit-report@v3
114+
if: always() # always run even if the previous step fails
115+
with:
116+
report_paths: test.xml
117+
annotate_only: true

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ null/*
1313
results*/*
1414
.nf-test/
1515
.nf-test.log
16-
16+
test/.nf-test.log

‎conf/modules.config

+3-3
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ process {
3939
]
4040
}
4141
withName: SVYNC {
42-
ext.prefix = {"${meta.id}.${meta.caller}"}
42+
ext.prefix = {"${meta.id}.${meta.caller}.svync"}
4343
publishDir = [
4444
path: {"${params.outdir}/${params.variant_type}/${meta.id}/preprocess"},
4545
pattern: "*{.vcf.gz,vcf.gz.tbi}",
@@ -192,7 +192,7 @@ process {
192192
]
193193
}
194194
withName: "HAPPY_PREPY" {
195-
ext.prefix = {"${meta.id}.${params.truth_id}.${meta.caller}"}
195+
ext.prefix = {"${meta.id}.${params.truth_id}.${meta.caller}.prepy"}
196196
ext.args = {"--fixchr --filter-nonref --bcftools-norm"}
197197
publishDir = [
198198
path: {"${params.outdir}/${params.variant_type}/${meta.id}/preprocess"},
@@ -315,7 +315,7 @@ process {
315315
]
316316
}
317317
withName: PICARD_LIFTOVERVCF {
318-
ext.prefix = {"${meta.id}}"}
318+
ext.prefix = {"${meta.id}"}
319319
ext.args = {"--WARN_ON_MISSING_CONTIG true"}
320320
}
321321
withName: BCFTOOLS_RENAME_CHR {

‎conf/tests/germline_small.config

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/variantbenchmarking -profile test,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
process {
14+
resourceLimits = [
15+
cpus: 4,
16+
memory: '15.GB',
17+
time: '1.h'
18+
]
19+
}
20+
21+
params {
22+
config_profile_name = 'Test profile'
23+
config_profile_description = 'Minimal test dataset to check pipeline function'
24+
25+
// Input data
26+
input = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/samplesheet_small.csv"
27+
outdir = "results"
28+
29+
// Genome references
30+
genome = 'GRCh38'
31+
32+
// Processes
33+
analysis = 'germline'
34+
variant_type = "small"
35+
method = 'happy,rtgtools'
36+
preprocess = "normalization,deduplication,prepy"
37+
include_expression = 'FILTER="."'
38+
39+
// truth information
40+
truth_id = "HG002"
41+
truth_vcf = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/CRMG_benchmark_dataset/HG002_GRCh38_CMRG_smallvar_v1.00.vcf.gz"
42+
regions_bed = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/CRMG_benchmark_dataset/HG002_GRCh38_CMRG_smallvar_v1.00.bed"
43+
}

‎conf/tests/germline_sv.config

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/variantbenchmarking -profile test,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
process {
14+
resourceLimits = [
15+
cpus: 4,
16+
memory: '15.GB',
17+
time: '1.h'
18+
]
19+
}
20+
21+
params {
22+
config_profile_name = 'Test profile'
23+
config_profile_description = 'Minimal test dataset to check pipeline function'
24+
25+
// Input data
26+
input = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/samplesheet_sv.csv"
27+
outdir = "results"
28+
29+
// Genome references
30+
genome = 'GRCh38'
31+
32+
// Processes
33+
analysis = 'germline'
34+
variant_type = "structural"
35+
method = 'svanalyzer,wittyer,truvari'
36+
preprocess = "normalization,deduplication"
37+
sv_standardization = "svync,homogenize"
38+
min_sv_size = 30
39+
truth_id = "HG002"
40+
include_expression = 'FILTER="."'
41+
42+
truth_vcf = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/hg38/truth/HG002_GRCh38_difficult_medical_gene_SV_benchmark_v0.01.chr21.vcf.gz"
43+
regions_bed = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/hg38/truth/HG002_GRCh38_difficult_medical_gene_SV_benchmark_v01.ch21.bed"
44+
}

‎conf/tests/liftover_hg37.config

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/benchmark -profile test_liftover,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
params {
14+
config_profile_name = 'Test profile'
15+
config_profile_description = 'Minimal test dataset to check pipeline function'
16+
17+
// Limit resources so that this can run on GitHub Actions
18+
max_cpus = 2
19+
max_memory = '6.5GB'
20+
max_time = '8.h'
21+
22+
// Input data
23+
input = 'https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/samplesheet_sv_hg37.csv'
24+
outdir = 'results'
25+
genome = 'GRCh37'
26+
27+
// Processes
28+
analysis = 'germline'
29+
variant_type = "structural"
30+
method = 'truvari'
31+
preprocess = "normalization,deduplication,filter_contigs"
32+
min_sv_size = 30
33+
34+
truth_id = "HG002"
35+
truth_vcf = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/hg38/truth/HG002_GRCh38_difficult_medical_gene_SV_benchmark_v0.01.chr21.vcf.gz"
36+
37+
//liftover files
38+
liftover = true
39+
chain = "http://ftp.ensembl.org/pub/assembly_mapping/homo_sapiens/GRCh38_to_GRCh37.chain.gz"
40+
rename_chr = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/grch38_grch37.txt"
41+
}

‎conf/tests/liftover_hg38.config

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/benchmark -profile test_liftover,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
params {
14+
config_profile_name = 'Test profile'
15+
config_profile_description = 'Minimal test dataset to check pipeline function'
16+
17+
// Limit resources so that this can run on GitHub Actions
18+
max_cpus = 2
19+
max_memory = '6.5GB'
20+
max_time = '8.h'
21+
22+
// Input data
23+
input = 'https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/samplesheet_small.csv'
24+
outdir = 'results'
25+
genome = 'GRCh38'
26+
27+
// Processes
28+
analysis = 'germline'
29+
truth_id = "HG002"
30+
variant_type = "small"
31+
method = 'rtgtools,happy'
32+
preprocess = "normalization,deduplication,filter_contigs"
33+
34+
truth_vcf = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/hg37/truth/HG002_GRCh37_1_22_v4.2.1_highconf.chr21.vcf.gz"
35+
regions_bed = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/hg37/truth/HG002_GRCh37_1_22_v4.2.1_highconf.bed"
36+
37+
//liftover files
38+
liftover = true
39+
chain = "https://raw.githubusercontent.com/broadinstitute/gatk/master/scripts/funcotator/data_sources/gnomAD/b37ToHg38.over.chain"
40+
rename_chr = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/grch37_grch38.txt"
41+
}

‎conf/tests/somatic_indel.config

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/variantbenchmarking -profile test_somatic,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
params {
14+
config_profile_name = 'Test profile'
15+
config_profile_description = 'Minimal test dataset to check pipeline function'
16+
17+
// Limit resources so that this can run on GitHub Actions
18+
max_cpus = 2
19+
max_memory = '6.5GB'
20+
max_time = '8.h'
21+
22+
// Input data
23+
input = 'https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/SV_testdata/samplesheet_somatic_indel.csv'
24+
outdir = 'results'
25+
26+
// Genome references
27+
genome = 'GRCh38'
28+
analysis = 'somatic'
29+
truth_id = "SEQC2"
30+
method = 'sompy'
31+
preprocess = "filter_contigs"
32+
variant_type = "indel"
33+
34+
truth_vcf = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/somatic/truth/hg38/sINDEL_truth_set_v1.0.chr21.vcf.gz"
35+
regions_bed = "https://raw.githubusercontent.com/kubranarci/benchmark_datasets/main/somatic/truth/hg38/high-confidence_sINDEL_in_HC_regions_v1.2.chr21.vcf.gz"
36+
37+
}

0 commit comments

Comments
 (0)