Skip to content

Commit 457d320

Browse files
marchoeppnermarchoeppner
authored
Initial commit
0 parents  commit 457d320

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2178
-0
lines changed

‎.github/workflows/linting.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: nextflow linting
2+
# This workflow is triggered on pushes and PRs to the repository to ensure
3+
# valid code is being contributed.
4+
5+
on:
6+
push:
7+
branches:
8+
- dev
9+
- TEMPLATE
10+
pull_request:
11+
release:
12+
types: [published]
13+
14+
jobs:
15+
groovy-lint:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- uses: actions/setup-node@v4
21+
22+
- name: Install groovy lint
23+
run: npm install -g npm-groovy-lint
24+
25+
- name: Run groovy lint check
26+
run: npm-groovy-lint ${GITHUB_WORKSPACE}
27+

‎.groovylintrc.json

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"extends" : "recommended",
3+
"rules": {
4+
"comments.ClassJavadoc": "off",
5+
"formatting.Indentation": {
6+
"spacesPerIndentLevel": 4,
7+
"severity": "info"
8+
},
9+
"security.JavaIoPackageAccess": "off",
10+
"security.SystemExit": "off",
11+
"convention.CompileStatic": "off",
12+
"unnecessary.UnnecessaryGetter": "off",
13+
"convention.MethodReturnTypeRequired": "off",
14+
"convention.MethodParameterTypeRequired": "off",
15+
"convention.VariableTypeRequired": "off",
16+
"exceptions.MissingNewInThrowStatement": "off",
17+
"convention.NoDef": "off",
18+
"design.ImplementationAsType": "off",
19+
"unnecessary.UnnecessaryPublicModifier": "off",
20+
"unnecessary.DuplicateStringLiteral": "off"
21+
}
22+
}

‎.prettierignore

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
email_template.html
2+
.nextflow*
3+
work/
4+
data/
5+
results/
6+
.DS_Store
7+
testing/
8+
testing*
9+
*.pyc
10+
bin/

‎.prettierrc.yml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
printWidth: 120

‎.vscode/settings.json

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"cSpell.words": [
3+
"cpus",
4+
"Javadoc",
5+
"marchoeppner",
6+
"multiqc",
7+
"nextflow",
8+
"outdir",
9+
"pipefail",
10+
"pipelinelogo",
11+
"println",
12+
"slurm",
13+
"subworkflow"
14+
]
15+
}

‎Dockerfile

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM nfcore/base
2+
LABEL authors="Marc Hoeppner" \
3+
description="Docker image containing all requirements for THIS pipeline"
4+
5+
COPY environment.yml /
6+
RUN conda env create -f /environment.yml && conda clean -a
7+
ENV PATH /opt/conda/envs/PIPELINE-VERSION/bin:$PATH
8+
9+
RUN apt-get -y update && apt-get -y install make wget

‎LICENSE

+674
Large diffs are not rendered by default.

‎README.md

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Pipeline
2+
3+
## Documentation
4+
5+
1. [What happens in this pipeline?](docs/pipeline.md)
6+
2. [Installation and configuration](docs/installation.md)
7+
3. [Running the pipeline](docs/usage.md)
8+
4. [Output](docs/output.md)
9+
5. [Software](docs/software.md)
10+
5. [Troubleshooting](docs/troubleshooting.md)
11+
6. [Developer guide](docs/developer.md)

‎assets/email_template.html

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
<html>
2+
<head>
3+
<head>
4+
<meta charset="utf-8">
5+
<meta http-equiv="X-UA-Compatible" content="IE=edge">
6+
<meta name="viewport" content="width=device-width, initial-scale=1">
7+
8+
<meta name="description" content="Pipeline">
9+
<title>Pipeline Report</title>
10+
</head>
11+
<body>
12+
<div style="font-family: Helvetica, Arial, sans-serif; padding: 30px; max-width: 800px; margin: 0 auto;">
13+
14+
<img src="cid:ikmblogo">
15+
16+
<h1>Pipeline: version ${version}</h1>
17+
<h2>Run Name: ${runName}</h2>
18+
19+
<% if (!success){
20+
out << """
21+
<div style="color: #a94442; background-color: #f2dede; border-color: #ebccd1; padding: 15px; margin-bottom: 20px; border: 1px solid transparent; border-radius: 4px;">
22+
<h4 style="margin-top:0; color: inherit;">Pipeline execution completed unsuccessfully!</h4>
23+
<p>The exit status of the task that caused the workflow execution to fail was: <code>$exitStatus</code>.</p>
24+
<p>The full error message was:</p>
25+
<pre style="white-space: pre-wrap; overflow: visible; margin-bottom: 0;">${errorReport}</pre>
26+
</div>
27+
"""
28+
} else {
29+
out << """
30+
<div style="color: #3c763d; background-color: #dff0d8; border-color: #d6e9c6; padding: 15px; margin-bottom: 20px; border: 1px solid transparent; border-radius: 4px;">
31+
Pipeline execution completed successfully!
32+
</div>
33+
"""
34+
}
35+
%>
36+
37+
<p>The workflow was completed at <strong>$dateComplete</strong> (duration: <strong>$duration</strong>)</p>
38+
<p>The command used to launch the workflow was as follows:</p>
39+
<pre style="white-space: pre-wrap; overflow: visible; background-color: #ededed; padding: 15px; border-radius: 4px; margin-bottom:30px;">$commandLine</pre>
40+
41+
<h3>Pipeline Configuration:</h3>
42+
<table style="width:100%; max-width:100%; border-spacing: 0; border-collapse: collapse; border:0; margin-bottom: 30px;">
43+
<tbody style="border-bottom: 1px solid #ddd;">
44+
<% out << summary.collect{ k,v -> "<tr><th style='text-align:left; padding: 8px 0; line-height: 1.42857143; vertical-align: top; border-top: 1px solid #ddd;'>$k</th><td style='text-align:left; padding: 8px; line-height: 1.42857143; vertical-align: top; border-top: 1px solid #ddd;'><pre style='white-space: pre-wrap; overflow: visible;'>$v</pre></td></tr>" }.join("\n") %>
45+
</tbody>
46+
</table>
47+
<p>
48+
Pipeline code available at: [email protected]:ikmb/xxx.git
49+
</p>
50+
51+
<hr style="height: 3px; padding: 0; margin: 24px 0; background-color: #e1e4e8; border: 0;">
52+
53+
54+
</div>
55+
56+
</body>
57+
</html>

‎assets/email_template.txt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
========================================
2+
Pipeline: version ${version}
3+
========================================
4+
Run Name: $runName
5+
6+
<% if (success){
7+
out << "## Pipeline execution completed successfully! ##"
8+
} else {
9+
out << """####################################################
10+
## Pipeline execution completed unsuccessfully! ##
11+
####################################################
12+
The exit status of the task that caused the workflow execution to fail was: $exitStatus.
13+
The full error message was:
14+
15+
${errorReport}
16+
"""
17+
} %>
18+
19+
The workflow was completed at $dateComplete (duration: $duration)
20+
21+
The command used to launch the workflow was as follows:
22+
23+
$commandLine
24+
25+
26+
27+
Pipeline Configuration:
28+
-----------------------
29+
<% out << summary.collect{ k,v -> " - $k: $v" }.join("\n") %>
30+
31+
32+
--

‎assets/pipelinelogo.png

593 Bytes
Loading

‎assets/sendmail_template.txt

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
To: $email
2+
Subject: $subject
3+
Mime-Version: 1.0
4+
Content-Type: multipart/related;boundary="ikmbmimeboundary"
5+
6+
--ikmbmimeboundary
7+
Content-Type: text/html; charset=utf-8
8+
9+
$email_html
10+
11+
--ikmbmimeboundary
12+
Content-Type: image/png;name="pipelinelogo.png"
13+
Content-Transfer-Encoding: base64
14+
Content-ID: <ikmblogo>
15+
Content-Disposition: inline; filename="pipelinelogo.png"
16+
17+
<% out << new File("$baseDir/assets/pipelinelogo.png").
18+
bytes.
19+
encodeBase64().
20+
toString().
21+
tokenize( '\n' )*.
22+
toList()*.
23+
collate( 76 )*.
24+
collect { it.join() }.
25+
flatten().
26+
join( '\n' ) %>
27+
%>
28+
29+
<%
30+
if (mqcFile){
31+
def mqcFileObj = new File("$mqcFile")
32+
if (mqcFileObj.length() < mqcMaxSize){
33+
out << """
34+
--ikmbmimeboundary
35+
Content-Type: text/html; name=\"multiqc_report\"
36+
Content-Transfer-Encoding: base64
37+
Content-ID: <mqcreport>
38+
Content-Disposition: attachment; filename=\"${mqcFileObj.getName()}\"
39+
40+
${mqcFileObj.
41+
bytes.
42+
encodeBase64().
43+
toString().
44+
tokenize( '\n' )*.
45+
toList()*.
46+
collate( 76 )*.
47+
collect { it.join() }.
48+
flatten().
49+
join( '\n' )}
50+
"""
51+
}}
52+
%>
53+
54+
--nfcoremimeboundary--

‎conf/base.config

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
2+
// Resource limits for different jobs
3+
4+
// Process setting
5+
6+
process {
7+
8+
// Global cluster parameters
9+
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
10+
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
11+
time = { check_max( 2.h * task.attempt, 'time' ) }
12+
13+
errorStrategy = { task.exitStatus in [143,137,140,7] ? 'retry' : 'finish' }
14+
maxRetries = 3
15+
maxErrors = '-1'
16+
17+
// Specific compute parameters for each process
18+
19+
withLabel: short_serial {
20+
time = { check_max( 2.h * task.attempt, 'time' ) }
21+
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
22+
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
23+
}
24+
withLabel: medium_serial {
25+
time = { check_max( 8.h * task.attempt, 'time' ) }
26+
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
27+
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
28+
}
29+
withLabel: long_serial {
30+
time = { check_max( 24.h * task.attempt, 'time' ) }
31+
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
32+
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
33+
}
34+
withLabel: short_parallel {
35+
time = { check_max( 2.h * task.attempt, 'time' ) }
36+
memory = { check_max( 24.GB * task.attempt, 'memory' ) }
37+
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
38+
}
39+
withLabel: medium_parallel {
40+
time = { check_max( 8.h * task.attempt, 'time' ) }
41+
memory = { check_max( 64.GB * task.attempt, 'memory' ) }
42+
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
43+
}
44+
withLabel: long_parallel {
45+
time = { check_max( 24.h * task.attempt, 'time' ) }
46+
memory = { check_max( 96.GB * task.attempt, 'memory' ) }
47+
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
48+
}
49+
withLabel: extra_long_parallel {
50+
time = { check_max( 96.h * task.attempt, 'time' ) }
51+
memory = { check_max( 120.GB * task.attempt, 'memory' ) }
52+
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
53+
}
54+
55+
}
56+
57+
58+

‎conf/local.config

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
// Job parameters to set for this environment
3+
4+
params {
5+
max_cpus = 8
6+
max_ram = 16.GB
7+
}
8+
9+
// Resource manager settings
10+
11+
executor {
12+
queueSize=4
13+
}
14+
15+
// Process setting
16+
17+
process {
18+
executor = 'local'
19+
}

‎conf/lsh.config

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
2+
// Job parameters to set for this environment
3+
4+
params {
5+
max_cpus = 40
6+
max_ram = 250
7+
}
8+
9+
// Resource manager settings
10+
11+
executor {
12+
queueSize=50
13+
}
14+
15+
// Process setting
16+
17+
process {
18+
executor = 'slurm'
19+
queue = 'all'
20+
}
21+
22+
singularity {
23+
enabled = true
24+
runOptions = "-B /work -B /scratch"
25+
cacheDir = "/work/singularity_cache"
26+
}
27+

‎conf/multiqc_config.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
title: "Pipeline report"
2+
subtitle: "Best-practice workflow for EDIT_IN_conf_multiqc_config.yaml"
3+
custom_logo: "pipelinelogo.png"
4+
custom_logo_title: "Pipeline title here"
5+
custom_logo_url: "http://www.github.com/marchoeppner"
6+
7+
extra_fn_clean_exts:
8+
- _R1
9+
- _R2
10+
- _duplicate_metrics.txt
11+
- .pass
12+
13+
report_comment: >
14+
This report has been generated automatically by EDIT_IN_conf_multiqc_config.yaml.
15+
For help interpreting the outputs, please see: https://github.com/marchoeppner/update_url
16+
report_header_info:
17+
- Contact E-mail: "[email protected]"
18+
- Application Type: "EDIT_IN_conf_multiqc_config.yaml"
19+
20+
top_modules:
21+
- 'general_stats'
22+

‎conf/resources.config

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
params {
2+
3+
references {
4+
5+
}
6+
7+
}

‎conf/test.config

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Example test config - requires a minimal set of input data (to be hosted in marchoeppner/nf-testdata)
2+
params {
3+
4+
samples = "https://raw.githubusercontent.com/marchoeppner/nf-testdata/master/deepvariant/samples.csv"
5+
intervals = "https://raw.githubusercontent.com/marchoeppner/nf-testdata/master/deepvariant/test-targets.bed"
6+
email = "[email protected]"
7+
run_name = "pipeline_test"
8+
max_cpus = 6
9+
}

0 commit comments

Comments
 (0)