Skip to content

Commit 0bbc41c

Browse files
Lehmann-FabianLehmann-Fabian
authored
Merge pull request #13 from CommonWorkflowScheduler/wow
WOW Scheduling strategy
2 parents 2aecf43 + a02b84b commit 0bbc41c

File tree

148 files changed

+9395
-249
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+9395
-249
lines changed

‎.github/workflows/workflow.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
strategy:
1414
fail-fast: false
1515
matrix:
16-
java_version: [ 17, 21 ]
16+
java_version: [ 21 ]
1717
steps:
1818
- name: Checkout
1919
uses: actions/checkout@v3
@@ -32,10 +32,10 @@ jobs:
3232
steps:
3333
- name: Checkout
3434
uses: actions/checkout@v3
35-
- name: Set up JDK 17
35+
- name: Set up JDK 21
3636
uses: actions/setup-java@v3
3737
with:
38-
java-version: 17
38+
java-version: 21
3939
distribution: 'temurin'
4040
- name: Set outputs
4141
id: vars

‎Dockerfile

+21-17
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
1-
FROM maven:3-openjdk-17-slim AS build
2-
WORKDIR /build
3-
COPY pom.xml pom.xml
4-
RUN mvn dependency:go-offline --no-transfer-progress -Dmaven.repo.local=/mvn/.m2nrepo/repository
5-
COPY src/ src/
6-
RUN mvn package --no-transfer-progress -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository
7-
8-
#
9-
# Package stage
10-
#
11-
FROM openjdk:17-alpine
1+
FROM eclipse-temurin:21-jdk-jammy AS builder
122
WORKDIR /app
13-
RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data
14-
COPY --from=build /build/target/cws-k8s-scheduler*.jar cws-k8s-scheduler.jar
15-
RUN chown -R javauser:javagroup /app
16-
USER javauser
17-
EXPOSE 8080
18-
ENTRYPOINT ["java","-jar","/app/cws-k8s-scheduler.jar"]
3+
4+
RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
5+
6+
COPY pom.xml .
7+
RUN mvn dependency:go-offline --no-transfer-progress
8+
9+
COPY src/ ./src/
10+
RUN mvn package --no-transfer-progress -DskipTests
11+
12+
FROM eclipse-temurin:21-jre-jammy
13+
14+
WORKDIR /app
15+
16+
RUN apt-get update && apt-get install -y \
17+
libglib2.0-0 \
18+
&& rm -rf /var/lib/apt/lists/*
19+
20+
COPY --from=builder /app/target/cws-k8s-scheduler-*-SNAPSHOT.jar app.jar
21+
22+
CMD ["java", "-jar", "app.jar"]

‎Dockerfile-development

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM maven:3-openjdk-17-slim AS build
1+
FROM maven:3-openjdk-18-slim AS build
22
WORKDIR /build
33
COPY pom.xml pom.xml
44
RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository

‎Jenkinsfile

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
pipeline {
2+
agent {
3+
kubernetes {
4+
yamlFile 'jenkins-pod.yaml'
5+
}
6+
}
7+
environment {
8+
// creates DOCKERHUB_USR and DOCKERHUB_PSW env variables
9+
DOCKERHUB = credentials('fondahub-dockerhub')
10+
}
11+
12+
stages {
13+
stage('Build') {
14+
steps {
15+
container('maven') {
16+
// run a clean build without tests to see if the project compiles
17+
sh 'mvn clean test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V'
18+
}
19+
}
20+
}
21+
22+
stage('Test') {
23+
steps {
24+
container('maven') {
25+
// run JUnit tests
26+
sh 'mvn test -B -V'
27+
}
28+
}
29+
post {
30+
// collect test results
31+
always {
32+
junit 'target/surefire-reports/TEST-*.xml'
33+
jacoco classPattern: 'target/classes,target/test-classes', execPattern: 'target/coverage-reports/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src/main/java,src/test/java'
34+
archiveArtifacts 'target/surefire-reports/TEST-*.xml'
35+
archiveArtifacts 'target/*.exec'
36+
}
37+
}
38+
}
39+
40+
stage('Package') {
41+
steps {
42+
container('maven') {
43+
sh 'mvn package -DskipTests=true -Dmaven.javadoc.skip=true -B -V'
44+
}
45+
}
46+
post {
47+
success {
48+
archiveArtifacts 'target/*.jar'
49+
}
50+
}
51+
}
52+
53+
stage('Static Code Analysis') {
54+
steps {
55+
container('maven') {
56+
withSonarQubeEnv('fonda-sonarqube') {
57+
sh '''
58+
mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \
59+
-Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \
60+
-Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \
61+
-Dsonar.junit.reportPaths=target/surefire-reports
62+
'''
63+
}
64+
}
65+
}
66+
}
67+
68+
stage('Build and push Docker') {
69+
// only push images from the master branch
70+
when {
71+
branch "master"
72+
}
73+
// agents are specified per stage to enable real parallel execution
74+
parallel {
75+
stage('workflow-k8s-scheduler') {
76+
agent {
77+
kubernetes {
78+
yamlFile 'jenkins-pod.yaml'
79+
}
80+
}
81+
steps {
82+
container('hadolint') {
83+
sh "hadolint --format json Dockerfile | tee -a hadolint_scheduler.json"
84+
}
85+
// build and push image to fondahub/workflow-k8s-scheduler
86+
container('docker') {
87+
sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin"
88+
sh "docker build . -t fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}"
89+
sh "docker tag fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]} fondahub/workflow-k8s-scheduler:latest"
90+
sh "docker push fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}"
91+
sh "docker push fondahub/workflow-k8s-scheduler:latest"
92+
}
93+
}
94+
post {
95+
always {
96+
archiveArtifacts "hadolint_scheduler.json"
97+
recordIssues(
98+
aggregatingResults: true,
99+
tools: [hadoLint(pattern: "hadolint_scheduler.json", id: "scheduler")]
100+
)
101+
}
102+
}
103+
}
104+
stage('vsftpd') {
105+
agent {
106+
kubernetes {
107+
yamlFile 'jenkins-pod.yaml'
108+
}
109+
}
110+
steps {
111+
container('hadolint') {
112+
sh "hadolint --format json daemons/ftp/Dockerfile | tee -a hadolint_vsftpd.json"
113+
}
114+
// build and push image to fondahub/vsftpd
115+
container('docker') {
116+
sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin"
117+
sh "docker build daemons/ftp/ -t fondahub/vsftpd:${GIT_COMMIT[0..7]}"
118+
sh "docker tag fondahub/vsftpd:${GIT_COMMIT[0..7]} fondahub/vsftpd:latest"
119+
sh "docker push fondahub/vsftpd:${GIT_COMMIT[0..7]}"
120+
sh "docker push fondahub/vsftpd:latest"
121+
}
122+
}
123+
post {
124+
always {
125+
archiveArtifacts "hadolint_vsftpd.json"
126+
recordIssues(
127+
aggregatingResults: true,
128+
tools: [hadoLint(pattern: "hadolint_vsftpd.json", id: "vsfptd")]
129+
)
130+
}
131+
}
132+
}
133+
}
134+
}
135+
}
136+
}

‎README.md

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
# Common Workflow Scheduler for Kubernetes
2-
[![DOI](https://zenodo.org/badge/596122315.svg)](https://zenodo.org/badge/latestdoi/596122315)
1+
# Kubernetes Workflow Scheduler
32

4-
In this repository, you will find the Common Workflow Scheduler for Kubernetes proposed in the paper "**How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**."
3+
SWAGGER: http://localhost:8080/swagger-ui.html
4+
5+
API-DOCS: http://localhost:8080/v3/api-docs/
56

67
---
78
#### Build
@@ -188,6 +189,7 @@ The following strategies are available:
188189
| random | Randomly prioritize tasks. |
189190
| max | Prioritize tasks with larger input size. |
190191
| min | Prioritize tasks with smaller input size. |
192+
| wow | WOW scheduler for data location awareness. This is scheduling + node assignment. Details are provided in our paper [tbd](tbd). |
191193

192194
| Node Assignment Strategy | Behaviour |
193195
|--------------------------|-----------------------------------------------------------------------------------------|
@@ -215,4 +217,4 @@ Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Le
215217
```
216218
---
217219
#### Acknowledgement:
218-
This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis."
220+
This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis."

‎daemons/ftp/Dockerfile

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM python:slim
2+
RUN apt update && apt install -y \
3+
vsftpd \
4+
&& rm -rf /var/lib/apt/lists/*
5+
6+
RUN mkdir -p /var/run/vsftpd/empty
7+
8+
COPY vsftpd.conf /etc/vsftpd.conf
9+
10+
USER root
11+
RUN echo 'root:password' | chpasswd
12+
13+
COPY ftp.py /code/ftp.py
14+
15+
WORKDIR /code
16+
17+
ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"]

0 commit comments

Comments
 (0)