WOW Scheduling strategy

.github/workflows/workflow.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [ 17, 21 ]
+        java_version: [ 21 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -32,10 +32,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 21
           distribution: 'temurin'
       - name: Set outputs
         id: vars

Dockerfile

@@ -1,18 +1,22 @@
-FROM maven:3-openjdk-17-slim AS build
-WORKDIR /build
-COPY pom.xml pom.xml
-RUN mvn dependency:go-offline --no-transfer-progress -Dmaven.repo.local=/mvn/.m2nrepo/repository
-COPY src/ src/
-RUN mvn package --no-transfer-progress -DskipTests -Dmaven.repo.local=/mvn/.m2nrepo/repository
-
-#
-# Package stage
-#
-FROM openjdk:17-alpine
+FROM eclipse-temurin:21-jdk-jammy AS builder
 WORKDIR /app
-RUN addgroup -S javagroup && adduser -S javauser -G javagroup && mkdir data
-COPY --from=build /build/target/cws-k8s-scheduler*.jar cws-k8s-scheduler.jar
-RUN chown -R javauser:javagroup /app
-USER javauser
-EXPOSE 8080
-ENTRYPOINT ["java","-jar","/app/cws-k8s-scheduler.jar"]
+
+RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
+
+COPY pom.xml .
+RUN mvn dependency:go-offline --no-transfer-progress
+
+COPY src/ ./src/
+RUN mvn package --no-transfer-progress -DskipTests
+
+FROM eclipse-temurin:21-jre-jammy
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /app/target/cws-k8s-scheduler-*-SNAPSHOT.jar app.jar
+
+CMD ["java", "-jar", "app.jar"]

Dockerfile-development

@@ -1,4 +1,4 @@
-FROM maven:3-openjdk-17-slim AS build
+FROM maven:3-openjdk-18-slim AS build
 WORKDIR /build
 COPY pom.xml pom.xml
 RUN mkdir data/ && mvn dependency:go-offline -B -Dmaven.repo.local=/mvn/.m2nrepo/repository

Jenkinsfile

@@ -0,0 +1,136 @@
+pipeline {
+    agent {
+        kubernetes {
+            yamlFile 'jenkins-pod.yaml'
+        }
+    }
+    environment {
+        // creates DOCKERHUB_USR and DOCKERHUB_PSW env variables
+        DOCKERHUB = credentials('fondahub-dockerhub')
+    }
+
+    stages {
+        stage('Build') {
+            steps {
+                container('maven') {
+                    // run a clean build without tests to see if the project compiles
+                    sh 'mvn clean test-compile -DskipTests=true -Dmaven.javadoc.skip=true -B -V'
+                }
+            }
+        }
+
+        stage('Test') {
+            steps {
+                container('maven') {
+                    // run JUnit tests
+                    sh 'mvn test -B -V'
+                }
+            }
+            post {
+                // collect test results
+                always {
+                    junit 'target/surefire-reports/TEST-*.xml'
+                    jacoco classPattern: 'target/classes,target/test-classes', execPattern: 'target/coverage-reports/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src/main/java,src/test/java'
+                    archiveArtifacts 'target/surefire-reports/TEST-*.xml'
+                    archiveArtifacts 'target/*.exec'
+                }
+            }
+        }
+
+        stage('Package') {
+            steps {
+                container('maven') {
+                    sh 'mvn package -DskipTests=true -Dmaven.javadoc.skip=true -B -V'
+                }
+            }
+            post {
+                success {
+                    archiveArtifacts 'target/*.jar'
+                }
+            }
+        }
+
+        stage('Static Code Analysis') {
+            steps {
+                container('maven') {
+                    withSonarQubeEnv('fonda-sonarqube') {
+                        sh '''
+                            mvn sonar:sonar -B -V -Dsonar.projectKey=workflow_k8s_scheduler \
+                                -Dsonar.branch.name=$BRANCH_NAME -Dsonar.sources=src/main/java -Dsonar.tests=src/test/java \
+                                -Dsonar.inclusions="**/*.java" -Dsonar.test.inclusions="src/test/java/**/*.java" \
+                                -Dsonar.junit.reportPaths=target/surefire-reports
+                        '''
+                    }
+                }
+            }
+        }
+
+        stage('Build and push Docker') {
+            // only push images from the master branch
+            when {
+                branch "master"
+            }
+            // agents are specified per stage to enable real parallel execution
+            parallel {
+                stage('workflow-k8s-scheduler') {
+                    agent {
+                        kubernetes {
+                            yamlFile 'jenkins-pod.yaml'
+                        }
+                    }
+                    steps {
+                        container('hadolint') {
+                            sh "hadolint --format json Dockerfile | tee -a hadolint_scheduler.json"
+                        }
+                        // build and push image to fondahub/workflow-k8s-scheduler
+                        container('docker') {
+                            sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin"
+                            sh "docker build . -t fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}"
+                            sh "docker tag fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]} fondahub/workflow-k8s-scheduler:latest"
+                            sh "docker push fondahub/workflow-k8s-scheduler:${GIT_COMMIT[0..7]}"
+                            sh "docker push fondahub/workflow-k8s-scheduler:latest"
+                        }
+                    }
+                    post {
+                        always {
+                            archiveArtifacts "hadolint_scheduler.json"
+                            recordIssues(
+                                aggregatingResults: true,
+                                tools: [hadoLint(pattern: "hadolint_scheduler.json", id: "scheduler")]
+                            )
+                        }
+                    }
+                }
+                stage('vsftpd') {
+                    agent {
+                        kubernetes {
+                            yamlFile 'jenkins-pod.yaml'
+                        }
+                    }
+                    steps {
+                        container('hadolint') {
+                            sh "hadolint --format json daemons/ftp/Dockerfile | tee -a hadolint_vsftpd.json"
+                        }
+                        // build and push image to fondahub/vsftpd
+                        container('docker') {
+                            sh "echo $DOCKERHUB_PSW | docker login -u $DOCKERHUB_USR --password-stdin"
+                            sh "docker build daemons/ftp/ -t fondahub/vsftpd:${GIT_COMMIT[0..7]}"
+                            sh "docker tag fondahub/vsftpd:${GIT_COMMIT[0..7]} fondahub/vsftpd:latest"
+                            sh "docker push fondahub/vsftpd:${GIT_COMMIT[0..7]}"    
+                            sh "docker push fondahub/vsftpd:latest"
+                        }
+                    }
+                    post {
+                        always {
+                            archiveArtifacts "hadolint_vsftpd.json"
+                            recordIssues(
+                                aggregatingResults: true,
+                                tools: [hadoLint(pattern: "hadolint_vsftpd.json", id: "vsfptd")]
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

README.md

@@ -1,7 +1,8 @@
-# Common Workflow Scheduler for Kubernetes
-[![DOI](https://zenodo.org/badge/596122315.svg)](https://zenodo.org/badge/latestdoi/596122315)
+# Kubernetes Workflow Scheduler
 
-In this repository, you will find the Common Workflow Scheduler for Kubernetes proposed in the paper "**How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**."
+SWAGGER: http://localhost:8080/swagger-ui.html
+
+API-DOCS: http://localhost:8080/v3/api-docs/
 
 ---
 #### Build
@@ -188,6 +189,7 @@ The following strategies are available:
 | random              | Randomly prioritize tasks.                                                                                                                               |
 | max                 | Prioritize tasks with larger input size.                                                                                                                 |
 | min                 | Prioritize tasks with smaller input size.                                                                                                                |
+| wow                 | WOW scheduler for data location awareness. This is scheduling + node assignment. Details are provided in our paper [tbd](tbd).                           |
 
 | Node Assignment Strategy | Behaviour                                                                               |
 |--------------------------|-----------------------------------------------------------------------------------------|
@@ -215,4 +217,4 @@ Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Le
 ```
 ---
 #### Acknowledgement:
-This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis." 
+This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis."

daemons/ftp/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:slim
+RUN apt update && apt install -y \
+  vsftpd \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/run/vsftpd/empty
+
+COPY vsftpd.conf /etc/vsftpd.conf
+
+USER root
+RUN echo 'root:password' | chpasswd
+
+COPY ftp.py /code/ftp.py
+
+WORKDIR /code
+
+ENTRYPOINT ["sh","-c","/usr/sbin/vsftpd /etc/vsftpd.conf"]