Skip to content

Commit

Permalink
[PF-2977] Azure Service Bus support for stairway (#122)
Browse files Browse the repository at this point in the history
  • Loading branch information
@v-hsompalli
v-hsompalli authored Dec 6, 2023
1 parent cd80783 commit 11dd6d9
Show file tree
Hide file tree
Showing 5 changed files with 257 additions and 11 deletions.
4 changes: 3 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@ dependencies {

// Terra libraries
implementation group: 'org.broadinstitute.dsde.workbench', name: 'sam-client_2.13', version: '0.1-5281c21'
api group: 'bio.terra', name: 'stairway-gcp', version: '0.0.76-SNAPSHOT'
var stairwayVersion= '0.0.80-SNAPSHOT'
api "bio.terra:stairway-gcp:${stairwayVersion}"
implementation "bio.terra:stairway-azure:${stairwayVersion}"

// Logging
implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36'
Expand Down
54 changes: 51 additions & 3 deletions gradle.lockfile
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,37 @@
# This is a Gradle generated file for dependency locking.
# Manual edits can break the build and are not advised.
# This file is expected to be part of source control.
bio.terra:stairway-gcp:0.0.76-SNAPSHOT=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
bio.terra:stairway:0.0.76-SNAPSHOT=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
bio.terra:stairway-azure:0.0.80-SNAPSHOT=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
bio.terra:stairway-gcp:0.0.80-SNAPSHOT=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
bio.terra:stairway:0.0.80-SNAPSHOT=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback.contrib:logback-jackson:0.1.5=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback.contrib:logback-json-classic:0.1.5=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback.contrib:logback-json-core:0.1.5=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback:logback-classic:1.4.14=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback:logback-core:1.4.14=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-core-amqp:2.9.0-beta.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.azure:azure-core-http-netty:1.13.9=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.azure:azure-core:1.44.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.azure:azure-identity:1.10.4=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.azure:azure-json:1.1.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.azure:azure-messaging-servicebus:7.14.0-beta.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.beust:jcommander:1.48=pmd
com.fasterxml.jackson.core:jackson-annotations:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.core:jackson-core:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-guava:2.15.2=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.module:jackson-module-parameter-names:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson:jackson-bom:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.woodstox:woodstox-core:6.5.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.flagsmith:flagsmith-java-client:6.1.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.github.ben-manes.caffeine:caffeine:2.9.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.github.spotbugs:spotbugs-annotations:4.7.1=spotbugs
com.github.spotbugs:spotbugs-annotations:4.7.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.github.spotbugs:spotbugs:4.7.1=spotbugs
com.github.stephenc.jcip:jcip-annotations:1.0-1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.google.android:annotations:4.1.1.4=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.google.api-client:google-api-client:1.34.0=compileClasspath,testCompileClasspath
com.google.api-client:google-api-client:2.2.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -72,6 +82,13 @@ com.google.protobuf:protobuf-java-util:3.21.12=compileClasspath,productionRuntim
com.google.protobuf:protobuf-java:3.21.12=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.google.re2j:re2j:1.6=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.jayway.jsonpath:json-path:2.8.0=testCompileClasspath,testRuntimeClasspath
com.microsoft.azure:msal4j-persistence-extension:1.2.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.microsoft.azure:msal4j:1.13.9=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.microsoft.azure:qpid-proton-j-extensions:1.2.4=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.nimbusds:content-type:2.2=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.nimbusds:lang-tag:1.7=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.nimbusds:nimbus-jose-jwt:9.30.2=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.nimbusds:oauth2-oidc-sdk:10.7.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
com.opencsv:opencsv:5.6=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.puppycrawl.tools:checkstyle:10.3.1=checkstyle
com.squareup.okhttp3:logging-interceptor:4.10.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -106,6 +123,27 @@ io.kubernetes:client-java-proto:16.0.0=compileClasspath,productionRuntimeClasspa
io.kubernetes:client-java:16.0.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.micrometer:micrometer-commons:1.10.9=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.micrometer:micrometer-observation:1.10.9=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.netty:netty-buffer:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-codec-dns:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-codec-http2:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-codec-http:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-codec-socks:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-codec:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-common:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-handler-proxy:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-handler:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-resolver-dns-classes-macos:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-resolver-dns-native-macos:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-resolver-dns:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-resolver:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-tcnative-boringssl-static:2.0.62.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-tcnative-classes:2.0.62.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport-classes-epoll:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport-classes-kqueue:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport-native-epoll:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport-native-kqueue:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport-native-unix-common:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.netty:netty-transport:4.1.100.Final=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.opencensus:opencensus-api:0.31.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.opencensus:opencensus-contrib-http-util:0.31.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.opencensus:opencensus-proto:0.2.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -141,6 +179,9 @@ io.opentelemetry:opentelemetry-sdk-trace:1.31.0=compileClasspath,productionRunti
io.opentelemetry:opentelemetry-sdk:1.31.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.opentelemetry:opentelemetry-semconv:1.24.0-alpha=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.perfmark:perfmark-api:0.26.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.projectreactor.netty:reactor-netty-core:1.0.38=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.projectreactor.netty:reactor-netty-http:1.0.38=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.projectreactor:reactor-core:3.4.33=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
io.prometheus:simpleclient:0.16.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.prometheus:simpleclient_common:0.16.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
io.prometheus:simpleclient_httpserver:0.16.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
Expand All @@ -159,8 +200,12 @@ javax.xml.bind:jaxb-api:2.3.1=compileClasspath,productionRuntimeClasspath,runtim
jaxen:jaxen:1.2.0=spotbugs
net.bytebuddy:byte-buddy-agent:1.14.4=testCompileClasspath,testRuntimeClasspath
net.bytebuddy:byte-buddy:1.14.4=testCompileClasspath,testRuntimeClasspath
net.java.dev.jna:jna-platform:5.13.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
net.java.dev.jna:jna:5.13.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
net.jcip:jcip-annotations:1.0=spotbugs
net.minidev:accessors-smart:2.4.11=testCompileClasspath,testRuntimeClasspath
net.minidev:accessors-smart:2.4.9=productionRuntimeClasspath,runtimeClasspath
net.minidev:json-smart:2.4.10=productionRuntimeClasspath,runtimeClasspath
net.minidev:json-smart:2.4.11=testCompileClasspath,testRuntimeClasspath
net.sf.saxon:Saxon-HE:11.3=checkstyle,spotbugs
net.sourceforge.pmd:pmd-core:6.31.0=pmd
Expand All @@ -185,6 +230,7 @@ org.apache.logging.log4j:log4j-to-slf4j:2.20.0=compileClasspath,productionRuntim
org.apache.maven:maven-artifact:3.6.3=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.apache.oltu.oauth2:org.apache.oltu.oauth2.client:1.0.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.apache.oltu.oauth2:org.apache.oltu.oauth2.common:1.0.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.apache.qpid:proton-j:0.33.8=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.apache.tomcat.embed:tomcat-embed-core:10.1.11=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.apache.tomcat.embed:tomcat-embed-el:10.1.11=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.apache.tomcat.embed:tomcat-embed-websocket:10.1.11=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
Expand All @@ -201,6 +247,7 @@ org.checkerframework:checker-qual:3.12.0=checkstyle
org.checkerframework:checker-qual:3.32.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.codehaus.mojo:animal-sniffer-annotations:1.23=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.codehaus.plexus:plexus-utils:3.2.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.codehaus.woodstox:stax2-api:4.2.1=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.dom4j:dom4j:2.1.3=spotbugs
org.hamcrest:hamcrest:2.2=testCompileClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -241,8 +288,9 @@ org.ow2.asm:asm-tree:9.3=spotbugs
org.ow2.asm:asm-util:9.3=spotbugs
org.ow2.asm:asm:9.0-beta=pmd
org.ow2.asm:asm:9.2=jacocoAnt
org.ow2.asm:asm:9.3=spotbugs,testCompileClasspath,testRuntimeClasspath
org.ow2.asm:asm:9.3=productionRuntimeClasspath,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
org.postgresql:postgresql:42.5.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.reactivestreams:reactive-streams:1.0.4=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.reflections:reflections:0.10.2=checkstyle
org.scala-lang:scala-library:2.13.10=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath
Expand Down
33 changes: 26 additions & 7 deletions src/main/java/bio/terra/common/stairway/StairwayComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,15 @@

import bio.terra.common.kubernetes.KubeProperties;
import bio.terra.common.kubernetes.KubeService;
import bio.terra.stairway.ExceptionSerializer;
import bio.terra.stairway.QueueInterface;
import bio.terra.stairway.Stairway;
import bio.terra.stairway.StairwayBuilder;
import bio.terra.stairway.StairwayHook;
import bio.terra.stairway.*;
import bio.terra.stairway.azure.AzureServiceBusQueue;
import bio.terra.stairway.exception.StairwayException;
import bio.terra.stairway.exception.StairwayExecutionException;
import bio.terra.stairway.gcp.GcpPubSubQueue;
import bio.terra.stairway.gcp.GcpQueueUtils;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -54,7 +53,7 @@ public StairwayComponent(
* clusterNameSuffix is present, then we create the queue. It is an error to try to run in
* Kubernetes without a work queue.
*/
private GcpPubSubQueue setupWorkQueue() {
private QueueInterface setupGcpWorkQueue() {
try {
String topicId = stairwayProperties.getGcpPubSubTopicId();
String subscriptionId = stairwayProperties.getGcpPubSubSubscriptionId();
Expand Down Expand Up @@ -91,6 +90,20 @@ private GcpPubSubQueue setupWorkQueue() {
}
}

/** Set up the Stairway Azure work queue. */
@VisibleForTesting
QueueInterface setupAzureWorkQueue() {
return AzureServiceBusQueue.newBuilder()
.connectionString(stairwayProperties.getAzureServiceBusConnectionString())
.subscriptionName(stairwayProperties.getAzureServiceBusSubscriptionName())
.topicName(stairwayProperties.getAzureServiceBusTopicName())
.maxAutoLockRenewDuration(
Duration.ofMinutes(stairwayProperties.getAzureServiceBusMaxAutoLockRenewDuration()))
.namespace(stairwayProperties.getAzureServiceBusNamespace())
.useManagedIdentity(stairwayProperties.isUseManagedIdentity())
.build();
}

/** convenience for getting a builder for initialize input */
public StairwayOptionsBuilder newStairwayOptionsBuilder() {
return new StairwayOptionsBuilder();
Expand All @@ -113,7 +126,13 @@ public void initialize(DataSource dataSource, Object context, List<StairwayHook>
* @param initializeBuilder collection of Stairway initialization parameters
*/
public void initialize(StairwayOptionsBuilder initializeBuilder) {
QueueInterface queue = (kubeProperties.isInKubernetes()) ? setupWorkQueue() : null;
QueueInterface queue;
// Using Azure WorkQueue if azureQueueEnabled set to true
if (stairwayProperties.isAzureQueueEnabled()) {
queue = setupAzureWorkQueue();
} else {
queue = (kubeProperties.isInKubernetes()) ? setupGcpWorkQueue() : null;
}

logger.info("Initializing Stairway...");
final StairwayBuilder builder =
Expand Down
Loading

0 comments on commit 11dd6d9

Please sign in to comment.