[DC-1269] Remove references to integration (#1825)

* Remove references to integration

* Fix spotless and remove terraform doc section

* remove unused autowired properties?

* Fix linting

* Add back unused dependencies

.github/dependabot.yml

@@ -3,13 +3,6 @@ updates:
   # Enable version updates for Gradle
   - package-ecosystem: "gradle"
     directory: "/"
-    # See workflows/int-and-connected-test-run.yml for the current list of k8s namespaces available
-    # for integration testing.
-    # Presently, there are only 4, and these tests often take 2 hours to run.
-    # Keeping the open PR limit low at 4 so that concurrent Dependabot PR integration test runs all
-    # stand a chance at success rather than some being likely to time out waiting for an available
-    # namespace.
-    open-pull-requests-limit: 4
     groups:
       # We group minor and patch updates together because they are less likely to break things.
       # Major updates will be PR-ed individually: they are more likely to need developer

.github/workflows/helmtagbumper.yaml

@@ -3,57 +3,6 @@ on:
   workflow_dispatch: {}
   workflow_call: {}
 jobs:
-# new integration image updater
-  integration_helm_tag_update:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: develop
-      - name: 'Get Previous tag'
-        id: apiprevioustag
-        run: |
-          TAG=$(git for-each-ref --sort=-creatordate --count 1 --format="%(refname:short)" "refs/tags/")
-          echo "tag=${TAG}" >> $GITHUB_OUTPUT
-      - name: 'Checkout datarepo-helm-definitions repo'
-        uses: actions/checkout@v3
-        with:
-          repository: 'broadinstitute/datarepo-helm-definitions'
-          token: ${{ secrets.BROADBOT_TOKEN}}
-          path: datarepo-helm-definitions
-      - name: "integration-1 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-1/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "integration-2 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-2/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "integration-3 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-3/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "integration-4 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-4/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "integration-5 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-5/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "integration-6 find and replace"
-        uses: docker://mikefarah/yq:3.3.4
-        with:
-          args: yq w -i datarepo-helm-definitions/integration/integration-6/datarepo-api.yaml image.tag ${{ steps.apiprevioustag.outputs.tag }}"
-      - name: "[datarepo-helm-definitions] Merge chart version update"
-        uses: broadinstitute/datarepo-actions/actions/[email protected]
-        env:
-          COMMIT_MESSAGE: "Datarepo api tag version update: ${{ steps.apiprevioustag.outputs.tag }}"
-          GITHUB_REPO: datarepo-helm-definitions
-          SWITCH_DIRECTORIES: true
-          MERGE_BRANCH: master
   datarepo_helm_chart_update:
     runs-on: ubuntu-20.04
     steps:

README.md

@@ -157,7 +157,7 @@ skaffold run
       googleAllowreuseexistingbuckets:
     ```
     - Release a new version of the chart. Talk to DevOps to do this.
-3. To override properties for specific environments (e.g. integration), modify the
+3. To override properties for specific environments (e.g. dev), modify the
 [environment-specific override Data Repo charts](https://github.com/broadinstitute/datarepo-helm-definitions).
     - Find the [deployment.yaml](https://github.com/broadinstitute/datarepo-helm-definitions/blob/master/integration/integration-1/integration-1Deployment.yaml)
     for the specific environment.
@@ -166,7 +166,6 @@ skaffold run
     googleAllowreuseexistingbuckets: true
     ```
    - It's a good idea to test out changes on your developer-namespace before making a PR.
-   - Changes to integration, temp, or developer-namespace environments are good with regular PR approval (1 thumb for this repository).
    - Changes to dev or prod need more eyes, and perhaps a group discussion to discuss possible side effects or failure modes.
 
 ## Developer Notes

build.gradle

@@ -92,16 +92,9 @@ if (hasProperty('buildScan')) {
         uploadInBackground = false
         publishAlways()
         def project = ""; def cluster = ""; def nschunk = ""
-        if (System.getenv("NAMESPACEINUSE")) {
-            project = "broad-jade-integration"
-            cluster = "integration-master"
-            def ns = System.getenv("NAMESPACEINUSE")
-            nschunk = "%0Aresource.labels.namespace_name%3D%22${ns}%22%0Alabels.k8s-pod%2Fcomponent%3D%22${ns}-jade-datarepo-api%22"
-        } else {
-            project = System.getenv("GOOGLE_CLOUD_PROJECT")
-            cluster = "jade-master-us-central1"
-            nschunk = ""
-        }
+        project = System.getenv("GOOGLE_CLOUD_PROJECT")
+        cluster = "jade-master-us-central1"
+        nschunk = ""
         def startTest = new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
         buildFinished { ->
             def endTest = new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))

datarepo-clienttests/src/main/java/common/commands/PrintHelp.java

@@ -40,7 +40,7 @@ public static void printHelp() throws IOException {
     System.out.println(
         ANSI_PURPLE
             + "Usage (2):\n"
-            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"mmdev.json\"\n"
+            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"localhost.json\"\n"
             + "./gradlew lockAndRunTest --args=\"configOrSuiteFileName outputDirectoryName\""
             + ANSI_RESET
             + System.lineSeparator()
@@ -51,7 +51,7 @@ public static void printHelp() throws IOException {
             + System.lineSeparator()
             + ANSI_PURPLE
             + "Can use manual lock/unlock gradle commands to reset the locks:\n"
-            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"mmdev.json\"\n"
+            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"localhost.json\"\n"
             + "./gradlew lockNamespace\n"
             + "./gradlew unlockNamespace"
             + ANSI_RESET
@@ -115,7 +115,7 @@ public static void printHelp() throws IOException {
             + "Example Workflow (7): Execute a test configuration while locking the server, collect the measurements generated by the server during the run, and upload the results"
             + ANSI_RESET
             + System.lineSeparator()
-            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"mmdev.json\"\n"
+            + "export TEST_RUNNER_SERVER_SPECIFICATION_FILE=\"localhost.json\"\n"
             + System.lineSeparator()
             + "  ./gradlew lockAndRunTest --args=\"configs/basicexamples/BasicUnauthenticated.json /tmp/TestRunnerResults\""
             + System.lineSeparator()
@@ -128,7 +128,7 @@ public static void printHelp() throws IOException {
             + "Example Workflow (8): Collect the measurements generated by the server for a particular time interval"
             + ANSI_RESET
             + System.lineSeparator()
-            + "  ./gradlew collectMeasurements --args=\"AllMeasurements.json /tmp/TestRunnerResults mmdev.json '2020-08-20 13:18:34' '2020-08-20 13:18:35.615628881'\""
+            + "  ./gradlew collectMeasurements --args=\"AllMeasurements.json /tmp/TestRunnerResults localhost.json '2020-08-20 13:18:34' '2020-08-20 13:18:35.615628881'\""
             + System.lineSeparator());
     System.out.println(
         ANSI_PURPLE

datarepo-clienttests/src/main/resources/configs/basicexamples/BasicAuthenticated.json

@@ -1,7 +1,7 @@
 {
   "name": "BasicAuthenticated",
   "description": "Two users fetch the available profiles repeatedly, concurrently. Authentication required.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {},
   "application": {},

datarepo-clienttests/src/main/resources/configs/basicexamples/BasicUnauthenticated.json

@@ -1,7 +1,7 @@
 {
   "name": "BasicUnauthenticated",
   "description": "Two users check the service status repeatedly, concurrently. No authentication required.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {},
   "application": {},

datarepo-clienttests/src/main/resources/configs/basicexamples/DRSLookup.json

@@ -1,7 +1,7 @@
 {
   "name": "DRSLookup",
   "description": "Run DRS lookups on the file refs in a snapshot.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 2

datarepo-clienttests/src/main/resources/configs/basicexamples/LoadSingleFile.json

@@ -1,7 +1,7 @@
 {
   "name": "LoadSingleFile",
   "description": "One user ingests several files of two different sizes (1KB, 10KB) concurrently, into two datasets.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 3

datarepo-clienttests/src/main/resources/configs/basicexamples/LookupDataset.json

@@ -1,7 +1,7 @@
 {
   "name": "LookupDataset",
   "description": "Two users retrieve a dataset repeatedly, concurrently.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 2

datarepo-clienttests/src/main/resources/configs/basicexamples/LookupSnapshot.json

@@ -1,7 +1,7 @@
 {
   "name": "LookupSnapshot",
   "description": "Run lookups on two different snapshots.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 2

datarepo-clienttests/src/main/resources/configs/basicexamples/SoftDeleteDataset.json

@@ -1,7 +1,7 @@
 {
   "name": "SoftDeleteDataset",
   "description": "Load data into a dataset and then try to soft delete the same row 3 times.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 3

datarepo-clienttests/src/main/resources/configs/functional/AzureCreateSnapshot.json

@@ -1,7 +1,7 @@
 {
   "name": "AzureCreateSnapshot",
   "description": "Create a snapshot with a lot of files",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "tenantId": "efc08443-0082-4d6c-8931-c5794c156abd",
   "subscriptionId": "71d52ec1-5886-480a-9d6e-ed98cbf1f69f",

datarepo-clienttests/src/main/resources/configs/functional/BasicAuthenticatedWithSAMTimeout.json

@@ -1,7 +1,7 @@
 {
   "name": "BasicAuthenticatedWithSAMTimeout",
   "description": "Two users fetch the available profiles repeatedly, concurrently. SAM timeout fault enabled.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {},
   "application": {},

datarepo-clienttests/src/main/resources/configs/functional/BillingProfileAccess.json

@@ -1,7 +1,7 @@
 {
   "name": "BillingProfileAccess",
   "description": "Test roles and operations against billing profiles",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/functional/BillingProfileInUse.json

@@ -1,7 +1,7 @@
 {
   "name": "BillingProfileInUseTest",
   "description": "Test attempts to delete a bill profile when it is in use",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/functional/BillingProfileOwnerHandoff.json

@@ -1,7 +1,7 @@
 {
   "name": "BillingProfileOwnerHandoffTest",
   "description": "Test of change of ownership with access tests",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/functional/DatasetCustodianPermissions.json

@@ -1,7 +1,7 @@
 {
   "name": "DatasetCustodianPermissions",
   "description": "Check custodian permissions on a dataset.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "isFunctional": true,
   "kubernetes": {},

datarepo-clienttests/src/main/resources/configs/functional/DeleteInitialPods.json

@@ -1,7 +1,7 @@
 {
   "name": "DeleteInitialPods",
   "description": "Two bulk file loads on two 2 different datasets, while incrementally deleting all initially created pods",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 3

datarepo-clienttests/src/main/resources/configs/functional/PodDeleteDuringBulkLoad.json

@@ -1,7 +1,7 @@
 {
   "name": "PodDeleteDuringBulkLoad",
   "description": "One user ingests many files into one dataset while we delete a random pod",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "isFunctional": true,
   "kubernetes": {

datarepo-clienttests/src/main/resources/configs/functional/ScalePodsToZero.json

@@ -1,7 +1,7 @@
 {
   "name": "ScalePods",
   "description": "One user ingests many files into one dataset while we scale the Kubernetes pods down to 0 and up to 3.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "isFunctional": true,
   "kubernetes": {

datarepo-clienttests/src/main/resources/configs/functional/ScalePodsUpDown.json

@@ -1,7 +1,7 @@
 {
   "name": "ScalePods",
   "description": "One user ingests many files into one dataset while we scale the Kubernetes pods down to 1 and up to 4.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "isFunctional": true,
   "kubernetes": {

datarepo-clienttests/src/main/resources/configs/functional/SnapshotScaleCreate.json

@@ -1,7 +1,7 @@
 {
   "name": "SnapshotScaleCreate",
   "description": "Create and delete 1+ snapshots based on the same source dataset.",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 2

datarepo-clienttests/src/main/resources/configs/functional/UserWorkflow.json

@@ -1,7 +1,7 @@
 {
   "name": "UserWorkflow",
   "description": "Use billing profile to create dataset, ingest data and create snapshot",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "application": {},
   "testScripts": [

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles1000_100.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles1000_100",
   "description": "One user ingests small files into the simple dataset and then builds a snapshot",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles1000_1000.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles1000_1000",
   "description": "One user ingests small files into the simple dataset and then builds snapshots",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles1000_200.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles1000_200",
   "description": "One user ingests small files into the simple dataset and then builds snapshots",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles1000_50.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles1000_50",
   "description": "One user ingests small files into the simple dataset and then builds a snapshot",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles1000_500.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles1000_500",
   "description": "One user ingests small files into the simple dataset and then builds snapshots",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles5000_1000.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles5000_1000",
   "description": "One user ingests small files into the simple dataset and then builds snapshots",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BuildSnapshotWithFiles5000_2000.json

@@ -1,7 +1,7 @@
 {
   "name": "BuildSnapshotWithFiles5000_2000",
   "description": "One user ingests small files into the simple dataset and then builds snapshots",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BulkLoadScale200_100.json

@@ -1,7 +1,7 @@
 {
   "name": "BulkLoadScale200_100",
   "description": "One user ingests many files into one dataset",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1

datarepo-clienttests/src/main/resources/configs/profiling/BulkLoadScale200_20.json

@@ -1,7 +1,7 @@
 {
   "name": "BulkLoadScale200_20",
   "description": "One user ingests many files into one dataset",
-  "serverSpecificationFile": "integration-6.json",
+  "serverSpecificationFile": "localhost.json",
   "billingAccount": "00708C-45D19D-27AAFA",
   "kubernetes": {
     "numberOfInitialPods" : 1