fixes the encode test (#146)

* fixes the encode test

* fixes integration, we will need to override this in dev and terra

* better way to have integration drop all before migrating and running tests

* styling

* utility scripts

ops/db-connect.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+: ${DB:?}
+
+PW=$( vault read -format=json secret/dsde/datarepo/integration/api-secrets-integration.json | \
+      jq -r .data.datarepoPassword )
+
+kubectl --namespace data-repo run psql -it --serviceaccount=jade-sa --restart=Never --rm --image postgres:9.6 -- \
+    psql "postgresql://drmanager:${PW}@cloudsql-proxy-service.data-repo/${DB}"

ops/k8s/secrets/api-secrets.yaml.ctmpl

@@ -1,5 +1,5 @@
 {{ with $environment := env "ENVIRONMENT" }}
-{{with $suffix := env "SUFFIX"}}
+{{ with $suffix := env "SUFFIX" }}
 {{ with $project := env "GOOGLE_CLOUD_PROJECT" }}
 {{ with secret (printf "secret/dsde/datarepo/%s/api-secrets-%s.json" $environment $suffix) }}
 

ops/update-secrets.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+: ${GOOGLE_CLOUD_PROJECT:?}
+: ${VAULT_ADDR:?}
+: ${ENVIRONMENT:?}
+: ${SUFFIX:?}
+
+if [ -z "$VAULT_TOKEN" ]; then
+    if [ ! -f ~/.vault-token ]; then
+        echo "VAULT_TOKEN needs to be set or ~/.vault-token needs to exist"
+        exit 1
+    fi
+    export VAULT_TOKEN=$( cat ~/.vault-token )
+fi
+
+# the paths we'll use will be relative to this script
+WD=$( dirname "${BASH_SOURCE[0]}" )
+NOW=$(date +%Y-%m-%d_%H-%M-%S)
+DATA_REPO_TAG="${GOOGLE_CLOUD_PROJECT}_${NOW}"
+SCRATCH=/tmp/deploy-scratch
+
+# Make sure kubectl is pointing at the right project
+KUBECTL_CONTEXT=$(kubectl config current-context)
+if [[ $KUBECTL_CONTEXT != *${GOOGLE_CLOUD_PROJECT}* ]]; then
+    echo "the kubernetes context (${KUBECTL_CONTEXT}) does not match your GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT}"
+    echo "the easiest way to change it is using the context menu after clicking on the Docker icon in your top bar"
+    exit 1
+fi
+
+# make a temporary directory for rendering, we'll delete it later
+mkdir -p $SCRATCH
+
+# render secrets, create or update on kubernetes
+consul-template -template "${WD}/k8s/secrets/api-secrets.yaml.ctmpl:${SCRATCH}/api-secrets.yaml" -once
+kubectl apply -f "${SCRATCH}/api-secrets.yaml"
+
+rm -r $SCRATCH

src/main/resources/application-integration.properties

@@ -0,0 +1 @@
+db.migrate.dropAllOnStart=true

src/test/java/bio/terra/filesystem/EncodeFileTest.java

@@ -196,8 +196,10 @@ public void encodeFileBadRowTest() throws Exception {
         MockHttpServletResponse response = connectedOperations.validateJobModelAndWait(result);
 
         ErrorModel ingestError = connectedOperations.handleAsyncFailureCase(response);
+        // NB: this used to return 2 errors. It seems like the BQ API changed recently and now returns 3, except two of
+        // them are the same error with one word changed.
         assertThat("correctly found bad row",
-            ingestError.getMessage(), equalTo("Ingest failed with 2 errors - see error details"));
+            ingestError.getMessage(), equalTo("Ingest failed with 3 errors - see error details"));
 
         List<String> errorDetails = ingestError.getErrorDetail();
         assertNotNull("Error details were returned", errorDetails);
@@ -206,6 +208,10 @@ public void encodeFileBadRowTest() throws Exception {
                 "JSON table encountered too many errors, giving up. Rows: 1; errors: 1. Please look into the " +
                 "errors[] collection for more details."));
         assertThat("Big query returned in details 1", errorDetails.get(1),
+            equalTo("BigQueryError: reason=invalid message=Error while reading data, error message: " +
+                "JSON processing encountered too many errors, giving up. " +
+                "Rows: 1; errors: 1; max bad: 0; error percent: 0"));
+        assertThat("Big query returned in details 2", errorDetails.get(2),
             equalTo("BigQueryError: reason=invalid message=Error while reading data, error message: " +
                 "JSON parsing error in row starting at position 0: Parser terminated before end of string"));