Add support for ARTIFACT_DIR in CI tests (opendatahub-io#215)

* Add support for ARTIFACT_DIR in CI tests

The env var ARTIFACS_DIR is used in prow tests to determine where CI
tests artifacts will be retrieved at the end of the run.

Signed-off-by: Landon LaSmith <[email protected]>

* FIXUP: Move end of test output out of error check

ODH pod and operator logs are now preserved with on success or failure
when SKIP_PODS_OUTPUT is not set

* FIXUP: Remove SKIP_PODS_OUTPUT

Signed-off-by: Landon LaSmith <[email protected]>

.gitignore

@@ -1,4 +1,5 @@
 kfdef/.cache/
 kfdef/kustomize/
 tests/kubeconfig
+tests/artifacts
 .idea

tests/Makefile

@@ -5,11 +5,13 @@ ODHPROJECT=opendatahub
 OPENSHIFT_USER=
 OPENSHIFT_PASS=
 OPENSHIFT_LOGIN_PROVIDER=
-SKIP_PODS_OUTPUT=
 # Setting SKIP_INSTALL will let you run the tests against an ODH instance that is already setup
 SKIP_INSTALL=
 # Setting TESTS_REGEX will allow you to change which tests are going to be run
 TESTS_REGEX=
+# Location inside the container where CI system will retrieve files after a test run
+ARTIFACT_DIR=/tmp/artifacts
+LOCAL_ARTIFACT_DIR="${PWD}/artifacts"
 
 all: test
 test: build run clean
@@ -18,11 +20,12 @@ build:
 	podman build -t $(IMAGE) --build-arg ORG=$(GIT_ORG) --build-arg BRANCH=$(GIT_BRANCH) .
 
 run:
-	mkdir -p `pwd`/screenshots/
+	# Confirm that we have a directory for storing any screenshots from selenium tests
+	mkdir -p ${LOCAL_ARTIFACT_DIR}/screenshots
 	oc config view --flatten --minify > /tmp/tests-kubeconfig
-	podman run -e SKIP_PODS_OUTPUT=$(SKIP_PODS_OUTPUT) -e SKIP_INSTALL=$(SKIP_INSTALL) -e TESTS_REGEX=$(TESTS_REGEX) -e ODHPROJECT=$(ODHPROJECT) \
-		-e OPENSHIFT_USER="$(OPENSHIFT_USER)" -e OPENSHIFT_PASS="$(OPENSHIFT_PASS)" -e OPENSHIFT_LOGIN_PROVIDER=$(OPENSHIFT_LOGIN_PROVIDER) \
-		-it -v `pwd`/screenshots/:/tmp/screenshots:z -v /tmp/tests-kubeconfig:/tmp/kubeconfig:z $(IMAGE)
+	podman run -e SKIP_INSTALL=$(SKIP_INSTALL) -e TESTS_REGEX=$(TESTS_REGEX) -e ODHPROJECT=$(ODHPROJECT) \
+		-e OPENSHIFT_USER="$(OPENSHIFT_USER)" -e OPENSHIFT_PASS="$(OPENSHIFT_PASS)" -e OPENSHIFT_LOGIN_PROVIDER=$(OPENSHIFT_LOGIN_PROVIDER) -e ARTIFACT_DIR=$(ARTIFACT_DIR) \
+		-it -v ${LOCAL_ARTIFACT_DIR}/:$(ARTIFACT_DIR):z -v /tmp/tests-kubeconfig:/tmp/kubeconfig:z $(IMAGE)
 
 clean:
 	oc delete -n $(ODHPROJECT) kfdef opendatahub || true

tests/README.md

@@ -40,6 +40,11 @@ only run the test that you specify instead of all of the tests.  example: `make
 For other possible configurations, you can look in the Makefile.  There are a set of
 variables at the top that you could change to meet the needs of your particular test run.
 
+## Test Artifacts
+The environment variable `ARTIFACT_DIR` specifies the root directory where all test artifacts should be
+stored for retrieval at the end of a test run. Any files created should be uniquely named to prevent
+a test from overwriting an artifact generated by another test.
+
 # Running tests manually
 
 Manual running of the tests relies on the test

tests/basictests/jupyterhub/jhtest.py

@@ -20,7 +20,7 @@
 
 logging.basicConfig(level=logging.INFO)
 _LOGGER = logging.getLogger()
-_SCREENSHOT_DIR="/tmp/screenshots"
+_SCREENSHOT_DIR=f"{os.environ.get('ARTIFACT_SCREENSHOT_DIR', '/tmp')}"
 
 class JHStress():
     def __init__(self):

tests/scripts/install.sh

@@ -26,7 +26,7 @@ if [ -z "$PULL_NUMBER" ]; then
 else
   curl -O -L https://github.com/${REPO_OWNER}/${REPO_NAME}/pull/${PULL_NUMBER}.patch
   echo "Applying followng patch:"
-  cat ${PULL_NUMBER}.patch
+  cat ${PULL_NUMBER}.patch > ${ARTIFACT_DIR}/github-pr-${PULL_NUMBER}.patch
   git apply ${PULL_NUMBER}.patch
 fi
 popd
@@ -61,7 +61,7 @@ if [ -z "${OPENSHIFT_USER}" ] || [ -z "${OPENSHIFT_PASS}" ]; then
 fi
 
 echo "Creating the following KfDef"
-cat ./kfctl_openshift.yaml
+cat ./kfctl_openshift.yaml > ${ARTIFACT_DIR}/kfctl_openshift.yaml
 oc apply -f ./kfctl_openshift.yaml
 kfctl_result=$?
 if [ "$kfctl_result" -ne 0 ]; then

tests/scripts/installandtest.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
 
 set -x
-env | sort
+env | sort >  ${ARTIFACT_DIR}/env.txt
 mkdir -p ~/.kube
 cp /tmp/kubeconfig ~/.kube/config 2> /dev/null || cp /var/run/secrets/ci.openshift.io/multi-stage/kubeconfig ~/.kube/config
 chmod 644 ~/.kube/config
 export KUBECONFIG=~/.kube/config
+export ARTIFACT_SCREENSHOT_DIR="${ARTIFACT_DIR}/screenshots"
+
+if [ ! -d "${ARTIFACT_SCREENSHOTS_DIR}" ]; then
+  echo "Creating the screenshot artifact directory: ${ARTIFACT_SCREENSHOT_DIR}"
+  mkdir -p ${ARTIFACT_SCREENSHOT_DIR}
+fi
 
 TESTS_REGEX=${TESTS_REGEX:-"basictests"}
 ODHPROJECT=${ODHPROJECT:-"opendatahub"}
@@ -23,14 +29,13 @@ if [ -z "${SKIP_INSTALL}" ]; then
 fi
 $HOME/peak/run.sh ${TESTS_REGEX}
 
+echo "Saving the dump of the pods logs in the artifacts directory"
+oc get pods -o yaml -n ${ODHPROJECT} > ${ARTIFACT_DIR}/${ODHPROJECT}.pods.yaml
+echo "Saving the logs from the opendatahub-operator pod in the artifacts directory"
+oc logs -n openshift-operators $(oc get pods -n openshift-operators -l name=opendatahub-operator -o jsonpath="{$.items[*].metadata.name}") > ${ARTIFACT_DIR}/opendatahub-operator.log
+
 if  [ "$?" -ne 0 ]; then
     echo "The tests failed"
-    if [ -z "${SKIP_PODS_OUTPUT}" ]; then
-        echo "Here's a dump of the pods:"
-        oc get pods -o json -n ${ODHPROJECT}
-        echo "Logs from the opendatahub-operator pod"
-        oc logs -n openshift-operators $(oc get pods -n openshift-operators -l name=opendatahub-operator -o jsonpath="{$.items[*].metadata.name}")
-    fi
     exit 1
 fi