From ed612d803701b07f963fbcdad6505a883807faa2 Mon Sep 17 00:00:00 2001 From: techfg Date: Fri, 7 Feb 2025 07:38:07 -0800 Subject: [PATCH] chore: refactor test scripts (#4577) * chore: refactor test scripts * chore: format * fix: UESIO_DEV needed for nx graph due to cypress.config.ts * fix: debug for nx failing to build graph * chore: add more diag info and explicitly ignore go-build-cache directory for nx * chore: finalize nx graph fix * chore: don't leak -e for env setup * chore: refactor build steps for docker tests * chore: ignore go-build-cache in .gitignore instead --- .github/workflows/ci.yaml | 11 ++-- .gitignore | 7 +++ README.md | 54 +++++++++++----- apps/platform-e2e/cypress.config.ts | 1 + .../cypress/e2e/wire_lookup_conditions.cy.ts | 2 +- ...ace.hurl => create_app_and_workspace.hurl} | 53 ---------------- .../hurl_seeds/delete_app.hurl | 62 +++++++++++++++++++ apps/platform/ssl/create.sh | 25 ++++++-- package.json | 8 ++- scripts/cleanup-tests.sh | 8 +++ scripts/init-tests.sh | 7 +++ scripts/run-all-tests.sh | 12 ++-- .../{run-tests-ci.sh => run-docker-tests.sh} | 15 +++-- scripts/run-e2e-tests.sh | 13 ++-- scripts/run-integration-tests.sh | 41 +++--------- scripts/tests-cleanup.sh | 37 ----------- scripts/tests/cleanup-common.sh | 46 ++++++++++++++ .../cleanup-docker.sh} | 0 .../{tests-init.sh => tests/init-common.sh} | 14 +++-- scripts/tests/setup-common.sh | 6 ++ .../{tests-setup.sh => tests/setup-docker.sh} | 23 ++----- scripts/tests/setup-env.sh | 9 +++ scripts/tests/start-e2e-tests.sh | 12 ++++ scripts/tests/start-integration-tests.sh | 37 +++++++++++ 24 files changed, 310 insertions(+), 193 deletions(-) rename apps/platform-integration-tests/hurl_seeds/{app_and_workspace.hurl => create_app_and_workspace.hurl} (81%) create mode 100644 apps/platform-integration-tests/hurl_seeds/delete_app.hurl create mode 100644 scripts/cleanup-tests.sh create mode 100644 scripts/init-tests.sh rename scripts/{run-tests-ci.sh => run-docker-tests.sh} (68%) delete mode 100644 scripts/tests-cleanup.sh create mode 100644 scripts/tests/cleanup-common.sh rename scripts/{tests-down.sh => tests/cleanup-docker.sh} (100%) rename scripts/{tests-init.sh => tests/init-common.sh} (70%) create mode 100644 scripts/tests/setup-common.sh rename scripts/{tests-setup.sh => tests/setup-docker.sh} (56%) create mode 100644 scripts/tests/setup-env.sh create mode 100644 scripts/tests/start-e2e-tests.sh create mode 100644 scripts/tests/start-integration-tests.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c9af224ca..e0d3b37bbf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,11 +81,6 @@ jobs: env: UESIO_DEV: "true" # required to ensure packui gets built run: | - # We lint/test/build affected but in order to build image, we need to ensure everything - # is built. Build anything that hasn't been built yet (takes advantage of nx cache - # for anything already built from above) - npx nx run-many -t build - # build values for docker image tags VERSION_SUFFIX=${{ github.run_number }}.${{ github.run_attempt }} FULL_SHA=${{ steps.setNxSHAs.outputs.head }} @@ -155,9 +150,13 @@ jobs: run: | ./scripts/seed-etc-hosts.sh + cd apps/platform/ssl + bash ./create.sh + cd ../../../ + # Start up the Uesio app, and dependencies, in Docker # then run all Integration and E2E tests against the app - npm run tests-ci + npm run tests-docker check: name: Check format and lint diff --git a/.gitignore b/.gitignore index bc3011ce24..865e2bce13 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,10 @@ __debug_bin queries/results .nx/cache .nx/workspace-data + +# CI workflows cache go which creates a go-build-cache directory that contains packages that contain package.json files +# so whenever nx is run, it "thinks" they are "projects" in the nx workspace and then fails to parse them (e.g., they +# do not have a "name" property). The below could be included in a .nxignore file since the problem is nx specific, +# however putting here instead because we would also never want to commit these files if we ever committed in +# our workflows. +/go-build-cache diff --git a/README.md b/README.md index e312d90c53..8497bc5775 100644 --- a/README.md +++ b/README.md @@ -447,38 +447,54 @@ docker compose up -d bash apps/platform/migrations_test/test_migrations.sh ``` -## End-to-end Testing and Integration testing +## Testing (Unit, Integration & E2E) -To run E2E and Integration tests locally, there are a number of commands available: +> ![IMPORTANT] +> The default behavior for all tests is to run against `https://studio.uesio-dev.com:3000` so you must ensure that [SSL](#set-up-ssl) and [local DNS](#set-up-your-local-dns) have been configured. -1. `npm run tests-all` - - Runs all Integration and E2E tests against your local Uesio app. +To run the various test suites, there are a number of commands available: + +1. `npm run tests` + - Runs all Unit tests +2. `npm run tests-all` + - Runs all Unit, Integration and E2E tests against your local Uesio app. - Use this when writing and debugging tests locally -1. `npm run tests-ci` - - This is what we run in Github Actions [CI](./.github/workflows/ci.yaml) builds. It spins up all dependencies, and a Dockerized version of the Uesio app, runs integration and E2E tests against the app, and then spins down all Docker containers. -1. `npm run tests-integration` +3. `npm run tests-docker` + - This is what we run in Github Actions [CI](./.github/workflows/ci.yaml) builds for integration & E2E tests. It builds the latest code, spins up all dependencies, and a Dockerized version of the Uesio app, runs integration and E2E tests against the app, and then spins down all Docker containers. +4. `npm run tests-integration` - Runs _just_ the Integration Tests (against your local app). -1. `npm run tests-e2e` +5. `npm run tests-e2e` - Runs _just_ the E2E Tests (against your local app). +6. `npm run tests-init` + - Deletes the `uesio/tests` app if it exists and then creates the `uesio/tests` app with related workspaces and sites and loads seed data. All of the above scripts execute this script automatically so there is not typically a need to run it. However, if you want to run individual tests (via hurl, cypress, etc.) separate from one of the above scripts which runs an entire suite, you will need to run this script to prepare for test execution. +7. `npm run tests-cleanup` + - Removes the `uesio/tests` app (if it exists). Similar to `tests-init`, the automated test suite scripts will execute this script prior to completion. However, if a test run terminates abnormally and/or if you ran `tests-init` manually, you can execute this script to remove the test related `uesio/tests` app. +8. `npm run tests-cypress-open` + - Runs the cypress visual UI where you can run E2E tests from. See [E2E testing with cypress](#e2e-testing-with-cypress) for details. +9. `npm run tests-cypress-run` + - Runs the cypress in headless mode, helpful when you want to run individual tests. See [E2E testing with cypress](#e2e-testing-with-cypress) for details. TO run just an individual E2E or Integration test, see the sections below. +> [!NOTE] +> You must manually run `npm run tests-init` in order to run individual tests. Depending on the test, you may need to re-run this script prior to every test execution. Additionally, ensure that `UESIO_DEV=true` environment variable is set prior to starting the server and for each test so that mock logins can be used. + ### E2E testing with Cypress -We use [Cypress](https://cypress.io) for writing end-to-end tests of the Uesio app. All E2E tests are defined in `cypress/e2e` directory. +We use [Cypress](https://cypress.io) for writing end-to-end tests of the Uesio app. All E2E tests are defined in `apps/platform-e2e` directory. E2E tests are the most expensive and most brittle, and as such should be used sparingly. -If you're running Uesio locally, you can use `npx cypress open` to launch Cypress' visual UI for running tests, or `npm run tests-e2e` to just run the tests in a headless runner. +If you're running Uesio locally, you can use `npm run tests-cypress-open` to launch Cypress' visual UI for running tests, or `npm run tests-e2e` to just run all the tests in a headless runner. Note that when running using the visual UI or when running individual tests as per the below, you must have the `UESIO_DEV=true` environment variable set and have run `npm run tests-init`. #### Running a single E2E spec If you want to _visually_ run a single spec, use the Cypress visual UI and then select the individual spec. -Or, use `npx cypress run --spec ` to run a specific file in a headless Electron instance, e.g. +Or, use `npm run tests-cypress-run -- --spec ` to run a specific file in a headless Electron instance, e.g. -``` -npx cypress run --spec cypress/e2e/builder.cy.ts +```bash +npx run tests-cypress-run -- --spec apps/platform-e2e/cypress/e2e/builder.cy.ts ``` ### Integration / API testing with Hurl @@ -489,13 +505,19 @@ To run API integration tests locally against your running Uesio container, use ` #### Running a single Integration Test -The easiest way to run a single Integration Test is to go into the `run-integration-tests.sh` file and comment out the lines where we run all tests, and uncomment the lines here: +The easiest way to run a single Integration Test is to go into the `scripts/tests/start-integration-tests.sh` file and comment out the lines where we run all tests, and uncomment the lines here and then run `npm run tests-integration`: ``` -npx hurl --very-verbose -k --variable host=studio.uesio-dev.com --variable domain=uesio-dev.com --variable port=3000 hurl_specs/wire_collection_dependencies.hurl +# npx hurl --very-verbose -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT apps/platform-integration-tests/hurl_specs/wire_collection_dependencies.hurl ``` -You could run this from the CLI, but you would have to make sure that you are (a) in the right directory (b) have the right environment variables set up. (See the top of this `run-integration-tests.sh` for a better understanding). +You could run the individual test from the CLI, but you would have to make sure that you have the test app created and the right environment variables set up. If you would like to run via the CLI: + +```bash +npm run tests-init # initialize test app/workspace/site/data/etc. +source scripts/tests/setup-env.sh # setup environment variables used in tests +npx hurl --very-verbose -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT apps/platform-integration-tests/hurl_specs/wire_collection_dependencies.hurl +``` # Continous integration (CI) diff --git a/apps/platform-e2e/cypress.config.ts b/apps/platform-e2e/cypress.config.ts index 4f9f5dd352..c52ef4e77b 100644 --- a/apps/platform-e2e/cypress.config.ts +++ b/apps/platform-e2e/cypress.config.ts @@ -7,6 +7,7 @@ const automationUsername = process.env.UESIO_AUTOMATION_USERNAME || "uesio" const automationPassword = process.env.UESIO_AUTOMATION_PASSWORD const useMockLogin = !automationPassword const inCi = process.env.CI === "true" + export default defineConfig({ e2e: { ...nxE2EPreset(__filename, { diff --git a/apps/platform-e2e/cypress/e2e/wire_lookup_conditions.cy.ts b/apps/platform-e2e/cypress/e2e/wire_lookup_conditions.cy.ts index 39247fff93..15958eb32d 100644 --- a/apps/platform-e2e/cypress/e2e/wire_lookup_conditions.cy.ts +++ b/apps/platform-e2e/cypress/e2e/wire_lookup_conditions.cy.ts @@ -25,7 +25,7 @@ describe("Wire Lookup Conditions", () => { ) cy.get("table[id$='contactsTable']>tbody>tr", { timeout: 4000, - }).should("have.length", 2) + }).should("have.length", 3) // Now initiate a manual requery by searching cy.getByIdFragment("input", "contactsSearch").type("Fred") diff --git a/apps/platform-integration-tests/hurl_seeds/app_and_workspace.hurl b/apps/platform-integration-tests/hurl_seeds/create_app_and_workspace.hurl similarity index 81% rename from apps/platform-integration-tests/hurl_seeds/app_and_workspace.hurl rename to apps/platform-integration-tests/hurl_seeds/create_app_and_workspace.hurl index aadfe39dc9..c347b16227 100644 --- a/apps/platform-integration-tests/hurl_seeds/app_and_workspace.hurl +++ b/apps/platform-integration-tests/hurl_seeds/create_app_and_workspace.hurl @@ -13,59 +13,6 @@ user_id: jsonpath "$.user.id" POST https://{{host}}:{{port}}/site/perf/stats/reset HTTP 204 -# Delete the test site if it already exists -DELETE https://{{host}}:{{port}}/site/api/v1/collection/uesio/studio/site?uesio/studio.name=eq.testsite -x-uesio-app: uesio/tests -HTTP 204 - -# Delete the tests app if it already exists, which should cascade delete the workspace and all of its metadata -DELETE https://{{host}}:{{port}}/site/api/v1/collection/uesio/studio/app?uesio/studio.fullname=eq.uesio%2Ftests -HTTP 204 - -# Verify that workspaces are cascade deleted -POST https://{{host}}:{{port}}/site/wires/load -{ - "wires": [ - { - "name": "workspaces", - "collection": "uesio/studio.workspace", - "conditions": [ - { - "field": "uesio/studio.app->uesio/studio.fullname", - "operator": "EQ", - "value": "uesio/tests" - } - ] - } - ] -} -HTTP 200 -[Asserts] -jsonpath "$.wires[0].errors" not exists -jsonpath "$.wires[0].data" count == 0 - -# Verify that sites are cascade deleted -POST https://{{host}}:{{port}}/site/wires/load -{ - "wires": [ - { - "name": "sites", - "collection": "uesio/studio.site", - "conditions": [ - { - "field": "uesio/studio.app->uesio/studio.fullname", - "operator": "EQ", - "value": "uesio/tests" - } - ] - } - ] -} -HTTP 200 -[Asserts] -jsonpath "$.wires[0].errors" not exists -jsonpath "$.wires[0].data" count == 0 - # Create the tests app POST https://{{host}}:{{port}}/site/wires/save { diff --git a/apps/platform-integration-tests/hurl_seeds/delete_app.hurl b/apps/platform-integration-tests/hurl_seeds/delete_app.hurl new file mode 100644 index 0000000000..dc3f03928c --- /dev/null +++ b/apps/platform-integration-tests/hurl_seeds/delete_app.hurl @@ -0,0 +1,62 @@ +POST https://{{host}}:{{port}}/site/auth/uesio/core/mock/login +{ + "token": "uesio" +} +# Extract the sessionid from set-cookie header +HTTP 200 +[Captures] +session_id: cookie "sessid" +user_id: jsonpath "$.user.id" + +# Delete the test site if it already exists +DELETE https://{{host}}:{{port}}/site/api/v1/collection/uesio/studio/site?uesio/studio.name=eq.testsite +x-uesio-app: uesio/tests +HTTP 204 + +# Delete the tests app if it already exists, which should cascade delete the workspace and all of its metadata +DELETE https://{{host}}:{{port}}/site/api/v1/collection/uesio/studio/app?uesio/studio.fullname=eq.uesio%2Ftests +HTTP 204 + +# Verify that workspaces are cascade deleted +POST https://{{host}}:{{port}}/site/wires/load +{ + "wires": [ + { + "name": "workspaces", + "collection": "uesio/studio.workspace", + "conditions": [ + { + "field": "uesio/studio.app->uesio/studio.fullname", + "operator": "EQ", + "value": "uesio/tests" + } + ] + } + ] +} +HTTP 200 +[Asserts] +jsonpath "$.wires[0].errors" not exists +jsonpath "$.wires[0].data" count == 0 + +# Verify that sites are cascade deleted +POST https://{{host}}:{{port}}/site/wires/load +{ + "wires": [ + { + "name": "sites", + "collection": "uesio/studio.site", + "conditions": [ + { + "field": "uesio/studio.app->uesio/studio.fullname", + "operator": "EQ", + "value": "uesio/tests" + } + ] + } + ] +} +HTTP 200 +[Asserts] +jsonpath "$.wires[0].errors" not exists +jsonpath "$.wires[0].data" count == 0 diff --git a/apps/platform/ssl/create.sh b/apps/platform/ssl/create.sh index 587d526cb5..bf05cfcd9e 100755 --- a/apps/platform/ssl/create.sh +++ b/apps/platform/ssl/create.sh @@ -1,9 +1,24 @@ #!/bin/bash -# Generate a new primary key -openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -keyout private.key -days 3560 -out certificate.crt -config config.txt +CERT_FILE="$PWD/certificate.crt" -# OS X shortcut to automatically trust the certificate -if [[ "$(uname)" == "Darwin" ]]; then - sudo security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "$PWD/certificate.crt" +if [ -f "$CERT_FILE" ]; then + echo "SSL certificate already exists. Delete the existing certificate and re-run to generate a new SSL certificate." +else + # Generate a new primary key + openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -keyout private.key -days 3560 -out "$CERT_FILE" -config config.txt + echo "SSL Certificate and private key created!" + + # OS X shortcut to automatically trust the certificate + if [[ "$(uname)" == "Darwin" ]]; then + sudo security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "$CERT_FILE" + else + # Update CA certificates so that CLI and Curl will not complain when connecting + # to our local Uesio instance with self-signed certificate + # TODO: update-ca-certificates should be fairly portable across distros but may need to handle differently in some cases + sudo cp "$CERT_FILE" /usr/local/share/ca-certificates/ + sudo update-ca-certificates + fi + echo "System certificate trust store updated to trust SSL certificate!" fi + diff --git a/package.json b/package.json index c1896c9d8e..cbf219cd5c 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,14 @@ "get-module-version": "node -p \"require('./package').devDependencies['$npm_config_modulename']\" | sed 's/^\\^//g'", "graceful-shutdown": "lsof -t -i :3000 | xargs kill -15", "kill:webserver": "lsof -t -i :3000 | xargs kill -9", - "tests-cleanup": "bash scripts/tests-cleanup.sh", + "tests-cleanup": "bash scripts/cleanup-tests.sh", "tests-e2e": "bash scripts/run-e2e-tests.sh", - "tests-init": "bash scripts/tests-init.sh", + "tests-init": "bash scripts/init-tests.sh", "tests-integration": "bash scripts/run-integration-tests.sh", "tests-all": "bash scripts/run-all-tests.sh", - "tests-ci": "bash scripts/run-tests-ci.sh", + "tests-docker": "bash scripts/run-docker-tests.sh", + "tests-cypress-open": "cypress open --project apps/platform-e2e", + "tests-cypress-run": "cypress run --project apps/platform-e2e", "format": "nx format:write", "format:check": "nx format:check", "format:write": "nx format:write", diff --git a/scripts/cleanup-tests.sh b/scripts/cleanup-tests.sh new file mode 100644 index 0000000000..b00a927137 --- /dev/null +++ b/scripts/cleanup-tests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Initial setup +source ./scripts/tests/setup-common.sh + +# cleans up the tests app and workspaces, +# and verifies that cleanup worked as expected +bash "scripts/tests/cleanup-common.sh" diff --git a/scripts/init-tests.sh b/scripts/init-tests.sh new file mode 100644 index 0000000000..7c12d9c9e7 --- /dev/null +++ b/scripts/init-tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Initial setup +source ./scripts/tests/setup-common.sh + +# Initialize the sample app and seed test data +bash "scripts/tests/init-common.sh" diff --git a/scripts/run-all-tests.sh b/scripts/run-all-tests.sh index e8f97d591f..bcac594a6a 100644 --- a/scripts/run-all-tests.sh +++ b/scripts/run-all-tests.sh @@ -8,15 +8,19 @@ set -e +# Initial setup +source ./scripts/tests/setup-common.sh + # Runs unit tests npm run test # Runs Hurl integration tests against the app -bash ./scripts/run-integration-tests.sh +bash ./scripts/tests/start-integration-tests.sh # Runs Cypress End-to-End tests against the app -bash ./scripts/run-e2e-tests.sh +bash ./scripts/tests/start-e2e-tests.sh -# Cleans up the tests app and workspaces -bash ./scripts/tests-cleanup.sh +# cleans up the tests app and workspaces, +# and verifies that cleanup worked as expected +bash ./scripts/tests/cleanup-common.sh diff --git a/scripts/run-tests-ci.sh b/scripts/run-docker-tests.sh similarity index 68% rename from scripts/run-tests-ci.sh rename to scripts/run-docker-tests.sh index 7f1396a06a..64f3ef53f5 100644 --- a/scripts/run-tests-ci.sh +++ b/scripts/run-docker-tests.sh @@ -11,19 +11,22 @@ set -e +# Initial setup +source ./scripts/tests/setup-common.sh + # Spins up dependencies and runs the app in Docker -bash ./scripts/tests-setup.sh +bash ./scripts/tests/setup-docker.sh # Runs Hurl integration tests against the app -bash ./scripts/run-integration-tests.sh +bash ./scripts/tests/start-integration-tests.sh # Runs Cypress End-to-End tests against the app -bash ./scripts/run-e2e-tests.sh +bash ./scripts/tests/start-e2e-tests.sh # cleans up the tests app and workspaces, -# and verifies that these routines work as expected -bash ./scripts/tests-cleanup.sh +# and verifies that cleanup worked as expected +bash ./scripts/tests/cleanup-common.sh # Spins down all docker containers -bash ./scripts/tests-down.sh +bash ./scripts/tests/cleanup-docker.sh diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index 20f9f82595..82fee5f523 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -2,7 +2,12 @@ set -e -# Run e2e tests with cypress -export UESIO_APP_URL="https://studio.uesio-dev.com:3000" -export UESIO_DEV=true -npx cypress run --project apps/platform-e2e +# Initial setup +source ./scripts/tests/setup-common.sh + +# Runs Cypress End-to-End tests against the app +bash ./scripts/tests/start-e2e-tests.sh + +# cleans up the tests app and workspaces, +# and verifies that cleanup worked as expected +bash ./scripts/tests/cleanup-common.sh diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index 7c983424ae..af0fc970bc 100644 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -2,39 +2,12 @@ set -e -export UESIO_CLI_LOGIN_METHOD=uesio/core.mock -export UESIO_CLI_USERNAME=uesio -export UESIO_CLI_HOST="https://studio.uesio-dev.com:3000" -# this enables {{unix_epoch_seconds}} variable to be used within tests to add some uniqueness to seed values -export HURL_unix_epoch_seconds=$(date +%s) -export HURL_num_common_fields=8 -export HURL_num_public_core_collections=14 -export HURL_num_tests_collections=11 -export HURL_num_aikit_collections=2 -export HURL_num_appkit_collections=1 -# this number should be the sum of the above two variables -export HURL_num_core_and_tests_collections=$(($HURL_num_public_core_collections + $HURL_num_tests_collections + $HURL_num_aikit_collections + $HURL_num_appkit_collections)) +# Initial setup +source ./scripts/tests/setup-common.sh -# Initialize and the sample app and seed test data -bash "scripts/tests-init.sh" +# Runs Cypress End-to-End tests against the app +bash ./scripts/tests/start-integration-tests.sh -echo "Running tests..." - -cd apps/platform-integration-tests - -# TODO: Hurl 5.0 added parallel test execution which is enabled by default. Currently, hurl tests fail when run in parallel. For now, forcing -# sequential execution via --jobs flag but this should be investigated and, assuming possible depending on root cause, changed to not limit parallel execution. -# See https://github.com/ues-io/uesio/issues/4457 -# Run specs -npx hurl --jobs 1 --error-format long -k --variable host=studio.uesio-dev.com --variable domain=uesio-dev.com --variable port=3000 --test hurl_specs/*.hurl -# Run field condition tests -npx hurl --jobs 1 --error-format long -k --variable host=studio.uesio-dev.com --variable domain=uesio-dev.com --variable port=3000 --test hurl_fields/*.hurl - -# FYI if you want to view the output of the request made by a specific hurl spec, -# you can comment out the assertions of the last hurl request made in a hurl file, and then run the spec -# without the "--test" flag, like this -# npx hurl --very-verbose -k --variable host=studio.uesio-dev.com --variable domain=uesio-dev.com --variable port=3000 hurl_specs/wire_collection_dependencies.hurl - -npx hurl --very-verbose -k --variable host=studio.uesio-dev.com --variable domain=uesio-dev.com --variable port=3000 --test hurl_specs_single_run/perf_stats.hurl - -cd - >> /dev/null +# cleans up the tests app and workspaces, +# and verifies that cleanup worked as expected +bash ./scripts/tests/cleanup-common.sh diff --git a/scripts/tests-cleanup.sh b/scripts/tests-cleanup.sh deleted file mode 100644 index d6d8b6e33c..0000000000 --- a/scripts/tests-cleanup.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -e -shopt -s expand_aliases - -export UESIO_CLI_LOGIN_METHOD=uesio/core.mock -export UESIO_CLI_USERNAME=uesio -export UESIO_CLI_HOST="https://studio.uesio-dev.com:3000" - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -alias uesio="$SCRIPT_DIR/../dist/cli/uesio" - -#Navigate -cd apps/platform-integration-tests - -echo "Logging in to Studio as uesio user..." -uesio logout -uesio sethost -uesio login - -# Truncate dev workspace -echo "Truncate dev workspace." -uesio work -n dev -uesio workspace truncate -echo "dev workspace should be clear" - -#Navigate back -cd - >> /dev/null - -# Run specs -npx hurl -k --variable host=studio.uesio-dev.com --variable port=3000 --test apps/platform-integration-tests/hurl_specs_single_run/truncate_tenant_data_cli.hurl - -# Delete the workspaces -cd apps/platform-integration-tests -uesio workspace delete -n truncatetests -uesio workspace delete -n dev -cd - >> /dev/null diff --git a/scripts/tests/cleanup-common.sh b/scripts/tests/cleanup-common.sh new file mode 100644 index 0000000000..dc7ba9149b --- /dev/null +++ b/scripts/tests/cleanup-common.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e +shopt -s expand_aliases + +export UESIO_CLI_LOGIN_METHOD=uesio/core.mock +export UESIO_CLI_USERNAME=uesio +export UESIO_CLI_HOST=$UESIO_TEST_APP_URL + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +alias uesio="$SCRIPT_DIR/../../dist/cli/uesio" + +#Navigate +cd apps/platform-integration-tests + +echo "Logging in to Studio as uesio user..." +uesio logout +uesio sethost +uesio login + +# Truncate dev workspace +echo "Truncate dev workspace." +uesio work -n dev +# TODO: This may fail because the workspace does not exist which is ultimately what we are after in this script. There +# is no built-in method to determine if a workspace exists currently (see https://github.com/ues-io/uesio/issues/4573) +# and we can't assume a failure is because it does not exist (it may not be running or could be for another reason). +# This needs to be improved to check if the workspace exists first and if it doesn't, then just return else continue. +uesio workspace truncate +echo "dev workspace should be clear" + +#Navigate back +cd - >> /dev/null + +# Run specs +npx hurl -k --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --test apps/platform-integration-tests/hurl_specs_single_run/truncate_tenant_data_cli.hurl + +# Delete the workspaces +cd apps/platform-integration-tests +uesio workspace delete -n truncatetests +uesio workspace delete -n dev +uesio workspace delete -n quickstart +cd - >> /dev/null + +# Remove anything that remains +echo "Deleting tests app..." +npx hurl -k --error-format long --no-output --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --variable domain=$UESIO_TEST_DOMAIN apps/platform-integration-tests/hurl_seeds/delete_app.hurl diff --git a/scripts/tests-down.sh b/scripts/tests/cleanup-docker.sh similarity index 100% rename from scripts/tests-down.sh rename to scripts/tests/cleanup-docker.sh diff --git a/scripts/tests-init.sh b/scripts/tests/init-common.sh similarity index 70% rename from scripts/tests-init.sh rename to scripts/tests/init-common.sh index 3a4e682a5f..c90d41d3ef 100755 --- a/scripts/tests-init.sh +++ b/scripts/tests/init-common.sh @@ -5,10 +5,10 @@ shopt -s expand_aliases export UESIO_CLI_LOGIN_METHOD=uesio/core.mock export UESIO_CLI_USERNAME=uesio -export UESIO_CLI_HOST="https://studio.uesio-dev.com:3000" +export UESIO_CLI_HOST=$UESIO_TEST_APP_URL SCRIPT_DIR=$(dirname "$(realpath "$0")") -alias uesio="$SCRIPT_DIR/../dist/cli/uesio" +alias uesio="$SCRIPT_DIR/../../dist/cli/uesio" # Deploy the sample app using Uesio cd apps/platform-integration-tests @@ -18,8 +18,10 @@ uesio logout uesio sethost uesio login -echo "Deleting and recreating the tests app and dev workspace..." -npx hurl -k --error-format long --no-output --variable host=studio.uesio-dev.com --variable port=3000 --variable domain=uesio-dev.com hurl_seeds/app_and_workspace.hurl +echo "Deleting tests app if it exists..." +npx hurl -k --error-format long --no-output --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --variable domain=$UESIO_TEST_DOMAIN hurl_seeds/delete_app.hurl +echo "Creating the tests app and dev workspace..." +npx hurl -k --error-format long --no-output --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --variable domain=$UESIO_TEST_DOMAIN hurl_seeds/create_app_and_workspace.hurl # truncatetests workspace echo "Changing to truncatetests workspace..." @@ -43,12 +45,12 @@ uesio upsert -f seed_data/contacts.csv -s seed_data/contacts_import.spec.json uesio upsert -f seed_data/tools.csv -s seed_data/tools_import.spec.json # Populate secrets and config values for the dev workspace -npx hurl -k --error-format long --no-output --variable host=studio.uesio-dev.com --variable port=3000 --variable domain=uesio-dev.com hurl_seeds/populate_secrets_and_config_values.hurl +npx hurl -k --error-format long --no-output --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --variable domain=$UESIO_TEST_DOMAIN hurl_seeds/populate_secrets_and_config_values.hurl echo "Successfully upserted seed data into our workspace. Creating a test site, domain, and bundle..." # Now that we have deployed our metadata to the workspace, we can create a bundle, site, and domain which uses its metadata -npx hurl -k --error-format long --no-output --variable host=studio.uesio-dev.com --variable port=3000 --variable domain=uesio-dev.com hurl_seeds/site_domain_bundle.hurl +npx hurl -k --error-format long --no-output --variable host=$UESIO_TEST_HOST_NAME --variable port=$UESIO_TEST_PORT --variable domain=$UESIO_TEST_DOMAIN hurl_seeds/site_domain_bundle.hurl echo "Seeding data into our test site..." uesio siteadmin -n=testsite diff --git a/scripts/tests/setup-common.sh b/scripts/tests/setup-common.sh new file mode 100644 index 0000000000..001e61a346 --- /dev/null +++ b/scripts/tests/setup-common.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +# Common environment variables +source scripts/tests/setup-env.sh diff --git a/scripts/tests-setup.sh b/scripts/tests/setup-docker.sh similarity index 56% rename from scripts/tests-setup.sh rename to scripts/tests/setup-docker.sh index b8ab1e2d43..7a97924c09 100644 --- a/scripts/tests-setup.sh +++ b/scripts/tests/setup-docker.sh @@ -2,25 +2,14 @@ set -e -# Ensure that ssl directory is non-empty, otherwise seed SSL -CERT_FILE="./apps/platform/ssl/certificate.crt" -if [ -f "$CERT_FILE" ]; then - echo "SSL certificate already exists." -else - echo "SSL certificate does not exist, creating..." - cd apps/platform/ssl - bash ./create.sh - echo "SSL Certificate and private key created!" - cd ../../../ - # Update CA certificates so that CLI and Curl will not complain when connecting - # to our local Uesio instance with self-signed certificate - sudo cp apps/platform/ssl/certificate.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates -fi +# Ensure everything is built with latest code +# UESIO_DEV set to ensure packui is built +UESIO_DEV=true npm run build-all # Ensure that we have a Uesio docker image to run # In CI, we should have the image built already but locally we want to re-build on every run if [[ -z "${APP_IMAGE}" ]]; then + echo "Building docker image..." # use a specific tag to avoid each built image remaining in docker IMAGE_TAG="uesio-test:latest" # force a (re)build to ensure we use current filesystem (e.g., uncommitted changes) @@ -30,14 +19,12 @@ if [[ -z "${APP_IMAGE}" ]]; then export APP_IMAGE="$IMAGE_TAG" fi -export APP_HOST="https://studio.uesio-dev.com:3000" - # Spin up dependencies and the app, and run migrations againt the DB docker compose -f docker-compose-tests.yaml down --volumes docker compose -f docker-compose-tests.yaml up -d echo "Waiting for Uesio app to start..." # curl the app's /health route in a loop and sleep 1 second until we get a 200 -until $(curl --output /dev/null --silent --fail $APP_HOST/health); do +until $(curl --output /dev/null --silent --fail $UESIO_TEST_APP_URL/health); do printf '.' sleep 1 done diff --git a/scripts/tests/setup-env.sh b/scripts/tests/setup-env.sh new file mode 100644 index 0000000000..914587d8fb --- /dev/null +++ b/scripts/tests/setup-env.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Intentionaly not including set -e to avoid leaking it since this script might be called directly by users via source command + +export UESIO_TEST_DOMAIN=uesio-dev.com +export UESIO_TEST_HOST_NAME=studio.$UESIO_TEST_DOMAIN +export UESIO_TEST_PORT=3000 +export UESIO_TEST_PROTOCOL=https +export UESIO_TEST_APP_URL="$UESIO_TEST_PROTOCOL://$UESIO_TEST_HOST_NAME:$UESIO_TEST_PORT" diff --git a/scripts/tests/start-e2e-tests.sh b/scripts/tests/start-e2e-tests.sh new file mode 100644 index 0000000000..fd9b9a831c --- /dev/null +++ b/scripts/tests/start-e2e-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +export UESIO_APP_URL=$UESIO_TEST_APP_URL +export UESIO_DEV=true + +# Initialize the sample app and seed test data +bash "scripts/tests/init-common.sh" + +# Run e2e tests with cypress +npx cypress run --project apps/platform-e2e diff --git a/scripts/tests/start-integration-tests.sh b/scripts/tests/start-integration-tests.sh new file mode 100644 index 0000000000..0e456565a7 --- /dev/null +++ b/scripts/tests/start-integration-tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e + +# this enables {{unix_epoch_seconds}} variable to be used within tests to add some uniqueness to seed values +export HURL_unix_epoch_seconds=$(date +%s) +export HURL_num_common_fields=8 +export HURL_num_public_core_collections=14 +export HURL_num_tests_collections=11 +export HURL_num_aikit_collections=2 +export HURL_num_appkit_collections=1 +# this number should be the sum of the above two variables +export HURL_num_core_and_tests_collections=$(($HURL_num_public_core_collections + $HURL_num_tests_collections + $HURL_num_aikit_collections + $HURL_num_appkit_collections)) + +# Initialize the sample app and seed test data +bash "scripts/tests/init-common.sh" + +echo "Running Integration tests..." + +cd apps/platform-integration-tests + +# TODO: Hurl 5.0 added parallel test execution which is enabled by default. Currently, hurl tests fail when run in parallel. For now, forcing +# sequential execution via --jobs flag but this should be investigated and, assuming possible depending on root cause, changed to not limit parallel execution. +# See https://github.com/ues-io/uesio/issues/4457 +# Run specs +npx hurl --jobs 1 --error-format long -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT --test hurl_specs/*.hurl +# Run field condition tests +npx hurl --jobs 1 --error-format long -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT --test hurl_fields/*.hurl + +# FYI if you want to view the output of the request made by a specific hurl spec, +# you can comment out the assertions of the last hurl request made in a hurl file, and then run the spec +# without the "--test" flag, like this +# npx hurl --very-verbose -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT hurl_specs/wire_collection_dependencies.hurl + +npx hurl --very-verbose -k --variable host=$UESIO_TEST_HOST_NAME --variable domain=$UESIO_TEST_DOMAIN --variable port=$UESIO_TEST_PORT --test hurl_specs_single_run/perf_stats.hurl + +cd - >> /dev/null