fix(tests): changed the expected format of the config files

.github/workflows/run-plugin-integration-tests.yaml

@@ -2,10 +2,6 @@ name: Run Plugin Integration Tests
 
 on:
   workflow_call:
-    inputs:
-      workspace-path:
-        required: true
-        type: string
     outputs:
       success:
         description: Overall integration test result
@@ -39,8 +35,6 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Start RHDH with Layered Test Config
-        env:
-          WORKSPACE_PATH: ${{ inputs.workspace-path }}
         run: |
           set -euo pipefail
           ls -la ./artifacts/ || true

.github/workflows/workspace-tests.yaml

@@ -4,8 +4,6 @@ on:
   workflow_run:
     workflows: [Pull Request Actions]
     types: [completed]
-    paths:
-      - 'workspaces/**'
 
 jobs:
   resolve:
@@ -80,38 +78,44 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Debug resolved metadata
+        env:
+          WORKSPACE: ${{ steps.meta.outputs.workspace }}
+          PR: ${{ steps.meta.outputs.pr }}
+          OVERLAY_SHA: ${{ steps.meta.outputs.overlayCommit }}
         run: |
-          echo "Workspace: ${{ steps.meta.outputs.workspace }}"
-          echo "PR: ${{ steps.meta.outputs.pr }}, Overlay SHA: ${{ steps.meta.outputs.overlayCommit }}"
+          echo "Workspace: $WORKSPACE"
+          echo "PR: $PR, Overlay SHA: $OVERLAY_SHA"
 
   prepare-test-config:
     needs: resolve
     if: ${{ needs.resolve.outputs.workspace != '' }}
     runs-on: ubuntu-latest
     outputs:
       plugins_metadata_complete: ${{ steps.build-dynamic-plugins.outputs.plugins_metadata_complete }}
+      skip_tests_missing_env: ${{ steps.build-dynamic-plugins.outputs.skip_tests_missing_env }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.resolve.outputs.overlayBranch }}
           repository: ${{ needs.resolve.outputs.overlayRepo }}
 
-      - name: Setup yq
-        uses: mikefarah/yq@v4
-
       - name: Build dynamic-plugins.test.yaml
         id: build-dynamic-plugins
+        env:
+          WORKSPACE_PATH: ${{ needs.resolve.outputs.workspace }}
+          PUBLISHED_EXPORTS: ${{ needs.resolve.outputs.publishedExports }}
         run: |
-          WORKSPACE_PATH="${{ needs.resolve.outputs.workspace }}"
-          PUBLISHED_EXPORTS="${{ needs.resolve.outputs.publishedExports }}"
           PLUGINS_FOUND=0
+          PLUGINS_SKIPPED_MISSING_ENV=0
           TOTAL_PLUGINS=0
           PLUGINS_METADATA_COMPLETE="false"
+          SKIP_TESTS_MISSING_ENV="false"
 
           if [ -z "$PUBLISHED_EXPORTS" ]; then
             echo "No published exports provided."
             echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT"
+            echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT"
             exit 0
           fi
 
@@ -128,8 +132,8 @@ jobs:
           declare -A META_MAP
           for file in "$WORKSPACE_PATH"/metadata/*.yaml; do
             [ -e "$file" ] || continue
-            pkg=$(yq -r '.packageName // ""' "$file")
-            if [ -n "$pkg" ]; then
+            pkg=$(yq -r '.spec.packageName // ""' "$file")
+            if [ -n "$pkg" ] && [ "$pkg" != "null" ]; then
               stripped=$(echo "$pkg" | sed 's|^@||; s|/|-|')
               META_MAP["$stripped"]="$file"
             fi
@@ -138,6 +142,16 @@ jobs:
           # Always include the root-level default config
           ROOT_CONFIG="$GITHUB_WORKSPACE/tests/app-config.yaml"
           [ -f "$ROOT_CONFIG" ] && cp "$ROOT_CONFIG" "$OUT_DIR/app-config.yaml"
+
+          # Read workspace-wide test.env if it exists
+          WORKSPACE_ENV_FILE="$WORKSPACE_PATH/tests/test.env"
+          WORKSPACE_ENV_CONTENT=""
+          if [ -f "$WORKSPACE_ENV_FILE" ]; then
+            echo "Found workspace test.env file: $WORKSPACE_ENV_FILE"
+            WORKSPACE_ENV_CONTENT=$(cat "$WORKSPACE_ENV_FILE")
+          else
+            echo "No workspace test.env file found at: $WORKSPACE_ENV_FILE"
+          fi
 
           # Start the resulting YAML file
           echo "plugins:" > "$OUT_FILE"
@@ -155,24 +169,52 @@ jobs:
                 continue
               fi
 
-              PACKAGE_NAME=$(yq -r '.packageName' "$METADATA_FILE")
+              PACKAGE_NAME=$(yq -r '.spec.packageName' "$METADATA_FILE")
               if [ -z "$PACKAGE_NAME" ] || [ "$PACKAGE_NAME" = "null" ]; then
-                echo "packageName not found in $METADATA_FILE, skipping"
+                echo "spec.packageName not found in $METADATA_FILE, skipping"
                 continue
               fi
 
-              # Extract plugin config block under configs.default
-              if ! yq -e '.configs.default' "$METADATA_FILE" >/dev/null 2>&1; then
-                echo "configs.default not found in $METADATA_FILE, skipping"
+              # First appConfigExamples item is used for testing
+              CONFIG_CONTENT=$(yq -o=yaml '.spec.appConfigExamples[0].content' "$METADATA_FILE" 2>/dev/null || echo "")
+              if [ -z "$CONFIG_CONTENT" ] || [ "$CONFIG_CONTENT" = "null" ]; then
+                echo "spec.appConfigExamples[0].content not found in $METADATA_FILE, skipping"
                 continue
               fi
 
+              ENV_VARS=$(echo "$CONFIG_CONTENT" | yq -o=yaml '.dynamicPlugins' 2>/dev/null | grep -oE '\$\{[A-Z_][A-Z0-9_]*\}|\$[A-Z_][A-Z0-9_]*' | sed 's/\${//; s/}//; s/^\$//' | sort -u || true)
+
+              if [ -n "$ENV_VARS" ]; then
+                # Config contains environment variables
+                if [ -z "$WORKSPACE_ENV_CONTENT" ]; then
+                  echo "  ::warning::Config for $PLUGIN_NAME contains environment variables but workspace test.env is missing. Tests will be skipped."
+                  SKIP_TESTS_MISSING_ENV="true"
+                  PLUGINS_SKIPPED_MISSING_ENV=$((PLUGINS_SKIPPED_MISSING_ENV + 1))
+                  continue
+                else
+                  # Validate all ENVs are present in merged env file
+                  MISSING_ENVS=()
+                  while IFS= read -r env_var; do
+                    [ -z "$env_var" ] && continue
+                    if ! echo "$WORKSPACE_ENV_CONTENT" | grep -qE "^[[:space:]]*${env_var}[[:space:]]*="; then
+                      MISSING_ENVS+=("$env_var")
+                    fi
+                  done <<< "$ENV_VARS"
+
+                  if [ ${#MISSING_ENVS[@]} -gt 0 ]; then
+                    echo "  ::error::Environment variables missing from test.env: ${MISSING_ENVS[*]}"
+                    echo "  Config for $PLUGIN_NAME references these environment variables but they are not defined in $WORKSPACE_ENV_FILE."
+                    exit 1
+                  fi
+                fi
+              fi
+              # If no ENVs in config, continue regardless of env file existence
+
               STRIPPED=$(echo "$PACKAGE_NAME" | sed 's|^@||; s|/|-|')
               echo "- package: \"oci://ghcr.io/${IMAGE_PATH_AND_PLUGIN}:${NEW_TAG}!${STRIPPED}\"" >> "$OUT_FILE"
               echo "  disabled: false" >> "$OUT_FILE"
               echo "  pluginConfig:" >> "$OUT_FILE"
-              # Append the configs.default block indented by 4 spaces under pluginConfig
-              yq -o=yaml '.configs.default' "$METADATA_FILE" | sed 's/^/    /' >> "$OUT_FILE"
+              echo "$CONFIG_CONTENT" | sed 's/^/    /' >> "$OUT_FILE"
               PLUGINS_FOUND=$((PLUGINS_FOUND + 1))
             else
               echo "Export did not match expected format, skipping: $export"
@@ -183,11 +225,17 @@ jobs:
             echo "[]" >> "$OUT_FILE"
           fi
 
-          if [ "$PLUGINS_FOUND" -eq "$TOTAL_PLUGINS" ] && [ "$TOTAL_PLUGINS" -gt 0 ]; then
+          # Check if all plugins were found (including those skipped due to missing env)
+          TOTAL_PROCESSED=$((PLUGINS_FOUND + PLUGINS_SKIPPED_MISSING_ENV))
+          echo "Plugins: $PLUGINS_FOUND/$TOTAL_PLUGINS processed successfully"
+          [ "${PLUGINS_SKIPPED_MISSING_ENV:-0}" -gt 0 ] && echo "Skipped $PLUGINS_SKIPPED_MISSING_ENV (missing test.env)"
+
+          if [ "$TOTAL_PROCESSED" -eq "$TOTAL_PLUGINS" ] && [ "$TOTAL_PLUGINS" -gt 0 ]; then
             PLUGINS_METADATA_COMPLETE="true"
           fi
 
           echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT"
+          echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT"
 
       - name: Upload integration-test artefact
         uses: actions/upload-artifact@v4
@@ -200,13 +248,11 @@ jobs:
     needs:
       - resolve
       - prepare-test-config
-    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' }}
+    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }}
     uses: ./.github/workflows/run-plugin-integration-tests.yaml
-    with:
-      workspace-path: ${{ needs.resolve.outputs.workspace }}
 
   add-skipped-test-comment:
-    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'false' && needs.resolve.outputs.prNumber != 'null' }}
+    if: ${{ (needs.prepare-test-config.outputs.plugins_metadata_complete == 'false' || needs.prepare-test-config.outputs.skip_tests_missing_env == 'true') && needs.resolve.outputs.prNumber != 'null' }}
     needs:
       - resolve
       - prepare-test-config
@@ -217,15 +263,24 @@ jobs:
     steps:
       - name: Post skipped-test comment
         uses: actions/github-script@v7
+        env:
+          INPUT_SKIP_MISSING_ENV: ${{ needs.prepare-test-config.outputs.skip_tests_missing_env }}
+          INPUT_PR_NUMBER: ${{ needs.resolve.outputs.prNumber }}
         with:
           script: |
-            const pr = Number('${{ needs.resolve.outputs.prNumber }}');
+            const pr = Number(core.getInput('pr_number') || '0');
             if (!pr) {
               console.log('No PR associated; skipping');
               return;
             }
             const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-            const body = `:warning: \n[Test workflow](${runUrl}) skipped because some plugin metadata files (\`<workspace>/metadata/*.yaml\`) were not found.\n\nAdd the corresponding metadata files to enable testing for this workspace.\n`;
+            const skipMissingEnv = core.getInput('skip_missing_env') === 'true';
+            let body;
+            if (skipMissingEnv) {
+              body = `:warning: \n[Test workflow](${runUrl}) skipped because some plugin configurations contain environment variables but the workspace \`tests/test.env\` file is missing.\n\nAdd the required \`test.env\` file to enable testing for this workspace.\n`;
+            } else {
+              body = `:warning: \n[Test workflow](${runUrl}) skipped because some plugin metadata files (\`<workspace>/metadata/*.yaml\`) were not found.\n\nAdd the corresponding metadata files to enable testing for this workspace.\n`;
+            }
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
@@ -234,7 +289,7 @@ jobs:
             });
 
   add-test-result-comment:
-    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' }}
+    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }}
     needs:
       - resolve
       - prepare-test-config
@@ -260,7 +315,7 @@ jobs:
           script: |
             const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             const success = process.env.SUCCESS === 'true';
-            const failed = process.env.FAILED_PLUGINS.trim();
+            const failed = (process.env.FAILED_PLUGINS || '').trim();
             const pr = Number(process.env.PR_NUMBER);
             const overlayCommit = process.env.OVERLAY_COMMIT;
 

README.md

@@ -133,10 +133,15 @@ The repository includes an automated integration testing workflow that verifies
 
 **Testing workflow steps:**
 1. **Resolve metadata**: Retrieves published OCI references and PR metadata from the `published-exports` artifact
-2. **Prepare test config**: Generates `dynamic-plugins.test.yaml` from plugin metadata and copies other configuration files (`tests/app-config.yaml` and workspace-specific `app-config.test.yaml`, `test.env`)
+2. **Prepare test config**: Generates `dynamic-plugins.test.yaml` from plugin metadata and copies other configuration files (`tests/app-config.yaml`, workspace-specific `app-config.test.yaml` and `test.env`)
 3. **Run integration tests**: Starts RHDH container with layered configuration, installs dynamic plugins from OCI artifacts, and verifies each plugin loads successfully
 4. **Report results**: Posts test status as a commit status check and PR comment with pass/fail results and links to the workflow run
 
+**Environment Variables in Tests:**
+If your plugin configuration (in `metadata/*.yaml`) uses environment variables (e.g., `${API_TOKEN}`), you must provide them in a `test.env` file located at `workspaces/<workspace>/tests/test.env`.
+- If the `.env` file is missing but required, tests are skipped.
+- If the `.env` file exists but is missing variables, the workflow fails.
+
 - **Results** are reported via PR comment and in the status check. The complete container logs are also available, in the `integration-tests/run` step.
 
 #### Manual Testing