From f56647615b7aabd0bb3939f8a9631d6c62526370 Mon Sep 17 00:00:00 2001 From: Jordi Massaguer Pla Date: Mon, 8 Jul 2024 17:36:45 +0200 Subject: [PATCH] GH Recommended tests plus some renaming of GH tests workflows --- .github/scripts/get_tests.py | 12 +++ .github/scripts/test_covering_pr.js | 2 +- .github/workflows/acceptance_tests_common.yml | 84 +++++++++++-------- .../workflows/acceptance_tests_feedback.yml | 7 +- .../workflows/acceptance_tests_secondary.yml | 14 +++- .../acceptance_tests_secondary_parallel.yml | 14 +++- .github/workflows/recommended_tests.yml | 48 +++++++++++ .github/workflows/test-covering-pr.yml | 37 -------- testsuite/Rakefile | 23 +++++ .../podman_runner/18_run_secondary_tests.sh | 3 +- .../19_generate_recommended_tests_yml.sh | 4 + .../podman_runner/19_run_recommended_tests.sh | 4 + 12 files changed, 166 insertions(+), 86 deletions(-) create mode 100644 .github/scripts/get_tests.py create mode 100644 .github/workflows/recommended_tests.yml delete mode 100644 .github/workflows/test-covering-pr.yml create mode 100755 testsuite/podman_runner/19_generate_recommended_tests_yml.sh create mode 100755 testsuite/podman_runner/19_run_recommended_tests.sh diff --git a/.github/scripts/get_tests.py b/.github/scripts/get_tests.py new file mode 100644 index 000000000000..624c9ef12c9b --- /dev/null +++ b/.github/scripts/get_tests.py @@ -0,0 +1,12 @@ +import re +import sys +import yaml + +if len(sys.argv) != 2: + print("Expecting the XML string as argument") + exit(-1) + +xml_string = sys.argv[1] +pattern = re.compile(r'
  • ([\w\-]+)
    • ') +matches = pattern.findall(xml_string) +print(yaml.dump(matches)) diff --git a/.github/scripts/test_covering_pr.js b/.github/scripts/test_covering_pr.js index 0240e52230ab..74a0093a058d 100644 --- a/.github/scripts/test_covering_pr.js +++ b/.github/scripts/test_covering_pr.js @@ -24,7 +24,7 @@ const main = async () => { }); } - console.log('
    Suggested tests to cover this Pull Request
    ', Array.from(tests).join('
  • ')); + console.log('
    Suggested tests to cover this Pull Request
    • %s
    ', Array.from(tests).join('
  • ')); process.exit(0); } main(); diff --git a/.github/workflows/acceptance_tests_common.yml b/.github/workflows/acceptance_tests_common.yml index a75e1d4638e5..426c2924e495 100644 --- a/.github/workflows/acceptance_tests_common.yml +++ b/.github/workflows/acceptance_tests_common.yml @@ -8,6 +8,10 @@ on: server_id: required: true type: string + recommended_tests: + required: false + type: string + env: UYUNI_PROJECT: uyuni-project UYUNI_VERSION: master @@ -16,7 +20,7 @@ env: AUTH_REGISTRY: "auth_registry" AUTH_REGISTRY_CREDENTIALS: "cucutest|cucutest" jobs: - paths-filter-1: + filter-paths: runs-on: ubuntu-latest outputs: require_acceptance_tests: ${{ steps.filter.outputs.java == 'true' || steps.filter.outputs.web == 'true' || steps.filter.outputs.testsuite == 'true' }} @@ -25,7 +29,6 @@ jobs: - uses: dorny/paths-filter@v3 id: filter with: - predicate-quantifier: 'every' filters: | java: - 'java/**' @@ -34,101 +37,108 @@ jobs: testsuite: - 'testsuite/**' - '!testsuite/features/build_validation/**' - test-uyuni: + tests: runs-on: ubuntu-22.04 - needs: paths-filter-1 - if: ${{ needs.paths-filter-1.outputs.require_acceptance_tests == 'true' }} + needs: filter-paths + if: ${{ needs.filter-paths.outputs.require_acceptance_tests == 'true' }} steps: - - name: fix podman + - name: Install Podman run: sudo apt install podman=3.4.4+ds1-1ubuntu1 --allow-downgrades - - name: welcome_message + - name: Install Python dependencies + run: pip install pyyaml + - name: Welcome message run: echo "Running acceptance tests. More info at https://github.com/uyuni-project/uyuni/wiki/Running-Acceptance-Tests-at-PR" - uses: actions/checkout@v4 - - name: Cache-jar-files + - name: Cache jar files uses: actions/cache@v4 with: path: java/buildconf/ivy/repository/ key: ${{ runner.os }}-build-cache-uyuni-jars-${{ hashFiles('**/java/buildconf/ivy/*.xml') }} restore-keys: | ${{ runner.os }}-build-cache-uyuni-jars- - - - name: Cache-obs-to-maven files + - name: Cache obs-to-maven files uses: actions/cache@v4 with: path: java/.obs-to-maven-cache key: ${{ runner.os }}-build-cache-uyuni-obs-to-maven-${{ hashFiles('**/java/buildconf/ivy/obs-maven-config.yaml') }} restore-keys: | ${{ runner.os }}-build-cache-uyuni-obs-to-maven- - - name: Cache-nodejs + - name: Cache NodeJS modules uses: actions/cache@v4 with: path: web/html/src/node_modules key: ${{ runner.os }}-build-cache-uyuni-nodejs-${{ hashFiles('**/web/html/src/yarn.lock') }} restore-keys: | ${{ runner.os }}-build-cache-uyuni-nodejs- - - name: create_tmp + - name: Create temporary directories run: ./testsuite/podman_runner/01_setup_tmp_dirs.sh - - name: create-podman-network + - name: Create Podman network run: ./testsuite/podman_runner/02_setup_network.sh - - name: start_controller_and_registry_and_buildhost + - name: Start controller, registry and build host run: ./testsuite/podman_runner/03_run_controller_and_registry_and_buildhost.sh - - name: create_ssh_conf + - name: Create SSH configuration in controller run: ./testsuite/podman_runner/04_setup_ssh_controller.sh - - name: install_gems_in_controller + - name: Install gems in controller run: ./testsuite/podman_runner/05_install_gems_in_controller.sh - - name: collect_and_tag_flaky_tests_in_controller + - name: Parse recommended tests coming from the input as XML + if: ${{ inputs.recommended_tests != '' }} + run: python .github/scripts/get_tests.py '${{ inputs.recommended_tests }}' > testsuite/run_sets/filter.yml + - name: Generate recommended tests as YAML + if: ${{ inputs.recommended_tests != '' }} + run: ./testsuite/podman_runner/19_generate_recommended_tests_yml.sh + - name: Collect and tag flaky tests in controller # Until we refactor the usage of the secret, it only runs on uyuni-project/uyuni repository branches if: github.repository == 'uyuni-project/uyuni' run: ./testsuite/podman_runner/06_collect_and_tag_flaky_tests_in_controller.sh env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_GALAXY_CI }} - - name: setup-server-container + - name: Setup server container run: ./testsuite/podman_runner/07_server_setup.sh - - name: start-server-container + - name: Start server container run: ./testsuite/podman_runner/08_start_server.sh - - name: build_code + - name: Build server code run: ./testsuite/podman_runner/09_build_server_code.sh # - name: copy_ca # run: podman exec server bash -c "cp /etc/pki/tls/certs/spacewalk.crt /tmp" # - name: update_ca_in_controller # run: podman exec controller bash -c "cat /tmp/spacewalk.crt >> /etc/ssl/ca-bundle.pem" - - name: sle-sshminion + - name: Run SSH minion container run: ./testsuite/podman_runner/10_run_sshminion.sh - - name: test_from_host + - name: Test access to server from controller run: curl --insecure https://localhost:8443/rhn/help/Copyright.do - - name: test_from_container + - name: Test access to server from SSH minion run: sudo -i podman exec opensusessh curl --insecure https://server:443/rhn/help/Copyright.do - - name: setup_sshd + - name: Configure SSHD in controller, server, build host and SSH minion run: ./testsuite/podman_runner/11_setup_sshd.sh - - name: run_cucumber_core + - name: Run Core tests run: ./testsuite/podman_runner/12_run_core_tests.sh - - name: sle-salt-minion + - name: Run and configure Salt Minion container run: ./testsuite/podman_runner/13_run_salt_sle_minion.sh - - name: rhlike-minion + - name: Run and configure RH-Like Minion container run: ./testsuite/podman_runner/14_run_salt_rhlike_minion.sh - - name: deblike-minion + - name: Run and configure Deb-Like Minion container run: ./testsuite/podman_runner/15_run_salt_deblike_minion.sh - - name: accept_keys + - name: Accept all keys run: ./testsuite/podman_runner/16_accept_all_keys.sh - - name: run_cucumber_clients + - name: Run init clients run: ./testsuite/podman_runner/17_run_init_clients_tests.sh - - name: split_secondary + - name: Split tests into multiple YAML files run: ./testsuite/podman_runner/19_split_secondary_p_tests.sh - - name: run_secondary_tests - run: ./testsuite/podman_runner/${{ inputs.secondary_tests }} - - name: get_server_logs + - name: Run acceptance tests + run: ./testsuite/podman_runner/${{ inputs.secondary_tests }} + - name: Get server logs if: ${{ failure() }} run: ./testsuite/podman_runner/20_get_server_logs.sh ${{ inputs.server_id }} - - name: get_client_logs + - name: Get client logs if: ${{ failure() }} run: ./testsuite/podman_runner/21_get_client_logs.sh ${{ inputs.server_id }} - - name: upload_server_log_artifacts + - name: Upload server log artifacts if: ${{ failure() }} uses: actions/upload-artifact@v4 with: name: server_rhn_logs_${{ inputs.server_id }} path: /tmp/testing/server-logs/${{ inputs.server_id }} - - name: upload_client_log_artifacts + - name: Upload client log artifacts if: ${{ failure() }} uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/acceptance_tests_feedback.yml b/.github/workflows/acceptance_tests_feedback.yml index fb7231282755..958622e1e233 100644 --- a/.github/workflows/acceptance_tests_feedback.yml +++ b/.github/workflows/acceptance_tests_feedback.yml @@ -1,4 +1,4 @@ -name: acceptance-tests-feedback +name: Ask for acceptance tests feedback on: pull_request_target: paths: @@ -25,7 +25,10 @@ jobs: with: body: | :wave: Hello! Thanks for contributing to our project. - Acceptance tests will take some time (aprox. 1h), please be patient :coffee: + + If you want to run all the acceptance tests, you can add the label "run-full-testsuite" to this PR, otherwise, only the tests that are related to your changes will run. + + Running all the acceptance tests will take some time (aprox. 1h), please be patient :coffee: You can see the progress at the end of this page and at https://github.com/uyuni-project/uyuni/pull/${{ github.event.pull_request.number }}/checks Once tests finish, if they fail, you can check :eyes: the cucumber report. See the link at the output of the action. You can also check the artifacts section, which contains the logs at https://github.com/uyuni-project/uyuni/pull/${{ github.event.pull_request.number }}/checks. diff --git a/.github/workflows/acceptance_tests_secondary.yml b/.github/workflows/acceptance_tests_secondary.yml index 4ab01b82562d..3cbcc31acfc9 100644 --- a/.github/workflows/acceptance_tests_secondary.yml +++ b/.github/workflows/acceptance_tests_secondary.yml @@ -1,19 +1,25 @@ -name: acceptance-tests-secondary +name: Acceptance Tests on: pull_request: types: - opened - reopened - synchronize + pull_request_target: + types: + - labeled # Trigger when a label is added schedule: - cron: '0 */12 * * *' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - acceptance-tests-secondary: - # Skip running scheduled jobs on forks - if: (github.repository == 'uyuni-project/uyuni' || github.event_name != 'schedule') + acceptance-tests: + name: "0" + # Skip running scheduled jobs on forks and only run if "run-full-testsuite" label is present + if: >- + (github.repository == 'uyuni-project/uyuni' || github.event_name != 'schedule') && + contains(github.event.pull_request.labels.*.name, 'run-full-testsuite') uses: ./.github/workflows/acceptance_tests_common.yml with: secondary_tests: "18_run_secondary_tests.sh" diff --git a/.github/workflows/acceptance_tests_secondary_parallel.yml b/.github/workflows/acceptance_tests_secondary_parallel.yml index 8657f72e56ad..195f0c2036ff 100644 --- a/.github/workflows/acceptance_tests_secondary_parallel.yml +++ b/.github/workflows/acceptance_tests_secondary_parallel.yml @@ -1,19 +1,25 @@ -name: acceptance-tests-secondary-parallel +name: Additional Tests on: pull_request: types: - opened - reopened - synchronize + pull_request_target: + types: + - labeled # Trigger when a label is added schedule: - cron: '0 */12 * * *' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} jobs: - acceptance-tests-secondary-parallel: - # Skip running scheduled jobs on forks - if: (github.repository == 'uyuni-project/uyuni' || github.event_name != 'schedule') + additional-acceptance-tests: + name: ${{ matrix.set }} + # Skip running scheduled jobs on forks and only run if "run-full-testsuite" label is present + if: >- + (github.repository == 'uyuni-project/uyuni' || github.event_name != 'schedule') && + contains(github.event.pull_request.labels.*.name, 'run-full-testsuite') uses: ./.github/workflows/acceptance_tests_common.yml strategy: fail-fast: false diff --git a/.github/workflows/recommended_tests.yml b/.github/workflows/recommended_tests.yml new file mode 100644 index 000000000000..87b244f372c9 --- /dev/null +++ b/.github/workflows/recommended_tests.yml @@ -0,0 +1,48 @@ +name: Recommended Tests + +on: + pull_request_target: + paths: + - '**.java' + +jobs: + generate_recommended_tests: + name: Generate + runs-on: ubuntu-latest + steps: + - name: Checkout Pull Request code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm i ioredis + - id: changed_files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: '*.java' + - id: tests_covering_pr + name: Tests covering PR + env: + REDIS_USER: ${{ secrets.REDIS_USER }} + REDIS_PASS: ${{ secrets.REDIS_PASS }} + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + run: | + recommended_tests=$(node .github/scripts/test_covering_pr.js ${{ steps.changed_files.outputs.added_modified }}) + echo "recommended_tests=$recommended_tests" >> $GITHUB_OUTPUT + echo "Recommended Tests: $recommended_tests" + outputs: + recommended_tests: ${{ steps.tests_covering_pr.outputs.recommended_tests }} + + run_recommended_tests: + name: Run + needs: generate_recommended_tests + uses: ./.github/workflows/acceptance_tests_common.yml + with: + secondary_tests: "19_run_recommended_tests.sh" + server_id: "secondary_f" + recommended_tests: ${{ needs.generate_recommended_tests.outputs.recommended_tests }} diff --git a/.github/workflows/test-covering-pr.yml b/.github/workflows/test-covering-pr.yml deleted file mode 100644 index e3f39921785a..000000000000 --- a/.github/workflows/test-covering-pr.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: 'Suggested tests to cover this Pull Request' - -on: - pull_request_target: - paths: - - '**.java' - -jobs: - tests-covering-pr: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Use Node.js 18 - uses: actions/setup-node@v4 - with: - node-version: 18 - - run: npm i ioredis - - id: files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: '*.java' - - id: tests_covering_pr - name: Tests covering PR - env: - REDIS_USER: ${{ secrets.REDIS_USER }} - REDIS_PASS: ${{ secrets.REDIS_PASS }} - REDIS_HOST: ${{ secrets.REDIS_HOST }} - REDIS_PORT: ${{ secrets.REDIS_PORT }} - run: echo "result=$(node .github/scripts/test_covering_pr.js ${{ steps.files.outputs.added_modified }})" >> $GITHUB_OUTPUT - - name: Write a comment into the Pull Request - uses: actions-cool/maintain-one-comment@v3 - with: - body: | - ${{ steps.tests_covering_pr.outputs.result }} - body-include: '' diff --git a/testsuite/Rakefile b/testsuite/Rakefile index 6bf28f93819a..e6a718b8f433 100644 --- a/testsuite/Rakefile +++ b/testsuite/Rakefile @@ -83,6 +83,29 @@ namespace :utils do end end + desc 'Filter set test' + task :filter_secondary, [:filter_yaml, :input_yaml, :output_yaml] do |_t, args| + filter_yaml = args[:filter_yaml] + input_yaml = args[:input_yaml] + output_yaml = args[:output_yaml] + + # Load test lists from the provided YAML files + filter_tests = YAML.load_file(filter_yaml) || [] + input_tests = YAML.load_file(input_yaml) || [] + + # Normalize the test names + filter_tests = filter_tests.map { |test| "features/secondary/#{test}.feature" } + + # Intersect filter_tests with input_tests to create the filtered set + filtered_tests = input_tests.select { |test| filter_tests.include?(test) } + + # Log for debugging + puts "Filtered Tests: #{filtered_tests}" + + # Write the filtered tests to the output YAML + File.write(output_yaml, filtered_tests.to_yaml) + end + desc 'Generate feature file for a client from a template and include it in a yaml file' task :generate_feature, [:template_path, :client_name, :output_path] do |_t, args| # TODO: We want to use English language in our Cucumber .feature files, diff --git a/testsuite/podman_runner/18_run_secondary_tests.sh b/testsuite/podman_runner/18_run_secondary_tests.sh index 9ec604b3bc76..9b3c977fdc93 100755 --- a/testsuite/podman_runner/18_run_secondary_tests.sh +++ b/testsuite/podman_runner/18_run_secondary_tests.sh @@ -1,3 +1,4 @@ #!/bin/bash set -xe -sudo -i podman exec controller bash -c "export BUILD_HOST=buildhost && export AUTH_REGISTRY=${AUTH_REGISTRY} && export AUTH_REGISTRY_CREDENTIALS=\"${AUTH_REGISTRY_CREDENTIALS}\" && export NO_AUTH_REGISTRY=${NO_AUTH_REGISTRY} && export CUCUMBER_PUBLISH_TOKEN=${CUCUMBER_PUBLISH_TOKEN} && export PROVIDER=podman && export SERVER=server && export HOSTNAME=controller && export SSH_MINION=opensusessh && export MINION=sle_minion && export RHLIKE_MINION=rhlike_minion && export TAGS=\"\\\"not @flaky\\\"\" && export DEBLIKE_MINION=deblike_minion && cd /testsuite && rake cucumber:secondary" +TESTS="$@" +sudo -i podman exec controller bash -c "export CUCUMBER_PUBLISH_TOKEN=${CUCUMBER_PUBLISH_TOKEN} && export PROVIDER=podman && export SERVER=server && export HOSTNAME=controller && export SSH_MINION=opensusessh && export MINION=sle_minion && export RHLIKE_MINION=rhlike_minion && export TAGS=\"\\\"not @flaky\\\"\" && cd /testsuite && rake cucumber:secondary ${TESTS}" diff --git a/testsuite/podman_runner/19_generate_recommended_tests_yml.sh b/testsuite/podman_runner/19_generate_recommended_tests_yml.sh new file mode 100755 index 000000000000..87cac68c408f --- /dev/null +++ b/testsuite/podman_runner/19_generate_recommended_tests_yml.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -xe +sudo -i podman exec controller bash -c "cd /testsuite && rake utils:filter_secondary[/testsuite/run_sets/filter.yml,/testsuite/run_sets/secondary.yml,/testsuite/run_sets/recommended_tests.yml] " + diff --git a/testsuite/podman_runner/19_run_recommended_tests.sh b/testsuite/podman_runner/19_run_recommended_tests.sh new file mode 100755 index 000000000000..aaf924497175 --- /dev/null +++ b/testsuite/podman_runner/19_run_recommended_tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -xe + +sudo -i podman exec controller bash -c "export CUCUMBER_PUBLISH_TOKEN=${CUCUMBER_PUBLISH_TOKEN} && export PROVIDER=podman && export SERVER=server && export HOSTNAME=controller && export SSH_MINION=opensusessh && export MINION=sle_minion && export RHLIKE_MINION=rhlike_minion && cd /testsuite && export TAGS=\"\\\"not @flaky\\\"\" && cat run_sets/recommended_tests.yml && rake cucumber:recommended_tests"