From 4dbf191c9fe951df4322b5f043ba2d436d6b7b75 Mon Sep 17 00:00:00 2001 From: Alex Yefimov <126113326+ayefimov-1@users.noreply.github.com> Date: Tue, 28 Jan 2025 18:23:06 -0500 Subject: [PATCH] Add to irrelevant_files list for graphing, logging and autoscaling jobs --- .ansible-lint | 1 + .zuul.yaml | 67 ++++++++++++++++--- ci/logging_tests_computes.yml | 7 ++ ci/logging_tests_controller.yml | 5 +- ci/run_autoscaling_osp18.yml | 63 ----------------- ci/use-master-containers.yml | 18 +++++ ci/vars-functional-test.yml | 6 +- ci/vars-use-master-containers.yml | 4 ++ roles/common/README.md | 1 + roles/common/tasks/container_test.yml | 4 +- roles/common/tasks/cr_tests.yml | 5 +- roles/common/tasks/crd_tests.yml | 4 +- roles/common/tasks/endpoint_tests.yml | 6 +- roles/common/tasks/file_tests.yml | 22 ++++-- roles/common/tasks/main.yml | 2 +- roles/common/tasks/manifest_tests.yml | 6 +- roles/common/tasks/pod_tests.yml | 4 +- roles/common/tasks/project_tests.yml | 6 +- roles/common/tasks/service_tests.yml | 2 +- roles/common/tasks/subscription_tests.yml | 4 +- .../tasks/test_autoscaling.yml | 24 +++++++ .../telemetry_logging/tasks/journal_tests.yml | 2 +- roles/telemetry_verify_metrics/README.md | 5 +- .../defaults/main.yml | 3 + roles/telemetry_verify_metrics/tasks/main.yml | 14 ++++ .../tasks/verify_ceilometer_ipmi_metrics.yml | 40 +++++++++++ .../tasks/verify_kepler_metrics.yml | 36 ++++++++++ 27 files changed, 262 insertions(+), 99 deletions(-) create mode 100644 ci/use-master-containers.yml create mode 100644 ci/vars-use-master-containers.yml create mode 100644 roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml create mode 100644 roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml diff --git a/.ansible-lint b/.ansible-lint index ff64212f..9b67ffad 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,6 +6,7 @@ skip_list: - command-instead-of-module # Using command rather than module - galaxy - meta-no-info # meta/main.yml should contain relevant info + - name[template] # Jinja templates should only be at the end of 'name' - risky-shell-pipe # Shells that use pipes should set the pipefail option - unnamed-task # All tasks should be named - var-naming[no-role-prefix] # Variables names from within roles should use role_name_ as a prefix diff --git a/.zuul.yaml b/.zuul.yaml index 93fc5248..5b687ea0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -10,6 +10,7 @@ - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework required-projects: &required_projects @@ -57,11 +58,26 @@ roles: - zuul: github.com/openstack-k8s-operators/ci-framework required-projects: *required_projects - irrelevant-files: - <<: *irrelevant_files - - roles/telemetry_graphing - - roles/telemetry_autoscaling - - roles/telemetry_verify_metrics + irrelevant-files: + # *irrelevant_files + - .github/.* + - ci/github/.* + - callback_plugins/.* + - README* + - .*/*.md + - roles/client_side_tests/.* + - roles/test_alerts/.* + - roles/test_collectd/.* + - roles/test_metrics_retention/.* + - roles/test_observability_strategy/.* + - roles/test_qdr/.* + - roles/test_sensubility/.* + - roles/test_snmp_traps/.* + - roles/test_verify_email/.* + # + - roles/telemetry_graphing + - roles/telemetry_autoscaling + - roles/telemetry_verify_metrics - job: name: functional-metric-verification-tests-osp18 @@ -74,15 +90,29 @@ - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework required-projects: *required_projects irrelevant-files: - <<: *irrelevant_files + # *irrelevant_files + - .github/.* + - ci/github/.* + - callback_plugins/.* + - README* + - .*/*.md + - roles/client_side_tests/.* + - roles/test_alerts/.* + - roles/test_collectd/.* + - roles/test_metrics_retention/.* + - roles/test_observability_strategy/.* + - roles/test_qdr/.* + - roles/test_sensubility/.* + - roles/test_snmp_traps/.* + - roles/test_verify_email/.* + # - roles/telemetry_graphing - roles/telemetry_logging - - roles/telemetry_verify_metrics - - job: name: feature-verification-tests-noop @@ -108,8 +138,23 @@ roles: - zuul: github.com/openstack-k8s-operators/ci-framework required-projects: *required_projects - irrelevant-files: - <<: *irrelevant_files + irrelevant-files: + # *irrelevant_files + - .github/.* + - ci/github/.* + - callback_plugins/.* + - README* + - .*/*.md + - roles/client_side_tests/.* + - roles/test_alerts/.* + - roles/test_collectd/.* + - roles/test_metrics_retention/.* + - roles/test_observability_strategy/.* + - roles/test_qdr/.* + - roles/test_sensubility/.* + - roles/test_snmp_traps/.* + - roles/test_verify_email/.* + # - roles/telemetry_logging - roles/telemetry_verify_metrics @@ -122,7 +167,7 @@ override-checkout: main irrelevant-files: *irrelevant_files - functional-tests-on-osp18: - voting: false + voting: true - functional-logging-tests-osp18 - functional-graphing-tests-osp18 - functional-metric-verification-tests-osp18 diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index d520a9c4..80c6f617 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -11,7 +11,14 @@ journal_list: - ceilometer_agent_compute - nova_compute + common_file_test_id: "RHOSO-12754" + common_file_list: + - /etc/rsyslog.d/10-telemetry.conf tasks: - name: "Run Telemetry Logging tests" ansible.builtin.import_role: name: telemetry_logging + + - name: "Run the tests from the common role" + ansible.builtin.import_role: + name: common diff --git a/ci/logging_tests_controller.yml b/ci/logging_tests_controller.yml index 9c314a4e..a7adbb36 100644 --- a/ci/logging_tests_controller.yml +++ b/ci/logging_tests_controller.yml @@ -7,7 +7,7 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - common_pod_test_id: "RHOS0-12672" + common_pod_test_id: "RHOSO-12672" common_pod_status_str: "Running" common_pod_nspace: openstack-operators common_pod_list: @@ -42,9 +42,6 @@ - logging-loki-query-frontend-grpc - logging-loki-query-frontend-http - openstack-logging - common_file_test_id: "RHOSO-12754" - common_file_list: - - /etc/rsyslog.d/10-telemetry.conf tasks: - name: "Verify logging infrastructure components" ansible.builtin.import_role: diff --git a/ci/run_autoscaling_osp18.yml b/ci/run_autoscaling_osp18.yml index 135c8b8c..bfa918da 100644 --- a/ci/run_autoscaling_osp18.yml +++ b/ci/run_autoscaling_osp18.yml @@ -20,26 +20,6 @@ tags: - setup - - name: Patch the openstackversions to use the master containers for aodh and heat - ansible.builtin.shell: - cmd: | - oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml - chdir: "{{ fvt_dir }}" - when: "{{ patch_openstackversions | bool }}" - tags: - - setup - - - name: Redeploy the dataplane - ansible.builtin.shell: - cmd: | - oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml - oc delete osdpd edpm-deployment - sleep 10 - oc apply -f /tmp/osdpd.yaml - when: "{{ patch_openstackversions | bool }}" - tags: - - setup - - name: Patch observabilityclient into openstackclient ansible.builtin.shell: cmd: | @@ -50,50 +30,7 @@ tags: - setup - - name: Wait until the oscp is resolved the changes to continue - ansible.builtin.shell: - cmd: | - oc get oscp | grep "Setup complete" - retries: 24 - timeout: 5 - until: output.stdout_lines | length == 1 - register: output - when: "{{ patch_openstackversions | bool }}" - tags: - - setup - - - name: Wait until the osdpd is redeployed to continue - ansible.builtin.shell: - cmd: | - oc get osdpd | grep "Setup complete" - retries: 60 - delay: 45 - until: output.stdout_lines | length == 1 - register: output - when: "{{ patch_openstackversions | bool }}" - tags: - - setup - - name: "Run Telemetry Autoscaling tests" ansible.builtin.import_role: name: telemetry_autoscaling ignore_errors: true - - always: - - name: Revert the version update - ansible.builtin.shell: - cmd: | - oc patch openstackversions controlplane --type json -p='[{"op": "replace", "path": "/spec/customContainerImages", "value": {} }]' - when: "{{ patch_openstackversions | bool }}" - - - name: Redeploy the dataplane - ansible.builtin.shell: - cmd: | - oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml - oc delete osdpd edpm-deployment - sleep 10 - oc apply -f /tmp/osdpd.yaml - when: "{{ patch_openstackversions | bool }}" - tags: - - setup - diff --git a/ci/use-master-containers.yml b/ci/use-master-containers.yml new file mode 100644 index 00000000..79b07fcf --- /dev/null +++ b/ci/use-master-containers.yml @@ -0,0 +1,18 @@ +- name: Run telemetry autoscaling tests on osp18 + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + - vars/osp18_env.yml + tasks: + - name: Patch the openstackversions to use the master containers for aodh, heat and ceilometer + ansible.builtin.shell: + cmd: | + oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml + chdir: "{{ fvt_dir }}" + tags: + - setup + diff --git a/ci/vars-functional-test.yml b/ci/vars-functional-test.yml index 036d39a8..081d86d5 100644 --- a/ci/vars-functional-test.yml +++ b/ci/vars-functional-test.yml @@ -1,5 +1,9 @@ --- -# temp: skip os-net-setup +# This is added so we skip the os_net_setup role, which would normally be used +# to create a network. +# A network gets created by `make edpm_deploy_instance` during the autoscaling +# tests. +# skip os-net-setup cifmw_os_net_setup_config: [] post_deploy_00_run_functional_test: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_autoscaling_osp18.yml" diff --git a/ci/vars-use-master-containers.yml b/ci/vars-use-master-containers.yml new file mode 100644 index 00000000..ba07741f --- /dev/null +++ b/ci/vars-use-master-containers.yml @@ -0,0 +1,4 @@ +--- +post_ctlplane_deploy_99_modify_openstackversions: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/use-master-containers.yml" + type: playbook diff --git a/roles/common/README.md b/roles/common/README.md index 7b06b9be..bbc72559 100644 --- a/roles/common/README.md +++ b/roles/common/README.md @@ -155,6 +155,7 @@ can be set at the play level. - ceilometer_agent_compute - ceilometer_agent_ipmi - node_exporter + - kepler - name: "Verify Endpoint" ansible.builtin.import_role: diff --git a/roles/common/tasks/container_test.yml b/roles/common/tasks/container_test.yml index 71b84ac5..2ccfa050 100644 --- a/roles/common/tasks/container_test.yml +++ b/roles/common/tasks/container_test.yml @@ -6,7 +6,9 @@ changed_when: false register: container_status -- name: Verify container status {{ common_container_test_id }} +- name: | + TEST Verify {{ container_name }} container status + {{ common_container_test_id }} ansible.builtin.assert: that: - "'unhealthy' not in container_status.stdout" diff --git a/roles/common/tasks/cr_tests.yml b/roles/common/tasks/cr_tests.yml index fcb646d7..03d05423 100644 --- a/roles/common/tasks/cr_tests.yml +++ b/roles/common/tasks/cr_tests.yml @@ -1,4 +1,7 @@ -- name: Verify that a CR exists {{ common_cr_test_id }} +--- +- name: | + TEST Verify that the {{ item.kind }} {{ item.name }} CR exists + {{ common_cr_test_id }} ansible.builtin.command: cmd: | oc get {{ item.kind }} {{ item.name }} diff --git a/roles/common/tasks/crd_tests.yml b/roles/common/tasks/crd_tests.yml index e618d204..afbe8930 100644 --- a/roles/common/tasks/crd_tests.yml +++ b/roles/common/tasks/crd_tests.yml @@ -6,7 +6,9 @@ changed_when: false register: output -- name: Verify CRD is found "{{ common_crd_test_id }}" +- name: | + TEST Verify CRD {{ item }} is found + {{ common_crd_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" diff --git a/roles/common/tasks/endpoint_tests.yml b/roles/common/tasks/endpoint_tests.yml index cdea10ea..b04b937a 100644 --- a/roles/common/tasks/endpoint_tests.yml +++ b/roles/common/tasks/endpoint_tests.yml @@ -6,9 +6,11 @@ changed_when: false register: output -- name: Check that endpoint exist {{ common_endpoint_test_id }} +- name: | + TEST Check that {{ item[0] }} {{ item[2] }} endpoint exists + {{ common_endpoint_test_id }} ansible.builtin.assert: that: - output.stdout != "" success_msg: "The endpoint {{ item[0] }} is found." - fail_msg: "The endpoint for service {{ item[0] }} " \ No newline at end of file + fail_msg: "The endpoint for service {{ item[0] }} on the {{ item[2] }} interface was not found" diff --git a/roles/common/tasks/file_tests.yml b/roles/common/tasks/file_tests.yml index 3a0b383f..c56d8634 100644 --- a/roles/common/tasks/file_tests.yml +++ b/roles/common/tasks/file_tests.yml @@ -3,8 +3,20 @@ ansible.builtin.stat: path: "{{ item }}" register: fstats - failed_when: - - fstats.stat.pw_name != "root" - - fstats.stat.size | int < 300 - - not fstats.stat.exists - - not fstats.stat.isreg \ No newline at end of file + +- name: | + TEST Verify the file {{ item }} exists + {{ common_file_test_id }} + ansible.builtin.assert: + that: + - fstats.stat.exists + - fstats.stat.pw_name == "root" + - fstats.stat.size | int > 300 + - fstats.stat.isreg + fail_msg: | + File {{ item }} does not meet the required conditions: + {%- if not fstats.stat.exists %} file does not exist.{% endif %} + {%- if fstats.stat.exists and fstats.stat.pw_name != "root" %} file owner must be root (got {{ fstats.stat.pw_name }});{% endif %} + {%- if fstats.stat.exists and fstats.stat.size | int < 300 %} file must be larger than 300 bytes (got {{ fstats.stat.size }} bytes);{% endif %} + {%- if fstats.stat.exists and not fstats.stat.isreg | bool %} file must be a regular file.{% endif %} + success_msg: "The test passed" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 21fcc337..1993bc9d 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -1,5 +1,5 @@ --- -- name: "Verify pod - {{ common_pod_test_id }}" +- name: "Run pod tests" when: - common_pod_test_id is defined - common_pod_list is defined diff --git a/roles/common/tasks/manifest_tests.yml b/roles/common/tasks/manifest_tests.yml index 63e03aa7..48c4ab86 100644 --- a/roles/common/tasks/manifest_tests.yml +++ b/roles/common/tasks/manifest_tests.yml @@ -13,10 +13,12 @@ register: pack_name changed_when: false -- name: Get packagemanifest "{{ common_manifest_test_id }}" +- name: | + TEST Get {{ item }} packagemanifest + {{ common_manifest_test_id }} ansible.builtin.shell: cmd: | oc get packagemanifests | grep "{{ pack_name.stdout }}" | wc -l register: num_found changed_when: false - failed_when: num_expected.stdout != num_found.stdout \ No newline at end of file + failed_when: num_expected.stdout != num_found.stdout diff --git a/roles/common/tasks/pod_tests.yml b/roles/common/tasks/pod_tests.yml index 97e2b5b7..e688c9fc 100644 --- a/roles/common/tasks/pod_tests.yml +++ b/roles/common/tasks/pod_tests.yml @@ -11,7 +11,9 @@ - podinstance.stdout_lines | length != 1 changed_when: false - - name: Check pod {{ common_pod_test_id }} + - name: | + TEST Check {{ item }} pod is {{ common_pod_status_str }} in {{ common_pod_nspace }} namespace + {{ common_pod_test_id }} ansible.builtin.command: cmd: | oc get pod -n "{{ common_pod_nspace }}" "{{ podinstance.stdout }}" diff --git a/roles/common/tasks/project_tests.yml b/roles/common/tasks/project_tests.yml index 1d0be76e..487961a7 100644 --- a/roles/common/tasks/project_tests.yml +++ b/roles/common/tasks/project_tests.yml @@ -1,12 +1,14 @@ --- -- name: Verify project "{{ item }}" +- name: Get the {{ item }} project ansible.builtin.shell: cmd: | oc get project "{{ item }}" changed_when: false register: output -- name: Verify project is found "{{ common_project_test_id }}" +- name: | + TEST Verify {{ item }} project exists + {{ common_project_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" diff --git a/roles/common/tasks/service_tests.yml b/roles/common/tasks/service_tests.yml index a6505d66..1a1fcf7e 100644 --- a/roles/common/tasks/service_tests.yml +++ b/roles/common/tasks/service_tests.yml @@ -7,7 +7,7 @@ register: output - name: | - Verify {{ item }} service is running + TEST Verify {{ item }} service is running in {{ common_service_nspace }} namespace {{ common_service_test_id }} ansible.builtin.assert: that: diff --git a/roles/common/tasks/subscription_tests.yml b/roles/common/tasks/subscription_tests.yml index 49f2b6db..4b27530c 100644 --- a/roles/common/tasks/subscription_tests.yml +++ b/roles/common/tasks/subscription_tests.yml @@ -6,7 +6,9 @@ changed_when: false register: output -- name: Check that subscription exist {{ common_subscription_test_id }} +- name: | + TEST Check that {{ item }} subscription exists in {{ common_subscription_nspace }} namespace + {{ common_subscription_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" diff --git a/roles/telemetry_autoscaling/tasks/test_autoscaling.yml b/roles/telemetry_autoscaling/tasks/test_autoscaling.yml index 76db5940..7dd29944 100644 --- a/roles/telemetry_autoscaling/tasks/test_autoscaling.yml +++ b/roles/telemetry_autoscaling/tasks/test_autoscaling.yml @@ -59,6 +59,30 @@ register: busy_process with_items: "{{ vnf_instance_ip.stdout_lines }}" +- name: Show ceilometer_cpu metrics from Prometheus + when: metrics_backend == "prometheus" + block: + - name: Register autoscaling query results + shell: | + export STACK_ID=$({{ openstack_cmd }} stack show {{ stack_name }} -c id -f value) + {{ openstack_cmd }} metric query '(rate(ceilometer_cpu{server_group=~"$STACK_ID"}[150s]))/10000000' + ignore_errors: true + register: result + + - name: Show the query result + ansible.builtin.debug: + var: result + + - name: Register ceilometer_cpu metrics + shell: | + {{ openstack_cmd }} metric show ceilometer_cpu --disable-rbac + ignore_errors: true + register: result + + - name: Show the ceilometer_cpu metrics + ansible.builtin.debug: + var: result + - name: | TEST Verify cpu high alarm has been triggered RHOSO-12660 diff --git a/roles/telemetry_logging/tasks/journal_tests.yml b/roles/telemetry_logging/tasks/journal_tests.yml index 1d27f2a4..cb5e14b6 100644 --- a/roles/telemetry_logging/tasks/journal_tests.yml +++ b/roles/telemetry_logging/tasks/journal_tests.yml @@ -1,6 +1,6 @@ --- - name: | - Get journals {{ item }} + TEST Get journals {{ item }} {{ journal_test_id }} become: true ansible.builtin.shell: diff --git a/roles/telemetry_verify_metrics/README.md b/roles/telemetry_verify_metrics/README.md index e48628e7..ddb6f360 100644 --- a/roles/telemetry_verify_metrics/README.md +++ b/roles/telemetry_verify_metrics/README.md @@ -24,13 +24,16 @@ Tests: - Verify Ceilometer metrics are being exposed and stored - Use openstack observabilityclient to verify Ceilometer central metrics are stored in Prometheus - Use openstack observabilityclient to verify Ceilometer compute metrics are stored in Prometheus + - Use openstack observabilityclient to verify Ceilometer ipmi metrics are stored in Prometheus - Verify NodeExporter metrics are being exposed and stored - Use openstack observabilityclient to verify NodeExporter metrics are stored in Prometheus +- Verify Kepler metrics are being exposed and stored + - Use openstack observabilityclient to verify Kepler metrics are stored in Prometheus Role Variables -------------- openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack" -telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, node\_exporter, rabbitmq +telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler Example Playbook ---------------- diff --git a/roles/telemetry_verify_metrics/defaults/main.yml b/roles/telemetry_verify_metrics/defaults/main.yml index 349c4370..74a3f89a 100644 --- a/roles/telemetry_verify_metrics/defaults/main.yml +++ b/roles/telemetry_verify_metrics/defaults/main.yml @@ -4,3 +4,6 @@ telemetry_verify_metrics_metric_sources_to_test: - ceilometer_central_agent - node_exporter - rabbitmq + - kepler + # NOTE: IPMI metrics can only be verified when compute nodes are baremetal + # - ceilometer_ipmi_agent diff --git a/roles/telemetry_verify_metrics/tasks/main.yml b/roles/telemetry_verify_metrics/tasks/main.yml index c055e9c6..2645e1ad 100644 --- a/roles/telemetry_verify_metrics/tasks/main.yml +++ b/roles/telemetry_verify_metrics/tasks/main.yml @@ -41,8 +41,22 @@ tags: test when: '"ceilometer_central_agent" in telemetry_verify_metrics_metric_sources_to_test' +- name: Verify Ceilometer ipmi metrics are being exposed and stored + ansible.builtin.include_tasks: + file: verify_ceilometer_ipmi_metrics.yml + tags: power_monitoring + when: '"ceilometer_ipmi_agent" in telemetry_verify_metrics_metric_sources_to_test' + - name: Verify NodeExporter metrics are being exposed and stored ansible.builtin.include_tasks: file: verify_node_exporter_metrics.yml tags: test when: '"node_exporter" in telemetry_verify_metrics_metric_sources_to_test' + +- name: Verify Kepler metrics are being exposed and stored + ansible.builtin.include_tasks: + file: verify_kepler_metrics.yml + tags: + - test + - power_monitoring + when: '"kepler" in telemetry_verify_metrics_metric_sources_to_test' diff --git a/roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml b/roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml new file mode 100644 index 00000000..2bba5e21 --- /dev/null +++ b/roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml @@ -0,0 +1,40 @@ +- name: Verify ceilometer scrapeconfig exists + ansible.builtin.include_role: + name: common + vars: + common_cr_test_id: RHOSO-1221 + common_cr_list: + - kind: scrapeconfigs.monitoring.rhobs + name: telemetry-ceilometer + +- name: Verify ceilometer ipmi agent container is up + ansible.builtin.include_tasks: + file: check_compute_node_containers.yml + vars: + common_container_test_id: RHOSO-1230 + common_container_list: + - ceilometer_agent_ipmi + loop: "{{ groups['computes'] }}" + loop_control: + loop_var: compute_node + +- name: Verify ceilometer controlplane pod is running + ansible.builtin.include_role: + name: common + vars: + common_pod_test_id: RHOSO-1242 + common_pod_status_str: "Running" + common_pod_nspace: openstack + common_pod_list: + - ceilometer-0 + +- name: | + TEST Use openstack observabilityclient to verify ceilometer ipmi metrics are stored in prometheus + RHOSO-1217 + ansible.builtin.shell: | + {{ openstack_cmd }} metric show ceilometer_hardware_ipmi_fan --disable-rbac + register: result + delay: 30 + retries: 10 + until: result.rc == 0 and "ceilometer_hardware_ipmi" in result.stdout + changed_when: false diff --git a/roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml b/roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml new file mode 100644 index 00000000..32a8ad4e --- /dev/null +++ b/roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml @@ -0,0 +1,36 @@ +- name: Verify node exporter scrapeconfigs exist + ansible.builtin.include_role: + name: common + vars: + common_cr_test_id: RHOSO-1222 + common_cr_list: + - kind: scrapeconfigs.monitoring.rhobs + name: telemetry-kepler + kepler_dashboard_metrics: + - kepler_vm_platform_joules_total + - kepler_node_platform_joules_total + - kepler_process_joules_total + - kepler_container_joules_total + - kepler_node_info + +- name: Verify Kepler container is up + ansible.builtin.include_tasks: + file: check_compute_node_containers.yml + vars: + common_container_test_id: RHOSO-1231 + common_container_list: + - kepler + loop: "{{ groups['computes'] }}" + loop_control: + loop_var: compute_node + +- name: | + TEST Use openstack observabilityclient to verify kepler metrics are stored in prometheus + RHOSO-1216 + ansible.builtin.shell: | + {{ openstack_cmd }} metric show --disable-rbac kepler_node_info + register: result + delay: 30 + retries: 10 + changed_when: false + until: result.rc == 0 and "kepler_node_info" in result.stdout