Merge branch 'master' into update_pod_test_completed_status

infrawatch · Jan 30, 2025 · 9cd7bc0 · 9cd7bc0
2 parents 7de292a + 24eae54
commit 9cd7bc0
Show file tree

Hide file tree

Showing 13 changed files with 155 additions and 69 deletions.
diff --git a/.zuul.yaml b/.zuul.yaml
@@ -10,6 +10,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: &required_projects
@@ -70,6 +71,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: *required_projects
@@ -111,7 +113,7 @@
             override-checkout: main
             irrelevant-files: *irrelevant_files
         - functional-tests-on-osp18:
-            voting: false
+            voting: true
         - functional-logging-tests-osp18
         - functional-graphing-tests-osp18
         - functional-metric-verification-tests-osp18
diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml
@@ -11,7 +11,14 @@
     journal_list:
       - ceilometer_agent_compute
       - nova_compute
+    common_file_test_id: "RHOSO-12754"
+    common_file_list:
+      - /etc/rsyslog.d/10-telemetry.conf
   tasks:
     - name: "Run Telemetry Logging tests"
       ansible.builtin.import_role:
         name: telemetry_logging
+
+    - name: "Run the tests from the common role"
+      ansible.builtin.import_role:
+        name: common
diff --git a/ci/logging_tests_controller.yml b/ci/logging_tests_controller.yml
@@ -7,7 +7,7 @@
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path }}"
   vars:
-    common_pod_test_id: "RHOS0-12672"
+    common_pod_test_id: "RHOSO-12672"
     common_pod_status_str: "Running"
     common_pod_nspace: openstack-operators
     common_pod_list:
@@ -42,9 +42,6 @@
       - logging-loki-query-frontend-grpc
       - logging-loki-query-frontend-http
       - openstack-logging
-    common_file_test_id: "RHOSO-12754"
-    common_file_list:
-      - /etc/rsyslog.d/10-telemetry.conf
   tasks:
     - name: "Verify logging infrastructure components"
       ansible.builtin.import_role:

diff --git a/ci/run_autoscaling_osp18.yml b/ci/run_autoscaling_osp18.yml
@@ -20,26 +20,6 @@
           tags:
             - setup
 
-        - name: Patch the openstackversions to use the master containers for aodh and heat
-          ansible.builtin.shell:
-            cmd: |
-              oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml
-            chdir: "{{ fvt_dir }}"
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
-        - name: Redeploy the dataplane
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml
-              oc delete osdpd edpm-deployment
-              sleep 10
-              oc apply -f /tmp/osdpd.yaml
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
         - name: Patch observabilityclient into openstackclient
           ansible.builtin.shell:
             cmd: |
@@ -50,50 +30,7 @@
           tags:
             - setup
 
-        - name: Wait until the oscp is resolved the changes to continue
-          ansible.builtin.shell:
-            cmd: |
-              oc get oscp | grep "Setup complete"
-          retries: 24
-          timeout: 5
-          until: output.stdout_lines | length == 1
-          register: output
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
-        - name: Wait until the osdpd is redeployed to continue
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd | grep "Setup complete"
-          retries: 60
-          delay: 45
-          until: output.stdout_lines | length == 1
-          register: output
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
         - name: "Run Telemetry Autoscaling tests"
           ansible.builtin.import_role:
             name: telemetry_autoscaling
           ignore_errors: true
-
-      always:
-        - name: Revert the version update
-          ansible.builtin.shell:
-            cmd: |
-              oc patch  openstackversions controlplane  --type json -p='[{"op": "replace", "path": "/spec/customContainerImages", "value": {} }]'
-          when: "{{ patch_openstackversions | bool }}"
-
-        - name: Redeploy the dataplane
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml
-              oc delete osdpd edpm-deployment
-              sleep 10
-              oc apply -f /tmp/osdpd.yaml
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
diff --git a/ci/use-master-containers.yml b/ci/use-master-containers.yml
@@ -0,0 +1,18 @@
+- name: Run telemetry autoscaling tests on osp18
+  hosts:  "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars_files:
+    - vars/common.yml
+    - vars/osp18_env.yml
+  tasks:
+    - name: Patch the openstackversions to use the master containers for aodh, heat and ceilometer
+      ansible.builtin.shell:
+        cmd: |
+          oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml
+        chdir: "{{ fvt_dir }}"
+      tags:
+        - setup
+
diff --git a/ci/vars-use-master-containers.yml b/ci/vars-use-master-containers.yml
@@ -0,0 +1,4 @@
+---
+post_ctlplane_deploy_99_modify_openstackversions:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/use-master-containers.yml"
+  type: playbook
diff --git a/roles/common/README.md b/roles/common/README.md
@@ -155,6 +155,7 @@ can be set at the play level.
                 - ceilometer_agent_compute
                 - ceilometer_agent_ipmi
                 - node_exporter
+                - kepler
 
       - name: "Verify Endpoint"
         ansible.builtin.import_role:

diff --git a/roles/telemetry_autoscaling/tasks/test_autoscaling.yml b/roles/telemetry_autoscaling/tasks/test_autoscaling.yml
@@ -59,6 +59,30 @@
   register: busy_process
   with_items: "{{ vnf_instance_ip.stdout_lines }}"
 
+- name: Show ceilometer_cpu metrics from Prometheus
+  when: metrics_backend == "prometheus"
+  block:
+    - name: Register autoscaling query results
+      shell: |
+        export STACK_ID=$({{ openstack_cmd }} stack show {{ stack_name }} -c id -f value)
+        {{ openstack_cmd }} metric query '(rate(ceilometer_cpu{server_group=~"$STACK_ID"}[150s]))/10000000'
+      ignore_errors: true
+      register: result
+
+    - name: Show the query result
+      ansible.builtin.debug:
+        var: result
+
+    - name: Register ceilometer_cpu metrics
+      shell: |
+        {{ openstack_cmd }} metric show ceilometer_cpu --disable-rbac
+      ignore_errors: true
+      register: result
+
+    - name: Show the ceilometer_cpu metrics
+      ansible.builtin.debug:
+        var: result
+
 - name: |
     TEST Verify cpu high alarm has been triggered
     RHOSO-12660

diff --git a/roles/telemetry_verify_metrics/README.md b/roles/telemetry_verify_metrics/README.md
@@ -24,13 +24,16 @@ Tests:
 - Verify Ceilometer metrics are being exposed and stored
     - Use openstack observabilityclient to verify Ceilometer central metrics are stored in Prometheus
     - Use openstack observabilityclient to verify Ceilometer compute metrics are stored in Prometheus
+    - Use openstack observabilityclient to verify Ceilometer ipmi metrics are stored in Prometheus
 - Verify NodeExporter metrics are being exposed and stored
     - Use openstack observabilityclient to verify NodeExporter metrics are stored in Prometheus
+- Verify Kepler metrics are being exposed and stored
+    - Use openstack observabilityclient to verify Kepler metrics are stored in Prometheus
 
 Role Variables
 --------------
 openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
-telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, node\_exporter, rabbitmq
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler
 
 Example Playbook
 ----------------

diff --git a/roles/telemetry_verify_metrics/defaults/main.yml b/roles/telemetry_verify_metrics/defaults/main.yml
@@ -4,3 +4,6 @@ telemetry_verify_metrics_metric_sources_to_test:
   - ceilometer_central_agent
   - node_exporter
   - rabbitmq
+  - kepler
+  # NOTE: IPMI metrics can only be verified when compute nodes are baremetal
+  # - ceilometer_ipmi_agent
diff --git a/roles/telemetry_verify_metrics/tasks/main.yml b/roles/telemetry_verify_metrics/tasks/main.yml
@@ -41,8 +41,22 @@
   tags: test
   when: '"ceilometer_central_agent" in telemetry_verify_metrics_metric_sources_to_test'
 
+- name: Verify Ceilometer ipmi metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_ipmi_metrics.yml
+  tags: power_monitoring
+  when: '"ceilometer_ipmi_agent" in telemetry_verify_metrics_metric_sources_to_test'
+
 - name: Verify NodeExporter metrics are being exposed and stored
   ansible.builtin.include_tasks:
     file: verify_node_exporter_metrics.yml
   tags: test
   when: '"node_exporter" in telemetry_verify_metrics_metric_sources_to_test'
+
+- name: Verify Kepler metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_kepler_metrics.yml
+  tags: 
+    - test
+    - power_monitoring
+  when: '"kepler" in telemetry_verify_metrics_metric_sources_to_test'
diff --git a/roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml b/roles/telemetry_verify_metrics/tasks/verify_ceilometer_ipmi_metrics.yml
@@ -0,0 +1,40 @@
+- name: Verify ceilometer scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1221
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ceilometer
+
+- name: Verify ceilometer ipmi agent container is up
+  ansible.builtin.include_tasks:
+    file: check_compute_node_containers.yml
+  vars:
+    common_container_test_id: RHOSO-1230
+    common_container_list:
+      - ceilometer_agent_ipmi
+  loop: "{{ groups['computes'] }}"
+  loop_control:
+    loop_var: compute_node
+
+- name: Verify ceilometer controlplane pod is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_test_id: RHOSO-1242
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - ceilometer-0
+
+- name: |
+    TEST Use openstack observabilityclient to verify ceilometer ipmi metrics are stored in prometheus
+    RHOSO-1217
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show ceilometer_hardware_ipmi_fan --disable-rbac
+  register: result
+  delay: 30
+  retries: 10
+  until: result.rc == 0 and "ceilometer_hardware_ipmi" in result.stdout
+  changed_when: false
diff --git a/roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml b/roles/telemetry_verify_metrics/tasks/verify_kepler_metrics.yml
@@ -0,0 +1,36 @@
+- name: Verify node exporter scrapeconfigs exist
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_test_id: RHOSO-1222
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-kepler
+    kepler_dashboard_metrics:
+      - kepler_vm_platform_joules_total
+      - kepler_node_platform_joules_total
+      - kepler_process_joules_total
+      - kepler_container_joules_total
+      - kepler_node_info
+
+- name: Verify Kepler container is up
+  ansible.builtin.include_tasks:
+    file: check_compute_node_containers.yml
+  vars:
+    common_container_test_id: RHOSO-1231
+    common_container_list:
+      - kepler
+  loop: "{{ groups['computes'] }}"
+  loop_control:
+    loop_var: compute_node
+
+- name: |
+    TEST Use openstack observabilityclient to verify kepler metrics are stored in prometheus
+    RHOSO-1216
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show --disable-rbac kepler_node_info
+  register: result
+  delay: 30
+  retries: 10
+  changed_when: false
+  until: result.rc == 0 and "kepler_node_info" in result.stdout