DNM Checking CRC Cloud with image based on OCP 4.16 rc4

This is early test PR that uses new way for deploying the CRC -
crc-cloud [1] and image that is base on OCP 4.16 rc4
with small modifications [2]. In the future, that image would be created
by the nodepool-builder, but right now it is just a PoC (Proof of concept).
Also is worth to mention, that the base job includes few workarounds
until the PR in the official projects are not merged [3][4][5]

[1] https://github.com/crc-org/crc-cloud
[2] https://softwarefactory-project.io/r/c/software-factory/sf-infra/+/31853
[3] crc-org/crc-cloud#188
[4] crc-org/crc-cloud#189
[5] crc-org/crc-cloud#190

Author: danpawlik

ci/playbooks/collect-logs.yml

@@ -119,6 +119,8 @@
               cp /etc/containers/registries.conf {{ ansible_user_dir }}/zuul-output/logs/
               cp -r /etc/containers/registries.conf.d {{ ansible_user_dir }}/zuul-output/logs/
               chown -R "{{ ansible_user }}" {{ ansible_user_dir }}/zuul-output/logs/registries.conf.d
+              cp -r {{ ansible_user_dir }}/workdir/*log {{ ansible_user_dir }}/zuul-output/logs/crc-cloud/
+              chown -R "{{ ansible_user }}" {{ ansible_user_dir }}/zuul-output/logs/crc-cloud/
 
         - name: Copy generated documentation if available
           when:

zuul.d/base.yaml

@@ -147,6 +147,92 @@
       - ci/playbooks/collect-logs.yml
       - ci/playbooks/multinode-autohold.yml
     vars: &multinode_edpm_vars
+      wait_for_mcp: true
+      zuul_log_collection: true
+      registry_login_enabled: true
+      push_registry: quay.rdoproject.org
+      quay_login_secret_name: quay_nextgen_zuulgithubci
+      cifmw_artifacts_crc_sshkey: "~/.ssh/id_cifw"
+      cifmw_openshift_user: kubeadmin
+      cifmw_openshift_password: "123456789"
+      cifmw_openshift_api: api.crc.testing:6443
+      cifmw_openshift_kubeconfig: "{{ ansible_user_dir }}/.crc/machines/crc/kubeconfig"
+      cifmw_openshift_skip_tls_verify: true
+      cifmw_use_libvirt: false
+      cifmw_zuul_target_host: controller
+      crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}"
+      crc_ci_bootstrap_networking:
+        networks:
+          default:
+            mtu: "{{ ('ibm' in nodepool.cloud) | ternary('1440', '1500') }}"
+            router_net: "{{ ('ibm' in nodepool.cloud) | ternary('hostonly', 'public') }}"
+            range: 192.168.122.0/24
+          internal-api:
+            vlan: 20
+            range: 172.17.0.0/24
+          storage:
+            vlan: 21
+            range: 172.18.0.0/24
+          tenant:
+            vlan: 22
+            range: 172.19.0.0/24
+        instances:
+          controller:
+            networks:
+              default:
+                ip: 192.168.122.11
+          crc:
+            networks:
+              default:
+                ip: 192.168.122.10
+              internal-api:
+                ip: 172.17.0.5
+              storage:
+                ip: 172.18.0.5
+              tenant:
+                ip: 172.19.0.5
+          compute-0:
+            networks:
+              default:
+                ip: 192.168.122.100
+              internal-api:
+                ip: 172.17.0.100
+                config_nm: false
+              storage:
+                ip: 172.18.0.100
+                config_nm: false
+              tenant:
+                ip: 172.19.0.100
+                config_nm: false
+
+- job:
+    name: cifmw-podified-multinode-edpm-base-crc-cloud
+    parent: base-crc-cloud
+    timeout: 10800
+    attempts: 1
+    nodeset: centos-9-medium-centos-9-crc-extracted-2-39-0-3xl
+    irrelevant-files: *ir_files
+    required-projects: &multinode_edpm_rp
+      - openstack-k8s-operators/ci-framework
+      - openstack-k8s-operators/install_yamls
+      - openstack-k8s-operators/infra-operator
+      - openstack-k8s-operators/openstack-baremetal-operator
+      - openstack-k8s-operators/openstack-must-gather
+      - openstack-k8s-operators/openstack-operator
+      - openstack-k8s-operators/repo-setup
+      - openstack-k8s-operators/edpm-ansible
+    roles: &multinode_edpm_roles
+      - zuul: github.com/openstack-k8s-operators/ci-framework
+    pre-run: &multinode_edpm_pre_run
+      - ci/playbooks/multinode-customizations.yml
+      - ci/playbooks/e2e-prepare.yml
+      - ci/playbooks/dump_zuul_data.yml
+    post-run: &multinode_edpm_post_run
+      - ci/playbooks/e2e-collect-logs.yml
+      - ci/playbooks/collect-logs.yml
+      - ci/playbooks/multinode-autohold.yml
+    vars: &multinode_edpm_vars
+      wait_for_mcp: true
       zuul_log_collection: true
       registry_login_enabled: true
       push_registry: quay.rdoproject.org
@@ -306,3 +392,18 @@
       - ci/playbooks/e2e-prepare.yml
       - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/pre-ci-bootstrap.yml
+
+- job:
+    name: cifmw-podified-multinode-edpm-base-crc-cloud
+    parent: base-crc-cloud
+    timeout: 10800
+    attempts: 1
+    nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-16-3xl
+    irrelevant-files: *ir_files
+    required-projects: *multinode_edpm_rp
+    roles: *multinode_edpm_roles
+    pre-run: *multinode_edpm_pre_run
+    post-run: *multinode_edpm_post_run
+    vars:
+      <<: *multinode_edpm_vars
+      wait_for_mcp: true

zuul.d/edpm_multinode.yaml

@@ -144,7 +144,7 @@
 - job:
     name: podified-multinode-hci-deployment-crc-3comp
     parent: podified-multinode-edpm-deployment-crc
-    nodeset: centos-9-medium-3x-centos-9-crc-extracted-2-39-0-xxl
+    nodeset: centos-9-medium-3x-centos-9-crc-cloud-ocp-4-16-xxl
     vars:
       cifmw_edpm_deploy_hci: true
       crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}"
@@ -312,7 +312,10 @@
 
 - job:
     name: podified-multinode-edpm-deployment-crc
-    parent: cifmw-podified-multinode-edpm-base-crc
+    parent: base-crc-cloud
+    timeout: 10800
+    attempts: 1
+    nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-16-3xl
     vars:
       cifmw_extras:
         - '@scenarios/centos-9/multinode-ci.yml'

zuul.d/kuttl_multinode.yaml

@@ -51,7 +51,79 @@
 
 - job:
     name: cifmw-multinode-kuttl
-    parent: cifmw-base-multinode-kuttl
+    parent: cifmw-base-multinode-kuttl-crc-cloud
+    files:
+      - ^ci/playbooks/kuttl/.*
+      - ^scenarios/centos-9/kuttl.yml
+      - ^zuul.d/kuttl.yaml
+    vars:
+      cifmw_extras:
+        - '@scenarios/centos-9/kuttl_multinode.yml'
+      cifmw_kuttl_tests_operator_list:
+        - openstack
+        - barbican
+        - keystone
+        - horizon
+      commands_before_kuttl_run:
+        - oc get pv
+        - oc get all
+      commands_after_kuttl_run:
+        - oc get pv
+        - oc get all
+
+- job:
+    name: cifmw-base-multinode-kuttl-crc-cloud
+    parent: cifmw-podified-multinode-edpm-base-crc-cloud
+    timeout: 7200
+    abstract: true
+    nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-16-3xl
+    vars:
+      zuul_log_collection: true
+    extra-vars:
+      crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}"
+      crc_ci_bootstrap_networking:
+        networks:
+          default:
+            range: 192.168.122.0/24
+            mtu: "{{ ('ibm' in nodepool.cloud) | ternary('1440', '1500') }}"
+            router_net: "{{ ('ibm' in nodepool.cloud) | ternary('hostonly', 'public') }}"
+          internal-api:
+            vlan: 20
+            range: 172.17.0.0/24
+          storage:
+            vlan: 21
+            range: 172.18.0.0/24
+          tenant:
+            vlan: 22
+            range: 172.19.0.0/24
+        instances:
+          controller:
+            networks:
+              default:
+                ip: 192.168.122.11
+          crc:
+            networks:
+              default:
+                ip: 192.168.122.10
+              internal-api:
+                ip: 172.17.0.5
+              storage:
+                ip: 172.18.0.5
+              tenant:
+                ip: 172.19.0.5
+    pre-run:
+      - ci/playbooks/e2e-prepare.yml
+    run:
+      - ci/playbooks/dump_zuul_data.yml
+      - ci/playbooks/kuttl/run.yml
+    post-run:
+      - ci/playbooks/collect-logs.yml
+    required-projects:
+      - github.com/openstack-k8s-operators/install_yamls
+
+- job:
+    name: cifmw-multinode-kuttl
+    parent: cifmw-base-multinode-kuttl-crc-cloud
     files:
       - ^ci/playbooks/kuttl/.*
       - ^scenarios/centos-9/kuttl.yml

zuul.d/nodeset.yaml

@@ -279,3 +279,43 @@
     nodes:
       - name: controller
         label: centos-9-stream-crc-2-39-0-xl
+
+- nodeset:
+    name: centos-9-medium-centos-9-crc-cloud-ocp-4-16-3xl
+    nodes:
+      - name: controller
+        label: cloud-centos-9-stream-tripleo-medium
+      - name: compute-0
+        label: cloud-centos-9-stream-tripleo
+      - name: crc
+        label: crc-cloud-ocp-4-16-0-3xl
+    groups:
+      - name: computes
+        nodes:
+          - compute-0
+      - name: ocps
+        nodes:
+          - crc
+
+- nodeset:
+    name: centos-9-medium-3x-centos-9-crc-cloud-ocp-4-16-xxl
+    nodes:
+      - name: controller
+        label: cloud-centos-9-stream-tripleo-medium
+      - name: compute-0
+        label: cloud-centos-9-stream-tripleo
+      - name: compute-1
+        label: cloud-centos-9-stream-tripleo
+      - name: compute-2
+        label: cloud-centos-9-stream-tripleo
+      - name: crc
+        label: crc-cloud-ocp-4-16-0-3xl
+    groups:
+      - name: computes
+        nodes:
+          - compute-0
+          - compute-1
+          - compute-2
+      - name: ocps
+        nodes:
+          - crc

zuul.d/tempest_multinode.yaml

@@ -117,6 +117,7 @@
 - job:
     name: cifmw-multinode-tempest
     parent: podified-multinode-edpm-deployment-crc-tempest
+    nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-16-3xl
     files:
       - ^roles/test_operator
       - ^scenarios/centos-9/multinode-ci.yml