Merge pull request #195 from omeryahud/backport-operator-upgrade

[Backport] Merge pull request #186 - Operator upgrade path

Author: ksimon1

_defaults.yml

@@ -27,3 +27,4 @@ virt_launcher_tag: "{{ lookup('env','VIRT_LAUNCHER_TAG')| default('v0.21.0', tru
 validator_tag: "{{ lookup('env','VALIDATOR_TAG')| default('v0.6.6', true) }}"
 image_name_prefix: "{{ lookup('env','IMAGE_NAME_PREFIX')| default('', true) }}"
 templates_version: v0.11.2
+operator_version: "{{ lookup('env', 'OPERATOR_VERSION') }}"

build/csv-generator.sh

@@ -21,12 +21,13 @@ replace_env_var() {
 }
 
 help_text() {
-    echo "USAGE: csv-generator --csv-version=<version> --namespace=<namespace> --operator-image=<operator image> [optional args]"
+    echo "USAGE: csv-generator --csv-version=<version> --namespace=<namespace> --operator-image=<operator image> --operator-version=<operator version> [optional args]"
     echo ""
     echo "ARGS:"
     echo "  --csv-version:    (REQUIRED) The version of the CSV file"
     echo "  --namespace:      (REQUIRED) The namespace set on the CSV file"
     echo "  --operator-image: (REQUIRED) The operator container image to use in the CSV file"
+    echo "  --operator-version: (REQUIRED) The version of the operator to use in the CSV file"
     echo "  --watch-namespace:   (OPTIONAL)"
     echo "  --kvm-info-tag:      (OPTIONAL)"
     echo "  --validator-tag:     (OPTIONAL)"
@@ -41,6 +42,7 @@ help_text() {
 CSV_VERSION=""
 NAMESPACE=""
 OPERATOR_IMAGE=""
+OPERATOR_VERSION=""
 
 # OPTIONAL ARGS
 WATCH_NAMESPACE=""
@@ -67,6 +69,9 @@ while (( "$#" )); do
     --operator-image)
         OPERATOR_IMAGE=$VAL
         ;;
+    --operator-version)
+        OPERATOR_VERSION=$VAL
+        ;;
     --watch-namespace)
         WATCH_NAMESPACE=$VAL
         ;;
@@ -101,7 +106,7 @@ while (( "$#" )); do
     esac
 done
 
-if [ -z "$CSV_VERSION" ] || [ -z "$NAMESPACE" ] || [ -z "$OPERATOR_IMAGE" ]; then
+if [ -z "$CSV_VERSION" ] || [ -z "$NAMESPACE" ] || [ -z "$OPERATOR_IMAGE" ] || [ -z "$OPERATOR_VERSION" ]; then
     echo "Error: Missing required arguments"
     help_text
     exit 1
@@ -114,6 +119,7 @@ cp ${MANIFESTS_GENERATED_CSV} ${TMP_FILE}
 sed -i "s/PLACEHOLDER_CSV_VERSION/${CSV_VERSION}/g" ${TMP_FILE}
 sed -i "s/namespace: placeholder/namespace: ${NAMESPACE}/g" ${TMP_FILE}
 sed -i "s|REPLACE_IMAGE|${OPERATOR_IMAGE}|g" ${TMP_FILE}
+sed -i "s|REPLACE_VERSION|${OPERATOR_VERSION}|g" ${TMP_FILE}
 
 replace_env_var "WATCH_NAMESPACE" $WATCH_NAMESPACE
 replace_env_var "KVM_INFO_TAG" $KVM_INFO_TAG

deploy/operator.yaml

@@ -50,3 +50,5 @@ spec:
               value: ""
             - name: OPERATOR_NAME
               value: "kubevirt-ssp-operator"
+            - name: OPERATOR_VERSION
+              value: REPLACE_VERSION

deploy/role.yaml

@@ -16,6 +16,7 @@ rules:
   - patch
   - update
   - watch
+  - delete
 - apiGroups:
   - monitoring.coreos.com
   resources:
@@ -26,6 +27,7 @@ rules:
   - list
   - patch
   - watch
+  - delete
 - apiGroups:
   - monitoring.coreos.com
   resources:
@@ -67,6 +69,7 @@ rules:
   - list
   - patch
   - watch
+  - delete
 - apiGroups:
   - ""
   resources:
@@ -81,6 +84,7 @@ rules:
   - patch
   - list
   - watch
+  - delete
 - apiGroups:
   - ""
   resources:

hack/make-manifests.sh

@@ -12,7 +12,7 @@ CHANNEL="beta"
 CLUSTER_VERSIONED_DIR="cluster/${VERSION}"
 MANIFESTS_DIR="manifests/kubevirt-ssp-operator"
 MANIFESTS_VERSIONED_DIR="${MANIFESTS_DIR}/${TAG}"
-IMAGE_PATH="quay.io/fromani/kubevirt-ssp-operator-container:latest"
+IMAGE_PATH="quay.io/fromani/kubevirt-ssp-operator-container:${TAG}"
 
 HAVE_COURIER=0
 if which operator-courier &> /dev/null; then
@@ -39,11 +39,11 @@ done
 (
 for MF in deploy/service_account.yaml deploy/role.yaml deploy/role_binding.yaml deploy/operator.yaml; do
 	echo "---"
-	sed "s|REPLACE_IMAGE|${IMAGE_PATH}|" < ${MF}
+	sed "s|REPLACE_IMAGE|${IMAGE_PATH}|g ; s|REPLACE_VERSION|${TAG}|g" < ${MF}
 done
 ) > ${CLUSTER_VERSIONED_DIR}/kubevirt-ssp-operator.yaml
 
-${BASEPATH}/../build/csv-generator.sh --csv-version=${VERSION} --namespace=placeholder --operator-image=REPLACE_IMAGE > ${MANIFESTS_VERSIONED_DIR}/kubevirt-ssp-operator.${TAG}.clusterserviceversion.yaml
+${BASEPATH}/../build/csv-generator.sh --csv-version=${VERSION} --namespace=placeholder --operator-image=REPLACE_IMAGE --operator-version=REPLACE_VERSION > ${MANIFESTS_VERSIONED_DIR}/kubevirt-ssp-operator.${TAG}.clusterserviceversion.yaml
 
 # caution: operator-courier (as in 5a4852c) wants *one* entity per yaml file (e.g. it does NOT use safe_load_all)
 for CRD in $( ls deploy/crds/kubevirt_*crd.yaml ); do

roles/ClaimOwnership/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# tasks file for ClaimOwnership
+- name: Claim Ownership
+  k8s: 
+    state: present
+    resource_definition:
+      apiVersion: "{{ object.apiVersion }}"
+      kind: "{{ object.kind }}"
+      metadata:
+        name: "{{ object.metadata.name }}"
+        namespace: "{{ object.metadata.namespace }}"
+        ownerReferences:
+        - apiVersion: "{{ owner.apiVersion }}"
+          kind: "{{ owner.kind }}"
+          name: "{{ owner.metadata.name }}"
+          uid: "{{ owner.metadata.uid }}"

roles/KubevirtCircuitBreaker/tasks/main.yml

@@ -2,7 +2,7 @@
 # tasks file for KubevirtCircuitBreaker
 - name: Extract the CR info
   set_fact:
-    cr_info: "{{ lookup('k8s', api_version='v1', kind=cr_kind, namespace=meta.namespace, resource_name=meta.name) | from_yaml }}"
+    cr_info: "{{ lookup('k8s', api_version='ssp.kubevirt.io/v1', kind=cr_kind, namespace=meta.namespace, resource_name=meta.name) | from_yaml }}"
 - name: Extract the disable info
   set_fact:
     is_paused: "{{ cr_info['metadata'].get('annotations', {})['kubevirt.io/operator.paused'] | default('false', true) | from_json }}"

roles/KubevirtCommonTemplatesBundle/filter_plugins/k8s_owned_by.py

@@ -0,0 +1,40 @@
+from ansible.errors import AnsibleError
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'k8s_owned_by': k8s_owned_by
+        }
+
+def k8s_owned_by(objects, owner):
+    owned = []
+
+    for obj in objects:
+        if object_owned_by(obj, owner):
+            owned.append(obj)
+    
+    return owned
+
+def object_owned_by(object, owner):
+    if owner is None:
+        raise AnsibleError('owner is empty')
+    if 'metadata' not in owner:
+        raise AnsibleError('owner is missing "metadata" field')
+    if 'uid' not in owner['metadata']:
+        raise AnsibleError('owner is missing "metadata.uid" field')
+
+    if object is None:
+        raise AnsibleError('object is empty')
+    if 'metadata' not in object:
+        raise AnsibleError('object is missing "metadata" field')
+    if 'ownerReferences' not in object['metadata']:
+        return False
+
+    ownerUID = owner['metadata']['uid']
+
+    if object['metadata']['ownerReferences'] is not None:
+        for ref in object['metadata']['ownerReferences']:
+            if ref['uid'] == ownerUID:
+                return True
+    
+    return False

roles/KubevirtCommonTemplatesBundle/tasks/main.yml

@@ -1,46 +1,113 @@
 ---
+- name: Set operatorVersion and targetVersion
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    status:
+      operatorVersion: "{{ operator_version }}"
+      targetVersion: "{{ operator_version }}"
+
 # tasks file for KubevirtCommonTemplatesBundle
 - name: Install VM templates
   k8s:
     state: present
-    namespace: "{{ meta.namespace }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
     definition: "{{ item | from_yaml }}"
+    apply: yes
   with_items: "{{ lookup('file', 'common-templates-'+ version +'.yaml').split('\n---\n') | select('search', '(^|\n)[^#]') | list }}"
   register: ct_status
 
+# Get all templates
+- name: Fetching all templates
+  set_fact:
+    templates: "{{ lookup('k8s', api_version=ct_status.results[0].result.apiVersion, kind='template') }}"
+
+- name: Fetch old cr
+  block:
+  - set_fact:
+      old_cr: "{{ lookup('k8s', api_version='kubevirt.io/v1', kind='KubevirtCommonTemplatesBundle') }}"
+  - set_fact:
+      old_cr_exists: true
+  rescue:
+  - set_fact:
+      old_cr_exists: false
+
+- name: Filter for templates owned by the old cr
+  set_fact:
+    old_cr_templates: "{{ templates | k8s_owned_by(old_cr) }}"
+  when: "{{ old_cr_exists==true }}"
+
+# Inject ownerReferences
+- name: Inject owner references for KubevirtCommonTemplatesBundle
+  include_role: 
+    name: ClaimOwnership
+  vars:
+    object: "{{ item }}"
+    owner: "{{ cr_info }}"
+  when: "{{ old_cr_exists==true }}"
+  with_items: "{{ old_cr_templates }}" # Templates
+
 - name: "Count all new templates in file"
   set_fact:
     new_templates: "{{ lookup('file', 'common-templates-'+ version +'.yaml').split('\n---\n') | select('search', '(^|\n)[^#]') | list | length }}"
 
-
 - name: "Set label"
   set_fact:
     label: "template.kubevirt.io/version={{ version }}"
-  
+
 - name: "Get all templates"
   set_fact:
-    deployed_templates_after: "{{ lookup('k8s', api_version=ct_status.results[0].result.apiVersion, kind='template', label_selector=label)|length }}"
+    deployed_templates_after: "{{ lookup('k8s', api_version=ct_status.results[0].result.apiVersion, kind='template', label_selector=label) | length }}"
+
+- name: "Set Available status"
+  set_fact:
+    available: "{{ true if new_templates <= deployed_templates_after else false }}"
 
 - name: Set progressing condition
   operator_sdk.util.k8s_status:
-    api_version: ssp.kubevirt.io/v1
-    kind: KubevirtCommonTemplatesBundle
-    name: "{{ meta.name }}"
-    namespace: "{{ meta.namespace }}"
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
     conditions:
     - type: Progressing
-      status: "{{ 'True' if new_templates > deployed_templates_after else 'False' }}"
+      status: "False"
       reason: "progressing"
-      message: "Templates progressing."
+      message: "Templates progressing (deployed templates: {{ deployed_templates_after }}, desired deployed templated: {{ new_templates }})."
 
 - name: Set available condition
   operator_sdk.util.k8s_status:
-    api_version: ssp.kubevirt.io/v1
-    kind: KubevirtCommonTemplatesBundle
-    name: "{{ meta.name }}"
-    namespace: "{{ meta.namespace }}"
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
     conditions:
     - type: Available
-      status: "{{ 'True' if new_templates <= deployed_templates_after else 'False' }}"
+      status: "True"
       reason: "available"
-      message: "Common templates available."
+      message: "Common templates available (deployed templates: {{ deployed_templates_after }}, desired deployed templated: {{ new_templates }})."
+
+- name: Set degraded condition
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    conditions:
+    - type: Degraded
+      status: "False"
+      reason: "degraded"
+      message: "Templates degraded (deployed templates: {{ deployed_templates_after }}, desired deployed templated: {{ new_templates }})."
+
+# Update observerVersion when the CR is Available to indicate a successfull upgrade
+- name: Set observedVersion
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    status:
+      observedVersion: "{{ operator_version }}"
+  when: "{{ available==true }}"

roles/KubevirtMetricsAggregation/tasks/main.yml

@@ -1,9 +1,75 @@
 ---
+- name: Set operatorVersion and targetVersion
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    status:
+      operatorVersion: "{{ operator_version }}"
+      targetVersion: "{{ operator_version }}"
+
 # tasks file for KubevirtMetricsAggregation
 - name: Install VMI count aggregation rule
   k8s:
     state: present
     namespace: "{{ meta.namespace }}"
     definition: "{{ item | from_yaml }}"
+    apply: yes
   with_items: "{{ lookup('template', 'aggregation-rule-vmi-count.yaml.j2').split('\n---\n') | select('search', '(^|\n)[^#]') |list }}"
+  register: promrules
+
+- name: Inject owner references for KubevirtMetricsAggregation
+  include_role:
+    name: ClaimOwnership
+  vars:
+    object: "{{ item.result }}"
+    owner: "{{ cr_info }}"
+  with_items: "{{ promrules.results }}"
+
+- name: Set available condition
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    conditions:
+    - type: Available
+      status: "True"
+      reason: "available"
+      message: "KubevirtMetricsAggregation is available."
+
+# There is no logic for the Progressing/Degraded conditions for KubevirtMetricsAggregation
+# so using these constant conditions
+- name: Set progressing condition
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    conditions:
+    - type: Progressing
+      status: "False"
+      reason: "progressing"
+      message: "KubevirtMetricsAggregation progressing"
+
+- name: Set degraded condition
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    conditions:
+    - type: Degraded
+      status: "False"
+      reason: "degraded"
+      message: "KubevirtMetricsAggregation degraded"
 
+- name: Set observedVersion
+  operator_sdk.util.k8s_status:
+    api_version: "{{ cr_info.apiVersion }}"
+    kind: "{{ cr_info.kind }}"
+    name: "{{ cr_info.metadata.name }}"
+    namespace: "{{ cr_info.metadata.namespace }}"
+    status:
+      observedVersion: "{{ operator_version }}"