fix: start/stop pods: Do not crash if related Deployment / Statefulset does not exist

Author: desaintmartin

CHANGELOG.md

@@ -1,5 +1,9 @@
 # wiremind-kubernetes
 
+## v7.4.2 (2024-07-25)
+### Fix
+- start/stop pods: Do not crash if related Deployment / Statefulset does not exist
+
 ## v7.4.4 (2024-04-22)
 ### Chore
 - mark package as PEP 561-compliant for type hints

VERSION

@@ -1 +1 @@
-7.4.4
+7.4.5

src/wiremind_kubernetes/kubernetes_helper.py

@@ -133,7 +133,7 @@ def scale_down_statefulset(self, statefulset_name: str) -> None:
         self.client_appsv1_api.patch_namespaced_stateful_set_scale(statefulset_name, self.namespace, body)
         logger.debug("Done deleting.")
 
-    @retry_kubernetes_request_no_ignore
+    @retry_kubernetes_request
     def scale_down_deployment(self, deployment_name: str) -> None:
         body = self.get_deployment_scale(deployment_name)
         logger.debug("Deleting all Pods for %s", deployment_name)
@@ -148,7 +148,7 @@ def scale_up_statefulset(self, statefulset_name: str, pod_amount: int = 1) -> No
         self.client_appsv1_api.patch_namespaced_stateful_set_scale(statefulset_name, self.namespace, body)
         logger.debug("Done recreating.")
 
-    @retry_kubernetes_request_no_ignore
+    @retry_kubernetes_request
     def scale_up_deployment(self, deployment_name: str, pod_amount: int) -> None:
         body = self.get_deployment_scale(deployment_name)
         logger.debug("Recreating backend Pods for %s", deployment_name)
@@ -183,7 +183,12 @@ def _get_pods_from_deployment(self, deployment_name: str, statefulset: bool = Fa
                 raise
 
     def is_deployment_stopped(self, deployment_name: str, statefulset: bool = False) -> bool:
-        pod_list: List = self._get_pods_from_deployment(deployment_name, statefulset)
+        try:
+            pod_list: List = self._get_pods_from_deployment(deployment_name, statefulset)
+        except kubernetes.client.rest.ApiException as e:
+            if e.status == 404:
+                logger.warning("Not found, ignoring.")
+                return True
 
         current_scale = 0
         for pod in pod_list:
@@ -197,10 +202,16 @@ def is_deployment_stopped(self, deployment_name: str, statefulset: bool = False)
         return True
 
     def is_deployment_ready(self, deployment_name: str, statefulset: bool = False) -> bool:
-        if statefulset:
-            status = self.client_appsv1_api.read_namespaced_stateful_set_status(deployment_name, self.namespace)
-        else:
-            status = self.client_appsv1_api.read_namespaced_deployment_status(deployment_name, self.namespace)
+        try:
+            if statefulset:
+                status = self.client_appsv1_api.read_namespaced_stateful_set_status(deployment_name, self.namespace)
+            else:
+                status = self.client_appsv1_api.read_namespaced_deployment_status(deployment_name, self.namespace)
+        except kubernetes.client.rest.ApiException as e:
+            if e.status == 404:
+                logger.warning("Not found, ignoring.")
+                return True
+
         expected_replicas = status.spec.replicas
         ready_replicas = status.status.ready_replicas
         resource_type = statefulset and "StatefulSet" or "Deployment"

src/wiremind_kubernetes/tests/e2e_tests/manifests/2_edss.yml

@@ -61,6 +61,19 @@ spec:
 
 ---
 
+apiVersion: "wiremind.io/v1"
+kind: ExpectedDeploymentScale
+metadata:
+  name: concerned-but-without-related-deployment
+  labels:
+    app.kubernetes.io/instance: concerned
+spec:
+  deploymentName: concerned-but-without-related-deployment
+  expectedScale: 1
+  priority: 10
+
+---
+
 apiVersion: "wiremind.io/v1"
 kind: ExpectedDeploymentScale
 metadata: