DS-392 Flaky Integrationtest Fix

test/integration/steps/functions/aws/cloudwatch.py

@@ -1,3 +1,4 @@
+import random
 from datetime import datetime
 from json import dumps
 from os import getenv
@@ -15,8 +16,8 @@ def get_logs(
     query: str,
     lambda_name: str,
     start_time: Timestamp | None = None,
-    retry_count: int = 32,
-    sleep_per_loop: int = 20,
+    retry_count: int = 32,  # Retry Count
+    sleep_per_loop: int = 20,  # Sleep time between retries.
 ) -> str:
     """Get logs from CloudWatch.
 
@@ -44,15 +45,19 @@ def get_logs(
             )
         except ClientError as error:
             if error.response["Error"]["Code"] == "LimitExceededException":
-                limit_exceeded_timeout += 30
+                limit_exceeded_timeout += random.uniform(30, 60) * (2**counter)  # Exponential backoff with jitter
+                print(f"Limit exceeded, retrying after {limit_exceeded_timeout} seconds.")
                 sleep(limit_exceeded_timeout)
                 continue
             raise
         query_id = start_query_response["queryId"]
         response = None
+
+        # Wait for the query to complete without a max timeout
         while response is None or response["status"] != "Complete":
             sleep(sleep_per_loop)
             response = LAMBDA_CLIENT_LOGS.get_query_results(queryId=query_id)
+
         counter += 1
         if response["results"] != []:
             logs_found = True

test/integration/steps/functions/utils.py

@@ -1,3 +1,5 @@
+import secrets
+import string
 from json import dumps, loads
 from os import getenv
 from random import randint, randrange
@@ -274,3 +276,9 @@ def quality_checker_negative_log_check(
 | filter message="{reason}"
 | sort @timestamp asc"""
     return negative_log_check(query=query, event_lambda="quality-checker", start_time=start_time)
+
+
+def generate_unique_ods_code() -> str:
+    """Generate a unique 5-character uppercase alphanumeric ODSCode."""
+    chars = string.ascii_uppercase + string.digits  # A-Z, 0-9
+    return "".join(secrets.choice(chars) for _ in range(5))  # 5-character random string

test/integration/steps/test_steps.py

@@ -70,6 +70,7 @@
     create_pending_change_for_service,
     generate_correlation_id,
     generate_random_int,
+    generate_unique_ods_code,
     get_address_string,
     get_expected_data,
     quality_checker_log_check,
@@ -136,7 +137,7 @@ def a_service_table_entry_is_created(context: Context, ods_code: int = 0, servic
         Context: The context object.
     """
     if ods_code == 0:
-        ods_code = str(randint(10000, 99999))
+        ods_code = generate_unique_ods_code()
     query_values = {
         "uid": f"test{randint(10000, 99999)!s}",
         "service_type": service_type,