Modify tests

Author: Zahed-Riyaz

apps/challenges/aws_utils.py

@@ -1914,20 +1914,16 @@ def calculate_retention_period_days(challenge_end_date, challenge=None):
             # Default 30-day retention when host has consented
             return 30
 
-    # No host consent - use conservative default (longer retention)
-    # Default retention calculation (90 days after challenge ends for safety)
+    # No host consent - use indefinite retention (no automatic cleanup)
+    # Without consent, data is retained indefinitely for safety
     if challenge_end_date > now:
-        # Challenge is still active, retain until end date + 90 days
-        # Round up to the nearest day to avoid flakiness
-        seconds_until_end = (challenge_end_date - now).total_seconds()
-        days_until_end = math.ceil(seconds_until_end / (24 * 3600.0))
-        return int(days_until_end) + 90
+        # Challenge is still active, retain indefinitely
+        # Return a very large number to effectively make it indefinite
+        return 3653  # Maximum AWS CloudWatch retention period (10 years)
     else:
-        # Challenge has ended, retain for 90 more days
-        # Round down to match original behavior of .days
-        seconds_since_end = (now - challenge_end_date).total_seconds()
-        days_since_end = math.floor(seconds_since_end / (24 * 3600.0))
-        return max(90 - int(days_since_end), 1)  # At least 1 day
+        # Challenge has ended, retain indefinitely
+        # Return maximum retention period
+        return 3653  # Maximum AWS CloudWatch retention period (10 years)
 
 
 def map_retention_days_to_aws_values(days):
@@ -1993,7 +1989,7 @@ def set_cloudwatch_log_retention(challenge_pk, retention_days=None):
             return {
                 "error": f"Challenge {challenge_pk} host has not consented to retention policy. "
                 "Please obtain consent before applying retention policies. "
-                "Without consent, data is retained for 90 days for safety.",
+                "Without consent, data is retained indefinitely for safety.",
                 "requires_consent": True,
                 "challenge_id": challenge_pk,
             }
@@ -2045,7 +2041,7 @@ def set_cloudwatch_log_retention(challenge_pk, retention_days=None):
             "retention_days": aws_retention_days,
             "log_group": log_group_name,
             "message": f"Retention policy set to {aws_retention_days} days "
-            f"({'30-day policy applied' if challenge_obj.retention_policy_consent else '90-day safety retention'})",
+            f"({'30-day policy applied' if challenge_obj.retention_policy_consent else 'indefinite retention (no consent)'})",
             "host_consent": challenge_obj.retention_policy_consent,
         }
 
@@ -2076,7 +2072,7 @@ def calculate_submission_retention_date(challenge_phase):
         challenge_phase: ChallengePhase object
 
     Returns:
-        datetime: Date when submission artifacts can be deleted
+        datetime: Date when submission artifacts can be deleted, or None if indefinite retention
     """
     from datetime import timedelta
 
@@ -2092,17 +2088,14 @@ def calculate_submission_retention_date(challenge_phase):
 
     # Check if challenge has host consent
     if challenge.retention_policy_consent:
-        # Use challenge-level retention policy
+        # Use challenge-level retention policy (30 days)
         retention_days = calculate_retention_period_days(
             challenge_phase.end_date, challenge
         )
+        return challenge_phase.end_date + timedelta(days=retention_days)
     else:
-        # No host consent, use default retention period
-        retention_days = calculate_retention_period_days(
-            challenge_phase.end_date, challenge
-        )
-
-    return challenge_phase.end_date + timedelta(days=retention_days)
+        # No host consent - indefinite retention (no automatic cleanup)
+        return None
 
 
 def delete_submission_files_from_storage(submission):
@@ -2209,7 +2202,9 @@ def cleanup_expired_submission_artifacts():
     # Find submissions eligible for cleanup
     now = timezone.now()
     eligible_submissions = Submission.objects.filter(
-        retention_eligible_date__lte=now, is_artifact_deleted=False
+        retention_eligible_date__lte=now,
+        retention_eligible_date__isnull=False,  # Exclude indefinite retention
+        is_artifact_deleted=False,
     ).select_related("challenge_phase__challenge")
 
     cleanup_stats = {
@@ -2307,36 +2302,26 @@ def update_submission_retention_dates():
 
         for phase in ended_phases:
             try:
-                # Process submissions by type
-                for submission_type in [
-                    "participant",
-                    "host",
-                    "baseline",
-                    "evaluation_output",
-                ]:
-                    retention_date = calculate_submission_retention_date(
-                        phase, submission_type
-                    )
-                    if retention_date:
-                        # Update submissions for this phase and type
-                        submissions_updated = Submission.objects.filter(
-                            challenge_phase=phase,
-                            submission_type=submission_type,
-                            retention_eligible_date__isnull=True,
-                            is_artifact_deleted=False,
-                        ).update(retention_eligible_date=retention_date)
-
-                        updated_count += submissions_updated
-
-                        if submissions_updated > 0:
-                            logger.info(
-                                f"Updated {submissions_updated} {submission_type} submissions for phase {phase.pk} "
-                                f"({phase.challenge.title}) with retention date {retention_date}"
-                            )
-                    else:
-                        logger.debug(
-                            f"No retention date calculated for phase {phase.pk} submission type {submission_type} - phase may still be public"
+                retention_date = calculate_submission_retention_date(phase)
+                if retention_date:
+                    # Update submissions for this phase
+                    submissions_updated = Submission.objects.filter(
+                        challenge_phase=phase,
+                        retention_eligible_date__isnull=True,
+                        is_artifact_deleted=False,
+                    ).update(retention_eligible_date=retention_date)
+
+                    updated_count += submissions_updated
+
+                    if submissions_updated > 0:
+                        logger.info(
+                            f"Updated {submissions_updated} submissions for phase {phase.pk} "
+                            f"({phase.challenge.title}) with retention date {retention_date}"
                         )
+                else:
+                    logger.debug(
+                        f"No retention date calculated for phase {phase.pk} - phase may still be public or indefinite retention"
+                    )
 
             except Exception as e:
                 error_msg = f"Failed to update retention dates for phase {phase.pk}: {str(e)}"
@@ -2480,6 +2465,7 @@ def weekly_retention_notifications_and_consent_log():
     warning_date = timezone.now() + timedelta(days=14)
     warning_submissions = Submission.objects.filter(
         retention_eligible_date__date=warning_date.date(),
+        retention_eligible_date__isnull=False,  # Exclude indefinite retention
         is_artifact_deleted=False,
     ).select_related("challenge_phase__challenge__creator")
 

apps/challenges/management/commands/manage_retention.py

@@ -247,8 +247,6 @@ def add_arguments(self, parser):
             help="Limit number of results (default: 50)",
         )
 
-
-
         # Check consent status
         subparsers.add_parser(
             "check-consent",
@@ -335,7 +333,9 @@ def handle_cleanup(self, options):
 
         now = timezone.now()
         eligible_submissions = Submission.objects.filter(
-            retention_eligible_date__lte=now, is_artifact_deleted=False
+            retention_eligible_date__lte=now,
+            retention_eligible_date__isnull=False,  # Exclude indefinite retention
+            is_artifact_deleted=False,
         ).select_related("challenge_phase__challenge")
 
         if not eligible_submissions.exists():
@@ -527,11 +527,11 @@ def show_challenge_status(self, challenge_id):
         else:
             self.stdout.write(
                 self.style.WARNING(
-                    "❌ HOST HAS NOT CONSENTED - 90-DAY SAFETY RETENTION APPLIED"
+                    "❌ HOST HAS NOT CONSENTED - INDEFINITE RETENTION APPLIED"
                 )
             )
             self.stdout.write(
-                f"   Retention policy: 90-day safety retention (default)"
+                f"   Retention policy: Indefinite retention (no automatic cleanup)"
             )
             self.stdout.write(
                 f"   Action needed: Host must provide consent for 30-day retention"
@@ -578,9 +578,18 @@ def show_challenge_status(self, challenge_id):
                     f"  Retention eligible date: {retention_date}"
                 )
             else:
-                self.stdout.write(
-                    "  Retention not applicable (phase still public or no end date)"
-                )
+                if phase.is_public:
+                    self.stdout.write(
+                        "  Retention not applicable (phase still public)"
+                    )
+                elif not phase.end_date:
+                    self.stdout.write(
+                        "  Retention not applicable (no end date)"
+                    )
+                else:
+                    self.stdout.write(
+                        "  Retention: Indefinite (no host consent)"
+                    )
 
             submissions = Submission.objects.filter(challenge_phase=phase)
             total_submissions = submissions.count()
@@ -630,6 +639,7 @@ def show_overall_status(self):
         ).count()
         eligible_submissions = Submission.objects.filter(
             retention_eligible_date__lte=timezone.now(),
+            retention_eligible_date__isnull=False,  # Exclude indefinite retention
             is_artifact_deleted=False,
         ).count()
 
@@ -650,7 +660,7 @@ def show_overall_status(self):
             f"With consent (30-day retention): {consented_challenges}"
         )
         self.stdout.write(
-            f"Without consent (90-day retention): {non_consented_challenges}"
+            f"Without consent (indefinite retention): {non_consented_challenges}"
         )
 
         if non_consented_challenges > 0:
@@ -671,6 +681,7 @@ def show_overall_status(self):
         upcoming_submissions = Submission.objects.filter(
             retention_eligible_date__lte=upcoming_date,
             retention_eligible_date__gt=timezone.now(),
+            retention_eligible_date__isnull=False,  # Exclude indefinite retention
             is_artifact_deleted=False,
         ).select_related("challenge_phase__challenge")
 
@@ -694,7 +705,7 @@ def show_overall_status(self):
                 consent_status = (
                     "✅ 30-day"
                     if challenge_data["has_consent"]
-                    else "❌ 90-day"
+                    else "❌ Indefinite"
                 )
                 self.stdout.write(
                     f"  - {challenge_data['name']}: {challenge_data['count']} submissions ({consent_status})"
@@ -902,7 +913,7 @@ def _build_retention_report(self, challenge_id=None):
                     "retention_policy": (
                         "30-day"
                         if challenge.retention_policy_consent
-                        else "90-day safety"
+                        else "indefinite"
                     ),
                 },
                 "admin_override": {
@@ -964,6 +975,7 @@ def _build_retention_report(self, challenge_id=None):
             challenge_data["submissions"]["eligible"] = (
                 challenge_submissions.filter(
                     retention_eligible_date__lte=now,
+                    retention_eligible_date__isnull=False,  # Exclude indefinite retention
                     is_artifact_deleted=False,
                 ).count()
             )
@@ -1160,14 +1172,18 @@ def handle_check_health(self, options):
                 f"Found {orphaned_submissions} submissions without challenge phases"
             )
 
-        # Check 3: Submissions with missing retention dates
+        # Check 3: Submissions with missing retention dates (excluding indefinite retention)
+        # Only count submissions that should have retention dates but don't
         missing_retention_dates = Submission.objects.filter(
             retention_eligible_date__isnull=True,
             is_artifact_deleted=False,
+            challenge_phase__end_date__isnull=False,  # Has end date
+            challenge_phase__is_public=False,  # Phase is not public
+            challenge_phase__challenge__retention_policy_consent=True,  # Has consent
         ).count()
         if missing_retention_dates > 0:
             health_status["warnings"].append(
-                f"Found {missing_retention_dates} submissions without retention dates"
+                f"Found {missing_retention_dates} submissions without retention dates (should have 30-day retention)"
             )
 
         # Check 4: Recent errors (if verbose)
@@ -1352,8 +1368,6 @@ def handle_find_submissions(self, options):
 
     # NEW: Consent management methods
 
-
-
     def handle_check_consent(self, options):
         """Handle checking consent status for challenges"""
         self.stdout.write("Checking retention policy consent status:")
@@ -1369,7 +1383,7 @@ def handle_check_consent(self, options):
                 status = "✅ CONSENTED (30-day retention allowed)"
             else:
                 consent_stats["without_consent"] += 1
-                status = "❌ NO CONSENT (90-day retention for safety)"
+                status = "❌ NO CONSENT (indefinite retention for safety)"
 
             self.stdout.write(
                 f"Challenge {challenge.pk}: {challenge.title[:40]:<40} | {status}"
@@ -1383,7 +1397,7 @@ def handle_check_consent(self, options):
             f"With consent (30-day retention allowed): {consent_stats['with_consent']}"
         )
         self.stdout.write(
-            f"Without consent (90-day retention for safety): {consent_stats['without_consent']}"
+            f"Without consent (indefinite retention for safety): {consent_stats['without_consent']}"
         )
 
         if consent_stats["without_consent"] > 0:

apps/challenges/migrations/0115_add_retention_consent_fields.py renamed to apps/challenges/migrations/0114_add_log_retention_and_consent_fields.py

@@ -9,10 +9,22 @@ class Migration(migrations.Migration):
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ("challenges", "0114_add_log_retention_override"),
+        ("challenges", "0113_add_github_branch_field_and_unique_constraint"),
     ]
 
     operations = [
+        # Log retention override field (from 0114)
+        migrations.AddField(
+            model_name="challenge",
+            name="log_retention_days_override",
+            field=models.PositiveIntegerField(
+                blank=True,
+                default=None,
+                help_text="Admin override for CloudWatch log retention period in days (defaults to 30 days when host has consented)",
+                null=True,
+            ),
+        ),
+        # Retention consent fields (from 0115)
         migrations.AddField(
             model_name="challenge",
             name="retention_policy_consent",
@@ -52,14 +64,4 @@ class Migration(migrations.Migration):
                 null=True,
             ),
         ),
-        migrations.AlterField(
-            model_name="challenge",
-            name="log_retention_days_override",
-            field=models.PositiveIntegerField(
-                blank=True,
-                default=None,
-                help_text="Admin override for CloudWatch log retention period in days (defaults to 30 days when host has consented)",
-                null=True,
-            ),
-        ),
     ]