Release 3.1.5

.github/workflows/ci2develop.yml

@@ -65,20 +65,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to Harbor
-        uses: docker/login-action@v1
+        uses: docker/login-action@v4
         with:
           registry: ${{ secrets.HARBOR_REGISTRY }}
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_PASSWORD }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v7
         with:
           context: ./frontend
           file: ./frontend/Dockerfile
@@ -91,6 +91,8 @@ jobs:
             SENTRY_ENVIRONMENT=development
             SENTRY_DSN_FRONTEND=${{ secrets.SENTRY_DSN_FRONTEND }}
             USE_SENTRY=1
+          secrets: |
+            "sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}"
 
   ci-hub-auth-dev:
     needs: [detect-changes-by-component]

.github/workflows/ci2production-and-release.yml

@@ -93,20 +93,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to Harbor
-        uses: docker/login-action@v1
+        uses: docker/login-action@v4
         with:
           registry: ${{ secrets.HARBOR_REGISTRY }}
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_PASSWORD }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v7
         with:
           context: ./frontend
           file: ./frontend/Dockerfile
@@ -119,6 +119,8 @@ jobs:
             SENTRY_ENVIRONMENT=production
             SENTRY_DSN_FRONTEND=${{ secrets.SENTRY_DSN_FRONTEND }}
             USE_SENTRY=1
+          secrets: |
+            "sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}"
 
   ci-hub-auth:
     needs: [detect-changes-by-component]

.github/workflows/monitoring-validate.yml

@@ -0,0 +1,188 @@
+name: Validate monitoring contracts
+
+on:
+  pull_request:
+    paths:
+      - "kubernetes/monitoring/**"
+      - "kubernetes/base/hub-auth/**"
+      - "kubernetes/base/backend/**"
+      - "kubernetes/base/celery-beat/**"
+      - "kubernetes/base/celery-worker/**"
+      - "kubernetes/overlays/dev/**"
+      - "kubernetes/overlays/prod/**"
+      - "hub/auth_server/**"
+      - "backend/account/middleware.py"
+      - "backend/account/tests.py"
+      - "backend/judge/tasks.py"
+      - "backend/judge/tests.py"
+      - "backend/oj/celery.py"
+      - "backend/problem/llm_hint.py"
+      - "backend/problem/views/oj.py"
+      - "backend/problem/tests.py"
+      - "backend/submission/views/oj.py"
+      - "backend/submission/tests.py"
+      - "backend/utils/observability_metrics.py"
+      - "backend/utils/observability_tracing.py"
+      - "backend/utils/tests.py"
+      - "backend/utils/test_observability_tracing.py"
+      - ".github/workflows/monitoring-validate.yml"
+  push:
+    branches: [develop, main]
+    paths:
+      - "kubernetes/monitoring/**"
+      - "kubernetes/base/hub-auth/**"
+      - "kubernetes/base/backend/**"
+      - "kubernetes/base/celery-beat/**"
+      - "kubernetes/base/celery-worker/**"
+      - "kubernetes/overlays/dev/**"
+      - "kubernetes/overlays/prod/**"
+      - "hub/auth_server/**"
+      - "backend/account/middleware.py"
+      - "backend/account/tests.py"
+      - "backend/judge/tasks.py"
+      - "backend/judge/tests.py"
+      - "backend/oj/celery.py"
+      - "backend/problem/llm_hint.py"
+      - "backend/problem/views/oj.py"
+      - "backend/problem/tests.py"
+      - "backend/submission/views/oj.py"
+      - "backend/submission/tests.py"
+      - "backend/utils/observability_metrics.py"
+      - "backend/utils/observability_tracing.py"
+      - "backend/utils/tests.py"
+      - "backend/utils/test_observability_tracing.py"
+      - ".github/workflows/monitoring-validate.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    env:
+      POSTGRES_HOST: 127.0.0.1
+      POSTGRES_PORT: "5435"
+      POSTGRES_DB: onlinejudge
+      POSTGRES_USER: onlinejudge
+      POSTGRES_PASSWORD: onlinejudge
+      REDIS_HOST: 127.0.0.1
+      REDIS_PORT: "6380"
+    services:
+      postgres:
+        image: postgres:14
+        env:
+          POSTGRES_DB: onlinejudge
+          POSTGRES_USER: onlinejudge
+          POSTGRES_PASSWORD: onlinejudge
+        ports:
+          - 5435:5432
+        options: >-
+          --health-cmd "pg_isready -U onlinejudge -d onlinejudge"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 6380:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: "v1.33.6"
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "v3.18.6"
+
+      - name: Install validation dependencies
+        run: python -m pip install --disable-pip-version-check -r backend/deploy/requirements.txt -r hub/auth_server/requirements.txt pytest PyYAML
+
+      - name: Test hub-auth health contract
+        working-directory: hub/auth_server
+        env:
+          GITHUB_OAUTH_APP_ID: test-client
+          GITHUB_OAUTH_APP_SECRET: test-secret
+        run: python -m pytest app/test_main.py
+
+      - name: Test trace resource contract
+        env:
+          PYTHONPATH: backend
+        run: python -m unittest utils.test_observability_tracing
+
+      - name: Test backend observability contracts
+        working-directory: backend
+        run: |
+          printf 'monitoring-validation-secret\n' >data/config/secret.key
+          python manage.py test \
+            utils.tests.CodePlaceCollectorTest \
+            utils.tests.CeleryRequestIDContextTest \
+            utils.tests.ObservabilityTracingTest \
+            utils.tests.CodePlaceMetricsEndpointTest \
+            judge.tests.JudgeTaskObservabilityTest \
+            account.tests.RequestLogMiddlewareTest \
+            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_api_success_outcome \
+            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_success_metric \
+            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_request_error_metric \
+            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_problem_limit_outcome \
+            submission.tests.SubmissionAPITest.test_create_submission_records_success_outcome \
+            submission.tests.SubmissionAPITest.test_create_submission_records_problem_not_found_outcome
+
+      - name: Validate dashboard contracts
+        run: python kubernetes/monitoring/validate_dashboards.py --prometheus-rules-output /tmp/codeplace-dashboard-rules.json
+
+      - name: Validate alert contracts
+        run: python kubernetes/monitoring/validate_alerts.py
+
+      - name: Render Kubernetes manifests
+        run: |
+          kubectl kustomize kubernetes/overlays/dev >/dev/null
+          kubectl kustomize kubernetes/overlays/prod >/dev/null
+          kubectl kustomize kubernetes/monitoring >/tmp/codeplace-monitoring.yaml
+          python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/codeplace-monitoring.yaml").read_text()) if doc]; assert any(doc.get("kind") == "PodMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "Traefik PodMonitor is missing"; assert not any(doc.get("kind") == "ServiceMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "legacy Traefik ServiceMonitor remains"'
+          test ! -e kubernetes/monitoring/traefik-service-monitor.yaml
+
+      - name: Render pinned monitoring charts
+        run: |
+          helm template kube-prometheus-stack kube-prometheus-stack \
+            --repo https://prometheus-community.github.io/helm-charts \
+            --version 86.3.1 \
+            --namespace monitoring \
+            --kube-version 1.33.6 \
+            --values kubernetes/monitoring/kube-prometheus-stack-values.yaml >/tmp/kube-prometheus-stack.yaml
+          python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/kube-prometheus-stack.yaml").read_text()) if doc]; assert any(doc.get("kind") == "Job" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-crds-upgrade" for doc in docs), "CRD upgrade Job is missing"; grafana=next(doc for doc in docs if doc.get("kind") == "Deployment" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-grafana"); assert grafana.get("spec", {}).get("strategy", {}).get("type") == "Recreate", "Grafana must use Recreate with its RWO PVC"'
+          helm template loki loki \
+            --repo https://grafana.github.io/helm-charts \
+            --version 6.55.0 \
+            --namespace monitoring \
+            --kube-version 1.33.6 \
+            --api-versions monitoring.coreos.com/v1/ServiceMonitor \
+            --values kubernetes/monitoring/logs/loki-values.yaml >/dev/null
+          helm template alloy alloy \
+            --repo https://grafana.github.io/helm-charts \
+            --version 1.10.0 \
+            --namespace monitoring \
+            --kube-version 1.33.6 \
+            --api-versions monitoring.coreos.com/v1/ServiceMonitor \
+            --values kubernetes/monitoring/logs/alloy-values.yaml >/dev/null
+
+      - name: Validate Prometheus rules
+        run: |
+          python -c 'import pathlib, yaml; source=yaml.safe_load(pathlib.Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()); pathlib.Path("/tmp/codeplace-rules.yaml").write_text(yaml.safe_dump({"groups": source["spec"]["groups"]}))'
+          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro prom/prometheus:v3.7.3 check rules /rules.yaml
+          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-dashboard-rules.json:/rules.json:ro prom/prometheus:v3.7.3 check rules /rules.json
+          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro -v "$PWD/kubernetes/monitoring/prometheus-rules.test.yaml:/rules.test.yaml:ro" prom/prometheus:v3.7.3 test rules /rules.test.yaml

backend/account/tests.py

@@ -18,7 +18,7 @@
 
 from .models import AdminType, ProblemPermission, User
 from .decorators import login_required, scheduler_only
-from .middleware import AdminRoleRequiredMiddleware, RequestIDMiddleware
+from .middleware import AdminRoleRequiredMiddleware, RequestIDMiddleware, RequestLogMiddleware
 from .tasks import calculate_user_score_basis, calculate_user_score_fluctuation
 
 
@@ -65,6 +65,23 @@ def test_generates_request_id_when_header_has_no_safe_characters(self):
         self.assertEqual(response["X-Request-ID"], request.request_id)
 
 
+class RequestLogMiddlewareTest(SimpleTestCase):
+
+    @mock.patch("account.middleware.HTTP_REQUEST_DURATION_SECONDS")
+    @mock.patch("account.middleware.HTTP_REQUESTS_TOTAL")
+    def test_records_request_metrics(self, requests_total, duration_seconds):
+        request = RequestFactory().get("/api/problem")
+        request.resolver_match = mock.Mock(view_name="problem_api")
+        middleware = RequestLogMiddleware(lambda _: JsonResponse({}, status=201))
+
+        middleware(request)
+
+        requests_total.labels.assert_called_once_with("GET", "problem_api", "201")
+        requests_total.labels.return_value.inc.assert_called_once()
+        duration_seconds.labels.assert_called_once_with("GET", "problem_api")
+        duration_seconds.labels.return_value.observe.assert_called_once()
+
+
 class AdminRoleRequiredMiddlewareTest(SimpleTestCase):
 
     def setUp(self):

backend/judge/tasks.py

@@ -4,7 +4,7 @@
 from account.models import User
 from utils.shortcuts import CELERY_TASK_ARGS
 from utils.observability_tracing import get_tracer
-from utils.observability_metrics import JUDGE_TASK_OUTCOME_TOTAL
+from utils.observability_metrics import record_judge_task_outcome
 from submission.models import Submission
 from judge.dispatcher import JudgeDispatcher
 
@@ -13,7 +13,7 @@
 
 
 def _record_judge_task_outcome(status, scope):
-    JUDGE_TASK_OUTCOME_TOTAL.labels(status=status, scope=scope).inc()
+    record_judge_task_outcome(status, scope)
 
 
 @celery.shared_task(**CELERY_TASK_ARGS())

backend/judge/tests.py

@@ -7,37 +7,29 @@
 
 class JudgeTaskObservabilityTest(SimpleTestCase):
 
-    @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL")
+    @mock.patch("judge.tasks.record_judge_task_outcome")
     @mock.patch("judge.tasks.JudgeDispatcher")
     @mock.patch("judge.tasks.User")
     @mock.patch("judge.tasks.Submission")
-    def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, outcome_total):
+    def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, record_outcome):
         submission = mock.Mock(user_id=1, contest_id=None)
         submission_model.objects.get.return_value = submission
         user_model.objects.get.return_value = mock.Mock(is_disabled=False)
-        labels = mock.Mock()
-        outcome_total.labels.return_value = labels
-
         judge_task.run(10, 20)
 
         dispatcher.assert_called_once_with(10, 20)
         dispatcher.return_value.judge.assert_called_once()
-        outcome_total.labels.assert_called_once_with(status="success", scope="practice")
-        labels.inc.assert_called_once()
+        record_outcome.assert_called_once_with("success", "practice")
 
-    @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL")
+    @mock.patch("judge.tasks.record_judge_task_outcome")
     @mock.patch("judge.tasks.JudgeDispatcher")
     @mock.patch("judge.tasks.User")
     @mock.patch("judge.tasks.Submission")
-    def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, outcome_total):
+    def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, record_outcome):
         submission = mock.Mock(user_id=1, contest_id=7)
         submission_model.objects.get.return_value = submission
         user_model.objects.get.return_value = mock.Mock(is_disabled=True)
-        labels = mock.Mock()
-        outcome_total.labels.return_value = labels
-
         judge_task.run(10, 20)
 
         dispatcher.assert_not_called()
-        outcome_total.labels.assert_called_once_with(status="user_disabled", scope="contest")
-        labels.inc.assert_called_once()
+        record_outcome.assert_called_once_with("user_disabled", "contest")

backend/oj/celery.py

@@ -1,6 +1,8 @@
 import os
 import celery
+from celery.signals import task_postrun, task_prerun
 
+from utils.observability_context import reset_request_id, set_request_id
 from utils.observability_tracing import configure_opentelemetry
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings')
@@ -9,3 +11,20 @@
 app = celery.Celery('scheduler')
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()
+
+_request_id_tokens = {}
+
+
+@task_prerun.connect
+def bind_request_id(task_id=None, task=None, **kwargs):
+    headers = getattr(getattr(task, "request", None), "headers", None) or {}
+    request_id = headers.get("x-request-id")
+    if task_id and request_id:
+        _request_id_tokens[task_id] = set_request_id(request_id)
+
+
+@task_postrun.connect
+def unbind_request_id(task_id=None, **kwargs):
+    token = _request_id_tokens.pop(task_id, None)
+    if token is not None:
+        reset_request_id(token)