feat(migrations): Adding a support of Flyway

.github/workflows/check_python.yml → .github/workflows/quality_gates.yml

@@ -1,4 +1,4 @@
-name: Python Check
+name: Quality Gates
 
 on:
   pull_request:
@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: static-python-check-${{ github.ref }}
+  group: quality-gates-${{ github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -21,35 +21,43 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       python_changed: ${{ steps.changes.outputs.python_changed }}
+      database_changed: ${{ steps.changes.outputs.database_changed }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
         with:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Check if Python files changed
+      - name: Check if Python or database files changed
         id: changes
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           set -euo pipefail
-          
+
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then
             CHANGED_FILES=$(gh api \
               "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
-              --jq '.[].filename | select(endswith(".py") or (startswith("requirements") and endswith(".txt")))')
+              --paginate \
+              --jq '.[].filename')
           else
-            CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}" -- '*.py' 'requirements*.txt')
+            CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}")
           fi
 
-          if [[ -n "$CHANGED_FILES" ]]; then
+          if grep -Eq '(^|/)[^/]+\.py$|^requirements[^/]*\.txt$' <<< "$CHANGED_FILES"; then
             echo "python_changed=true" >> "$GITHUB_OUTPUT"
           else
             echo "python_changed=false" >> "$GITHUB_OUTPUT"
           fi
 
+          if grep -Eq '^database/|^flyway\.toml$' <<< "$CHANGED_FILES"; then
+            echo "database_changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "database_changed=false" >> "$GITHUB_OUTPUT"
+          fi
+
   pylint-analysis:
     name: Pylint Static Code Analysis
     needs: detect
@@ -139,7 +147,7 @@ jobs:
   integration-tests:
     name: Pytest Integration Tests
     needs: detect
-    if: needs.detect.outputs.python_changed == 'true'
+    if: needs.detect.outputs.python_changed == 'true' || needs.detect.outputs.database_changed == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -152,13 +160,26 @@ jobs:
       - name: Set up dev Python environment
         uses: ./.github/actions/setup-dev-python-env
 
+      - name: Set up Java
+        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Set up Flyway
+        uses: red-gate/setup-flyway@e024a17cd0890383f6996ed7edbded24c54ed86c
+        with:
+          version: '13.3.0'
+          edition: community
+          i-agree-to-the-eula: true
+
       - name: Run integration tests
         run: pytest tests/integration/ -v --tb=short --log-cli-level=INFO
 
   noop:
     name: No Operation
     needs: detect
-    if: needs.detect.outputs.python_changed != 'true'
+    if: needs.detect.outputs.python_changed != 'true' && needs.detect.outputs.database_changed != 'true'
     runs-on: ubuntu-latest
     steps:
       - run: echo "No changes in the *.py files — passing."

database/README.md

@@ -0,0 +1,79 @@
+# EventGate Database
+
+All database code lives here and is deployed with [Flyway](https://documentation.red-gate.com/flyway).
+The migrations are the single source of truth for the schema, roles, and grants — the
+same migrations build local, CI (integration tests), and real environments.
+
+## Layout
+
+```text
+flyway.toml                            # Flyway configuration (locations, baseline, placeholders) — repo root
+database/
+├── README.md
+└── migrations/
+    ├── 00_databases.ddl               # One-off DB bootstrap (NOT a Flyway migration; no `V` prefix)
+    ├── V1.4.0.1__create_roles.ddl     # owner / writer / reader roles
+    ├── V1.4.0.2__initial_schema.ddl   # tables
+    └── V1.4.0.3__grants.ddl           # ownership + least-privilege grants
+    ...
+```
+
+## Conventions
+
+- Versioned migrations follow Flyway's `V<major>.<minor>.<patch>.<step>__description.ext` format,
+  where `<major>.<minor>.<patch>` tracks the EventGate release the migration ships in and `<step>`
+  increments per migration within that release.
+- Extensions carry intent: `.ddl` for structural changes (tables, roles, constraints, indexes),
+  `.sql` for DML / data.
+
+## Roles
+
+| Role               | Purpose                                             | Used by             |
+|--------------------|-----------------------------------------------------|---------------------|
+| master (superuser) | Runs the migrations                                 | Flyway (deployment) |
+| `eventgate_owner`  | Owns the schema objects, may run DDL                | Migrations          |
+| `eventgate_writer` | `SELECT` / `INSERT` / `UPDATE` on data tables       | EventGate Lambda    |
+| `eventgate_reader` | `SELECT` only                                       | EventStats Lambda   |
+
+Role passwords are required Flyway placeholders (`eventgate_owner_password`,
+`eventgate_writer_password`, `eventgate_reader_password`). Supply them from secrets in real
+environments.
+
+## Local setup
+
+Requires the Flyway CLI (needs a JDK 17+) and Docker.
+
+```zsh
+# 1. Start a local Postgres docker container
+docker run --name=eventgate_db -e POSTGRES_PASSWORD=changeme -e POSTGRES_DB=eventgate_db -p 5432:5432 -d postgres:16
+
+# 2. Apply the migrations (run from the repo root, where flyway.toml lives)
+export FLYWAY_PLACEHOLDERS_EVENTGATE_OWNER_PASSWORD=changeme
+export FLYWAY_PLACEHOLDERS_EVENTGATE_WRITER_PASSWORD=changeme
+export FLYWAY_PLACEHOLDERS_EVENTGATE_READER_PASSWORD=changeme
+flyway migrate
+
+# Inspect state / clean up
+flyway info
+docker kill eventgate_db && docker rm eventgate_db
+```
+
+## Adopting an existing database
+
+On a database that already contains the tables but has no Flyway history (i.e. production), a
+plain `flyway migrate` fails because Flyway sees existing objects it didn't create. The first
+migration against such a database must instead pass baseline flags explicitly, one time only:
+
+```zsh
+flyway -baselineOnMigrate=true -baselineVersion=1.4.0.0 migrate
+```
+
+This records a baseline at `1.4.0.0` in `flyway_schema_history` and then applies `V1.4.0.1+` on
+top.
+
+Before the first production migration:
+
+1. Compare the deployed schema with `V1.4.0.2__initial_schema.ddl`.
+2. Back up the database and cluster roles.
+3. Confirm the migration account can create roles and change ownership of every EventGate table.
+4. Run `flyway info`, then the baseline command above with all role-password placeholders supplied from secrets.

database/migrations/00_databases.ddl

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2026 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Database bootstrap (NOT a Flyway migration).
+--
+-- Flyway connects to an existing database, so it cannot create the database it migrates.
+-- This script is intentionally NOT prefixed with `V`, so Flyway ignores it.
+
+CREATE DATABASE eventgate_db
+    WITH
+    ENCODING = 'UTF8'
+    CONNECTION LIMIT = -1;

database/migrations/V1.4.0.1__create_roles.ddl

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2026 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Application database roles.
+--
+-- eventgate_owner  - owns the schema objects and may run DDL.
+-- eventgate_writer - inserts/updates event data (main EventGate Lambda).
+-- eventgate_reader - read-only access (EventStats Lambda).
+
+DO
+$do$
+    BEGIN
+        IF EXISTS (
+                SELECT FROM pg_catalog.pg_roles
+                WHERE rolname = 'eventgate_owner') THEN
+
+            RAISE NOTICE 'Role "eventgate_owner" already exists. Skipping.';
+        ELSE
+            CREATE ROLE eventgate_owner WITH
+                LOGIN
+                NOSUPERUSER
+                INHERIT
+                NOCREATEDB
+                NOCREATEROLE
+                NOREPLICATION
+                PASSWORD '${eventgate_owner_password}';
+        END IF;
+    END
+$do$;
+
+DO
+$do$
+    BEGIN
+        IF EXISTS (
+                SELECT FROM pg_catalog.pg_roles
+                WHERE rolname = 'eventgate_writer') THEN
+            RAISE NOTICE 'Role "eventgate_writer" already exists. Skipping.';
+        ELSE
+            CREATE ROLE eventgate_writer WITH
+                LOGIN
+                NOSUPERUSER
+                INHERIT
+                NOCREATEDB
+                NOCREATEROLE
+                NOREPLICATION
+                PASSWORD '${eventgate_writer_password}';
+        END IF;
+    END
+$do$;
+
+DO
+$do$
+    BEGIN
+        IF EXISTS (
+                SELECT FROM pg_catalog.pg_roles
+                WHERE rolname = 'eventgate_reader') THEN
+            RAISE NOTICE 'Role "eventgate_reader" already exists. Skipping.';
+        ELSE
+            CREATE ROLE eventgate_reader WITH
+                LOGIN
+                NOSUPERUSER
+                INHERIT
+                NOCREATEDB
+                NOCREATEROLE
+                NOREPLICATION
+                PASSWORD '${eventgate_reader_password}';
+        END IF;
+    END
+$do$;

tests/integration/schemas/postgres_schema.py → database/migrations/V1.4.0.2__initial_schema.ddl

@@ -1,23 +1,21 @@
-#
-# Copyright 2026 ABSA Group Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+/*
+ * Copyright 2026 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
-"""PostgreSQL schema for integration tests."""
+-- Initial EventGate schema.
 
-SCHEMA_SQL = """
--- Table matching WriterPostgres._postgres_run_write columns
+-- Run header rows for the runs topic.
 CREATE TABLE IF NOT EXISTS public_cps_za_runs (
     event_id VARCHAR(255) NOT NULL,
     job_ref VARCHAR(255) NOT NULL,
@@ -29,7 +27,7 @@
     timestamp_end BIGINT
 );
 
--- Table matching WriterPostgres._postgres_run_write job rows
+-- Per-job rows belonging to a run.
 CREATE TABLE IF NOT EXISTS public_cps_za_runs_jobs (
     internal_id SERIAL PRIMARY KEY,
     event_id VARCHAR(255) NOT NULL,
@@ -42,7 +40,7 @@
     additional_info JSONB
 );
 
--- Table matching WriterPostgres._postgres_edla_write columns
+-- Data lake change events.
 CREATE TABLE IF NOT EXISTS public_cps_za_dlchange (
     event_id VARCHAR(255) NOT NULL,
     tenant_id VARCHAR(255) NOT NULL,
@@ -59,7 +57,7 @@
     additional_info JSONB
 );
 
--- Table matching WriterPostgres._postgres_test_write columns
+-- Test topic events.
 CREATE TABLE IF NOT EXISTS public_cps_za_test (
     event_id VARCHAR(255) NOT NULL,
     tenant_id VARCHAR(255) NOT NULL,
@@ -69,7 +67,7 @@
     additional_info JSONB
 );
 
--- Table for test_status_change_writer
+-- Aggregated latest status per job (see ADR 001).
 CREATE TABLE IF NOT EXISTS public_cps_za_status_change_aggregated_job (
     job_id               UUID PRIMARY KEY,
     job_group_id         UUID,
@@ -97,4 +95,3 @@
     finished_at          TIMESTAMPTZ,
     last_updated_at      TIMESTAMPTZ NOT NULL
 );
-"""