feat: highly available clickhouse deployment

config/components/clickhouse-database/README.md

@@ -1,50 +1,40 @@
 # ClickHouse Database Component
 
-Deploys a ClickHouse cluster for storing audit events.
+Deploys a highly-available ClickHouse cluster for storing audit events.
 
-## What It Does
+## Configuration
 
-Creates a ClickHouse instance with:
-- Single shard, single replica (test/dev configuration)
-- Hot/cold storage tiering (local SSD + S3)
-- Audit database pre-configured
-- TTL policy to move old data to S3 after 90 days
+- **3 replicas, 1 shard** - Survives 1 node failure
+- **ReplicatedReplacingMergeTree** - Automatic replication with deduplication
+- **Quorum writes** (2/3) - Strong consistency guarantees
+- **Hot/cold tiering** - Local SSD → S3 after 90 days
+- **Keeper coordination** - Replaces ZooKeeper for HA
 
-## Files
+## Prerequisites
 
-- `clickhouse-installation.yaml` - ClickHouseInstallation CR
-- `kustomization.yaml` - Component definition
-
-## Storage Configuration
-
-- **Hot Storage**: Recent data on local disks for fast queries
-- **Cold Storage**: Old data on S3 for cost-effective archival
-
-The storage policy is configured via patches in each overlay (test-infra, production, etc).
+1. Deploy ClickHouse Keeper first (see `../clickhouse-keeper/`)
+2. ClickHouse Operator v0.25.6+
+3. S3-compatible storage for cold tier
 
 ## Usage
 
-Include in your overlay:
-
 ```yaml
 # config/overlays/{env}/kustomization.yaml
 components:
+  - ../../components/clickhouse-keeper
   - ../../components/clickhouse-database
 ```
 
-Add environment-specific storage patches as needed.
-
-## Access
+Override storage settings via patches in your overlay.
 
-```bash
-# Connect to ClickHouse
-kubectl exec -it clickhouse-activity-0-0-0 -n activity-system -- clickhouse-client
+## How It Works
 
-# Query audit events
-SELECT * FROM audit.events LIMIT 10;
-```
+- **Writes**: Insert on any replica → replicate via Keeper → quorum (2/3)
+  acknowledges
+- **Reads**: Query any replica with read-after-write consistency
+- **Failures**: Tolerates 1 replica down (2/3 quorum maintained)
 
-## Requirements
+**Learn more**:
 
-- ClickHouse Operator must be installed first
-- S3 storage (RustFS in test, AWS S3 in production)
+- [Data Replication](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication)
+- [ClickHouse Keeper](https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper)

config/components/clickhouse-database/clickhouse-installation.yaml

@@ -5,21 +5,70 @@ metadata:
   namespace: activity-system
 spec:
   configuration:
+    # Configure the clickhouse cluster with the nodes created by the clickhouse
+    # keeper installation.
+    zookeeper:
+      nodes:
+        - host: chk-activity-keeper-activity-keeper-0-0.activity-system.svc.cluster.local
+          port: 2181
+        - host: chk-activity-keeper-activity-keeper-0-1.activity-system.svc.cluster.local
+          port: 2181
+        - host: chk-activity-keeper-activity-keeper-0-2.activity-system.svc.cluster.local
+          port: 2181
+      # Session timeout: 30 seconds (default)
+      # Keeper will expire the session if no heartbeat received within this time
+      session_timeout_ms: 30000
+      # Operation timeout: 10 seconds
+      # Maximum time to wait for a Keeper operation to complete
+      operation_timeout_ms: 10000
     clusters:
       - name: activity
         layout:
           shardsCount: 1
-          replicasCount: 1
+          replicasCount: 3
+
     users:
       default/password: ""  # Empty password for test environments
       default/networks/ip:
         - "0.0.0.0/0"
     settings:
       # Default settings for ClickHouse
       default/default_database: audit
+
+      # These control how CREATE/ALTER/DROP with ON CLUSTER are executed
+      default/distributed_ddl_task_timeout: 3600
+      default/distributed_ddl_entry_format_version: 5
+
+      # Require 2 out of 3 replicas to acknowledge writes before returning success
+      # This provides strong consistency guarantees
+      default/insert_quorum: 2
+      default/insert_quorum_timeout: 60000  # 60 seconds to wait for quorum
+      default/insert_quorum_parallel: 1     # Enable parallel quorum inserts
+
+      # Ensure you can read your own writes (read-after-write consistency)
+      default/select_sequential_consistency: 1
+
+      # Maximum ratio of errors before marking replica as failed
+      default/replicated_max_ratio_of_errors_to_be_ignored: 0.5
+      # Deduplication window: keep track of recent inserts to prevent duplicates
+      default/replicated_deduplication_window: 100
+      # Deduplication time window: 7 days
+      default/replicated_deduplication_window_seconds: 604800
+
     profiles:
       default/max_memory_usage: 3000000000  # 3GB max memory usage
     files:
+      config.d/log_rotation.xml: |-
+        <clickhouse>
+            <logger>
+                <level>information</level>
+                <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+                <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+                <size>1000M</size>
+                <count>5</count>
+                <console>1</console>
+            </logger>
+        </clickhouse>
       # Storage configuration for hot/cold tiering with S3
       # Hot storage: local disk for recent data (fast queries)
       # Cold storage: S3-compatible storage for older data (cost-effective archival)

config/components/clickhouse-database/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Component
 
 resources:
   - clickhouse-installation.yaml
+  - pod-disruption-budget.yaml

config/components/clickhouse-database/pod-disruption-budget.yaml

@@ -0,0 +1,12 @@
+---
+# PodDisruptionBudget for ClickHouse Database StatefulSet Ensures high availability during voluntary disruptions (node
+# drains, upgrades, etc.) Allows only 1 pod to be unavailable at a time to maintain quorum
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: clickhouse-database-pdb
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      clickhouse.altinity.com/chi: activity-clickhouse

config/components/clickhouse-keeper/README.md

@@ -0,0 +1,33 @@
+# ClickHouse Keeper Component
+
+This component deploys a 3-node ClickHouse Keeper cluster to provide
+coordination and metadata management for ClickHouse replication. ClickHouse
+Keeper is the modern replacement for ZooKeeper.
+
+Our configuration provides high availability (survives 1 node failure) with pod
+anti-affinity ensuring nodes are distributed across the cluster.
+
+## Usage
+
+Include in your overlay:
+
+```yaml
+# config/overlays/{env}/kustomization.yaml
+components:
+  - ../../components/clickhouse-keeper
+  - ../../components/clickhouse-database
+```
+
+> [!IMPORTANT]
+>
+> Deploy Keeper before ClickHouse or update ClickHouse after Keeper is ready.
+
+## Learn More
+
+For detailed information about ClickHouse Keeper, including architecture,
+configuration, monitoring, and troubleshooting:
+
+- [ClickHouse Keeper
+  Documentation](https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper)
+- [Altinity Kubernetes Operator - ClickHouse
+  Keeper](https://docs.altinity.com/altinitykubernetesoperator/)

config/components/clickhouse-keeper/keeper-installation.yaml

@@ -0,0 +1,94 @@
+---
+apiVersion: clickhouse-keeper.altinity.com/v1
+kind: ClickHouseKeeperInstallation
+metadata:
+  name: activity-keeper
+  namespace: activity-system
+spec:
+  configuration:
+    settings:
+      prometheus/endpoint: /metrics
+      prometheus/port: 7000
+      prometheus/metrics: true
+      prometheus/events: true
+      prometheus/asynchronous_metrics: true
+
+    clusters:
+      - name: activity-keeper
+        layout:
+          # 3 replicas for HA (odd number required for Raft quorum)
+          # Survives 1 node failure while maintaining quorum
+          replicasCount: 3
+
+  defaults:
+    templates:
+      podTemplate: clickhouse-keeper
+      dataVolumeClaimTemplate: clickhouse-keeper
+
+  templates:
+    podTemplates:
+      - name: clickhouse-keeper
+        spec:
+          containers:
+            - name: clickhouse-keeper
+              imagePullPolicy: IfNotPresent
+              image: clickhouse/clickhouse-keeper:25.12.3-alpine
+              resources:
+                requests:
+                  cpu: 1
+                  memory: 2Gi
+                limits:
+                  cpu: 2
+                  memory: 4Gi
+              ports:
+                - name: chk-metrics
+                  containerPort: 7000
+          # Configure the pod's to spread across nodes and availability zones.
+          topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: topology.kubernetes.io/zone
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                clickhouse.altinity.com/chi: activity-clickhouse
+                app.kubernetes.io/part-of: activity.miloapis.com
+                clickhouse.altinity.com/cluster: activity
+                clickhouse.altinity.com/namespace: activity-system
+          - maxSkew: 1
+            topologyKey: kubernetes.io/hostname
+            whenUnsatisfiable: ScheduleAnyway
+            labelSelector:
+              matchLabels:
+                clickhouse.altinity.com/chi: activity-clickhouse
+                app.kubernetes.io/part-of: activity.miloapis.com
+                clickhouse.altinity.com/cluster: activity
+                clickhouse.altinity.com/namespace: activity-system
+    volumeClaimTemplates:
+      - name: clickhouse-keeper
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 5Gi
+---
+# Configure a pod monitor to scrape metrics from the clickhouse keeper pods
+# using the port defined in the configuration above.
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: clickhouse-keeper
+  namespace: activity-system
+  labels:
+    app: clickhouse-keeper
+    app.kubernetes.io/component: coordination
+spec:
+  selector:
+    matchLabels:
+      # Match operator-created pods for this ClickHouseKeeperInstallation
+      clickhouse-keeper.altinity.com/chk: activity-keeper
+  podMetricsEndpoints:
+    - port: chk-metrics
+      path: /metrics
+      interval: 30s
+      scheme: http

config/components/clickhouse-keeper/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - keeper-installation.yaml
+  - pod-disruption-budget.yaml
+
+# Component metadata
+metadata:
+  name: clickhouse-keeper
+  annotations:
+    config.kubernetes.io/local-config: "true"

config/components/clickhouse-keeper/pod-disruption-budget.yaml

@@ -0,0 +1,12 @@
+---
+# PodDisruptionBudget for ClickHouse Keeper StatefulSet Ensures high availability during voluntary disruptions (node
+# drains, upgrades, etc.) Allows only 1 pod to be unavailable at a time to maintain quorum
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: clickhouse-keeper-pdb
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      clickhouse-keeper.altinity.com/chk: activity-keeper