Added sink connector

.gitignore

@@ -0,0 +1 @@
+.idea/

README.md

@@ -9,7 +9,15 @@ You'll need a Kubernetes cluster. EKS, GKE, or minikube should all be OK.
 
 Install the following tools in your environment. ArgoCD should be installed
 in the argocd namespace. 
+You can use the script
+```
+./install-argocd.sh
+```
 
+```
+kubectl create namespace argocd
+kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
+```
 * [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
 * [argocd](https://argo-cd.readthedocs.io/en/stable/getting_started/)
 * [kustomize](https://kubectl.docs.kubernetes.io/installation/kustomize/)
@@ -21,16 +29,35 @@ kubectl get ns
 
 ## Login to ArgoCD
 
+ArgoCD default password can be retrieved from k8s secrets
+```
+kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d
+```
 All commands with argocd require an accessible ArgoCD API endpoint and a
 valid token. Get them as follows.
 ```
 kubectl port-forward svc/argocd-server -n argocd 8080:443 &
 argocd login localhost:8080 --username=admin --password="yourpassword" --insecure
 ```
 
+You can also use the script.
+```
+./login-argocd.sh
+```
 ## Bring up the analytic stack
 
 Run the `create-stack.sh` script in the apps directory.
+If you need to the run changes from a specific branch
+use the following command
+```
+./create-stack.sh -b <branch-name>
+```
+
+To run the ETL stack with MySQL, Clickhouse sink connector
+use the following command
+```
+./create-stack-mysql-etl.sh -b <branch-name>
+```
 
 ## Make endpoints accessible
 
@@ -48,3 +75,37 @@ following steps to clean up fully.
 
 1. Run 'kubectl edit' on any chi resource and remove the finalizer. 
 2. Drop the namespace. 
+
+### Grafana (Manual steps)
+The admin password can be reset by executing the following command on the grafana pod.
+```
+grafana-cli admin reset-admin-password admin
+```
+
+The `admin` password is stored in `grafana-admin-credentials` secret.
+```
+kubectl get secret grafana-admin-credentials --template={{.data.GF_SECURITY_ADMIN_PASSWORD}} -n ch|base64 -d
+```
+
+Open the grafana UI by setting up port-forward on the grafana pod.
+```
+kubectl port-forward -n ch svc/grafana-service 3000:3000
+```
+Login as admin and the password retrieved in the previous step.
+
+### Prometheus Operator
+ArgoCD has a bug where the prometheus operator CRD cannot be installed
+because of the `CRD too long` error.
+This is a known issue with ArgoCD. See
+https://github.com/prometheus-community/helm-charts/issues/2479
+To work around this, stripped down CRDs are used.
+`make stripped-down-crd` will generate the stripped down CRDs in prometheus operator.
+
+
+## ArgoCD SyncWaves
+App of Apps pattern
+https://codefresh.io/blog/argo-cd-application-dependencies/
+https://redhat-scholars.github.io/argocd-tutorial/argocd-tutorial/04-syncwaves-hooks.html
+
+## ToDO (Use sops to encrypt passwords)
+https://cloud.redhat.com/blog/a-guide-to-gitops-and-secret-management-with-argocd-operator-and-sops

apps/clickhouse-operator/Chart.yaml

@@ -22,6 +22,10 @@ version: 0.21.2
 # follow Semantic Versioning. They should reflect the version the application is using.
 appVersion: "1.0"
 
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+
 dependencies:
 - name: altinity-clickhouse-operator
   version: 0.21.2

apps/clickhouse/demo.yaml

@@ -1,9 +1,39 @@
+## Prometheus operator podMonitor
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: clickhouse-metrics
+  namespace: ch
+  labels:
+    app: prometheus
+spec:
+  namespaceSelector:
+    matchNames:
+      - test
+  selector:
+    matchLabels:
+      clickhouse.altinity.com/app: chop
+  podMetricsEndpoints:
+    - port: "metrics"
+
+---
+
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
   name: "argocd"
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
 spec:
   configuration:
+    settings:
+      # listen clickhosue-server embedded prometheus endpoint
+      prometheus/endpoint: /metrics
+      prometheus/port: 9363
+      prometheus/metrics: true
+      prometheus/events: true
+      prometheus/asynchronous_metrics: true
+      prometheus/status_info: true
     clusters:
       - name: "demo"
         layout:
@@ -32,11 +62,21 @@ spec:
         - host: zookeeper
   templates:
     podTemplates:
+      - name: pod-template-with-metrics
+        metadata:
+          # for `additionalScrapeConfigs`, look https://github.com/Altinity/clickhouse-operator/blob/master/deploy/prometheus/prometheus-template.yaml#L11
+          annotations:
+            prometheus.io/scrape: "true"
+            prometheus.io/path: "/metrics"
+            prometheus.io/port: "9363"
       - name: server
         spec:
           containers:
           - name: clickhouse
-            image: altinity/clickhouse-server:23.3.8.22.altinitystable
+            image: altinity/clickhouse-server:22.3.15.34.altinitystable
+            ports:
+              - name: "metrics"
+                containerPort: 9363
     volumeClaimTemplates:
       - name: storage
         # Do not delete PVC if installation is dropped.
@@ -47,3 +87,33 @@ spec:
           resources:
             requests:
               storage: 50Gi
+---
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: "clickhouse-service"
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  defaults:
+    templates:
+      serviceTemplate: svc-template
+  configuration:
+    settings:
+      http_port: 8124
+      tcp_port: 9001
+      interserver_http_port: 9010
+  templates:
+    serviceTemplates:
+      - name: svc-template
+        generateName: chendpoint-{chi}
+        metadata:
+          labels:
+            custom.label: "custom.value"
+        spec:
+          ports:
+            - name: http
+              port: 8123
+            - name: tcp
+              port: 9000
+          type: ClusterIP

apps/grafana-datasource/datasource.yaml

@@ -0,0 +1,20 @@
+#apiVersion: grafana.integreatly.org/v1beta1
+#kind: GrafanaDatasource
+#metadata:
+#  name: clickhouse-grafana-datasource
+#  namespace: ch
+#spec:
+#  datasource:
+#    access: proxy
+#    type: vertamedia-clickhouse-datasource
+#    jsonData:
+#      timeInterval: 5s
+#      tlsSkipVerify: true
+#    name: clickhouse
+#    url: http://clickhouse:8123
+#  instanceSelector:
+#    matchLabels:
+#      dashboards: grafana
+#  plugins:
+#    - name: vertamedia-clickhouse-datasource
+#      version: 1.3.0

apps/grafana-datasource/grafana-dashboard-clickhouse-keeper-cr-template.yaml

@@ -0,0 +1,16 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: clickhouse-keeper-dashboard
+  labels:
+    app: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  url: https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/grafana-dashboard/ClickHouseKeeper_dashboard.json
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: clickhouse-operator-prometheus
+  # look to https://github.com/integr8ly/grafana-operator/issues/416#issuecomment-849498860, json used as fallback
+  json: ""

apps/grafana-datasource/grafana-dashboard-kafka-cr-template.yaml

@@ -0,0 +1,15 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: clickhouse-kafka-dashboard
+  labels:
+    app: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  url: https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/grafana-dashboard/Kafka_dashboard.json
+  json: ""
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: clickhouse-operator-prometheus

apps/grafana-datasource/grafana-dashboard-operator-cr-template.yaml

@@ -0,0 +1,22 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: "clickhouse-operator-dashboard"
+  labels:
+    app: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  url: https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/grafana-dashboard/Altinity_ClickHouse_Operator_dashboard.json
+  # look to https://github.com/integr8ly/grafana-operator/issues/416#issuecomment-849498860, json used as fallback
+  json: ""
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: clickhouse-operator-prometheus
+  # TODO remove this plugin definition after resolve https://github.com/integr8ly/grafana-operator/issues/155
+  plugins:
+    - name: "vertamedia-clickhouse-datasource"
+      version: "2.5.3"
+    - name: "grafana-piechart-panel"
+      version: "1.6.2"

apps/grafana-datasource/grafana-dashboard-queries-cr-template.yaml

@@ -0,0 +1,19 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: "clickhouse-queries-dashboard"
+  labels:
+    app: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  url: https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/grafana-dashboard/ClickHouse_Queries_dashboard.json
+  # look to https://github.com/integr8ly/grafana-operator/issues/416#issuecomment-849498860, json used as fallback
+  json: ""
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: "clickhouse-operator-prometheus"
+  plugins:
+    - name: "vertamedia-clickhouse-datasource"
+      version: "2.5.3"

apps/grafana-datasource/grafana-dashboard-zookeeper-cr-template.yaml

@@ -0,0 +1,16 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: "zookeeper-dashboard"
+  labels:
+    app: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  url: https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/grafana-dashboard/Zookeeper_dashboard.json
+  # look to https://github.com/integr8ly/grafana-operator/issues/416#issuecomment-849498860, json used as fallback
+  json: ""
+  datasources:
+    - inputName: "DS_PROMETHEUS"
+      datasourceName: "clickhouse-operator-prometheus"

apps/grafana-datasource/grafana-data-source-clickhouse-cr-template.yaml

@@ -0,0 +1,27 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: clickhouse-datasource
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+      name: "clickhouse"
+      type: vertamedia-clickhouse-datasource
+      access: proxy
+      url: "http://chi-argocd-demo-0-0:8123"
+      basicAuth: false
+      basicAuthUser: "root"
+      basicAuthPassword: "secretsecret"
+      isDefault: false
+      version: 1
+      editable: true
+      jsonData:
+        tlsSkipVerify: true
+        addCorsHeader: true
+        defaultDatabase: "default"
+        usePOST: true
+        useYandexCloudAuthorization: true
+        xHeaderUser: "root"
+        xHeaderKey: "secretsecret"

apps/grafana-datasource/grafana-data-source-prometheus-cr-template.yaml

@@ -0,0 +1,19 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: "clickhouse-grafana-datasource"
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+      name: "clickhouse-operator-prometheus"
+      type: prometheus
+      access: proxy
+      url: "http://prometheus:9090"
+      isDefault: true
+      version: 1
+      editable: false
+      jsonData:
+        tlsSkipVerify: true
+        timeInterval: "5s"

apps/prometheus/Chart.yaml → apps/grafana-operator/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-name: prometheus
+name: grafana-operator
 description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.
@@ -15,14 +15,14 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 23.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "1.0"
+# It is recommended to use it with quotes.
+appVersion: "v5.3.0"
 
-dependencies:
-- name: prometheus
-  version: 23.1.0
-  repository: https://prometheus-community.github.io/helm-charts/
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"