Grafana operator dashboard

.gitignore

@@ -0,0 +1 @@
+.idea/

README.md

@@ -9,7 +9,15 @@ You'll need a Kubernetes cluster. EKS, GKE, or minikube should all be OK.
 
 Install the following tools in your environment. ArgoCD should be installed
 in the argocd namespace. 
+You can use the script
+```
+./install-argocd.sh
+```
 
+```
+kubectl create namespace argocd
+kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
+```
 * [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
 * [argocd](https://argo-cd.readthedocs.io/en/stable/getting_started/)
 * [kustomize](https://kubectl.docs.kubernetes.io/installation/kustomize/)
@@ -21,16 +29,34 @@ kubectl get ns
 
 ## Login to ArgoCD
 
+ArgoCD default password can be retrieved from k8s secrets
+```
+kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d
+```
 All commands with argocd require an accessible ArgoCD API endpoint and a
 valid token. Get them as follows.
 ```
 kubectl port-forward svc/argocd-server -n argocd 8080:443 &
 argocd login localhost:8080 --username=admin --password="yourpassword" --insecure
 ```
 
+You can also use the script.
+```
+./login-argocd.sh
+```
 ## Bring up the analytic stack
 
+Run the `create-infra-stack.sh`  to create the infra applications
+```
+cd apps/infra
+./create-infra-stack.sh -b <branch-name> 
+```
 Run the `create-stack.sh` script in the apps directory.
+If you need to the run changes from a specific branch
+use the following command
+```
+./create-stack.sh -b <branch-name>
+```
 
 ## Make endpoints accessible
 
@@ -48,3 +74,37 @@ following steps to clean up fully.
 
 1. Run 'kubectl edit' on any chi resource and remove the finalizer. 
 2. Drop the namespace. 
+
+### Grafana (Manual steps)
+The admin password can be reset by executing the following command on the grafana pod.
+```
+grafana-cli admin reset-admin-password admin
+```
+
+The `admin` password is stored in `grafana-admin-credentials` secret.
+```
+kubectl get secret grafana-admin-credentials --template={{.data.GF_SECURITY_ADMIN_PASSWORD}} -n ch|base64 -d
+```
+
+Open the grafana UI by setting up port-forward on the grafana pod.
+```
+kubectl port-forward -n ch svc/grafana-service 3000:3000
+```
+Login as admin and the password retrieved in the previous step.
+
+### Prometheus Operator
+ArgoCD has a bug where the prometheus operator CRD cannot be installed
+because of the `CRD too long` error.
+This is a known issue with ArgoCD. See
+https://github.com/prometheus-community/helm-charts/issues/2479
+To work around this, stripped down CRDs are used.
+`make stripped-down-crd` will generate the stripped down CRDs in prometheus operator.
+
+
+## ArgoCD SyncWaves
+App of Apps pattern
+https://codefresh.io/blog/argo-cd-application-dependencies/
+https://redhat-scholars.github.io/argocd-tutorial/argocd-tutorial/04-syncwaves-hooks.html
+
+## ToDO (Use sops to encrypt passwords)
+https://cloud.redhat.com/blog/a-guide-to-gitops-and-secret-management-with-argocd-operator-and-sops

apps/clickhouse-operator/Chart.yaml

@@ -22,6 +22,10 @@ version: 0.21.2
 # follow Semantic Versioning. They should reflect the version the application is using.
 appVersion: "1.0"
 
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+
 dependencies:
 - name: altinity-clickhouse-operator
   version: 0.21.2

apps/clickhouse/demo.yaml

@@ -1,9 +1,41 @@
+## Prometheus operator podMonitor
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: clickhouse-metrics
+  namespace: ch
+  labels:
+    app: prometheus
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  namespaceSelector:
+    matchNames:
+      - test
+  selector:
+    matchLabels:
+      clickhouse.altinity.com/app: chop
+  podMetricsEndpoints:
+    - port: "metrics"
+
+---
+
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseInstallation"
 metadata:
   name: "argocd"
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
 spec:
   configuration:
+    settings:
+      # listen clickhosue-server embedded prometheus endpoint
+      prometheus/endpoint: /metrics
+      prometheus/port: 9363
+      prometheus/metrics: true
+      prometheus/events: true
+      prometheus/asynchronous_metrics: true
+      prometheus/status_info: true
     clusters:
       - name: "demo"
         layout:
@@ -32,11 +64,21 @@ spec:
         - host: zookeeper
   templates:
     podTemplates:
+      - name: pod-template-with-metrics
+        metadata:
+          # for `additionalScrapeConfigs`, look https://github.com/Altinity/clickhouse-operator/blob/master/deploy/prometheus/prometheus-template.yaml#L11
+          annotations:
+            prometheus.io/scrape: "true"
+            prometheus.io/path: "/metrics"
+            prometheus.io/port: "9363"
       - name: server
         spec:
           containers:
           - name: clickhouse
-            image: altinity/clickhouse-server:23.3.8.22.altinitystable
+            image: altinity/clickhouse-server:22.3.15.34.altinitystable
+            ports:
+              - name: "metrics"
+                containerPort: 9363
     volumeClaimTemplates:
       - name: storage
         # Do not delete PVC if installation is dropped.