Skip to content

Commit df458de

Browse files
aclevernameaclevername
committed
feat: add elasticcloud promise
0 parents  commit df458de

17 files changed

+16275
-0
lines changed

Diff for: ‎.circleci/config.yml

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
version: 2.1
2+
3+
orbs:
4+
k8s: circleci/[email protected]
5+
retry: kimh/[email protected]
6+
7+
executors:
8+
machine-medium:
9+
machine: true
10+
resource_class: medium
11+
working_directory: ~/repo
12+
machine-large:
13+
machine: true
14+
resource_class: large
15+
working_directory: ~/repo
16+
17+
commands:
18+
install_software:
19+
steps:
20+
- k8s/install
21+
- run:
22+
name: Install tools and deps
23+
command: |
24+
if [ ! -f ~/bin/kind ]; then
25+
curl -L https://github.com/kubernetes-sigs/kind/releases/download/v0.14.0/kind-linux-amd64 -o ~/bin/kind
26+
chmod +x ~/bin/kind
27+
fi
28+
29+
curl -L https://github.com/syntasso/kratix/releases/download/v0.0.1/worker-resource-builder-v0.0.0-1-linux-amd64 -o ~/bin/worker-resource-builder
30+
chmod +x ~/bin/worker-resource-builder
31+
- run:
32+
name: Install Kratix
33+
command: |
34+
kind create cluster --name platform
35+
kubectl apply --filename https://raw.githubusercontent.com/syntasso/kratix/main/distribution/single-cluster/install-all-in-one.yaml
36+
sleep 5
37+
kubectl apply --filename https://raw.githubusercontent.com/syntasso/kratix/main/distribution/single-cluster/config-all-in-one.yaml
38+
sleep 5
39+
- attach_workspace:
40+
at: .
41+
42+
jobs:
43+
test-and-push:
44+
parameters:
45+
promise:
46+
type: string
47+
executor:
48+
name: machine-large
49+
steps:
50+
- install_software
51+
- checkout
52+
- run:
53+
name: Validate no pending changes
54+
command: |
55+
./internal/scripts/inject-wcr
56+
57+
if ! git diff --exit-code .; then
58+
echo ""
59+
echo "Changes in the WCR were detected"
60+
echo "Injection of WCR via CI is not supported. Please commit and push them manually."
61+
exit 1
62+
fi
63+
- run:
64+
name: Install Promise
65+
command: |
66+
kubectl create --filename promise.yaml
67+
./internal/scripts/pipeline-image build load
68+
- retry/run-with-retry:
69+
command: ./internal/scripts/test promise
70+
# 5 minutes total retry (3 sec * 100 times = 300 sec)
71+
sleep: 3
72+
retry-count: 100
73+
- run:
74+
name: Apply resource-request
75+
command: |
76+
if test -f "resource-request.yaml"; then
77+
kubectl apply --filename resource-request.yaml
78+
fi
79+
- retry/run-with-retry:
80+
command: ./internal/scripts/test resource-request
81+
# 5 minutes total retry (3 sec * 100 times = 300 sec)
82+
sleep: 3
83+
retry-count: 100
84+
- when:
85+
condition:
86+
equal: [main, << pipeline.git.branch >>]
87+
steps:
88+
- run:
89+
name: GHCR Push
90+
command: |
91+
echo "$GITHUB_TOKEN" | docker login ghcr.io -u syntassodev --password-stdin
92+
./internal/scripts/pipeline-image push
93+
- run:
94+
name: Verify package is public
95+
command: |
96+
if test -f "internal/request-pipeline/Dockerfile"; then
97+
visibility=$(curl -sL \
98+
-H "Accept: application/vnd.github+json"\
99+
-H "Authorization: Bearer $GITHUB_TOKEN" \
100+
-H "X-GitHub-Api-Version: 2022-11-28" \
101+
"https://api.github.com/orgs/syntasso/packages?package_type=container" |
102+
jq -r '.[] | select(.name | contains ("<< parameters.promise >>")) | .visibility')
103+
104+
test "${visibility}" = "public"
105+
fi
106+
107+
workflows:
108+
promises:
109+
jobs:
110+
- test-and-push:
111+
matrix:
112+
parameters:
113+
promise:
114+
- elasticcloud

Diff for: ‎.envrc

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export PATH="$(pwd)/internal/scripts:$PATH"

Diff for: ‎README.md

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Elasitc Cloud on Kubernetes
2+
3+
This Promise deploys an Elasticsearch and a Kibana instance, using [Elastic Cloud on
4+
Kubernetes
5+
(ECK)](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html). The
6+
promise has a single field:
7+
* `spec.env`
8+
9+
Check the CRD documentation for more information.
10+
11+
To install, run the following command while targeting your Platform cluster:
12+
```
13+
kubectl create -f https://raw.githubusercontent.com/syntasso/promise-elasticcloud/main/promise.yaml
14+
```
15+
16+
This will start the Elastic Operator on Worker Cluster. To verify the installation, run the following
17+
command while targeting the Worker cluster (it may take a couple of minutes):
18+
19+
```shell-session
20+
$ kubectl get all --namespace elastic-system
21+
NAME READY STATUS RESTARTS AGE
22+
pod/elastic-operator-0 1/1 Running 0 49s
23+
24+
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
25+
service/elastic-webhook-server ClusterIP 10.96.99.51 <none> 443/TCP 49s
26+
27+
NAME READY AGE
28+
statefulset.apps/elastic-operator 1/1 49s
29+
```
30+
31+
To create an instance of Elastic Search with Kibana, run the following command while targeting the Platform cluster:
32+
```
33+
kubectl apply -f https://raw.githubusercontent.com/syntasso/promise-elasticcloud/main/resource-request.yaml
34+
```
35+
36+
To verify that the Elastic Stack is created, run the following command while targeting the Worker cluster:
37+
```shell-session
38+
$ kubectl get elasticsearches.elasticsearch.k8s.elastic.co
39+
NAME HEALTH NODES VERSION PHASE AGE
40+
example yellow 1 8.5.3 Ready 4m15s
41+
```
42+
## Accessing Kibana
43+
44+
A ClusterIP Service is automatically created for Kibana:
45+
46+
```
47+
kubectl get service example-kb-http
48+
```
49+
50+
Use kubectl port-forward to access Kibana from your local workstation:
51+
52+
```
53+
kubectl port-forward service/example-kb-http 5601
54+
```
55+
56+
Open https://localhost:5601 in your browser. Login as the `elastic` user. The password can be obtained with the following command:
57+
58+
```
59+
kubectl get secret quickstart-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode
60+
```
61+
62+
## Development
63+
64+
For development see [README.md](./internal/README.md)

Diff for: ‎internal/README.md

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Development
2+
3+
## Build promise.yaml
4+
The `promise.yaml` is generated by interpolating the contents of `resources/` into
5+
the `workerClusterResources`.
6+
7+
These resources were downloaded using `./scripts/fetch-crds`.
8+
9+
10+
To build run:
11+
12+
```
13+
./scripts/inject-wcr
14+
```
15+
16+
## Pipeline image
17+
18+
The resources used in the pipeline were downloaded using `./scripts/fetch-pipeline-resources`.
19+
20+
21+
To build the image:
22+
```
23+
./scripts/pipeline-image build
24+
```
25+
26+
To load the image to the local kind platform cluster:
27+
```
28+
./scripts/pipeline-image load
29+
```
30+
31+
To push the image to ghcr.io:
32+
```
33+
./scripts/pipeline-image push
34+
```
35+
36+
## Testing
37+
To test the promise install kratix, and then:
38+
```
39+
kubectl apply -f promise.yaml
40+
./scripts/test promise
41+
```
42+
43+
This asserts the Promise is installed correctly.
44+
45+
To test the resource request:
46+
```
47+
kubectl apply -f resource-request.yaml
48+
./scripts/test resource-request
49+
```

Diff for: ‎internal/request-pipeline/Dockerfile

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
FROM "alpine"
2+
3+
LABEL org.opencontainers.image.authors "[email protected]"
4+
LABEL org.opencontainers.image.source https://github.com/syntasso/kratix-marketplace
5+
6+
RUN [ "mkdir", "/tmp/transfer" ]
7+
RUN apk update && apk add --no-cache yq
8+
9+
ADD resources/* /tmp/transfer/
10+
ADD execute-pipeline execute-pipeline
11+
12+
CMD [ "sh", "-c", "./execute-pipeline" ]
13+
ENTRYPOINT []

Diff for: ‎internal/request-pipeline/execute-pipeline

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env sh
2+
3+
set -ex
4+
5+
name="$(yq eval '.metadata.name' /input/object.yaml)"
6+
env_type="$(yq eval '.spec.env // "dev"' /input/object.yaml)"
7+
8+
sed "s/TBDNAME/${name}/g" /tmp/transfer/baseline-resources.yaml > /output/baseline-resources.yaml
9+
if [ $env_type = "prod" ]; then
10+
sed "s/TBDNAME/${name}/g" /tmp/transfer/autoscaling.yaml > /output/autoscaling.yaml
11+
fi

Diff for: ‎internal/request-pipeline/resources/autoscaling.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
---
2+
apiVersion: autoscaling.k8s.elastic.co/v1alpha1
3+
kind: ElasticsearchAutoscaler
4+
metadata:
5+
name: TBDNAME
6+
namespace: default
7+
spec:
8+
elasticsearchRef:
9+
name: TBDNAME
10+
policies:
11+
- name: data-ingest
12+
roles: ["data", "ingest" , "transform"]
13+
resources:
14+
nodeCount: {min: 1, max: 3}
15+
cpu: {min: 1, max: 2}
16+
memory: {min: 2Gi, max: 3Gi}
17+
storage: {min: 1Gi, max: 5Gi}
18+
- name: ml
19+
roles:
20+
- ml
21+
resources:
22+
nodeCount: {min: 1, max: 3}
23+
cpu: {min: 1, max: 2}
24+
memory: {min: 2Gi, max: 3Gi}
25+
storage: {min: 1Gi, max: 5Gi}
26+
---
27+
apiVersion: autoscaling/v2beta2
28+
kind: HorizontalPodAutoscaler
29+
metadata:
30+
name: TBDNAME
31+
namespace: default
32+
spec:
33+
scaleTargetRef:
34+
apiVersion: kibana.k8s.elastic.co/v1
35+
kind: Kibana
36+
name: TBDNAME
37+
minReplicas: 1
38+
maxReplicas: 4
39+
metrics:
40+
- type: Resource
41+
resource:
42+
name: cpu
43+
target:
44+
type: Utilization
45+
averageUtilization: 50

Diff for: ‎internal/request-pipeline/resources/baseline-resources.yaml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
apiVersion: elasticsearch.k8s.elastic.co/v1
3+
kind: Elasticsearch
4+
metadata:
5+
name: TBDNAME
6+
namespace: default
7+
spec:
8+
version: 8.5.3
9+
nodeSets:
10+
- name: default
11+
count: 1
12+
config:
13+
node.store.allow_mmap: false
14+
---
15+
apiVersion: kibana.k8s.elastic.co/v1
16+
kind: Kibana
17+
metadata:
18+
name: TBDNAME
19+
namespace: default
20+
spec:
21+
version: 8.5.3
22+
count: 1
23+
elasticsearchRef:
24+
name: TBDNAME
25+
---
26+
apiVersion: beat.k8s.elastic.co/v1beta1
27+
kind: Beat
28+
metadata:
29+
name: TBDNAME
30+
namespace: default
31+
spec:
32+
type: filebeat
33+
version: 8.5.3
34+
elasticsearchRef:
35+
name: TBDNAME
36+
config:
37+
filebeat.inputs:
38+
- type: container
39+
paths:
40+
- /var/log/containers/*.log
41+
daemonSet:
42+
podTemplate:
43+
spec:
44+
dnsPolicy: ClusterFirstWithHostNet
45+
hostNetwork: true
46+
securityContext:
47+
runAsUser: 0
48+
containers:
49+
- name: filebeat
50+
volumeMounts:
51+
- name: varlogcontainers
52+
mountPath: /var/log/containers
53+
- name: varlogpods
54+
mountPath: /var/log/pods
55+
- name: varlibdockercontainers
56+
mountPath: /var/lib/docker/containers
57+
volumes:
58+
- name: varlogcontainers
59+
hostPath:
60+
path: /var/log/containers
61+
- name: varlogpods
62+
hostPath:
63+
path: /var/log/pods
64+
- name: varlibdockercontainers
65+
hostPath:
66+
path: /var/lib/docker/containers

0 commit comments

Comments
 (0)