Skip to content

Commit 2d7ce91

Browse files
fsrv-xyzfsrv-xyz
committed
squash: feat: mysql support
Squashed commit of the following: * tests: optimizations ; local run script for integration tests Signed-off-by: Florian Bauer <[email protected]> * tests: add test configuration for percona Signed-off-by: Florian Bauer <[email protected]> * fix: improve integration tests; do not use manifests as a template any more Signed-off-by: Florian Bauer <[email protected]> * fix(mysql): add some logging Signed-off-by: Florian Bauer <[email protected]> * chore(.dockerignore): Add newline at end of file * chore(integration-tests): Add newline at end of setup.sh * ci: use custom build container image Signed-off-by: Florian Bauer <[email protected]> * tests(kubernetes): remove parallel execution from TestDatabase Signed-off-by: Florian Bauer <[email protected]> * fix(deployment): Update image version to latest * feat: mysql support Signed-off-by: Florian Bauer <[email protected]> See merge request https://ref.ci/fsrvcorp/integration/external-db-operator/-/merge_requests/19
1 parent 4b5a9a4 commit 2d7ce91

19 files changed

+636
-41
lines changed

‎.dockerignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
manifests/
2+
integration-tests/
3+
assets/
4+
.gitlab/
5+
.idea/
6+
README.md
7+
.gitlab-ci.yml

‎.gitlab-ci.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@ Integration Tests:
2525
variables:
2626
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
2727
before_script:
28-
- (apt update && apt install -y curl) > /dev/null 2>&1
28+
- (apt update && apt install -y curl git) > /dev/null 2>&1
2929
- curl -sSL https://go.dev/dl/go1.21.5.linux-amd64.tar.gz | tar -C /usr/local -xzf -
3030
- /usr/local/go/bin/go install gotest.tools/gotestsum@latest
3131
script:
3232
- bash -x integration-tests/setup.sh
33+
- eval $(cat .env)
3334
- /usr/local/go/bin/go test -v ./integration-tests/... --tags integration -json | /root/go/bin/gotestsum --junitfile report.xml --format testname --raw-command -- cat
3435
artifacts:
3536
reports:

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Operator Features:
1010

1111
Supported Databases:
1212
- PostgreSQL (via [pgx](https://github.com/jackc/pgx))
13+
- MySQL / MariaDB / Percona (via [go-sql-driver/mysql](https://github.com/go-sql-driver/mysql))
1314

1415
Other databases can be added by implementing the [Provider](internal/database/database.go) interface.
1516

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.21
44

55
require (
66
github.com/alecthomas/kingpin/v2 v2.4.0
7+
github.com/go-sql-driver/mysql v1.7.1
78
github.com/google/uuid v1.5.0
89
github.com/hellofresh/health-go/v5 v5.5.1
910
github.com/jackc/pgx/v5 v5.5.1

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ github.com/go-openapi/jsonreference v0.20.3 h1:EjGcjTW8pD1mRis6+w/gmoBdqv5+RbE9B
1919
github.com/go-openapi/jsonreference v0.20.3/go.mod h1:FviDZ46i9ivh810gqzFLl5NttD5q3tSlMLqLr6okedM=
2020
github.com/go-openapi/swag v0.22.5 h1:fVS63IE3M0lsuWRzuom3RLwUMVI2peDH01s6M70ugys=
2121
github.com/go-openapi/swag v0.22.5/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0=
22+
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
23+
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
2224
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
2325
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
2426
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=

‎integration-tests/kubernetes_test.go

+289-26
Original file line numberDiff line numberDiff line change
@@ -3,63 +3,326 @@
33
package integration_tests
44

55
import (
6-
"io"
6+
"context"
77
"os"
88
"os/exec"
99
"strings"
1010
"testing"
11+
"time"
12+
13+
"github.com/google/uuid"
14+
appsv1 "k8s.io/api/apps/v1"
15+
batchv1 "k8s.io/api/batch/v1"
16+
corev1 "k8s.io/api/core/v1"
17+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
18+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
19+
"k8s.io/apimachinery/pkg/runtime/schema"
20+
"k8s.io/apimachinery/pkg/util/intstr"
21+
"k8s.io/client-go/dynamic"
22+
"k8s.io/client-go/kubernetes"
23+
"k8s.io/client-go/tools/clientcmd"
1124
)
1225

1326
func kubectl(args ...string) *exec.Cmd {
1427
return exec.Command("kubectl", args...)
1528
}
1629

30+
func mapIncludesKeys[T any](m map[string]T, keys ...string) bool {
31+
for _, key := range keys {
32+
if _, ok := m[key]; !ok {
33+
return false
34+
}
35+
}
36+
return true
37+
}
38+
39+
func deployOperator(kubernetesApiClient *kubernetes.Clientset, name, provider, dsn string) error {
40+
operatorLabels := map[string]string{
41+
"app.kubernetes.io/name": "external-db-operator",
42+
"app.kubernetes.io/instance": name,
43+
}
44+
45+
image, operatorImageGiven := os.LookupEnv("OPERATOR_IMAGE")
46+
if !operatorImageGiven {
47+
image = "registry.fsrv.services/fsrvcorp/integration/external-db-operator:latest"
48+
}
49+
50+
deploymentConfiguration := &appsv1.Deployment{
51+
ObjectMeta: metav1.ObjectMeta{
52+
Name: "external-db-operator-" + name,
53+
},
54+
Spec: appsv1.DeploymentSpec{
55+
Selector: &metav1.LabelSelector{
56+
MatchLabels: operatorLabels,
57+
},
58+
Template: corev1.PodTemplateSpec{
59+
ObjectMeta: metav1.ObjectMeta{
60+
Labels: operatorLabels,
61+
},
62+
Spec: corev1.PodSpec{
63+
ServiceAccountName: "external-db-operator-sa",
64+
RestartPolicy: corev1.RestartPolicyAlways,
65+
InitContainers: []corev1.Container{
66+
{
67+
Name: "wait-for-database",
68+
Image: "alpine:latest",
69+
Command: []string{
70+
"sh",
71+
"-c",
72+
"until getent hosts " + name + ".databases.svc.cluster.local; do echo 'Waiting for database connection...' && sleep 1; done",
73+
},
74+
},
75+
},
76+
Containers: []corev1.Container{
77+
{
78+
Name: "external-db-operator",
79+
Image: image,
80+
ReadinessProbe: &corev1.Probe{
81+
InitialDelaySeconds: 5,
82+
PeriodSeconds: 2,
83+
ProbeHandler: corev1.ProbeHandler{
84+
HTTPGet: &corev1.HTTPGetAction{
85+
Path: "/status",
86+
Port: intstr.FromInt32(8080),
87+
},
88+
},
89+
},
90+
Env: []corev1.EnvVar{
91+
{
92+
Name: "DATABASE_PROVIDER",
93+
Value: provider,
94+
},
95+
{
96+
Name: "DATABASE_DSN",
97+
Value: dsn,
98+
},
99+
{
100+
Name: "INSTANCE_NAME",
101+
Value: name,
102+
},
103+
},
104+
},
105+
},
106+
},
107+
},
108+
},
109+
}
110+
111+
// check if deployment already exists
112+
_, getDeploymentError := kubernetesApiClient.AppsV1().Deployments("default").Get(context.Background(), deploymentConfiguration.Name, metav1.GetOptions{})
113+
if getDeploymentError == nil {
114+
// deployment already exists, delete it
115+
deleteDeploymentError := kubernetesApiClient.AppsV1().Deployments("default").Delete(context.Background(), deploymentConfiguration.Name, metav1.DeleteOptions{})
116+
if deleteDeploymentError != nil {
117+
return deleteDeploymentError
118+
}
119+
}
120+
121+
_, operatorDeployError := kubernetesApiClient.AppsV1().Deployments("default").Create(context.Background(), deploymentConfiguration, metav1.CreateOptions{})
122+
123+
return operatorDeployError
124+
}
125+
17126
func TestDatabase(t *testing.T) {
127+
clientConfig, _ := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
128+
kubernetesApiClient := kubernetes.NewForConfigOrDie(clientConfig)
129+
kubernetesDynamicClient := dynamic.NewForConfigOrDie(clientConfig)
130+
18131
kubectl("apply", "-f", "../manifests/crd.yaml").Run()
19132
kubectl("apply", "-f", "../manifests/rbac.yaml").Run()
20133

21-
deploymentFile, deploymentFileOpenError := os.Open("../manifests/deployment.yaml")
22-
if deploymentFileOpenError != nil {
23-
t.Fatalf("failed to open deployment manifest: %v", deploymentFileOpenError)
24-
}
25-
deploymentFileContent, manifestLoadError := io.ReadAll(deploymentFile)
26-
if manifestLoadError != nil {
27-
t.Fatalf("failed to load manifest: %v", manifestLoadError)
134+
databaseNamespace := "databases"
135+
kubectl("delete", "namespace", databaseNamespace).Run()
136+
kubectl("create", "namespace", databaseNamespace).Run()
137+
138+
type check struct {
139+
packages []string
140+
command string
28141
}
29-
deploymentManifest := string(deploymentFileContent)
142+
143+
mysqlProviderCheckCommand := "mysql -h $(cat /etc/db-credentials/host) -P $(cat /etc/db-credentials/port) -u $(cat /etc/db-credentials/username) -p$(cat /etc/db-credentials/password) $(cat /etc/db-credentials/database) -e 'SELECT 1'"
30144

31145
for _, testCase := range []struct {
32146
name string
33147
provider string
34148
dsn string
149+
check check
35150
}{
36151
{
37152
name: "postgres",
38153
provider: "postgres",
39-
dsn: "postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable",
154+
dsn: "postgres://postgres:[email protected]:5432/postgres?sslmode=disable",
155+
check: check{
156+
packages: []string{"postgresql-client"},
157+
command: "PGPASSWORD=$(cat /etc/db-credentials/password) psql -h $(cat /etc/db-credentials/host) -p $(cat /etc/db-credentials/port) -U $(cat /etc/db-credentials/username) -d $(cat /etc/db-credentials/database) -c 'SELECT 1'",
158+
},
159+
},
160+
{
161+
name: "mysql",
162+
provider: "mysql",
163+
dsn: "root:password@tcp(mysql.databases.svc.cluster.local:3306)/mysql?charset=utf8mb4&parseTime=True&loc=Local",
164+
check: check{
165+
packages: []string{"default-mysql-client"},
166+
command: mysqlProviderCheckCommand,
167+
},
168+
},
169+
{
170+
name: "mariadb",
171+
provider: "mysql",
172+
dsn: "root:password@tcp(mariadb.databases.svc.cluster.local:3306)/mysql?charset=utf8mb4&parseTime=True&loc=Local",
173+
check: check{
174+
packages: []string{"default-mysql-client"},
175+
command: mysqlProviderCheckCommand,
176+
},
177+
},
178+
{
179+
name: "percona",
180+
provider: "mysql",
181+
dsn: "root:password@tcp(percona.databases.svc.cluster.local:3306)/mysql?charset=utf8mb4&parseTime=True&loc=Local",
182+
check: check{
183+
packages: []string{"default-mysql-client"},
184+
command: mysqlProviderCheckCommand,
185+
},
40186
},
41187
} {
42188
t.Run(testCase.name, func(t *testing.T) {
43189
// Deploy test database
44-
kubectl("apply", "-f", "../manifests/databases/"+testCase.provider+".yaml").Run()
45-
46-
testCaseDeploymentManifest := strings.Replace(deploymentManifest, "<DSN>", testCase.dsn, -1)
47-
testCaseDeploymentManifest = strings.Replace(testCaseDeploymentManifest, "<PROVIDER>", testCase.provider, -1)
48-
apply := kubectl("apply", "-f", "-")
49-
apply.Stdin = strings.NewReader(testCaseDeploymentManifest)
50-
if applyOperatorOutput, applyOperatorError := apply.CombinedOutput(); applyOperatorError != nil {
51-
t.Errorf("failed to apply operator: %v \n %v", applyOperatorError, string(applyOperatorOutput))
52-
return
53-
} else {
54-
t.Log(string(applyOperatorOutput))
55-
}
190+
t.Run("deploy test database", func(t *testing.T) {
191+
if deployTestDatabaseOutput, deployTestDatabaseError := kubectl("apply", "-n", databaseNamespace, "-f", "../manifests/databases/"+testCase.name+".yaml").CombinedOutput(); deployTestDatabaseError != nil {
192+
t.Errorf("failed to deploy test database: %v \n %v", deployTestDatabaseError, string(deployTestDatabaseOutput))
193+
return
194+
} else {
195+
t.Log(string(deployTestDatabaseOutput))
196+
}
197+
})
198+
199+
// Deploy operator
200+
t.Run("deploy operator", func(t *testing.T) {
201+
deployOperatorError := deployOperator(kubernetesApiClient, testCase.name, testCase.provider, testCase.dsn)
202+
if deployOperatorError != nil {
203+
t.Fatalf("failed to deploy operator: %v", deployOperatorError)
204+
}
56205

57-
if output, waitError := kubectl("wait", "--for=condition=Available", "--timeout=1m", "deployment/external-db-operator-"+testCase.provider).CombinedOutput(); waitError != nil {
58-
t.Errorf("failed to wait for operator: %v \n %v", waitError, string(output))
206+
// Wait for operator to be ready
207+
if output, waitError := kubectl("wait", "--for=condition=Available", "--timeout=1m", "deployment/external-db-operator-"+testCase.name).CombinedOutput(); waitError != nil {
208+
t.Errorf("failed to wait for operator: %v \n %v", waitError, string(output))
209+
return
210+
} else {
211+
t.Log(string(output))
212+
}
213+
})
214+
215+
databaseResourceNamespace := "test-namespace-" + testCase.name
216+
217+
// Create test database
218+
kubectl("delete", "namespace", databaseResourceNamespace).Run()
219+
kubectl("create", "namespace", databaseResourceNamespace).Run()
220+
_, createTestDatabaseError := kubernetesDynamicClient.Resource(schema.GroupVersionResource(metav1.GroupVersionResource{
221+
Group: "bonsai-oss.org",
222+
Version: "v1",
223+
Resource: "databases",
224+
})).Namespace(databaseResourceNamespace).Create(context.Background(), &unstructured.Unstructured{
225+
Object: map[string]interface{}{
226+
"apiVersion": "bonsai-oss.org/v1",
227+
"kind": "Database",
228+
"metadata": map[string]interface{}{
229+
"name": "demo",
230+
"labels": map[string]interface{}{
231+
"bonsai-oss.org/external-db-operator": testCase.provider + "-" + testCase.name,
232+
},
233+
},
234+
},
235+
}, metav1.CreateOptions{})
236+
237+
if createTestDatabaseError != nil {
238+
t.Errorf("failed to create test database: %v", createTestDatabaseError)
59239
return
60-
} else {
61-
t.Log(string(output))
62240
}
241+
242+
time.Sleep(5 * time.Second)
243+
t.Run("check secret integrity", func(t *testing.T) {
244+
secret, secretGetError := kubernetesApiClient.
245+
CoreV1().
246+
Secrets(databaseResourceNamespace).
247+
Get(context.Background(), "edb-demo", metav1.GetOptions{})
248+
if secretGetError != nil {
249+
t.Errorf("failed to get secret: %v", secretGetError)
250+
return
251+
}
252+
253+
if !mapIncludesKeys(secret.Data, "username", "password", "host", "port", "database") {
254+
t.Errorf("secret does not contain all required keys")
255+
return
256+
}
257+
})
258+
259+
t.Run("check database integrity", func(t *testing.T) {
260+
_, jobError := kubernetesApiClient.BatchV1().Jobs(databaseResourceNamespace).Create(context.Background(), &batchv1.Job{
261+
ObjectMeta: metav1.ObjectMeta{
262+
Name: "db-integrity-check-" + uuid.NewString(),
263+
},
264+
Spec: batchv1.JobSpec{
265+
BackoffLimit: func() *int32 { i := int32(0); return &i }(),
266+
Template: corev1.PodTemplateSpec{
267+
Spec: corev1.PodSpec{
268+
RestartPolicy: corev1.RestartPolicyNever,
269+
Volumes: []corev1.Volume{
270+
{
271+
Name: "db-credentials",
272+
VolumeSource: corev1.VolumeSource{
273+
Secret: &corev1.SecretVolumeSource{
274+
SecretName: "edb-demo",
275+
},
276+
},
277+
},
278+
},
279+
Containers: []corev1.Container{
280+
{
281+
Name: "db-integrity-check",
282+
Image: "debian:sid",
283+
Command: []string{
284+
"bash",
285+
"-xc",
286+
"(apt update && apt install -y " + strings.Join(testCase.check.packages, " ") + ") >/dev/null 2>&1 && " + testCase.check.command,
287+
},
288+
VolumeMounts: []corev1.VolumeMount{
289+
{
290+
Name: "db-credentials",
291+
MountPath: "/etc/db-credentials",
292+
},
293+
},
294+
},
295+
},
296+
},
297+
},
298+
},
299+
}, metav1.CreateOptions{})
300+
if jobError != nil {
301+
t.Errorf("failed to create job: %v", jobError)
302+
return
303+
}
304+
305+
// check for job completion
306+
watcher, _ := kubernetesApiClient.BatchV1().Jobs(databaseResourceNamespace).Watch(context.Background(), metav1.ListOptions{
307+
Watch: true,
308+
})
309+
defer watcher.Stop()
310+
311+
for event := range watcher.ResultChan() {
312+
job, _ := event.Object.(*batchv1.Job)
313+
if len(job.Status.Conditions) == 0 {
314+
continue
315+
}
316+
317+
switch job.Status.Conditions[0].Type {
318+
case batchv1.JobComplete:
319+
return
320+
default:
321+
t.Errorf("job failed: %v", job.Status.Conditions[0].Message)
322+
return
323+
}
324+
}
325+
})
63326
})
64327
}
65328
}

0 commit comments

Comments
 (0)