Skip to content

Commit a8f9a46

Browse files
agneumagneum
committed
Merge branch 'check-dle-for-all-postrges-versions' into 'master'
chore: run DLE tests for all supported PostgreSQL versions See merge request postgres-ai/database-lab!350
2 parents a6839b3 + 9e53b15 commit a8f9a46

File tree

11 files changed

+72
-30
lines changed

11 files changed

+72
-30
lines changed

‎.gitlab-ci.yml

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ lint:
2222
script:
2323
- make lint
2424

25-
2625
.only_var_template: &only_tag_release
2726
only:
2827
variables:
@@ -320,7 +319,8 @@ build-image-swagger-latest:
320319
- export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
321320
- export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
322321

323-
bash-test:
322+
323+
.bash-test: &bash_test
324324
<<: *only_feature
325325
stage: integration-test
326326
variables:
@@ -333,9 +333,40 @@ bash-test:
333333
tags:
334334
- dle-test
335335

336+
bash-test-9-6:
337+
<<: *bash_test
338+
variables:
339+
POSTGRES_VERSION: "9.6"
340+
341+
bash-test-10:
342+
<<: *bash_test
343+
variables:
344+
POSTGRES_VERSION: 10
345+
346+
bash-test-11:
347+
<<: *bash_test
348+
variables:
349+
POSTGRES_VERSION: 11
350+
351+
bash-test-12:
352+
<<: *bash_test
353+
variables:
354+
POSTGRES_VERSION: 12
355+
356+
bash-test-13:
357+
<<: *bash_test
358+
variables:
359+
POSTGRES_VERSION: 13
360+
361+
bash-test-14:
362+
<<: *bash_test
363+
variables:
364+
POSTGRES_VERSION: 14
365+
336366
integration-test:
337367
services:
338-
- docker:dind
368+
- name: docker:dind
369+
command: ["--tls=false"]
339370
<<: *only_feature
340371
stage: integration-test
341372
variables:

‎Makefile

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,6 @@ test:
5959
test-ci-integration:
6060
GO111MODULE=on go test -race -tags=integration ./...
6161

62-
test-local-integration:
63-
sudo GO111MODULE=on /usr/local/go/bin/go test -race -tags=integration ./...
64-
6562
fmt:
6663
go fmt $$(go list ./... | grep -v /vendor/)
6764

‎configs/standard/postgres/default/14/postgresql.conf renamed to ‎configs/standard/postgres/default/14/postgresql.dblab.postgresql.conf

File renamed without changes.

‎pkg/retrieval/engine/postgres/snapshot/physical.go

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"context"
1212
"fmt"
1313
"io"
14-
"os"
1514
"path"
1615
"strings"
1716
"sync"
@@ -652,14 +651,18 @@ func (p *PhysicalInitial) getDSAFromWAL(ctx context.Context, pgVersion float64,
652651

653652
walDirectory := walDir(cloneDir, pgVersion)
654653

655-
infos, err := os.ReadDir(walDirectory)
654+
output, err := tools.ExecCommandWithOutput(ctx, p.dockerClient, containerID, types.ExecConfig{
655+
Cmd: []string{"ls", "-t", walDirectory},
656+
})
656657
if err != nil {
657-
return "", errors.Wrap(err, "failed to read the pg_wal dir")
658+
return "", errors.Wrap(err, "failed to read the wal directory")
658659
}
659660

661+
walFileList := strings.Fields(output)
662+
660663
// Walk in the reverse order.
661-
for i := len(infos) - 1; i >= 0; i-- {
662-
fileName := infos[i].Name()
664+
for i := len(walFileList) - 1; i >= 0; i-- {
665+
fileName := walFileList[i]
663666
walFilePath := path.Join(walDirectory, fileName)
664667

665668
log.Dbg("Look up into file: ", walFilePath)

‎pkg/retrieval/engine/postgres/snapshot/physical_integration_test.go

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,8 @@ import (
1111
"fmt"
1212
"os"
1313
"testing"
14-
"time"
1514

1615
"github.com/docker/docker/client"
17-
"github.com/docker/go-connections/nat"
1816
_ "github.com/lib/pq"
1917
"github.com/stretchr/testify/assert"
2018
"github.com/stretchr/testify/require"
@@ -83,6 +81,7 @@ const (
8381
dbname = "postgres"
8482
user = "postgres"
8583
testPassword = "password"
84+
pgdata = "/var/lib/postgresql/data/"
8685
)
8786

8887
func TestParsingWAL96(t *testing.T) {
@@ -100,7 +99,7 @@ func TestParsingWAL(t *testing.T) {
10099
t.Fatal("Failed to create a Docker client:", err)
101100
}
102101

103-
postgresVersions := []float64{10, 11, 12, 13}
102+
postgresVersions := []float64{10, 11, 12, 13, 14}
104103

105104
for _, pgVersion := range postgresVersions {
106105
testWALParsing(t, dockerCLI, pgVersion, initialScript)
@@ -122,26 +121,17 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
122121
logStrategyForAcceptingConnections := wait.NewLogStrategy("database system is ready to accept connections")
123122
logStrategyForAcceptingConnections.Occurrence = 2
124123

125-
dbURL := func(port nat.Port) string {
126-
return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
127-
"localhost", port.Port(), user, testPassword, dbname)
128-
}
129-
130124
req := testcontainers.ContainerRequest{
131125
Name: "pg_test_" + pgVersionString,
132126
Image: "postgres:" + pgVersionString,
133127
ExposedPorts: []string{port},
134128
WaitingFor: wait.ForAll(
135129
logStrategyForAcceptingConnections,
136130
wait.ForLog("PostgreSQL init process complete; ready for start up."),
137-
wait.ForSQL(nat.Port(port), "postgres", dbURL).Timeout(10*time.Second),
138131
),
139-
BindMounts: map[string]string{
140-
dir: "/var/lib/postgresql/data",
141-
"/tmp": "/tmp",
142-
},
143132
Env: map[string]string{
144133
"POSTGRES_PASSWORD": testPassword,
134+
"PGDATA": pgdata,
145135
},
146136
}
147137

@@ -155,7 +145,6 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
155145

156146
// Prepare test data.
157147
code, err := postgresContainer.Exec(ctx, []string{"psql", "-U", user, "-d", dbname, "-XAtc", initialSQL})
158-
159148
require.Nil(t, err)
160149
assert.Equal(t, 0, code)
161150

@@ -164,7 +153,7 @@ func testWALParsing(t *testing.T, dockerCLI *client.Client, pgVersion float64, i
164153
}
165154

166155
// Check WAL parsing.
167-
dsa, err := p.getDSAFromWAL(ctx, pgVersion, postgresContainer.GetContainerID(), dir)
156+
dsa, err := p.getDSAFromWAL(ctx, pgVersion, postgresContainer.GetContainerID(), pgdata)
168157
assert.Nil(t, err)
169158
assert.NotEmpty(t, dsa)
170159

‎test/1.synthetic.sh

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
TAG="${TAG:-"master"}"
4+
TAG=${TAG:-${CI_COMMIT_REF_SLUG}}
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
66
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
77

@@ -25,16 +25,24 @@ sudo docker run \
2525
postgres:"${POSTGRES_VERSION}"-alpine
2626

2727
for i in {1..300}; do
28-
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
28+
sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
29+
sleep 1
30+
done
31+
32+
# Restart container explicitly after initdb to make sure that the server will not receive a shutdown request and queries will not be interrupted.
33+
sudo docker restart dblab_pg_initdb
34+
35+
for i in {1..300}; do
36+
sudo docker exec dblab_pg_initdb psql -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
2937
sleep 1
3038
done
3139

3240
# Create the test database
33-
sudo docker exec -it dblab_pg_initdb psql -U postgres -c 'create database test'
41+
sudo docker exec dblab_pg_initdb psql -U postgres -c 'create database test'
3442

3543
# Generate data in the test database using pgbench
3644
# 1,000,000 accounts, ~0.14 GiB of data.
37-
sudo docker exec -it dblab_pg_initdb pgbench -U postgres -i -s 10 test
45+
sudo docker exec dblab_pg_initdb pgbench -U postgres -i -s 10 test
3846

3947
# Stop and remove the container
4048
sudo docker stop dblab_pg_initdb
@@ -51,11 +59,18 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
5159

5260
# Edit the following options
5361
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' "${configDir}/server.yml"
62+
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
5463
sed -ri 's/^(\s*)(- logicalDump$)/\1/' "${configDir}/server.yml"
5564
sed -ri 's/^(\s*)(- logicalRestore$)/\1/' "${configDir}/server.yml"
5665
# replace postgres version
5766
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
5867

68+
69+
# logerrors is not supported in PostgreSQL 9.6
70+
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
71+
sed -ri 's/^(\s*)(shared_preload_libraries:.*$)/\1shared_preload_libraries: "pg_stat_statements, auto_explain"/' "${configDir}/server.yml"
72+
fi
73+
5974
## Launch Database Lab server
6075
sudo docker run \
6176
--name dblab_server \

‎test/2.logical_generic.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
6060

6161
# Edit the following options
6262
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
63+
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
6364
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
6465
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${SOURCE_HOST}/" "${configDir}/server.yml"
6566
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${SOURCE_PORT}/" "${configDir}/server.yml"

‎test/3.physical_walg.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
3838

3939
# Edit the following options
4040
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
41+
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
4142
# set WAL-G envs
4243
sed -ri "s/^(\s*)(backupName:.*$)/\1backupName: ${WALG_BACKUP_NAME}/" "${configDir}/server.yml"
4344
set +euxo pipefail # ---- do not display secrets

‎test/4.physical_basebackup.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
6464

6565
# Edit the following options
6666
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
67+
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
6768
sed -ri "s/^(\s*)(PGUSER:.*$)/\1PGUSER: ${SOURCE_USERNAME}/" "${configDir}/server.yml"
6869
sed -ri "s/^(\s*)(PGPASSWORD:.*$)/\1PGPASSWORD: ${SOURCE_PASSWORD}/" "${configDir}/server.yml"
6970
sed -ri "s/^(\s*)(PGHOST:.*$)/\1PGHOST: ${SOURCE_HOST}/" "${configDir}/server.yml"

‎test/5.logical_rds.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.e
3333

3434
# Edit the following options
3535
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
36+
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
3637
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
3738
sed -ri "s/^(\s*)(username: test_user.*$)/\1username: \"${SOURCE_USERNAME}\"/" "${configDir}/server.yml"
3839
sed -ri "s/^(\s*)(awsRegion:.*$)/\1awsRegion: \"${AWS_REGION}\"/" "${configDir}/server.yml"

0 commit comments

Comments
 (0)