Skip to content

Commit 350ce6e

Browse files
dveritedverite
authored
Merge pull request #60 from qonto/improve-tests
Improve end-to-end tests
2 parents cc54734 + b071150 commit 350ce6e

File tree

7 files changed

+174
-22
lines changed

7 files changed

+174
-22
lines changed

‎internal/infra/partition/bounds.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func (r PartitionRange) IsEqual(r2 PartitionRange) bool {
5454
return r.LowerBound.Equal(r2.LowerBound) && r.UpperBound.Equal(r2.UpperBound)
5555
}
5656

57-
// Intersection returns the intersection between the intervals r1 and r2
57+
// Intersection returns the intersection between the intervals r and r2
5858
func (r PartitionRange) Intersection(r2 PartitionRange) PartitionRange {
5959
var res PartitionRange // initialized with {time.Time{}, time.Time{}}
6060

‎scripts/bats/20_check.bats

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ load 'test/libs/partitions'
33
load 'test/libs/seeds'
44
load 'test/libs/sql'
55

6+
setup_file() {
7+
reset_database
8+
}
9+
610
setup() {
711
bats_load_library bats-support
812
bats_load_library bats-assert
9-
10-
reset_database
1113
}
1214

1315
@test "Test exit code on PostgreSQL connection error" {
@@ -106,3 +108,42 @@ EOF
106108
assert_failure
107109
assert_output --partial "Found missing tables"
108110
}
111+
112+
@test "Test check succeeding with an UUID partition key" {
113+
local TABLE="test_uuid_1"
114+
local INTERVAL=monthly
115+
local RETENTION=2
116+
local PREPROVISIONED=2
117+
118+
create_table_uuid_range ${TABLE}
119+
120+
declare -a PARTS=(
121+
test_uuid_1_2024_12 01937f84-5800-7000-0000-000000000000 01941f29-7c00-7000-0000-000000000000
122+
test_uuid_1_2025_01 01941f29-7c00-7000-0000-000000000000 0194bece-a000-7000-0000-000000000000
123+
test_uuid_1_2025_02 0194bece-a000-7000-0000-000000000000 01954f00-b000-7000-0000-000000000000
124+
test_uuid_1_2025_03 01954f00-b000-7000-0000-000000000000 0195eea5-d400-7000-0000-000000000000
125+
test_uuid_1_2025_04 0195eea5-d400-7000-0000-000000000000 01968924-9c00-7000-0000-000000000000
126+
)
127+
128+
create_partitions "$TABLE" "${PARTS[@]}"
129+
130+
local CONFIGURATION=$(cat << EOF
131+
partitions:
132+
unittest:
133+
schema: public
134+
table: ${TABLE}
135+
interval: ${INTERVAL}
136+
partitionKey: id
137+
cleanupPolicy: drop
138+
retention: ${RETENTION}
139+
preProvisioned: ${PREPROVISIONED}
140+
EOF
141+
)
142+
local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
143+
144+
PPM_WORK_DATE="2025-02-10" run postgresql-partition-manager run check -c ${CONFIGURATION_FILE}
145+
146+
assert_success
147+
assert_output --partial "All partitions are correctly configured"
148+
rm "$CONFIGURATION_FILE"
149+
}

‎scripts/bats/30_provisioning.bats

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ load 'test/libs/seeds'
44
load 'test/libs/sql'
55
load 'test/libs/time'
66

7+
setup_file() {
8+
reset_database
9+
}
10+
711
setup() {
812
bats_load_library bats-support
913
bats_load_library bats-assert
10-
11-
reset_database
1214
}
1315

1416
@test "Test that provisioning succeed on up-to-date partitioning" {
@@ -315,7 +317,10 @@ EOF
315317

316318
@test "Test that provisioning continues after an error on a partition set" {
317319
# Have a non-existing parent table, plus a normal set of partitions
318-
local CONFIGURATION=$(cat << EOF
320+
321+
local TABLE=$(generate_table_name)
322+
323+
local CONFIGURATION=$(cat << EOF
319324
partitions:
320325
unittest1:
321326
schema: public
@@ -327,7 +332,7 @@ partitions:
327332
preProvisioned: 1
328333
unittest2:
329334
schema: public
330-
table: table_unittest2
335+
table: ${TABLE}
331336
interval: daily
332337
partitionKey: created_at
333338
cleanupPolicy: drop
@@ -337,23 +342,23 @@ EOF
337342
)
338343
local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
339344

340-
create_partitioned_table "table_unittest2"
345+
create_partitioned_table "${TABLE}"
341346

342347
PPM_WORK_DATE="2025-02-01" run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE}
343348

344349
# The status must reflect the fact that one partition set failed
345350
[ "$status" -eq 4 ] # PartitionsProvisioningFailedExitCode
346351

347352
# Check the success of the second set of partitions
348-
local expected2=$(cat <<'EOF'
349-
public|table_unittest2_2025_01_30|2025-01-30|2025-01-31
350-
public|table_unittest2_2025_01_31|2025-01-31|2025-02-01
351-
public|table_unittest2_2025_02_01|2025-02-01|2025-02-02
352-
public|table_unittest2_2025_02_02|2025-02-02|2025-02-03
353-
public|table_unittest2_2025_02_03|2025-02-03|2025-02-04
353+
local expected2=$(cat <<EOF
354+
public|${TABLE}_2025_01_30|2025-01-30|2025-01-31
355+
public|${TABLE}_2025_01_31|2025-01-31|2025-02-01
356+
public|${TABLE}_2025_02_01|2025-02-01|2025-02-02
357+
public|${TABLE}_2025_02_02|2025-02-02|2025-02-03
358+
public|${TABLE}_2025_02_03|2025-02-03|2025-02-04
354359
EOF
355360
)
356361

357-
run list_existing_partitions "unittest" "public" "table_unittest2"
362+
run list_existing_partitions "unittest" "public" "${TABLE}"
358363
assert_output "$expected2"
359364
}

‎scripts/bats/40_cleanup.bats

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ load 'test/libs/partitions'
33
load 'test/libs/seeds'
44
load 'test/libs/sql'
55

6+
setup_file() {
7+
reset_database
8+
}
9+
610
setup() {
711
bats_load_library bats-support
812
bats_load_library bats-assert
9-
10-
reset_database
1113
}
1214

1315
@test "Test that useless partitions are removed by the cleanup" {
@@ -60,3 +62,62 @@ EOF
6062
assert_table_not_exists public $(generate_daily_partition_name ${TABLE} ${i})
6163
done
6264
}
65+
66+
@test "Test that gaps in the partition set prevent any partition removal" {
67+
local TABLE="test_uuid_gap"
68+
local INTERVAL=monthly
69+
local RETENTION=1
70+
local PREPROVISIONED=2
71+
72+
create_table_uuid_range ${TABLE}
73+
74+
declare -a PARTS=(
75+
${TABLE}_2024_12 01937f84-5800-7000-0000-000000000000 01941f29-7c00-7000-0000-000000000000
76+
# next partition is missing
77+
# ${TABLE}_2025_01 01941f29-7c00-7000-0000-000000000000 0194bece-a000-7000-0000-000000000000
78+
${TABLE}_2025_02 0194bece-a000-7000-0000-000000000000 01954f00-b000-7000-0000-000000000000
79+
${TABLE}_2025_03 01954f00-b000-7000-0000-000000000000 0195eea5-d400-7000-0000-000000000000
80+
${TABLE}_2025_04 0195eea5-d400-7000-0000-000000000000 01968924-9c00-7000-0000-000000000000
81+
)
82+
83+
create_partitions "$TABLE" "${PARTS[@]}"
84+
85+
local CONFIGURATION=$(cat << EOF
86+
partitions:
87+
unittest:
88+
schema: public
89+
table: ${TABLE}
90+
interval: ${INTERVAL}
91+
partitionKey: id
92+
cleanupPolicy: drop
93+
retention: ${RETENTION}
94+
preProvisioned: ${PREPROVISIONED}
95+
EOF
96+
)
97+
local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
98+
99+
# When run on 2025-03-15 with a retention of 1 month, the partition for 2024-12
100+
# is old enough to be dropped. But since 2025-01 is missing, it is an error that
101+
# should prevent the drop.
102+
PPM_WORK_DATE="2025-03-15" run postgresql-partition-manager run cleanup -c ${CONFIGURATION_FILE}
103+
104+
assert_failure
105+
assert_output --partial 'level=ERROR msg="Partition Gap"'
106+
107+
# Check that all the partitions are still there
108+
local i=0
109+
local expected=""
110+
while (( i < ${#PARTS[*]} ))
111+
do
112+
# bats does not expect any trailing newline, so append it to each previous line
113+
# except the first
114+
if (( i > 0 )); then expected+=$'\n'; fi
115+
expected+="public|${PARTS[i]}|${PARTS[i+1]}|${PARTS[i+2]}"
116+
(( i+=3 ))
117+
done
118+
run list_existing_partitions "unittest" "public" "${TABLE}"
119+
120+
assert_output "$expected"
121+
122+
rm "$CONFIGURATION_FILE"
123+
}

‎scripts/bats/test/libs/partitions.bash

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,36 @@ generate_daily_partition_name() {
5454
generate_table_name() {
5555
cat /dev/urandom | head -n 1 | base64 | tr -dc '[:alnum:]' | tr '[:upper:]' '[:lower:]' | cut -c -13 | sed -e 's/^[0-9]/a/g'
5656
}
57+
58+
# Generic method to create child partitions with specified ranges
59+
# Arguments: parent table and an array of partition definitions
60+
# The array elements are:
61+
# [0] partition name
62+
# [1] lower bound
63+
# [2] upper bound
64+
# [3] name of next partition
65+
# [4] lower bound for next partition
66+
# ...and so on (3 elements per partition)
67+
create_partitions() {
68+
local tbl="$1"
69+
shift
70+
local parts=("$@")
71+
72+
local len=${#parts[*]}
73+
if ! (( len % 3 == 0 )); then
74+
echo >&2 "The list must have 3 elements per partition (found length=$len)"
75+
return 1
76+
fi
77+
i=0
78+
local sql_block="BEGIN;"
79+
while (( i < len ))
80+
do
81+
# Partition names and ranges must be SQL-quoted by the caller if needed
82+
sql_block="$sql_block
83+
CREATE TABLE ${parts[i]} PARTITION OF ${tbl} FOR VALUES FROM ('${parts[((i+1))]}') TO ('${parts[((i+2))]}') ;"
84+
(( i+=3 ))
85+
done
86+
sql_block="$sql_block
87+
COMMIT;";
88+
execute_sql_commands "$sql_block"
89+
}

‎scripts/bats/test/libs/seeds.bash

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,19 @@ EOQ
2626
execute_sql "${QUERY}"
2727
}
2828

29+
create_table_uuid_range() {
30+
local TABLE="$1"
31+
32+
read -r -d '' QUERY <<EOQ ||
33+
CREATE TABLE ${TABLE} (
34+
id uuid,
35+
value INT,
36+
created_at DATE NOT NULL
37+
) PARTITION BY RANGE (id);
38+
EOQ
39+
execute_sql "${QUERY}"
40+
}
41+
2942
generate_configuration_file() {
3043
local PARTITION_CONFIGURATION=$1
3144
local CONFIGURATION_TEMPLATE_FILE=configuration/template.yaml

‎scripts/bats/test/libs/sql.bash

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,11 @@ execute_sql_file() {
2222

2323
# Execute multiple semicolon-separated commands
2424
execute_sql_commands() {
25-
local SQL_COMMANDS=$1
26-
local DATABASE=$2
25+
# $1: sql commands
26+
# $2: dbname (if unset, psql defaults to $PGDATABASE and then $USER)
2727

28-
PGDATABASE="$DATABASE" psql --tuples-only --no-align --quiet <<EOSQL
29-
$SQL_COMMANDS
30-
;
28+
psql --tuples-only --no-align --quiet "$2" <<EOSQL
29+
$1
3130
EOSQL
3231
}
3332

0 commit comments

Comments
 (0)