From 1f5664a630d0b2573e42b02006a7d9025ea14c5a Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 6 Jan 2025 13:52:28 +0100 Subject: [PATCH 1/3] feat: make create search index idempotent, it shouldn't fail when it's already there. --- internal/etl/etl_test.go | 36 +++++++++++++++++++++++++++++------ internal/etl/load/postgres.go | 13 ++++++++++--- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/internal/etl/etl_test.go b/internal/etl/etl_test.go index 01d2b59..1f0f54f 100644 --- a/internal/etl/etl_test.go +++ b/internal/etl/etl_test.go @@ -37,8 +37,7 @@ func TestCreateSearchIndex(t *testing.T) { t.Error(err) } defer terminateContainer(ctx, t, postgisContainer) - - dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") + dbConn := makeDbConnection(dbPort) // when/then err = CreateSearchIndex(dbConn, "search_index") @@ -47,6 +46,31 @@ func TestCreateSearchIndex(t *testing.T) { assert.NoError(t, err) } +func TestCreateSearchIndexIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + ctx := context.Background() + + // given + dbPort, postgisContainer, err := setupPostgis(ctx, t) + if err != nil { + t.Error(err) + } + defer terminateContainer(ctx, t, postgisContainer) + dbConn := makeDbConnection(dbPort) + + // when/then + err = CreateSearchIndex(dbConn, "search_index") + assert.NoError(t, err) + err = CreateSearchIndex(dbConn, "search_index") // second time, should not fail + assert.NoError(t, err) +} + +func makeDbConnection(dbPort nat.Port) string { + return fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") +} + func TestImportGeoPackage(t *testing.T) { if testing.Short() { t.Skip("Skipping integration test in short mode") @@ -75,9 +99,7 @@ func TestImportGeoPackage(t *testing.T) { if err != nil { t.Error(err) } - defer terminateContainer(ctx, t, postgisContainer) - - dbConn := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", dbPort.Int(), "test_db") + dbConn := makeDbConnection(dbPort) cfg, err := config.NewConfig(pwd + "/testdata/config.yaml") if err != nil { @@ -105,9 +127,11 @@ func TestImportGeoPackage(t *testing.T) { assert.NoError(t, err) var count int err = db.QueryRow(ctx, "select count(*) from search_index").Scan(&count) - defer db.Close(ctx) + db.Close(ctx) assert.NoError(t, err) assert.Equal(t, tt.count, count) + + terminateContainer(ctx, t, postgisContainer) } } diff --git a/internal/etl/load/postgres.go b/internal/etl/load/postgres.go index ac29070..89e30d3 100644 --- a/internal/etl/load/postgres.go +++ b/internal/etl/load/postgres.go @@ -49,7 +49,14 @@ func (p *Postgres) Load(records []t.SearchIndexRecord, index string) (int64, err // Init initialize search index func (p *Postgres) Init(index string) error { - geometryType := `create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON');` + // since "create type if not exists" isn't supported by Postgres we use a bit + // of pl/pgsql to avoid to create the geometry_type in an idempotent way + geometryType := ` + do $$ begin + create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON'); + exception + when duplicate_object then null; + end $$;` _, err := p.db.Exec(p.ctx, geometryType) if err != nil { return fmt.Errorf("error creating geometry type: %w", err) @@ -73,14 +80,14 @@ func (p *Postgres) Init(index string) error { } fullTextSearchColumn := fmt.Sprintf(` - alter table %[1]s add column ts tsvector + alter table %[1]s add column if not exists ts tsvector generated always as (to_tsvector('dutch', suggest || display_name )) stored;`, index) _, err = p.db.Exec(p.ctx, fullTextSearchColumn) if err != nil { return fmt.Errorf("error creating full-text search column: %w", err) } - ginIndex := fmt.Sprintf(`create index ts_idx on %[1]s using gin(ts);`, index) + ginIndex := fmt.Sprintf(`create index if not exists ts_idx on %[1]s using gin(ts);`, index) _, err = p.db.Exec(p.ctx, ginIndex) if err != nil { return fmt.Errorf("error creating GIN index: %w", err) From 9a358b27759ae7a5b534cd0357db2f7ebad99a9c Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Mon, 6 Jan 2025 13:53:46 +0100 Subject: [PATCH 2/3] feat: make create search index idempotent, it shouldn't fail when it's already there. --- internal/etl/load/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/etl/load/postgres.go b/internal/etl/load/postgres.go index 89e30d3..3029a4c 100644 --- a/internal/etl/load/postgres.go +++ b/internal/etl/load/postgres.go @@ -50,7 +50,7 @@ func (p *Postgres) Load(records []t.SearchIndexRecord, index string) (int64, err // Init initialize search index func (p *Postgres) Init(index string) error { // since "create type if not exists" isn't supported by Postgres we use a bit - // of pl/pgsql to avoid to create the geometry_type in an idempotent way + // of pl/pgsql to avoid to creating the geometry_type when it already exists. geometryType := ` do $$ begin create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON'); From c769e5e8fafa4ce8e0a1e49d6821bb464df264ba Mon Sep 17 00:00:00 2001 From: Richard Kettelerij <291572+rkettelerij@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:19:53 +0100 Subject: [PATCH 3/3] fix typo --- internal/etl/load/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/etl/load/postgres.go b/internal/etl/load/postgres.go index 3029a4c..18c415b 100644 --- a/internal/etl/load/postgres.go +++ b/internal/etl/load/postgres.go @@ -50,7 +50,7 @@ func (p *Postgres) Load(records []t.SearchIndexRecord, index string) (int64, err // Init initialize search index func (p *Postgres) Init(index string) error { // since "create type if not exists" isn't supported by Postgres we use a bit - // of pl/pgsql to avoid to creating the geometry_type when it already exists. + // of pl/pgsql to avoid creating the geometry_type when it already exists. geometryType := ` do $$ begin create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON');