Handle nullable columns

destination.go

@@ -39,7 +39,7 @@ type Destination struct {
 	getTableName destination.TableFn
 
 	conn        *pgx.Conn
-	dbInfo      *internal.DbInfo
+	dbInfo      *internal.NumericScaleInfo
 	stmtBuilder sq.StatementBuilderType
 }
 
@@ -347,7 +347,7 @@ func (d *Destination) formatBigRat(ctx context.Context, table string, column str
 
 	// we need to get the scale of the column so we that we can properly
 	// round the result of dividing the input big.Rat's numerator and denominator.
-	scale, err := d.dbInfo.GetNumericColumnScale(ctx, table, column)
+	scale, err := d.dbInfo.Get(ctx, table, column)
 	if err != nil {
 		return "", fmt.Errorf("failed getting scale of numeric column: %w", err)
 	}

destination_integration_test.go

@@ -36,7 +36,7 @@ func TestDestination_Write(t *testing.T) {
 
 	// tables with capital letters should be quoted
 	tableName := strings.ToUpper(test.RandomIdentifier(t))
-	test.SetupTestTableWithName(ctx, t, conn, tableName)
+	test.SetupTableWithName(ctx, t, conn, tableName)
 
 	d := NewDestination()
 	err := sdk.Util.ParseConfig(
@@ -70,10 +70,11 @@ func TestDestination_Write(t *testing.T) {
 				Key:       opencdc.StructuredData{"id": 5000},
 				Payload: opencdc.Change{
 					After: opencdc.StructuredData{
+						"key":              []uint8("123"),
 						"column1":          "foo",
 						"column2":          123,
 						"column3":          true,
-						"column4":          nil,
+						"column4":          big.NewRat(123, 10),
 						"UppercaseColumn1": 222,
 					},
 				},
@@ -88,10 +89,11 @@ func TestDestination_Write(t *testing.T) {
 				Key:       opencdc.StructuredData{"id": 5},
 				Payload: opencdc.Change{
 					After: opencdc.StructuredData{
+						"key":              []uint8("234"),
 						"column1":          "foo",
 						"column2":          456,
 						"column3":          false,
-						"column4":          nil,
+						"column4":          big.NewRat(123, 10),
 						"UppercaseColumn1": 333,
 					},
 				},
@@ -106,10 +108,11 @@ func TestDestination_Write(t *testing.T) {
 				Key:       opencdc.StructuredData{"id": 6},
 				Payload: opencdc.Change{
 					After: opencdc.StructuredData{
+						"key":              []uint8("345"),
 						"column1":          "bar",
 						"column2":          567,
 						"column3":          true,
-						"column4":          nil,
+						"column4":          big.NewRat(123, 10),
 						"UppercaseColumn1": 444,
 					},
 				},
@@ -124,10 +127,11 @@ func TestDestination_Write(t *testing.T) {
 				Key:       opencdc.StructuredData{"id": 1},
 				Payload: opencdc.Change{
 					After: opencdc.StructuredData{
+						"key":              []uint8("456"),
 						"column1":          "foobar",
 						"column2":          567,
 						"column3":          true,
-						"column4":          nil,
+						"column4":          big.NewRat(123, 10),
 						"UppercaseColumn1": 555,
 					},
 				},
@@ -151,6 +155,7 @@ func TestDestination_Write(t *testing.T) {
 				Key:       opencdc.StructuredData{"id": 123},
 				Payload: opencdc.Change{
 					After: opencdc.StructuredData{
+						"key":              []uint8("567"),
 						"column1":          "abcdef",
 						"column2":          567,
 						"column3":          true,
@@ -179,9 +184,7 @@ func TestDestination_Write(t *testing.T) {
 					cmp.Diff(
 						tt.record.Payload.After,
 						got,
-						cmp.Comparer(func(x, y *big.Rat) bool {
-							return x.Cmp(y) == 0
-						}),
+						test.BigRatComparer,
 					),
 				) // -want, +got
 			case opencdc.OperationDelete:
@@ -197,7 +200,7 @@ func TestDestination_Batch(t *testing.T) {
 	conn := test.ConnectSimple(ctx, t, test.RegularConnString)
 
 	tableName := strings.ToUpper(test.RandomIdentifier(t))
-	test.SetupTestTableWithName(ctx, t, conn, tableName)
+	test.SetupTableWithName(ctx, t, conn, tableName)
 
 	d := NewDestination()
 
@@ -223,10 +226,11 @@ func TestDestination_Batch(t *testing.T) {
 			Key:       opencdc.StructuredData{"id": 5},
 			Payload: opencdc.Change{
 				After: opencdc.StructuredData{
+					"key":              []uint8("123"),
 					"column1":          "foo1",
 					"column2":          1,
 					"column3":          false,
-					"column4":          nil,
+					"column4":          big.NewRat(123, 10),
 					"UppercaseColumn1": 111,
 				},
 			},
@@ -237,10 +241,11 @@ func TestDestination_Batch(t *testing.T) {
 			Key:       opencdc.StructuredData{"id": 6},
 			Payload: opencdc.Change{
 				After: opencdc.StructuredData{
+					"key":              []uint8("234"),
 					"column1":          "foo2",
 					"column2":          2,
 					"column3":          true,
-					"column4":          nil,
+					"column4":          big.NewRat(123, 10),
 					"UppercaseColumn1": 222,
 				},
 			},
@@ -251,10 +256,11 @@ func TestDestination_Batch(t *testing.T) {
 			Key:       opencdc.StructuredData{"id": 7},
 			Payload: opencdc.Change{
 				After: opencdc.StructuredData{
+					"key":              []uint8("345"),
 					"column1":          "foo3",
 					"column2":          3,
 					"column3":          false,
-					"column4":          nil,
+					"column4":          big.NewRat(123, 10),
 					"UppercaseColumn1": 333,
 				},
 			},
@@ -275,19 +281,20 @@ func TestDestination_Batch(t *testing.T) {
 func queryTestTable(ctx context.Context, conn test.Querier, tableName string, id any) (opencdc.StructuredData, error) {
 	row := conn.QueryRow(
 		ctx,
-		fmt.Sprintf(`SELECT column1, column2, column3, column4, "UppercaseColumn1" FROM %q WHERE id = $1`, tableName),
+		fmt.Sprintf(`SELECT key, column1, column2, column3, column4, "UppercaseColumn1" FROM %q WHERE id = $1`, tableName),
 		id,
 	)
 
 	var (
+		key           []uint8
 		col1          string
 		col2          int
 		col3          bool
 		col4Str       *string
 		uppercaseCol1 int
 	)
 
-	err := row.Scan(&col1, &col2, &col3, &col4Str, &uppercaseCol1)
+	err := row.Scan(&key, &col1, &col2, &col3, &col4Str, &uppercaseCol1)
 	if err != nil {
 		return nil, err
 	}
@@ -301,6 +308,7 @@ func queryTestTable(ctx context.Context, conn test.Querier, tableName string, id
 	}
 
 	return opencdc.StructuredData{
+		"key":              key,
 		"column1":          col1,
 		"column2":          col2,
 		"column3":          col3,

go.mod

@@ -61,7 +61,7 @@ require (
 	github.com/conduitio/evolviconf/evolviyaml v0.1.0 // indirect
 	github.com/conduitio/yaml/v3 v3.3.0 // indirect
 	github.com/curioswitch/go-reassign v0.3.0 // indirect
-	github.com/daixiang0/gci v0.13.6 // indirect
+	github.com/daixiang0/gci v0.13.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/denis-tingaikin/go-header v0.5.0 // indirect
 	github.com/ettle/strcase v0.2.0 // indirect

go.sum

@@ -96,8 +96,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs=
 github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88=
-github.com/daixiang0/gci v0.13.6 h1:RKuEOSkGpSadkGbvZ6hJ4ddItT3cVZ9Vn9Rybk6xjl8=
-github.com/daixiang0/gci v0.13.6/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
+github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c=
+github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

internal/db_info.go → internal/numeric_scale_info.go

@@ -22,52 +22,52 @@ import (
 	"github.com/jackc/pgx/v5"
 )
 
-// DbInfo provides information about tables in a database.
-type DbInfo struct {
+// NumericScaleInfo provides information about the scale of numeric columns
+// in a database.
+type NumericScaleInfo struct {
 	conn  *pgx.Conn
 	cache map[string]*tableCache
 }
 
 // tableCache stores information about a table.
-// The information is cached and refreshed every 'cacheExpiration'.
 type tableCache struct {
 	columns map[string]int
 }
 
-func NewDbInfo(conn *pgx.Conn) *DbInfo {
-	return &DbInfo{
+func NewDbInfo(conn *pgx.Conn) *NumericScaleInfo {
+	return &NumericScaleInfo{
 		conn:  conn,
 		cache: map[string]*tableCache{},
 	}
 }
 
-func (d *DbInfo) GetNumericColumnScale(ctx context.Context, table string, column string) (int, error) {
-	// Check if table exists in cache and is not expired
-	tableInfo, ok := d.cache[table]
+func (i *NumericScaleInfo) Get(ctx context.Context, table string, column string) (int, error) {
+	// Check if table exists in cache
+	tableInfo, ok := i.cache[table]
 	if ok {
 		scale, ok := tableInfo.columns[column]
 		if ok {
 			return scale, nil
 		}
 	} else {
 		// Table info has expired, refresh the cache
-		d.cache[table] = &tableCache{
+		i.cache[table] = &tableCache{
 			columns: map[string]int{},
 		}
 	}
 
 	// Fetch scale from database
-	scale, err := d.numericScaleFromDb(ctx, table, column)
+	scale, err := i.fetchFromDB(ctx, table, column)
 	if err != nil {
 		return 0, err
 	}
 
-	d.cache[table].columns[column] = scale
+	i.cache[table].columns[column] = scale
 
 	return scale, nil
 }
 
-func (d *DbInfo) numericScaleFromDb(ctx context.Context, table string, column string) (int, error) {
+func (i *NumericScaleInfo) fetchFromDB(ctx context.Context, table string, column string) (int, error) {
 	// Query to get the column type and numeric scale
 	query := `
 		SELECT 
@@ -83,7 +83,7 @@ func (d *DbInfo) numericScaleFromDb(ctx context.Context, table string, column st
 	var dataType string
 	var numericScale *int
 
-	err := d.conn.QueryRow(ctx, query, table, column).Scan(&dataType, &numericScale)
+	err := i.conn.QueryRow(ctx, query, table, column).Scan(&dataType, &numericScale)
 	if err != nil {
 		if errors.Is(err, pgx.ErrNoRows) {
 			return 0, fmt.Errorf("column %s not found in table %s", column, table)

internal/table_info.go

@@ -0,0 +1,114 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	sdk "github.com/conduitio/conduit-connector-sdk"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+type TableInfo struct {
+	Name    string
+	Columns map[string]*ColumnInfo
+}
+
+func NewTableInfo(tableName string) *TableInfo {
+	return &TableInfo{
+		Name:    tableName,
+		Columns: make(map[string]*ColumnInfo),
+	}
+}
+
+type ColumnInfo struct {
+	IsNotNull bool
+}
+
+type TableInfoFetcher struct {
+	connPool  *pgxpool.Pool
+	tableInfo map[string]*TableInfo
+}
+
+func NewTableInfoFetcher(connPool *pgxpool.Pool) *TableInfoFetcher {
+	return &TableInfoFetcher{
+		connPool:  connPool,
+		tableInfo: make(map[string]*TableInfo),
+	}
+}
+
+func (i TableInfoFetcher) Refresh(ctx context.Context, tableName string) error {
+	tx, err := i.connPool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to start tx for getting table info: %w", err)
+	}
+	defer func() {
+		if err := tx.Rollback(ctx); err != nil {
+			sdk.Logger(ctx).Warn().
+				Err(err).
+				Msgf("error on tx rollback for getting table info")
+		}
+	}()
+
+	query := `
+		SELECT a.attname as column_name, a.attnotnull as is_not_null
+		FROM pg_catalog.pg_attribute a
+		WHERE a.attrelid = $1::regclass
+			AND a.attnum > 0 
+			AND NOT a.attisdropped
+		ORDER BY a.attnum;
+	`
+
+	rows, err := tx.Query(ctx, query, WrapSQLIdent(tableName))
+	if err != nil {
+		sdk.Logger(ctx).
+			Err(err).
+			Str("query", query).
+			Msgf("failed to execute table info query")
+
+		return fmt.Errorf("failed to get table info: %w", err)
+	}
+	defer rows.Close()
+
+	ti := NewTableInfo(tableName)
+	for rows.Next() {
+		var columnName string
+		var isNotNull bool
+
+		err := rows.Scan(&columnName, &isNotNull)
+		if err != nil {
+			return fmt.Errorf("failed to scan table info row: %w", err)
+		}
+
+		ci := ti.Columns[columnName]
+		if ci == nil {
+			ci = &ColumnInfo{}
+			ti.Columns[columnName] = ci
+		}
+		ci.IsNotNull = isNotNull
+	}
+
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("failed to get table info rows: %w", err)
+	}
+
+	i.tableInfo[tableName] = ti
+	return nil
+}
+
+func (i TableInfoFetcher) GetTable(name string) *TableInfo {
+	return i.tableInfo[name]
+}