Improve data loading performance (#134)

Author: taylorchu

dbr.go

@@ -116,7 +116,7 @@ func exec(ctx context.Context, runner runner, log EventReceiver, builder Builder
 		Dialect:      d,
 		IgnoreBinary: true,
 	}
-	err := i.interpolate(placeholder, []interface{}{builder}, true)
+	err := i.encodePlaceholder(builder, true)
 	query, value := i.String(), i.Value()
 	if err != nil {
 		return nil, log.EventErrKv("dbr.exec.interpolate", err, kvs{
@@ -154,7 +154,7 @@ func query(ctx context.Context, runner runner, log EventReceiver, builder Builde
 		Dialect:      d,
 		IgnoreBinary: true,
 	}
-	err := i.interpolate(placeholder, []interface{}{builder}, true)
+	err := i.encodePlaceholder(builder, true)
 	query, value := i.String(), i.Value()
 	if err != nil {
 		return 0, log.EventErrKv("dbr.select.interpolate", err, kvs{

insert.go

@@ -150,20 +150,23 @@ func (b *InsertStmt) Record(structValue interface{}) *InsertStmt {
 	v := reflect.Indirect(reflect.ValueOf(structValue))
 
 	if v.Kind() == reflect.Struct {
-		m := structMap(v)
-		if v.CanSet() {
-			// ID is recommended by golint here
-			if field, ok := m["id"]; ok && field.Kind() == reflect.Int64 {
-				b.RecordID = field.Addr().Interface().(*int64)
+		found := make([]interface{}, len(b.Column)+1)
+		// ID is recommended by golint here
+		findValueByName(v, append(b.Column, "id"), found, false)
+
+		value := found[:len(found)-1]
+		for i, v := range value {
+			if v != nil {
+				value[i] = v.(reflect.Value).Interface()
 			}
 		}
 
-		var value []interface{}
-		for _, key := range b.Column {
-			if val, ok := m[key]; ok {
-				value = append(value, val.Interface())
-			} else {
-				value = append(value, nil)
+		if v.CanSet() {
+			switch idField := found[len(found)-1].(type) {
+			case reflect.Value:
+				if idField.Kind() == reflect.Int64 {
+					b.RecordID = idField.Addr().Interface().(*int64)
+				}
 			}
 		}
 		b.Values(value...)

interpolate.go

@@ -62,7 +62,7 @@ func (i *interpolator) interpolate(query string, value []interface{}, topLevel b
 		}
 
 		if valueIndex >= len(value) {
-			break
+			return ErrPlaceholderCount
 		}
 
 		i.WriteString(query[:index])
@@ -103,10 +103,8 @@ func (i *interpolator) encodePlaceholder(value interface{}, topLevel bool) error
 		}
 		paren := false
 		switch value.(type) {
-		case *SelectStmt:
+		case *SelectStmt, *union:
 			paren = !topLevel
-		case *union:
-			paren = true
 		}
 		if paren {
 			i.WriteString("(")

load.go

@@ -46,9 +46,9 @@ func Load(rows *sql.Rows, value interface{}) (int, error) {
 		var err error
 
 		if isMapOfSlices {
-			elem = reflect.New(v.Type().Elem().Elem()).Elem()
+			elem = reflectAlloc(v.Type().Elem().Elem())
 		} else if isSlice || isMap {
-			elem = reflect.New(v.Type().Elem()).Elem()
+			elem = reflectAlloc(v.Type().Elem())
 		} else {
 			elem = v
 		}
@@ -58,7 +58,7 @@ func Load(rows *sql.Rows, value interface{}) (int, error) {
 			if err != nil {
 				return 0, err
 			}
-			keyElem = reflect.New(v.Type().Key()).Elem()
+			keyElem = reflectAlloc(v.Type().Key())
 			keyPtr, err := findPtr(column[0:1], keyElem)
 			if err != nil {
 				return 0, err
@@ -95,6 +95,13 @@ func Load(rows *sql.Rows, value interface{}) (int, error) {
 	return count, nil
 }
 
+func reflectAlloc(typ reflect.Type) reflect.Value {
+	if typ.Kind() == reflect.Ptr {
+		return reflect.New(typ.Elem())
+	}
+	return reflect.New(typ).Elem()
+}
+
 type dummyScanner struct{}
 
 func (dummyScanner) Scan(interface{}) error {
@@ -107,18 +114,16 @@ var (
 )
 
 func findPtr(column []string, value reflect.Value) ([]interface{}, error) {
-	if value.Addr().Type().Implements(typeScanner) {
+	if value.CanAddr() && value.Addr().Type().Implements(typeScanner) {
 		return []interface{}{value.Addr().Interface()}, nil
 	}
 	switch value.Kind() {
 	case reflect.Struct:
-		var ptr []interface{}
-		m := structMap(value)
-		for _, key := range column {
-			if val, ok := m[key]; ok {
-				ptr = append(ptr, val.Addr().Interface())
-			} else {
-				ptr = append(ptr, dummyDest)
+		ptr := make([]interface{}, len(column))
+		findValueByName(value, column, ptr, true)
+		for i := range ptr {
+			if ptr[i] == nil {
+				ptr[i] = dummyDest
 			}
 		}
 		return ptr, nil

load_benchmark_test.go

@@ -0,0 +1,75 @@
+package dbr
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkLoadValues(b *testing.B) {
+	sess := mysqlSession
+	for _, v := range []string{
+		`DROP TABLE IF EXISTS suggestions`,
+		`CREATE TABLE suggestions (
+			id serial PRIMARY KEY,
+			title varchar(255),
+			body text
+		)`,
+	} {
+		_, err := sess.Exec(v)
+		require.NoError(b, err)
+	}
+	tx, err := sess.Begin()
+	require.NoError(b, err)
+
+	const maxRows = 100000
+
+	for i := 0; i < maxRows; i++ {
+		_, err := tx.InsertInto("suggestions").
+			Columns("title", "body").
+			Values("title", "body").
+			Exec()
+		require.NoError(b, err)
+	}
+	err = tx.Commit()
+	require.NoError(b, err)
+
+	type Suggestion struct {
+		Title *string
+		Body  *string
+	}
+	for n := 10; n <= maxRows; n *= 10 {
+		query := fmt.Sprintf("SELECT * FROM suggestions ORDER BY id ASC LIMIT %d", n)
+
+		b.Run(fmt.Sprintf("sqlx_%d", n), func(b *testing.B) {
+			b.StopTimer()
+			db, err := sqlx.Connect("mysql", mysqlDSN)
+			require.NoError(b, err)
+			db = db.Unsafe()
+			defer db.Close()
+
+			for i := 0; i < b.N; i++ {
+				var suggs []*Suggestion
+				b.StartTimer()
+				err := db.Select(&suggs, query)
+				b.StopTimer()
+				require.NoError(b, err)
+				require.Len(b, suggs, n)
+			}
+		})
+		b.Run(fmt.Sprintf("dbr_%d", n), func(b *testing.B) {
+			b.StopTimer()
+
+			for i := 0; i < b.N; i++ {
+				var suggs []*Suggestion
+				b.StartTimer()
+				_, err := sess.SelectBySql(query).Load(&suggs)
+				b.StopTimer()
+				require.NoError(b, err)
+				require.Len(b, suggs, n)
+			}
+		})
+	}
+}

util.go

@@ -43,17 +43,11 @@ func camelCaseToSnakeCase(name string) string {
 	return buf.String()
 }
 
-func structMap(value reflect.Value) map[string]reflect.Value {
-	m := make(map[string]reflect.Value)
-	structValue(m, value)
-	return m
-}
-
 var (
 	typeValuer = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
 )
 
-func structValue(m map[string]reflect.Value, value reflect.Value) {
+func findValueByName(value reflect.Value, name []string, found []interface{}, retPtr bool) {
 	if value.Type().Implements(typeValuer) {
 		return
 	}
@@ -62,7 +56,7 @@ func structValue(m map[string]reflect.Value, value reflect.Value) {
 		if value.IsNil() {
 			return
 		}
-		structValue(m, value.Elem())
+		findValueByName(value.Elem(), name, found, retPtr)
 	case reflect.Struct:
 		t := value.Type()
 		for i := 0; i < t.NumField(); i++ {
@@ -81,10 +75,19 @@ func structValue(m map[string]reflect.Value, value reflect.Value) {
 				tag = NameMapping(field.Name)
 			}
 			fieldValue := value.Field(i)
-			if _, ok := m[tag]; !ok {
-				m[tag] = fieldValue
+			for i, want := range name {
+				if want != tag {
+					continue
+				}
+				if found[i] == nil {
+					if retPtr {
+						found[i] = fieldValue.Addr().Interface()
+					} else {
+						found[i] = fieldValue
+					}
+				}
 			}
-			structValue(m, fieldValue)
+			findValueByName(fieldValue, name, found, retPtr)
 		}
 	}
 }

util_test.go

@@ -56,54 +56,58 @@ func BenchmarkCamelCaseToSnakeCase(b *testing.B) {
 	}
 }
 
-func TestStructMap(t *testing.T) {
+func TestFindValueByName(t *testing.T) {
 	for _, test := range []struct {
-		in  interface{}
-		ok  []string
-		bad []string
+		in   interface{}
+		name []string
+		want []string
 	}{
 		{
 			in: struct {
 				CreatedAt time.Time
 			}{},
-			ok: []string{"created_at"},
+			name: []string{"created_at"},
+			want: []string{"created_at"},
 		},
 		{
 			in: struct {
 				intVal int
 			}{},
-			bad: []string{"int_val"},
+			name: []string{"int_val"},
 		},
 		{
 			in: struct {
 				IntVal int `db:"test"`
 			}{},
-			ok:  []string{"test"},
-			bad: []string{"int_val"},
+			name: []string{"test"},
+			want: []string{"test"},
 		},
 		{
 			in: struct {
 				IntVal int `db:"-"`
 			}{},
-			bad: []string{"int_val"},
+			name: []string{"int_val"},
 		},
 		{
 			in: struct {
 				Test1 struct {
 					Test2 int
 				}
 			}{},
-			ok: []string{"test2"},
+			name: []string{"test2"},
+			want: []string{"test2"},
 		},
 	} {
-		m := structMap(reflect.ValueOf(test.in))
-		for _, c := range test.ok {
-			_, ok := m[c]
-			require.True(t, ok)
-		}
-		for _, c := range test.bad {
-			_, ok := m[c]
-			require.False(t, ok)
+		found := make([]interface{}, len(test.name))
+		findValueByName(reflect.ValueOf(test.in), test.name, found, false)
+
+		var got []string
+		for i, v := range found {
+			if v != nil {
+				got = append(got, test.name[i])
+			}
 		}
+
+		require.Equal(t, test.want, got)
 	}
 }