Skip to content

:copyfrom for MySQL LOAD DATA INFILE #2220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

:copyfrom for MySQL LOAD DATA INFILE #2220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions internal/codegen/golang/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,14 @@ func generate(req *plugin.CodeGenRequest, enums []Enum, structs []Struct, querie
SqlcVersion: req.SqlcVersion,
}

if tctx.UsesCopyFrom && !tctx.SQLDriver.IsPGX() {
return nil, errors.New(":copyfrom is only supported by pgx")
if tctx.UsesCopyFrom && !tctx.SQLDriver.IsPGX() && golang.SqlDriver != SQLDriverGoSQLDriverMySQL {
return nil, errors.New(":copyfrom is only supported by pgx and github.com/go-sql-driver/mysql")
}

if tctx.UsesCopyFrom && golang.SqlDriver == SQLDriverGoSQLDriverMySQL {
if err := checkNoTimesForMySQLCopyFrom(queries); err != nil {
return nil, err
}
}

if tctx.UsesBatch && !tctx.SQLDriver.IsPGX() {
Expand Down Expand Up @@ -288,3 +294,14 @@ func usesBatch(queries []Query) bool {
}
return false
}

func checkNoTimesForMySQLCopyFrom(queries []Query) error {
for _, q := range queries {
for _, f := range q.Arg.Fields() {
if f.Type == "time.Time" {
return fmt.Errorf("values with a timezone are not yet supported")
}
}
}
return nil
}
7 changes: 7 additions & 0 deletions internal/codegen/golang/imports.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,13 @@ func (i *importer) copyfromImports() fileImports {
})

std["context"] = struct{}{}
if i.Settings.Go.SqlDriver == SQLDriverGoSQLDriverMySQL {
std["io"] = struct{}{}
std["fmt"] = struct{}{}
std["sync/atomic"] = struct{}{}
pkg[ImportSpec{Path: "github.com/go-sql-driver/mysql"}] = struct{}{}
pkg[ImportSpec{Path: "github.com/hexon/mysqltsv"}] = struct{}{}
}

return sortedImports(std, pkg)
}
Expand Down
38 changes: 36 additions & 2 deletions internal/codegen/golang/query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,18 @@ func (v QueryValue) Params() string {
return "\n" + strings.Join(out, ",\n")
}

func (v QueryValue) ColumnNames() string {
func (v QueryValue) ColumnNames() []string {
if v.Struct == nil {
return []string{v.DBName}
}
names := make([]string, len(v.Struct.Fields))
for i, f := range v.Struct.Fields {
names[i] = f.DBName
}
return names
}

func (v QueryValue) ColumnNamesAsGoSlice() string {
if v.Struct == nil {
return fmt.Sprintf("[]string{%q}", v.DBName)
}
Expand Down Expand Up @@ -189,6 +200,19 @@ func (v QueryValue) Scan() string {
return "\n" + strings.Join(out, ",\n")
}

func (v QueryValue) Fields() []Field {
if v.Struct != nil {
return v.Struct.Fields
}
return []Field{
{
Name: v.Name,
DBName: v.DBName,
Type: v.Typ,
},
}
}

// A struct used to generate methods and fields on the Queries struct
type Query struct {
Cmd string
Expand All @@ -210,7 +234,7 @@ func (q Query) hasRetType() bool {
return scanned && !q.Ret.isEmpty()
}

func (q Query) TableIdentifier() string {
func (q Query) TableIdentifierAsGoSlice() string {
escapedNames := make([]string, 0, 3)
for _, p := range []string{q.Table.Catalog, q.Table.Schema, q.Table.Name} {
if p != "" {
Expand All @@ -219,3 +243,13 @@ func (q Query) TableIdentifier() string {
}
return "[]string{" + strings.Join(escapedNames, ", ") + "}"
}

func (q Query) TableIdentifierForMySQL() string {
escapedNames := make([]string, 0, 3)
for _, p := range []string{q.Table.Catalog, q.Table.Schema, q.Table.Name} {
if p != "" {
escapedNames = append(escapedNames, fmt.Sprintf("`%s`", p))
}
}
return strings.Join(escapedNames, ".")
}
45 changes: 45 additions & 0 deletions internal/codegen/golang/templates/go-sql-driver-mysql/copyfromCopy.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{{define "copyfromCodeGoSqlDriver"}}
{{range .GoQueries}}
{{if eq .Cmd ":copyfrom" }}
var readerHandlerSequenceFor{{.MethodName}} uint32 = 1

func convertRowsFor{{.MethodName}}(w *io.PipeWriter, {{.Arg.SlicePair}}) {
e := mysqltsv.NewEncoder(w, {{ len .Arg.Fields }}, nil)
for _, row := range {{.Arg.Name}} {
{{- with $arg := .Arg }}
{{- range $arg.Fields}}
{{- if eq .Type "string"}}
e.AppendString({{if eq (len $arg.Fields) 1}}row{{else}}row.{{.Name}}{{end}})
{{- else if eq .Type "[]byte"}}
e.AppendBytes({{if eq (len $arg.Fields) 1}}row{{else}}row.{{.Name}}{{end}})
{{- else}}
e.AppendValue({{if eq (len $arg.Fields) 1}}row{{else}}row.{{.Name}}{{end}})
{{- end}}
{{- end}}
{{- end}}
}
w.CloseWithError(e.Close())
}

{{range .Comments}}//{{.}}
{{end -}}
// {{.MethodName}} uses MySQL's LOAD DATA LOCAL INFILE and is not atomic. Errors and duplicate keys are treated as warnings and insertion will continue, even without an error for some cases.
// Use this in a transaction and use SHOW WARNINGS to check for any problems and roll back if you want to.
// This is a MySQL limitation, not sqlc. Check the documentation for more information: https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-error-handling
Comment on lines +26 to +28
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment is too wide. Can we edit it to be this instead?

// {{.MethodName}} uses MySQL's LOAD DATA LOCAL INFILE and is not atomic.
//
// Errors and duplicate keys are treated as warnings and insertion will
// continue, even without an error for some cases.  Use this in a transaction
// and use SHOW WARNINGS to check for any problems and roll back if you want to.
//
// Check the documentation for more information:
// https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-error-handling

func (q *Queries) {{.MethodName}}(ctx context.Context{{if $.EmitMethodsWithDBArgument}}, db DBTX{{end}}, {{.Arg.SlicePair}}) (int64, error) {
pr, pw := io.Pipe()
defer pr.Close()
rh := fmt.Sprintf("{{.MethodName}}_%d", atomic.AddUint32(&readerHandlerSequenceFor{{.MethodName}}, 1))
mysql.RegisterReaderHandler(rh, func() io.Reader { return pr })
defer mysql.DeregisterReaderHandler(rh)
go convertRowsFor{{.MethodName}}(pw, {{.Arg.Name}})
result, err := {{if (not $.EmitMethodsWithDBArgument)}}q.{{end}}db.ExecContext(ctx, fmt.Sprintf("LOAD DATA LOCAL INFILE '%s' INTO TABLE {{.TableIdentifierForMySQL}} %s ({{range $index, $name := .Arg.ColumnNames}}{{if gt $index 0}}, {{end}}{{$name}}{{end}})", "Reader::" + rh, mysqltsv.Escaping))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add a comment here explaining that LOAD DATA can't be used with parameters and link to this part of the docs

The LOAD DATA INFILE statement reads rows from a text file into a table at a very high speed. The file name must be given as a literal string.

if err != nil {
return 0, err
}
return result.RowsAffected()
}

{{end}}
{{end}}
{{end}}
4 changes: 2 additions & 2 deletions internal/codegen/golang/templates/pgx/copyfromCopy.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ func (r iteratorFor{{.MethodName}}) Err() error {
{{end -}}
{{- if $.EmitMethodsWithDBArgument -}}
func (q *Queries) {{.MethodName}}(ctx context.Context, db DBTX, {{.Arg.SlicePair}}) (int64, error) {
return db.CopyFrom(ctx, {{.TableIdentifier}}, {{.Arg.ColumnNames}}, &iteratorFor{{.MethodName}}{rows: {{.Arg.Name}}})
return db.CopyFrom(ctx, {{.TableIdentifierAsGoSlice}}, {{.Arg.ColumnNamesAsGoSlice}}, &iteratorFor{{.MethodName}}{rows: {{.Arg.Name}}})
{{- else -}}
func (q *Queries) {{.MethodName}}(ctx context.Context, {{.Arg.SlicePair}}) (int64, error) {
return q.db.CopyFrom(ctx, {{.TableIdentifier}}, {{.Arg.ColumnNames}}, &iteratorFor{{.MethodName}}{rows: {{.Arg.Name}}})
return q.db.CopyFrom(ctx, {{.TableIdentifierAsGoSlice}}, {{.Arg.ColumnNamesAsGoSlice}}, &iteratorFor{{.MethodName}}{rows: {{.Arg.Name}}})
{{- end}}
}

Expand Down
2 changes: 2 additions & 0 deletions internal/codegen/golang/templates/template.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,8 @@ import (
{{define "copyfromCode"}}
{{if .SQLDriver.IsPGX }}
{{- template "copyfromCodePgx" .}}
{{else}}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of {{else}}, let's check for the MySQL driver explicitly.

{{- template "copyfromCodeGoSqlDriver" .}}
{{end}}
{{end}}

Expand Down
74 changes: 74 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/go/copyfrom.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 31 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/go/db.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/go/models.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 25 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/go/query.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE TABLE foo (a text, b integer, c DATETIME, d DATE);

-- name: InsertValues :copyfrom
INSERT INTO foo (a, b, c, d) VALUES (?, ?, ?, ?);

-- name: InsertSingleValue :copyfrom
INSERT INTO foo (a) VALUES (?);
14 changes: 14 additions & 0 deletions internal/endtoend/testdata/copyfrom/mysql/sqlc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"version": "1",
"packages": [
{
"path": "go",
"sql_package": "database/sql",
"sql_driver": "github.com/go-sql-driver/mysql",
"engine": "mysql",
"name": "querytest",
"schema": "query.sql",
"queries": "query.sql"
}
]
}
2 changes: 2 additions & 0 deletions internal/endtoend/testdata/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ module github.com/kyleconroy/sqlc/endtoend
go 1.18

require (
github.com/go-sql-driver/mysql v1.7.0
github.com/gofrs/uuid v4.0.0+incompatible
github.com/google/uuid v1.3.0
github.com/hexon/mysqltsv v0.1.0
github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853
github.com/jackc/pgtype v1.6.2
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904
Expand Down
4 changes: 4 additions & 0 deletions internal/endtoend/testdata/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,17 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk=
github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI=
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hexon/mysqltsv v0.1.0 h1:48wYQlsPH8ZEkKAVCdsOYzMYAlEoevw8ZWD8rqYPdlg=
github.com/hexon/mysqltsv v0.1.0/go.mod h1:p3vPBkpxebjHWF1bWKYNcXx5pFu+yAG89QZQEKSvVrY=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
Expand Down