Skip to content

Commit 3872990

Browse files
andrewmbentonandrewmbenton
authored
feat: Add remote execution for codegen (sqlc-dev#2214)
1 parent d70dc32 commit 3872990

24 files changed

+782
-12
lines changed

‎Makefile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,9 @@ mysqlsh:
4646

4747
proto:
4848
buf generate
49+
50+
remote-proto:
51+
protoc \
52+
--go_out=. --go_opt="Minternal/remote/gen.proto=github.com/kyleconroy/sqlc/internal/remote" --go_opt=module=github.com/kyleconroy/sqlc \
53+
--go-grpc_out=. --go-grpc_opt="Minternal/remote/gen.proto=github.com/kyleconroy/sqlc/internal/remote" --go-grpc_opt=module=github.com/kyleconroy/sqlc \
54+
internal/remote/gen.proto

‎go.mod

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,17 @@ require (
1515
github.com/lib/pq v1.10.7
1616
github.com/mattn/go-sqlite3 v1.14.16
1717
github.com/pganalyze/pg_query_go/v4 v4.2.0
18+
github.com/riza-io/grpc-go v0.1.0
1819
github.com/spf13/cobra v1.7.0
1920
github.com/spf13/pflag v1.0.5
2021
golang.org/x/sync v0.1.0
22+
google.golang.org/grpc v1.54.0
2123
google.golang.org/protobuf v1.30.0
2224
gopkg.in/yaml.v3 v3.0.1
2325
)
2426

2527
require (
2628
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
27-
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
28-
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
29-
)
30-
31-
require (
3229
github.com/golang/protobuf v1.5.2 // indirect
3330
github.com/inconshreveable/mousetrap v1.1.0 // indirect
3431
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -41,10 +38,15 @@ require (
4138
github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 // indirect
4239
github.com/pingcap/tidb/parser v0.0.0-20220725134311-c80026e61f00
4340
github.com/pkg/errors v0.9.1 // indirect
41+
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
4442
go.uber.org/atomic v1.9.0 // indirect
4543
go.uber.org/multierr v1.7.0 // indirect
4644
go.uber.org/zap v1.19.1 // indirect
4745
golang.org/x/crypto v0.6.0 // indirect
48-
golang.org/x/text v0.7.0 // indirect
46+
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
47+
golang.org/x/net v0.8.0 // indirect
48+
golang.org/x/sys v0.6.0 // indirect
49+
golang.org/x/text v0.8.0 // indirect
50+
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
4951
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
5052
)

‎go.sum

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
134134
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
135135
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
136136
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
137+
github.com/riza-io/grpc-go v0.1.0 h1:qm0j1YT0mqvtaGQ4A+sFe5odQSEE2OGnXGjTJAzQdUM=
138+
github.com/riza-io/grpc-go v0.1.0/go.mod h1:2bDvR9KkKC3KhtlSHfR3dAXjUMT86kg4UfWFyVGWqi8=
137139
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
138140
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
139141
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
@@ -216,6 +218,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
216218
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
217219
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
218220
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
221+
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
222+
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
219223
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
220224
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
221225
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -238,6 +242,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
238242
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
239243
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
240244
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
245+
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
246+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
241247
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
242248
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
243249
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -248,8 +254,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
248254
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
249255
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
250256
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
251-
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
252257
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
258+
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
259+
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
253260
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
254261
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
255262
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -268,6 +275,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
268275
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
269276
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
270277
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
278+
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
279+
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
280+
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
281+
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
271282
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
272283
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
273284
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

‎internal/cmd/cmd.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
3333
rootCmd := &cobra.Command{Use: "sqlc", SilenceUsage: true}
3434
rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)")
3535
rootCmd.PersistentFlags().BoolP("experimental", "x", false, "DEPRECATED: enable experimental features (default: false)")
36+
rootCmd.PersistentFlags().BoolP("no-remote", "nr", false, "disable remote execution (default: false)")
3637

3738
rootCmd.AddCommand(checkCmd)
3839
rootCmd.AddCommand(diffCmd)
@@ -107,15 +108,18 @@ var initCmd = &cobra.Command{
107108
}
108109

109110
type Env struct {
110-
DryRun bool
111-
Debug opts.Debug
111+
DryRun bool
112+
Debug opts.Debug
113+
NoRemote bool
112114
}
113115

114116
func ParseEnv(c *cobra.Command) Env {
115117
dr := c.Flag("dry-run")
118+
nr := c.Flag("no-remote")
116119
return Env{
117-
DryRun: dr != nil && dr.Changed,
118-
Debug: opts.DebugFromEnv(),
120+
DryRun: dr != nil && dr.Changed,
121+
Debug: opts.DebugFromEnv(),
122+
NoRemote: nr != nil && nr.Value.String() == "true",
119123
}
120124
}
121125

‎internal/cmd/generate.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"sync"
1414

1515
"golang.org/x/sync/errgroup"
16+
"google.golang.org/grpc/status"
1617

1718
"github.com/kyleconroy/sqlc/internal/codegen/golang"
1819
"github.com/kyleconroy/sqlc/internal/codegen/json"
@@ -23,9 +24,12 @@ import (
2324
"github.com/kyleconroy/sqlc/internal/ext"
2425
"github.com/kyleconroy/sqlc/internal/ext/process"
2526
"github.com/kyleconroy/sqlc/internal/ext/wasm"
27+
"github.com/kyleconroy/sqlc/internal/info"
2628
"github.com/kyleconroy/sqlc/internal/multierr"
2729
"github.com/kyleconroy/sqlc/internal/opts"
2830
"github.com/kyleconroy/sqlc/internal/plugin"
31+
"github.com/kyleconroy/sqlc/internal/remote"
32+
"github.com/kyleconroy/sqlc/internal/sql/sqlpath"
2933
)
3034

3135
const errMessageNoVersion = `The configuration file must have a version number.
@@ -141,6 +145,10 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
141145
return nil, err
142146
}
143147

148+
if conf.Cloud.Hostname != "" && !e.NoRemote {
149+
return remoteGenerate(ctx, configPath, conf, dir, stderr)
150+
}
151+
144152
output := map[string]string{}
145153
errored := false
146154

@@ -259,6 +267,69 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
259267
return output, nil
260268
}
261269

270+
func remoteGenerate(ctx context.Context, configPath string, conf *config.Config, dir string, stderr io.Writer) (map[string]string, error) {
271+
rpcClient, err := remote.NewClient(conf.Cloud)
272+
if err != nil {
273+
fmt.Fprintf(stderr, "error creating rpc client: %s\n", err)
274+
return nil, err
275+
}
276+
277+
configBytes, err := os.ReadFile(configPath)
278+
if err != nil {
279+
fmt.Fprintf(stderr, "error reading config file %s: %s\n", configPath, err)
280+
return nil, err
281+
}
282+
283+
rpcReq := remote.GenerateRequest{
284+
Version: info.Version,
285+
Inputs: []*remote.File{{Path: filepath.Base(configPath), Bytes: configBytes}},
286+
}
287+
288+
for _, pkg := range conf.SQL {
289+
for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
290+
for i, relFilePath := range paths {
291+
paths[i] = filepath.Join(dir, relFilePath)
292+
}
293+
files, err := sqlpath.Glob(paths)
294+
if err != nil {
295+
fmt.Fprintf(stderr, "error globbing paths: %s\n", err)
296+
return nil, err
297+
}
298+
for _, filePath := range files {
299+
fileBytes, err := os.ReadFile(filePath)
300+
if err != nil {
301+
fmt.Fprintf(stderr, "error reading file %s: %s\n", filePath, err)
302+
return nil, err
303+
}
304+
fileRelPath, _ := filepath.Rel(dir, filePath)
305+
rpcReq.Inputs = append(rpcReq.Inputs, &remote.File{Path: fileRelPath, Bytes: fileBytes})
306+
}
307+
}
308+
}
309+
310+
rpcResp, err := rpcClient.Generate(ctx, &rpcReq)
311+
if err != nil {
312+
rpcStatus, ok := status.FromError(err)
313+
if !ok {
314+
return nil, err
315+
}
316+
fmt.Fprintf(stderr, "rpc error: %s", rpcStatus.Message())
317+
return nil, rpcStatus.Err()
318+
}
319+
320+
if rpcResp.ExitCode != 0 {
321+
fmt.Fprintf(stderr, "%s", rpcResp.Stderr)
322+
return nil, errors.New("remote execution returned with non-zero exit code")
323+
}
324+
325+
output := map[string]string{}
326+
for _, file := range rpcResp.Outputs {
327+
output[filepath.Join(dir, file.Path)] = string(file.Bytes)
328+
}
329+
330+
return output, nil
331+
}
332+
262333
func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
263334
defer trace.StartRegion(ctx, "parse").End()
264335
c := compiler.NewCompiler(sql, combo)

‎internal/endtoend/endtoend_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ func TestReplay(t *testing.T) {
112112
}
113113

114114
env := cmd.Env{
115-
Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]),
115+
Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]),
116+
NoRemote: true,
116117
}
117118
switch args.Command {
118119
case "diff":

‎internal/endtoend/testdata/remote_missing_semicolon/mysql/go/db.go

Lines changed: 31 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/endtoend/testdata/remote_missing_semicolon/mysql/go/models.go

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/endtoend/testdata/remote_missing_semicolon/mysql/go/query.sql.go

Lines changed: 26 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/endtoend/testdata/remote_missing_semicolon/mysql/query.sql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-- https://github.com/kyleconroy/sqlc/issues/1198
2+
CREATE TABLE authors (
3+
id INT PRIMARY KEY,
4+
name VARCHAR(255) NOT NULL,
5+
bio text
6+
);
7+
8+
-- name: SetAuthor :exec
9+
UPDATE authors
10+
SET name = ?
11+
WHERE id = ?

‎internal/endtoend/testdata/remote_missing_semicolon/mysql/sqlc.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"version": "1",
3+
"cloud": {
4+
"hostname": "localhost",
5+
"organization": "foo",
6+
"project": "bar"
7+
},
8+
"packages": [
9+
{
10+
"path": "go",
11+
"engine": "mysql",
12+
"name": "querytest",
13+
"schema": "query.sql",
14+
"queries": "query.sql"
15+
}
16+
]
17+
}

‎internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/query.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
CREATE TABLE foo (email text not null);
2+
3+
-- name: FirstQuery :many
4+
SELECT * FROM foo;
5+
6+
-- name: SecondQuery :many
7+
SELECT * FROM foo WHERE email = $1

‎internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/sqlc.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"version": "1",
3+
"cloud": {
4+
"hostname": "localhost",
5+
"organization": "foo",
6+
"project": "bar"
7+
},
8+
"packages": [
9+
{
10+
"path": "go",
11+
"engine": "postgresql",
12+
"sql_package": "pgx/v4",
13+
"name": "querytest",
14+
"schema": "query.sql",
15+
"queries": "query.sql"
16+
}
17+
]
18+
}

‎internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/stderr.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# package querytest
2+
query.sql:7:1: missing semicolon at end of file

‎internal/endtoend/testdata/remote_missing_semicolon/pgx/v5/query.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
CREATE TABLE foo (email text not null);
2+
3+
-- name: FirstQuery :many
4+
SELECT * FROM foo;
5+
6+
-- name: SecondQuery :many
7+
SELECT * FROM foo WHERE email = $1

0 commit comments

Comments
 (0)