Skip to content

Commit bb58088

Browse files
harry-hovharry-hov
authored
feat(mod): fetch indirect dependencies (#602)
1 parent a2966a7 commit bb58088

File tree

5 files changed

+220
-42
lines changed

5 files changed

+220
-42
lines changed

‎cmd/gnodev/mod.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,12 @@ func execModDownload(cfg *modDownloadCfg, args []string, io *commands.IO) error
9292
return fmt.Errorf("validate: %w", err)
9393
}
9494

95+
gnoModPath, err := gnomod.GetGnoModPath()
96+
if err != nil {
97+
return fmt.Errorf("get gno.mod path: %w", err)
98+
}
9599
// fetch dependencies
96-
if err := gnoMod.FetchDeps(cfg.remote); err != nil {
100+
if err := gnoMod.FetchDeps(gnoModPath, cfg.remote); err != nil {
97101
return fmt.Errorf("fetch: %w", err)
98102
}
99103

‎pkgs/gnolang/gnomod/file.go‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import (
66
"log"
77
"os"
88
"path/filepath"
9+
"strings"
910

11+
"github.com/gnolang/gno/pkgs/gnolang"
1012
"golang.org/x/mod/modfile"
1113
"golang.org/x/mod/module"
1214
)
@@ -32,12 +34,7 @@ func (f *File) Validate() error {
3234

3335
// FetchDeps fetches and writes gno.mod packages
3436
// in GOPATH/pkg/gnomod/
35-
func (f *File) FetchDeps(remote string) error {
36-
gnoModPath, err := GetGnoModPath()
37-
if err != nil {
38-
return fmt.Errorf("get gno.mod path: %w", err)
39-
}
40-
37+
func (f *File) FetchDeps(path string, remote string) error {
4138
for _, r := range f.Require {
4239
mod, replaced := isReplaced(r.Mod, f.Replace)
4340
if replaced {
@@ -46,21 +43,63 @@ func (f *File) FetchDeps(remote string) error {
4643
}
4744
r.Mod = *mod
4845
}
49-
log.Println("fetching", r.Mod.Path)
50-
err := writePackage(remote, gnoModPath, r.Mod.Path)
46+
indirect := ""
47+
if r.Indirect {
48+
indirect = "// indirect"
49+
}
50+
51+
_, err := os.Stat(filepath.Join(path, r.Mod.Path))
52+
if !os.IsNotExist(err) {
53+
log.Println("cached", r.Mod.Path, indirect)
54+
continue
55+
}
56+
log.Println("fetching", r.Mod.Path, indirect)
57+
requirements, err := writePackage(remote, path, r.Mod.Path)
5158
if err != nil {
5259
return fmt.Errorf("writepackage: %w", err)
5360
}
5461

55-
f := &File{
62+
modFile := &File{
5663
Module: &modfile.Module{
5764
Mod: module.Version{
5865
Path: r.Mod.Path,
5966
},
6067
},
6168
}
69+
for _, req := range requirements {
70+
path := req[1 : len(req)-1] // trim leading and trailing `"`
71+
if strings.HasSuffix(path, modFile.Module.Mod.Path) {
72+
continue
73+
}
74+
// skip if `std`, special case.
75+
if path == gnolang.GnoStdPkgAfter {
76+
continue
77+
}
6278

63-
f.WriteToPath(filepath.Join(gnoModPath, r.Mod.Path))
79+
if strings.HasPrefix(path, gnolang.ImportPrefix) {
80+
path = strings.TrimPrefix(path, gnolang.ImportPrefix+"/examples/")
81+
modFile.Require = append(modFile.Require, &modfile.Require{
82+
Mod: module.Version{
83+
Path: path,
84+
Version: "v0.0.0", // TODO: Use latest?
85+
},
86+
Indirect: true,
87+
})
88+
}
89+
}
90+
91+
err = modFile.FetchDeps(path, remote)
92+
if err != nil {
93+
return err
94+
}
95+
goMod, err := GnoToGoMod(*modFile)
96+
if err != nil {
97+
return err
98+
}
99+
err = goMod.WriteToPath(filepath.Join(path, r.Mod.Path))
100+
if err != nil {
101+
return err
102+
}
64103
}
65104

66105
return nil

‎pkgs/gnolang/gnomod/file_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package gnomod
2+
3+
import (
4+
"bytes"
5+
"log"
6+
"os"
7+
"path/filepath"
8+
"testing"
9+
10+
"github.com/gnolang/gno/pkgs/testutils"
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
"golang.org/x/mod/modfile"
14+
"golang.org/x/mod/module"
15+
)
16+
17+
const testRemote string = "test3.gno.land:36657"
18+
19+
func TestFetchDeps(t *testing.T) {
20+
for _, tc := range []struct {
21+
desc string
22+
modFile File
23+
requirements []string
24+
stdOutContains []string
25+
cachedStdOutContains []string
26+
}{
27+
{
28+
desc: "fetch_gno.land/p/demo/avl",
29+
modFile: File{
30+
Module: &modfile.Module{
31+
Mod: module.Version{
32+
Path: "testFetchDeps",
33+
},
34+
},
35+
Require: []*modfile.Require{
36+
{
37+
Mod: module.Version{
38+
Path: "gno.land/p/demo/avl",
39+
Version: "v0.0.0",
40+
},
41+
},
42+
},
43+
},
44+
requirements: []string{"avl"},
45+
stdOutContains: []string{
46+
"fetching gno.land/p/demo/avl",
47+
},
48+
cachedStdOutContains: []string{
49+
"cached gno.land/p/demo/avl",
50+
},
51+
}, {
52+
desc: "fetch_gno.land/p/demo/blog",
53+
modFile: File{
54+
Module: &modfile.Module{
55+
Mod: module.Version{
56+
Path: "testFetchDeps",
57+
},
58+
},
59+
Require: []*modfile.Require{
60+
{
61+
Mod: module.Version{
62+
Path: "gno.land/p/demo/blog",
63+
Version: "v0.0.0",
64+
},
65+
},
66+
},
67+
},
68+
requirements: []string{"avl", "blog", "ufmt"},
69+
stdOutContains: []string{
70+
"fetching gno.land/p/demo/blog",
71+
"fetching gno.land/p/demo/avl // indirect",
72+
"fetching gno.land/p/demo/ufmt // indirect",
73+
},
74+
cachedStdOutContains: []string{
75+
"cached gno.land/p/demo/blog",
76+
},
77+
},
78+
} {
79+
t.Run(tc.desc, func(t *testing.T) {
80+
var buf bytes.Buffer
81+
log.SetOutput(&buf)
82+
defer func() {
83+
log.SetOutput(os.Stderr)
84+
}()
85+
86+
// Create test dir
87+
dirPath, cleanUpFn := testutils.NewTestCaseDir(t)
88+
assert.NotNil(t, dirPath)
89+
defer cleanUpFn()
90+
91+
// Fetching dependencies
92+
tc.modFile.FetchDeps(dirPath, testRemote)
93+
94+
// Read dir
95+
entries, err := os.ReadDir(filepath.Join(dirPath, "gno.land", "p", "demo"))
96+
require.Nil(t, err)
97+
98+
// Check dir entries
99+
assert.Equal(t, len(tc.requirements), len(entries))
100+
for _, e := range entries {
101+
assert.Contains(t, tc.requirements, e.Name())
102+
}
103+
104+
// Check logs
105+
for _, c := range tc.stdOutContains {
106+
assert.Contains(t, buf.String(), c)
107+
}
108+
109+
buf.Reset()
110+
111+
// Try fetching again. Should be cached
112+
tc.modFile.FetchDeps(dirPath, testRemote)
113+
for _, c := range tc.cachedStdOutContains {
114+
assert.Contains(t, buf.String(), c)
115+
}
116+
})
117+
}
118+
}

‎pkgs/gnolang/gnomod/gnomod.go‎

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -25,28 +25,30 @@ func GetGnoModPath() (string, error) {
2525
return filepath.Join(goPath, "pkg", "gnomod"), nil
2626
}
2727

28-
func writePackage(remote, basePath, pkgPath string) error {
28+
func writePackage(remote, basePath, pkgPath string) (requirements []string, err error) {
2929
res, err := queryChain(remote, queryPathFile, []byte(pkgPath))
3030
if err != nil {
31-
return fmt.Errorf("querychain: %w", err)
31+
return nil, fmt.Errorf("querychain: %w", err)
3232
}
3333

3434
dirPath, fileName := std.SplitFilepath(pkgPath)
3535
if fileName == "" {
3636
// Is Dir
3737
// Create Dir if not exists
3838
dirPath := filepath.Join(basePath, dirPath)
39-
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
40-
if err := os.MkdirAll(dirPath, 0o755); err != nil {
41-
return fmt.Errorf("mkdir %q: %w", dirPath, err)
39+
if _, err = os.Stat(dirPath); os.IsNotExist(err) {
40+
if err = os.MkdirAll(dirPath, 0o755); err != nil {
41+
return nil, fmt.Errorf("mkdir %q: %w", dirPath, err)
4242
}
4343
}
4444

4545
files := strings.Split(string(res.Data), "\n")
4646
for _, file := range files {
47-
if err := writePackage(remote, basePath, filepath.Join(pkgPath, file)); err != nil {
48-
return fmt.Errorf("writepackage: %w", err)
47+
reqs, err := writePackage(remote, basePath, filepath.Join(pkgPath, file))
48+
if err != nil {
49+
return nil, fmt.Errorf("writepackage: %w", err)
4950
}
51+
requirements = append(requirements, reqs...)
5052
}
5153
} else {
5254
// Is File
@@ -55,17 +57,21 @@ func writePackage(remote, basePath, pkgPath string) error {
5557
targetFilename, _ := gnolang.GetPrecompileFilenameAndTags(filePath)
5658
precompileRes, err := gnolang.Precompile(string(res.Data), "", fileName)
5759
if err != nil {
58-
return fmt.Errorf("precompile: %w", err)
60+
return nil, fmt.Errorf("precompile: %w", err)
61+
}
62+
63+
for _, i := range precompileRes.Imports {
64+
requirements = append(requirements, i.Path.Value)
5965
}
6066

6167
fileNameWithPath := filepath.Join(basePath, dirPath, targetFilename)
6268
err = os.WriteFile(fileNameWithPath, []byte(precompileRes.Translated), 0o644)
6369
if err != nil {
64-
return fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
70+
return nil, fmt.Errorf("writefile %q: %w", fileNameWithPath, err)
6571
}
6672
}
6773

68-
return nil
74+
return removeDuplicateStr(requirements), nil
6975
}
7076

7177
// GnoToGoMod make necessary modifications in the gno.mod
@@ -76,9 +82,9 @@ func GnoToGoMod(f File) (*File, error) {
7682
return nil, err
7783
}
7884

79-
if strings.HasPrefix(f.Module.Mod.Path, "gno.land/r/") ||
80-
strings.HasPrefix(f.Module.Mod.Path, "gno.land/p/demo/") {
81-
f.Module.Mod.Path = "github.com/gnolang/gno/examples/" + f.Module.Mod.Path
85+
if strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
86+
strings.HasPrefix(f.Module.Mod.Path, gnolang.GnoPackagePrefixBefore) {
87+
f.Module.Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Module.Mod.Path
8288
}
8389

8490
for i := range f.Require {
@@ -89,9 +95,9 @@ func GnoToGoMod(f File) (*File, error) {
8995
}
9096
}
9197
path := f.Require[i].Mod.Path
92-
if strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/r/") ||
93-
strings.HasPrefix(f.Require[i].Mod.Path, "gno.land/p/demo/") {
94-
f.Require[i].Mod.Path = "github.com/gnolang/gno/examples/" + f.Require[i].Mod.Path
98+
if strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoRealmPkgsPrefixBefore) ||
99+
strings.HasPrefix(f.Require[i].Mod.Path, gnolang.GnoPackagePrefixBefore) {
100+
f.Require[i].Mod.Path = gnolang.ImportPrefix + "/examples/" + f.Require[i].Mod.Path
95101
}
96102

97103
f.Replace = append(f.Replace, &modfile.Replace{
@@ -153,3 +159,14 @@ func isReplaced(module module.Version, repl []*modfile.Replace) (*module.Version
153159
}
154160
return nil, false
155161
}
162+
163+
func removeDuplicateStr(str []string) (res []string) {
164+
m := make(map[string]struct{}, len(str))
165+
for _, s := range str {
166+
if _, ok := m[s]; !ok {
167+
m[s] = struct{}{}
168+
res = append(res, s)
169+
}
170+
}
171+
return
172+
}

‎pkgs/gnolang/precompile.go‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ import (
2020
)
2121

2222
const (
23-
gnoRealmPkgsPrefixBefore = "gno.land/r/"
24-
gnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
25-
gnoPackagePrefixBefore = "gno.land/p/demo/"
26-
gnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
27-
gnoStdPkgBefore = "std"
28-
gnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
23+
GnoRealmPkgsPrefixBefore = "gno.land/r/"
24+
GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/"
25+
GnoPackagePrefixBefore = "gno.land/p/demo/"
26+
GnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/"
27+
GnoStdPkgBefore = "std"
28+
GnoStdPkgAfter = "github.com/gnolang/gno/stdlibs/stdshim"
2929
)
3030

3131
var stdlibWhitelist = []string{
@@ -263,11 +263,11 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
263263
for _, importSpec := range paragraph {
264264
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)
265265

266-
if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
266+
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
267267
continue
268268
}
269269

270-
if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
270+
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
271271
continue
272272
}
273273

@@ -303,24 +303,24 @@ func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.N
303303
importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`)
304304

305305
// std package
306-
if importPath == gnoStdPkgBefore {
307-
if !astutil.RewriteImport(fset, f, gnoStdPkgBefore, gnoStdPkgAfter) {
308-
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", gnoStdPkgBefore, gnoStdPkgAfter))
306+
if importPath == GnoStdPkgBefore {
307+
if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) {
308+
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter))
309309
}
310310
}
311311

312312
// p/pkg packages
313-
if strings.HasPrefix(importPath, gnoPackagePrefixBefore) {
314-
target := gnoPackagePrefixAfter + strings.TrimPrefix(importPath, gnoPackagePrefixBefore)
313+
if strings.HasPrefix(importPath, GnoPackagePrefixBefore) {
314+
target := GnoPackagePrefixAfter + strings.TrimPrefix(importPath, GnoPackagePrefixBefore)
315315

316316
if !astutil.RewriteImport(fset, f, importPath, target) {
317317
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))
318318
}
319319
}
320320

321321
// r/realm packages
322-
if strings.HasPrefix(importPath, gnoRealmPkgsPrefixBefore) {
323-
target := gnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, gnoRealmPkgsPrefixBefore)
322+
if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) {
323+
target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore)
324324

325325
if !astutil.RewriteImport(fset, f, importPath, target) {
326326
errs = multierr.Append(errs, fmt.Errorf("failed to replace the %q package with %q", importPath, target))

0 commit comments

Comments
 (0)