Merge pull request #163 from kcl-lang/pull-mod-spec

feat: 'kcl mod pull' supports ModSpec

.github/workflows/ci.yaml

@@ -30,6 +30,12 @@ jobs:
           GO111MODULE: on
         run: |
           make cover
+
+      - name: Running e2e tests with coverage
+        env:
+          GO111MODULE: on
+        run: |
+          make e2e
       - name: Send coverage
         uses: shogo82148/actions-goveralls@v1
         with:

.gitignore

@@ -37,4 +37,5 @@ _kcl_test.k
 
 # e2e test cases dir
 scripts/e2e/pkg_in_reg/*
-scripts/e2e/registry_auth/*
+scripts/e2e/registry_auth/*
+scripts/registry_auth/*

cmd/kcl/commands/args.go

@@ -1,11 +1,14 @@
 package cmd
 
 import (
+	"fmt"
 	"net/url"
 
 	"kcl-lang.io/kpm/pkg/client"
 	"kcl-lang.io/kpm/pkg/constants"
+	"kcl-lang.io/kpm/pkg/downloader"
 	"kcl-lang.io/kpm/pkg/opt"
+	"kcl-lang.io/kpm/pkg/utils"
 )
 
 func argsGet(a []string, n int) string {
@@ -15,6 +18,105 @@ func argsGet(a []string, n int) string {
 	return ""
 }
 
+func ParseSourceFromArgs(cli *client.KpmClient, args []string) (*downloader.Source, error) {
+	source := downloader.Source{}
+	modSpec := downloader.ModSpec{}
+
+	// Parse the source from the args
+	// Parse the input like: kcl mod pull k8s:1.28 or kcl mod pull oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0
+	if len(args) != 0 {
+		// Iterate through the args to find the source
+		for _, arg := range args {
+			// if arg is a path and exists, it is a local source
+			if utils.DirExists(arg) {
+				source.Local = &downloader.Local{
+					Path: arg,
+				}
+				continue
+			}
+
+			// if arg is not path, it is Modspec
+			if err := modSpec.FromString(arg); err == nil {
+				// check if modspec already exists
+				if source.ModSpec == nil {
+					source.ModSpec = &modSpec
+				} else {
+					return nil, fmt.Errorf("only one modspec is allowed")
+				}
+				continue
+			} else {
+				modSpec = downloader.ModSpec{}
+			}
+
+			// if arg is a url, set the source url
+			if source.IsNilSource() {
+				err := source.FromString(arg)
+				if err != nil {
+					return nil, err
+				}
+				continue
+			}
+
+			if !source.IsNilSource() {
+				return nil, fmt.Errorf("only one source is allowed")
+			}
+		}
+		// For the source parsed from the args, set the tag, commit, branch
+		if source.Git != nil {
+			source.Git.Tag = tag
+			source.Git.Commit = commit
+			source.Git.Branch = branch
+		} else if source.Oci != nil {
+			source.Oci.Tag = tag
+		}
+	}
+
+	// Parse the source from the flags
+	// Parse the input like: kcl mod pull --oci oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0
+	if source.IsNilSource() || source.SpecOnly() {
+		if len(git) != 0 {
+			source.Git = &downloader.Git{
+				Url:    git,
+				Tag:    tag,
+				Commit: commit,
+				Branch: branch,
+			}
+		} else if len(oci) != 0 {
+			ociUrl, err := url.Parse(oci)
+			if err != nil {
+				return nil, err
+			}
+
+			ociUrl.Scheme = constants.OciScheme
+			query := ociUrl.Query()
+			query.Add(constants.Tag, tag)
+			ociUrl.RawQuery = query.Encode()
+			err = source.FromString(ociUrl.String())
+			if err != nil {
+				return nil, err
+			}
+		} else if len(path) != 0 {
+			source.Local = &downloader.Local{
+				Path: path,
+			}
+		}
+	} else if len(git) != 0 || len(oci) != 0 || len(path) != 0 {
+		return nil, fmt.Errorf("only one source is allowed")
+	}
+
+	source.ModSpec = &modSpec
+	// Set the default oci registry and repo if the source is spec only
+	if source.SpecOnly() {
+		source.Oci = &downloader.Oci{
+			Reg:  cli.GetSettings().DefaultOciRegistry(),
+			Repo: utils.JoinPath(cli.GetSettings().DefaultOciRepo(), source.ModSpec.Name),
+			Tag:  source.ModSpec.Version,
+		}
+	}
+
+	return &source, nil
+}
+
 func ParseUrlFromArgs(cli *client.KpmClient, args []string) (*url.URL, error) {
 	var sourceUrl url.URL
 

cmd/kcl/commands/mod_pull.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"kcl-lang.io/kpm/pkg/client"
-	"kcl-lang.io/kpm/pkg/downloader"
 )
 
 const (
@@ -28,7 +27,19 @@ const (
   kcl mod pull --git ssh://github.com/kcl-lang/konfig --tag v0.4.0
 
   # Pull the module from the OCI Registry by flag
-  kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0`
+  kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0
+  
+  # Pull the module from the OCI Registry by flag and specify the module spce
+  kcl mod pull subhelloworld --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.4
+  
+  # Pull the module from the OCI Registry by flag and specify the module spce with version
+  kcl mod pull subhelloworld:0.0.1 --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.4
+  
+  # Pull the module from the Git Repo by flag and specify the module spce
+  kcl mod pull cc --git git://github.com/kcl-lang/flask-demo-kcl-manifests.git --commit 8308200
+  
+  # Pull the module from the Git Repo by flag and specify the module spce with version
+  kcl mod pull cc:0.0.1 --git git://github.com/kcl-lang/flask-demo-kcl-manifests.git --commit 8308200`
 )
 
 // NewModPullCmd returns the mod pull command.
@@ -56,11 +67,7 @@ func NewModPullCmd(cli *client.KpmClient) *cobra.Command {
 }
 
 func pull(cli *client.KpmClient, args []string, localPath string) error {
-	sourceUrl, err := ParseUrlFromArgs(cli, args)
-	if err != nil {
-		return err
-	}
-	source, err := downloader.NewSourceFromStr(sourceUrl.String())
+	source, err := ParseSourceFromArgs(cli, args)
 	if err != nil {
 		return err
 	}

go.mod

@@ -13,7 +13,7 @@ require (
 	kcl-lang.io/kcl-go v0.10.8
 	kcl-lang.io/kcl-openapi v0.10.0
 	kcl-lang.io/kcl-plugin v0.6.0
-	kcl-lang.io/kpm v0.10.0
+	kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05
 )
 
 require (
@@ -33,6 +33,7 @@ require (
 	github.com/containers/ocicrypt v1.2.0 // indirect
 	github.com/containers/storage v1.55.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dchest/siphash v1.2.3 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/ebitengine/purego v0.7.1 // indirect
 	github.com/elliotchance/orderedmap/v2 v2.4.0 // indirect
@@ -47,6 +48,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-getter v1.7.6 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/kubescape/go-git-url v0.0.30 // indirect

go.sum

@@ -336,6 +336,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
+github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
@@ -664,6 +666,8 @@ github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
 github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -1698,8 +1702,8 @@ kcl-lang.io/kcl-openapi v0.10.0 h1:yetZMSnn/HHaMcfiLt1P2zhCF06O33jxkjtHrm08VR8=
 kcl-lang.io/kcl-openapi v0.10.0/go.mod h1:kGCf0AZygrZyB+xpmMtiC3FYoiV/1rCLXuAq2QtuLf8=
 kcl-lang.io/kcl-plugin v0.6.0 h1:rBdoqKDPdOtojeOHCFnXoB/I7ltFjV61r0KkfOcL5sE=
 kcl-lang.io/kcl-plugin v0.6.0/go.mod h1:LoIouleHYRKAvFcdW30yUlhsMYH2W9zD5Ji1XHfbht4=
-kcl-lang.io/kpm v0.10.0 h1:VnsJ5IS8YSvgXYnItLdaJp/1tTrSSmThzmNCWmhm5Bg=
-kcl-lang.io/kpm v0.10.0/go.mod h1:MhQh9kewILcUlhuhVBksxWVZfgdwkpkX6xFj1pxF0QM=
+kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05 h1:J+Leim2cR1h6KQ0DhurhrrctgVnLrhCC/wsgJWhcBw8=
+kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05/go.mod h1:1ndoNvUQdYNgoiQHIkGGRUQIWLU8BSslIbJhk9B5Kco=
 kcl-lang.io/lib v0.10.8 h1:/Mhko6fngIstvdx9dAS3H6N1utogkWfoARVj643l5nU=
 kcl-lang.io/lib v0.10.8/go.mod h1:0Dw/MQwRMjLDksxl4JerGBn/ueaxRyCCKBCCwQwJ1MI=
 oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk=

scripts/e2e/pull_pkg.sh

@@ -13,11 +13,15 @@ cd ./scripts/e2e/pkg_in_reg/
 
 
 # Check if file exists
-if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.28" ]; then
+if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.28/k8s/1.28" ]; then
   $current_dir/bin/kcl mod pull k8s:1.28
 fi
 
-if [ ! -d "./oci/ghcr.io/kcl-lang/helloworld/0.1.1" ]; then
+if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.31.2/k8s/1.31.2" ]; then
+  $current_dir/bin/kcl mod pull k8s:1.31.2
+fi
+
+if [ ! -d "./oci/ghcr.io/kcl-lang/helloworld/0.1.1/helloworld/0.1.1" ]; then
   $current_dir/bin/kcl mod pull helloworld:0.1.1
 fi
 

scripts/e2e/push_pkg.sh

@@ -11,13 +11,18 @@ echo $current_dir
 
 $current_dir/bin/kcl registry login -u test -p 1234 localhost:5001
 
-cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.28
+cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.28/k8s/1.28
+$current_dir/bin/kcl mod push
+
+cd "$current_dir"
+
+cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.31.2/k8s/1.31.2
 $current_dir/bin/kcl mod push
 
 cd "$current_dir"
 
 # Push the package helloworld/0.1.1 to the registry
-cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/helloworld/0.1.1
+cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/helloworld/0.1.1/helloworld/0.1.1
 $current_dir/bin/kcl mod push
 
 cd "$current_dir"

test/e2e/test_suites/test_kcl_run_26/stdout

@@ -1 +1 @@
-The_first_kcl_program: Hello World!
+sub: SUB

test/e2e/test_suites/test_kcl_run_27/stdout

@@ -1 +1 @@
-The_first_kcl_program: Hello World!
+The_sub_kcl_program: Hello Sub World!

test/e2e/test_suites/test_kcl_run_invalid/input

@@ -1 +1 @@
-kcl run --oci oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld
+kcl run --oci oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/helloworld

test/e2e/test_suites/test_kcl_run_invalid/stderr

@@ -1 +1 @@
-cannot specify multiple KCL modules [oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld]
+cannot specify multiple KCL modules [oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/helloworld]

test/e2e/test_suites/test_kcl_run_invalid_1/input

@@ -1 +1 @@
-kcl run --oci oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld --tag v0.1.0
+kcl run --oci oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/hellworld --tag v0.1.0

test/e2e/test_suites/test_kcl_run_invalid_1/stderr

@@ -1 +1 @@
-cannot specify tag, commit, or branch with multiple modules [oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0 oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0]
+cannot specify tag, commit, or branch with multiple modules [oci://ghcr.io/kcl-lang/helloworld?tag=v0.1.0 oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0]