diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7c1fd97..0bc9bf8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,6 +30,12 @@ jobs: GO111MODULE: on run: | make cover + + - name: Running e2e tests with coverage + env: + GO111MODULE: on + run: | + make e2e - name: Send coverage uses: shogo82148/actions-goveralls@v1 with: diff --git a/.gitignore b/.gitignore index 32da575..07d2800 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ _kcl_test.k # e2e test cases dir scripts/e2e/pkg_in_reg/* -scripts/e2e/registry_auth/* \ No newline at end of file +scripts/e2e/registry_auth/* +scripts/registry_auth/* \ No newline at end of file diff --git a/cmd/kcl/commands/args.go b/cmd/kcl/commands/args.go index 8bcf372..5822169 100644 --- a/cmd/kcl/commands/args.go +++ b/cmd/kcl/commands/args.go @@ -1,11 +1,14 @@ package cmd import ( + "fmt" "net/url" "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/opt" + "kcl-lang.io/kpm/pkg/utils" ) func argsGet(a []string, n int) string { @@ -15,6 +18,105 @@ func argsGet(a []string, n int) string { return "" } +func ParseSourceFromArgs(cli *client.KpmClient, args []string) (*downloader.Source, error) { + source := downloader.Source{} + modSpec := downloader.ModSpec{} + + // Parse the source from the args + // Parse the input like: kcl mod pull k8s:1.28 or kcl mod pull oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0 + if len(args) != 0 { + // Iterate through the args to find the source + for _, arg := range args { + // if arg is a path and exists, it is a local source + if utils.DirExists(arg) { + source.Local = &downloader.Local{ + Path: arg, + } + continue + } + + // if arg is not path, it is Modspec + if err := modSpec.FromString(arg); err == nil { + // check if modspec already exists + if source.ModSpec == nil { + source.ModSpec = &modSpec + } else { + return nil, fmt.Errorf("only one modspec is allowed") + } + continue + } else { + modSpec = downloader.ModSpec{} + } + + // if arg is a url, set the source url + if source.IsNilSource() { + err := source.FromString(arg) + if err != nil { + return nil, err + } + continue + } + + if !source.IsNilSource() { + return nil, fmt.Errorf("only one source is allowed") + } + } + // For the source parsed from the args, set the tag, commit, branch + if source.Git != nil { + source.Git.Tag = tag + source.Git.Commit = commit + source.Git.Branch = branch + } else if source.Oci != nil { + source.Oci.Tag = tag + } + } + + // Parse the source from the flags + // Parse the input like: kcl mod pull --oci oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0 + if source.IsNilSource() || source.SpecOnly() { + if len(git) != 0 { + source.Git = &downloader.Git{ + Url: git, + Tag: tag, + Commit: commit, + Branch: branch, + } + } else if len(oci) != 0 { + ociUrl, err := url.Parse(oci) + if err != nil { + return nil, err + } + + ociUrl.Scheme = constants.OciScheme + query := ociUrl.Query() + query.Add(constants.Tag, tag) + ociUrl.RawQuery = query.Encode() + err = source.FromString(ociUrl.String()) + if err != nil { + return nil, err + } + } else if len(path) != 0 { + source.Local = &downloader.Local{ + Path: path, + } + } + } else if len(git) != 0 || len(oci) != 0 || len(path) != 0 { + return nil, fmt.Errorf("only one source is allowed") + } + + source.ModSpec = &modSpec + // Set the default oci registry and repo if the source is spec only + if source.SpecOnly() { + source.Oci = &downloader.Oci{ + Reg: cli.GetSettings().DefaultOciRegistry(), + Repo: utils.JoinPath(cli.GetSettings().DefaultOciRepo(), source.ModSpec.Name), + Tag: source.ModSpec.Version, + } + } + + return &source, nil +} + func ParseUrlFromArgs(cli *client.KpmClient, args []string) (*url.URL, error) { var sourceUrl url.URL diff --git a/cmd/kcl/commands/mod_pull.go b/cmd/kcl/commands/mod_pull.go index 689a745..abb5b45 100644 --- a/cmd/kcl/commands/mod_pull.go +++ b/cmd/kcl/commands/mod_pull.go @@ -3,7 +3,6 @@ package cmd import ( "github.com/spf13/cobra" "kcl-lang.io/kpm/pkg/client" - "kcl-lang.io/kpm/pkg/downloader" ) const ( @@ -28,7 +27,19 @@ const ( kcl mod pull --git ssh://github.com/kcl-lang/konfig --tag v0.4.0 # Pull the module from the OCI Registry by flag - kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0` + kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0 + + # Pull the module from the OCI Registry by flag and specify the module spce + kcl mod pull subhelloworld --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.4 + + # Pull the module from the OCI Registry by flag and specify the module spce with version + kcl mod pull subhelloworld:0.0.1 --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.4 + + # Pull the module from the Git Repo by flag and specify the module spce + kcl mod pull cc --git git://github.com/kcl-lang/flask-demo-kcl-manifests.git --commit 8308200 + + # Pull the module from the Git Repo by flag and specify the module spce with version + kcl mod pull cc:0.0.1 --git git://github.com/kcl-lang/flask-demo-kcl-manifests.git --commit 8308200` ) // NewModPullCmd returns the mod pull command. @@ -56,11 +67,7 @@ func NewModPullCmd(cli *client.KpmClient) *cobra.Command { } func pull(cli *client.KpmClient, args []string, localPath string) error { - sourceUrl, err := ParseUrlFromArgs(cli, args) - if err != nil { - return err - } - source, err := downloader.NewSourceFromStr(sourceUrl.String()) + source, err := ParseSourceFromArgs(cli, args) if err != nil { return err } diff --git a/go.mod b/go.mod index ba1f835..aeeab48 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( kcl-lang.io/kcl-go v0.10.8 kcl-lang.io/kcl-openapi v0.10.0 kcl-lang.io/kcl-plugin v0.6.0 - kcl-lang.io/kpm v0.10.0 + kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05 ) require ( @@ -33,6 +33,7 @@ require ( github.com/containers/ocicrypt v1.2.0 // indirect github.com/containers/storage v1.55.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dchest/siphash v1.2.3 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/ebitengine/purego v0.7.1 // indirect github.com/elliotchance/orderedmap/v2 v2.4.0 // indirect @@ -47,6 +48,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.6 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kubescape/go-git-url v0.0.30 // indirect diff --git a/go.sum b/go.sum index 39c05be..fbfc8ab 100644 --- a/go.sum +++ b/go.sum @@ -336,6 +336,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= @@ -664,6 +666,8 @@ github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -1698,8 +1702,8 @@ kcl-lang.io/kcl-openapi v0.10.0 h1:yetZMSnn/HHaMcfiLt1P2zhCF06O33jxkjtHrm08VR8= kcl-lang.io/kcl-openapi v0.10.0/go.mod h1:kGCf0AZygrZyB+xpmMtiC3FYoiV/1rCLXuAq2QtuLf8= kcl-lang.io/kcl-plugin v0.6.0 h1:rBdoqKDPdOtojeOHCFnXoB/I7ltFjV61r0KkfOcL5sE= kcl-lang.io/kcl-plugin v0.6.0/go.mod h1:LoIouleHYRKAvFcdW30yUlhsMYH2W9zD5Ji1XHfbht4= -kcl-lang.io/kpm v0.10.0 h1:VnsJ5IS8YSvgXYnItLdaJp/1tTrSSmThzmNCWmhm5Bg= -kcl-lang.io/kpm v0.10.0/go.mod h1:MhQh9kewILcUlhuhVBksxWVZfgdwkpkX6xFj1pxF0QM= +kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05 h1:J+Leim2cR1h6KQ0DhurhrrctgVnLrhCC/wsgJWhcBw8= +kcl-lang.io/kpm v0.10.1-0.20241107094147-b05e196b2c05/go.mod h1:1ndoNvUQdYNgoiQHIkGGRUQIWLU8BSslIbJhk9B5Kco= kcl-lang.io/lib v0.10.8 h1:/Mhko6fngIstvdx9dAS3H6N1utogkWfoARVj643l5nU= kcl-lang.io/lib v0.10.8/go.mod h1:0Dw/MQwRMjLDksxl4JerGBn/ueaxRyCCKBCCwQwJ1MI= oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk= diff --git a/scripts/e2e/pull_pkg.sh b/scripts/e2e/pull_pkg.sh index bc022d7..24ab401 100755 --- a/scripts/e2e/pull_pkg.sh +++ b/scripts/e2e/pull_pkg.sh @@ -13,11 +13,15 @@ cd ./scripts/e2e/pkg_in_reg/ # Check if file exists -if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.28" ]; then +if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.28/k8s/1.28" ]; then $current_dir/bin/kcl mod pull k8s:1.28 fi -if [ ! -d "./oci/ghcr.io/kcl-lang/helloworld/0.1.1" ]; then +if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.31.2/k8s/1.31.2" ]; then + $current_dir/bin/kcl mod pull k8s:1.31.2 +fi + +if [ ! -d "./oci/ghcr.io/kcl-lang/helloworld/0.1.1/helloworld/0.1.1" ]; then $current_dir/bin/kcl mod pull helloworld:0.1.1 fi diff --git a/scripts/e2e/push_pkg.sh b/scripts/e2e/push_pkg.sh index 517e472..e052fd8 100755 --- a/scripts/e2e/push_pkg.sh +++ b/scripts/e2e/push_pkg.sh @@ -11,13 +11,18 @@ echo $current_dir $current_dir/bin/kcl registry login -u test -p 1234 localhost:5001 -cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.28 +cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.28/k8s/1.28 +$current_dir/bin/kcl mod push + +cd "$current_dir" + +cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.31.2/k8s/1.31.2 $current_dir/bin/kcl mod push cd "$current_dir" # Push the package helloworld/0.1.1 to the registry -cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/helloworld/0.1.1 +cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/helloworld/0.1.1/helloworld/0.1.1 $current_dir/bin/kcl mod push cd "$current_dir" diff --git a/test/e2e/test_suites/test_kcl_run_26/stdout b/test/e2e/test_suites/test_kcl_run_26/stdout index f405925..e4c703e 100644 --- a/test/e2e/test_suites/test_kcl_run_26/stdout +++ b/test/e2e/test_suites/test_kcl_run_26/stdout @@ -1 +1 @@ -The_first_kcl_program: Hello World! \ No newline at end of file +sub: SUB \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_run_27/stdout b/test/e2e/test_suites/test_kcl_run_27/stdout index d2dcf2a..5ead6da 100644 --- a/test/e2e/test_suites/test_kcl_run_27/stdout +++ b/test/e2e/test_suites/test_kcl_run_27/stdout @@ -1 +1 @@ -The_first_kcl_program: Hello World! +The_sub_kcl_program: Hello Sub World! \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_run_invalid/input b/test/e2e/test_suites/test_kcl_run_invalid/input index 2b4e70f..2564c77 100644 --- a/test/e2e/test_suites/test_kcl_run_invalid/input +++ b/test/e2e/test_suites/test_kcl_run_invalid/input @@ -1 +1 @@ -kcl run --oci oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld \ No newline at end of file +kcl run --oci oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/helloworld \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_run_invalid/stderr b/test/e2e/test_suites/test_kcl_run_invalid/stderr index 57daf60..8674d21 100644 --- a/test/e2e/test_suites/test_kcl_run_invalid/stderr +++ b/test/e2e/test_suites/test_kcl_run_invalid/stderr @@ -1 +1 @@ -cannot specify multiple KCL modules [oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld] \ No newline at end of file +cannot specify multiple KCL modules [oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/helloworld] \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_run_invalid_1/input b/test/e2e/test_suites/test_kcl_run_invalid_1/input index d69ff2e..f4259f7 100644 --- a/test/e2e/test_suites/test_kcl_run_invalid_1/input +++ b/test/e2e/test_suites/test_kcl_run_invalid_1/input @@ -1 +1 @@ -kcl run --oci oci://ghcr.io/kcl-lang/hellworld oci://ghcr.io/kcl-lang/hellworld --tag v0.1.0 \ No newline at end of file +kcl run --oci oci://ghcr.io/kcl-lang/helloworld oci://ghcr.io/kcl-lang/hellworld --tag v0.1.0 \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_run_invalid_1/stderr b/test/e2e/test_suites/test_kcl_run_invalid_1/stderr index 033aa36..3bbe81d 100644 --- a/test/e2e/test_suites/test_kcl_run_invalid_1/stderr +++ b/test/e2e/test_suites/test_kcl_run_invalid_1/stderr @@ -1 +1 @@ -cannot specify tag, commit, or branch with multiple modules [oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0 oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0] \ No newline at end of file +cannot specify tag, commit, or branch with multiple modules [oci://ghcr.io/kcl-lang/helloworld?tag=v0.1.0 oci://ghcr.io/kcl-lang/hellworld?tag=v0.1.0] \ No newline at end of file