Allow ship to be pointed to git file paths (#646)

Author: Rob0h

integration/init/git-single-file/expected/.ship/state.json

@@ -0,0 +1,8 @@
+{
+  "v1": {
+    "config": {},
+    "upstream": "github.com/replicatedhq/test-charts/blob/3427d6997bd150c60caa00ba0298fdfe17e3ed04/plain-k8s/frontend-deployment.yaml",
+    "metadata": null,
+    "contentSHA": "9fa025c3fdbcc02c7de8e9c74c8058d16b2aada04917895685504b387563afda"
+  }
+}

integration/init/git-single-file/expected/base/kustomization.yaml

@@ -0,0 +1,4 @@
+kind: ""
+apiversion: ""
+resources:
+- plain-k8s/frontend-deployment.yaml

integration/init/git-single-file/expected/base/plain-k8s/frontend-deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1 #  for k8s versions before 1.9.0 use apps/v1beta2  and before 1.8.0 use extensions/v1beta1
+kind: Deployment
+metadata:
+  name: frontend
+spec:
+  selector:
+    matchLabels:
+      app: guestbook
+      tier: frontend
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: guestbook
+        tier: frontend
+    spec:
+      containers:
+      - name: php-redis
+        image: gcr.io/google-samples/gb-frontend:v4
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: GET_HOSTS_FROM
+          value: dns
+          # If your cluster config does not include a dns service, then to
+          # instead access environment variables to find service host
+          # info, comment out the 'value: dns' line above, and uncomment the
+          # line below:
+          # value: env
+        ports:
+        - containerPort: 80

integration/init/git-single-file/expected/overlays/ship/kustomization.yaml

@@ -0,0 +1,4 @@
+kind: ""
+apiversion: ""
+bases:
+- ../../base

integration/init/git-single-file/expected/rendered.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: guestbook
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: guestbook
+        tier: frontend
+    spec:
+      containers:
+      - env:
+        - name: GET_HOSTS_FROM
+          value: dns
+        image: gcr.io/google-samples/gb-frontend:v4
+        name: php-redis
+        ports:
+        - containerPort: 80
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi

integration/init/git-single-file/metadata.yaml

@@ -0,0 +1,3 @@
+upstream: "github.com/replicatedhq/test-charts/blob/3427d6997bd150c60caa00ba0298fdfe17e3ed04/plain-k8s/frontend-deployment.yaml"
+args: []
+skip_cleanup: false

pkg/specs/githubclient/client.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -125,17 +126,6 @@ func (g *GithubClient) downloadAndExtractFiles(
 		}
 
 		switch header.Typeflag {
-		case tar.TypeDir:
-			dirName := strings.Join(strings.Split(header.Name, "/")[1:], "/")
-			if !strings.HasPrefix(dirName, basePath) {
-				continue
-			}
-			basePathFound = true
-
-			dirName = strings.TrimPrefix(dirName, basePath)
-			if err := g.fs.MkdirAll(filepath.Join(filePath, dirName), 0755); err != nil {
-				return errors.Wrapf(err, "extract tar gz, mkdir")
-			}
 		case tar.TypeReg:
 			// need this in a func because defer in a loop was leaking handles
 			err := func() error {
@@ -145,7 +135,13 @@ func (g *GithubClient) downloadAndExtractFiles(
 				}
 				basePathFound = true
 
-				fileName = strings.TrimPrefix(fileName, basePath)
+				if fileName != basePath {
+					fileName = strings.TrimPrefix(fileName, basePath)
+				}
+				dirPath, _ := path.Split(fileName)
+				if err := g.fs.MkdirAll(filepath.Join(filePath, dirPath), 0755); err != nil {
+					return errors.Wrapf(err, "extract tar gz, mkdir")
+				}
 				outFile, err := g.fs.Create(filepath.Join(filePath, fileName))
 				if err != nil {
 					return errors.Wrapf(err, "extract tar gz, create")
@@ -177,7 +173,7 @@ func decodeGitHubURL(chartPath string) (owner string, repo string, branch string
 	branch = ""
 	path = ""
 	if len(splitPath) > 3 {
-		if splitPath[3] == "tree" {
+		if splitPath[3] == "tree" || splitPath[3] == "blob" {
 			branch = splitPath[4]
 			path = strings.Join(splitPath[5:], "/")
 		} else {

pkg/specs/githubclient/client_test.go

@@ -42,10 +42,13 @@ func TestGithubClient(t *testing.T) {
 
 var _ = Describe("GithubClient", func() {
 	client, mux, serverURL, teardown = setupGitClient()
-	mux.HandleFunc("/repos/o/r/tarball", func(w http.ResponseWriter, r *http.Request) {
+	redirectArchive := func(w http.ResponseWriter, r *http.Request) {
 		http.Redirect(w, r, serverURL+"/archive.tar.gz", http.StatusFound)
 		return
-	})
+	}
+	mux.HandleFunc("/repos/o/r/tarball/", redirectArchive)
+	mux.HandleFunc("/repos/o/r/tarball", redirectArchive)
+
 	mux.HandleFunc("/archive.tar.gz", func(w http.ResponseWriter, r *http.Request) {
 		archiveData := `H4sIAJKjXFsAA+3WXW6CQBQFYJbCBmrv/D831ce+uIOpDtGEKQaoibt3qERbEmiNI6TxfC8TIwkXTg65lfW73D3ZcrXZ7t1zcg9EZJRKv059OonL09lKmRDcMM6k0SkxSYolqbrLNB2fVW3LMIoPr2DounBZlg383z7H+fwnqp/5v25sWc8O1ucR7xHeh5ZyKH9xzl+TDPkroylJKeIMvR48//fw8PC4Ov1fLl7mb4uZX8e8xzX9V4Y1/RdMof9jyIpi6hFgQp3+1y78tLWrYm6CV+1/oum/JqGx/42hN/+12+XFwbuPsA7euA3++v1n/LL/sZA/JyM4vv9juMQ89SQwhd7+V67cb1fu5vInf9n/zLf+y6b/nDP0fwxtzFOPAQAAAAAAAAAAAACRHQEZehxJACgAAA==`
 		dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(archiveData))
@@ -127,6 +130,43 @@ var _ = Describe("GithubClient", func() {
 				Expect(err.Error()).To(Equal("http://gitlab.com/o/r is not a Github URL"))
 			})
 		})
+
+		Context("With a url path to a single file at the base of the repo", func() {
+			It("should fetch and persist the file", func() {
+				validGithubURLSingle := "github.com/o/r/blob/master/Chart.yaml"
+				mockFs := afero.Afero{Fs: afero.NewMemMapFs()}
+				gitClient := &GithubClient{
+					client: client,
+					fs:     mockFs,
+					logger: log.NewNopLogger(),
+				}
+
+				err := gitClient.GetFiles(context.Background(), validGithubURLSingle, constants.HelmChartPath)
+
+				chart, err := gitClient.fs.ReadFile(path.Join(constants.HelmChartPath, "Chart.yaml"))
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(string(chart)).To(Equal("bar"))
+			})
+		})
+
+		Context("With a url path to a single nested file", func() {
+			It("should fetch and persist the file", func() {
+				validGithubURLSingle := "github.com/o/r/blob/master/templates/service.yml"
+				mockFs := afero.Afero{Fs: afero.NewMemMapFs()}
+				gitClient := &GithubClient{
+					client: client,
+					fs:     mockFs,
+					logger: log.NewNopLogger(),
+				}
+
+				err := gitClient.GetFiles(context.Background(), validGithubURLSingle, constants.HelmChartPath)
+				chart, err := gitClient.fs.ReadFile(path.Join(constants.HelmChartPath, "templates", "service.yml"))
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(string(chart)).To(Equal("service"))
+			})
+		})
 	})
 
 	Describe("decodeGitHubURL", func() {