feat(features): make direct-tarball download match the TS CLI

downloadFeatureTarball was a bare GET: no auth, no User-Agent, no integrity
check, 200-only. It worked for public tarballs but 404'd on private GitHub
release assets and never recorded a digest. Bring it to parity with the TS
request/getRequestHeaders/fetchContentsAtTarballUri path:

  - User-Agent: devcontainer on every request.
  - Authorization: Bearer $GITHUB_TOKEN for github.com / api.github.com URLs
    (from the process env via osEnvMap), so private assets resolve. Go strips
    the header on the cross-host redirect to the presigned CDN URL, which is the
    correct behavior for release assets.
  - Accept any 2xx, not only 200 (TS treats 200-299 as success).
  - Warn when fetching over plaintext HTTP (http:// or a localhost host).
  - Compute and return the tarball's sha256; the direct-tarball feature Set now
    carries ComputedDigest so the lockfile pins it, and a fetch is rejected when
    its digest does not match an existing lockfile integrity.
  - Thread ctx/logger/env; keep the shared proxy- and custom-CA-aware transport.

Header/auth/digest logic is factored into small pure helpers with unit tests
(featureRequestHeaders, isGitHubTarballURI, isPlainHTTPURL) plus httptest
coverage of the real download path. A sweep confirms every HTTP path in the
CLI (httpx, oras/OCI, tarball) now goes through httpx.NewTransport.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: aledbf

internal/cli/feature_deps.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -63,7 +64,7 @@ func newMetadataProcessFeature(client oci.Registry, logger log.Logger, basePath
 			}, nil
 
 		case features.SourceDirectTarball:
-			meta, err := readTarballFeatureMetadata(id)
+			meta, err := readTarballFeatureMetadata(logger, id)
 			if err != nil {
 				return nil, fmt.Errorf("ERR: Feature '%s' could not be processed.  %w", id, err)
 			}
@@ -177,8 +178,8 @@ func readLocalFeatureMetadata(dir string) (features.Feature, error) {
 	return meta, nil
 }
 
-func readTarballFeatureMetadata(url string) (features.Feature, error) {
-	blobData, err := downloadFeatureTarball(url)
+func readTarballFeatureMetadata(logger log.Logger, tarballURL string) (features.Feature, error) {
+	blobData, _, err := downloadFeatureTarball(context.Background(), logger, tarballURL, osEnvMap())
 	if err != nil {
 		return features.Feature{}, err
 	}

internal/cli/feature_install.go

@@ -4,10 +4,13 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -255,10 +258,17 @@ func processInstallFeature(
 		pfs.MkdirAll(featureDir)
 
 		logger.Write(fmt.Sprintf("Fetching feature tarball %s...", id), log.LevelInfo)
-		blobData, dlErr := downloadFeatureTarball(id)
+		blobData, computedDigest, dlErr := downloadFeatureTarball(context.Background(), logger, id, osEnvMap())
 		if dlErr != nil {
 			return nil, fmt.Errorf("ERR: Feature '%s' could not be processed.  %w", id, dlErr)
 		}
+		// Verify integrity against the lockfile pin when present (TS parity: it
+		// rejects a tarball whose digest does not match the expected one).
+		if lockfile != nil {
+			if entry, ok := lockfile.Features[id]; ok && entry.Integrity != "" && entry.Integrity != computedDigest {
+				return nil, fmt.Errorf("ERR: Feature '%s' tarball digest %s did not match lockfile integrity %s", id, computedDigest, entry.Integrity)
+			}
+		}
 		tgzPath := filepath.Join(featureDir, "feature.tgz")
 		if err := pfs.WriteFile(tgzPath, blobData); err != nil {
 			return nil, fmt.Errorf("write feature tarball: %w", err)
@@ -281,6 +291,9 @@ func processInstallFeature(
 			SourceInfo:      &features.TarballSource{TarballURI: id, UserID: id},
 			Features:        []features.Feature{feat},
 			InternalVersion: "2",
+			// Record the tarball digest so the lockfile pins it (TS parity: a
+			// direct-tarball feature is locked by its computed sha256).
+			ComputedDigest: computedDigest,
 		}, nil
 
 	default: // OCI (and legacy shorthand resolved to OCI)
@@ -779,18 +792,82 @@ func extractUserOptions(v interface{}) map[string]interface{} {
 	return nil
 }
 
-// downloadFeatureTarball fetches a Feature published as a direct HTTP(S) tarball.
-func downloadFeatureTarball(url string) ([]byte, error) {
+// downloadFeatureTarball fetches a Feature published as a direct HTTP(S) tarball,
+// matching the TS CLI's request/getRequestHeaders behavior instead of a bare GET:
+//   - sends a "devcontainer" User-Agent;
+//   - attaches Authorization: Bearer $GITHUB_TOKEN for github.com / api.github.com
+//     URLs, so private release assets resolve (the naive GET 404'd on them);
+//   - accepts any 2xx (TS treats 200-299 as success), not only 200;
+//   - warns when downloading over plaintext HTTP;
+//   - goes through the shared proxy- and custom-CA-aware transport.
+//
+// It returns the tarball bytes and their computed "sha256:<hex>" digest so the
+// caller can pin/verify integrity (TS records and checks this digest).
+func downloadFeatureTarball(ctx context.Context, logger log.Logger, tarballURL string, env map[string]string) ([]byte, string, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarballURL, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	for k, v := range featureRequestHeaders(tarballURL, env, logger) {
+		req.Header.Set(k, v)
+	}
+	if isPlainHTTPURL(tarballURL) && logger != nil {
+		logger.Write("Sending as plain HTTP request", log.LevelWarning)
+	}
+
 	client := &http.Client{Transport: httpx.NewTransport(), Timeout: 60 * time.Second}
-	resp, err := client.Get(url)
+	resp, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("download %s: HTTP %d", url, resp.StatusCode)
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, "", fmt.Errorf("download %s: HTTP %d", tarballURL, resp.StatusCode)
+	}
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 512<<20)) // 512 MiB cap
+	if err != nil {
+		return nil, "", err
+	}
+	sum := sha256.Sum256(data)
+	return data, "sha256:" + hex.EncodeToString(sum[:]), nil
+}
+
+// featureRequestHeaders mirrors the TS getRequestHeaders for a direct-tarball
+// source: always a "devcontainer" User-Agent, plus a bearer GITHUB_TOKEN when the
+// tarball is hosted on GitHub. Redirects to a signed CDN URL drop the header —
+// Go's http.Client strips Authorization on a cross-host redirect, which is the
+// correct behavior for GitHub release assets (the redirect target is presigned).
+func featureRequestHeaders(tarballURL string, env map[string]string, logger log.Logger) map[string]string {
+	headers := map[string]string{"User-Agent": "devcontainer"}
+	if isGitHubTarballURI(tarballURL) {
+		if token := env["GITHUB_TOKEN"]; token != "" {
+			if logger != nil {
+				logger.Write("Using environment GITHUB_TOKEN.", log.LevelInfo)
+			}
+			headers["Authorization"] = "Bearer " + token
+		} else if logger != nil {
+			logger.Write("No environment GITHUB_TOKEN available.", log.LevelInfo)
+		}
+	}
+	return headers
+}
+
+// isGitHubTarballURI matches the TS isGitHubUri check (raw prefix, no parsing).
+func isGitHubTarballURI(u string) bool {
+	return strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "https://api.github.com")
+}
+
+// isPlainHTTPURL reports whether the URL would travel unencrypted (TS warns on
+// this): a http:// scheme, or a localhost host regardless of scheme.
+func isPlainHTTPURL(u string) bool {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return false
 	}
-	return io.ReadAll(io.LimitReader(resp.Body, 512<<20)) // 512 MiB cap
+	return parsed.Scheme == "http" || parsed.Hostname() == "localhost"
 }
 
 // addFeatureOption returns the feature options with key=true added, preserving

internal/cli/feature_tarball_test.go

@@ -0,0 +1,134 @@
+package cli
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestIsGitHubTarballURI(t *testing.T) {
+	cases := map[string]bool{
+		"https://github.com/owner/repo/releases/download/v1/feat.tgz": true,
+		"https://api.github.com/repos/owner/repo/releases/assets/1":   true,
+		"https://raw.githubusercontent.com/owner/repo/main/feat.tgz":  false, // not github.com host
+		"https://example.com/feat.tgz":                                false,
+		"http://github.com/owner/repo/feat.tgz":                       false, // http, not https
+	}
+	for u, want := range cases {
+		if got := isGitHubTarballURI(u); got != want {
+			t.Errorf("isGitHubTarballURI(%q) = %v, want %v", u, got, want)
+		}
+	}
+}
+
+func TestIsPlainHTTPURL(t *testing.T) {
+	cases := map[string]bool{
+		"http://example.com/feat.tgz":   true,
+		"https://example.com/feat.tgz":  false,
+		"https://localhost:8080/x.tgz":  true, // localhost even over https
+		"http://localhost/x.tgz":        true,
+		"https://127.0.0.1:5000/x.tgz":  false, // only the literal "localhost" host is special
+	}
+	for u, want := range cases {
+		if got := isPlainHTTPURL(u); got != want {
+			t.Errorf("isPlainHTTPURL(%q) = %v, want %v", u, got, want)
+		}
+	}
+}
+
+func TestFeatureRequestHeaders(t *testing.T) {
+	// Always a devcontainer User-Agent.
+	h := featureRequestHeaders("https://example.com/x.tgz", nil, nil)
+	if h["User-Agent"] != "devcontainer" {
+		t.Errorf("User-Agent = %q, want devcontainer", h["User-Agent"])
+	}
+	if _, ok := h["Authorization"]; ok {
+		t.Error("non-GitHub URL must not get an Authorization header")
+	}
+
+	// GitHub URL + token → bearer auth.
+	h = featureRequestHeaders("https://github.com/o/r/releases/download/v1/f.tgz",
+		map[string]string{"GITHUB_TOKEN": "secret"}, nil)
+	if h["Authorization"] != "Bearer secret" {
+		t.Errorf("Authorization = %q, want 'Bearer secret'", h["Authorization"])
+	}
+
+	// GitHub URL without a token → no auth header (not an empty bearer).
+	h = featureRequestHeaders("https://github.com/o/r/f.tgz", map[string]string{}, nil)
+	if _, ok := h["Authorization"]; ok {
+		t.Error("GitHub URL without GITHUB_TOKEN must not set Authorization")
+	}
+
+	// Non-GitHub URL never leaks the token.
+	h = featureRequestHeaders("https://example.com/f.tgz",
+		map[string]string{"GITHUB_TOKEN": "secret"}, nil)
+	if _, ok := h["Authorization"]; ok {
+		t.Error("non-GitHub URL must not receive the GITHUB_TOKEN")
+	}
+}
+
+// TestDownloadFeatureTarball_HeadersAndDigest exercises the real HTTP path
+// against a local server: it must send the devcontainer UA + bearer token and
+// return the correct sha256 digest.
+func TestDownloadFeatureTarball_HeadersAndDigest(t *testing.T) {
+	payload := []byte("fake-tarball-bytes")
+	var gotUA, gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotUA = r.Header.Get("User-Agent")
+		gotAuth = r.Header.Get("Authorization")
+		w.WriteHeader(http.StatusOK)
+		w.Write(payload)
+	}))
+	defer srv.Close()
+
+	// The test server is not a github.com host, so no auth is expected here; use
+	// the header helper test above for the GitHub-auth branch. Verify UA + digest.
+	data, digest, err := downloadFeatureTarball(context.Background(), nil, srv.URL, map[string]string{"GITHUB_TOKEN": "secret"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != string(payload) {
+		t.Errorf("body = %q, want %q", data, payload)
+	}
+	if gotUA != "devcontainer" {
+		t.Errorf("server saw User-Agent %q, want devcontainer", gotUA)
+	}
+	if gotAuth != "" {
+		t.Errorf("non-GitHub host must not receive Authorization, got %q", gotAuth)
+	}
+	sum := sha256.Sum256(payload)
+	want := "sha256:" + hex.EncodeToString(sum[:])
+	if digest != want {
+		t.Errorf("digest = %q, want %q", digest, want)
+	}
+}
+
+// TestDownloadFeatureTarball_Non2xx verifies non-2xx responses surface an error.
+func TestDownloadFeatureTarball_Non2xx(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	_, _, err := downloadFeatureTarball(context.Background(), nil, srv.URL, nil)
+	if err == nil || !strings.Contains(err.Error(), "404") {
+		t.Errorf("want an HTTP 404 error, got %v", err)
+	}
+}
+
+// TestDownloadFeatureTarball_2xxAccepted verifies a 2xx that is not 200 (e.g. 206)
+// is accepted, matching the TS 200-299 range.
+func TestDownloadFeatureTarball_2xxAccepted(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent) // 204, still 2xx
+	}))
+	defer srv.Close()
+
+	if _, _, err := downloadFeatureTarball(context.Background(), nil, srv.URL, nil); err != nil {
+		t.Errorf("204 should be accepted as success, got %v", err)
+	}
+}