test(cli): supabox-backed live test suite + cli-e2e-ci dispatch

.github/workflows/dispatch-cli-e2e-ci.yml

@@ -0,0 +1,57 @@
+name: Dispatch cli-e2e-ci
+
+# Asks the supabase/cli-e2e-ci harness to run the cli `test:live` suite against
+# a full supabox stack, built from THIS PR's head commit (CLI-1825 / CLI-1831).
+#
+# This is distinct from `live-e2e.yml`, which runs the cli-e2e package against
+# real staging (api.supabase.green). Here the suite runs against a local supabox
+# stack stood up inside the private cli-e2e-ci repo; we only fire the trigger and
+# pass our head SHA — cli-e2e-ci checks that SHA out into its `cli` submodule.
+#
+# Opt-in by label to keep the expensive full-stack run off every PR: add the
+# `run-live-e2e-ci` label (re-dispatches on each subsequent push while labeled).
+# cli-e2e-ci reports a `cli-e2e-ci / live` commit status back onto the head SHA.
+#
+# Fork PRs cannot dispatch (no access to the App secret); run cli-e2e-ci's own
+# workflow_dispatch with `cli_ref` for those.
+on:
+  pull_request:
+    types: [labeled, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  dispatch:
+    # Same-repo PRs only: fork PRs don't receive secrets (GH_APP_PRIVATE_KEY), so
+    # the App-token step would fail and leave a red check. Skip them cleanly —
+    # fork PRs use cli-e2e-ci's workflow_dispatch with `cli_ref` instead.
+    if: >-
+      contains(github.event.pull_request.labels.*.name, 'run-live-e2e-ci')
+      && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      # App token scoped to cli-e2e-ci with contents:write — the
+      # repository_dispatch REST endpoint requires write on the target repo.
+      - name: Create GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          client-id: ${{ vars.GH_APP_CLIENT_ID }}
+          private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
+          owner: supabase
+          repositories: cli-e2e-ci
+          permission-contents: write
+
+      - name: Dispatch live run to cli-e2e-ci
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          CLI_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          echo "Dispatching cli-e2e-ci live run for PR #${PR_NUMBER} @ ${CLI_SHA}"
+          # Build the nested client_payload with jq — `gh api -f` sends a flat
+          # body and would not nest `client_payload.*` correctly.
+          jq -n --arg sha "$CLI_SHA" --argjson pr "$PR_NUMBER" \
+            '{event_type: "cli-pr", client_payload: {cli_sha: $sha, pr_number: $pr}}' \
+            | gh api -X POST repos/supabase/cli-e2e-ci/dispatches --input -

apps/cli/package.json

@@ -107,7 +107,8 @@
     "entry": [
       "src/shared/cli/bin.ts",
       "src/**/*.test.ts",
-      "src/**/*.e2e.test.ts"
+      "src/**/*.e2e.test.ts",
+      "src/**/*.live.test.ts"
     ],
     "ignore": [
       "scripts/*.ts",

apps/cli/src/legacy/commands/branches/list/list.live.test.ts

@@ -0,0 +1,30 @@
+import { expect, test } from "vitest";
+
+import {
+  describeLiveProject,
+  requireLiveProjectRef,
+  runSupabaseLive,
+} from "../../../../../tests/helpers/live.ts";
+
+const LIVE_TIMEOUT_MS = 120_000;
+
+// Project-scoped read-only scenario. Skipped unless SUPABASE_LIVE_PROJECT_REF is
+// set — i.e. a project has been provisioned on the stack (the cli-e2e-ci runner
+// does this; a control-plane-only stack, like local macOS, skips it).
+//
+// Entry point for the branching lifecycle tracked in CLI-1834
+// (create / switch / delete) — extend here once a provisioned project is
+// available on the full stack.
+describeLiveProject("supabase branches list (live)", () => {
+  test("lists branches for the project", { timeout: LIVE_TIMEOUT_MS }, async () => {
+    const ref = requireLiveProjectRef();
+    const { exitCode, stdout, stderr } = await runSupabaseLive([
+      "branches",
+      "list",
+      "--project-ref",
+      ref,
+    ]);
+    expect(`${stdout}${stderr}`).not.toContain("Unauthorized");
+    expect(exitCode).toBe(0);
+  });
+});

apps/cli/src/legacy/commands/functions/list/list.live.test.ts

@@ -0,0 +1,52 @@
+import { expect, test } from "vitest";
+
+import {
+  describeLive,
+  describeLiveProject,
+  requireLiveProjectRef,
+  runSupabaseLive,
+} from "../../../../../tests/helpers/live.ts";
+
+const LIVE_TIMEOUT_MS = 120_000;
+
+// Project-scoped read-only scenario. Skipped unless SUPABASE_LIVE_PROJECT_REF is
+// set — i.e. a project has been provisioned on the stack (the cli-e2e-ci runner
+// does this; a control-plane-only stack, like local macOS, skips it).
+//
+// This is the entry point for the broader edge-functions coverage tracked in
+// CLI-1834 (deploy + invoke over :443 / {ref}.supabase.red), which needs the
+// project's gateway reachable from the host — author those here as they become
+// runnable on the full stack.
+describeLiveProject("supabase functions list (live)", () => {
+  test("lists edge functions for the project", { timeout: LIVE_TIMEOUT_MS }, async () => {
+    const ref = requireLiveProjectRef();
+    const { exitCode, stdout, stderr } = await runSupabaseLive([
+      "functions",
+      "list",
+      "--project-ref",
+      ref,
+    ]);
+    expect(`${stdout}${stderr}`).not.toContain("Unauthorized");
+    expect(exitCode).toBe(0);
+  });
+});
+
+// Project-scoped error path that needs NO provisioned project: a valid token
+// with an unknown `--project-ref` must reach the live Management API, come back
+// 404, and surface as a non-zero exit (not a crash, not "Unauthorized"). This
+// exercises the `--project-ref` request path + error mapping on a control-plane-
+// only stack, so it runs under `describeLive`, not `describeLiveProject`.
+describeLive("supabase functions list — unknown project (live)", () => {
+  test("fails with a 404 for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => {
+    const { exitCode, stdout, stderr } = await runSupabaseLive([
+      "functions",
+      "list",
+      "--project-ref",
+      "a".repeat(20), // well-formed (20 lowercase chars) but nonexistent ref
+    ]);
+    const out = `${stdout}${stderr}`;
+    expect(exitCode).not.toBe(0);
+    expect(out).not.toContain("Unauthorized");
+    expect(out).toContain("404");
+  });
+});

apps/cli/src/legacy/commands/orgs/list/list.live.test.ts

@@ -0,0 +1,52 @@
+import { expect, test } from "vitest";
+import { describeLive, runSupabaseLive } from "../../../../../tests/helpers/live.ts";
+
+const LIVE_TIMEOUT_MS = 60_000;
+
+// Harness smoke for the `live` Vitest project: the canonical example of a live
+// test. It exercises the full path — built binary → SUPABASE_PROFILE resolution
+// → authenticated Management API request against the running platform — with a
+// read-only call, so it is safe to run repeatedly and creates no resources.
+//
+// Gated by `describeLive`: skipped unless SUPABASE_ACCESS_TOKEN is set (the
+// cli-e2e-ci runner provides supabox's seeded PAT). Broader lifecycle scenarios
+// (projects, functions, branching, db, storage) build on this same harness.
+describeLive("supabase orgs list (live)", () => {
+  test(
+    "lists organizations for the authenticated token",
+    { timeout: LIVE_TIMEOUT_MS },
+    async () => {
+      const { exitCode, stdout, stderr } = await runSupabaseLive(["orgs", "list"]);
+      expect(`${stdout}${stderr}`).not.toContain("Unauthorized");
+      expect(exitCode).toBe(0);
+    },
+  );
+
+  test(
+    "emits machine-readable JSON with --output-format json",
+    { timeout: LIVE_TIMEOUT_MS },
+    async () => {
+      const { exitCode, stdout } = await runSupabaseLive([
+        "orgs",
+        "list",
+        "--output-format",
+        "json",
+      ]);
+      expect(exitCode).toBe(0);
+      // stdout must be payload-only valid JSON in json mode (no spinner/log noise).
+      expect(() => JSON.parse(stdout)).not.toThrow();
+    },
+  );
+
+  // Negative path: a bad token must round-trip to the real Management API, come
+  // back 401, and surface as a non-zero exit with the upstream "Unauthorized"
+  // message — i.e. the cli's auth + error mapping work against the live stack,
+  // not just the golden path. Overrides only the token (profile stays set).
+  test("fails with Unauthorized for an invalid token", { timeout: LIVE_TIMEOUT_MS }, async () => {
+    const { exitCode, stdout, stderr } = await runSupabaseLive(["orgs", "list"], {
+      env: { SUPABASE_ACCESS_TOKEN: `sbp_${"0".repeat(40)}` },
+    });
+    expect(exitCode).not.toBe(0);
+    expect(`${stdout}${stderr}`).toContain("Unauthorized");
+  });
+});

apps/cli/src/legacy/commands/projects/list/list.live.test.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "vitest";
+
+import { describeLive, runSupabaseLive } from "../../../../../tests/helpers/live.ts";
+
+const LIVE_TIMEOUT_MS = 60_000;
+
+// Account-level read-only live scenario, alongside `orgs list`. Lists every
+// project the authenticated token can access — no project ref required, so it
+// runs against just the control plane (no provisioned project instance needed).
+// Safe to run repeatedly; creates nothing.
+describeLive("supabase projects list (live)", () => {
+  test("lists projects for the authenticated token", { timeout: LIVE_TIMEOUT_MS }, async () => {
+    const { exitCode, stdout, stderr } = await runSupabaseLive(["projects", "list"]);
+    expect(`${stdout}${stderr}`).not.toContain("Unauthorized");
+    expect(exitCode).toBe(0);
+  });
+
+  test(
+    "emits machine-readable JSON with --output-format json",
+    { timeout: LIVE_TIMEOUT_MS },
+    async () => {
+      const { exitCode, stdout } = await runSupabaseLive([
+        "projects",
+        "list",
+        "--output-format",
+        "json",
+      ]);
+      expect(exitCode).toBe(0);
+      // stdout must be payload-only valid JSON in json mode (no spinner/log noise).
+      expect(() => JSON.parse(stdout)).not.toThrow();
+    },
+  );
+});

apps/cli/tests/helpers/live-env.ts

@@ -0,0 +1,73 @@
+/**
+ * Environment-only helpers for the `live` Vitest project, with **no Vitest test
+ * APIs imported**. Vitest evaluates `globalSetup` (live-global-setup.ts) in a
+ * separate context before the test workers, where importing `describe`/`test`
+ * is not valid — so the global setup imports the env helpers from here, while
+ * the test-facing pieces (`describeLive`, `runSupabaseLive`, …) live in
+ * `live.ts` and re-export these.
+ *
+ * Environment contract (provided by the cli-e2e-ci runner):
+ * - `SUPABASE_ACCESS_TOKEN` — required; the platform PAT (supabox seeds a
+ *   deterministic `sbp_…` token into its mgmt-api database).
+ * - `SUPABASE_PROFILE` — selects the API base URL; defaults to `supabase-local`
+ *   (→ `http://localhost:8080`, `project_host: supabase.red`). Note the cli does
+ *   NOT honor `SUPABASE_API_URL` (Go parity) — the profile is the override.
+ * - `SUPABASE_LIVE_API_URL` — base URL the readiness check probes; defaults to
+ *   `http://localhost:8080`.
+ * - `SUPABASE_LIVE_PROJECT_REF` — a provisioned project; gates project-scoped
+ *   suites (functions, branches, db, storage).
+ * - `NODE_EXTRA_CA_CERTS` — trusts the supabox CA for `*.supabase.red` TLS;
+ *   inherited by the subprocess via the parent environment.
+ */
+
+/** Default profile for the host runner: api_url → localhost:8080, project_host → supabase.red. */
+export const LIVE_DEFAULT_PROFILE = "supabase-local";
+
+/**
+ * Default subprocess exit timeout for live runs. `runSupabase` otherwise caps at
+ * 60s, which would kill a slow-but-valid supabox call before the live tests'
+ * own (60–120s+) timeouts fire. Generous, but under the `live` project's 300s
+ * cap so the per-test timeout stays the real gate. Callers may override.
+ */
+export const LIVE_EXIT_TIMEOUT_MS = 240_000;
+
+/** Management API base URL probed by the live readiness check. */
+export function liveApiBaseUrl(): string {
+  return process.env["SUPABASE_LIVE_API_URL"] ?? "http://localhost:8080";
+}
+
+/**
+ * True when the environment carries a platform access token, i.e. the live
+ * suite is expected to run. Used to gate `describeLive` so live tests are inert
+ * in the default test loop.
+ */
+export function isLiveConfigured(): boolean {
+  return Boolean(process.env["SUPABASE_ACCESS_TOKEN"]);
+}
+
+/**
+ * Project ref for project-scoped live scenarios (functions, branches, db,
+ * storage, …). The cli-e2e-ci runner sets this once a project has been
+ * provisioned on the stack; absent → those suites skip. Returns `undefined`
+ * when unset so callers can branch; use `requireLiveProjectRef` inside a
+ * `describeLiveProject` block where presence is already guaranteed.
+ */
+export function liveProjectRef(): string | undefined {
+  return process.env["SUPABASE_LIVE_PROJECT_REF"];
+}
+
+/**
+ * The live project ref, or a thrown error if unset. Safe to call inside a
+ * `describeLiveProject` block (the gate guarantees it is present) and gives a
+ * typed `string` without a non-null assertion.
+ */
+export function requireLiveProjectRef(): string {
+  const ref = liveProjectRef();
+  if (!ref) {
+    throw new Error(
+      "SUPABASE_LIVE_PROJECT_REF must be set for project-scoped live tests " +
+        "(the cli-e2e-ci runner sets it after provisioning a project).",
+    );
+  }
+  return ref;
+}

apps/cli/tests/helpers/live.ts

@@ -0,0 +1,66 @@
+import { describe } from "vitest";
+
+import { runSupabase } from "./cli.ts";
+import {
+  isLiveConfigured,
+  LIVE_DEFAULT_PROFILE,
+  LIVE_EXIT_TIMEOUT_MS,
+  liveProjectRef,
+} from "./live-env.ts";
+
+/**
+ * Test-facing helpers for the `live` Vitest project (`*.live.test.ts`):
+ * black-box CLI subprocess tests that run against a *real* Supabase platform —
+ * in CI a local supabox stack (see the `supabase/cli-e2e-ci` harness).
+ *
+ * This module imports Vitest test APIs (`describe`), so it must NOT be imported
+ * from `globalSetup` (Vitest evaluates that in a different context). The
+ * env-only helpers live in `./live-env.ts`; `globalSetup` imports from there.
+ * They are re-exported below so test files have a single import site.
+ */
+
+// Re-export the env-only helpers so `*.live.test.ts` files import everything
+// from `helpers/live.ts`.
+export {
+  isLiveConfigured,
+  LIVE_DEFAULT_PROFILE,
+  LIVE_EXIT_TIMEOUT_MS,
+  liveApiBaseUrl,
+  liveProjectRef,
+  requireLiveProjectRef,
+} from "./live-env.ts";
+
+/**
+ * `describe` that runs only when the live environment is configured. Use this
+ * for every live suite so the file is inert (skipped, not failed) outside the
+ * cli-e2e-ci runner.
+ */
+export const describeLive = describe.skipIf(!isLiveConfigured());
+
+/**
+ * `describe` for project-scoped live suites: runs only when the live env is
+ * configured AND a project ref is available. On a control-plane-only stack
+ * (e.g. local macOS where project instances can't be built) these skip rather
+ * than fail. See `requireLiveProjectRef`.
+ */
+export const describeLiveProject = describe.skipIf(!isLiveConfigured() || !liveProjectRef());
+
+/**
+ * Spawn the built CLI against the live platform, injecting the profile so the
+ * Management API base resolves to the stack. Defaults to the `legacy` shell,
+ * which hosts the platform commands (orgs, projects, branches, functions, …).
+ */
+export function runSupabaseLive(
+  args: string[],
+  options?: Parameters<typeof runSupabase>[1],
+): ReturnType<typeof runSupabase> {
+  return runSupabase(args, {
+    entrypoint: "legacy",
+    ...options,
+    exitTimeoutMs: options?.exitTimeoutMs ?? LIVE_EXIT_TIMEOUT_MS,
+    env: {
+      SUPABASE_PROFILE: process.env["SUPABASE_PROFILE"] ?? LIVE_DEFAULT_PROFILE,
+      ...options?.env,
+    },
+  });
+}