-
Notifications
You must be signed in to change notification settings - Fork 485
test(cli): expand live e2e coverage across read-only commands #5708
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
avallete
wants to merge
5
commits into
develop
Choose a base branch
from
test/deepen-live-assertions
base: develop
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
9c23cd7
test(cli): deepen live JSON assertions for orgs/projects list
avallete 40c50f6
test(cli): add secrets list live coverage
avallete f09bdfc
test(cli): cover read-only command surface with live tests
avallete 1456b0f
test(cli): align live negatives with real error mapping
avallete 3b2049a
test(cli): demote network-bans get live to negative-only
avallete File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
66 changes: 66 additions & 0 deletions
66
apps/cli/src/legacy/commands/backups/list/list.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { | ||
| describeLive, | ||
| describeLiveProject, | ||
| requireLiveProjectRef, | ||
| runSupabaseLive, | ||
| } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 120_000; | ||
|
|
||
| // Project-scoped read-only scenario. Skipped unless SUPABASE_LIVE_PROJECT_REF is | ||
| // set — i.e. a project has been provisioned on the stack (the cli-e2e-ci runner | ||
| // does this; a control-plane-only stack, like local macOS, skips it). | ||
| // | ||
| // Backups are listed via the Management API control plane (no project DB query), | ||
| // so this runs against a freshly provisioned project regardless of data-plane | ||
| // health — a new project simply has an empty backups list. | ||
| describeLiveProject("supabase backups list (live)", () => { | ||
| test("lists backups for the project", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const ref = requireLiveProjectRef(); | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "backups", | ||
| "list", | ||
| "--project-ref", | ||
| ref, | ||
| ]); | ||
| expect(`${stdout}${stderr}`).not.toContain("Unauthorized"); | ||
| expect(exitCode).toBe(0); | ||
| }); | ||
|
|
||
| test("emits backups as machine-readable JSON", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const ref = requireLiveProjectRef(); | ||
| const { exitCode, stdout } = await runSupabaseLive([ | ||
| "backups", | ||
| "list", | ||
| "--project-ref", | ||
| ref, | ||
| "--output-format", | ||
| "json", | ||
| ]); | ||
| expect(exitCode).toBe(0); | ||
| // Payload-only JSON shaped like { backups: [...], ... }. A fresh project may | ||
| // have zero backups, but the array must always be present. | ||
| const parsed = JSON.parse(stdout) as { backups: unknown[] }; | ||
| expect(Array.isArray(parsed.backups)).toBe(true); | ||
| }); | ||
| }); | ||
|
|
||
| // Project-scoped error path needing NO provisioned project: a valid token with | ||
| // an unknown --project-ref must reach the live Management API, come back 404, | ||
| // and exit non-zero (not a crash, not "Unauthorized"). | ||
| describeLive("supabase backups list — unknown project (live)", () => { | ||
| test("fails with a 404 for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "backups", | ||
| "list", | ||
| "--project-ref", | ||
| "a".repeat(20), | ||
| ]); | ||
| const out = `${stdout}${stderr}`; | ||
| expect(exitCode).not.toBe(0); | ||
| expect(out).not.toContain("Unauthorized"); | ||
| expect(out).toContain("404"); | ||
| }); | ||
| }); |
34 changes: 34 additions & 0 deletions
34
apps/cli/src/legacy/commands/branches/get/get.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { describeLive, runSupabaseLive } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 120_000; | ||
|
|
||
| // `branches get` resolves a branch within a project, so a stable success path | ||
| // needs a project that has branching enabled and a known branch — not | ||
| // guaranteed on a freshly provisioned project (branch lifecycle coverage is | ||
| // tracked separately in CLI-1834). | ||
| // | ||
| // The portable live signal is the request path + error mapping: a valid token | ||
| // with an unknown --project-ref must reach the live Management API, come back | ||
| // 404 (the find-branch error includes the status code), and exit non-zero. | ||
| // | ||
| // A branch name is passed explicitly: omitting the optional [name] makes | ||
| // `legacyBranchesGet` prompt for a branch id, which in a non-TTY live subprocess | ||
| // (e.g. detached HEAD) fails before the API call and would not exercise the | ||
| // intended path. Runs under `describeLive` so it needs no provisioned project. | ||
| describeLive("supabase branches get — unknown project (live)", () => { | ||
| test("fails with a 404 for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "branches", | ||
| "get", | ||
| "main", // placeholder branch name to skip the non-TTY prompt | ||
| "--project-ref", | ||
| "a".repeat(20), // well-formed (20 lowercase chars) but nonexistent ref | ||
| ]); | ||
| const out = `${stdout}${stderr}`; | ||
| expect(exitCode).not.toBe(0); | ||
| expect(out).not.toContain("Unauthorized"); | ||
| expect(out).toContain("404"); | ||
| }); | ||
| }); | ||
36 changes: 36 additions & 0 deletions
36
apps/cli/src/legacy/commands/db/advisors/advisors.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { | ||
| describeLiveDb, | ||
| requireLiveDbUrl, | ||
| runSupabaseLive, | ||
| } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 180_000; | ||
|
|
||
| // Data-plane scenario: `db advisors` runs lint queries against the project | ||
| // Postgres via --db-url, so it is gated by `describeLiveDb` — it runs only when | ||
| // SUPABASE_LIVE_DB_URL is set (the cli-e2e-ci runner resolves the provisioned | ||
| // project's pooler URL). Skipped otherwise. | ||
| describeLiveDb("supabase db advisors (live)", () => { | ||
| test("emits advisor results as machine-readable JSON", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const dbUrl = requireLiveDbUrl(); | ||
| // `--fail-on none` keeps the exit code 0 regardless of which advisories the | ||
| // project happens to have, so the test asserts the command path, not the | ||
| // project's current lint state. | ||
| const { exitCode, stdout } = await runSupabaseLive([ | ||
| "db", | ||
| "advisors", | ||
| "--db-url", | ||
| dbUrl, | ||
| "--fail-on", | ||
| "none", | ||
| "--output-format", | ||
| "json", | ||
| ]); | ||
| expect(exitCode).toBe(0); | ||
| // Payload-only JSON shaped like { results: [...] }. | ||
| const parsed = JSON.parse(stdout) as { results: unknown[] }; | ||
| expect(Array.isArray(parsed.results)).toBe(true); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { | ||
| describeLiveDb, | ||
| requireLiveDbUrl, | ||
| runSupabaseLive, | ||
| } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 180_000; | ||
|
|
||
| // Data-plane scenario: `db dump` connects to the project Postgres directly via | ||
| // --db-url (not the Management API), so it is gated by `describeLiveDb` — it | ||
| // runs only when SUPABASE_LIVE_DB_URL is set (the cli-e2e-ci runner resolves the | ||
| // provisioned project's pooler URL). Skipped otherwise. | ||
| describeLiveDb("supabase db dump (live)", () => { | ||
| test("dumps the project schema to stdout", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const dbUrl = requireLiveDbUrl(); | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive(["db", "dump", "--db-url", dbUrl]); | ||
| expect(stderr).not.toContain("Unauthorized"); | ||
| expect(exitCode).toBe(0); | ||
| // A real pg_dump of a Supabase project emits SQL DDL to stdout; assert it is | ||
| // non-empty rather than pinning an exact header that varies by pg version. | ||
| expect(stdout.trim().length).toBeGreaterThan(0); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { describeLive, runSupabaseLive } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 120_000; | ||
|
|
||
| // `domains get` reads the custom-hostname config, which the Management API only | ||
| // returns once a custom hostname has been configured — a freshly provisioned | ||
| // project legitimately has none, so there is no stable success path to assert. | ||
| // | ||
| // The valuable live signal is the request path + error mapping: a valid token | ||
| // with an unknown --project-ref must reach the live Management API, come back | ||
| // 404, and exit non-zero (not a crash, not "Unauthorized"). Runs under | ||
| // `describeLive` so it needs no provisioned project. | ||
| describeLive("supabase domains get — unknown project (live)", () => { | ||
| test("fails with a 404 for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "domains", | ||
| "get", | ||
| "--project-ref", | ||
| "a".repeat(20), // well-formed (20 lowercase chars) but nonexistent ref | ||
| ]); | ||
| const out = `${stdout}${stderr}`; | ||
| expect(exitCode).not.toBe(0); | ||
| expect(out).not.toContain("Unauthorized"); | ||
| expect(out).toContain("404"); | ||
| }); | ||
| }); |
30 changes: 30 additions & 0 deletions
30
apps/cli/src/legacy/commands/migration/list/list.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { | ||
| describeLiveDb, | ||
| requireLiveDbUrl, | ||
| runSupabaseLive, | ||
| } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 180_000; | ||
|
|
||
| // Data-plane scenario: `migration list` reads the remote migration history table | ||
| // over a direct Postgres connection via --db-url (not the Management API), so it | ||
| // is gated by `describeLiveDb` — it runs only when SUPABASE_LIVE_DB_URL is set | ||
| // (the cli-e2e-ci runner resolves the provisioned project's pooler URL). Skipped | ||
| // otherwise. | ||
| describeLiveDb("supabase migration list (live)", () => { | ||
| test("lists remote migrations for the project", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const dbUrl = requireLiveDbUrl(); | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "migration", | ||
| "list", | ||
| "--db-url", | ||
| dbUrl, | ||
| ]); | ||
| expect(`${stdout}${stderr}`).not.toContain("Unauthorized"); | ||
| // A freshly provisioned project may have no applied migrations; the command | ||
| // still exits 0 and prints the (possibly empty) history table. | ||
| expect(exitCode).toBe(0); | ||
| }); | ||
| }); |
29 changes: 29 additions & 0 deletions
29
apps/cli/src/legacy/commands/network-bans/get/get.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { describeLive, runSupabaseLive } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 120_000; | ||
|
|
||
| // `network-bans get` retrieves bans via a dedicated Management API endpoint that | ||
| // supabox returns a non-200 for on a freshly provisioned project (verified | ||
| // against the live stack: the request reaches the API — not Unauthorized — but | ||
| // exits non-zero), so there is no stable success path here. | ||
| // | ||
| // The portable live signal is the unknown-project path: a valid token with an | ||
| // unknown --project-ref must reach the live Management API, come back 404 (the | ||
| // status mapper includes the code), and exit non-zero. Runs under `describeLive` | ||
| // so it needs no provisioned project. | ||
| describeLive("supabase network-bans get — unknown project (live)", () => { | ||
| test("fails with a 404 for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "network-bans", | ||
| "get", | ||
| "--project-ref", | ||
| "a".repeat(20), // well-formed (20 lowercase chars) but nonexistent ref | ||
| ]); | ||
| const out = `${stdout}${stderr}`; | ||
| expect(exitCode).not.toBe(0); | ||
| expect(out).not.toContain("Unauthorized"); | ||
| expect(out).toContain("404"); | ||
| }); | ||
| }); |
68 changes: 68 additions & 0 deletions
68
apps/cli/src/legacy/commands/network-restrictions/get/get.live.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| import { expect, test } from "vitest"; | ||
|
|
||
| import { | ||
| describeLive, | ||
| describeLiveProject, | ||
| requireLiveProjectRef, | ||
| runSupabaseLive, | ||
| } from "../../../../../tests/helpers/live.ts"; | ||
|
|
||
| const LIVE_TIMEOUT_MS = 120_000; | ||
|
|
||
| // Project-scoped read-only scenario. Skipped unless SUPABASE_LIVE_PROJECT_REF is | ||
| // set (the cli-e2e-ci runner provisions a project; a control-plane-only stack | ||
| // skips it). Reads the project's network restrictions config via the Management | ||
| // API control plane — every project has a config object. | ||
| describeLiveProject("supabase network-restrictions get (live)", () => { | ||
| test("gets network restrictions for the project", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const ref = requireLiveProjectRef(); | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "network-restrictions", | ||
| "get", | ||
| "--project-ref", | ||
| ref, | ||
| ]); | ||
| expect(`${stdout}${stderr}`).not.toContain("Unauthorized"); | ||
| expect(exitCode).toBe(0); | ||
| }); | ||
|
|
||
| test( | ||
| "emits network restrictions as machine-readable JSON", | ||
| { timeout: LIVE_TIMEOUT_MS }, | ||
| async () => { | ||
| const ref = requireLiveProjectRef(); | ||
| const { exitCode, stdout } = await runSupabaseLive([ | ||
| "network-restrictions", | ||
| "get", | ||
| "--project-ref", | ||
| ref, | ||
| "--output-format", | ||
| "json", | ||
| ]); | ||
| expect(exitCode).toBe(0); | ||
| // Payload-only JSON: the restrictions config is a single object, not an array. | ||
| const parsed: unknown = JSON.parse(stdout); | ||
| expect(typeof parsed).toBe("object"); | ||
| expect(parsed).not.toBeNull(); | ||
| expect(Array.isArray(parsed)).toBe(false); | ||
| }, | ||
| ); | ||
| }); | ||
|
|
||
| // Error path needing NO provisioned project: a valid token with an unknown | ||
| // --project-ref must reach the live Management API and exit non-zero. The | ||
| // handler formats non-200s as "failed to retrieve network restrictions; | ||
| // received: <body>" without the HTTP status code, so we assert behavior rather | ||
| // than a literal "404". | ||
| describeLive("supabase network-restrictions get — unknown project (live)", () => { | ||
| test("fails cleanly for an unknown project ref", { timeout: LIVE_TIMEOUT_MS }, async () => { | ||
| const { exitCode, stdout, stderr } = await runSupabaseLive([ | ||
| "network-restrictions", | ||
| "get", | ||
| "--project-ref", | ||
| "a".repeat(20), | ||
| ]); | ||
| expect(exitCode).not.toBe(0); | ||
| expect(`${stdout}${stderr}`).not.toContain("Unauthorized"); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.