Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions misc/python/materialize/mzbuild.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def run_and_detect_rust_incremental_build_failure(
base_env = env if env is not None else os.environ
p = subprocess.Popen(
cmd,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
Expand Down
14 changes: 7 additions & 7 deletions test/canary-environment/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,13 @@ the split clusters are new), so the `USAGE` grants are reapplied on every
granted in the testdrive block of `workflow_create`, since mz-deploy has no
cluster-grant modifier and doesn't manage those schemas.

**Iceberg sinks: AWS/S3-Tables enabled, GCS still disabled.** The five
S3-Tables model sinks and the `public_table.table_mv_iceberg_sink` testdrive
sink are active. The five `*_gcs_iceberg_sink` model sinks remain
`*.sql.disabled` (mz-deploy only reads `*.sql`, so it skips them) and the
`table_mv_gcs_iceberg_sink` testdrive sink is left out of `mzcompose.py`. To
re-enable a GCS sink, rename its `.sql.disabled` file back to `.sql` (and add
the testdrive sink back to `mzcompose.py`).
**Iceberg sinks: AWS/S3-Tables and GCS both enabled.** The five S3-Tables model
sinks, the five `*_gcs_iceberg_sink` GCS model sinks, and both the
`public_table.table_mv_iceberg_sink` (S3-Tables) and
`public_table.table_mv_gcs_iceberg_sink` (GCS) testdrive sinks are active. To
disable a GCS sink, rename its `.sql` file to `.sql.disabled` (mz-deploy only
reads `*.sql`, so it skips them) and remove the corresponding testdrive sink
from `mzcompose.py`.

## What `mz-deploy` does not manage

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-- the Business Source License, use of this software will be governed
-- by the Apache License, Version 2.0.

-- TODO: Reenable when SS-282 is fixed
CREATE SINK customer_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM qa_canary_environment.public_loadgen_sources.customer_tbl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-- the Business Source License, use of this software will be governed
-- by the Apache License, Version 2.0.

-- TODO: Reenable when SS-282 is fixed
CREATE SINK sales_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM qa_canary_environment.public_loadgen_sources.sales_tbl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-- the Business Source License, use of this software will be governed
-- by the Apache License, Version 2.0.

-- TODO: Reenable when SS-282 is fixed
CREATE SINK mysql_wmr_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM qa_canary_environment.public_mysql_cdc.mysql_wmr
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-- the Business Source License, use of this software will be governed
-- by the Apache License, Version 2.0.

-- TODO: Reenable when SS-282 is fixed
CREATE SINK pg_relationships_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM qa_canary_environment.public_pg_cdc_sources.pg_relationships
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
-- the Business Source License, use of this software will be governed
-- by the Apache License, Version 2.0.

-- TODO: Reenable when SS-282 is fixed
CREATE SINK tpch_q18_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM qa_canary_environment.public_tpch.tpch_q18
Expand Down
11 changes: 8 additions & 3 deletions test/canary-environment/mzcompose.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,9 +447,12 @@ def workflow_create(c: Composition, parser: WorkflowArgumentParser) -> None:
MODE APPEND
WITH (COMMIT INTERVAL = '60s')

# The GCS Iceberg sink (table_mv_gcs_iceberg_sink) stays disabled, as
# do the *_gcs_iceberg_sink.sql.disabled model sinks. Only the
# AWS/S3-Tables Iceberg sinks are enabled.
> CREATE SINK IF NOT EXISTS public_table.table_mv_gcs_iceberg_sink
IN CLUSTER qa_canary_environment_sinks
FROM public_table.table_mv
INTO ICEBERG CATALOG CONNECTION public.qa_canary_gcs_iceberg_catalog (NAMESPACE = 'qa_canary_environment', TABLE = 'table_mv')
MODE APPEND
WITH (COMMIT INTERVAL = '60s')

# Seed data for the loadgen product/category tables (created by `apply`).
> DELETE FROM public_loadgen_sources.product_category
Expand Down Expand Up @@ -859,9 +862,11 @@ def workflow_mz_deploy(c: Composition, parser: WorkflowArgumentParser) -> None:
if not cli_args.mz_args:
raise ValueError("usage: ./mzcompose run mz <mz-deploy command> [args...]")
write_profiles()
write_project_toml()
mz_deploy(c, *cli_args.mz_args)


def workflow_clean(c: Composition, parser: WorkflowArgumentParser) -> None:
write_profiles()
write_project_toml()
mz_deploy(c, "clean")
Loading