Skip to content

update mzcompose to support 2000 object search #32478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

update mzcompose to support 2000 object search #32478

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 37 additions & 29 deletions test/limits/mzcompose.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from materialize.mzcompose.services.materialized import Materialized
from materialize.mzcompose.services.mysql import MySql
from materialize.mzcompose.services.mz import Mz
from materialize.mzcompose.services.postgres import Postgres
from materialize.mzcompose.services.postgres import Postgres, PostgresMetadata
from materialize.mzcompose.services.schema_registry import SchemaRegistry
from materialize.mzcompose.services.test_certs import TestCerts
from materialize.mzcompose.services.testdrive import Testdrive
Expand Down Expand Up @@ -74,6 +74,7 @@ class Generator:
# For tests that deal with records, the number of records processed
# is usually COUNT * 1000
COUNT: int = 1000
MAX_COUNT: int = 2000

VERSION: str = "1.0.0"

Expand Down Expand Up @@ -131,7 +132,7 @@ def body(cls) -> None:
# three extra connections for mz_system, default connection, and one
# since sqlparse 0.4.4. 3 reserved superuser connections since materialize#25666
# try bumping limit a bit further since this is sometimes flaky
print(f"ALTER SYSTEM SET max_connections = {Connections.COUNT+10};")
print(f"ALTER SYSTEM SET max_connections = {Connections.COUNT + 10};")

for i in cls.all():
print(
Expand Down Expand Up @@ -328,7 +329,7 @@ def body(cls) -> None:
'$ set value-schema={"type": "record", "name": "r", "fields": [{"name": "f1", "type": "string"}]}'
)
print(
f"$ kafka-create-topic topic=kafka-partitions partitions={round(cls.COUNT/2)}"
f"$ kafka-create-topic topic=kafka-partitions partitions={round(cls.COUNT / 2)}"
)
print(
"$ kafka-ingest format=avro topic=kafka-partitions key-format=avro key-schema=${key-schema} schema=${value-schema} partition=-1"
Expand Down Expand Up @@ -668,7 +669,7 @@ def body(cls) -> None:
print("> CREATE TABLE t1 (f1 INTEGER);")
print("> INSERT INTO t1 VALUES (1);")
table_list = ", ".join(f"t1 as a{i}" for i in cls.all())
condition_list = " AND ".join(f"a{i}.f1 = a{i+1}.f1" for i in cls.no_last())
condition_list = " AND ".join(f"a{i}.f1 = a{i + 1}.f1" for i in cls.no_last())
cls.store_explain_and_run(f"SELECT * FROM {table_list} WHERE {condition_list};")
print(" ".join("1" for i in cls.all()))

Expand Down Expand Up @@ -708,7 +709,7 @@ def body(cls) -> None:
print("> CREATE TABLE t1 (f1 INTEGER);")
print("> INSERT INTO t1 VALUES (1);")
table_list = " LEFT JOIN ".join(
f"t1 as a{i} ON (a{i-1}.f1 = a{i}.f1)" for i in cls.no_first()
f"t1 as a{i} ON (a{i - 1}.f1 = a{i}.f1)" for i in cls.no_first()
)
cls.store_explain_and_run(f"SELECT * FROM t1 AS a1 LEFT JOIN {table_list};")
print(" ".join("1" for i in cls.all()))
Expand Down Expand Up @@ -845,7 +846,7 @@ def body(cls) -> None:
print("> CREATE VIEW v0 (f1) AS SELECT f1 FROM t;")

for i in cls.all():
print(f"> CREATE VIEW v{i} AS SELECT f1 + 1 AS f1 FROM v{i-1};")
print(f"> CREATE VIEW v{i} AS SELECT f1 + 1 AS f1 FROM v{i - 1};")

cls.store_explain_and_run(f"SELECT * FROM v{cls.COUNT};")
print(f"{cls.COUNT}")
Expand All @@ -871,7 +872,7 @@ def body(cls) -> None:

for i in cls.all():
print(
f"> CREATE MATERIALIZED VIEW v{i} AS SELECT f1 + 1 AS f1 FROM v{i-1};"
f"> CREATE MATERIALIZED VIEW v{i} AS SELECT f1 + 1 AS f1 FROM v{i - 1};"
)

cls.store_explain_and_run(f"SELECT * FROM v{cls.COUNT};")
Expand Down Expand Up @@ -907,7 +908,7 @@ def body(cls) -> None:
print("> CREATE TABLE t1 (f1 INTEGER);")
print("> INSERT INTO t1 VALUES " + ", ".join(f"({i})" for i in cls.all()))
cte_list = ", ".join(
f"a{i} AS (SELECT f1 + 1 AS f1 FROM a{i-1} WHERE f1 <= {i})"
f"a{i} AS (SELECT f1 + 1 AS f1 FROM a{i - 1} WHERE f1 <= {i})"
for i in cls.no_first()
)
table_list = ", ".join(f"a{i}" for i in cls.all())
Expand All @@ -927,7 +928,7 @@ def body(cls) -> None:
print("> CREATE TABLE t1 (f1 INTEGER);")
print("> INSERT INTO t1 VALUES (1)")
cte_list = ", ".join(
f"a{i} AS (SELECT a{i-1}.f1 + 0 AS f1 FROM a{i-1}, t1 WHERE a{i-1}.f1 = t1.f1)"
f"a{i} AS (SELECT a{i - 1}.f1 + 0 AS f1 FROM a{i - 1}, t1 WHERE a{i - 1}.f1 = t1.f1)"
for i in cls.no_first()
)
cls.store_explain_and_run(
Expand All @@ -952,7 +953,7 @@ def body(cls) -> None:
for i in cls.all()
)
cls.store_explain_and_run(f"SELECT * FROM {table_list};")
print(" ".join(f"{i+1}" for i in cls.all()))
print(" ".join(f"{i + 1}" for i in cls.all()))


class Lateral(Generator):
Expand All @@ -967,7 +968,7 @@ def body(cls) -> None:
print("> CREATE TABLE t1 (f1 INTEGER);")
print("> INSERT INTO t1 VALUES (1)")
table_list = ", LATERAL ".join(
f"(SELECT t1.f1 + {i-1} AS f1 FROM t1 WHERE f1 <= a{i-1}.f1) AS a{i}"
f"(SELECT t1.f1 + {i - 1} AS f1 FROM t1 WHERE f1 <= a{i - 1}.f1) AS a{i}"
for i in cls.no_first()
)
cls.store_explain_and_run(f"SELECT * FROM t1 AS a1 , LATERAL {table_list};")
Expand Down Expand Up @@ -1275,7 +1276,7 @@ def body(cls) -> None:
)
print("> CREATE DEFAULT INDEX ON v")
cls.store_explain_and_run("SELECT LENGTH(c) FROM v")
print(f"{cls.COUNT*1024}")
print(f"{cls.COUNT * 1024}")


class ArrayAgg(Generator):
Expand All @@ -1286,25 +1287,27 @@ def body(cls) -> None:
print("> SET statement_timeout='300s'")
print(
f"""> CREATE TABLE t ({
", ".join(
", ".join([
f"a{i} STRING",
f"b{i} STRING",
f"c{i} STRING",
f"d{i} STRING[]",
])
for i in cls.all()
)
});"""
", ".join(
", ".join(
[
f"a{i} STRING",
f"b{i} STRING",
f"c{i} STRING",
f"d{i} STRING[]",
]
)
Comment on lines +1291 to +1298
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

auto formatter from my editor.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bin/pyfmt is your friend.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah thanks!

for i in cls.all()
)
});"""
)
print("> INSERT INTO t DEFAULT VALUES;")
print(
f"""> CREATE MATERIALIZED VIEW v2 AS SELECT {
", ".join(
f"ARRAY_AGG(a{i} ORDER BY b1) FILTER (WHERE 's{i}' = ANY(d{i})) AS r{i}"
for i in cls.all()
)
} FROM t GROUP BY a1;"""
", ".join(
f"ARRAY_AGG(a{i} ORDER BY b1) FILTER (WHERE 's{i}' = ANY(d{i})) AS r{i}"
for i in cls.all()
)
} FROM t GROUP BY a1;"""
Comment on lines +1290 to +1310
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah sorry this is my auto formatter

)
print("> CREATE DEFAULT INDEX ON v2;")

Expand Down Expand Up @@ -1790,6 +1793,7 @@ def app_password(email: str) -> str:
],
),
Cockroach(in_memory=True),
PostgresMetadata(),
Materialized(
memory="8G",
cpu="2",
Expand All @@ -1810,7 +1814,7 @@ def app_password(email: str) -> str:
],
sanity_restart=False,
external_metadata_store=True,
metadata_store="cockroach",
metadata_store="postgres-metadata",
),
Mz(app_password=""),
]
Expand All @@ -1819,7 +1823,11 @@ def app_password(email: str) -> str:
for replica_id in range(1, MAX_REPLICAS + 1):
for node_id in range(1, MAX_NODES + 1):
SERVICES.append(
Clusterd(name=f"clusterd_{cluster_id}_{replica_id}_{node_id}", cpu="2")
Clusterd(
name=f"clusterd_{cluster_id}_{replica_id}_{node_id}",
cpu="2",
memory="4G",
)
)


Expand Down