Skip to content

perf/hardware summary perf #2064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alanpeixinho wants to merge 1 commit into
base: main
Choose a base branch
from
+217 −116
Open

perf/hardware summary perf #2064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 101 additions & 108 deletions backend/kernelCI_app/queries/hardware.py
View file
Open in desktop

@felipebergamin felipebergamin Aug 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is probably intentional, but just checking. The builds are still joining tests, so they are not really being optimized like boots/tests, right?

@alanpeixinho alanpeixinho Aug 28, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. I am including this on builds as well.

@felipebergamin felipebergamin Aug 27, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: For this kind of change, it would be nice to have a before-and-after comparison to highlight the improvement. Just a nit though — no need to do it now.

@alanpeixinho alanpeixinho Aug 28, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. I am including a small local benchmark on the PR.

Original file line number Diff line number Diff line change
Expand Up @@ -12,44 +12,71 @@
from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree


def _get_hardware_tree_heads_clause(*, id_only: bool) -> str:
"""Returns the tree_heads for the hardware queries,
where the checkout is not filtered by origin.

This is done because tests from a origin can be
related to checkouts from another origin."""
if id_only is True:
fields = "C.id"
else:
fields = """C.id,
C.origin,
def _get_hardware_tree_heads_with_ctes() -> str:
"""Latest checkouts in the window, then probe tests via builds.build_id.

Checkout origin is not filtered: tests from one origin can belong to
checkouts from another.
"""
return """
tree_heads AS MATERIALIZED (
SELECT DISTINCT
ON (
C.tree_name,
C.start_time,
C.git_repository_branch,
C.git_repository_url,
C.git_commit_name,
C.git_commit_hash,
C.git_commit_tags"""

return f"""SELECT DISTINCT
ON (
C.tree_name,
C.git_repository_branch,
C.git_repository_url,
C.origin
)
{fields}
FROM
checkouts C
WHERE
C.start_time >= %(start_date)s
AND C.start_time <= %(end_date)s
ORDER BY
C.tree_name ASC,
C.git_repository_branch ASC,
C.git_repository_url ASC,
C.origin ASC,
C.start_time DESC"""
C.origin
)
C.id,
C.origin,
C.tree_name,
C.start_time,
C.git_repository_branch,
C.git_repository_url,
C.git_commit_name,
C.git_commit_hash,
C.git_commit_tags
FROM
checkouts C
WHERE
C.start_time >= %(start_date)s
AND C.start_time <= %(end_date)s
ORDER BY
C.tree_name ASC,
C.git_repository_branch ASC,
C.git_repository_url ASC,
C.origin ASC,
C.start_time DESC
),
period_builds AS MATERIALIZED (
SELECT
builds.id,
builds.checkout_id
FROM
tree_heads TH
INNER JOIN builds ON builds.checkout_id = TH.id
),
hardware_checkouts AS (
SELECT DISTINCT
period_builds.checkout_id AS id
FROM
period_builds
INNER JOIN LATERAL (
SELECT
1
FROM
tests
WHERE
tests.build_id = period_builds.id
AND tests.origin = %(origin)s
AND (
tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
OR tests.environment_misc ->> 'platform' = %(hardware)s
)
LIMIT
1
) t ON true
)"""


# TODO: unify with get_tree_listing_count_clause
Expand Down Expand Up @@ -455,7 +482,6 @@ def get_hardware_details_summary(
start_datetime: datetime,
end_datetime: datetime,
):

if builds_duration is None:
builds_duration = (None, None)
if boots_duration is None:
Expand All @@ -481,12 +507,30 @@ def get_hardware_details_summary(
if query_rows is not None:
return query_rows

builds_duration_clause = get_build_duration_clause(builds_duration)
boots_tests_duration_clause = get_boot_test_duration_clause(
boots_duration, tests_duration
)

query = """
WITH platform_tests AS (
SELECT
id, build_id, status, path, duration, misc,
environment_misc, environment_compatible, lab_id
FROM
tests
WHERE
tests.origin = %(origin)s
AND tests.start_time >= %(start_date)s
AND tests.start_time <= %(end_date)s
AND tests.environment_compatible @> ARRAY[%(platform)s]::TEXT[]
UNION
SELECT
id, build_id, status, path, duration, misc,
environment_misc, environment_compatible, lab_id
FROM
tests
WHERE
tests.origin = %(origin)s
AND tests.start_time >= %(start_date)s
AND tests.start_time <= %(end_date)s
AND tests.environment_misc ->> 'platform' = %(platform)s
)
(SELECT
COUNT(DISTINCT builds.id) AS count,
checkouts.origin,
Expand All @@ -500,7 +544,6 @@ def get_hardware_details_summary(
COALESCE(bl.name, builds.misc->>'lab') AS lab,
tests.environment_misc->>'platform' AS platform,
tests.environment_compatible,
checkouts.origin,
checkouts.tree_name,
checkouts.git_repository_url,
checkouts.git_commit_tags,
Expand All @@ -512,24 +555,20 @@ def get_hardware_details_summary(
false AS is_boot
FROM
builds
INNER JOIN tests ON
INNER JOIN platform_tests tests ON
tests.build_id = builds.id
INNER JOIN checkouts ON
builds.checkout_id = checkouts.id
LEFT JOIN labs bl ON builds.lab_id = bl.id
LEFT OUTER JOIN incidents ON
builds.id = incidents.build_id
WHERE
(
builds.config_name IS NOT NULL
AND builds.id not like 'maestro:dummy_%%'
AND (tests.environment_compatible @> ARRAY[%(platform)s]::TEXT[]
OR tests.environment_misc ->> 'platform' = %(platform)s)
)
builds.config_name IS NOT NULL
AND builds.id not like 'maestro:dummy_%%'
AND builds.origin = %(origin)s
AND builds.start_time >= %(start_date)s
AND builds.start_time <= %(end_date)s
AND (checkouts.git_commit_hash = ANY(%(commits)s)) {0}
AND (checkouts.git_commit_hash = ANY(%(commits)s)) {builds_duration_clause}
GROUP BY checkouts.id, builds.status, tests.environment_compatible, compiler_arch,
builds.config_name, lab, platform, is_boot)
UNION ALL
Expand All @@ -546,7 +585,6 @@ def get_hardware_details_summary(
COALESCE(tl.name, tests.misc->>'runtime') AS lab,
tests.environment_misc->>'platform' AS platform,
tests.environment_compatible,
checkouts.origin,
checkouts.tree_name,
checkouts.git_repository_url,
checkouts.git_commit_tags,
Expand All @@ -558,27 +596,22 @@ def get_hardware_details_summary(
(tests.path like 'boot.%%' or tests.path = 'boot') AS is_boot
FROM
builds
INNER JOIN tests ON
INNER JOIN platform_tests tests ON
tests.build_id = builds.id
INNER JOIN checkouts ON
builds.checkout_id = checkouts.id
LEFT JOIN labs tl ON tests.lab_id = tl.id
LEFT OUTER JOIN incidents ON
tests.id = incidents.test_id
WHERE
(
(tests.environment_compatible @> ARRAY[%(platform)s]::TEXT[]
OR tests.environment_misc ->> 'platform' = %(platform)s)
)
AND tests.origin = %(origin)s
AND tests.start_time >= %(start_date)s
AND tests.start_time <= %(end_date)s
AND (checkouts.git_commit_hash = ANY(%(commits)s)) {1}
(checkouts.git_commit_hash = ANY(%(commits)s)) {boots_tests_duration_clause}
GROUP BY checkouts.id, tests.status, tests.environment_compatible, compiler_arch,
builds.config_name, lab, platform, is_boot);
""".format(
builds_duration_clause,
boots_tests_duration_clause,
builds_duration_clause=get_build_duration_clause(builds_duration),
boots_tests_duration_clause=get_boot_test_duration_clause(
boots_duration, tests_duration
),
)

build_duration_min, build_duration_max = builds_duration
Expand Down Expand Up @@ -796,10 +829,6 @@ def get_hardware_trees_head_commits(
start_datetime: datetime,
end_datetime: datetime,
) -> list[tuple[str, str]]:

# similar to the get_hardware_trees_data, except we limit the information
# being brought to the commit hash

cache_key = "hardwareTreesHeadCommits"

cache_params = {
Expand All @@ -814,16 +843,9 @@ def get_hardware_trees_head_commits(
if trees:
return trees

tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)

# We need a subquery because if we filter by any hardware, it will get the
# last head that has that hardware, but not the real head of the trees
query = f"""
WITH
-- Selects the data of the latest checkout of all trees in the given period
tree_heads AS (
{tree_head_clause}
)
{_get_hardware_tree_heads_with_ctes()}
SELECT DISTINCT
ON (
TH.tree_name,
Expand All @@ -833,19 +855,8 @@ def get_hardware_trees_head_commits(
) TH.tree_name,
TH.git_commit_hash
FROM
tests
INNER JOIN builds ON tests.build_id = builds.id
INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
WHERE
(
(
tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
OR tests.environment_misc ->> 'platform' = %(hardware)s
)
AND tests.origin = %(origin)s
AND TH.start_time >= %(start_date)s
AND TH.start_time <= %(end_date)s
)
tree_heads TH
INNER JOIN hardware_checkouts HC ON HC.id = TH.id
ORDER BY
TH.tree_name ASC,
TH.git_repository_branch ASC,
Expand Down Expand Up @@ -891,17 +902,10 @@ def get_hardware_trees_data(

trees: list[Tree] = get_query_cache(cache_key, params)

tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)

if not trees:
# We need a subquery because if we filter by any hardware, it will get the
# last head that has that hardware, but not the real head of the trees
query = f"""
WITH
-- Selects the data of the latest checkout of all trees in the given period
tree_heads AS (
{tree_head_clause}
)
{_get_hardware_tree_heads_with_ctes()}
SELECT DISTINCT
ON (
TH.tree_name,
Expand All @@ -916,19 +920,8 @@ def get_hardware_trees_data(
TH.git_commit_hash,
TH.git_commit_tags
FROM
tests
INNER JOIN builds ON tests.build_id = builds.id
INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
WHERE
(
(
tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
OR tests.environment_misc ->> 'platform' = %(hardware)s
)
AND tests.origin = %(origin)s
AND TH.start_time >= %(start_date)s
AND TH.start_time <= %(end_date)s
)
tree_heads TH
INNER JOIN hardware_checkouts HC ON HC.id = TH.id
ORDER BY
TH.tree_name ASC,
TH.git_repository_branch ASC,
Expand Down
Loading
Loading