Reapply feat/labs migration (#1968)

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -153,6 +153,7 @@ def get_build_typed(record: Dict, tree_idx: int) -> HardwareBuildHistoryItem:
         tree_name=record["build__checkout__tree_name"],
         issue_id=record["build__incidents__issue__id"],
         issue_version=record["build__incidents__issue__version"],
+        lab=record.get("build_lab") or UNKNOWN_STRING,
     )
 
 
@@ -290,8 +291,6 @@ def handle_test_history(
     full_environment_misc: bool = False,
 ) -> None:
     create_record_test_platform(record=record)
-    record_misc = sanitize_dict(record.get("misc"))
-
     environment_misc_dict = get_environment_misc_value(
         full_environment_misc=full_environment_misc,
         parsed_environment_misc=record.get("parsed_environment_misc"),
@@ -313,11 +312,7 @@ def handle_test_history(
         environment_misc=environment_misc,
         tree_name=record["build__checkout__tree_name"],
         git_repository_branch=record["build__checkout__git_repository_branch"],
-        lab=(
-            record_misc.get("runtime", UNKNOWN_STRING)
-            if record_misc
-            else UNKNOWN_STRING
-        ),
+        lab=record.get("lab") or UNKNOWN_STRING,
     )
 
     task.append(test_history_item)
@@ -380,16 +375,14 @@ def handle_test_summary(
         getattr(task.origins[origin], status) + 1,
     )
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
-    if lab:
-        if task.labs.get(lab) is None:
-            task.labs[lab] = StatusCount()
-        setattr(
-            task.labs[lab],
-            status,
-            getattr(task.labs[lab], status) + 1,
-        )
+    lab = record.get("lab") or UNKNOWN_STRING
+    if task.labs.get(lab) is None:
+        task.labs[lab] = StatusCount()
+    setattr(
+        task.labs[lab],
+        status,
+        getattr(task.labs[lab], status) + 1,
+    )
 
 
 def handle_build_history(
@@ -458,18 +451,16 @@ def handle_build_summary(
             getattr(builds_summary.origins[origin], status_key) + 1,
         )
 
-    misc = sanitize_dict(build.misc) or {}
-    lab = misc.get("lab", UNKNOWN_STRING)
-    if lab:
-        build_lab_summary = builds_summary.labs.get(lab)
-        if not build_lab_summary:
-            build_lab_summary = StatusCount()
-            builds_summary.labs[lab] = build_lab_summary
-        setattr(
-            builds_summary.labs[lab],
-            status_key,
-            getattr(builds_summary.labs[lab], status_key) + 1,
-        )
+    lab = record.get("build_lab") or UNKNOWN_STRING
+    build_lab_summary = builds_summary.labs.get(lab)
+    if not build_lab_summary:
+        build_lab_summary = StatusCount()
+        builds_summary.labs[lab] = build_lab_summary
+    setattr(
+        builds_summary.labs[lab],
+        status_key,
+        getattr(builds_summary.labs[lab], status_key) + 1,
+    )
 
     process_issue(record=record, task_issues_dict=issue_dict, issue_from="build")
 
@@ -579,8 +570,7 @@ def decide_if_is_full_record_filtered_out(
     if not is_current_tree_selected:
         return True
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
+    lab = record.get("lab") or UNKNOWN_STRING
 
     is_record_filtered_out_result = instance.filters.is_record_filtered_out(
         hardwares=record["environment_compatible"],
@@ -730,10 +720,8 @@ def process_filters(*, instance, record: Dict) -> None:
 
         instance.unfiltered_origins["build"].add(record["build__origin"])
 
-        build_misc = sanitize_dict(record.get("build__misc")) or {}
-        build_lab = build_misc.get("lab")
-        if build_lab:
-            instance.unfiltered_labs["build"].add(build_lab)
+        if record.get("build_lab"):
+            instance.unfiltered_labs["build"].add(record["build_lab"])
 
     if record["id"] is not None:
         if is_boot(record["path"]):
@@ -771,10 +759,8 @@ def process_filters(*, instance, record: Dict) -> None:
         platform_set.add(test_platform)
         origin_set.add(record["test_origin"])
 
-        test_misc = sanitize_dict(record.get("misc")) or {}
-        test_lab = test_misc.get("runtime")
-        if test_lab:
-            instance.unfiltered_labs[flag_tab].add(test_lab)
+        if record.get("lab"):
+            instance.unfiltered_labs[flag_tab].add(record["lab"])
 
 
 def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool:

backend/kernelCI_app/helpers/treeDetails.py

@@ -25,7 +25,6 @@
     create_issue_typed,
     extract_error_message,
     is_boot,
-    sanitize_dict,
 )
 
 
@@ -87,42 +86,41 @@ def get_current_row_data(
         "test_number_value": current_row[10],
         "test_misc": current_row[11],
         "test_environment_compatible": current_row[tmp_test_env_comp_key],
-        "build_id": current_row[13],
-        "build_origin": current_row[14],
-        "build_comment": current_row[15],
-        "build_start_time": current_row[16],
-        "build_duration": current_row[17],
-        "build_architecture": current_row[18],
-        "build_command": current_row[19],
-        "build_compiler": current_row[20],
-        "build_config_name": current_row[21],
-        "build_config_url": current_row[22],
-        "build_log_url": current_row[23],
-        "build_status": current_row[24],
-        "build_misc": current_row[25],
-        "checkout_id": current_row[26],
-        "checkout_git_repository_url": current_row[27],
-        "checkout_git_repository_branch": current_row[28],
-        "checkout_git_commit_tags": current_row[29],
-        "checkout_origin": current_row[30],
-        "incident_id": current_row[31],
-        "incident_test_id": current_row[32],
-        "incident_present": current_row[33],
-        "issue_id": current_row[34],
-        "issue_version": current_row[35],
-        "issue_comment": current_row[36],
-        "issue_report_url": current_row[37],
+        "test_lab": current_row[13],
+        "build_id": current_row[14],
+        "build_origin": current_row[15],
+        "build_comment": current_row[16],
+        "build_start_time": current_row[17],
+        "build_duration": current_row[18],
+        "build_architecture": current_row[19],
+        "build_command": current_row[20],
+        "build_compiler": current_row[21],
+        "build_config_name": current_row[22],
+        "build_config_url": current_row[23],
+        "build_log_url": current_row[24],
+        "build_status": current_row[25],
+        "build_misc": current_row[26],
+        "build_lab": current_row[27],
+        "checkout_id": current_row[28],
+        "checkout_git_repository_url": current_row[29],
+        "checkout_git_repository_branch": current_row[30],
+        "checkout_git_commit_tags": current_row[31],
+        "checkout_origin": current_row[32],
+        "incident_id": current_row[33],
+        "incident_test_id": current_row[34],
+        "incident_present": current_row[35],
+        "issue_id": current_row[36],
+        "issue_version": current_row[37],
+        "issue_comment": current_row[38],
+        "issue_report_url": current_row[39],
     }
 
     parsed_environment_misc = handle_misc(
         misc_value_or_default(current_row_data["test_environment_misc"])
     )
     current_row_data["test_platform"] = parsed_environment_misc.get("platform")
     current_row_data["parsed_environment_misc"] = parsed_environment_misc
-    test_misc = sanitize_dict(current_row_data["test_misc"])
-    test_runtime_lab = UNKNOWN_STRING
-    if test_misc is not None:
-        test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING)
+    test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING
 
     if current_row_data["test_status"] is None:
         current_row_data["test_status"] = NULL_STATUS
@@ -219,6 +217,7 @@ def get_build(row_data: dict) -> BuildHistoryItem:
         start_time=row_data["build_start_time"],
         git_repository_url=row_data["checkout_git_repository_url"],
         git_repository_branch=row_data["checkout_git_repository_branch"],
+        lab=row_data.get("build_lab") or UNKNOWN_STRING,
     )
 
 
@@ -495,8 +494,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False)
         instance.global_architectures.add(row_data["build_architecture"])
         instance.global_compilers.add(row_data["build_compiler"])
         instance.unfiltered_origins["build"].add(row_data["build_origin"])
-        if (build_misc := row_data["build_misc"]) is not None:
-            instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING))
+        instance.unfiltered_labs["build"].add(
+            row_data.get("build_lab") or UNKNOWN_STRING
+        )
 
         build_issue_id, build_issue_version, is_build_issue = (
             should_increment_build_issue(

backend/kernelCI_app/helpers/treeDetailsRollup.py

@@ -55,9 +55,7 @@ def process_build_filters(instance, row_data: dict) -> None:
     instance.global_architectures.add(row_data["build_architecture"])
     instance.global_compilers.add(row_data["build_compiler"])
     instance.unfiltered_origins["build"].add(row_data["build_origin"])
-
-    if (build_misc := row_data["build_misc"]) is not None:
-        instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING))
+    instance.unfiltered_labs["build"].add(row_data.get("build_lab") or UNKNOWN_STRING)
 
     build_issue_id, build_issue_version, is_build_issue = should_increment_build_issue(
         issue_id=issue_id,

backend/kernelCI_app/management/commands/generated/insert_queries.py

@@ -151,6 +151,7 @@
             "log_url",
             "log_excerpt",
             "misc",
+            "lab_id",
             "status",
         ],
         "query": """
@@ -172,10 +173,11 @@
                     log_url,
                     log_excerpt,
                     misc,
+                    lab_id,
                     status
                 )
                 VALUES (
-                    %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
+                    %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
                 )
                 ON CONFLICT (id)
                 DO UPDATE SET
@@ -193,6 +195,7 @@
                     log_url = COALESCE(builds.log_url, EXCLUDED.log_url),
                     log_excerpt = COALESCE(builds.log_excerpt, EXCLUDED.log_excerpt),
                     misc = COALESCE(builds.misc, EXCLUDED.misc),
+                    lab_id = COALESCE(builds.lab_id, EXCLUDED.lab_id),
                     status = COALESCE(builds.status, EXCLUDED.status);
             """,
     },
@@ -213,6 +216,7 @@
             "duration",
             "output_files",
             "misc",
+            "lab_id",
             "number_value",
             "environment_compatible",
             "number_prefix",
@@ -236,14 +240,15 @@
                     duration,
                     output_files,
                     misc,
+                    lab_id,
                     number_value,
                     environment_compatible,
                     number_prefix,
                     number_unit,
                     input_files
                 )
                 VALUES (
-                    %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
+                    %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
                 )
                 ON CONFLICT (id)
                 DO UPDATE SET
@@ -259,6 +264,7 @@
                     duration = COALESCE(tests.duration, EXCLUDED.duration),
                     output_files = COALESCE(tests.output_files, EXCLUDED.output_files),
                     misc = COALESCE(tests.misc, EXCLUDED.misc),
+                    lab_id = COALESCE(tests.lab_id, EXCLUDED.lab_id),
                     number_value = COALESCE(tests.number_value, EXCLUDED.number_value),
                     environment_compatible = COALESCE(tests.environment_compatible, EXCLUDED.environment_compatible),
                     number_prefix = COALESCE(tests.number_prefix, EXCLUDED.number_prefix),

backend/kernelCI_app/management/commands/helpers/aggregation_helpers.py

@@ -52,7 +52,8 @@ def convert_test(t: Tests) -> PendingTest:
         is_boot=is_boot(t.path) if t.path else False,
         path=t.path,
         start_time=t.start_time,
-        lab=misc.get("runtime"),
+        # TODO remove misc->>'runtime' fallback after lab backfill
+        lab=t._lab_name or misc.get("runtime"),
         full_status=t.status,
     )
 

backend/kernelCI_app/management/commands/helpers/kcidbng_ingester.py

@@ -105,6 +105,9 @@ def _standardize_lab_field(item: dict[str, Any], field: str) -> None:
     """
     lab = item.get("misc", {}).get(field)
     is_automatic = lab and AUTOMATIC_LABS.match(lab)
+    # Real lab for the lab_id FK, captured before the origin fallback (#1752) below
+    # overwrites misc, which would otherwise pollute the lab dimension.
+    item["_real_lab"] = None if is_automatic else lab
     if is_automatic:
         item["misc"][AUTOMATIC_LAB_FIELD] = lab
         item["misc"].pop(field, None)
@@ -193,6 +196,39 @@ def prepare_file_data(
         }
 
 
+def assign_lab_ids(builds_buf: list[Builds], tests_buf: list[Tests]) -> None:
+    """Resolve each instance's real lab name to a labs.id and set its lab_id FK.
+
+    Select-first so we only INSERT genuinely new labs (avoids burning the id
+    sequence on every flush). New labs are committed outside the fact-insert
+    transaction (autocommit).
+    """
+    objs = [*builds_buf, *tests_buf]
+    names = {name for obj in objs if (name := obj._lab_name)}
+
+    id_map: dict[str, int] = {}
+    if names:
+        with connections["default"].cursor() as cursor:
+            cursor.execute(
+                "SELECT id, name FROM labs WHERE name = ANY(%s)", [list(names)]
+            )
+            id_map = {name: lab_id for lab_id, name in cursor.fetchall()}
+
+            missing = [name for name in names if name not in id_map]
+            if missing:
+                cursor.executemany(
+                    "INSERT INTO labs (name) VALUES (%s) ON CONFLICT (name) DO NOTHING",
+                    [(name,) for name in missing],
+                )
+                cursor.execute(
+                    "SELECT id, name FROM labs WHERE name = ANY(%s)", [missing]
+                )
+                id_map.update({name: lab_id for lab_id, name in cursor.fetchall()})
+
+    for obj in objs:
+        obj.lab_id = id_map.get(obj._lab_name) if obj._lab_name else None
+
+
 def consume_buffer(buffer: list[TableModels], table_name: TableNames) -> None:
     """
     Consume a buffer of items and insert them into the database.
@@ -253,6 +289,8 @@ def flush_buffers(
     # Insert in dependency-safe order
     flush_start = time.time()
     try:
+        # Autocommit, outside the transaction: avoids holding lab locks per flush.
+        assign_lab_ids(builds_buf, tests_buf)
         # Single transaction for all tables in the flush
         with transaction.atomic():
             consume_buffer(issues_buf, "issues")

backend/kernelCI_app/management/commands/helpers/process_submissions.py

@@ -117,6 +117,7 @@ def make_build_instance(build: dict[str, Any]) -> Builds:
     filtered_build = {key: value for key, value in build.items() if key in BUILD_FIELDS}
     obj = Builds(**filtered_build)
     obj.field_timestamp = timezone.now()
+    obj._lab_name = build.get("_real_lab")
     return obj
 
 
@@ -127,6 +128,7 @@ def make_test_instance(test: dict[str, Any]) -> Tests:
     }
     obj = Tests(**filtered_test)
     obj.field_timestamp = timezone.now()
+    obj._lab_name = test.get("_real_lab")
     return obj
 
 
@@ -187,7 +189,8 @@ def _process(items, item_type: TableNames):
 
                         try:
                             misc = build.misc
-                            lab = misc.get("lab")
+                            # TODO remove misc->>'lab' fallback after lab backfill
+                            lab = build._lab_name or misc.get("lab")
                         except AttributeError:
                             lab = None
 
@@ -202,7 +205,8 @@ def _process(items, item_type: TableNames):
 
                         try:
                             misc = test.misc
-                            lab = misc.get("lab", misc.get("runtime"))
+                            # TODO remove misc->>'runtime' fallback after lab backfill
+                            lab = test._lab_name or misc.get("runtime")
                         except AttributeError:
                             lab = None