feat/backfill db

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -1,4 +1,3 @@
-import bisect
 import json
 from collections import defaultdict
 from datetime import datetime, timezone
@@ -28,7 +27,6 @@
     misc_value_or_default,
 )
 from kernelCI_app.typeModels.commonDetails import (
-    BuildArchitectures,
     BuildSummary,
     EnvironmentMisc,
     StatusCount,
@@ -290,8 +288,6 @@ def handle_test_history(
     full_environment_misc: bool = False,
 ) -> None:
     create_record_test_platform(record=record)
-    record_misc = sanitize_dict(record.get("misc"))
-
     environment_misc_dict = get_environment_misc_value(
         full_environment_misc=full_environment_misc,
         parsed_environment_misc=record.get("parsed_environment_misc"),
@@ -313,11 +309,7 @@ def handle_test_history(
         environment_misc=environment_misc,
         tree_name=record["build__checkout__tree_name"],
         git_repository_branch=record["build__checkout__git_repository_branch"],
-        lab=(
-            record_misc.get("runtime", UNKNOWN_STRING)
-            if record_misc
-            else UNKNOWN_STRING
-        ),
+        lab=record.get("lab") or UNKNOWN_STRING,
     )
 
     task.append(test_history_item)
@@ -380,16 +372,14 @@ def handle_test_summary(
         getattr(task.origins[origin], status) + 1,
     )
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
-    if lab:
-        if task.labs.get(lab) is None:
-            task.labs[lab] = StatusCount()
-        setattr(
-            task.labs[lab],
-            status,
-            getattr(task.labs[lab], status) + 1,
-        )
+    lab = record.get("lab") or UNKNOWN_STRING
+    if task.labs.get(lab) is None:
+        task.labs[lab] = StatusCount()
+    setattr(
+        task.labs[lab],
+        status,
+        getattr(task.labs[lab], status) + 1,
+    )
 
 
 def handle_build_history(
@@ -402,79 +392,7 @@ def handle_build_history(
     builds.append(build)
 
 
-def handle_build_summary(
-    *,
-    record: Dict,
-    builds_summary: BuildSummary,
-    issue_dict: Dict,
-    tree_index: int,
-) -> None:
-    build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index)
-
-    status_key = build.status
-    setattr(
-        builds_summary.status,
-        status_key,
-        getattr(builds_summary.status, status_key) + 1,
-    )
-
-    if config := build.config_name:
-        build_config_summary = builds_summary.configs.get(config)
-        if not build_config_summary:
-            build_config_summary = StatusCount()
-            builds_summary.configs[config] = build_config_summary
-        setattr(
-            builds_summary.configs[config],
-            status_key,
-            getattr(builds_summary.configs[config], status_key) + 1,
-        )
-
-    if arch := build.architecture:
-        build_arch_summary = builds_summary.architectures.get(arch)
-        if not build_arch_summary:
-            build_arch_summary = BuildArchitectures()
-            builds_summary.architectures[arch] = build_arch_summary
-        setattr(
-            builds_summary.architectures[arch],
-            status_key,
-            getattr(builds_summary.architectures[arch], status_key) + 1,
-        )
-
-        compiler = build.compiler
-        if (
-            compiler is not None
-            and compiler not in builds_summary.architectures.get(arch).compilers
-        ):
-            bisect.insort(builds_summary.architectures[arch].compilers, compiler)
-
-    if origin := build.origin:
-        build_origin_summary = builds_summary.origins.get(origin)
-        if not build_origin_summary:
-            build_origin_summary = StatusCount()
-            builds_summary.origins[origin] = build_origin_summary
-        setattr(
-            builds_summary.origins[origin],
-            status_key,
-            getattr(builds_summary.origins[origin], status_key) + 1,
-        )
-
-    misc = sanitize_dict(build.misc) or {}
-    lab = misc.get("lab", UNKNOWN_STRING)
-    if lab:
-        build_lab_summary = builds_summary.labs.get(lab)
-        if not build_lab_summary:
-            build_lab_summary = StatusCount()
-            builds_summary.labs[lab] = build_lab_summary
-        setattr(
-            builds_summary.labs[lab],
-            status_key,
-            getattr(builds_summary.labs[lab], status_key) + 1,
-        )
-
-    process_issue(record=record, task_issues_dict=issue_dict, issue_from="build")
-
-
-# deprecated, use handle_build_history and handle_build_summary separately instead, with typing
+# deprecated, use handle_build_history separately instead, with typing
 def handle_build(*, instance, record: Dict, build: Dict) -> None:
     instance.builds["items"].append(build)
     update_issues(
@@ -579,8 +497,7 @@ def decide_if_is_full_record_filtered_out(
     if not is_current_tree_selected:
         return True
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
+    lab = record.get("lab") or UNKNOWN_STRING
 
     is_record_filtered_out_result = instance.filters.is_record_filtered_out(
         hardwares=record["environment_compatible"],
@@ -730,10 +647,8 @@ def process_filters(*, instance, record: Dict) -> None:
 
         instance.unfiltered_origins["build"].add(record["build__origin"])
 
-        build_misc = sanitize_dict(record.get("build__misc")) or {}
-        build_lab = build_misc.get("lab")
-        if build_lab:
-            instance.unfiltered_labs["build"].add(build_lab)
+        if record.get("build_lab"):
+            instance.unfiltered_labs["build"].add(record["build_lab"])
 
     if record["id"] is not None:
         if is_boot(record["path"]):
@@ -771,10 +686,8 @@ def process_filters(*, instance, record: Dict) -> None:
         platform_set.add(test_platform)
         origin_set.add(record["test_origin"])
 
-        test_misc = sanitize_dict(record.get("misc")) or {}
-        test_lab = test_misc.get("runtime")
-        if test_lab:
-            instance.unfiltered_labs[flag_tab].add(test_lab)
+        if record.get("lab"):
+            instance.unfiltered_labs[flag_tab].add(record["lab"])
 
 
 def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool:

backend/kernelCI_app/helpers/treeDetails.py

@@ -25,7 +25,6 @@
     create_issue_typed,
     extract_error_message,
     is_boot,
-    sanitize_dict,
 )
 
 
@@ -87,42 +86,41 @@ def get_current_row_data(
         "test_number_value": current_row[10],
         "test_misc": current_row[11],
         "test_environment_compatible": current_row[tmp_test_env_comp_key],
-        "build_id": current_row[13],
-        "build_origin": current_row[14],
-        "build_comment": current_row[15],
-        "build_start_time": current_row[16],
-        "build_duration": current_row[17],
-        "build_architecture": current_row[18],
-        "build_command": current_row[19],
-        "build_compiler": current_row[20],
-        "build_config_name": current_row[21],
-        "build_config_url": current_row[22],
-        "build_log_url": current_row[23],
-        "build_status": current_row[24],
-        "build_misc": current_row[25],
-        "checkout_id": current_row[26],
-        "checkout_git_repository_url": current_row[27],
-        "checkout_git_repository_branch": current_row[28],
-        "checkout_git_commit_tags": current_row[29],
-        "checkout_origin": current_row[30],
-        "incident_id": current_row[31],
-        "incident_test_id": current_row[32],
-        "incident_present": current_row[33],
-        "issue_id": current_row[34],
-        "issue_version": current_row[35],
-        "issue_comment": current_row[36],
-        "issue_report_url": current_row[37],
+        "test_lab": current_row[13],
+        "build_id": current_row[14],
+        "build_origin": current_row[15],
+        "build_comment": current_row[16],
+        "build_start_time": current_row[17],
+        "build_duration": current_row[18],
+        "build_architecture": current_row[19],
+        "build_command": current_row[20],
+        "build_compiler": current_row[21],
+        "build_config_name": current_row[22],
+        "build_config_url": current_row[23],
+        "build_log_url": current_row[24],
+        "build_status": current_row[25],
+        "build_misc": current_row[26],
+        "build_lab": current_row[27],
+        "checkout_id": current_row[28],
+        "checkout_git_repository_url": current_row[29],
+        "checkout_git_repository_branch": current_row[30],
+        "checkout_git_commit_tags": current_row[31],
+        "checkout_origin": current_row[32],
+        "incident_id": current_row[33],
+        "incident_test_id": current_row[34],
+        "incident_present": current_row[35],
+        "issue_id": current_row[36],
+        "issue_version": current_row[37],
+        "issue_comment": current_row[38],
+        "issue_report_url": current_row[39],
     }
 
     parsed_environment_misc = handle_misc(
         misc_value_or_default(current_row_data["test_environment_misc"])
     )
     current_row_data["test_platform"] = parsed_environment_misc.get("platform")
     current_row_data["parsed_environment_misc"] = parsed_environment_misc
-    test_misc = sanitize_dict(current_row_data["test_misc"])
-    test_runtime_lab = UNKNOWN_STRING
-    if test_misc is not None:
-        test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING)
+    test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING
 
     if current_row_data["test_status"] is None:
         current_row_data["test_status"] = NULL_STATUS
@@ -495,8 +493,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False)
         instance.global_architectures.add(row_data["build_architecture"])
         instance.global_compilers.add(row_data["build_compiler"])
         instance.unfiltered_origins["build"].add(row_data["build_origin"])
-        if (build_misc := row_data["build_misc"]) is not None:
-            instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING))
+        instance.unfiltered_labs["build"].add(
+            row_data.get("build_lab") or UNKNOWN_STRING
+        )
 
         build_issue_id, build_issue_version, is_build_issue = (
             should_increment_build_issue(