[BE] Move benchmark configs & types around for easy modification

torchci/components/benchmark_v3/components/benchmarkRegressionReport/common.tsx

@@ -2,13 +2,13 @@ import CircleIcon from "@mui/icons-material/Circle";
 import OpenInNewIcon from "@mui/icons-material/OpenInNew";
 import { Chip, IconButton, Tooltip, Typography } from "@mui/material";
 import { Box, Stack } from "@mui/system";
+import { getBenchmarkIdFromReportId } from "components/benchmark_v3/configs/configurations";
 import { getBenchmarkFields } from "components/benchmark_v3/configs/utils/urlHandling";
 import { getBenchmarkMainRouteById } from "components/benchmark_v3/pages/BenchmarkListPage";
 import { queryObjectToSearchParams } from "components/uiModules/UMCopyLink";
 import dayjs from "dayjs";
 import {
   BenchmarkCommitMeta,
-  getBenchmarkIdFromReportId,
   TimeRange,
 } from "lib/benchmark/store/benchmark_regression_store";
 import { stateToQuery } from "lib/helpers/urlQuery";

torchci/components/benchmark_v3/components/benchmarkSideBar/BenchmarkTopBar.tsx

@@ -1,6 +1,6 @@
 import { Divider, Paper, Typography } from "@mui/material";
 import { Stack } from "@mui/system";
-import { BenchmarkUIConfigHandler } from "lib/benchmark/store/benchmark_config_book";
+import { BenchmarkUIConfigHandler } from "components/benchmark_v3/configs/benchmark_config_book";
 import { BenchmarkReportFeatureNotification } from "../benchmarkRegressionReport/BenchmarkReportFeatureNotification";
 import { BenchmarkReportFeatureSidePanel } from "../benchmarkRegressionReport/BenchmarkReportFeatureSidePanel";
 import { CommitWorflowSelectSection } from "./components/commits/CommitWorkfowSelectSection";

torchci/components/benchmark_v3/components/benchmarkSideBar/components/SideBarMainSection.tsx

@@ -2,14 +2,14 @@
 import Divider from "@mui/material/Divider";
 import Stack from "@mui/material/Stack";
 import Typography from "@mui/material/Typography";
+import { useBenchmarkBook } from "components/benchmark_v3/configs/benchmark_config_book";
 import { QueryParameterConverterInputs } from "components/benchmark_v3/configs/utils/dataBindingRegistration";
 import { CenteredLoader } from "components/common/LoadingIcon";
 import { UMCopyLink } from "components/uiModules/UMCopyLink";
 import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker";
 import { UMDenseButtonLight } from "components/uiModules/UMDenseComponents";
 import dayjs from "dayjs";
 import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks";
-import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book";
 import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider";
 import { useRouter } from "next/router";
 import { useEffect, useRef, useState } from "react";

torchci/components/benchmark_v3/components/benchmarkSideBar/components/commits/CommitWorkfowSelectSection.tsx

@@ -1,10 +1,10 @@
 import { Typography } from "@mui/material";
 import { Stack } from "@mui/system";
+import { useBenchmarkBook } from "components/benchmark_v3/configs/benchmark_config_book";
 import { QueryParameterConverterInputs } from "components/benchmark_v3/configs/utils/dataBindingRegistration";
 import { CenteredLoader } from "components/common/LoadingIcon";
 import { UMDenseCommitDropdown } from "components/uiModules/UMDenseComponents";
 import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks";
-import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book";
 import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider";
 import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store";
 import { useEffect, useState } from "react";

torchci/components/benchmark_v3/components/benchmarkSideBar/components/filters/BenchmarkFilterDropdownGroup.tsx

@@ -1,9 +1,9 @@
 import { Stack } from "@mui/system";
+import { BenchmarkUIConfigFilterConstarintConfig } from "components/benchmark_v3/configs/config_book_types";
 import {
   UMDenseDropdown,
   UMDenseDropdownOption,
 } from "components/uiModules/UMDenseComponents";
-import { BenchmarkUIConfigFilterConstarintConfig } from "lib/benchmark/store/benchmark_config_book";
 
 /**
  * The input item for benchmark dashboard dropdown

torchci/components/benchmark_v3/components/dataRender/auto/autoComponents.tsx

@@ -6,12 +6,13 @@ import {
   useBenchmarkCommittedContext,
   useBenchmarkTimeSeriesData,
 } from "lib/benchmark/api_helper/fe/hooks";
-import { UIRenderConfig } from "lib/benchmark/store/benchmark_config_book";
 import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider";
 import BenchmarkRawDataTable from "../components/benchmarkTimeSeries/components/BenchmarkRawDataTable";
 
 import { LOG_PREFIX } from "components/benchmark/common";
+import { UIRenderConfig } from "components/benchmark_v3/configs/config_book_types";
 import { BenchmarkLogSidePanelWrapper } from "../../common/BenchmarkLogViewer/BenchmarkSidePanel";
+import { RenderRawContent } from "../../common/RawContentDialog";
 import BenchmarkTimeSeriesChartGroup from "../components/benchmarkTimeSeries/components/BenchmarkTimeSeriesChart/BenchmarkTimeSeriesChartGroup";
 import { ComparisonTable } from "../components/benchmarkTimeSeries/components/BenchmarkTimeSeriesComparisonSection/BenchmarkTimeSeriesComparisonTable/ComparisonTable";
 
@@ -243,6 +244,14 @@ export function AutoBenchmarkPairwiseTable({ config }: AutoComponentProps) {
     );
   }
 
+  if (error) {
+    return (
+      <Alert severity="error">
+        (AutoBenchmarkTimeSeriesTable){error.message}
+      </Alert>
+    );
+  }
+
   if (isLoading || !resp) {
     return (
       <LoadingPage
@@ -252,14 +261,6 @@ export function AutoBenchmarkPairwiseTable({ config }: AutoComponentProps) {
     );
   }
 
-  if (error) {
-    return (
-      <Alert severity="error">
-        (AutoBenchmarkTimeSeriesTable){error.message}
-      </Alert>
-    );
-  }
-
   if (!resp?.data?.data) {
     return <div>no data</div>;
   }
@@ -268,6 +269,7 @@ export function AutoBenchmarkPairwiseTable({ config }: AutoComponentProps) {
   return (
     <Grid container sx={{ m: 1 }}>
       <Grid sx={{ p: 0.2 }} size={{ xs: 12 }}>
+        <RenderRawContent data={data["table"]} />
         <ComparisonTable
           data={data["table"]}
           config={uiRenderConfig.config}

torchci/components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/components/BenchmarkTimeSeriesComparisonSection/BenchmarkTimeSeriesComparisonTable/ComparisonTable.tsx

@@ -87,6 +87,11 @@ export function ComparisonTable({
     [allColumns, lWorkflowId, rWorkflowId, title]
   );
 
+  const columnVisibilityModel = Object.fromEntries(
+    Object.entries(config?.renderOptions?.tableRenderingBook ?? {})
+      .filter(([_, v]) => v?.hide === true)
+      .map(([k]) => [k, false])
+  );
   return (
     <Box>
       <Typography variant="h6">{title.text}</Typography>
@@ -122,6 +127,9 @@ export function ComparisonTable({
           pagination: {
             paginationModel: { pageSize: 25 },
           },
+          columns: {
+            columnVisibilityModel: columnVisibilityModel,
+          },
         }}
         getRowId={(r) => {
           return r.id;

torchci/components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/components/BenchmarkTimeSeriesComparisonSection/BenchmarkTimeSeriesComparisonTable/ComparisonTableColumnRendering.tsx

@@ -100,11 +100,16 @@ export function getComparisionTableConlumnRendering(
       const L = valOf(ldata);
       const R = valOf(rdata);
 
+      // get target field name, for instance, metric
       const targetField = getBenchmarkTimeSeriesComparisonTableTarget();
       const findFieldValueFromColData =
         ldata?.[targetField] ?? rdata?.[targetField];
       const targetVal = findFieldValueFromColData;
 
+      const failureConfig =
+        config?.renderOptions?.tableRenderingBook?.[targetVal]?.failure;
+      // lobr snf gs∂
+
       const { result, text } = getComparisonResult(
         L,
         R,
@@ -121,7 +126,6 @@ export function getComparisionTableConlumnRendering(
       };
     },
     valueFormatter: (value: any, row: any) => {
-      // this export as value in the export csv file
       return value?.text ?? "";
     },
     renderCell: (params: GridRenderCellParams<any, GridRowModel>) => (
@@ -212,7 +216,7 @@ export function ComparisonTableColumnFieldValueCell({
 }) {
   // pick background color based on result signals
   let bgColor = "";
-  switch (result.verdict) {
+  switch (result?.verdict) {
     case "good":
       bgColor = IMPROVEMENT_COLOR;
       break;
@@ -243,7 +247,8 @@ export function ComparisonTableColumnFieldValueCell({
   );
 }
 
-function renderComparisonResult(result: ComparisonResult) {
+function renderComparisonResult(result?: ComparisonResult) {
+  if (!result) return "";
   return (
     <Box sx={{ p: 1 }}>
       {Object.entries(result).map(([key, value]) => (
@@ -262,6 +267,8 @@ export function getFieldRender(
   config?: ComparisonTableConfig,
   ldisplay?: string,
   rdisplay?: string,
+  lfailed?: boolean,
+  rfailed?: boolean,
   missingText: string = "missing data"
 ) {
   if (ldisplay || rdisplay) {
@@ -271,13 +278,24 @@ export function getFieldRender(
     targetField,
     config
   );
-  return formatTransitionWithUnit(L, R, rc?.unit, missingText);
+
+  return formatTransitionWithUnit(
+    L,
+    R,
+    lfailed,
+    rfailed,
+    rc?.unit,
+    missingText
+  );
 }
 export function formatTransitionWithUnit(
   L: any,
   R: any,
+  lfailed?: boolean,
+  rfailed?: boolean,
   table_unit?: BenchmarkUnitConfig,
-  missingText: string = "missing data"
+  missingText: string = "missing data",
+  failureText: string = "Failure"
 ): string {
   const formatValue = (v: any) => {
     if (v == null || v == undefined) return missingText;
@@ -290,6 +308,17 @@ export function formatTransitionWithUnit(
     return String(v);
   };
 
+  if (lfailed && rfailed) {
+    return failureText;
+  }
+
+  if (lfailed) {
+    return `${failureText}→${formatValue(R)}`;
+  }
+  if (rfailed) {
+    return `${formatValue(L)}→${failureText}`;
+  }
+
   if (L == null && R == null) {
     return missingText;
   }
@@ -327,7 +356,22 @@ export function getComparisonResult(
 
   const ldisplay = displayNameOf(ldata);
   const rdisplay = displayNameOf(rdata);
-  const text = getFieldRender(targetVal, L, R, config, ldisplay, rdisplay);
+
+  if (ldata?.is_failure || rdata?.is_failure) {
+    result.verdict = "warning";
+    result.reason = "detect failure";
+  }
+
+  const text = getFieldRender(
+    targetVal,
+    L,
+    R,
+    config,
+    ldisplay,
+    rdisplay,
+    ldata?.is_failure,
+    rdata?.is_failure
+  );
   return {
     result,
     text,

torchci/components/benchmark_v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx

@@ -38,6 +38,11 @@ export interface BenchmarkTimeSeriesChartRenderingConfig {
 export interface BenchmarkComparisonTableRenderingConfig {
   displayName?: string;
   unit: BenchmarkUnitConfig;
+  hide: boolean; // hide the column in the table
+  failure: {
+    value: any;
+    dependence?: string; // the field name to check the failure condition
+  };
 }
 
 // The book: dictionary keyed by field name

torchci/components/benchmark_v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx

@@ -1,10 +1,10 @@
 import { Alert, Divider, Typography } from "@mui/material";
 import { Box, Stack } from "@mui/system";
 import { HighlightStyles } from "components/benchmark_v3/components/common/highlight";
+import { useBenchmarkBook } from "components/benchmark_v3/configs/benchmark_config_book";
 import { getFanoutRenderComponent } from "components/benchmark_v3/configs/utils/fanoutRegistration";
 import LoadingPage from "components/common/LoadingPage";
 import { useBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/fe/hooks";
-import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book";
 import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider";
 import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store";
 import { useState } from "react";