chore: add logger write through evaluate advise (#895)

Co-authored-by: yanzhi.yan <[email protected]>

Author: Yzing

__tests__/evaluation/advise/shared.ts

@@ -1,3 +1,5 @@
+import winston from 'winston';
+import fs from 'fs';
 import { loadDataset } from '../utils/load-dataset';
 import { AVA, Spec } from '../../../src';
 
@@ -20,32 +22,107 @@ export type TestData = {
  */
 export const runAdviseEvaluation = (
   selectQuestion: (data: TestData) => string,
-  isPass: (spec: Spec, answer: TestData['answer']) => boolean
+  isPass: (spec: Spec, answer: TestData['answer']) => boolean,
+  loggerPath?: {
+    info?: string;
+    error?: string;
+  }
 ) => {
   jest.setTimeout(3600000);
 
+  const errorLogPath = loggerPath?.error || '__tests__/evaluation/advise/error.jsonl';
+  const infoLogPath = loggerPath?.info || '__tests__/evaluation/advise/info.jsonl';
+
+  // remove old log file
+  if (fs.existsSync(errorLogPath)) {
+    fs.rmSync(errorLogPath);
+  }
+  if (fs.existsSync(infoLogPath)) {
+    fs.rmSync(infoLogPath);
+  }
+
+  const logger = winston.createLogger({
+    format: winston.format.json(),
+    defaultMeta: { service: 'evaluate-service' },
+    transports: [
+      new winston.transports.File({ filename: errorLogPath, level: 'error' }),
+      new winston.transports.File({ filename: infoLogPath, level: 'info' }),
+    ],
+  });
+
   const ava = new AVA({
     llm: {
-      appId: process.env.TBOX_APP_ID!,
-      authorization: process.env.TBOX_AUTHORIZATION!,
+      appId: process.env.TBOX_LLM_APP_ID!,
+      authorization: process.env.TBOX_LLM_AUTH!,
     },
   });
 
-  const evaluateChartAdvise = (chartId: string) => {
+  const evaluateChartAdvise = async (chartId: string) => {
     const dataset = loadDataset(chartId);
 
-    dataset.forEach((data: TestData, i: number) => {
-      it(`evaluate ${chartId} case ${i}`, async () => {
-        console.log(`evaluate ${chartId} case ${i}`);
-        const { answer } = data;
-        const question = selectQuestion(data);
-        const dataShards = await ava.extract(question);
-        const advises = await ava.advise(dataShards);
-        const { spec } = advises?.[0]?.charts?.[0] || {};
-        const success = isPass(spec, answer);
-        expect(success).toEqual(true);
+    const promises = dataset.map((data: TestData, i: number) => {
+      return new Promise((resolve) => {
+        it(`evaluate ${chartId} case ${i}`, async () => {
+          console.log(`evaluate ${chartId} case ${i}`);
+          const { answer } = data;
+          const question = selectQuestion(data);
+          let dataShards = [];
+          try {
+            dataShards = await ava.extract(question);
+          } catch (e) {
+            logger.error({
+              msg: 'extract error',
+              input: question,
+            });
+          }
+          if (dataShards.length === 0) {
+            logger.error({
+              msg: 'extract empty',
+              input: question,
+            });
+            resolve(null);
+            return;
+          }
+          try {
+            const advises = await ava.advise(dataShards);
+            const { spec } = advises?.[0]?.charts?.[0] || {};
+            logger.info({
+              msg: 'advise success',
+              input: question,
+              dataShards,
+              output: spec,
+              source: answer,
+            });
+          } catch (e) {
+            logger.error({
+              msg: 'advise error',
+              input: question,
+            });
+          }
+          resolve(null);
+        });
       });
     });
+
+    await Promise.all(promises);
+
+    it('evaluate pass rate should >= 0.95', () => {
+      const data = fs.readFileSync(loggerPath?.info);
+      const lines = data
+        .toString()
+        .split('\n')
+        .filter((v) => !!v.length);
+      const passCount = lines.filter((line) => {
+        try {
+          const log = JSON.parse(line);
+          return isPass(log.spec, log.source);
+        } catch (e) {
+          return false;
+        }
+      }).length;
+      console.log('pass rate: ', passCount / lines.length);
+      expect(passCount / lines.length).toBeGreaterThanOrEqual(0.95);
+    });
   };
 
   const chartIds = [

__tests__/evaluation/advise/specify-chart.test.ts

@@ -10,5 +10,9 @@ runAdviseEvaluation(
   (spec: Spec, answer: TestData['answer']) => {
     const { type, ...finalSpec } = spec || {};
     return type === answer.type && CHARTS[type] && validateObject(CHARTS[type].zodSchema, finalSpec) === true;
+  },
+  {
+    info: '__tests__/evaluation/advise/specify-chart-info.jsonl',
+    error: '__tests__/evaluation/advise/specify-chart-error.jsonl',
   }
 );

__tests__/evaluation/advise/unspecified-chart.test.ts

@@ -26,5 +26,9 @@ runAdviseEvaluation(
   (spec: Spec, answer: TestData['answer']) => {
     const { type, ...finalSpec } = spec || {};
     return typeMatches(type, answer.type) && validateObject(CHARTS[type].zodSchema, finalSpec) === true;
+  },
+  {
+    info: '__tests__/evaluation/advise/unspecified-chart-info.jsonl',
+    error: '__tests__/evaluation/advise/unspecified-chart-error.jsonl',
   }
 );

__tests__/evaluation/extract/index.test.ts

@@ -4,7 +4,7 @@ import { generate } from './generate';
 import { evaluate } from './evaluate';
 import { EVALUATE_RESULT_LOG_PATH } from './constants';
 
-describe('expect evaluate pass rate >= 90%', () => {
+describe('expect evaluate pass rate >= 95%', () => {
   it('generate success rate >= 95%', async () => {
     const { success, total } = await generate();
     expect(success / total).toBeGreaterThanOrEqual(0.95);
@@ -15,9 +15,12 @@ describe('expect evaluate pass rate >= 90%', () => {
     expect(success / total).toBeGreaterThanOrEqual(0.95);
   }, 6000000);
 
-  it('evaluate extract pass rate >= 90%', () => {
+  it('evaluate extract pass rate >= 95%', () => {
     const content = fs.readFileSync(EVALUATE_RESULT_LOG_PATH);
-    const lines = content.toString().split('\n').filter(line => line.length > 0);
+    const lines = content
+      .toString()
+      .split('\n')
+      .filter((line) => line.length > 0);
     let total = lines.length;
     let pass = 0;