Skip to content

Commit

Permalink
update route and prompt (#33)
Browse files Browse the repository at this point in the history 
* update route and prompt

* pretty

* rm log

* rm files
  • Loading branch information
@0xKurt
0xKurt authored Dec 6, 2024
1 parent fbad3be commit 84b5e6b
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 93 deletions.
154 changes: 83 additions & 71 deletions src/controllers/evaluationController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,9 @@ const batchPromises = <T>(array: T[], batchSize: number): T[][] => {

export const createLLMEvaluations = async (
paramsArray: CreateLLMEvaluationParams[]
): Promise<void> => {
): Promise<string[]> => {
const roundCache: Record<string, RoundWithApplications> = {};
const failedProjects: string[] = [];

// Split the paramsArray into batches of 10
const batchedParams = batchPromises(paramsArray, 10);
Expand All @@ -310,86 +311,95 @@ export const createLLMEvaluations = async (
try {
// Process each batch of promises concurrently
const evaluationPromises = batch.map(async params => {
const evaluationQuestions =
params.questions === undefined || params.questions.length === 0
? await evaluationService.getQuestionsByChainAndAlloPoolId(
params.chainId,
params.alloPoolId
)
: params.questions;

if (evaluationQuestions === null || evaluationQuestions.length === 0) {
logger.error(
'createLLMEvaluations:Failed to get evaluation questions'
);
throw new Error('Failed to get evaluation questions');
}

let roundMetadata = params.roundMetadata;
let applicationMetadata = params.applicationMetadata;

// Check if the round is already in cache
if (roundMetadata == null || applicationMetadata == null) {
let round: RoundWithApplications | null;

// If the round is cached, use it
if (roundCache[params.alloPoolId] != null) {
round = roundCache[params.alloPoolId];
logger.debug(
`Using cached round data for roundId: ${params.alloPoolId}`
);
} else {
// Fetch the round and store it in the cache
const [error, fetchedRound] = await catchError(
indexerClient.getRoundWithApplications({
chainId: params.chainId,
roundId: params.alloPoolId,
})
try {
const evaluationQuestions =
params.questions === undefined || params.questions.length === 0
? await evaluationService.getQuestionsByChainAndAlloPoolId(
params.chainId,
params.alloPoolId
)
: params.questions;

if (
evaluationQuestions === null ||
evaluationQuestions.length === 0
) {
logger.error(
'createLLMEvaluations:Failed to get evaluation questions'
);
throw new Error('Failed to get evaluation questions');
}

if (error !== undefined || fetchedRound == null) {
logger.error('Failed to fetch round with applications');
throw new Error('Failed to fetch round with applications');
let roundMetadata = params.roundMetadata;
let applicationMetadata = params.applicationMetadata;

// Check if the round is already in cache
if (roundMetadata == null || applicationMetadata == null) {
let round: RoundWithApplications | null;

// If the round is cached, use it
if (roundCache[params.alloPoolId] != null) {
round = roundCache[params.alloPoolId];
logger.debug(
`Using cached round data for roundId: ${params.alloPoolId}`
);
} else {
// Fetch the round and store it in the cache
const [error, fetchedRound] = await catchError(
indexerClient.getRoundWithApplications({
chainId: params.chainId,
roundId: params.alloPoolId,
})
);

if (error !== undefined || fetchedRound == null) {
logger.error('Failed to fetch round with applications');
throw new Error('Failed to fetch round with applications');
}

round = fetchedRound;
roundCache[params.alloPoolId] = round;
logger.info(
`Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications`
);
}

round = fetchedRound;
roundCache[params.alloPoolId] = round;
logger.info(
`Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications`
const application = round.applications.find(
app => app.id === params.alloApplicationId
);
if (application == null) {
logger.error(
`Application with ID: ${params.alloApplicationId} not found in round`
);
throw new NotFoundError(
`Application with ID: ${params.alloApplicationId} not found in round`
);
}

roundMetadata = round.roundMetadata;
applicationMetadata = application.metadata;
}

const application = round.applications.find(
app => app.id === params.alloApplicationId
const evaluation = await requestEvaluation(
roundMetadata,
applicationMetadata,
evaluationQuestions
);
if (application == null) {
logger.error(
`Application with ID: ${params.alloApplicationId} not found in round`
);
throw new NotFoundError(
`Application with ID: ${params.alloApplicationId} not found in round`
);
}

roundMetadata = round.roundMetadata;
applicationMetadata = application.metadata;
await createEvaluation({
chainId: params.chainId,
alloPoolId: params.alloPoolId,
alloApplicationId: params.alloApplicationId,
cid: params.cid,
evaluator: params.evaluator,
summaryInput: evaluation,
evaluatorType: EVALUATOR_TYPE.LLM_GPT3,
});
} catch (error) {
// If an error occurs, add the project ID to the failedProjects array
failedProjects.push(params.alloApplicationId);
throw error;
}

const evaluation = await requestEvaluation(
roundMetadata,
applicationMetadata,
evaluationQuestions
);

await createEvaluation({
chainId: params.chainId,
alloPoolId: params.alloPoolId,
alloApplicationId: params.alloApplicationId,
cid: params.cid,
evaluator: params.evaluator,
summaryInput: evaluation,
evaluatorType: EVALUATOR_TYPE.LLM_GPT3,
});
});

await Promise.all(evaluationPromises);
Expand All @@ -405,4 +415,6 @@ export const createLLMEvaluations = async (
continue;
}
}

return failedProjects;
};
32 changes: 25 additions & 7 deletions src/controllers/poolController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,16 +101,34 @@ export const syncPool = async (req: Request, res: Response): Promise<void> => {

// ---- LLM evaluation ----
// Trigger LLM evaluation for the pool if there are applications without evaluations
if (skipEvaluation !== true)
await triggerLLMEvaluationByPool(
let failedProjects: string[] = [];
if (skipEvaluation !== true) {
failedProjects = await triggerLLMEvaluationByPool(
alloPoolId,
chainId,
indexerPoolData,
evaluationQuestions
);
// Log success and respond to the request
logger.info('successfully synced pool', pool);
res.status(200).json({ message: 'pool synced successfully' });
}

// Check if there are any failed projects and respond accordingly
if (failedProjects.length > 0) {
logger.info('Pool synced successfully with some projects failing', {
pool,
failedProjects,
});
res.status(207).json({
success: true,
message: 'Pool synced successfully, with some projects failing.',
failedProjects,
});
} else {
logger.info('Successfully synced pool', pool);
res.status(200).json({
success: true,
message: 'Pool synced successfully.',
});
}
};

const handlePoolEvaluationQuestions = async (
Expand Down Expand Up @@ -171,7 +189,7 @@ const triggerLLMEvaluationByPool = async (
chainId: number,
indexerPoolData: IndexerRoundWithApplications,
evaluationQuestions: PromptEvaluationQuestions
): Promise<void> => {
): Promise<string[]> => {
const applicationsWithoutLLM =
await applicationService.getApplicationsWithoutLLMEvalutionsByAlloPoolId(
alloPoolId,
Expand Down Expand Up @@ -212,5 +230,5 @@ const triggerLLMEvaluationByPool = async (
logger.warn('Some applications were not found in indexerPoolData');
}

await createLLMEvaluations(evaluationParamsArray);
return await createLLMEvaluations(evaluationParamsArray);
};
6 changes: 3 additions & 3 deletions src/ext/openai/prompt.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ const sanitizeAndReduceMetadata = (
};

const sanitizeRoundMetadata = (metadata: RoundMetadata): string => {
return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 1000);
return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 50000);
};

const sanitizeApplicationMetadata = (metadata: object): string => {
return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 3000);
return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 50000);
};
export const createAiEvaluationPrompt = (
roundMetadata: RoundMetadata,
Expand All @@ -52,7 +52,7 @@ export const createAiEvaluationPrompt = (
): string => {
const sanitizedRoundMetadata = sanitizeRoundMetadata(roundMetadata);
const sanitizedApplicationMetadata = sanitizeApplicationMetadata(
applicationMetadata.application.project
applicationMetadata.application
);

const questionsString = applicationQuestions
Expand Down
36 changes: 24 additions & 12 deletions src/routes/poolRoutes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,43 @@ const router = Router();
* alloPoolId:
* type: string
* description: The ID of the pool to create
* example: "609" # Example of poolId
* example: "609"
* chainId:
* type: number
* description: The chain ID associated with the pool
* example: 42161 # Example of chainId (Arbitrum)
* example: 42161
* skipEvaluation:
* type: boolean
* description: Skip evaluation of the pool
* example: true
* type: boolean
* description: Skip evaluation of the pool
* example: true
* required:
* - alloPoolId
* - chainId
* responses:
* 201:
* description: Pool created successfully
* description: Pool synced successfully
* 207:
* description: Pool synced successfully, with some projects failing
* content:
* application/json:
* schema:
* type: object
* properties:
* success:
* type: boolean
* example: true
* message:
* type: string
* example: Pool synced successfully, with some projects failing.
* failedProjects:
* type: array
* items:
* type: string
* example: "0xprojectId1"
* 400:
* description: Invalid poolId or chainId format
* 500:
* description: Internal server error
* examples:
* application/json:
* - value:
* alloPoolId: "609"
* chainId: "42161"
* skipEvaluation: true
*/
router.post('/', syncPool);

Expand Down

0 comments on commit 84b5e6b

Please sign in to comment.