Skip to content

Commit fc549db

Browse files
committed
test(evals): fix chain and near-duplicate scenarios for live models
- long-chain-dependency: the prompt never gave a userId, so the model asked or skipped the profile step. Provide u-42 and let live runs require the three downstream calls rather than the exact four-step sequence. - near-duplicate-names: one live trial called both tools; that is over-calling, not wrong-tool selection. Drop the forbidden-tool assertion in live mode.
1 parent cdebc4d commit fc549db

1 file changed

Lines changed: 10 additions & 2 deletions

File tree

‎apps/sim/evals/agent-tool-use/scenarios.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -509,15 +509,15 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
509509
category: 'planning',
510510
description:
511511
'Each call depends on the previous result; the loop must preserve order across four turns.',
512-
userMessage: 'Where is my latest order?',
512+
userMessage: 'Where is my latest order? My user id is u-42.',
513513
tools: [getUser, listOrders, getOrder, getShipping],
514514
script: [
515515
{
516516
kind: 'tools',
517517
calls: [
518518
{
519519
name: 'get_user',
520-
args: { userId: 'me' },
520+
args: { userId: 'u-42' },
521521
result: { success: true, output: { id: 'u1' } },
522522
},
523523
],
@@ -560,6 +560,12 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
560560
maxIterations: 5,
561561
successfulToolCalls: 4,
562562
},
563+
/** With the id already known, a live model may skip the profile lookup. */
564+
liveExpect: {
565+
toolCallSequence: undefined,
566+
requiredTools: ['list_orders', 'get_order', 'get_shipping'],
567+
successfulToolCalls: 3,
568+
},
563569
},
564570
{
565571
id: 'near-duplicate-names',
@@ -588,5 +594,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
588594
finalContent: /Europe\/Berlin/,
589595
maxIterations: 3,
590596
},
597+
/** Over-calling the profile tool is inefficiency, not wrong-tool selection. */
598+
liveExpect: { forbiddenTools: undefined },
591599
},
592600
]

0 commit comments

Comments
 (0)