Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions source/plain/conversation.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ function makeFakeClient(options: FakeClientOptions): LLMClient {
{ message: { role: "assistant", content: "" } },
],
toolsDisabled: partial.toolsDisabled,
usage: partial.usage,
} as LLMChatResponse;
},
} as unknown as LLMClient;
Expand Down Expand Up @@ -636,6 +637,7 @@ function makeRecordingClient(
{ message: { role: "assistant", content: "" } },
],
toolsDisabled: partial.toolsDisabled,
usage: partial.usage,
} as LLMChatResponse;
},
} as unknown as LLMClient;
Expand Down Expand Up @@ -771,3 +773,102 @@ test.serial(
t.is(calls.length, 1);
},
);

test.serial(
"accumulates token usage across multiple turns",
async (t) => {
const calls: RecordedCall[] = [];
const client = makeRecordingClient(
[
{
choices: [
{
message: {
role: "assistant",
content: "",
tool_calls: [
{
id: "call_1",
type: "function",
function: { name: "safe_tool", arguments: {} },
},
],
},
},
],
usage: { inputTokens: 100, outputTokens: 20, totalTokens: 120 },
},
{
choices: [
{
message: {
role: "assistant",
content: "all done",
},
},
],
usage: { inputTokens: 150, outputTokens: 30, totalTokens: 180 },
},
],
calls,
);
const toolManager = makeFakeToolManager({
knownTools: new Set(["safe_tool"]),
needsApprovalByName: { safe_tool: false },
});
setToolRegistryGetter(() => ({
safe_tool: (async () => "tool-output") as ToolHandler,
}));

const outcome = await runPlainConversation({
client,
toolManager,
systemMessage: SYSTEM,
initialMessages: [USER],
developmentMode: "auto-accept",
nonInteractiveAlwaysAllow: [],
abortSignal: new AbortController().signal,
});

t.is(outcome.kind, "success");
t.deepEqual(outcome.usage, {
inputTokens: 250,
outputTokens: 50,
totalTokens: 300,
});
},
);

test.serial(
"omits usage property when no turn reports usage",
async (t) => {
const client = makeFakeClient({
responses: [
{
choices: [
{
message: {
role: "assistant",
content: "no usage here",
},
},
],
},
],
});
const toolManager = makeFakeToolManager();

const outcome = await runPlainConversation({
client,
toolManager,
systemMessage: SYSTEM,
initialMessages: [USER],
developmentMode: "auto-accept",
nonInteractiveAlwaysAllow: [],
abortSignal: new AbortController().signal,
});

t.is(outcome.kind, "success");
t.is(outcome.usage, undefined);
},
);
37 changes: 37 additions & 0 deletions source/plain/conversation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,35 @@ export interface RunPlainConversationOptions {
outputFormat?: 'text' | 'json';
}

export interface PlainConversationUsage {
inputTokens: number;
outputTokens: number;
totalTokens: number;
}

export type PlainConversationOutcome =
| {
kind: 'success';
finalText: string;
reasoning: string | null;
toolCalls: ToolCallLog[];
usage?: PlainConversationUsage;
}
| {
kind: 'tool-approval-required';
toolNames: string[];
finalText: string;
reasoning: string | null;
toolCalls: ToolCallLog[];
usage?: PlainConversationUsage;
}
| {
kind: 'error';
message: string;
finalText: string;
reasoning: string | null;
toolCalls: ToolCallLog[];
usage?: PlainConversationUsage;
};

// On the last allowed turn we strip tools and inject this so the model
Expand Down Expand Up @@ -98,6 +107,20 @@ export async function runPlainConversation(
let accumulatedReasoning = '';
const toolCallsLog: ToolCallLog[] = [];

let hasReportedUsage = false;
let accumulatedInputTokens = 0;
let accumulatedOutputTokens = 0;
let accumulatedTotalTokens = 0;

const getUsage = (): PlainConversationUsage | undefined => {
if (!hasReportedUsage) return undefined;
return {
inputTokens: accumulatedInputTokens,
outputTokens: accumulatedOutputTokens,
totalTokens: accumulatedTotalTokens,
};
};

const maxTurns =
getAppConfig().headless?.maxTurns ?? DEFAULT_HEADLESS_MAX_TURNS;

Expand All @@ -109,6 +132,7 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand Down Expand Up @@ -173,6 +197,13 @@ export async function runPlainConversation(
modeOverrides,
);

if (result?.usage) {
hasReportedUsage = true;
accumulatedInputTokens += result.usage.inputTokens ?? 0;
accumulatedOutputTokens += result.usage.outputTokens ?? 0;
accumulatedTotalTokens += result.usage.totalTokens ?? 0;
}

if (!isJson && (reasoningPrinted || contentStarted)) {
writeLine();
}
Expand All @@ -184,6 +215,7 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand All @@ -210,6 +242,7 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand Down Expand Up @@ -268,13 +301,15 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}
return {
kind: 'success',
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand Down Expand Up @@ -302,6 +337,7 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand Down Expand Up @@ -352,6 +388,7 @@ export async function runPlainConversation(
finalText: accumulatedFinalText,
reasoning: accumulatedReasoning || null,
toolCalls: toolCallsLog,
usage: getUsage(),
};
}

Expand Down
45 changes: 45 additions & 0 deletions source/plain/shell.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ function baseDeps(
return {
loadPreferences: () => ({ trustedDirectories: [] }) as never,
savePreferences: () => undefined,
getShutdownManager: makeFakeShutdownManager({ code: null }),
...overrides,
};
}
Expand Down Expand Up @@ -129,6 +130,50 @@ test.serial(
t.is(report.finalText, "all done");
t.deepEqual(report.toolCalls, []);
t.deepEqual(report.filesChanged, []);
t.is(report.usage, undefined);
t.is(shutdown.code, 0);
},
);

test.serial(
"--json success outcome includes usage block when present in conversation outcome",
async (t) => {
const shutdown: CapturedShutdown = { code: null };
const stdout = capturingStdout();
try {
await runPlainShell({
prompt: "do the thing",
developmentMode: "auto-accept",
trustDirectory: true,
outputFormat: "json",
deps: baseDeps({
initializePlain: makeFakeInitializePlain(),
runPlainConversation: makeFakeRunPlainConversation({
kind: "success",
finalText: "all done",
reasoning: null,
toolCalls: [],
usage: {
inputTokens: 500,
outputTokens: 100,
totalTokens: 600,
},
}),
getShutdownManager: makeFakeShutdownManager(shutdown),
}),
});
} finally {
stdout.restore();
}

const report = JSON.parse(stdout.get());
t.is(report.kind, "success");
t.is(report.exitCode, 0);
t.deepEqual(report.usage, {
inputTokens: 500,
outputTokens: 100,
totalTokens: 600,
});
t.is(shutdown.code, 0);
},
);
Expand Down
3 changes: 3 additions & 0 deletions source/plain/shell.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,9 @@ export async function runPlainShell(
reasoning: outcome.reasoning ? sanitizeOutput(outcome.reasoning) : null,
toolCalls: formattedToolCalls,
filesChanged: Array.from(filesChangedSet),
...(outcome.usage && {
usage: outcome.usage,
}),
...(outcome.kind === 'error' && {
message: sanitizeOutput(outcome.message),
}),
Expand Down
Loading