Support ACP audio prompts

src/CodexAcpClient.ts

@@ -934,7 +934,7 @@ function buildPromptItems(prompt: acp.ContentBlock[]): UserInput[] {
                 return {type: "text", text: `${link}\n${context}`, text_elements: []};
             }
             case "audio":
-                return null;
+                return {type: "audio", url: audioDataUrl(block)};
         }
     }).filter((block): block is UserInput => block !== null);
 }
@@ -943,6 +943,10 @@ function imageDataUrl(block: acp.ContentBlock & { type: "image" }): string {
     return `data:${block.mimeType};base64,${block.data}`;
 }
 
+function audioDataUrl(block: acp.ContentBlock & { type: "audio" }): string {
+    return `data:${block.mimeType};base64,${block.data}`;
+}
+
 function isImageMimeType(mimeType: string | null | undefined): mimeType is string {
     return mimeType?.startsWith("image/") ?? false;
 }

src/CodexAcpServer.ts

@@ -228,7 +228,8 @@ export class CodexAcpServer {
                 loadSession: true,
                 promptCapabilities: {
                     embeddedContext: true,
-                    image: true
+                    image: true,
+                    audio: true
                 },
                 sessionCapabilities: {
                     resume: { },
@@ -958,13 +959,17 @@ export class CodexAcpServer {
 
     /**
      * Rejects a steering prompt whose content the active model cannot accept
-     * (currently: image blocks on a text-only model).
+     * (currently: image or audio blocks when their modality is unsupported).
      */
     private assertSteerInputSupported(params: SessionSteerRequest, sessionState: SessionState): void {
         const hasImage = params.prompt.some(block => block.type === "image");
         if (hasImage && !sessionState.supportedInputModalities.includes("image")) {
             throw RequestError.invalidRequest("The current model does not support image input");
         }
+        const hasAudio = params.prompt.some(block => block.type === "audio");
+        if (hasAudio && !sessionState.supportedInputModalities.includes("audio")) {
+            throw RequestError.invalidRequest("The current model does not support audio input");
+        }
     }
 
     /**
@@ -1975,6 +1980,9 @@ export class CodexAcpServer {
             if (!sessionState.supportedInputModalities.includes("image") && params.prompt.some(b => b.type === "image")) {
                 throw RequestError.invalidRequest("The current model does not support image input");
             }
+            if (!sessionState.supportedInputModalities.includes("audio") && params.prompt.some(b => b.type === "audio")) {
+                throw RequestError.invalidRequest("The current model does not support audio input");
+            }
             const agentMode = sessionState.agentMode;
             const serviceTier = resolveFastServiceTier(
                 sessionState.fastModeEnabled,

src/__tests__/CodexACPAgent/CodexAcpClient.test.ts

@@ -1270,7 +1270,9 @@ describe('ACP server test', { timeout: 40_000 }, () => {
             }
         });
 
-        const sessionState: SessionState = createTestSessionState();
+        const sessionState: SessionState = createTestSessionState({
+            supportedInputModalities: ["text", "image", "audio"],
+        });
         vi.spyOn(codexAcpAgent, "getSessionState").mockReturnValue(sessionState);
 
         const prompt: acp.ContentBlock[] = [
@@ -1280,6 +1282,7 @@ describe('ACP server test', { timeout: 40_000 }, () => {
             { type: "resource", resource: { uri: "file:///tmp/notes.txt", text: "Notes body" } as acp.EmbeddedResourceResource },
             { type: "resource", resource: { uri: "file:///tmp/pixel.png", mimeType: "image/png", blob: "iVBORw0KGgo=" } as acp.EmbeddedResourceResource },
             { type: "resource", resource: { uri: "file:///tmp/archive.bin", mimeType: "application/octet-stream", blob: "AAEC" } as acp.EmbeddedResourceResource },
+            { type: "audio", mimeType: "audio/wav", data: "UklGRg==" },
         ];
 
         await codexAcpAgent.prompt({ sessionId: "session-id", prompt });
@@ -3272,6 +3275,19 @@ describe('ACP server test', { timeout: 40_000 }, () => {
         }));
     });
 
+    it ('should reject prompt with audio when model does not support audio input', async () => {
+        const { mockFixture } = setupPromptFixture({
+            supportedInputModalities: ["text", "image"],
+        });
+
+        const prompt: acp.ContentBlock[] = [
+            { type: "audio", mimeType: "audio/wav", data: "UklGRg==" },
+        ];
+
+        await expect(mockFixture.getCodexAcpAgent().prompt({ sessionId: "id", prompt }))
+            .rejects.toThrow("Invalid request");
+    });
+
     it ('should use inline image data for internal image URIs', async () => {
         const { mockFixture, turnStartSpy } = setupPromptFixture({
             supportedInputModalities: ["text", "image"],

src/__tests__/CodexACPAgent/data/send-attachments-turn-start.json

@@ -44,6 +44,10 @@
         "type": "text",
         "text": "[@archive.bin](file:///tmp/archive.bin)\n<context ref=\"file:///tmp/archive.bin\" mimeType=\"application/octet-stream\" encoding=\"base64\">\nAAEC\n</context>",
         "text_elements": []
+      },
+      {
+        "type": "audio",
+        "url": "data:audio/wav;base64,UklGRg=="
       }
     ],
     "approvalPolicy": "on-request",

src/__tests__/CodexACPAgent/initialize.test.ts

@@ -45,7 +45,8 @@ describe('CodexACPAgent - initialize', () => {
                 loadSession: true,
                 promptCapabilities: {
                     embeddedContext: true,
-                    image: true
+                    image: true,
+                    audio: true
                 },
                 sessionCapabilities: {
                     resume: {},

src/__tests__/CodexACPAgent/steer-events.test.ts

@@ -50,7 +50,9 @@ describe('_session/steering', () => {
     });
 
     it('reports injected when the input joins the active turn', async () => {
-        const {mockFixture, sessionState, turnCompleted} = startActiveTurn();
+        const {mockFixture, sessionState, turnCompleted} = startActiveTurn({
+            supportedInputModalities: ["text", "image", "audio"],
+        });
         const turnSteerSpy = vi.spyOn(mockFixture.getCodexAppServerClient(), "turnSteer")
             .mockResolvedValue({turnId: "turn-id"});
 
@@ -64,13 +66,19 @@ describe('_session/steering', () => {
 
         await expect(mockFixture.getCodexAcpAgent().extMethod(SESSION_STEERING_METHOD, {
             sessionId: "session-id",
-            prompt: [{type: "text", text: "also keep backward compatibility"}],
+            prompt: [
+                {type: "text", text: "also keep backward compatibility"},
+                {type: "audio", mimeType: "audio/webm", data: "T2dnUw=="},
+            ],
         })).resolves.toEqual({outcome: "injected"});
 
         expect(turnSteerSpy).toHaveBeenCalledWith({
             threadId: "session-id",
             expectedTurnId: "turn-id",
-            input: [{type: "text", text: "also keep backward compatibility", text_elements: []}],
+            input: [
+                {type: "text", text: "also keep backward compatibility", text_elements: []},
+                {type: "audio", url: "data:audio/webm;base64,T2dnUw=="},
+            ],
         });
 
         turnCompleted.resolve({
@@ -231,4 +239,24 @@ describe('_session/steering', () => {
         expect((error as RequestError).data).toContain("does not support image input");
         expect(turnSteerSpy).not.toHaveBeenCalled();
     });
+
+    it('rejects audio input when the model does not support it', async () => {
+        const {mockFixture} = startActiveTurn({supportedInputModalities: ["text", "image"]});
+        const turnSteerSpy = vi.spyOn(mockFixture.getCodexAppServerClient(), "turnSteer");
+
+        const audio: acp.ContentBlock = {
+            type: "audio",
+            mimeType: "audio/wav",
+            data: "UklGRg==",
+        };
+
+        const error = await mockFixture.getCodexAcpAgent().extMethod(SESSION_STEERING_METHOD, {
+            sessionId: "session-id",
+            prompt: [audio],
+        }).catch((err: unknown) => err);
+
+        expect(error).toBeInstanceOf(RequestError);
+        expect((error as RequestError).data).toContain("does not support audio input");
+        expect(turnSteerSpy).not.toHaveBeenCalled();
+    });
 });