Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,13 @@ To bring your own key with a custom model manually:
Full worked example (custom model parameters + API key) and protocol details:
[Configuration](./docs/configuration.md#bring-your-own-api-key).

## Slow Providers

Provider responses are not subject to an application-level response timeout, so
local models may take as long as necessary to load or generate output. Provider
connections still have an internal connection-establishment deadline, and every
request can be cancelled by the user.

## Docs

- [Offline Installation](./docs/offline-installation.md)
Expand Down
5 changes: 5 additions & 0 deletions README.zh-Hans.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,11 @@ devo resume <session-id>
完整示例(自定义模型参数 + API key)与协议说明见
[配置](./docs/configuration.zh-Hans.md#接入自有-api-key)。

## 响应较慢的 Provider

Provider 响应没有应用层总超时,因此本地模型可以按需要花费时间加载或生成输出。
Provider 连接建立仍有内部期限,并且用户可以随时取消请求。

## Docs

- [离线安装](./docs/offline-installation.zh-Hans.md)
Expand Down
2 changes: 2 additions & 0 deletions apps/desktop/src/main/native-stdio-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,10 +182,12 @@ describe("StdioNativeClient", () => {
sessionList: requestTimeoutMsForMethod("session/list", 10_000),
mcpTools: requestTimeoutMsForMethod("mcp/tools", 10_000),
mcpSetEnabled: requestTimeoutMsForMethod("mcp/set_enabled", 5),
providerValidate: requestTimeoutMsForMethod("provider/validate", 10_000),
}).toEqual({
sessionList: 10_000,
mcpTools: 60_000,
mcpSetEnabled: 60_000,
providerValidate: undefined,
})
})

Expand Down
29 changes: 19 additions & 10 deletions apps/desktop/src/main/native-stdio-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,17 @@ export type JsonRpcId = number | string
type PendingRequest = {
resolve: (value: unknown) => void
reject: (error: Error) => void
timer: ReturnType<typeof setTimeout>
timer?: ReturnType<typeof setTimeout>
}

const REQUEST_TIMEOUT_MS = 10_000
/** MCP admin RPCs may start a lazy server before listing tools. */
export const MCP_ADMIN_REQUEST_TIMEOUT_MS = 60_000

export function requestTimeoutMsForMethod(method: string, fallbackMs: number): number {
export function requestTimeoutMsForMethod(method: string, fallbackMs: number): number | undefined {
if (method === "provider/validate") {
return undefined
}
if (method === "mcp/tools" || method === "mcp/set_enabled") {
return Math.max(fallbackMs, MCP_ADMIN_REQUEST_TIMEOUT_MS)
}
Expand Down Expand Up @@ -295,11 +298,17 @@ export class StdioNativeClient implements NativeTransport {
const scopedParams = scopeRequestParams(method, params, directory)
const payload = { jsonrpc: "2.0", id, method, params: scopedParams }
const response = new Promise<unknown>((resolve, reject) => {
const timer = setTimeout(() => {
if (!this.pending.delete(id)) return
this.pendingMethods.delete(id)
reject(new Error(`${method} request ${id} timed out`))
}, requestTimeoutMsForMethod(method, this.options.requestTimeoutMs ?? REQUEST_TIMEOUT_MS))
const timeoutMs = requestTimeoutMsForMethod(
method,
this.options.requestTimeoutMs ?? REQUEST_TIMEOUT_MS,
)
const timer = timeoutMs === undefined
? undefined
: setTimeout(() => {
if (!this.pending.delete(id)) return
this.pendingMethods.delete(id)
reject(new Error(`${method} request ${id} timed out`))
}, timeoutMs)
this.pending.set(id, { resolve, reject, timer })
})
this.pendingMethods.set(id, method)
Expand All @@ -315,7 +324,7 @@ export class StdioNativeClient implements NativeTransport {
} catch (error) {
const reason = toError(error)
const pending = this.pending.get(id)
if (pending) clearTimeout(pending.timer)
if (pending?.timer !== undefined) clearTimeout(pending.timer)
this.pending.delete(id)
this.pendingMethods.delete(id)
this.close(reason)
Expand Down Expand Up @@ -399,7 +408,7 @@ export class StdioNativeClient implements NativeTransport {
const pending = this.pending.get(routed.id)
if (!pending) return
this.pending.delete(routed.id)
clearTimeout(pending.timer)
if (pending.timer !== undefined) clearTimeout(pending.timer)
const error = routed.message.error as { message?: string } | undefined
if (error) {
pending.reject(new Error(error.message ?? "Devo Native request failed"))
Expand Down Expand Up @@ -485,7 +494,7 @@ export class StdioNativeClient implements NativeTransport {
payload: { error: error.message },
})
for (const pending of this.pending.values()) {
clearTimeout(pending.timer)
if (pending.timer !== undefined) clearTimeout(pending.timer)
pending.reject(error)
}
this.pending.clear()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,27 @@ describe("ProviderSettings", () => {
expect(calls).toEqual(["validate"])
})

test("cancelled validation does not upsert", async () => {
const calls: string[] = []
const params = buildProviderUpsertParams(formValues, null)
const client = {
provider: {
validate: async () => {
calls.push("validate")
return { data: { reply_preview: "OK" } }
},
upsert: async () => {
calls.push("upsert")
return { data: { provider_vendor: providerVendor } }
},
},
}

const cancelled = true
await saveProviderVendor(client, params, () => !cancelled)
expect(calls).toEqual([])
})

test("provider dialog scrolls form body while keeping footer actions outside", () => {
const queryClient = new QueryClient()
const markup = renderToStaticMarkup(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import { Spinner } from "@devo/ui/components/spinner"
import { Textarea } from "@devo/ui/components/textarea"
import { useQueryClient } from "@tanstack/react-query"
import { SaveIcon } from "lucide-react"
import { useCallback, useEffect, useState } from "react"
import { useCallback, useEffect, useRef, useState } from "react"
import { queryKeys } from "../../hooks/use-devo-data"
import { createLogger } from "../../lib/logger"
import { getBaseClient, invalidateConfigOptionCaches } from "../../services/connection-manager"
Expand Down Expand Up @@ -170,16 +170,19 @@ export function buildProviderUpsertParams(
export async function saveProviderVendor(
client: ProviderVendorClient,
params: ProviderVendorUpsertParams,
shouldContinue: () => boolean = () => true,
) {
if (!params.model_binding) {
throw new Error("Model binding is required")
}
if (!shouldContinue()) return
const validateParams: ProviderValidateParams = {
provider_vendor: params.provider_vendor,
model_binding: params.model_binding,
...(params.api_key ? { api_key: params.api_key } : {}),
}
await client.provider.validate(validateParams)
if (!shouldContinue()) return
return client.provider.upsert(params)
}

Expand All @@ -193,8 +196,10 @@ export function ProviderVendorDialog({
const [values, setValues] = useState<ProviderVendorFormValues>(() => initialValues(providerVendor))
const [error, setError] = useState<string | null>(null)
const [saving, setSaving] = useState(false)
const saveAttemptRef = useRef(0)

useEffect(() => {
saveAttemptRef.current += 1
if (!open) return
setValues(initialValues(providerVendor))
setError(null)
Expand All @@ -211,13 +216,15 @@ export function ProviderVendorDialog({
const handleSubmit = useCallback(
async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault()
const saveAttempt = ++saveAttemptRef.current
setSaving(true)
setError(null)
try {
const client = getBaseClient()
if (!client) throw new Error("Not connected to server")
const params = buildProviderUpsertParams(values, providerVendor)
await saveProviderVendor(client, params)
await saveProviderVendor(client, params, () => saveAttemptRef.current === saveAttempt)
if (saveAttemptRef.current !== saveAttempt) return
invalidateConfigOptionCaches()
queryClient.invalidateQueries({ queryKey: queryKeys.providerVendors })
queryClient.invalidateQueries({
Expand All @@ -226,18 +233,32 @@ export function ProviderVendorDialog({
onSaved()
onOpenChange(false)
} catch (err) {
if (saveAttemptRef.current !== saveAttempt) return
const message = err instanceof Error ? err.message : "Failed to save provider"
log.error("Failed to save provider", { error: err })
setError(message)
} finally {
setSaving(false)
if (saveAttemptRef.current === saveAttempt) setSaving(false)
}
},
[values, providerVendor, queryClient, onSaved, onOpenChange],
)

const handleDialogOpenChange = useCallback(
(nextOpen: boolean) => {
if (!nextOpen) {
// Closing while validation is pending invalidates the continuation so
// a late validation response cannot persist the provider.
saveAttemptRef.current += 1
setSaving(false)
}
onOpenChange(nextOpen)
},
[onOpenChange],
)

return (
<Dialog open={open} onOpenChange={onOpenChange}>
<Dialog open={open} onOpenChange={handleDialogOpenChange}>
<DialogContent className="flex max-h-[calc(100dvh-2rem)] overflow-hidden p-0 sm:max-w-xl">
<form onSubmit={handleSubmit} className="flex min-h-0 flex-col">
<DialogHeader className="px-6 pt-6 pb-4">
Expand Down Expand Up @@ -399,7 +420,7 @@ export function ProviderVendorDialog({
className="shrink-0 bg-background px-6 py-4"
data-testid="provider-dialog-footer"
>
<Button type="button" variant="outline" onClick={() => onOpenChange(false)} disabled={saving}>
<Button type="button" variant="outline" onClick={() => handleDialogOpenChange(false)}>
Cancel
</Button>
<Button type="submit" disabled={saving}>
Expand Down
4 changes: 4 additions & 0 deletions apps/web/content/docs/reference/config-reference.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ Supported wire API values:
- `openai_chat_completions`
- `openai_responses`

Provider responses do not have an application-level response timeout. Slow local
models may take as long as needed to load or generate output; users can cancel a
request at any time. Connection establishment retains an internal deadline.

## `[model_bindings.<id>]`

| Field | Type | Meaning |
Expand Down
3 changes: 3 additions & 0 deletions apps/web/content/docs/reference/config-reference.zh.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ Devo server methods。
- `openai_chat_completions`
- `openai_responses`

Provider 响应没有应用层总超时。本地模型可以按需要花费时间加载或生成输出,
用户可以随时取消请求;连接建立仍保留内部期限。

## `[model_bindings.<id>]`

| Field | Type | Meaning |
Expand Down
68 changes: 56 additions & 12 deletions crates/client/src/client_core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ use crate::native_approval::resolve_approval_response;

const SERVER_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10);

#[derive(Clone, Copy)]
enum ServerResponseWait {
Standard,
Unbounded,
}

/// Synthetic notifications emitted when falling back to detached `session/prompt`.
#[derive(Debug, Clone, PartialEq)]
pub struct ServerNotificationMessage {
Expand Down Expand Up @@ -358,6 +364,29 @@ impl ServerClientCore {
}

pub(crate) async fn request<P, R>(&mut self, method: &str, params: P) -> Result<R>
where
P: Serialize,
R: DeserializeOwned,
{
self.request_with_wait(method, params, ServerResponseWait::Standard)
.await
}

async fn request_without_timeout<P, R>(&mut self, method: &str, params: P) -> Result<R>
where
P: Serialize,
R: DeserializeOwned,
{
self.request_with_wait(method, params, ServerResponseWait::Unbounded)
.await
}

async fn request_with_wait<P, R>(
&mut self,
method: &str,
params: P,
wait: ServerResponseWait,
) -> Result<R>
where
P: Serialize,
R: DeserializeOwned,
Expand All @@ -373,18 +402,32 @@ impl ServerClientCore {
return Err(error);
}

let response = match timeout(SERVER_RESPONSE_TIMEOUT, response_rx).await {
Ok(Ok(response)) => response,
Ok(Err(error)) => {
self.pending.lock().await.remove(&request_id);
return Err(error)
.with_context(|| format!("server dropped response for request {request_id}"));
}
Err(error) => {
self.pending.lock().await.remove(&request_id);
return Err(error)
.with_context(|| format!("{method} request {request_id} timed out"));
let response = match wait {
ServerResponseWait::Standard => {
match timeout(SERVER_RESPONSE_TIMEOUT, response_rx).await {
Ok(Ok(response)) => response,
Ok(Err(error)) => {
self.pending.lock().await.remove(&request_id);
return Err(error).with_context(|| {
format!("server dropped response for request {request_id}")
});
}
Err(error) => {
self.pending.lock().await.remove(&request_id);
return Err(error)
.with_context(|| format!("{method} request {request_id} timed out"));
}
}
}
ServerResponseWait::Unbounded => match response_rx.await {
Ok(response) => response,
Err(error) => {
self.pending.lock().await.remove(&request_id);
return Err(error).with_context(|| {
format!("server dropped response for request {request_id}")
});
}
},
};
if response.get("error").is_some() {
bail_server_error(&response)?;
Expand Down Expand Up @@ -1021,7 +1064,8 @@ impl ServerClientCore {
&mut self,
params: devo_protocol::native::rpc_admin::ProviderValidateParams,
) -> Result<devo_protocol::native::rpc_admin::ProviderValidateResult> {
self.request("provider/validate", params).await
self.request_without_timeout("provider/validate", params)
.await
}

pub(crate) async fn command_exec(
Expand Down
8 changes: 0 additions & 8 deletions crates/core/src/query/provider_retry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,6 @@ pub(crate) fn classify_error(e: &anyhow::Error) -> ErrorClass {
}
}

if e.chain().any(|cause| {
cause
.downcast_ref::<devo_provider::timeout::StreamIdleTimeoutError>()
.is_some()
}) {
return ErrorClass::NetworkError;
}

if e.chain().any(|cause| {
cause.downcast_ref::<reqwest::Error>().is_some_and(|error| {
error.is_timeout()
Expand Down
13 changes: 0 additions & 13 deletions crates/core/src/query/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,19 +165,6 @@ fn network_errors_are_retryable() {
message: "provider request timed out".into(),
provider_name: Some("test-provider".into()),
}),
anyhow::Error::new(devo_provider::timeout::stream_idle_timeout_provider_error(
"openai",
"gpt-test",
devo_provider::timeout::StreamIdleTimeoutError {
idle_timeout: std::time::Duration::from_secs(60),
},
)),
anyhow::Error::new(devo_provider::timeout::StreamIdleTimeoutError {
idle_timeout: std::time::Duration::from_secs(60),
}),
anyhow::anyhow!(
"openai stream idle timeout for model gpt-test: provider stream idle timeout after 60s without receiving data"
),
];

for error in cases {
Expand Down
Loading
Loading