better retry display

This commit is contained in:
Dax Raad
2025-11-17 11:30:55 -05:00
parent a5365ce294
commit 8b19c6c7e4
7 changed files with 118 additions and 211 deletions

View File

@@ -4,7 +4,7 @@
"provider": { "provider": {
"opencode": { "opencode": {
"options": { "options": {
// "baseURL": "http://localhost:8080" "baseURL": "http://localhost:8080",
}, },
}, },
}, },

View File

@@ -6,6 +6,8 @@ import {
For, For,
Match, Match,
on, on,
onCleanup,
onMount,
Show, Show,
Switch, Switch,
useContext, useContext,
@@ -972,11 +974,32 @@ function AssistantMessage(props: { message: AssistantMessage; parts: Part[]; las
<box paddingLeft={3} flexDirection="row" gap={1} marginTop={1}> <box paddingLeft={3} flexDirection="row" gap={1} marginTop={1}>
<text fg={local.agent.color(props.message.mode)}>{Locale.titlecase(props.message.mode)}</text> <text fg={local.agent.color(props.message.mode)}>{Locale.titlecase(props.message.mode)}</text>
<Shimmer text={props.message.modelID} color={theme.text} /> <Shimmer text={props.message.modelID} color={theme.text} />
<Show when={status().type === "retry"}> {(() => {
const retry = createMemo(() => {
const s = status()
if (s.type !== "retry") return
return s
})
const [seconds, setSeconds] = createSignal(0)
onMount(() => {
const timer = setInterval(() => {
const next = retry()?.next
if (next) setSeconds(Math.round((next - Date.now()) / 1000))
}, 1000)
onCleanup(() => {
clearInterval(timer)
})
})
return (
<Show when={retry()}>
<text fg={theme.error}> <text fg={theme.error}>
{(status() as any).message} [attempt #{(status() as any).attempt}] {retry()!.message} [attempt #{retry()!.attempt}
{seconds() > 0 ? `, retrying in ${seconds()}s` : ""}]
</text> </text>
</Show> </Show>
)
})()}
</box> </box>
</Show> </Show>
<Show <Show

View File

@@ -325,17 +325,16 @@ export namespace SessionProcessor {
const error = MessageV2.fromError(e, { providerID: input.providerID }) const error = MessageV2.fromError(e, { providerID: input.providerID })
if (error?.name === "APIError" && error.data.isRetryable) { if (error?.name === "APIError" && error.data.isRetryable) {
attempt++ attempt++
const delay = SessionRetry.getRetryDelayInMs(error, attempt) const delay = SessionRetry.delay(error, attempt)
if (delay) {
SessionStatus.set(input.sessionID, { SessionStatus.set(input.sessionID, {
type: "retry", type: "retry",
attempt, attempt,
message: error.data.message, message: error.data.message,
next: Date.now() + delay,
}) })
await SessionRetry.sleep(delay, input.abort).catch(() => {}) await SessionRetry.sleep(delay, input.abort).catch(() => {})
continue continue
} }
}
input.assistantMessage.error = error input.assistantMessage.error = error
Bus.publish(Session.Event.Error, { Bus.publish(Session.Event.Error, {
sessionID: input.assistantMessage.sessionID, sessionID: input.assistantMessage.sessionID,

View File

@@ -4,7 +4,7 @@ import { MessageV2 } from "./message-v2"
export namespace SessionRetry { export namespace SessionRetry {
export const RETRY_INITIAL_DELAY = 2000 export const RETRY_INITIAL_DELAY = 2000
export const RETRY_BACKOFF_FACTOR = 2 export const RETRY_BACKOFF_FACTOR = 2
export const RETRY_MAX_DELAY = 600_000 // 10 minutes export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
export async function sleep(ms: number, signal: AbortSignal): Promise<void> { export async function sleep(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@@ -20,8 +20,7 @@ export namespace SessionRetry {
}) })
} }
export function getRetryDelayInMs(error: MessageV2.APIError, attempt: number) { export function delay(error: MessageV2.APIError, attempt: number) {
const delay = iife(() => {
const headers = error.data.responseHeaders const headers = error.data.responseHeaders
if (headers) { if (headers) {
const retryAfterMs = headers["retry-after-ms"] const retryAfterMs = headers["retry-after-ms"]
@@ -45,32 +44,10 @@ export namespace SessionRetry {
return Math.ceil(parsed) return Math.ceil(parsed)
} }
} }
}
return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1) return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1)
})
// dont retry if wait is too far from now
if (delay > RETRY_MAX_DELAY) return undefined
return delay
} }
export function getBoundedDelay(input: { return Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS)
error: MessageV2.APIError
attempt: number
startTime: number
maxDuration?: number
}) {
const elapsed = Date.now() - input.startTime
const maxDuration = input.maxDuration ?? RETRY_MAX_DELAY
const remaining = maxDuration - elapsed
if (remaining <= 0) return undefined
const delay = getRetryDelayInMs(input.error, input.attempt)
if (!delay) return undefined
return Math.min(delay, remaining)
} }
} }

View File

@@ -12,6 +12,7 @@ export namespace SessionStatus {
type: z.literal("retry"), type: z.literal("retry"),
attempt: z.number(), attempt: z.number(),
message: z.string(), message: z.string(),
next: z.number(),
}), }),
z.object({ z.object({
type: z.literal("busy"), type: z.literal("busy"),

View File

@@ -10,163 +10,52 @@ function apiError(headers?: Record<string, string>): MessageV2.APIError {
}).toObject() as MessageV2.APIError }).toObject() as MessageV2.APIError
} }
describe("session.retry.getRetryDelayInMs", () => { describe("session.retry.delay", () => {
test("doubles delay on each attempt when headers missing", () => { test("caps delay at 30 seconds when headers missing", () => {
const error = apiError() const error = apiError()
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.getRetryDelayInMs(error, index + 1)) const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(error, index + 1))
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000, 512000, undefined]) expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 30000, 30000, 30000, 30000, 30000, 30000])
}) })
test("prefers retry-after-ms when shorter than exponential", () => { test("prefers retry-after-ms when shorter than exponential", () => {
const error = apiError({ "retry-after-ms": "1500" }) const error = apiError({ "retry-after-ms": "1500" })
expect(SessionRetry.getRetryDelayInMs(error, 4)).toBe(1500) expect(SessionRetry.delay(error, 4)).toBe(1500)
}) })
test("uses retry-after seconds when reasonable", () => { test("uses retry-after seconds when reasonable", () => {
const error = apiError({ "retry-after": "30" }) const error = apiError({ "retry-after": "30" })
expect(SessionRetry.getRetryDelayInMs(error, 3)).toBe(30000) expect(SessionRetry.delay(error, 3)).toBe(30000)
}) })
test("accepts http-date retry-after values", () => { test("accepts http-date retry-after values", () => {
const date = new Date(Date.now() + 20000).toUTCString() const date = new Date(Date.now() + 20000).toUTCString()
const error = apiError({ "retry-after": date }) const error = apiError({ "retry-after": date })
const delay = SessionRetry.getRetryDelayInMs(error, 1) const d = SessionRetry.delay(error, 1)
expect(delay).toBeGreaterThanOrEqual(19000) expect(d).toBeGreaterThanOrEqual(19000)
expect(delay).toBeLessThanOrEqual(20000) expect(d).toBeLessThanOrEqual(20000)
}) })
test("ignores invalid retry hints", () => { test("ignores invalid retry hints", () => {
const error = apiError({ "retry-after": "not-a-number" }) const error = apiError({ "retry-after": "not-a-number" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000) expect(SessionRetry.delay(error, 1)).toBe(2000)
}) })
test("ignores malformed date retry hints", () => { test("ignores malformed date retry hints", () => {
const error = apiError({ "retry-after": "Invalid Date String" }) const error = apiError({ "retry-after": "Invalid Date String" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000) expect(SessionRetry.delay(error, 1)).toBe(2000)
}) })
test("ignores past date retry hints", () => { test("ignores past date retry hints", () => {
const pastDate = new Date(Date.now() - 5000).toUTCString() const pastDate = new Date(Date.now() - 5000).toUTCString()
const error = apiError({ "retry-after": pastDate }) const error = apiError({ "retry-after": pastDate })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000) expect(SessionRetry.delay(error, 1)).toBe(2000)
}) })
test("returns undefined when delay exceeds 10 minutes", () => { test("returns undefined when retry-after exceeds 10 minutes with headers", () => {
const error = apiError()
expect(SessionRetry.getRetryDelayInMs(error, 10)).toBeUndefined()
})
test("returns undefined when retry-after exceeds 10 minutes", () => {
const error = apiError({ "retry-after": "50" }) const error = apiError({ "retry-after": "50" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(50000) expect(SessionRetry.delay(error, 1)).toBe(50000)
const longError = apiError({ "retry-after-ms": "700000" }) const longError = apiError({ "retry-after-ms": "700000" })
expect(SessionRetry.getRetryDelayInMs(longError, 1)).toBeUndefined() expect(SessionRetry.delay(longError, 1)).toBeUndefined()
})
})
describe("session.retry.getBoundedDelay", () => {
test("returns full delay when under time budget", () => {
const error = apiError()
const startTime = Date.now()
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBe(2000)
})
test("returns remaining time when delay exceeds budget", () => {
const error = apiError()
const startTime = Date.now() - 598_000 // 598 seconds elapsed, 2 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeGreaterThanOrEqual(1900)
expect(delay).toBeLessThanOrEqual(2100)
})
test("returns undefined when time budget exhausted", () => {
const error = apiError()
const startTime = Date.now() - 600_000 // exactly 10 minutes elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeUndefined()
})
test("returns undefined when time budget exceeded", () => {
const error = apiError()
const startTime = Date.now() - 700_000 // 11+ minutes elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeUndefined()
})
test("respects custom maxDuration", () => {
const error = apiError()
const startTime = Date.now() - 58_000 // 58 seconds elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
maxDuration: 60_000, // 1 minute max
})
expect(delay).toBeGreaterThanOrEqual(1900)
expect(delay).toBeLessThanOrEqual(2100)
})
test("caps exponential backoff to remaining time", () => {
const error = apiError()
const startTime = Date.now() - 595_000 // 595 seconds elapsed, 5 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 5, // would normally be 32 seconds
startTime,
})
expect(delay).toBeGreaterThanOrEqual(4900)
expect(delay).toBeLessThanOrEqual(5100)
})
test("respects server retry-after within budget", () => {
const error = apiError({ "retry-after": "30" })
const startTime = Date.now() - 550_000 // 550 seconds elapsed, 50 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBe(30000)
})
test("caps server retry-after to remaining time", () => {
const error = apiError({ "retry-after": "30" })
const startTime = Date.now() - 590_000 // 590 seconds elapsed, 10 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeGreaterThanOrEqual(9900)
expect(delay).toBeLessThanOrEqual(10100)
})
test("returns undefined when getRetryDelayInMs returns undefined", () => {
const error = apiError()
const startTime = Date.now()
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 10, // exceeds RETRY_MAX_DELAY
startTime,
})
expect(delay).toBeUndefined()
}) })
}) })

View File

@@ -367,6 +367,15 @@ export type CompactionPart = {
export type Part = export type Part =
| TextPart | TextPart
| {
id: string
sessionID: string
messageID: string
type: "subtask"
prompt: string
description: string
agent: string
}
| ReasoningPart | ReasoningPart
| FilePart | FilePart
| ToolPart | ToolPart
@@ -425,6 +434,28 @@ export type EventPermissionReplied = {
} }
} }
export type SessionStatus =
| {
type: "idle"
}
| {
type: "retry"
attempt: number
message: string
next: number
}
| {
type: "busy"
}
export type EventSessionStatus = {
type: "session.status"
properties: {
sessionID: string
status: SessionStatus
}
}
export type EventSessionCompacted = { export type EventSessionCompacted = {
type: "session.compacted" type: "session.compacted"
properties: { properties: {
@@ -476,27 +507,6 @@ export type EventCommandExecuted = {
} }
} }
export type SessionStatus =
| {
type: "idle"
}
| {
type: "retry"
attempt: number
message: string
}
| {
type: "busy"
}
export type EventSessionStatus = {
type: "session.status"
properties: {
sessionID: string
status: SessionStatus
}
}
export type EventSessionIdle = { export type EventSessionIdle = {
type: "session.idle" type: "session.idle"
properties: { properties: {
@@ -639,11 +649,11 @@ export type Event =
| EventMessagePartRemoved | EventMessagePartRemoved
| EventPermissionUpdated | EventPermissionUpdated
| EventPermissionReplied | EventPermissionReplied
| EventSessionStatus
| EventSessionCompacted | EventSessionCompacted
| EventFileEdited | EventFileEdited
| EventTodoUpdated | EventTodoUpdated
| EventCommandExecuted | EventCommandExecuted
| EventSessionStatus
| EventSessionIdle | EventSessionIdle
| EventSessionCreated | EventSessionCreated
| EventSessionUpdated | EventSessionUpdated
@@ -1248,6 +1258,14 @@ export type AgentPartInput = {
} }
} }
export type SubtaskPartInput = {
id?: string
type: "subtask"
prompt: string
description: string
agent: string
}
export type Command = { export type Command = {
name: string name: string
description?: string description?: string
@@ -2142,7 +2160,7 @@ export type SessionPromptData = {
tools?: { tools?: {
[key: string]: boolean [key: string]: boolean
} }
parts: Array<TextPartInput | FilePartInput | AgentPartInput> parts: Array<TextPartInput | FilePartInput | AgentPartInput | SubtaskPartInput>
} }
path: { path: {
/** /**