better retry display

This commit is contained in:
Dax Raad
2025-11-17 11:30:55 -05:00
parent a5365ce294
commit 8b19c6c7e4
7 changed files with 118 additions and 211 deletions

View File

@@ -4,7 +4,7 @@
"provider": {
"opencode": {
"options": {
// "baseURL": "http://localhost:8080"
"baseURL": "http://localhost:8080",
},
},
},

View File

@@ -6,6 +6,8 @@ import {
For,
Match,
on,
onCleanup,
onMount,
Show,
Switch,
useContext,
@@ -972,11 +974,32 @@ function AssistantMessage(props: { message: AssistantMessage; parts: Part[]; las
<box paddingLeft={3} flexDirection="row" gap={1} marginTop={1}>
<text fg={local.agent.color(props.message.mode)}>{Locale.titlecase(props.message.mode)}</text>
<Shimmer text={props.message.modelID} color={theme.text} />
<Show when={status().type === "retry"}>
{(() => {
const retry = createMemo(() => {
const s = status()
if (s.type !== "retry") return
return s
})
const [seconds, setSeconds] = createSignal(0)
onMount(() => {
const timer = setInterval(() => {
const next = retry()?.next
if (next) setSeconds(Math.round((next - Date.now()) / 1000))
}, 1000)
onCleanup(() => {
clearInterval(timer)
})
})
return (
<Show when={retry()}>
<text fg={theme.error}>
{(status() as any).message} [attempt #{(status() as any).attempt}]
{retry()!.message} [attempt #{retry()!.attempt}
{seconds() > 0 ? `, retrying in ${seconds()}s` : ""}]
</text>
</Show>
)
})()}
</box>
</Show>
<Show

View File

@@ -325,17 +325,16 @@ export namespace SessionProcessor {
const error = MessageV2.fromError(e, { providerID: input.providerID })
if (error?.name === "APIError" && error.data.isRetryable) {
attempt++
const delay = SessionRetry.getRetryDelayInMs(error, attempt)
if (delay) {
const delay = SessionRetry.delay(error, attempt)
SessionStatus.set(input.sessionID, {
type: "retry",
attempt,
message: error.data.message,
next: Date.now() + delay,
})
await SessionRetry.sleep(delay, input.abort).catch(() => {})
continue
}
}
input.assistantMessage.error = error
Bus.publish(Session.Event.Error, {
sessionID: input.assistantMessage.sessionID,

View File

@@ -4,7 +4,7 @@ import { MessageV2 } from "./message-v2"
export namespace SessionRetry {
export const RETRY_INITIAL_DELAY = 2000
export const RETRY_BACKOFF_FACTOR = 2
export const RETRY_MAX_DELAY = 600_000 // 10 minutes
export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
export async function sleep(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
@@ -20,8 +20,7 @@ export namespace SessionRetry {
})
}
export function getRetryDelayInMs(error: MessageV2.APIError, attempt: number) {
const delay = iife(() => {
export function delay(error: MessageV2.APIError, attempt: number) {
const headers = error.data.responseHeaders
if (headers) {
const retryAfterMs = headers["retry-after-ms"]
@@ -45,32 +44,10 @@ export namespace SessionRetry {
return Math.ceil(parsed)
}
}
}
return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1)
})
// dont retry if wait is too far from now
if (delay > RETRY_MAX_DELAY) return undefined
return delay
}
export function getBoundedDelay(input: {
error: MessageV2.APIError
attempt: number
startTime: number
maxDuration?: number
}) {
const elapsed = Date.now() - input.startTime
const maxDuration = input.maxDuration ?? RETRY_MAX_DELAY
const remaining = maxDuration - elapsed
if (remaining <= 0) return undefined
const delay = getRetryDelayInMs(input.error, input.attempt)
if (!delay) return undefined
return Math.min(delay, remaining)
return Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS)
}
}

View File

@@ -12,6 +12,7 @@ export namespace SessionStatus {
type: z.literal("retry"),
attempt: z.number(),
message: z.string(),
next: z.number(),
}),
z.object({
type: z.literal("busy"),

View File

@@ -10,163 +10,52 @@ function apiError(headers?: Record<string, string>): MessageV2.APIError {
}).toObject() as MessageV2.APIError
}
describe("session.retry.getRetryDelayInMs", () => {
test("doubles delay on each attempt when headers missing", () => {
describe("session.retry.delay", () => {
test("caps delay at 30 seconds when headers missing", () => {
const error = apiError()
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.getRetryDelayInMs(error, index + 1))
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000, 512000, undefined])
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(error, index + 1))
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 30000, 30000, 30000, 30000, 30000, 30000])
})
test("prefers retry-after-ms when shorter than exponential", () => {
const error = apiError({ "retry-after-ms": "1500" })
expect(SessionRetry.getRetryDelayInMs(error, 4)).toBe(1500)
expect(SessionRetry.delay(error, 4)).toBe(1500)
})
test("uses retry-after seconds when reasonable", () => {
const error = apiError({ "retry-after": "30" })
expect(SessionRetry.getRetryDelayInMs(error, 3)).toBe(30000)
expect(SessionRetry.delay(error, 3)).toBe(30000)
})
test("accepts http-date retry-after values", () => {
const date = new Date(Date.now() + 20000).toUTCString()
const error = apiError({ "retry-after": date })
const delay = SessionRetry.getRetryDelayInMs(error, 1)
expect(delay).toBeGreaterThanOrEqual(19000)
expect(delay).toBeLessThanOrEqual(20000)
const d = SessionRetry.delay(error, 1)
expect(d).toBeGreaterThanOrEqual(19000)
expect(d).toBeLessThanOrEqual(20000)
})
test("ignores invalid retry hints", () => {
const error = apiError({ "retry-after": "not-a-number" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
expect(SessionRetry.delay(error, 1)).toBe(2000)
})
test("ignores malformed date retry hints", () => {
const error = apiError({ "retry-after": "Invalid Date String" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
expect(SessionRetry.delay(error, 1)).toBe(2000)
})
test("ignores past date retry hints", () => {
const pastDate = new Date(Date.now() - 5000).toUTCString()
const error = apiError({ "retry-after": pastDate })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
expect(SessionRetry.delay(error, 1)).toBe(2000)
})
test("returns undefined when delay exceeds 10 minutes", () => {
const error = apiError()
expect(SessionRetry.getRetryDelayInMs(error, 10)).toBeUndefined()
})
test("returns undefined when retry-after exceeds 10 minutes", () => {
test("returns undefined when retry-after exceeds 10 minutes with headers", () => {
const error = apiError({ "retry-after": "50" })
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(50000)
expect(SessionRetry.delay(error, 1)).toBe(50000)
const longError = apiError({ "retry-after-ms": "700000" })
expect(SessionRetry.getRetryDelayInMs(longError, 1)).toBeUndefined()
})
})
describe("session.retry.getBoundedDelay", () => {
test("returns full delay when under time budget", () => {
const error = apiError()
const startTime = Date.now()
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBe(2000)
})
test("returns remaining time when delay exceeds budget", () => {
const error = apiError()
const startTime = Date.now() - 598_000 // 598 seconds elapsed, 2 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeGreaterThanOrEqual(1900)
expect(delay).toBeLessThanOrEqual(2100)
})
test("returns undefined when time budget exhausted", () => {
const error = apiError()
const startTime = Date.now() - 600_000 // exactly 10 minutes elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeUndefined()
})
test("returns undefined when time budget exceeded", () => {
const error = apiError()
const startTime = Date.now() - 700_000 // 11+ minutes elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeUndefined()
})
test("respects custom maxDuration", () => {
const error = apiError()
const startTime = Date.now() - 58_000 // 58 seconds elapsed
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
maxDuration: 60_000, // 1 minute max
})
expect(delay).toBeGreaterThanOrEqual(1900)
expect(delay).toBeLessThanOrEqual(2100)
})
test("caps exponential backoff to remaining time", () => {
const error = apiError()
const startTime = Date.now() - 595_000 // 595 seconds elapsed, 5 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 5, // would normally be 32 seconds
startTime,
})
expect(delay).toBeGreaterThanOrEqual(4900)
expect(delay).toBeLessThanOrEqual(5100)
})
test("respects server retry-after within budget", () => {
const error = apiError({ "retry-after": "30" })
const startTime = Date.now() - 550_000 // 550 seconds elapsed, 50 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBe(30000)
})
test("caps server retry-after to remaining time", () => {
const error = apiError({ "retry-after": "30" })
const startTime = Date.now() - 590_000 // 590 seconds elapsed, 10 seconds remaining
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 1,
startTime,
})
expect(delay).toBeGreaterThanOrEqual(9900)
expect(delay).toBeLessThanOrEqual(10100)
})
test("returns undefined when getRetryDelayInMs returns undefined", () => {
const error = apiError()
const startTime = Date.now()
const delay = SessionRetry.getBoundedDelay({
error,
attempt: 10, // exceeds RETRY_MAX_DELAY
startTime,
})
expect(delay).toBeUndefined()
expect(SessionRetry.delay(longError, 1)).toBeUndefined()
})
})

View File

@@ -367,6 +367,15 @@ export type CompactionPart = {
export type Part =
| TextPart
| {
id: string
sessionID: string
messageID: string
type: "subtask"
prompt: string
description: string
agent: string
}
| ReasoningPart
| FilePart
| ToolPart
@@ -425,6 +434,28 @@ export type EventPermissionReplied = {
}
}
export type SessionStatus =
| {
type: "idle"
}
| {
type: "retry"
attempt: number
message: string
next: number
}
| {
type: "busy"
}
export type EventSessionStatus = {
type: "session.status"
properties: {
sessionID: string
status: SessionStatus
}
}
export type EventSessionCompacted = {
type: "session.compacted"
properties: {
@@ -476,27 +507,6 @@ export type EventCommandExecuted = {
}
}
export type SessionStatus =
| {
type: "idle"
}
| {
type: "retry"
attempt: number
message: string
}
| {
type: "busy"
}
export type EventSessionStatus = {
type: "session.status"
properties: {
sessionID: string
status: SessionStatus
}
}
export type EventSessionIdle = {
type: "session.idle"
properties: {
@@ -639,11 +649,11 @@ export type Event =
| EventMessagePartRemoved
| EventPermissionUpdated
| EventPermissionReplied
| EventSessionStatus
| EventSessionCompacted
| EventFileEdited
| EventTodoUpdated
| EventCommandExecuted
| EventSessionStatus
| EventSessionIdle
| EventSessionCreated
| EventSessionUpdated
@@ -1248,6 +1258,14 @@ export type AgentPartInput = {
}
}
export type SubtaskPartInput = {
id?: string
type: "subtask"
prompt: string
description: string
agent: string
}
export type Command = {
name: string
description?: string
@@ -2142,7 +2160,7 @@ export type SessionPromptData = {
tools?: {
[key: string]: boolean
}
parts: Array<TextPartInput | FilePartInput | AgentPartInput>
parts: Array<TextPartInput | FilePartInput | AgentPartInput | SubtaskPartInput>
}
path: {
/**