mirror of
https://github.com/aljazceru/opencode.git
synced 2025-12-19 16:54:22 +01:00
better retry display
This commit is contained in:
@@ -4,7 +4,7 @@
|
|||||||
"provider": {
|
"provider": {
|
||||||
"opencode": {
|
"opencode": {
|
||||||
"options": {
|
"options": {
|
||||||
// "baseURL": "http://localhost:8080"
|
"baseURL": "http://localhost:8080",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import {
|
|||||||
For,
|
For,
|
||||||
Match,
|
Match,
|
||||||
on,
|
on,
|
||||||
|
onCleanup,
|
||||||
|
onMount,
|
||||||
Show,
|
Show,
|
||||||
Switch,
|
Switch,
|
||||||
useContext,
|
useContext,
|
||||||
@@ -972,11 +974,32 @@ function AssistantMessage(props: { message: AssistantMessage; parts: Part[]; las
|
|||||||
<box paddingLeft={3} flexDirection="row" gap={1} marginTop={1}>
|
<box paddingLeft={3} flexDirection="row" gap={1} marginTop={1}>
|
||||||
<text fg={local.agent.color(props.message.mode)}>{Locale.titlecase(props.message.mode)}</text>
|
<text fg={local.agent.color(props.message.mode)}>{Locale.titlecase(props.message.mode)}</text>
|
||||||
<Shimmer text={props.message.modelID} color={theme.text} />
|
<Shimmer text={props.message.modelID} color={theme.text} />
|
||||||
<Show when={status().type === "retry"}>
|
{(() => {
|
||||||
|
const retry = createMemo(() => {
|
||||||
|
const s = status()
|
||||||
|
if (s.type !== "retry") return
|
||||||
|
return s
|
||||||
|
})
|
||||||
|
const [seconds, setSeconds] = createSignal(0)
|
||||||
|
onMount(() => {
|
||||||
|
const timer = setInterval(() => {
|
||||||
|
const next = retry()?.next
|
||||||
|
if (next) setSeconds(Math.round((next - Date.now()) / 1000))
|
||||||
|
}, 1000)
|
||||||
|
|
||||||
|
onCleanup(() => {
|
||||||
|
clearInterval(timer)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
return (
|
||||||
|
<Show when={retry()}>
|
||||||
<text fg={theme.error}>
|
<text fg={theme.error}>
|
||||||
{(status() as any).message} [attempt #{(status() as any).attempt}]
|
{retry()!.message} [attempt #{retry()!.attempt}
|
||||||
|
{seconds() > 0 ? `, retrying in ${seconds()}s` : ""}]
|
||||||
</text>
|
</text>
|
||||||
</Show>
|
</Show>
|
||||||
|
)
|
||||||
|
})()}
|
||||||
</box>
|
</box>
|
||||||
</Show>
|
</Show>
|
||||||
<Show
|
<Show
|
||||||
|
|||||||
@@ -325,17 +325,16 @@ export namespace SessionProcessor {
|
|||||||
const error = MessageV2.fromError(e, { providerID: input.providerID })
|
const error = MessageV2.fromError(e, { providerID: input.providerID })
|
||||||
if (error?.name === "APIError" && error.data.isRetryable) {
|
if (error?.name === "APIError" && error.data.isRetryable) {
|
||||||
attempt++
|
attempt++
|
||||||
const delay = SessionRetry.getRetryDelayInMs(error, attempt)
|
const delay = SessionRetry.delay(error, attempt)
|
||||||
if (delay) {
|
|
||||||
SessionStatus.set(input.sessionID, {
|
SessionStatus.set(input.sessionID, {
|
||||||
type: "retry",
|
type: "retry",
|
||||||
attempt,
|
attempt,
|
||||||
message: error.data.message,
|
message: error.data.message,
|
||||||
|
next: Date.now() + delay,
|
||||||
})
|
})
|
||||||
await SessionRetry.sleep(delay, input.abort).catch(() => {})
|
await SessionRetry.sleep(delay, input.abort).catch(() => {})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
|
||||||
input.assistantMessage.error = error
|
input.assistantMessage.error = error
|
||||||
Bus.publish(Session.Event.Error, {
|
Bus.publish(Session.Event.Error, {
|
||||||
sessionID: input.assistantMessage.sessionID,
|
sessionID: input.assistantMessage.sessionID,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { MessageV2 } from "./message-v2"
|
|||||||
export namespace SessionRetry {
|
export namespace SessionRetry {
|
||||||
export const RETRY_INITIAL_DELAY = 2000
|
export const RETRY_INITIAL_DELAY = 2000
|
||||||
export const RETRY_BACKOFF_FACTOR = 2
|
export const RETRY_BACKOFF_FACTOR = 2
|
||||||
export const RETRY_MAX_DELAY = 600_000 // 10 minutes
|
export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
|
||||||
|
|
||||||
export async function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
export async function sleep(ms: number, signal: AbortSignal): Promise<void> {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
@@ -20,8 +20,7 @@ export namespace SessionRetry {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getRetryDelayInMs(error: MessageV2.APIError, attempt: number) {
|
export function delay(error: MessageV2.APIError, attempt: number) {
|
||||||
const delay = iife(() => {
|
|
||||||
const headers = error.data.responseHeaders
|
const headers = error.data.responseHeaders
|
||||||
if (headers) {
|
if (headers) {
|
||||||
const retryAfterMs = headers["retry-after-ms"]
|
const retryAfterMs = headers["retry-after-ms"]
|
||||||
@@ -45,32 +44,10 @@ export namespace SessionRetry {
|
|||||||
return Math.ceil(parsed)
|
return Math.ceil(parsed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1)
|
return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1)
|
||||||
})
|
|
||||||
|
|
||||||
// dont retry if wait is too far from now
|
|
||||||
if (delay > RETRY_MAX_DELAY) return undefined
|
|
||||||
|
|
||||||
return delay
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getBoundedDelay(input: {
|
return Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS)
|
||||||
error: MessageV2.APIError
|
|
||||||
attempt: number
|
|
||||||
startTime: number
|
|
||||||
maxDuration?: number
|
|
||||||
}) {
|
|
||||||
const elapsed = Date.now() - input.startTime
|
|
||||||
const maxDuration = input.maxDuration ?? RETRY_MAX_DELAY
|
|
||||||
const remaining = maxDuration - elapsed
|
|
||||||
|
|
||||||
if (remaining <= 0) return undefined
|
|
||||||
|
|
||||||
const delay = getRetryDelayInMs(input.error, input.attempt)
|
|
||||||
if (!delay) return undefined
|
|
||||||
|
|
||||||
return Math.min(delay, remaining)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export namespace SessionStatus {
|
|||||||
type: z.literal("retry"),
|
type: z.literal("retry"),
|
||||||
attempt: z.number(),
|
attempt: z.number(),
|
||||||
message: z.string(),
|
message: z.string(),
|
||||||
|
next: z.number(),
|
||||||
}),
|
}),
|
||||||
z.object({
|
z.object({
|
||||||
type: z.literal("busy"),
|
type: z.literal("busy"),
|
||||||
|
|||||||
@@ -10,163 +10,52 @@ function apiError(headers?: Record<string, string>): MessageV2.APIError {
|
|||||||
}).toObject() as MessageV2.APIError
|
}).toObject() as MessageV2.APIError
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("session.retry.getRetryDelayInMs", () => {
|
describe("session.retry.delay", () => {
|
||||||
test("doubles delay on each attempt when headers missing", () => {
|
test("caps delay at 30 seconds when headers missing", () => {
|
||||||
const error = apiError()
|
const error = apiError()
|
||||||
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.getRetryDelayInMs(error, index + 1))
|
const delays = Array.from({ length: 10 }, (_, index) => SessionRetry.delay(error, index + 1))
|
||||||
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000, 512000, undefined])
|
expect(delays).toStrictEqual([2000, 4000, 8000, 16000, 30000, 30000, 30000, 30000, 30000, 30000])
|
||||||
})
|
})
|
||||||
|
|
||||||
test("prefers retry-after-ms when shorter than exponential", () => {
|
test("prefers retry-after-ms when shorter than exponential", () => {
|
||||||
const error = apiError({ "retry-after-ms": "1500" })
|
const error = apiError({ "retry-after-ms": "1500" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 4)).toBe(1500)
|
expect(SessionRetry.delay(error, 4)).toBe(1500)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("uses retry-after seconds when reasonable", () => {
|
test("uses retry-after seconds when reasonable", () => {
|
||||||
const error = apiError({ "retry-after": "30" })
|
const error = apiError({ "retry-after": "30" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 3)).toBe(30000)
|
expect(SessionRetry.delay(error, 3)).toBe(30000)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("accepts http-date retry-after values", () => {
|
test("accepts http-date retry-after values", () => {
|
||||||
const date = new Date(Date.now() + 20000).toUTCString()
|
const date = new Date(Date.now() + 20000).toUTCString()
|
||||||
const error = apiError({ "retry-after": date })
|
const error = apiError({ "retry-after": date })
|
||||||
const delay = SessionRetry.getRetryDelayInMs(error, 1)
|
const d = SessionRetry.delay(error, 1)
|
||||||
expect(delay).toBeGreaterThanOrEqual(19000)
|
expect(d).toBeGreaterThanOrEqual(19000)
|
||||||
expect(delay).toBeLessThanOrEqual(20000)
|
expect(d).toBeLessThanOrEqual(20000)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("ignores invalid retry hints", () => {
|
test("ignores invalid retry hints", () => {
|
||||||
const error = apiError({ "retry-after": "not-a-number" })
|
const error = apiError({ "retry-after": "not-a-number" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
|
expect(SessionRetry.delay(error, 1)).toBe(2000)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("ignores malformed date retry hints", () => {
|
test("ignores malformed date retry hints", () => {
|
||||||
const error = apiError({ "retry-after": "Invalid Date String" })
|
const error = apiError({ "retry-after": "Invalid Date String" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
|
expect(SessionRetry.delay(error, 1)).toBe(2000)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("ignores past date retry hints", () => {
|
test("ignores past date retry hints", () => {
|
||||||
const pastDate = new Date(Date.now() - 5000).toUTCString()
|
const pastDate = new Date(Date.now() - 5000).toUTCString()
|
||||||
const error = apiError({ "retry-after": pastDate })
|
const error = apiError({ "retry-after": pastDate })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(2000)
|
expect(SessionRetry.delay(error, 1)).toBe(2000)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("returns undefined when delay exceeds 10 minutes", () => {
|
test("returns undefined when retry-after exceeds 10 minutes with headers", () => {
|
||||||
const error = apiError()
|
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 10)).toBeUndefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns undefined when retry-after exceeds 10 minutes", () => {
|
|
||||||
const error = apiError({ "retry-after": "50" })
|
const error = apiError({ "retry-after": "50" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(error, 1)).toBe(50000)
|
expect(SessionRetry.delay(error, 1)).toBe(50000)
|
||||||
|
|
||||||
const longError = apiError({ "retry-after-ms": "700000" })
|
const longError = apiError({ "retry-after-ms": "700000" })
|
||||||
expect(SessionRetry.getRetryDelayInMs(longError, 1)).toBeUndefined()
|
expect(SessionRetry.delay(longError, 1)).toBeUndefined()
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe("session.retry.getBoundedDelay", () => {
|
|
||||||
test("returns full delay when under time budget", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now()
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBe(2000)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns remaining time when delay exceeds budget", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now() - 598_000 // 598 seconds elapsed, 2 seconds remaining
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeGreaterThanOrEqual(1900)
|
|
||||||
expect(delay).toBeLessThanOrEqual(2100)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns undefined when time budget exhausted", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now() - 600_000 // exactly 10 minutes elapsed
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeUndefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns undefined when time budget exceeded", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now() - 700_000 // 11+ minutes elapsed
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeUndefined()
|
|
||||||
})
|
|
||||||
|
|
||||||
test("respects custom maxDuration", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now() - 58_000 // 58 seconds elapsed
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
maxDuration: 60_000, // 1 minute max
|
|
||||||
})
|
|
||||||
expect(delay).toBeGreaterThanOrEqual(1900)
|
|
||||||
expect(delay).toBeLessThanOrEqual(2100)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("caps exponential backoff to remaining time", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now() - 595_000 // 595 seconds elapsed, 5 seconds remaining
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 5, // would normally be 32 seconds
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeGreaterThanOrEqual(4900)
|
|
||||||
expect(delay).toBeLessThanOrEqual(5100)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("respects server retry-after within budget", () => {
|
|
||||||
const error = apiError({ "retry-after": "30" })
|
|
||||||
const startTime = Date.now() - 550_000 // 550 seconds elapsed, 50 seconds remaining
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBe(30000)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("caps server retry-after to remaining time", () => {
|
|
||||||
const error = apiError({ "retry-after": "30" })
|
|
||||||
const startTime = Date.now() - 590_000 // 590 seconds elapsed, 10 seconds remaining
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 1,
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeGreaterThanOrEqual(9900)
|
|
||||||
expect(delay).toBeLessThanOrEqual(10100)
|
|
||||||
})
|
|
||||||
|
|
||||||
test("returns undefined when getRetryDelayInMs returns undefined", () => {
|
|
||||||
const error = apiError()
|
|
||||||
const startTime = Date.now()
|
|
||||||
const delay = SessionRetry.getBoundedDelay({
|
|
||||||
error,
|
|
||||||
attempt: 10, // exceeds RETRY_MAX_DELAY
|
|
||||||
startTime,
|
|
||||||
})
|
|
||||||
expect(delay).toBeUndefined()
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -367,6 +367,15 @@ export type CompactionPart = {
|
|||||||
|
|
||||||
export type Part =
|
export type Part =
|
||||||
| TextPart
|
| TextPart
|
||||||
|
| {
|
||||||
|
id: string
|
||||||
|
sessionID: string
|
||||||
|
messageID: string
|
||||||
|
type: "subtask"
|
||||||
|
prompt: string
|
||||||
|
description: string
|
||||||
|
agent: string
|
||||||
|
}
|
||||||
| ReasoningPart
|
| ReasoningPart
|
||||||
| FilePart
|
| FilePart
|
||||||
| ToolPart
|
| ToolPart
|
||||||
@@ -425,6 +434,28 @@ export type EventPermissionReplied = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type SessionStatus =
|
||||||
|
| {
|
||||||
|
type: "idle"
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "retry"
|
||||||
|
attempt: number
|
||||||
|
message: string
|
||||||
|
next: number
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "busy"
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EventSessionStatus = {
|
||||||
|
type: "session.status"
|
||||||
|
properties: {
|
||||||
|
sessionID: string
|
||||||
|
status: SessionStatus
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export type EventSessionCompacted = {
|
export type EventSessionCompacted = {
|
||||||
type: "session.compacted"
|
type: "session.compacted"
|
||||||
properties: {
|
properties: {
|
||||||
@@ -476,27 +507,6 @@ export type EventCommandExecuted = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SessionStatus =
|
|
||||||
| {
|
|
||||||
type: "idle"
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "retry"
|
|
||||||
attempt: number
|
|
||||||
message: string
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
type: "busy"
|
|
||||||
}
|
|
||||||
|
|
||||||
export type EventSessionStatus = {
|
|
||||||
type: "session.status"
|
|
||||||
properties: {
|
|
||||||
sessionID: string
|
|
||||||
status: SessionStatus
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export type EventSessionIdle = {
|
export type EventSessionIdle = {
|
||||||
type: "session.idle"
|
type: "session.idle"
|
||||||
properties: {
|
properties: {
|
||||||
@@ -639,11 +649,11 @@ export type Event =
|
|||||||
| EventMessagePartRemoved
|
| EventMessagePartRemoved
|
||||||
| EventPermissionUpdated
|
| EventPermissionUpdated
|
||||||
| EventPermissionReplied
|
| EventPermissionReplied
|
||||||
|
| EventSessionStatus
|
||||||
| EventSessionCompacted
|
| EventSessionCompacted
|
||||||
| EventFileEdited
|
| EventFileEdited
|
||||||
| EventTodoUpdated
|
| EventTodoUpdated
|
||||||
| EventCommandExecuted
|
| EventCommandExecuted
|
||||||
| EventSessionStatus
|
|
||||||
| EventSessionIdle
|
| EventSessionIdle
|
||||||
| EventSessionCreated
|
| EventSessionCreated
|
||||||
| EventSessionUpdated
|
| EventSessionUpdated
|
||||||
@@ -1248,6 +1258,14 @@ export type AgentPartInput = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type SubtaskPartInput = {
|
||||||
|
id?: string
|
||||||
|
type: "subtask"
|
||||||
|
prompt: string
|
||||||
|
description: string
|
||||||
|
agent: string
|
||||||
|
}
|
||||||
|
|
||||||
export type Command = {
|
export type Command = {
|
||||||
name: string
|
name: string
|
||||||
description?: string
|
description?: string
|
||||||
@@ -2142,7 +2160,7 @@ export type SessionPromptData = {
|
|||||||
tools?: {
|
tools?: {
|
||||||
[key: string]: boolean
|
[key: string]: boolean
|
||||||
}
|
}
|
||||||
parts: Array<TextPartInput | FilePartInput | AgentPartInput>
|
parts: Array<TextPartInput | FilePartInput | AgentPartInput | SubtaskPartInput>
|
||||||
}
|
}
|
||||||
path: {
|
path: {
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user