From cb608975f60ac6c934d854b992c12246b52f0155 Mon Sep 17 00:00:00 2001 From: Dawid Bednarczyk Date: Sun, 15 Mar 2026 23:52:50 +0100 Subject: [PATCH] fix: add retry circuit breaker and backoff cap to prevent infinite retry loops When API errors trigger retries with response headers present but no retry-after header, the exponential backoff grows without bound (observed 202s+ delays in production). Combined with the while(true) loop in processor.ts having no exit condition, this causes sessions to hang indefinitely burning CPU and tokens. Changes: - Add RETRY_MAX_ATTEMPTS (10) to cap total retry count - Add RETRY_MAX_DELAY_WITH_HEADERS (60s) to cap backoff when headers are present but missing retry-after - Add circuit breaker in processor.ts that breaks the retry loop after max attempts, publishes error event, and sets session to idle Validated against production logs showing 11 retries over 542 seconds with AI_APICallError: Could not relay message upstream. Fixes #17648 --- packages/opencode/src/session/processor.ts | 10 ++++++++++ packages/opencode/src/session/retry.ts | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 38dac41b0589..968dee0259ed 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -367,6 +367,16 @@ export namespace SessionProcessor { const retry = SessionRetry.retryable(error) if (retry !== undefined) { attempt++ + if (attempt > SessionRetry.RETRY_MAX_ATTEMPTS) { + log.error("max retries exceeded", { attempt, sessionID: input.sessionID }) + input.assistantMessage.error = error + Bus.publish(Session.Event.Error, { + sessionID: input.sessionID, + error, + }) + SessionStatus.set(input.sessionID, { type: "idle" }) + break + } const delay = SessionRetry.delay(attempt, error.name === "APIError" ? error : undefined) SessionStatus.set(input.sessionID, { type: "retry", diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 6d057f539f81..b06a768b3ff5 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -6,7 +6,9 @@ export namespace SessionRetry { export const RETRY_INITIAL_DELAY = 2000 export const RETRY_BACKOFF_FACTOR = 2 export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds + export const RETRY_MAX_DELAY_WITH_HEADERS = 60_000 // 60 seconds — cap for when response headers are present but missing retry-after export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout + export const RETRY_MAX_ATTEMPTS = 10 // maximum retry attempts before giving up export async function sleep(ms: number, signal: AbortSignal): Promise { return new Promise((resolve, reject) => { @@ -51,7 +53,7 @@ export namespace SessionRetry { } } - return RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1) + return Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_WITH_HEADERS) } }