Files
CLIProxyAPI/internal/runtime/executor/codex_websockets_stream.go
Viggo95 cb73cd9936 fix(codex): buffer keepalive and empty item announcements during codex bootstrap
The bootstrap buffer released the stream on the first frame outside the
handshake allow-list. Upstream sends `keepalive` heartbeats and
`response.output_item.added` before the first token, so the downstream headers
were committed while nothing had been produced, and an overload rejection
arriving afterwards could no longer be retried on another credential.

Extend the allow-list with `keepalive`, and with `response.output_item.added`,
`response.content_part.added` and `response.reasoning_summary_part.added` when
they announce something that is still empty. `response.output_item.added` is
accepted only for `message`, `reasoning`, `function_call` and `custom_tool_call`
items, and only while their content, summary, arguments or input is empty: every
other item type stands for a server-side operation that may already be running -
a `web_search_call` is announced with status "in_progress" and its `searching`
event follows immediately - and failing the attempt over after one would run
that operation again on another credential. Part announcements are matched on a
closed list of textual part types for the same reason.

The list stays closed. "Nothing has happened yet" cannot be derived from the
absence of a TTFT token, because TTFT deliberately ignores server-side tool
traffic such as `response.shell_call_output_content.delta` and its `.done`
counterpart, so an unrecognised frame, an unrecognised item type and an
unrecognised part type all release the stream exactly as before.

Holding those frames also required fixing the bound on how much may be held.
codexBootstrapMaxBufferedEvents was enforced with len(bufferedChunks), and a
chunk count bounds only the downstream formats that render every upstream frame:
the OpenAI Chat Completions and Gemini translators return zero chunks for a frame
they do not recognise, which is true of every frame added here. Count frames read
from the upstream instead, and cap what a bootstrap accumulates over the upstream
frames and the chunks they translate into. The SSE scanner walks physical lines
and an event can arrive as one line (": keepalive"), two, or three
(event:/data:/blank), so the frame budget is sized for the widest framing rather
than derived from any separator. The websocket loop counts a message as soon as
it is read, before the branches that skip non-text and whitespace-only messages,
so a peer sending only frames the loop skips cannot hold the downstream headers
open; the message that exhausts the budget is still processed and delivered
rather than dropped. Both caps are checked before a frame is admitted, so one
oversized frame cannot be taken on the strength of an empty buffer. This widens
the window where one frame rendered one chunk - the websocket path effectively
moves from 16 frames to 48.

An empty `response.incomplete` seen while buffering is now delivered in-stream
instead of failing the attempt over, matching the websocket executor and the
documented contract that only overload and rate-limit rejections trigger
failover.

`isCodexHandshakeMetadataEvent` is renamed to `isCodexBootstrapBufferableEvent`
because the list is no longer only handshake metadata. Both config doc sites
describe what is held, that heartbeats are held too, that SSE also holds the
lines around a `data:` frame, and that the hold is bounded by frames and bytes
rather than by wall-clock time.
2026-09-12 22:27:44 +08:00

725 lines
29 KiB
Go

package executor
import (
"bytes"
"context"
"fmt"
"net/http"
"strings"
"github.com/gorilla/websocket"
"github.com/router-for-me/CLIProxyAPI/v7/internal/config"
"github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps"
"github.com/router-for-me/CLIProxyAPI/v7/internal/thinking"
cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/executor"
sdktranslator "github.com/router-for-me/CLIProxyAPI/v7/sdk/translator"
log "github.com/sirupsen/logrus"
"github.com/tidwall/gjson"
)
func (e *CodexWebsocketsExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (_ *cliproxyexecutor.StreamResult, err error) {
log.Debugf("Executing Codex Websockets stream request with auth ID: %s, model: %s", auth.ID, req.Model)
if ctx == nil {
ctx = context.Background()
}
if opts.Alt == "responses/compact" {
return nil, statusErr{code: http.StatusBadRequest, msg: "streaming not supported for /responses/compact"}
}
baseModel := thinking.ParseSuffix(req.Model).ModelName
apiKey, baseURL := codexCreds(auth)
if baseURL == "" {
baseURL = "https://chatgpt.com/backend-api/codex"
}
reporter := helps.NewExecutorUsageReporter(ctx, e, baseModel, auth)
defer reporter.TrackFailure(ctx, &err)
from := opts.SourceFormat
responseFormat := cliproxyexecutor.ResponseFormatOrSource(opts)
to := sdktranslator.FromString("codex")
originalPayloadSource := req.Payload
if len(opts.OriginalRequest) > 0 {
originalPayloadSource = opts.OriginalRequest
}
originalPayload := originalPayloadSource
originalTranslated, body := translateCodexRequestPair(from, to, baseModel, originalPayload, req.Payload, true)
body, err = helps.ApplyRequestThinking(body, req, opts, from.String(), to.String(), e.Identifier())
if err != nil {
return nil, err
}
requestedModel := helps.PayloadRequestedModel(opts, req.Model)
requestPath := helps.PayloadRequestPath(opts)
body = helps.ApplyPayloadConfigWithRequest(e.cfg, baseModel, to.String(), from.String(), "", body, originalTranslated, requestedModel, requestPath, opts.Headers)
body = helps.SetStringIfDifferent(body, "model", baseModel)
body = normalizeCodexInstructions(body)
if e.cfg == nil || e.cfg.DisableImageGeneration == config.DisableImageGenerationOff {
body = ensureImageGenerationTool(body, baseModel, auth, opts.Headers)
}
body = sanitizeOpenAIResponsesReasoningEncryptedContent(ctx, "codex websockets executor", body)
body = normalizeCodexWebsocketParallelToolCalls(body, opts.Headers)
body = helps.NormalizeCodexToolSchemas(body)
multiAgentV2Conflict := helps.HasCodexMultiAgentV2NamespaceConflict(body)
body, optimizeMultiAgentV2 := helps.OptimizeCodexMultiAgentV2RequestForAuth(ctx, opts.Headers, body, e.cfg, auth, baseModel)
body, replayScope, errReplay := applyCodexReasoningReplayCacheRequired(ctx, from, req, opts, body)
if errReplay != nil {
return nil, errReplay
}
httpURL := strings.TrimSuffix(baseURL, "/") + "/responses"
wsURL, err := buildCodexResponsesWebsocketURL(httpURL)
if err != nil {
return nil, err
}
body, wsHeaders, errPromptCache := applyCodexPromptCacheHeadersWithContext(ctx, from, req, body, opts.Headers)
if errPromptCache != nil {
return nil, errPromptCache
}
clientBody := body
var identityState codexIdentityConfuseState
upstreamBody, identityState := applyCodexIdentityConfuseBody(e.cfg, auth, originalPayloadSource, body)
reporter.SetTranslatedReasoningEffort(clientBody, to.String())
wsHeaders = applyCodexWebsocketHeaders(ctx, wsHeaders, auth, apiKey, e.cfg, opts.Headers)
applyModelHeaderOverrides(wsHeaders, baseModel)
applyCodexIdentityConfuseHeaders(wsHeaders, &identityState)
var authID, authLabel, authType, authValue string
authID = auth.ID
authLabel = auth.Label
authType, authValue = auth.AccountInfo()
executionSessionID := executionSessionIDFromOptions(opts)
var sess *codexWebsocketSession
if executionSessionID != "" {
sess = e.getOrCreateSession(executionSessionID)
if sess != nil {
sess.reqMu.Lock()
}
}
streamSessionLocked := sess != nil
unlockStreamSession := func() {
if sess != nil && streamSessionLocked {
sess.reqMu.Unlock()
streamSessionLocked = false
}
}
wsReqBody := buildCodexWebsocketRequestBody(upstreamBody)
wsReqLog := helps.UpstreamRequestLog{
URL: wsURL,
Method: "WEBSOCKET",
Headers: wsHeaders.Clone(),
Body: wsReqBody,
Provider: e.Identifier(),
AuthID: authID,
AuthLabel: authLabel,
AuthType: authType,
AuthValue: authValue,
}
helps.RecordAPIWebsocketRequest(ctx, e.cfg, wsReqLog)
var conn *websocket.Conn
var closer *websocketConnectionCloser
var respHS *http.Response
var errDial error
dialCtx := ctx
if cliproxyexecutor.RequiredUpstreamWebsocket(ctx) {
conn, closer = existingWebsocketSessionConn(sess, authID, wsURL)
if conn == nil {
if sess != nil {
sess.reqMu.Unlock()
}
return nil, cliproxyexecutor.NewUpstreamWebsocketReplayRequiredError()
}
} else {
dialCtx = cliproxyexecutor.WithUpstreamAttemptTracker(ctx)
conn, closer, respHS, errDial = e.ensureUpstreamConn(dialCtx, auth, sess, authID, wsURL, wsHeaders)
}
var upstreamHeaders http.Header
if respHS != nil {
upstreamHeaders = respHS.Header.Clone()
}
if errDial != nil {
bodyErr := websocketHandshakeBody(respHS)
if respHS != nil {
helps.RecordAPIWebsocketUpgradeRejection(ctx, e.cfg, websocketUpgradeRequestLog(wsReqLog), respHS.StatusCode, respHS.Header.Clone(), bodyErr)
}
if respHS != nil && respHS.StatusCode == http.StatusUpgradeRequired {
if sess != nil {
sess.reqMu.Unlock()
}
if opts.ExecutionLifecycle == nil && !cliproxyexecutor.DownstreamWebsocket(ctx) {
return e.CodexExecutor.ExecuteStream(ctx, auth, req, opts)
}
if cliproxyexecutor.UpstreamAttempted(dialCtx) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
}
return nil, statusErr{code: respHS.StatusCode, msg: string(bodyErr)}
}
if cliproxyexecutor.UpstreamAttempted(dialCtx) {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
}
if respHS != nil && respHS.StatusCode > 0 {
if sess != nil {
sess.reqMu.Unlock()
}
return nil, newCodexStatusErrWithCooling(respHS.StatusCode, bodyErr, e.modelLevelCooling())
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "dial", errDial)
if sess != nil {
sess.reqMu.Unlock()
}
return nil, errDial
}
if errBind := sess.bindExecutionLifecycle(opts, conn, closer, req.Model); errBind != nil {
if sess != nil {
sess.reqMu.Unlock()
}
closeWebsocketAfterBindFailure(sess, conn, closer)
return nil, errBind
}
recordAPIWebsocketHandshake(ctx, e.cfg, respHS)
reporter.StartResponseTTFT()
if sess == nil {
logCodexWebsocketConnected(executionSessionID, authID, wsURL)
}
var readCh chan codexWebsocketRead
if sess != nil {
readCh = sess.activate(conn)
}
restoreMultiAgentV2 := !multiAgentV2Conflict && (optimizeMultiAgentV2 || sess.isMultiAgentV2Optimized(conn))
cliproxyexecutor.MarkUpstreamAttempt(ctx)
if errSend := writeCodexWebsocketMessage(sess, conn, wsReqBody); errSend != nil {
errSend = mapCodexWebsocketWriteError(sess, conn, errSend)
helps.RecordAPIWebsocketError(ctx, e.cfg, "send", errSend)
if sess != nil {
if cliproxyexecutor.RequiredUpstreamWebsocket(ctx) {
e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "send_error", errSend)
sess.clearActive(conn, readCh)
sess.reqMu.Unlock()
if !shouldRetryCodexWebsocketSend(errSend) {
return nil, errSend
}
return nil, cliproxyexecutor.NewUpstreamWebsocketReplayRequiredError()
}
e.invalidateUpstreamConn(sess, conn, "send_error", errSend)
if !shouldRetryCodexWebsocketSend(errSend) {
sess.clearActive(conn, readCh)
sess.reqMu.Unlock()
return nil, errSend
}
// Retry once with a new websocket connection for the same execution session.
connRetry, closerRetry, respHSRetry, errDialRetry := e.ensureUpstreamConn(ctx, auth, sess, authID, wsURL, wsHeaders)
if errDialRetry != nil || connRetry == nil {
closeHTTPResponseBody(respHSRetry, "codex websockets executor: close handshake response body error")
helps.RecordAPIWebsocketError(ctx, e.cfg, "dial_retry", errDialRetry)
sess.clearActive(conn, readCh)
sess.reqMu.Unlock()
return nil, errDialRetry
}
previousConn, previousReadCh := conn, readCh
conn = connRetry
closer = closerRetry
if errBind := sess.bindExecutionLifecycle(opts, conn, closer, req.Model); errBind != nil {
clearRetryActiveState(sess, previousConn, previousReadCh)
sess.reqMu.Unlock()
closeWebsocketAfterBindFailure(sess, conn, closer)
return nil, errBind
}
readCh = sess.activate(conn)
restoreMultiAgentV2 = !multiAgentV2Conflict && (optimizeMultiAgentV2 || sess.isMultiAgentV2Optimized(conn))
wsReqBodyRetry := buildCodexWebsocketRequestBody(upstreamBody)
helps.RecordAPIWebsocketRequest(ctx, e.cfg, helps.UpstreamRequestLog{
URL: wsURL,
Method: "WEBSOCKET",
Headers: wsHeaders.Clone(),
Body: wsReqBodyRetry,
Provider: e.Identifier(),
AuthID: authID,
AuthLabel: authLabel,
AuthType: authType,
AuthValue: authValue,
})
recordAPIWebsocketHandshake(ctx, e.cfg, respHSRetry)
reporter.StartResponseTTFT()
cliproxyexecutor.MarkUpstreamAttempt(ctx)
if errSendRetry := writeCodexWebsocketMessage(sess, conn, wsReqBodyRetry); errSendRetry != nil {
errSendRetry = mapCodexWebsocketWriteError(sess, conn, errSendRetry)
helps.RecordAPIWebsocketError(ctx, e.cfg, "send_retry", errSendRetry)
e.invalidateUpstreamConn(sess, conn, "send_error", errSendRetry)
sess.clearActive(conn, readCh)
sess.reqMu.Unlock()
return nil, errSendRetry
}
wsReqBody = wsReqBodyRetry
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "send_error", errSend)
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
return nil, errSend
}
}
if optimizeMultiAgentV2 || multiAgentV2Conflict {
sess.setMultiAgentV2Optimized(conn, optimizeMultiAgentV2 && !multiAgentV2Conflict)
}
buffering := e.cfg != nil && e.cfg.Codex.StreamBootstrapBuffering
claudeInputTokens := helps.NewClaudeInputTokenState(from, to, responseFormat, originalPayload)
var param any
outputItemsByIndex := make(map[int64][]byte)
var outputItemsFallback [][]byte
var bufferedChunks [][]byte
// bufferedFrames counts every websocket message read during bootstrap, including the ones the
// loop skips, so a peer that only sends frames the loop ignores cannot keep the window open.
bufferedFrames := 0
bufferedBytes := 0
var initialChunks [][]byte
immediateTerminal := false
// bootstrapTerminalErr holds a non-overload terminal failure seen while buffering. It is
// delivered as an in-stream chunk after the buffered handshake so downstream behaviour stays
// identical to the unbuffered path instead of silently turning into a credential failover.
var bootstrapTerminalErr error
sawOutputDelta := false
if buffering {
for {
if ctx != nil && ctx.Err() != nil {
if sess != nil {
sess.clearActive(conn, readCh)
unlockStreamSession()
} else {
_ = closer.Close()
}
return nil, ctx.Err()
}
msgType, payload, errRead := readCodexWebsocketMessage(ctx, sess, conn, readCh)
if errRead != nil {
mappedErr := mapCodexWebsocketReadError(errRead)
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "read_error", mappedErr)
sess.clearActive(conn, readCh)
unlockStreamSession()
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "read_error", mappedErr)
_ = closer.Close()
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "read", mappedErr)
reporter.PublishFailure(ctx, mappedErr)
return nil, mappedErr
}
// Count every message ReadMessage returns, including the ones this loop goes on to skip,
// so a peer sending only skippable text frames still closes the window. Control frames
// are not counted: the websocket library answers ping and pong inside ReadMessage and
// never returns them, so only the read deadline bounds a peer that sends nothing else.
// windowOpen is carried into the skip branches below rather than breaking here, because
// this message has not been processed yet and dropping it would lose a token, or a
// terminal event, from the turn.
bufferedFrames++
windowOpen := bufferedFrames <= codexBootstrapMaxBufferedFrames
if !windowOpen && bufferedFrames == codexBootstrapMaxBufferedFrames+1 {
helps.LogWithRequestID(ctx).Debugf("codex websockets executor: bootstrap frame budget exhausted after %d messages read; this is the first message that may no longer be held", bufferedFrames)
}
if msgType != websocket.TextMessage {
if msgType == websocket.BinaryMessage {
errBinary := fmt.Errorf("codex websockets executor: unexpected binary message")
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "unexpected_binary", errBinary)
sess.clearActive(conn, readCh)
unlockStreamSession()
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "unexpected_binary", errBinary)
_ = closer.Close()
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "unexpected_binary", errBinary)
reporter.PublishFailure(ctx, errBinary)
return nil, errBinary
}
// No window check here: ReadMessage only ever returns text or binary, and binary
// returned just above, so nothing reaches this line. The empty-payload skip below
// is the reachable one and does consult the window.
continue
}
payload = bytes.TrimSpace(payload)
if len(payload) == 0 {
if !windowOpen {
break
}
continue
}
observeCodexTokenEvent(reporter, payload)
payload = applyCodexIdentityConfuseResponsePayload(payload, identityState)
helps.AppendCodexAPIWebsocketResponse(ctx, e.cfg, payload)
helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload)
payload = helps.RestoreCodexMultiAgentV2Response(payload, restoreMultiAgentV2)
if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok {
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "upstream_error", wsErr)
sess.clearActive(conn, readCh)
unlockStreamSession()
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "upstream_error", wsErr)
_ = closer.Close()
}
if errClearReplay := clearCodexReasoningReplayOnWebsocketError(ctx, replayScope, payload); errClearReplay != nil {
helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", errClearReplay)
reporter.PublishFailure(ctx, errClearReplay)
return nil, errClearReplay
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", wsErr)
reporter.PublishFailure(ctx, wsErr)
return nil, wsErr
}
if streamErr, terminalBody, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok {
// A transient capacity rejection is retried on another credential, so the
// downstream websocket session must survive this upstream teardown. Notifying
// the disconnect here would close the client connection before the retry can
// deliver anything. Every other terminal failure is forwarded in-stream and
// legitimately terminates the session, so it keeps the notifying variant.
failoverPending := isCodexOverloadBootstrapFailure(terminalBody)
if sess != nil {
unlockStreamSession()
if failoverPending {
e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "terminal_failure", streamErr)
} else {
e.invalidateUpstreamConn(sess, conn, "terminal_failure", streamErr)
}
sess.clearActive(conn, readCh)
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "terminal_failure", streamErr)
_ = closer.Close()
}
if errClearReplay := clearCodexReasoningReplayOnInvalidSignature(ctx, replayScope, streamErr.StatusCode(), terminalBody); errClearReplay != nil {
helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", errClearReplay)
reporter.PublishFailure(ctx, errClearReplay)
return nil, errClearReplay
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", streamErr)
reporter.PublishFailure(ctx, streamErr)
if failoverPending {
// Fail the attempt before the downstream headers are committed so the
// conductor can transparently retry on another credential, and report the
// status the upstream refused to put on the wire.
helps.LogWithRequestID(ctx).Debugf("codex websockets executor: bootstrap overload rejection after %d messages read, failing over", bufferedFrames)
return nil, newCodexBootstrapOverloadErr(terminalBody)
}
bootstrapTerminalErr = streamErr
break
}
eventType := gjson.GetBytes(payload, "type").String()
isTerminalEvent := eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" || eventType == "response.failed" || eventType == "error"
if helps.HasMeaningfulCodexOutputDelta(payload) {
sawOutputDelta = true
}
if helps.IsCodexTerminalEmptyIncomplete(payload, len(outputItemsByIndex)+len(outputItemsFallback), sawOutputDelta) {
streamErr := newCodexEmptyIncompleteStreamError()
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", streamErr)
reporter.PublishFailure(ctx, streamErr)
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "terminal_empty_incomplete", streamErr)
sess.clearActive(conn, readCh)
unlockStreamSession()
} else if closer != nil {
_ = closer.Close()
}
bootstrapTerminalErr = streamErr
break
}
if eventType == "response.output_item.done" {
collectCodexOutputItemDone(payload, outputItemsByIndex, &outputItemsFallback)
}
completedPayload := payload
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
completedPayload = normalizeCodexWebsocketCompletion(completedPayload)
completedPayload = patchCodexCompletedOutput(completedPayload, outputItemsByIndex, outputItemsFallback)
if eventType != "response.incomplete" {
cacheCodexReasoningReplayFromCompleted(replayScope, completedPayload)
}
if detail, ok := helps.ParseCodexUsage(completedPayload); ok {
reporter.Publish(ctx, detail)
} else {
reporter.EnsurePublished(ctx)
}
}
var currentChunks [][]byte
if cliproxyexecutor.DownstreamWebsocket(ctx) {
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
payload = completedPayload
}
clientPayload := applyCodexIdentityExposeResponsePayload(payload, identityState)
downstreamPayload := helps.EnsureResponsesUsageDetails(clientPayload)
currentChunks = [][]byte{downstreamPayload}
} else {
payload = normalizeCodexWebsocketCompletion(payload)
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
payload = completedPayload
}
clientPayload := applyCodexIdentityExposeResponsePayload(payload, identityState)
line := encodeCodexWebsocketAsSSE(clientPayload)
currentChunks = helps.TranslateStreamWithClaudeInputTokens(ctx, to, responseFormat, req.Model, originalPayload, clientBody, line, &param, claudeInputTokens)
}
// !isTerminalEvent is redundant against the closed allow-list, which admits no terminal
// type, and the empty-payload rule cannot fire on a payload already known non-empty. It
// stays as the guard a reader expects to find, and its SSE counterpart is !terminalSuccess.
if windowOpen && isCodexBootstrapBufferableEvent(eventType, payload) && !isTerminalEvent {
frameBytes := len(payload)
for i := range currentChunks {
frameBytes += len(currentChunks[i])
}
if bufferedBytes+frameBytes <= codexBootstrapMaxBufferedBytes {
bufferedBytes += frameBytes
bufferedChunks = append(bufferedChunks, currentChunks...)
continue
}
helps.LogWithRequestID(ctx).Debugf("codex websockets executor: bootstrap byte limit reached after %d messages / %d bytes, releasing stream without overload probing", bufferedFrames, bufferedBytes)
}
initialChunks = currentChunks
if isTerminalEvent {
immediateTerminal = true
}
break
}
}
chanCapacity := len(bufferedChunks) + len(initialChunks)
if bootstrapTerminalErr != nil {
chanCapacity++
}
out := make(chan cliproxyexecutor.StreamChunk, chanCapacity)
for _, chunk := range bufferedChunks {
out <- cliproxyexecutor.StreamChunk{Payload: chunk}
}
for _, chunk := range initialChunks {
out <- cliproxyexecutor.StreamChunk{Payload: chunk}
}
if bootstrapTerminalErr != nil {
// The upstream connection was already invalidated and released in the terminal-failure
// branch above, so only the buffered payloads plus the in-stream error remain to emit.
out <- cliproxyexecutor.StreamChunk{Err: bootstrapTerminalErr}
close(out)
return &cliproxyexecutor.StreamResult{Headers: upstreamHeaders, Chunks: out}, nil
}
if immediateTerminal {
if sess != nil {
sess.clearActive(conn, readCh)
unlockStreamSession()
} else {
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, "completed", nil)
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
}
close(out)
return &cliproxyexecutor.StreamResult{Headers: upstreamHeaders, Chunks: out}, nil
}
go func() {
terminateReason := "completed"
var terminateErr error
defer close(out)
defer func() {
if sess != nil {
sess.clearActive(conn, readCh)
unlockStreamSession()
return
}
logCodexWebsocketDisconnected(executionSessionID, authID, wsURL, terminateReason, terminateErr)
if errClose := closer.Close(); errClose != nil {
log.Errorf("codex websockets executor: close websocket error: %v", errClose)
}
}()
send := func(chunk cliproxyexecutor.StreamChunk) bool {
if ctx == nil {
out <- chunk
return true
}
select {
case out <- chunk:
return true
case <-ctx.Done():
return false
}
}
for {
if ctx != nil && ctx.Err() != nil {
terminateReason = "context_done"
terminateErr = ctx.Err()
_ = send(cliproxyexecutor.StreamChunk{Err: ctx.Err()})
return
}
msgType, payload, errRead := readCodexWebsocketMessage(ctx, sess, conn, readCh)
if errRead != nil {
if sess != nil && ctx != nil && ctx.Err() != nil {
terminateReason = "context_done"
terminateErr = ctx.Err()
_ = send(cliproxyexecutor.StreamChunk{Err: ctx.Err()})
return
}
mappedErr := mapCodexWebsocketReadError(errRead)
terminateReason = "read_error"
terminateErr = mappedErr
helps.RecordAPIWebsocketError(ctx, e.cfg, "read", mappedErr)
reporter.PublishFailure(ctx, mappedErr)
_ = send(cliproxyexecutor.StreamChunk{Err: mappedErr})
return
}
if msgType != websocket.TextMessage {
if msgType == websocket.BinaryMessage {
unexpectedErr := fmt.Errorf("codex websockets executor: unexpected binary message")
terminateReason = "unexpected_binary"
terminateErr = unexpectedErr
helps.RecordAPIWebsocketError(ctx, e.cfg, "unexpected_binary", unexpectedErr)
reporter.PublishFailure(ctx, unexpectedErr)
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "unexpected_binary", unexpectedErr)
}
_ = send(cliproxyexecutor.StreamChunk{Err: unexpectedErr})
return
}
continue
}
payload = bytes.TrimSpace(payload)
if len(payload) == 0 {
continue
}
observeCodexTokenEvent(reporter, payload)
payload = applyCodexIdentityConfuseResponsePayload(payload, identityState)
helps.AppendCodexAPIWebsocketResponse(ctx, e.cfg, payload)
helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload)
payload = helps.RestoreCodexMultiAgentV2Response(payload, restoreMultiAgentV2)
if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok {
terminateReason = "upstream_error"
terminateErr = wsErr
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "upstream_error", wsErr)
}
if errClearReplay := clearCodexReasoningReplayOnWebsocketError(ctx, replayScope, payload); errClearReplay != nil {
terminateErr = errClearReplay
helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", errClearReplay)
reporter.PublishFailure(ctx, errClearReplay)
_ = send(cliproxyexecutor.StreamChunk{Err: errClearReplay})
return
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", wsErr)
reporter.PublishFailure(ctx, wsErr)
_ = send(cliproxyexecutor.StreamChunk{Err: wsErr})
return
}
if streamErr, terminalBody, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok {
terminateReason = "upstream_error"
terminateErr = streamErr
if sess != nil {
unlockStreamSession()
e.invalidateUpstreamConn(sess, conn, "terminal_failure", streamErr)
}
if errClearReplay := clearCodexReasoningReplayOnInvalidSignature(ctx, replayScope, streamErr.StatusCode(), terminalBody); errClearReplay != nil {
terminateErr = errClearReplay
helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", errClearReplay)
reporter.PublishFailure(ctx, errClearReplay)
_ = send(cliproxyexecutor.StreamChunk{Err: errClearReplay})
return
}
helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", streamErr)
reporter.PublishFailure(ctx, streamErr)
_ = send(cliproxyexecutor.StreamChunk{Err: streamErr})
return
}
eventType := gjson.GetBytes(payload, "type").String()
isTerminalEvent := eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" || eventType == "response.failed" || eventType == "error"
if helps.HasMeaningfulCodexOutputDelta(payload) {
sawOutputDelta = true
}
if helps.IsCodexTerminalEmptyIncomplete(payload, len(outputItemsByIndex)+len(outputItemsFallback), sawOutputDelta) {
streamErr := newCodexEmptyIncompleteStreamError()
helps.RecordAPIResponseError(ctx, e.cfg, streamErr)
reporter.PublishFailure(ctx, streamErr)
if sess != nil {
e.invalidateUpstreamConn(sess, conn, "terminal_empty_incomplete", streamErr)
unlockStreamSession()
}
_ = send(cliproxyexecutor.StreamChunk{Err: streamErr})
terminateReason = "terminal_empty_incomplete"
terminateErr = streamErr
return
}
if eventType == "response.output_item.done" {
collectCodexOutputItemDone(payload, outputItemsByIndex, &outputItemsFallback)
}
completedPayload := payload
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
completedPayload = normalizeCodexWebsocketCompletion(completedPayload)
completedPayload = patchCodexCompletedOutput(completedPayload, outputItemsByIndex, outputItemsFallback)
if eventType != "response.incomplete" {
cacheCodexReasoningReplayFromCompleted(replayScope, completedPayload)
}
if detail, ok := helps.ParseCodexUsage(completedPayload); ok {
reporter.Publish(ctx, detail)
} else {
reporter.EnsurePublished(ctx)
}
}
clientPayload := applyCodexIdentityExposeResponsePayload(payload, identityState)
if cliproxyexecutor.DownstreamWebsocket(ctx) {
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
clientPayload = applyCodexIdentityExposeResponsePayload(completedPayload, identityState)
}
downstreamPayload := helps.EnsureResponsesUsageDetails(clientPayload)
if !send(cliproxyexecutor.StreamChunk{Payload: downstreamPayload}) {
terminateReason = "context_done"
terminateErr = ctx.Err()
return
}
if isTerminalEvent {
return
}
continue
}
payload = normalizeCodexWebsocketCompletion(payload)
if eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
payload = completedPayload
}
eventType = gjson.GetBytes(payload, "type").String()
clientPayload = applyCodexIdentityExposeResponsePayload(payload, identityState)
line := encodeCodexWebsocketAsSSE(clientPayload)
chunks := helps.TranslateStreamWithClaudeInputTokens(ctx, to, responseFormat, req.Model, originalPayload, clientBody, line, &param, claudeInputTokens)
for i := range chunks {
if !send(cliproxyexecutor.StreamChunk{Payload: chunks[i]}) {
terminateReason = "context_done"
terminateErr = ctx.Err()
return
}
}
if isTerminalEvent || eventType == "response.completed" || eventType == "response.done" || eventType == "response.incomplete" {
return
}
}
}()
return &cliproxyexecutor.StreamResult{Headers: upstreamHeaders, Chunks: out}, nil
}