Files
CLIProxyAPI/internal/runtime/executor/codex_websockets_errors.go
Luis Pater deacbf2199 fix(codex): preserve nested Retry-After headers on overload errors
- Extract error headers from nested Codex error and response.error payloads while keeping outer header precedence.
- Parse Retry-After hints on 502/503 and other recoverable transient status codes across terminal, WebSocket, and bootstrap failover paths.
- Add regression tests covering captured WS overload payloads, explicit 503 responses, header precedence, and cooldown scheduling.

Closes: #7600

 Conflicts:
	internal/runtime/executor/codex_executor_stream.go
	internal/runtime/executor/codex_websockets_errors.go
	internal/runtime/executor/codex_websockets_stream.go
2026-11-16 19:21:23 +08:00

367 lines
11 KiB
Go

package executor
import (
"context"
"io"
"net/http"
"strings"
"time"
"github.com/MoeForge/moego/ojson"
"github.com/router-for-me/CLIProxyAPI/v8/internal/config"
"github.com/router-for-me/CLIProxyAPI/v8/internal/runtime/executor/helps"
log "github.com/sirupsen/logrus"
)
type statusErrWithHeaders struct {
statusErr
headers http.Header
}
func (e statusErrWithHeaders) Headers() http.Header {
if e.headers == nil {
return nil
}
return e.headers.Clone()
}
func parseCodexWebsocketError(payload []byte) (error, bool) {
return parseCodexWebsocketErrorWithCooling(payload, false)
}
func codexWebsocketErrorStatus(payload []byte) int {
for _, path := range []string{"status", "status_code"} {
if status := int(ojson.GetBytesNoCopy(payload, path).Int()); status > 0 {
return status
}
}
for _, path := range []string{
"body.status", "body.status_code",
"error.status", "error.status_code",
"body.error.status", "body.error.status_code",
"response.error.status", "response.error.status_code",
} {
if status := int(ojson.GetBytesNoCopy(payload, path).Int()); status >= 400 && status <= 599 {
return status
}
}
return 0
}
func codexWebsocketErrorStatusWithHeaders(payload []byte, headers http.Header) int {
if status := codexWebsocketErrorStatus(payload); status > 0 {
return status
}
if len(headers) == 0 {
return 0
}
return inferCodexWebsocketErrorStatus(payload)
}
func inferCodexWebsocketErrorStatus(payload []byte) int {
if isCodexWebsocketConnectionLimitError(payload) {
return http.StatusTooManyRequests
}
body, ok := codexTerminalFailureBody(payload)
if !ok {
return 0
}
if isCodexUsageLimitError(body) || isCodexTransientCapacityError(body) {
return http.StatusTooManyRequests
}
if codexTerminalErrorIsContextLength(body) {
return http.StatusBadRequest
}
if code, _, classified := codexStatusErrorClassification(http.StatusBadRequest, body); classified {
if code == "auth_unavailable" {
return http.StatusUnauthorized
}
return http.StatusBadRequest
}
errorType := strings.ToLower(strings.TrimSpace(ojson.GetBytesNoCopy(body, "error.type").String()))
errorCode := strings.ToLower(strings.TrimSpace(ojson.GetBytesNoCopy(body, "error.code").String()))
if errorType == "service_unavailable_error" || errorCode == "server_is_overloaded" {
return http.StatusServiceUnavailable
}
return codexTerminalFailureStatus(body)
}
func parseCodexWebsocketErrorWithCooling(payload []byte, modelLevelCooling bool) (error, bool) {
if len(payload) == 0 {
return nil, false
}
if helps.CodexResponseEventType(payload) != "error" {
return nil, false
}
headers := parseCodexWebsocketErrorHeaders(payload)
status := codexWebsocketErrorStatusWithHeaders(payload, headers)
if status <= 0 {
return nil, false
}
out := buildCodexWebsocketErrorPayload(payload, status)
isUsageLimit := isCodexUsageLimitError(out)
transientCapacity := !isUsageLimit && isCodexTransientCapacityError(out)
statusError := statusErr{
code: status,
msg: string(out),
credentialScoped: isUsageLimit && !modelLevelCooling,
transientRateLimit: transientCapacity && status == http.StatusTooManyRequests,
}
// Hint precedence: the quota body reset wins, then the connection-limit
// immediate-retry contract, then the generic Retry-After header.
if retryAfter := parseCodexRetryAfter(status, out, time.Now()); retryAfter != nil {
statusError.retryAfter = retryAfter
} else if isCodexWebsocketConnectionLimitError(payload) {
retryAfter := time.Duration(0)
statusError.retryAfter = &retryAfter
} else if shouldParseCodexRetryAfterHeader(status) {
if retryAfter := parseRetryAfterHeader(headers, time.Now()); retryAfter != nil {
statusError.retryAfter = retryAfter
}
}
return statusErrWithHeaders{
statusErr: statusError,
headers: headers,
}, true
}
func clearCodexReasoningReplayOnWebsocketError(ctx context.Context, scope codexReasoningReplayScope, payload []byte) error {
headers := parseCodexWebsocketErrorHeaders(payload)
status := codexWebsocketErrorStatusWithHeaders(payload, headers)
if status <= 0 {
return nil
}
return clearCodexReasoningReplayOnInvalidSignature(ctx, scope, status, buildCodexWebsocketErrorPayload(payload, status))
}
func buildCodexWebsocketErrorPayload(payload []byte, status int) []byte {
doc := ojson.NewDocString(`{"type":"error"}`)
_ = doc.Set("status", status)
if seq := ojson.GetBytesNoCopy(payload, "sequence_number"); seq.Exists() {
_ = doc.Set("sequence_number", seq.Int())
}
fillBlankErrorMessage := func() {
if node := doc.Get("error"); !node.IsObject() {
scalar := ""
if node.Type != ojson.Null && strings.TrimSpace(node.String()) != "" {
scalar = node.String()
}
_ = doc.SetRawBytes("error", []byte(`{}`))
if scalar != "" {
_ = doc.Set("error.message", scalar)
return
}
}
if strings.TrimSpace(doc.Get("error.message").String()) != "" {
return
}
if code := strings.TrimSpace(doc.Get("error.code").String()); code != "" {
_ = doc.Set("error.message", codexPolicyFallbackMessage(code))
} else if errType := strings.TrimSpace(doc.Get("error.type").String()); errType != "" {
_ = doc.Set("error.message", errType)
} else {
_ = doc.Set("error.message", http.StatusText(status))
}
}
if bodyNode := ojson.GetBytesNoCopy(payload, "body"); bodyNode.Exists() {
_ = doc.SetRawBytes("body", []byte(bodyNode.Raw))
if bodyErrorNode := bodyNode.Get("error"); bodyErrorNode.Exists() && bodyErrorNode.Type != ojson.Null {
_ = doc.SetRawBytes("error", []byte(bodyErrorNode.Raw))
fillBlankErrorMessage()
return doc.Bytes()
}
}
for _, path := range []string{"error", "response.error"} {
if errNode := ojson.GetBytesNoCopy(payload, path); errNode.Exists() && errNode.Type != ojson.Null {
_ = doc.SetRawBytes("error", []byte(errNode.Raw))
fillBlankErrorMessage()
return doc.Bytes()
}
}
if topLevel := codexTerminalTopLevelErrorBody(payload); topLevel != nil {
_ = doc.SetRawBytes("error", []byte(topLevel.Get("error").Raw))
fillBlankErrorMessage()
return doc.Bytes()
}
_ = doc.Set("error.type", "server_error")
_ = doc.Set("error.message", http.StatusText(status))
return doc.Bytes()
}
func isCodexWebsocketConnectionLimitError(payload []byte) bool {
if len(payload) == 0 {
return false
}
for _, path := range []string{"error.code", "error.type", "body.error.code", "body.error.type", "code", "error"} {
if strings.TrimSpace(ojson.GetBytesNoCopy(payload, path).String()) == "websocket_connection_limit_reached" {
return true
}
}
return false
}
func shouldParseCodexRetryAfterHeader(statusCode int) bool {
switch statusCode {
case http.StatusTooManyRequests,
http.StatusRequestTimeout,
http.StatusInternalServerError,
http.StatusBadGateway,
http.StatusServiceUnavailable,
http.StatusGatewayTimeout,
520, 521, 522, 523, 524, 525, 526:
return true
default:
return false
}
}
func parseCodexWebsocketErrorHeaders(payload []byte) http.Header {
if len(payload) == 0 {
return nil
}
mapped := make(http.Header)
seen := make(map[string]struct{})
for _, path := range []string{"headers", "body.headers", "error.headers", "body.error.headers", "response.error.headers"} {
headersNode := ojson.GetBytesNoCopy(payload, path)
if !headersNode.Exists() || !headersNode.IsObject() {
continue
}
headersNode.ForEach(func(key, value ojson.Result) bool {
name := strings.TrimSpace(key.String())
if name == "" {
return true
}
canonical := http.CanonicalHeaderKey(name)
if _, exists := seen[canonical]; exists {
return true
}
// Mark the header as seen before validating its value so an explicit outer
// blank or unsupported entry still shadows any nested fallback value.
seen[canonical] = struct{}{}
switch value.Type {
case ojson.String:
if v := strings.TrimSpace(value.String()); v != "" {
mapped.Set(strings.Clone(name), strings.Clone(v))
}
case ojson.Number, ojson.True, ojson.False:
if v := strings.TrimSpace(value.Raw); v != "" {
mapped.Set(strings.Clone(name), strings.Clone(v))
}
default:
}
return true
})
}
if len(mapped) == 0 {
return nil
}
return mapped
}
func normalizeCodexWebsocketCompletion(payload []byte) []byte {
if strings.TrimSpace(ojson.GetBytesNoCopy(payload, "type").String()) == "response.done" {
updated, err := ojson.SetBytes(payload, "type", "response.completed")
if err == nil && len(updated) > 0 {
return updated
}
}
return payload
}
func encodeCodexWebsocketAsSSE(payload []byte) []byte {
if len(payload) == 0 {
return nil
}
line := make([]byte, 0, len("data: ")+len(payload))
line = append(line, []byte("data: ")...)
line = append(line, payload...)
return line
}
func websocketUpgradeRequestLog(info helps.UpstreamRequestLog) helps.UpstreamRequestLog {
upgradeInfo := info
upgradeInfo.URL = helps.WebsocketUpgradeRequestURL(info.URL)
upgradeInfo.Method = http.MethodGet
upgradeInfo.Body = nil
upgradeInfo.Headers = info.Headers.Clone()
if upgradeInfo.Headers == nil {
upgradeInfo.Headers = make(http.Header)
}
if strings.TrimSpace(upgradeInfo.Headers.Get("Connection")) == "" {
upgradeInfo.Headers.Set("Connection", "Upgrade")
}
if strings.TrimSpace(upgradeInfo.Headers.Get("Upgrade")) == "" {
upgradeInfo.Headers.Set("Upgrade", "websocket")
}
return upgradeInfo
}
func recordAPIWebsocketHandshake(ctx context.Context, cfg *config.Config, resp *http.Response) {
if resp == nil {
return
}
helps.RecordAPIWebsocketHandshake(ctx, cfg, resp.StatusCode, resp.Header.Clone())
closeHTTPResponseBody(resp, "codex websockets executor: close handshake response body error")
}
func websocketHandshakeBody(resp *http.Response) []byte {
body, _ := websocketHandshakeBodyWithReadErr(resp)
return body
}
func websocketHandshakeBodyWithReadErr(resp *http.Response) ([]byte, error) {
if resp == nil || resp.Body == nil {
return nil, nil
}
body, errRead := io.ReadAll(resp.Body)
closeHTTPResponseBody(resp, "codex websockets executor: close handshake response body error")
if len(body) == 0 {
return nil, errRead
}
return body, errRead
}
func recordWebsocketUpgradeRejection(ctx context.Context, cfg *config.Config, wsReqLog helps.UpstreamRequestLog, resp *http.Response) []byte {
bodyErr, errRead := websocketHandshakeBodyWithReadErr(resp)
if resp != nil {
helps.RecordAPIWebsocketUpgradeRejection(ctx, cfg, websocketUpgradeRequestLog(wsReqLog), resp.StatusCode, resp.Header.Clone(), bodyErr)
if errRead != nil {
helps.RecordAPIResponseError(ctx, cfg, errRead)
}
}
return bodyErr
}
func recordWebsocketDialFailure(ctx context.Context, cfg *config.Config, wsReqLog helps.UpstreamRequestLog, resp *http.Response, errDial error) []byte {
bodyErr := recordWebsocketUpgradeRejection(ctx, cfg, wsReqLog, resp)
helps.RecordAPIWebsocketError(ctx, cfg, "dial", errDial)
return bodyErr
}
func recordWebsocketDialRetryFailure(ctx context.Context, cfg *config.Config, wsReqLog helps.UpstreamRequestLog, resp *http.Response, errDial error) []byte {
if resp != nil {
bodyErr := recordWebsocketUpgradeRejection(ctx, cfg, wsReqLog, resp)
helps.RecordAPIWebsocketError(ctx, cfg, "dial_retry", errDial)
return bodyErr
}
if cfg != nil && !cfg.CommercialMode && !cfg.RequestLog {
helps.RecordAPIRequest(ctx, cfg, websocketUpgradeRequestLog(wsReqLog))
}
helps.RecordAPIWebsocketError(ctx, cfg, "dial_retry", errDial)
return nil
}
func closeHTTPResponseBody(resp *http.Response, logPrefix string) {
if resp == nil || resp.Body == nil {
return
}
if errClose := resp.Body.Close(); errClose != nil {
log.Errorf("%s: %v", logPrefix, errClose)
}
}