fix(xai): remap bad-credentials 403s to unauthorized

- normalize xAI `403` bad-credentials responses to `401` before conductor retry handling
- preserve websocket error headers while applying the normalized status/retry hints
- share bad-credentials detection across flat and nested payload shapes for HTTP/websocket paths

Closes: #4046
This commit is contained in:
Luis Pater
2026-08-07 06:48:09 +08:00
parent 0a95fa62a1
commit dd67f56f26
4 changed files with 130 additions and 5 deletions

View File

@@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by
// cli-chat-proxy ("Usage resets over a rolling 24-hour window").
const xaiFreeUsageExhaustedCooldown = 24 * time.Hour
// xaiStatusErr wraps upstream error bodies so free-tier exhaustion
// (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for
// auth cooldown / account rotation. Generic 429s stay without an explicit
// retry hint so conductor backoff still applies.
// xaiStatusErr normalizes upstream xAI error bodies for conductor behavior:
// - credential invalidation (403 bad-credentials) is remapped to 401 so the
// existing OAuth refresh-once-and-retry path runs instead of payment cooldown
// - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h
// RetryAfter hint for auth cooldown / account rotation
//
// Generic 429s stay without an explicit retry hint so conductor backoff still applies.
func xaiStatusErr(code int, body []byte) statusErr {
err := statusErr{code: code, msg: string(body)}
if code != http.StatusTooManyRequests || len(body) == 0 {
if len(body) == 0 {
return err
}
if code == http.StatusForbidden && isXAIBadCredentialsBody(body) {
// Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so
// tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies.
err.code = http.StatusUnauthorized
return err
}
if code != http.StatusTooManyRequests {
return err
}
codeStr := strings.ToLower(gjson.GetBytes(body, "code").String())
@@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr {
}
return err
}
// isXAIBadCredentialsBody reports whether an xAI error body indicates an
// invalidated/unusable OAuth access token rather than a generic permission or
// payment failure. HTTP and websocket payloads both use this helper, so nested
// error.code / error.message shapes are checked as well as flat bodies.
func isXAIBadCredentialsBody(body []byte) bool {
for _, path := range []string{"code", "error.code", "body.error.code"} {
if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") {
return true
}
}
for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} {
msg := strings.ToLower(gjson.GetBytes(body, path).String())
if strings.Contains(msg, "access token could not be validated") {
return true
}
}
raw := strings.ToLower(string(body))
return strings.Contains(raw, "bad-credentials") ||
strings.Contains(raw, "access token could not be validated")
}

View File

@@ -2,6 +2,7 @@ package executor
import (
"net/http"
"strings"
"testing"
"time"
)
@@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) {
func TestXAIStatusErr_Non429Unchanged(t *testing.T) {
body := []byte(`{"error":"nope"}`)
err := xaiStatusErr(http.StatusBadRequest, body)
if err.StatusCode() != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", err.StatusCode())
}
if err.RetryAfter() != nil {
t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter())
}
}
func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) {
body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`)
err := xaiStatusErr(http.StatusForbidden, body)
if err.StatusCode() != http.StatusUnauthorized {
t.Fatalf("status = %d, want 401", err.StatusCode())
}
if !strings.Contains(err.Error(), "bad-credentials") {
t.Fatalf("error body should be preserved, got %q", err.Error())
}
if err.RetryAfter() != nil {
t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter())
}
}
func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) {
body := []byte(`{"error":"The OAuth2 access token could not be validated."}`)
err := xaiStatusErr(http.StatusForbidden, body)
if err.StatusCode() != http.StatusUnauthorized {
t.Fatalf("status = %d, want 401", err.StatusCode())
}
}
func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) {
body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
err := xaiStatusErr(http.StatusForbidden, body)
if err.StatusCode() != http.StatusUnauthorized {
t.Fatalf("status = %d, want 401", err.StatusCode())
}
}
func TestXAIStatusErr_Generic403Unchanged(t *testing.T) {
body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`)
err := xaiStatusErr(http.StatusForbidden, body)
if err.StatusCode() != http.StatusForbidden {
t.Fatalf("status = %d, want 403", err.StatusCode())
}
if err.RetryAfter() != nil {
t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter())
}
}
func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) {
err := xaiStatusErr(http.StatusForbidden, nil)
if err.StatusCode() != http.StatusForbidden {
t.Fatalf("status = %d, want 403", err.StatusCode())
}
}

View File

@@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) {
if wsErr, ok := parseCodexWebsocketError(payload); ok {
if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus {
xaiError := xaiStatusErr(statusError.code, payload)
// Apply normalized status (e.g. 403 bad-credentials -> 401) and any
// provider-specific retry hint while preserving websocket headers.
statusError.code = xaiError.code
if xaiError.retryAfter != nil {
statusError.retryAfter = xaiError.retryAfter
}

View File

@@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) {
}
}
func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
err, ok := parseXAIWebsocketError(payload)
if !ok {
t.Fatal("expected xAI websocket error")
}
status, okStatus := err.(interface{ StatusCode() int })
if !okStatus || status.StatusCode() != http.StatusUnauthorized {
t.Fatalf("status = %#v, want 401", err)
}
headerSource, okHeaders := err.(interface{ Headers() http.Header })
if !okHeaders {
t.Fatalf("expected websocket error to preserve headers, got %#v", err)
}
if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" {
t.Fatalf("x-request-id = %q, want req-bad-credentials", got)
}
parsed := gjson.Parse(err.Error())
if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" {
t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err)
}
}
func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
err, ok := parseXAIWebsocketError(payload)
if !ok {
t.Fatal("expected bare xAI websocket error")
}
status, okStatus := err.(interface{ StatusCode() int })
if !okStatus || status.StatusCode() != http.StatusUnauthorized {
t.Fatalf("status = %#v, want 401", err)
}
}
func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) {
payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`)
err, ok := parseXAIWebsocketError(payload)