diff --git a/internal/runtime/executor/xai_executor_response.go b/internal/runtime/executor/xai_executor_response.go index a0bdbae08..7923cd597 100644 --- a/internal/runtime/executor/xai_executor_response.go +++ b/internal/runtime/executor/xai_executor_response.go @@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by // cli-chat-proxy ("Usage resets over a rolling 24-hour window"). const xaiFreeUsageExhaustedCooldown = 24 * time.Hour -// xaiStatusErr wraps upstream error bodies so free-tier exhaustion -// (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for -// auth cooldown / account rotation. Generic 429s stay without an explicit -// retry hint so conductor backoff still applies. +// xaiStatusErr normalizes upstream xAI error bodies for conductor behavior: +// - credential invalidation (403 bad-credentials) is remapped to 401 so the +// existing OAuth refresh-once-and-retry path runs instead of payment cooldown +// - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h +// RetryAfter hint for auth cooldown / account rotation +// +// Generic 429s stay without an explicit retry hint so conductor backoff still applies. func xaiStatusErr(code int, body []byte) statusErr { err := statusErr{code: code, msg: string(body)} - if code != http.StatusTooManyRequests || len(body) == 0 { + if len(body) == 0 { + return err + } + if code == http.StatusForbidden && isXAIBadCredentialsBody(body) { + // Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so + // tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies. + err.code = http.StatusUnauthorized + return err + } + if code != http.StatusTooManyRequests { return err } codeStr := strings.ToLower(gjson.GetBytes(body, "code").String()) @@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr { } return err } + +// isXAIBadCredentialsBody reports whether an xAI error body indicates an +// invalidated/unusable OAuth access token rather than a generic permission or +// payment failure. HTTP and websocket payloads both use this helper, so nested +// error.code / error.message shapes are checked as well as flat bodies. +func isXAIBadCredentialsBody(body []byte) bool { + for _, path := range []string{"code", "error.code", "body.error.code"} { + if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") { + return true + } + } + for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} { + msg := strings.ToLower(gjson.GetBytes(body, path).String()) + if strings.Contains(msg, "access token could not be validated") { + return true + } + } + raw := strings.ToLower(string(body)) + return strings.Contains(raw, "bad-credentials") || + strings.Contains(raw, "access token could not be validated") +} diff --git a/internal/runtime/executor/xai_status_err_test.go b/internal/runtime/executor/xai_status_err_test.go index 3142ae50d..5ca74c9a6 100644 --- a/internal/runtime/executor/xai_status_err_test.go +++ b/internal/runtime/executor/xai_status_err_test.go @@ -2,6 +2,7 @@ package executor import ( "net/http" + "strings" "testing" "time" ) @@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) { func TestXAIStatusErr_Non429Unchanged(t *testing.T) { body := []byte(`{"error":"nope"}`) err := xaiStatusErr(http.StatusBadRequest, body) + if err.StatusCode() != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", err.StatusCode()) + } if err.RetryAfter() != nil { t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter()) } } + +func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) { + body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`) + err := xaiStatusErr(http.StatusForbidden, body) + if err.StatusCode() != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", err.StatusCode()) + } + if !strings.Contains(err.Error(), "bad-credentials") { + t.Fatalf("error body should be preserved, got %q", err.Error()) + } + if err.RetryAfter() != nil { + t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter()) + } +} + +func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) { + body := []byte(`{"error":"The OAuth2 access token could not be validated."}`) + err := xaiStatusErr(http.StatusForbidden, body) + if err.StatusCode() != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", err.StatusCode()) + } +} + +func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) { + body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) + err := xaiStatusErr(http.StatusForbidden, body) + if err.StatusCode() != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", err.StatusCode()) + } +} + +func TestXAIStatusErr_Generic403Unchanged(t *testing.T) { + body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`) + err := xaiStatusErr(http.StatusForbidden, body) + if err.StatusCode() != http.StatusForbidden { + t.Fatalf("status = %d, want 403", err.StatusCode()) + } + if err.RetryAfter() != nil { + t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter()) + } +} + +func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) { + err := xaiStatusErr(http.StatusForbidden, nil) + if err.StatusCode() != http.StatusForbidden { + t.Fatalf("status = %d, want 403", err.StatusCode()) + } +} diff --git a/internal/runtime/executor/xai_websockets_executor.go b/internal/runtime/executor/xai_websockets_executor.go index a28cb9c62..841bfe0e1 100644 --- a/internal/runtime/executor/xai_websockets_executor.go +++ b/internal/runtime/executor/xai_websockets_executor.go @@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) { if wsErr, ok := parseCodexWebsocketError(payload); ok { if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus { xaiError := xaiStatusErr(statusError.code, payload) + // Apply normalized status (e.g. 403 bad-credentials -> 401) and any + // provider-specific retry hint while preserving websocket headers. + statusError.code = xaiError.code if xaiError.retryAfter != nil { statusError.retryAfter = xaiError.retryAfter } diff --git a/internal/runtime/executor/xai_websockets_executor_test.go b/internal/runtime/executor/xai_websockets_executor_test.go index 065e73f5a..c05ba0a39 100644 --- a/internal/runtime/executor/xai_websockets_executor_test.go +++ b/internal/runtime/executor/xai_websockets_executor_test.go @@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { } } +func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { + payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) + err, ok := parseXAIWebsocketError(payload) + if !ok { + t.Fatal("expected xAI websocket error") + } + + status, okStatus := err.(interface{ StatusCode() int }) + if !okStatus || status.StatusCode() != http.StatusUnauthorized { + t.Fatalf("status = %#v, want 401", err) + } + headerSource, okHeaders := err.(interface{ Headers() http.Header }) + if !okHeaders { + t.Fatalf("expected websocket error to preserve headers, got %#v", err) + } + if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" { + t.Fatalf("x-request-id = %q, want req-bad-credentials", got) + } + parsed := gjson.Parse(err.Error()) + if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" { + t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err) + } +} + +func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { + payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) + err, ok := parseXAIWebsocketError(payload) + if !ok { + t.Fatal("expected bare xAI websocket error") + } + + status, okStatus := err.(interface{ StatusCode() int }) + if !okStatus || status.StatusCode() != http.StatusUnauthorized { + t.Fatalf("status = %#v, want 401", err) + } +} + func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`) err, ok := parseXAIWebsocketError(payload)