From 42d8e746e540506cff12cba71e814b8e1864e17a Mon Sep 17 00:00:00 2001 From: Luis Pater Date: Fri, 21 Aug 2026 02:02:46 +0800 Subject: [PATCH] fix(claude): avoid long global cooldowns for Fable-only 7d_oi rate limits Closes: #5101 --- .../claude_executor_fable_ratelimit_test.go | 96 ++++++++++--------- .../executor/helps/claude_ratelimit.go | 16 ++-- .../executor/helps/claude_ratelimit_test.go | 52 ++++++++++ 3 files changed, 112 insertions(+), 52 deletions(-) diff --git a/internal/runtime/executor/claude_executor_fable_ratelimit_test.go b/internal/runtime/executor/claude_executor_fable_ratelimit_test.go index 8efe77cf9..f0d2f9159 100644 --- a/internal/runtime/executor/claude_executor_fable_ratelimit_test.go +++ b/internal/runtime/executor/claude_executor_fable_ratelimit_test.go @@ -95,58 +95,49 @@ func TestClassifyClaudeUpstreamError_SharedOrAmbiguousRejectionRemainsCredential } } -func TestClassifyClaudeUpstreamError_FableRetryDurationRemainsAvailable(t *testing.T) { - tests := []struct { - name string - headers http.Header - min, max time.Duration - }{ - { - name: "7d_oi reset", - headers: http.Header{ - "Anthropic-Ratelimit-Unified-Status": []string{"rejected"}, - "Anthropic-Ratelimit-Unified-5h-Status": []string{"allowed"}, - "Anthropic-Ratelimit-Unified-7d-Status": []string{"allowed"}, - "Anthropic-Ratelimit-Unified-7d_oi-Status": []string{"rejected"}, - "Anthropic-Ratelimit-Unified-7d_oi-Reset": []string{strconv.FormatInt(time.Now().Add(2*time.Hour).Unix(), 10)}, - }, - min: 2*time.Hour - 5*time.Second, - max: 2*time.Hour + 35*time.Second, - }, - { - name: "retry-after", - headers: http.Header{ - "Anthropic-Ratelimit-Unified-Status": []string{"rejected"}, - "Anthropic-Ratelimit-Unified-5h-Status": []string{"allowed"}, - "Anthropic-Ratelimit-Unified-7d-Status": []string{"allowed"}, - "Anthropic-Ratelimit-Unified-7d_oi-Status": []string{"rejected"}, - "Retry-After": []string{"120"}, - }, - min: 2 * time.Minute, - max: 2*time.Minute + 30*time.Second, - }, - } +func TestClassifyClaudeUpstreamError_FableRetryDuration(t *testing.T) { + t.Run("retry-after header is respected", func(t *testing.T) { + headers := http.Header{ + "Anthropic-Ratelimit-Unified-Status": []string{"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": []string{"allowed"}, + "Anthropic-Ratelimit-Unified-7d-Status": []string{"allowed"}, + "Anthropic-Ratelimit-Unified-7d_oi-Status": []string{"rejected"}, + "Anthropic-Ratelimit-Unified-7d_oi-Reset": []string{strconv.FormatInt(time.Now().Add(7*24*time.Hour).Unix(), 10)}, + "Retry-After": []string{"120"}, + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // When - err := classifyClaudeUpstreamError(http.StatusTooManyRequests, tt.headers, []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"Fable usage window rejected."}}`)) + err := classifyClaudeUpstreamError(http.StatusTooManyRequests, headers, []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"Fable usage window rejected."}}`)) - // Then - var retry retryAfterProvider - if !errors.As(err, &retry) || retry == nil || retry.RetryAfter() == nil { - t.Fatalf("expected Fable rate-limit error to retain a retry duration, got %v", err) - } - if got := *retry.RetryAfter(); got < tt.min || got > tt.max { - t.Fatalf("RetryAfter = %v, want between %v and %v", got, tt.min, tt.max) - } - }) - } + var retry retryAfterProvider + if !errors.As(err, &retry) || retry == nil || retry.RetryAfter() == nil { + t.Fatalf("expected Fable rate-limit error to retain a retry duration, got %v", err) + } + if got := *retry.RetryAfter(); got < 2*time.Minute || got > 2*time.Minute+30*time.Second { + t.Fatalf("RetryAfter = %v, want ~120s with fuzz, but not 7d", got) + } + }) + + t.Run("7d_oi reset only does not set week-long retry duration", func(t *testing.T) { + headers := http.Header{ + "Anthropic-Ratelimit-Unified-Status": []string{"rejected"}, + "Anthropic-Ratelimit-Unified-5h-Status": []string{"allowed"}, + "Anthropic-Ratelimit-Unified-7d-Status": []string{"allowed"}, + "Anthropic-Ratelimit-Unified-7d_oi-Status": []string{"rejected"}, + "Anthropic-Ratelimit-Unified-7d_oi-Reset": []string{strconv.FormatInt(time.Now().Add(7*24*time.Hour).Unix(), 10)}, + } + + err := classifyClaudeUpstreamError(http.StatusTooManyRequests, headers, []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"Fable usage window rejected."}}`)) + + var retry retryAfterProvider + if errors.As(err, &retry) && retry != nil && retry.RetryAfter() != nil { + t.Fatalf("expected Fable 7d_oi-only reset to yield nil RetryAfter, got %v", *retry.RetryAfter()) + } + }) } func TestClaudeExecutor_AuthManager_FableOnlyRejectionDoesNotBlockOpus(t *testing.T) { var fableAttempts, opusAttempts atomic.Int32 - reset := time.Now().Add(2 * time.Hour).Unix() + reset := time.Now().Add(7 * 24 * time.Hour).Unix() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, errRead := io.ReadAll(r.Body) if errRead != nil { @@ -207,6 +198,19 @@ func TestClaudeExecutor_AuthManager_FableOnlyRejectionDoesNotBlockOpus(t *testin t.Fatalf("Fable upstream attempts = %d, want 1", got) } + // Verify that Fable model state cooldown is driven by Retry-After (~120s) and not 7 days. + updatedAuth, ok := manager.GetByID(auth.ID) + if !ok || updatedAuth == nil { + t.Fatal("auth not found") + } + fableState := updatedAuth.ModelStates["claude-fable-5"] + if fableState == nil { + t.Fatal("fable model state not found") + } + if fableState.Quota.NextRecoverAt.After(time.Now().Add(5 * time.Minute)) { + t.Fatalf("fable model state cooldown too long: NextRecoverAt = %v (want ~120s, not 7 days)", fableState.Quota.NextRecoverAt) + } + payloadOpus := []byte(`{"model":"claude-opus-5","messages":[{"role":"user","content":[{"type":"text","text":"test"}]}]}`) _, errOpus := manager.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{ Model: "claude-opus-5", diff --git a/internal/runtime/executor/helps/claude_ratelimit.go b/internal/runtime/executor/helps/claude_ratelimit.go index 88a84a66b..3e2656945 100644 --- a/internal/runtime/executor/helps/claude_ratelimit.go +++ b/internal/runtime/executor/helps/claude_ratelimit.go @@ -36,8 +36,11 @@ func ClaudeHeadersIndicateUnifiedRateLimitRejection(headers http.Header) bool { return false } status7dOI := strings.ToLower(strings.TrimSpace(getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-7d_oi-Status"))) - fableOnlyRejection := status5h == "allowed" && status7d == "allowed" && status7dOI == "rejected" - return !fableOnlyRejection + return !isFableOnlyRejection(status5h, status7d, status7dOI) +} + +func isFableOnlyRejection(status5h, status7d, status7dOI string) bool { + return status5h == "allowed" && status7d == "allowed" && status7dOI == "rejected" } // ParseClaudeRateLimitReset inspects Anthropic response headers for shared and Fable-specific @@ -57,6 +60,7 @@ func parseClaudeRateLimitResetWithFuzz(headers http.Header, now time.Time, minFu status5h := strings.ToLower(strings.TrimSpace(getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-5h-Status"))) status7d := strings.ToLower(strings.TrimSpace(getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-7d-Status"))) status7dOI := strings.ToLower(strings.TrimSpace(getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-7d_oi-Status"))) + fableOnlyRejection := isFableOnlyRejection(status5h, status7d, status7dOI) var candidateDeadlines []time.Time var rejectedWindows []string @@ -102,8 +106,8 @@ func parseClaudeRateLimitResetWithFuzz(headers http.Header, now time.Time, minFu } } - // 4. Fable-specific 7-day window reset (only when rejected) - if status7dOI == "rejected" { + // 4. Fable-specific 7-day window reset (only when rejected and not a Fable-only rejection) + if status7dOI == "rejected" && !fableOnlyRejection { if raw := getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-7d_oi-Reset"); raw != "" { if t, ok := parseUnixOrTimestamp(raw); ok && t.After(now) { candidateDeadlines = append(candidateDeadlines, t) @@ -112,8 +116,8 @@ func parseClaudeRateLimitResetWithFuzz(headers http.Header, now time.Time, minFu } // 5. Unified reset header: - unifiedRejected := unifiedStatus == "rejected" || status5h == "rejected" || status7d == "rejected" || status7dOI == "rejected" || - (unifiedStatus == "" && status5h != "allowed" && status7d != "allowed") + unifiedRejected := !fableOnlyRejection && (unifiedStatus == "rejected" || status5h == "rejected" || status7d == "rejected" || status7dOI == "rejected" || + (unifiedStatus == "" && status5h != "allowed" && status7d != "allowed")) if unifiedRejected { if raw := getHeaderCaseInsensitive(headers, "Anthropic-Ratelimit-Unified-Reset"); raw != "" { diff --git a/internal/runtime/executor/helps/claude_ratelimit_test.go b/internal/runtime/executor/helps/claude_ratelimit_test.go index 5a9e928f5..c58b8f5f9 100644 --- a/internal/runtime/executor/helps/claude_ratelimit_test.go +++ b/internal/runtime/executor/helps/claude_ratelimit_test.go @@ -113,6 +113,58 @@ func TestParseClaudeRateLimitReset_AllCases(t *testing.T) { } }) + t.Run("fable-only rejection with 7d_oi reset and retry-after uses retry-after only", func(t *testing.T) { + h := make(http.Header) + h.Set("Anthropic-Ratelimit-Unified-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-5h-Status", "allowed") + h.Set("Anthropic-Ratelimit-Unified-7d-Status", "allowed") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Reset", strconv.FormatInt(now.Add(7*24*time.Hour).Unix(), 10)) + h.Set("Anthropic-Ratelimit-Unified-Reset", strconv.FormatInt(now.Add(7*24*time.Hour).Unix(), 10)) + h.Set("Retry-After", "60") + + got := parseClaudeRateLimitResetWithFuzz(h, now, 0, 0) + if got == nil { + t.Fatal("expected non-nil RetryAfter") + } + if *got != 60*time.Second { + t.Fatalf("expected 60s from Retry-After, got %v", *got) + } + }) + + t.Run("fable-only rejection with 7d_oi reset only returns nil for exponential backoff", func(t *testing.T) { + h := make(http.Header) + h.Set("Anthropic-Ratelimit-Unified-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-5h-Status", "allowed") + h.Set("Anthropic-Ratelimit-Unified-7d-Status", "allowed") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Reset", strconv.FormatInt(now.Add(7*24*time.Hour).Unix(), 10)) + h.Set("Anthropic-Ratelimit-Unified-Reset", strconv.FormatInt(now.Add(7*24*time.Hour).Unix(), 10)) + + got := ParseClaudeRateLimitReset(h, now) + if got != nil { + t.Fatalf("expected nil for fable-only rejection without retry-after, got %v", *got) + } + }) + + t.Run("non-fable combined rejection with 7d_oi reset keeps longer duration", func(t *testing.T) { + h := make(http.Header) + h.Set("Anthropic-Ratelimit-Unified-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-5h-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-5h-Reset", strconv.FormatInt(now.Add(5*time.Hour).Unix(), 10)) + h.Set("Anthropic-Ratelimit-Unified-7d-Status", "allowed") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Status", "rejected") + h.Set("Anthropic-Ratelimit-Unified-7d_oi-Reset", strconv.FormatInt(now.Add(7*24*time.Hour).Unix(), 10)) + + got := parseClaudeRateLimitResetWithFuzz(h, now, 0, 0) + if got == nil { + t.Fatal("expected non-nil RetryAfter") + } + if *got < 7*24*time.Hour-5*time.Second || *got > 7*24*time.Hour+5*time.Second { + t.Fatalf("expected ~7d, got %v", *got) + } + }) + t.Run("past timestamp returns nil", func(t *testing.T) { h := make(http.Header) h.Set("Anthropic-Ratelimit-Unified-5h-Status", "rejected")