Files
CLIProxyAPI/internal/runtime/executor/devin_quota_cooldown_regression_test.go
Luis Pater b946d54e4b feat(devin): add account-wide quota exhaustion tracking and credential cooldown refresh
- Introduce Devin quota exhaustion detection for daily and weekly usage windows with reset timestamp parsing.
- Apply credential-scoped 429 status errors with reset-based Retry-After for Devin quota limits.
- Support automatic quota revalidation and early credential cooldown release upon confirmed usage recovery.

Closes: #6690
2026-10-17 02:18:41 +08:00

269 lines
10 KiB
Go

package executor
import (
"bytes"
"context"
"fmt"
"net/http"
"testing"
"time"
"github.com/router-for-me/CLIProxyAPI/v8/internal/runtime/executor/helps"
cliproxyauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
)
// Tests in this file cover issue #6690: Devin daily/weekly usage quota
// exhaustion must cool down the whole credential until the quota reset
// (when a valid reset timestamp is known), instead of the short
// per-model exponential backoff.
func TestDevinQuotaExhaustedError_StreamWeeklyIsCredentialScopedWithResetRetryAfter(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
auth := devinTestAuthWithQuotaResets(now.Add(24*time.Hour), now.Add(6*24*time.Hour))
err := devinStreamTrailerError(t, auth, "failed_precondition", "Your weekly usage quota has been exhausted.")
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if se.code != http.StatusTooManyRequests {
t.Fatalf("code = %d, want 429", se.code)
}
if !se.IsCredentialScoped() {
t.Fatalf("weekly quota exhaustion must be credential-scoped (issue #6690), got model-scoped: %+v", se)
}
if se.RetryAfter() == nil {
t.Fatalf("weekly quota exhaustion must carry reset-based RetryAfter, got nil")
}
want := 6 * 24 * time.Hour
if got := *se.RetryAfter(); got < want-time.Minute || got > want+time.Minute {
t.Fatalf("RetryAfter = %v, want ~%v (weekly reset)", got, want)
}
}
func TestDevinQuotaExhaustedError_StreamDailyUsesDailyReset(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
auth := devinTestAuthWithQuotaResets(now.Add(3*time.Hour), now.Add(6*24*time.Hour))
err := devinStreamTrailerError(t, auth, "failed_precondition", "Your daily usage quota has been exhausted.")
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if !se.IsCredentialScoped() {
t.Fatalf("daily quota exhaustion must be credential-scoped (issue #6690), got model-scoped")
}
if se.RetryAfter() == nil {
t.Fatalf("daily quota exhaustion must carry daily-reset-based RetryAfter, got nil")
}
if got := *se.RetryAfter(); got < 2*time.Hour || got > 4*time.Hour {
t.Fatalf("RetryAfter = %v, want ~3h (daily reset, not weekly)", got)
}
}
func TestDevinQuotaExhaustedError_StreamWithoutSignalsStillCredentialScoped(t *testing.T) {
auth := &cliproxyauth.Auth{ID: "devin-quota-no-signals", Provider: "devin"}
err := devinStreamTrailerError(t, auth, "failed_precondition", "Your weekly usage quota has been exhausted.")
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if !se.IsCredentialScoped() {
t.Fatalf("weekly quota exhaustion without known reset must still be credential-scoped (issue #6690)")
}
if se.RetryAfter() != nil {
t.Fatalf("RetryAfter = %v, want nil (no valid reset signal; conductor credential backoff applies)", *se.RetryAfter())
}
}
func TestDevinQuotaExhaustedError_StreamPastResetYieldsNoRetryAfter(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
auth := devinTestAuthWithQuotaResets(now.Add(-time.Hour), now.Add(-time.Hour))
err := devinStreamTrailerError(t, auth, "failed_precondition", "Your weekly usage quota has been exhausted.")
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if !se.IsCredentialScoped() {
t.Fatalf("weekly quota exhaustion must be credential-scoped even with a stale reset (issue #6690)")
}
if se.RetryAfter() != nil {
t.Fatalf("RetryAfter = %v, want nil for past reset timestamp", *se.RetryAfter())
}
}
func TestDevinQuotaExhaustedError_StreamFarFutureResetIsRejected(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
// A reset further out than any weekly window (plus slack) is bogus data
// and must not pin the credential for months.
auth := devinTestAuthWithQuotaResets(now.Add(90*24*time.Hour), now.Add(90*24*time.Hour))
err := devinStreamTrailerError(t, auth, "failed_precondition", "Your weekly usage quota has been exhausted.")
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if !se.IsCredentialScoped() {
t.Fatalf("weekly quota exhaustion must be credential-scoped (issue #6690)")
}
if se.RetryAfter() != nil {
t.Fatalf("RetryAfter = %v, want nil for implausibly far reset", *se.RetryAfter())
}
}
func TestDevinQuotaExhaustedError_NonStreamConsumeWeeklyIsCredentialScoped(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
auth := devinTestAuthWithQuotaResets(now.Add(24*time.Hour), now.Add(72*time.Hour))
var buf bytes.Buffer
buf.Write(helps.WrapConnectEnvelopeWithFlag(helps.ConnectFlagEndStream, []byte(`{"error":{"code":"failed_precondition","message":"Your weekly usage quota has been exhausted."}}`)))
_, _, errConsume := consumeDevinFramesToInteractionsForAuth(&buf, "devin/claude-opus-5-5", "claude-opus-5-5", auth)
if errConsume == nil {
t.Fatal("expected trailer error, got nil")
}
se, ok := errConsume.(statusErr)
if !ok {
t.Fatalf("consume error type = %T, want statusErr (err=%v)", errConsume, errConsume)
}
if se.code != http.StatusTooManyRequests {
t.Fatalf("code = %d, want 429", se.code)
}
if !se.IsCredentialScoped() {
t.Fatalf("non-stream weekly quota exhaustion must be credential-scoped (issue #6690), err=%+v", se)
}
if se.RetryAfter() == nil {
t.Fatalf("non-stream weekly quota exhaustion must carry reset-based RetryAfter, got nil")
}
want := 72 * time.Hour
if got := *se.RetryAfter(); got < want-time.Minute || got > want+time.Minute {
t.Fatalf("RetryAfter = %v, want ~%v (weekly reset)", got, want)
}
}
func TestDevinTransientErrors_StayModelScoped(t *testing.T) {
cases := []struct {
name string
code string
message string
}{
{name: "high demand", code: "permission_denied", message: "We are experiencing high demand. Please try again later."},
{name: "resource exhausted", code: "resource_exhausted", message: "Resource exhausted, please retry shortly."},
{name: "monthly ACU out of scope", code: "failed_precondition", message: "User monthly ACU quota exhausted (trace ID: 12345)"},
{name: "generic rate limit", code: "failed_precondition", message: "Rate limit exceeded. Please slow down."},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
auth := &cliproxyauth.Auth{ID: "devin-transient", Provider: "devin"}
err := devinStreamTrailerError(t, auth, tc.code, tc.message)
se, ok := err.(statusErr)
if !ok {
t.Fatalf("stream error type = %T, want statusErr (err=%v)", err, err)
}
if se.code != http.StatusTooManyRequests {
t.Fatalf("code = %d, want 429", se.code)
}
if se.IsCredentialScoped() {
t.Fatalf("%s must stay model-scoped, got credential-scoped", tc.name)
}
if se.RetryAfter() != nil {
t.Fatalf("%s RetryAfter = %v, want nil (short backoff path)", tc.name, *se.RetryAfter())
}
})
}
}
func TestDevinStatusError_HTTP429WeeklyQuotaUsesResetWhenAuthKnown(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
auth := devinTestAuthWithQuotaResets(now.Add(24*time.Hour), now.Add(48*time.Hour))
se := newDevinStatusErrorForAuth(http.StatusTooManyRequests, http.Header{}, []byte("Your weekly usage quota has been exhausted."), auth)
if !se.IsCredentialScoped() {
t.Fatalf("HTTP 429 weekly quota exhaustion must be credential-scoped (issue #6690), got %+v", se)
}
if se.RetryAfter() == nil {
t.Fatal("HTTP 429 weekly quota exhaustion must use the known weekly reset when auth is available")
}
want := 48 * time.Hour
if got := *se.RetryAfter(); got < want-time.Minute || got > want+time.Minute {
t.Fatalf("RetryAfter = %v, want ~%v", got, want)
}
// The auth-less helper still guarantees credential scope (nil delay).
plain := newDevinStatusError(http.StatusTooManyRequests, http.Header{}, []byte("Your weekly usage quota has been exhausted."))
if !plain.IsCredentialScoped() || plain.RetryAfter() != nil {
t.Fatalf("auth-less HTTP 429 = %+v, want credential-scoped with nil RetryAfter", plain)
}
// An explicit Retry-After header keeps precedence over signal data.
hdr := http.Header{}
hdr.Set("Retry-After", "30")
withHeader := newDevinStatusErrorForAuth(http.StatusTooManyRequests, hdr, []byte("Your weekly usage quota has been exhausted."), auth)
if withHeader.RetryAfter() == nil || *withHeader.RetryAfter() != 30*time.Second {
t.Fatalf("explicit Retry-After must win, got %v", withHeader.RetryAfter())
}
}
func devinTestAuthWithQuotaResets(daily, weekly time.Time) *cliproxyauth.Auth {
auth := &cliproxyauth.Auth{ID: "devin-quota-auth", Provider: "devin"}
auth.Quota.Signals = map[string]string{
"daily_quota_remaining_percent": "0%",
"weekly_quota_remaining_percent": "0%",
}
if !daily.IsZero() {
auth.Quota.Signals["daily_quota_reset_at"] = daily.Format(time.RFC3339)
}
if !weekly.IsZero() {
auth.Quota.Signals["weekly_quota_reset_at"] = weekly.Format(time.RFC3339)
}
return auth
}
func devinStreamTrailerError(t *testing.T, auth *cliproxyauth.Auth, code, message string) error {
t.Helper()
trailer := fmt.Sprintf(`{"error":{"code":%q,"message":%q}}`, code, message)
var buf bytes.Buffer
buf.Write(helps.WrapConnectEnvelopeWithFlag(helps.ConnectFlagEndStream, []byte(trailer)))
e := &DevinExecutor{}
out := make(chan cliproxyexecutor.StreamChunk, 16)
opts := cliproxyexecutor.Options{
SourceFormat: sdktranslator.FormatOpenAI,
OriginalRequest: []byte(`{"model":"devin/claude-opus-5-5","stream":true,"messages":[{"role":"user","content":"hi"}]}`),
}
go func() {
defer close(out)
e.streamDevinFramesForAuth(
context.Background(),
&buf,
cliproxyexecutor.Request{Model: "devin/claude-opus-5-5", Payload: opts.OriginalRequest},
opts,
"claude-opus-5-5",
sdktranslator.FormatOpenAI,
nil,
out,
auth,
)
}()
var streamErr error
for chunk := range out {
if chunk.Err != nil {
streamErr = chunk.Err
}
}
if streamErr == nil {
t.Fatal("expected stream trailer error chunk, got none")
}
return streamErr
}