diff --git a/sdk/cliproxy/auth/conductor.go b/sdk/cliproxy/auth/conductor.go index a91a12bf2..716a6645c 100644 --- a/sdk/cliproxy/auth/conductor.go +++ b/sdk/cliproxy/auth/conductor.go @@ -2733,7 +2733,15 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, if ra := retryAfterFromError(errExec); ra != nil { result.RetryAfter = ra } - m.MarkResult(execCtx, result) + // Some Anthropic-compatible upstreams do not implement the + // count_tokens route and return a generic endpoint 404. Record + // the failure for hooks and metrics without suspending a model + // that remains usable through the messages endpoint. + if isCountTokensEndpointNotFoundError(errExec, execReq.Model) { + m.recordAvailabilityNeutralResult(execCtx, result) + } else { + m.MarkResult(execCtx, result) + } if isRequestInvalidError(errExec) { return cliproxyexecutor.Response{}, errExec } @@ -3891,6 +3899,30 @@ func (m *Manager) MarkResult(ctx context.Context, result Result) { m.publishErrorEvent(result, authSnapshot) } +func (m *Manager) recordAvailabilityNeutralResult(ctx context.Context, result Result) { + if result.AuthID == "" { + return + } + + var authSnapshot *Auth + m.mu.Lock() + if auth, ok := m.auths[result.AuthID]; ok && auth != nil { + now := time.Now() + auth.recordRecentRequest(now, result.Success) + if result.Success { + auth.Success++ + } else { + auth.Failed++ + } + _ = m.persist(ctx, auth) + authSnapshot = auth.Clone() + } + m.mu.Unlock() + + m.hook.OnResult(ctx, result) + m.publishErrorEvent(result, authSnapshot) +} + func ensureModelState(auth *Auth, model string) *ModelState { if auth == nil || model == "" { return nil @@ -4096,9 +4128,15 @@ func resultErrorFromError(err error) *Error { if err == nil { return nil } - resultErr := &Error{ - Message: err.Error(), - HTTPStatus: statusCodeFromError(err), + var sourceErr *Error + var resultErr *Error + if errors.As(err, &sourceErr) && sourceErr != nil { + resultErr = cloneError(sourceErr) + } else { + resultErr = &Error{Message: err.Error()} + } + if resultErr.HTTPStatus == 0 { + resultErr.HTTPStatus = statusCodeFromError(err) } if isRequestScopedError(err) || isRequestInvalidError(err) { resultErr.Code = requestScopedErrorCode @@ -4296,6 +4334,199 @@ func isRequestScopedResultError(err *Error) bool { return err != nil && (err.IsRequestScoped() || isRequestScopedNotFoundResultError(err)) } +func isCountTokensEndpointNotFoundError(err error, requestedModel string) bool { + if err == nil || statusCodeFromError(err) != http.StatusNotFound { + return false + } + baseModel := thinking.ParseSuffix(requestedModel).ModelName + return !isExplicitModelNotFoundError(err, baseModel) +} + +func isExplicitModelNotFoundError(err error, requestedModel string) bool { + if err == nil { + return false + } + if authErr, ok := err.(*Error); ok && authErr != nil { + if isModelNotFoundIdentifier(authErr.Code) || isStructuredModelNotFoundError(authErr.Message, requestedModel) { + return true + } + } else if isStructuredModelNotFoundError(err.Error(), requestedModel) { + return true + } + + switch wrapped := err.(type) { + case interface{ Unwrap() []error }: + for _, nested := range wrapped.Unwrap() { + if isExplicitModelNotFoundError(nested, requestedModel) { + return true + } + } + case interface{ Unwrap() error }: + return isExplicitModelNotFoundError(wrapped.Unwrap(), requestedModel) + } + return false +} + +func isStructuredModelNotFoundError(message, requestedModel string) bool { + var payload any + if errJSON := json.Unmarshal([]byte(strings.TrimSpace(message)), &payload); errJSON != nil { + return false + } + return containsStructuredModelNotFound(payload, requestedModel) +} + +func containsStructuredModelNotFound(value any, requestedModel string) bool { + switch typed := value.(type) { + case map[string]any: + notFoundType := false + exactModelReference := false + for key, item := range typed { + text, isString := item.(string) + if isString { + switch strings.ToLower(strings.TrimSpace(key)) { + case "code": + if isModelNotFoundIdentifier(text) { + return true + } + case "type": + if isModelNotFoundIdentifier(text) { + return true + } + notFoundType = notFoundType || isNotFoundErrorIdentifier(text) + case "error", "message", "detail", "error_description", "title": + if isExplicitModelNotFoundMessage(text, requestedModel) { + return true + } + exactModelReference = exactModelReference || isExactRequestedModelReference(text, requestedModel) + } + } + switch item.(type) { + case map[string]any, []any: + if containsStructuredModelNotFound(item, requestedModel) { + return true + } + } + } + return notFoundType && exactModelReference + case []any: + for _, item := range typed { + if text, isString := item.(string); isString && isExplicitModelNotFoundMessage(text, requestedModel) { + return true + } + if containsStructuredModelNotFound(item, requestedModel) { + return true + } + } + } + return false +} + +func isModelNotFoundIdentifier(value string) bool { + candidate := strings.ToLower(strings.TrimSpace(value)) + if fragment := strings.LastIndex(candidate, "#"); fragment >= 0 && fragment+1 < len(candidate) { + candidate = candidate[fragment+1:] + } else { + if query := strings.Index(candidate, "?"); query >= 0 { + candidate = candidate[:query] + } + candidate = strings.TrimRight(candidate, "/") + if separator := strings.LastIndexAny(candidate, "/:"); separator >= 0 { + candidate = candidate[separator+1:] + } + } + normalized := strings.NewReplacer("-", "_", " ", "_").Replace(candidate) + switch normalized { + case "model_not_found", "model_not_found_error", "unknown_model", "model_does_not_exist", "model_not_exist": + return true + default: + return false + } +} + +func isNotFoundErrorIdentifier(value string) bool { + normalized := strings.NewReplacer("-", "_", " ", "_").Replace(strings.ToLower(strings.TrimSpace(value))) + return normalized == "not_found" || normalized == "not_found_error" +} + +func isExplicitModelNotFoundMessage(message, requestedModel string) bool { + lower := strings.Trim(strings.ToLower(strings.TrimSpace(message)), " .!;\t\r\n") + if lower == "" { + return false + } + normalized := strings.NewReplacer("-", "_", " ", "_").Replace(lower) + if strings.Contains(normalized, "model_not_found") || strings.Contains(normalized, "unknown_model") { + return true + } + for _, prefix := range []string{"no such model", "unknown model"} { + if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") { + continue + } + remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix)) + remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":")) + if remainder == "" { + return true + } + missingSuffix, matches := trimRequestedModelReference(remainder, requestedModel) + return matches && missingSuffix == "" + } + for _, prefix := range []string{"the requested model", "requested model", "the model", "model"} { + if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") { + continue + } + remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix)) + remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":")) + if isMissingModelPhrase(remainder) { + return true + } + missingSuffix, matches := trimRequestedModelReference(remainder, requestedModel) + return matches && isMissingModelPhrase(missingSuffix) + } + return false +} + +func isExactRequestedModelReference(message, requestedModel string) bool { + lower := strings.Trim(strings.ToLower(strings.TrimSpace(message)), " .!;\t\r\n") + for _, prefix := range []string{"the requested model", "requested model", "the model", "model"} { + if lower != prefix && !strings.HasPrefix(lower, prefix+" ") && !strings.HasPrefix(lower, prefix+":") { + continue + } + remainder := strings.TrimSpace(strings.TrimPrefix(lower, prefix)) + remainder = strings.TrimSpace(strings.TrimPrefix(remainder, ":")) + suffix, matches := trimRequestedModelReference(remainder, requestedModel) + return matches && suffix == "" + } + return false +} + +func trimRequestedModelReference(value, requestedModel string) (string, bool) { + model := strings.ToLower(strings.TrimSpace(requestedModel)) + if model == "" { + return "", false + } + for _, candidate := range []string{model, "'" + model + "'", `"` + model + `"`, "`" + model + "`"} { + if value == candidate { + return "", true + } + if !strings.HasPrefix(value, candidate) { + continue + } + remainder := value[len(candidate):] + if remainder == "" || strings.ContainsRune(" :,", rune(remainder[0])) { + return strings.TrimLeft(remainder, " :,"), true + } + } + return "", false +} + +func isMissingModelPhrase(value string) bool { + switch strings.Trim(value, " .!;\t\r\n") { + case "not found", "was not found", "could not be found", "does not exist", "doesn't exist", "not exist", "is unknown": + return true + default: + return false + } +} + // isRequestInvalidError returns true if the error represents a client request // error that should not be retried. Specifically, it treats 400 responses with // "invalid_request_error", request-scoped 404 item misses caused by `store=false`, diff --git a/sdk/cliproxy/auth/conductor_overrides_test.go b/sdk/cliproxy/auth/conductor_overrides_test.go index 1cb0b2490..c4b606f8a 100644 --- a/sdk/cliproxy/auth/conductor_overrides_test.go +++ b/sdk/cliproxy/auth/conductor_overrides_test.go @@ -2,6 +2,8 @@ package auth import ( "context" + "errors" + "fmt" "net/http" "sync" "testing" @@ -162,6 +164,7 @@ type authFallbackExecutor struct { streamCalls []string executeErrors map[string]error streamFirstErrors map[string]error + countTokenErrors map[string]error } func (e *authFallbackExecutor) Identifier() string { @@ -200,8 +203,14 @@ func (e *authFallbackExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, er return auth, nil } -func (e *authFallbackExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { - return cliproxyexecutor.Response{}, &Error{HTTPStatus: 500, Message: "not implemented"} +func (e *authFallbackExecutor) CountTokens(_ context.Context, auth *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + e.mu.Lock() + err := e.countTokenErrors[auth.ID] + e.mu.Unlock() + if err != nil { + return cliproxyexecutor.Response{}, err + } + return cliproxyexecutor.Response{Payload: []byte(auth.ID)}, nil } func (e *authFallbackExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) { @@ -224,6 +233,27 @@ func (e *authFallbackExecutor) StreamCalls() []string { return out } +type resultCaptureHook struct { + NoopHook + + mu sync.Mutex + results []Result +} + +func (h *resultCaptureHook) OnResult(_ context.Context, result Result) { + h.mu.Lock() + h.results = append(h.results, result) + h.mu.Unlock() +} + +func (h *resultCaptureHook) Results() []Result { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]Result, len(h.results)) + copy(out, h.results) + return out +} + type retryAfterStatusError struct { status int message string @@ -1269,6 +1299,369 @@ func TestManager_MarkResult_RequestScopedNotFoundDoesNotCooldownAuth(t *testing. } } +func TestManager_ExecuteCount_GenericRouteNotFoundDoesNotSuspendModel(t *testing.T) { + previous := quotaCooldownDisabled.Load() + quotaCooldownDisabled.Store(false) + t.Cleanup(func() { quotaCooldownDisabled.Store(previous) }) + + hook := &resultCaptureHook{} + m := NewManager(nil, nil, hook) + executor := &authFallbackExecutor{ + id: "claude", + countTokenErrors: map[string]error{ + "count-route-not-found-auth": &Error{ + HTTPStatus: http.StatusNotFound, + Message: "404 page not found", + }, + }, + } + m.RegisterExecutor(executor) + + model := "count-route-not-found-model" + auth := &Auth{ID: "count-route-not-found-auth", Provider: "claude"} + reg := registry.GetGlobalRegistry() + reg.RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { reg.UnregisterClient(auth.ID) }) + + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + if _, errCount := m.ExecuteCount(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}); errCount == nil { + t.Fatal("expected count_tokens route 404 error") + } + + updated, ok := m.GetByID(auth.ID) + if !ok || updated == nil { + t.Fatal("expected auth to remain registered") + } + if updated.Failed != 1 { + t.Fatalf("failed request count = %d, want 1", updated.Failed) + } + results := hook.Results() + if len(results) != 1 || results[0].Success || results[0].Error == nil || results[0].Error.HTTPStatus != http.StatusNotFound { + t.Fatalf("recorded results = %#v, want one failed 404", results) + } + if updated.Unavailable { + t.Fatal("expected route 404 to keep auth available") + } + if state := updated.ModelStates[model]; state != nil { + t.Fatalf("expected route 404 to avoid model cooldown state, got %#v", state) + } + if count := reg.GetModelCount(model); count != 1 { + t.Fatalf("available model count = %d, want 1", count) + } + + resp, errExecute := m.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}) + if errExecute != nil { + t.Fatalf("execute after count_tokens route 404: %v", errExecute) + } + if string(resp.Payload) != auth.ID { + t.Fatalf("execute payload = %q, want %q", string(resp.Payload), auth.ID) + } +} + +func TestManager_ExecuteCount_ExplicitModelNotFoundSuspendsModel(t *testing.T) { + previous := quotaCooldownDisabled.Load() + quotaCooldownDisabled.Store(false) + t.Cleanup(func() { quotaCooldownDisabled.Store(previous) }) + + hook := &resultCaptureHook{} + m := NewManager(nil, nil, hook) + executor := &authFallbackExecutor{ + id: "claude", + countTokenErrors: map[string]error{ + "count-model-not-found-auth": &Error{ + Code: "model_not_found", + HTTPStatus: http.StatusNotFound, + Message: `{"type":"error","error":{"type":"not_found_error","message":"model count-explicitly-missing-model was not found"}}`, + }, + }, + } + m.RegisterExecutor(executor) + + model := "count-explicitly-missing-model" + auth := &Auth{ID: "count-model-not-found-auth", Provider: "claude"} + reg := registry.GetGlobalRegistry() + reg.RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { reg.UnregisterClient(auth.ID) }) + + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + if _, errCount := m.ExecuteCount(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}); errCount == nil { + t.Fatal("expected count_tokens model-not-found error") + } + + updated, ok := m.GetByID(auth.ID) + if !ok || updated == nil { + t.Fatal("expected auth to remain registered") + } + state := updated.ModelStates[model] + if state == nil || !state.Unavailable { + t.Fatalf("expected model-not-found cooldown state, got %#v", state) + } + if state.LastError == nil || state.LastError.Code != "model_not_found" { + t.Fatalf("model state error = %#v, want preserved model_not_found code", state.LastError) + } + results := hook.Results() + if len(results) != 1 || results[0].Error == nil || results[0].Error.Code != "model_not_found" { + t.Fatalf("hook results = %#v, want preserved model_not_found code", results) + } + remaining := time.Until(state.NextRetryAfter) + if remaining < 11*time.Hour || remaining > 12*time.Hour { + t.Fatalf("model-not-found cooldown = %v, want about 12h", remaining) + } + if count := reg.GetModelCount(model); count != 0 { + t.Fatalf("available model count = %d, want 0", count) + } +} + +func TestIsCountTokensEndpointNotFoundError(t *testing.T) { + tests := []struct { + name string + err error + model string + want bool + }{ + { + name: "empty router 404", + err: &Error{HTTPStatus: http.StatusNotFound}, + want: true, + }, + { + name: "plain router 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: "404 page not found"}, + want: true, + }, + { + name: "wrapped router 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: "upstream request failed: 404 page not found"}, + want: true, + }, + { + name: "fastapi route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"detail":"Not Found"}`}, + want: true, + }, + { + name: "problem details route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"title":"Not Found","status":404}`}, + want: true, + }, + { + name: "nested generic route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"Not Found"}}`}, + want: true, + }, + { + name: "generic model api route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"type":"not_found_error","title":"Model API","detail":"Not Found"}`}, + want: true, + }, + { + name: "generic model metadata route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"model metadata route not found"}}`}, + want: true, + }, + { + name: "generic model provider 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"model provider was not found"}}`}, + want: true, + }, + { + name: "generic route with misleading metadata", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"message":"Not Found","request_id":"model_not_found"}`}, + want: true, + }, + { + name: "express count route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: "Cannot POST /v1/messages/count_tokens"}, + want: true, + }, + { + name: "html route 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: "404 Not Found"}, + want: true, + }, + { + name: "structured model 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"model claude-missing was not found"}}`}, + want: false, + }, + { + name: "anthropic exact model reference", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"model: claude-missing"}}`}, + want: false, + }, + { + name: "anthropic model reference with thinking suffix", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"model: claude-missing"}}`}, + model: "claude-missing(high)", + want: false, + }, + { + name: "requested model does not exist", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"The requested model does not exist"}}`}, + want: false, + }, + { + name: "requested quoted model could not be found", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":{"type":"not_found_error","message":"The requested model 'foo' could not be found"}}`}, + model: "foo", + want: false, + }, + { + name: "problem details model type uri", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"type":"https://example.com/problems/model-not-found","title":"Not Found","status":404}`}, + want: false, + }, + { + name: "structured model error string", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"error":"model claude-missing does not exist"}`}, + want: false, + }, + { + name: "model code with generic message", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"message":"Not Found","code":"model_not_found","model":"claude-missing"}`}, + want: false, + }, + { + name: "typed model not found code", + err: &Error{Code: "model_not_found", HTTPStatus: http.StatusNotFound, Message: "Not Found"}, + want: false, + }, + { + name: "typed wrapper with structured model code", + err: &Error{Code: "not_found", HTTPStatus: http.StatusNotFound, Message: `{"error":{"code":"model_not_found","message":"Not Found"}}`}, + want: false, + }, + { + name: "wrapped structured model code", + err: fmt.Errorf("upstream failed: %w", &requestScopedStatusError{ + status: http.StatusNotFound, + message: `{"error":{"code":"model_not_found","message":"Not Found"}}`, + }), + want: false, + }, + { + name: "joined structured model code", + err: errors.Join( + errors.New("upstream failed"), + &requestScopedStatusError{ + status: http.StatusNotFound, + message: `{"error":{"code":"model_not_found","message":"Not Found"}}`, + }, + ), + want: false, + }, + { + name: "outer generic inner model 404", + err: &Error{HTTPStatus: http.StatusNotFound, Message: `{"message":"Not Found","error":{"type":"not_found_error","message":"model claude-missing does not exist"}}`}, + want: false, + }, + { + name: "unstructured model text", + err: &Error{HTTPStatus: http.StatusNotFound, Message: "model claude-missing was not found"}, + want: true, + }, + { + name: "non 404", + err: &Error{HTTPStatus: http.StatusInternalServerError, Message: "404 page not found"}, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + model := tc.model + if model == "" { + model = "claude-missing" + } + if got := isCountTokensEndpointNotFoundError(tc.err, model); got != tc.want { + t.Fatalf("isCountTokensEndpointNotFoundError() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestManager_Execute_GenericRouteNotFoundStillSuspendsModel(t *testing.T) { + previous := quotaCooldownDisabled.Load() + quotaCooldownDisabled.Store(false) + t.Cleanup(func() { quotaCooldownDisabled.Store(previous) }) + + m := NewManager(nil, nil, nil) + executor := &authFallbackExecutor{ + id: "claude", + executeErrors: map[string]error{ + "messages-route-not-found-auth": &Error{ + HTTPStatus: http.StatusNotFound, + Message: "404 page not found", + }, + }, + } + m.RegisterExecutor(executor) + + model := "messages-route-not-found-model" + auth := &Auth{ID: "messages-route-not-found-auth", Provider: "claude"} + reg := registry.GetGlobalRegistry() + reg.RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { reg.UnregisterClient(auth.ID) }) + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + if _, errExecute := m.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}); errExecute == nil { + t.Fatal("expected messages route 404") + } + + updated, ok := m.GetByID(auth.ID) + if !ok || updated == nil { + t.Fatal("expected auth to remain registered") + } + state := updated.ModelStates[model] + if state == nil || !state.Unavailable || state.NextRetryAfter.IsZero() { + t.Fatalf("expected ordinary messages 404 to suspend model, got %#v", state) + } +} + +func TestManager_RecordResult_AvailabilityNeutralSkipsSchedulerUpdate(t *testing.T) { + m := NewManager(nil, nil, nil) + auth := &Auth{ID: "availability-neutral-auth", Provider: "claude"} + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + m.scheduler.mu.Lock() + provider := m.scheduler.providers[auth.Provider] + if provider == nil || provider.auths[auth.ID] == nil { + m.scheduler.mu.Unlock() + t.Fatal("expected scheduler auth metadata") + } + before := provider.auths[auth.ID].auth + m.scheduler.mu.Unlock() + + m.recordAvailabilityNeutralResult(context.Background(), Result{ + AuthID: auth.ID, + Provider: auth.Provider, + Model: "availability-neutral-model", + Success: false, + Error: &Error{HTTPStatus: http.StatusNotFound, Message: "404 page not found"}, + }) + + updated, ok := m.GetByID(auth.ID) + if !ok || updated == nil || updated.Failed != 1 { + t.Fatalf("updated auth = %#v, want one recorded failure", updated) + } + m.scheduler.mu.Lock() + after := m.scheduler.providers[auth.Provider].auths[auth.ID].auth + m.scheduler.mu.Unlock() + if after != before { + t.Fatal("availability-neutral result unexpectedly replaced scheduler auth snapshot") + } +} + func TestManager_RequestScopedNotFoundStopsRetryWithoutSuspendingAuth(t *testing.T) { m := NewManager(nil, nil, nil) executor := &authFallbackExecutor{