From b285660f84f6e7e02097bbbc8a421ae4bb5d6fca Mon Sep 17 00:00:00 2001 From: Luis Pater Date: Fri, 13 Nov 2026 03:05:54 +0800 Subject: [PATCH] feat(api): expose standalone in-flight request count on management API - Add a lightweight per-request in-flight tracker on the auth manager that covers selection, upstream calls, retry waits, and the full stream lifetime; it stays disabled in Home mode where the execution registry is authoritative. - Register GET /v8/management/observability/requests/in-flight returning counts only (in_flight, oldest_started_at, by_provider) with no request bodies or credential identifiers. Closes: #7476 Co-authored-by: Dylan <1990016@gmail.com> Co-authored-by: lzt404 <2596933790@qq.com> --- internal/api/handlers/management/in_flight.go | 40 ++ .../api/handlers/management/in_flight_test.go | 63 +++ internal/api/server_management_v8.go | 1 + sdk/cliproxy/auth/conductor.go | 2 + sdk/cliproxy/auth/conductor_execution.go | 25 +- sdk/cliproxy/auth/conductor_home.go | 2 + sdk/cliproxy/auth/conductor_inflight.go | 227 ++++++++++ sdk/cliproxy/auth/conductor_inflight_test.go | 409 ++++++++++++++++++ 8 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 internal/api/handlers/management/in_flight.go create mode 100644 internal/api/handlers/management/in_flight_test.go create mode 100644 sdk/cliproxy/auth/conductor_inflight.go create mode 100644 sdk/cliproxy/auth/conductor_inflight_test.go diff --git a/internal/api/handlers/management/in_flight.go b/internal/api/handlers/management/in_flight.go new file mode 100644 index 0000000000..b4ba9bc50f --- /dev/null +++ b/internal/api/handlers/management/in_flight.go @@ -0,0 +1,40 @@ +package management + +import ( + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +// inFlightResponse is the read-only payload for the standalone in-flight +// request count endpoint. It intentionally exposes counts only: no request +// bodies, prompts, client addresses, or credential identifiers. +type inFlightResponse struct { + InFlight int64 `json:"in_flight"` + OldestStartedAt *time.Time `json:"oldest_started_at,omitempty"` + ByProvider map[string]int64 `json:"by_provider"` +} + +// GetInFlightRequests reports the current in-flight request count on a +// standalone instance. In Home mode the snapshot stays empty because the +// execution registry is the authoritative in-flight source there. +func (h *Handler) GetInFlightRequests(c *gin.Context) { + if h == nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "handler unavailable"}) + return + } + resp := inFlightResponse{ByProvider: map[string]int64{}} + if h.authManager != nil { + snapshot := h.authManager.InFlightSnapshot() + resp.InFlight = snapshot.InFlight + if !snapshot.OldestStartedAt.IsZero() { + oldest := snapshot.OldestStartedAt.UTC() + resp.OldestStartedAt = &oldest + } + if len(snapshot.ByProvider) > 0 { + resp.ByProvider = snapshot.ByProvider + } + } + c.JSON(http.StatusOK, resp) +} diff --git a/internal/api/handlers/management/in_flight_test.go b/internal/api/handlers/management/in_flight_test.go new file mode 100644 index 0000000000..755c490fc1 --- /dev/null +++ b/internal/api/handlers/management/in_flight_test.go @@ -0,0 +1,63 @@ +package management + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" +) + +func TestGetInFlightRequestsReturnsSnapshot(t *testing.T) { + gin.SetMode(gin.TestMode) + manager := coreauth.NewManager(nil, nil, nil) + h := &Handler{authManager: manager} + + rec := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(rec) + ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil) + + h.GetInFlightRequests(ginCtx) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var payload struct { + InFlight int64 `json:"in_flight"` + OldestStartedAt string `json:"oldest_started_at"` + ByProvider map[string]int64 `json:"by_provider"` + } + if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil { + t.Fatalf("unmarshal response: %v", errUnmarshal) + } + if payload.InFlight != 0 { + t.Fatalf("in_flight = %d, want 0", payload.InFlight) + } +} + +func TestGetInFlightRequestsNilManager(t *testing.T) { + gin.SetMode(gin.TestMode) + h := &Handler{} + + rec := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(rec) + ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil) + + h.GetInFlightRequests(ginCtx) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + var payload struct { + InFlight int64 `json:"in_flight"` + } + if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil { + t.Fatalf("unmarshal response: %v", errUnmarshal) + } + if payload.InFlight != 0 { + t.Fatalf("in_flight = %d, want 0", payload.InFlight) + } +} diff --git a/internal/api/server_management_v8.go b/internal/api/server_management_v8.go index e73ad89313..2b4dee0408 100644 --- a/internal/api/server_management_v8.go +++ b/internal/api/server_management_v8.go @@ -41,6 +41,7 @@ func (s *Server) registerManagementV8Routes() { v8.GET("/observability/logs/requests/:id", s.mgmt.GetRequestLogByID) v8.GET("/observability/usage/api-keys", s.mgmt.GetAPIKeyUsage) v8.GET("/observability/usage/queue", s.mgmt.GetUsageQueue) + v8.GET("/observability/requests/in-flight", s.mgmt.GetInFlightRequests) v8.GET("/credentials", s.mgmt.ListAuthFiles) v8.POST("/credentials", s.mgmt.UploadAuthFileV8) diff --git a/sdk/cliproxy/auth/conductor.go b/sdk/cliproxy/auth/conductor.go index 3f485fcf10..a18c263e45 100644 --- a/sdk/cliproxy/auth/conductor.go +++ b/sdk/cliproxy/auth/conductor.go @@ -172,6 +172,7 @@ type resultPolicyHolder struct { // Manager orchestrates auth lifecycle, selection, execution, and persistence. type Manager struct { + inFlightTracker *inFlightTracker store Store cooldownStore CooldownStateStore pendingCooldownStateStore CooldownStateStore @@ -258,6 +259,7 @@ func NewManager(store Store, selector Selector, hook Hook) *Manager { hook = NoopHook{} } manager := &Manager{ + inFlightTracker: newInFlightTracker(), store: store, authLoadGate: semaphore.NewWeighted(math.MaxInt64), executors: make(map[string]ProviderExecutor), diff --git a/sdk/cliproxy/auth/conductor_execution.go b/sdk/cliproxy/auth/conductor_execution.go index 0a9c9735a1..a05f57e584 100644 --- a/sdk/cliproxy/auth/conductor_execution.go +++ b/sdk/cliproxy/auth/conductor_execution.go @@ -200,10 +200,13 @@ func isOpenAIImageExecutionOptions(opts cliproxyexecutor.Options) bool { } func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + scope := m.beginInFlight("") + defer func() { scope.end() }() ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL) req, opts = cliproxysession.Enrich(req, opts) ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers)) ctx = coreusage.CaptureIdentity(ctx) + ctx = contextWithInFlightScope(ctx, scope) normalized := m.normalizeProviders(providers) if len(normalized) == 0 { return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} @@ -276,10 +279,13 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye // It supports multiple providers for the same model and round-robins the starting provider per model. func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + scope := m.beginInFlight("") + defer func() { scope.end() }() ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL) req, opts = cliproxysession.Enrich(req, opts) ctx = WithCodexClientRequest(ctx, IsCodexClientRequest(ctx, opts.Headers)) ctx = coreusage.CaptureIdentity(ctx) + ctx = contextWithInFlightScope(ctx, scope) normalized := m.normalizeProviders(providers) if len(normalized) == 0 { return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} @@ -338,11 +344,19 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip // ExecuteStream performs a streaming execution using the configured selector and executor. // It supports multiple providers for the same model and round-robins the starting provider per model. -func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { +func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (result *cliproxyexecutor.StreamResult, err error) { + scope := m.beginInFlight("") + defer func() { + if scope != nil { + result = wrapInFlightStreamResult(result, scope, ctx) + scope = nil + } + }() ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL) req, opts = cliproxysession.Enrich(req, opts) ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers)) ctx = coreusage.CaptureIdentity(ctx) + ctx = contextWithInFlightScope(ctx, scope) if m.HomeEnabled() { if unlockSession := m.lockHomeWebsocketSession(ctx, opts); unlockSession != nil { defer unlockSession() @@ -658,6 +672,9 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req pickOpts = withHomeExcludedAuthIDs(pickOpts, tried) } auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried) + if errPick == nil { + updateInFlightScopeProvider(ctx, provider) + } if errPick != nil { if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) { return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick) @@ -876,6 +893,9 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, pickOpts = withHomeExcludedAuthIDs(pickOpts, tried) } auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried) + if errPick == nil { + updateInFlightScopeProvider(ctx, provider) + } if errPick != nil { if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) { return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick) @@ -1124,6 +1144,9 @@ func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string } } else { auth, executor, provider, errPick = m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried) + if errPick == nil { + updateInFlightScopeProvider(ctx, provider) + } } if errPick != nil { preferredErr := preferredExecutionAttemptError(lastErr, upstreamErr) diff --git a/sdk/cliproxy/auth/conductor_home.go b/sdk/cliproxy/auth/conductor_home.go index ddd7a2bd23..ba9fce8a3d 100644 --- a/sdk/cliproxy/auth/conductor_home.go +++ b/sdk/cliproxy/auth/conductor_home.go @@ -1473,6 +1473,7 @@ func shouldAttemptAntigravityCreditsFallback(m *Manager, lastErr error, provider } func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, bool, error) { + updateInFlightScopeProvider(ctx, "antigravity") if m != nil && m.HomeEnabled() { return cliproxyexecutor.Response{}, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable} } @@ -1541,6 +1542,7 @@ func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxy } func (m *Manager) tryAntigravityCreditsExecuteStream(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, bool, error) { + updateInFlightScopeProvider(ctx, "antigravity") if m != nil && m.HomeEnabled() { return nil, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable} } diff --git a/sdk/cliproxy/auth/conductor_inflight.go b/sdk/cliproxy/auth/conductor_inflight.go new file mode 100644 index 0000000000..45cbd6a949 --- /dev/null +++ b/sdk/cliproxy/auth/conductor_inflight.go @@ -0,0 +1,227 @@ +package auth + +import ( + "context" + "sync" + "time" + + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +// InFlightSnapshot is an immutable view of standalone in-flight executions. +type InFlightSnapshot struct { + InFlight int64 + OldestStartedAt time.Time + ByProvider map[string]int64 +} + +// inFlightTracker counts in-flight local executions for standalone instances. +// It stays disabled while Home mode is active because the execution registry +// already tracks Home-dispatched executions for that lifetime. +type inFlightTracker struct { + mu sync.Mutex + count int64 + oldestStarted time.Time + byProvider map[string]int64 + inflightScopes map[*inFlightScope]time.Time +} + +func newInFlightTracker() *inFlightTracker { + return &inFlightTracker{ + byProvider: make(map[string]int64), + inflightScopes: make(map[*inFlightScope]time.Time), + } +} + +// inFlightScope represents one tracked execution until its end is called. +type inFlightScope struct { + tracker *inFlightTracker + provider string + done sync.Once +} + +// setProvider updates the provider attribution of an in-flight scope without +// changing the total count or its start time. +func (s *inFlightScope) setProvider(provider string) { + if s == nil || s.tracker == nil { + return + } + t := s.tracker + t.mu.Lock() + defer t.mu.Unlock() + if _, ok := t.inflightScopes[s]; !ok || s.provider == provider { + return + } + if next := t.byProvider[s.provider] - 1; next > 0 { + t.byProvider[s.provider] = next + } else { + delete(t.byProvider, s.provider) + } + s.provider = provider + t.byProvider[provider]++ +} + +// end releases the scope exactly once. +func (s *inFlightScope) end() { + if s == nil || s.tracker == nil { + return + } + s.done.Do(func() { + t := s.tracker + t.mu.Lock() + defer t.mu.Unlock() + startedAt, ok := t.inflightScopes[s] + if !ok { + return + } + delete(t.inflightScopes, s) + t.count-- + if next := t.byProvider[s.provider] - 1; next > 0 { + t.byProvider[s.provider] = next + } else { + delete(t.byProvider, s.provider) + } + if t.count == 0 { + t.oldestStarted = time.Time{} + } else if startedAt.Equal(t.oldestStarted) { + t.recomputeOldestLocked() + } + }) +} + +func (t *inFlightTracker) recomputeOldestLocked() { + var oldest time.Time + for _, startedAt := range t.inflightScopes { + if oldest.IsZero() || startedAt.Before(oldest) { + oldest = startedAt + } + } + t.oldestStarted = oldest +} + +// begin tracks a new execution. The returned scope must be ended exactly once. +func (t *inFlightTracker) begin(provider string) *inFlightScope { + if t == nil { + return nil + } + scope := &inFlightScope{tracker: t, provider: provider} + t.mu.Lock() + defer t.mu.Unlock() + now := time.Now() + t.inflightScopes[scope] = now + t.count++ + t.byProvider[provider]++ + if t.oldestStarted.IsZero() || now.Before(t.oldestStarted) { + t.oldestStarted = now + } + return scope +} + +// snapshot copies the current in-flight state into an immutable view. +func (t *inFlightTracker) snapshot() InFlightSnapshot { + if t == nil { + return InFlightSnapshot{ByProvider: map[string]int64{}} + } + t.mu.Lock() + defer t.mu.Unlock() + byProvider := make(map[string]int64, len(t.byProvider)) + for provider, count := range t.byProvider { + byProvider[provider] = count + } + return InFlightSnapshot{ + InFlight: t.count, + OldestStartedAt: t.oldestStarted, + ByProvider: byProvider, + } +} + +// InFlightSnapshot returns the current standalone in-flight execution counts. +// It reports an empty snapshot while Home mode is enabled, where the execution +// registry is the authoritative in-flight source instead. +func (m *Manager) InFlightSnapshot() InFlightSnapshot { + if m == nil || m.inFlightTracker == nil || m.HomeEnabled() { + return InFlightSnapshot{ByProvider: map[string]int64{}} + } + return m.inFlightTracker.snapshot() +} + +// beginInFlight starts tracking one local execution. The returned scope must be +// ended exactly once; it is nil-safe and ends as a no-op in Home mode. +func (m *Manager) beginInFlight(provider string) *inFlightScope { + if m == nil || m.inFlightTracker == nil || m.HomeEnabled() { + return nil + } + return m.inFlightTracker.begin(provider) +} + +// wrapInFlightStreamResult extends an in-flight scope until the returned +// stream finishes or ctx is canceled, whichever comes first. +func wrapInFlightStreamResult(result *cliproxyexecutor.StreamResult, scope *inFlightScope, ctx context.Context) *cliproxyexecutor.StreamResult { + if result == nil || scope == nil { + if scope != nil { + scope.end() + } + return result + } + out := make(chan cliproxyexecutor.StreamChunk) + go func() { + // Release the in-flight scope as soon as forwarding stops (stream end + // or cancellation). The output channel closes only after the inner + // channel is fully drained so callers keep the original happens-before + // between stream end and upstream result bookkeeping. + defer scope.end() + if result.Chunks == nil { + close(out) + return + } + forward := true + for forward { + select { + case <-ctx.Done(): + forward = false + case chunk, ok := <-result.Chunks: + if !ok { + close(out) + return + } + select { + case <-ctx.Done(): + forward = false + case out <- chunk: + } + } + } + // After cancellation, drain the inner channel on the same goroutine so + // the inner stream wrapper finishes its bookkeeping before out closes. + // The scope is released explicitly here before the drain can block; + // the deferred end above remains as the safety net for other exits. + scope.end() + for range result.Chunks { + } + close(out) + }() + return &cliproxyexecutor.StreamResult{Headers: result.Headers, Chunks: out} +} + +// inFlightScopeContextKey carries the request in-flight scope to inner +// selection code so the provider attribution can be corrected after pick. +type inFlightScopeContextKey struct{} + +// contextWithInFlightScope returns a context carrying the scope, or ctx when +// scope is nil. +func contextWithInFlightScope(ctx context.Context, scope *inFlightScope) context.Context { + if ctx == nil || scope == nil { + return ctx + } + return context.WithValue(ctx, inFlightScopeContextKey{}, scope) +} + +// updateInFlightScopeProvider re-attributes the request scope to the actually +// selected provider after auth selection. It is a no-op without a scope. +func updateInFlightScopeProvider(ctx context.Context, provider string) { + if ctx == nil { + return + } + scope, _ := ctx.Value(inFlightScopeContextKey{}).(*inFlightScope) + scope.setProvider(provider) +} diff --git a/sdk/cliproxy/auth/conductor_inflight_test.go b/sdk/cliproxy/auth/conductor_inflight_test.go new file mode 100644 index 0000000000..2f16f9b78c --- /dev/null +++ b/sdk/cliproxy/auth/conductor_inflight_test.go @@ -0,0 +1,409 @@ +package auth + +import ( + "context" + "errors" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +// inFlightBlockingExecutor blocks inside Execute until released so the test can +// observe the manager in-flight snapshot mid-request. +type inFlightBlockingExecutor struct { + id string + release chan struct{} + entered chan struct{} + releaseMu sync.Once +} + +func newInFlightBlockingExecutor(id string) *inFlightBlockingExecutor { + return &inFlightBlockingExecutor{ + id: id, + release: make(chan struct{}), + entered: make(chan struct{}, 4), + } +} + +func (e *inFlightBlockingExecutor) Identifier() string { return e.id } + +func (e *inFlightBlockingExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + e.entered <- struct{}{} + <-e.release + return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil +} + +func (e *inFlightBlockingExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { + return nil, errors.New("stream not supported") +} + +func (e *inFlightBlockingExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) { + return auth, nil +} + +func (e *inFlightBlockingExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + return cliproxyexecutor.Response{}, errors.New("not supported") +} + +func (e *inFlightBlockingExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) { + return nil, errors.New("not supported") +} + +// inFlightStreamExecutor returns a stream whose chunks channel stays open until +// the test closes it, letting the stream lifetime be observed via the snapshot. +type inFlightStreamExecutor struct { + id string + chunks chan cliproxyexecutor.StreamChunk +} + +func (e *inFlightStreamExecutor) Identifier() string { return e.id } + +func (e *inFlightStreamExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + return cliproxyexecutor.Response{}, errors.New("not supported") +} + +func (e *inFlightStreamExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { + return &cliproxyexecutor.StreamResult{Chunks: e.chunks}, nil +} + +func (e *inFlightStreamExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) { + return auth, nil +} + +func (e *inFlightStreamExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + return cliproxyexecutor.Response{}, errors.New("not supported") +} + +func (e *inFlightStreamExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) { + return nil, errors.New("not supported") +} + +func registerInFlightAuth(t *testing.T, m *Manager, authID, provider, model string) { + t.Helper() + auth := &Auth{ID: authID, Provider: provider, Status: StatusActive} + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("Register auth: %v", errRegister) + } + registry.GetGlobalRegistry().RegisterClient(authID, provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient(authID) + }) +} + +func waitForInFlightCount(t *testing.T, m *Manager, want int) InFlightSnapshot { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + snapshot := m.InFlightSnapshot() + if snapshot.InFlight == int64(want) { + return snapshot + } + if time.Now().After(deadline) { + t.Fatalf("in-flight count = %d, want %d (snapshot: %+v)", snapshot.InFlight, want, snapshot) + } + time.Sleep(5 * time.Millisecond) + } +} + +// TestManager_InFlightSnapshot_NonStreaming verifies that an in-flight +// non-streaming request is visible in the manager snapshot while it executes. +func TestManager_InFlightSnapshot_NonStreaming(t *testing.T) { + executor := newInFlightBlockingExecutor("inflight-provider") + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + + model := "inflight-model" + registerInFlightAuth(t, m, "auth-inflight-1", executor.Identifier(), model) + + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)} + done := make(chan error, 1) + go func() { + _, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{}) + done <- errExec + }() + + <-executor.entered + snapshot := waitForInFlightCount(t, m, 1) + if snapshot.ByProvider[executor.Identifier()] != 1 { + t.Fatalf("by_provider[%q] = %d, want 1 (snapshot: %+v)", executor.Identifier(), snapshot.ByProvider[executor.Identifier()], snapshot) + } + if snapshot.OldestStartedAt.IsZero() { + t.Fatal("expected oldest_started_at to be set while a request is in flight") + } + + executor.releaseMu.Do(func() { close(executor.release) }) + if errExec := <-done; errExec != nil { + t.Fatalf("Execute() error = %v", errExec) + } + waitForInFlightCount(t, m, 0) +} + +// TestManager_InFlightSnapshot_Streaming verifies that a long-lived streaming +// request remains counted until the chunk channel closes. +func TestManager_InFlightSnapshot_Streaming(t *testing.T) { + chunks := make(chan cliproxyexecutor.StreamChunk) + executor := &inFlightStreamExecutor{id: "inflight-stream-provider", chunks: chunks} + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + + model := "inflight-stream-model" + registerInFlightAuth(t, m, "auth-inflight-stream-1", executor.Identifier(), model) + + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)} + type streamOutcome struct { + result *cliproxyexecutor.StreamResult + err error + } + outcome := make(chan streamOutcome, 1) + go func() { + result, errStream := m.ExecuteStream(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true}) + outcome <- streamOutcome{result: result, err: errStream} + }() + // readStreamBootstrap blocks until the first payload chunk arrives. + chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")} + streamRes := <-outcome + result, errStream := streamRes.result, streamRes.err + if errStream != nil { + t.Fatalf("ExecuteStream() error = %v", errStream) + } + + waitForInFlightCount(t, m, 1) + + chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: hello\n\n")} + if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 { + t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload) + } + if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 { + t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot) + } + + close(chunks) + for range result.Chunks { + } + waitForInFlightCount(t, m, 0) +} + +// TestManager_InFlightSnapshot_DisabledInHomeMode verifies that the standalone +// in-flight tracker stays empty when Home mode is enabled, since the execution +// registry covers that mode. +func TestManager_InFlightSnapshot_DisabledInHomeMode(t *testing.T) { + executor := newInFlightBlockingExecutor("inflight-home-provider") + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + m.SetConfigSnapshot(&internalconfig.Config{Home: internalconfig.HomeConfig{Enabled: true}}) + + snapshot := m.InFlightSnapshot() + if snapshot.InFlight != 0 { + t.Fatalf("expected empty snapshot in home mode, got %+v", snapshot) + } +} + +// inFlightRetryableStatusError is a retryable upstream error for retry tests. +type inFlightRetryableStatusError struct{} + +func (inFlightRetryableStatusError) Error() string { return "upstream unavailable" } +func (inFlightRetryableStatusError) StatusCode() int { return http.StatusServiceUnavailable } + +// inFlightFailThenBlockExecutor fails the first Execute call with a retryable +// error and blocks in subsequent calls until released. +type inFlightFailThenBlockExecutor struct { + id string + calls atomic.Int32 + release chan struct{} + releaseOnce sync.Once +} + +func newInFlightFailThenBlockExecutor(id string) *inFlightFailThenBlockExecutor { + return &inFlightFailThenBlockExecutor{id: id, release: make(chan struct{})} +} + +func (e *inFlightFailThenBlockExecutor) Identifier() string { return e.id } + +func (e *inFlightFailThenBlockExecutor) Execute(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + call := e.calls.Add(1) + if call == 1 { + cliproxyexecutor.MarkUpstreamAttempt(ctx) + return cliproxyexecutor.Response{}, inFlightRetryableStatusError{} + } + <-e.release + return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil +} + +func (e *inFlightFailThenBlockExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) { + return nil, errors.New("stream not supported") +} + +func (e *inFlightFailThenBlockExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) { + return auth, nil +} + +func (e *inFlightFailThenBlockExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { + return cliproxyexecutor.Response{}, errors.New("not supported") +} + +func (e *inFlightFailThenBlockExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) { + return nil, errors.New("not supported") +} + +// TestManager_InFlightSnapshot_CoversRetryWait verifies that a request waiting +// between retry rounds still appears in the snapshot instead of blinking to 0. +func TestManager_InFlightSnapshot_CoversRetryWait(t *testing.T) { + executor := newInFlightFailThenBlockExecutor("inflight-retry-provider") + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + m.SetRetryConfig(1, 20*time.Second, 0) + + model := "inflight-retry-model" + if _, errRegister := m.Register(context.Background(), &Auth{ + ID: "auth-inflight-retry-1", + Provider: executor.Identifier(), + Metadata: map[string]any{"request_retry": 1, "disable_cooling": true}, + }); errRegister != nil { + t.Fatalf("Register auth: %v", errRegister) + } + registry.GetGlobalRegistry().RegisterClient("auth-inflight-retry-1", executor.Identifier(), []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { + registry.GetGlobalRegistry().UnregisterClient("auth-inflight-retry-1") + }) + + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)} + done := make(chan error, 1) + go func() { + _, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{}) + done <- errExec + }() + + // Wait for the first call to record its retryable failure. The request then + // enters the retry wait window while the second call blocks on release. + deadline := time.Now().Add(5 * time.Second) + for executor.calls.Load() < 1 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if executor.calls.Load() < 1 { + t.Fatalf("expected the first upstream call to happen, calls = %d", executor.calls.Load()) + } + // Allow the retry loop to record the failure and enter its wait or the + // blocking second call; either way the request is still active and must + // remain visible in the snapshot. + deadline = time.Now().Add(2 * time.Second) + for { + snapshot := m.InFlightSnapshot() + if snapshot.InFlight == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("in_flight = %d during retry, want 1 (snapshot: %+v)", snapshot.InFlight, snapshot) + } + time.Sleep(5 * time.Millisecond) + } + + executor.releaseOnce.Do(func() { close(executor.release) }) + if errExec := <-done; errExec != nil { + t.Fatalf("Execute() error = %v", errExec) + } + waitForInFlightCount(t, m, 0) +} + +// TestManager_InFlightSnapshot_StreamCancel verifies that canceling the request +// context releases the in-flight scope even while the upstream chunk channel +// stays open and silent after the first chunk was already forwarded. +func TestManager_InFlightSnapshot_StreamCancel(t *testing.T) { + chunks := make(chan cliproxyexecutor.StreamChunk) + executor := &inFlightStreamExecutor{id: "inflight-cancel-provider", chunks: chunks} + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + + model := "inflight-cancel-model" + registerInFlightAuth(t, m, "auth-inflight-cancel-1", executor.Identifier(), model) + + ctx, cancel := context.WithCancel(context.Background()) + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)} + + type streamOutcome struct { + result *cliproxyexecutor.StreamResult + err error + } + outcome := make(chan streamOutcome, 1) + go func() { + result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true}) + outcome <- streamOutcome{result: result, err: errStream} + }() + chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")} + res := <-outcome + if res.err != nil { + t.Fatalf("ExecuteStream() error = %v", res.err) + } + result := res.result + + waitForInFlightCount(t, m, 1) + + // Consume the forwarded bootstrap chunk so the wrapper is blocked waiting + // for the next upstream chunk (receive branch) when cancel happens. + chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: second\n\n")} + if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 { + t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload) + } + if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 { + t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot) + } + + // The upstream keeps its chunk channel open but silent; canceling the + // request context must release the scope without needing another chunk. + cancel() + waitForInFlightCount(t, m, 0) + + // Close the source so the drain path and the output channel can finish + // instead of leaking a background goroutine. + close(chunks) + for range result.Chunks { + } +} + +// TestManager_InFlightSnapshot_StreamCancelBeforeConsume verifies the send +// branch: cancellation while the first chunk is still queued for the caller +// also releases the in-flight scope. +func TestManager_InFlightSnapshot_StreamCancelBeforeConsume(t *testing.T) { + chunks := make(chan cliproxyexecutor.StreamChunk) + executor := &inFlightStreamExecutor{id: "inflight-cancel-early-provider", chunks: chunks} + m := NewManager(nil, nil, nil) + m.RegisterExecutor(executor) + + model := "inflight-cancel-early-model" + registerInFlightAuth(t, m, "auth-inflight-cancel-early-1", executor.Identifier(), model) + + ctx, cancel := context.WithCancel(context.Background()) + req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)} + + type streamOutcome struct { + result *cliproxyexecutor.StreamResult + err error + } + outcome := make(chan streamOutcome, 1) + go func() { + result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true}) + outcome <- streamOutcome{result: result, err: errStream} + }() + chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")} + res := <-outcome + if res.err != nil { + t.Fatalf("ExecuteStream() error = %v", res.err) + } + result := res.result + + waitForInFlightCount(t, m, 1) + + // Cancel without consuming the queued bootstrap chunk; the send branch of + // the wrapper must observe the cancellation and release the scope. + cancel() + waitForInFlightCount(t, m, 0) + + close(chunks) + for range result.Chunks { + } +}