feat(api): expose standalone in-flight request count on management API

- Add a lightweight per-request in-flight tracker on the auth manager that covers selection, upstream calls, retry waits, and the full stream lifetime; it stays disabled in Home mode where the execution registry is authoritative.
- Register GET /v8/management/observability/requests/in-flight returning counts only (in_flight, oldest_started_at, by_provider) with no request bodies or credential identifiers.

Closes: #7476
Co-authored-by: Dylan <1990016@gmail.com>
Co-authored-by: lzt404 <2596933790@qq.com>
This commit is contained in:
Luis Pater
2026-11-13 03:05:54 +08:00
parent 1aaf56bc6a
commit b285660f84
8 changed files with 768 additions and 1 deletions

View File

@@ -0,0 +1,40 @@
package management
import (
"net/http"
"time"
"github.com/gin-gonic/gin"
)
// inFlightResponse is the read-only payload for the standalone in-flight
// request count endpoint. It intentionally exposes counts only: no request
// bodies, prompts, client addresses, or credential identifiers.
type inFlightResponse struct {
InFlight int64 `json:"in_flight"`
OldestStartedAt *time.Time `json:"oldest_started_at,omitempty"`
ByProvider map[string]int64 `json:"by_provider"`
}
// GetInFlightRequests reports the current in-flight request count on a
// standalone instance. In Home mode the snapshot stays empty because the
// execution registry is the authoritative in-flight source there.
func (h *Handler) GetInFlightRequests(c *gin.Context) {
if h == nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "handler unavailable"})
return
}
resp := inFlightResponse{ByProvider: map[string]int64{}}
if h.authManager != nil {
snapshot := h.authManager.InFlightSnapshot()
resp.InFlight = snapshot.InFlight
if !snapshot.OldestStartedAt.IsZero() {
oldest := snapshot.OldestStartedAt.UTC()
resp.OldestStartedAt = &oldest
}
if len(snapshot.ByProvider) > 0 {
resp.ByProvider = snapshot.ByProvider
}
}
c.JSON(http.StatusOK, resp)
}

View File

@@ -0,0 +1,63 @@
package management
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
)
func TestGetInFlightRequestsReturnsSnapshot(t *testing.T) {
gin.SetMode(gin.TestMode)
manager := coreauth.NewManager(nil, nil, nil)
h := &Handler{authManager: manager}
rec := httptest.NewRecorder()
ginCtx, _ := gin.CreateTestContext(rec)
ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil)
h.GetInFlightRequests(ginCtx)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
var payload struct {
InFlight int64 `json:"in_flight"`
OldestStartedAt string `json:"oldest_started_at"`
ByProvider map[string]int64 `json:"by_provider"`
}
if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil {
t.Fatalf("unmarshal response: %v", errUnmarshal)
}
if payload.InFlight != 0 {
t.Fatalf("in_flight = %d, want 0", payload.InFlight)
}
}
func TestGetInFlightRequestsNilManager(t *testing.T) {
gin.SetMode(gin.TestMode)
h := &Handler{}
rec := httptest.NewRecorder()
ginCtx, _ := gin.CreateTestContext(rec)
ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil)
h.GetInFlightRequests(ginCtx)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
var payload struct {
InFlight int64 `json:"in_flight"`
}
if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil {
t.Fatalf("unmarshal response: %v", errUnmarshal)
}
if payload.InFlight != 0 {
t.Fatalf("in_flight = %d, want 0", payload.InFlight)
}
}

View File

@@ -41,6 +41,7 @@ func (s *Server) registerManagementV8Routes() {
v8.GET("/observability/logs/requests/:id", s.mgmt.GetRequestLogByID)
v8.GET("/observability/usage/api-keys", s.mgmt.GetAPIKeyUsage)
v8.GET("/observability/usage/queue", s.mgmt.GetUsageQueue)
v8.GET("/observability/requests/in-flight", s.mgmt.GetInFlightRequests)
v8.GET("/credentials", s.mgmt.ListAuthFiles)
v8.POST("/credentials", s.mgmt.UploadAuthFileV8)

View File

@@ -172,6 +172,7 @@ type resultPolicyHolder struct {
// Manager orchestrates auth lifecycle, selection, execution, and persistence.
type Manager struct {
inFlightTracker *inFlightTracker
store Store
cooldownStore CooldownStateStore
pendingCooldownStateStore CooldownStateStore
@@ -258,6 +259,7 @@ func NewManager(store Store, selector Selector, hook Hook) *Manager {
hook = NoopHook{}
}
manager := &Manager{
inFlightTracker: newInFlightTracker(),
store: store,
authLoadGate: semaphore.NewWeighted(math.MaxInt64),
executors: make(map[string]ProviderExecutor),

View File

@@ -200,10 +200,13 @@ func isOpenAIImageExecutionOptions(opts cliproxyexecutor.Options) bool {
}
func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
scope := m.beginInFlight("")
defer func() { scope.end() }()
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
req, opts = cliproxysession.Enrich(req, opts)
ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers))
ctx = coreusage.CaptureIdentity(ctx)
ctx = contextWithInFlightScope(ctx, scope)
normalized := m.normalizeProviders(providers)
if len(normalized) == 0 {
return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"}
@@ -276,10 +279,13 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
// It supports multiple providers for the same model and round-robins the starting provider per model.
func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
scope := m.beginInFlight("")
defer func() { scope.end() }()
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
req, opts = cliproxysession.Enrich(req, opts)
ctx = WithCodexClientRequest(ctx, IsCodexClientRequest(ctx, opts.Headers))
ctx = coreusage.CaptureIdentity(ctx)
ctx = contextWithInFlightScope(ctx, scope)
normalized := m.normalizeProviders(providers)
if len(normalized) == 0 {
return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"}
@@ -338,11 +344,19 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
// ExecuteStream performs a streaming execution using the configured selector and executor.
// It supports multiple providers for the same model and round-robins the starting provider per model.
func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (result *cliproxyexecutor.StreamResult, err error) {
scope := m.beginInFlight("")
defer func() {
if scope != nil {
result = wrapInFlightStreamResult(result, scope, ctx)
scope = nil
}
}()
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
req, opts = cliproxysession.Enrich(req, opts)
ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers))
ctx = coreusage.CaptureIdentity(ctx)
ctx = contextWithInFlightScope(ctx, scope)
if m.HomeEnabled() {
if unlockSession := m.lockHomeWebsocketSession(ctx, opts); unlockSession != nil {
defer unlockSession()
@@ -658,6 +672,9 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req
pickOpts = withHomeExcludedAuthIDs(pickOpts, tried)
}
auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
if errPick == nil {
updateInFlightScopeProvider(ctx, provider)
}
if errPick != nil {
if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) {
return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick)
@@ -876,6 +893,9 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string,
pickOpts = withHomeExcludedAuthIDs(pickOpts, tried)
}
auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
if errPick == nil {
updateInFlightScopeProvider(ctx, provider)
}
if errPick != nil {
if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) {
return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick)
@@ -1124,6 +1144,9 @@ func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string
}
} else {
auth, executor, provider, errPick = m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
if errPick == nil {
updateInFlightScopeProvider(ctx, provider)
}
}
if errPick != nil {
preferredErr := preferredExecutionAttemptError(lastErr, upstreamErr)

View File

@@ -1473,6 +1473,7 @@ func shouldAttemptAntigravityCreditsFallback(m *Manager, lastErr error, provider
}
func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, bool, error) {
updateInFlightScopeProvider(ctx, "antigravity")
if m != nil && m.HomeEnabled() {
return cliproxyexecutor.Response{}, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable}
}
@@ -1541,6 +1542,7 @@ func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxy
}
func (m *Manager) tryAntigravityCreditsExecuteStream(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, bool, error) {
updateInFlightScopeProvider(ctx, "antigravity")
if m != nil && m.HomeEnabled() {
return nil, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable}
}

View File

@@ -0,0 +1,227 @@
package auth
import (
"context"
"sync"
"time"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
// InFlightSnapshot is an immutable view of standalone in-flight executions.
type InFlightSnapshot struct {
InFlight int64
OldestStartedAt time.Time
ByProvider map[string]int64
}
// inFlightTracker counts in-flight local executions for standalone instances.
// It stays disabled while Home mode is active because the execution registry
// already tracks Home-dispatched executions for that lifetime.
type inFlightTracker struct {
mu sync.Mutex
count int64
oldestStarted time.Time
byProvider map[string]int64
inflightScopes map[*inFlightScope]time.Time
}
func newInFlightTracker() *inFlightTracker {
return &inFlightTracker{
byProvider: make(map[string]int64),
inflightScopes: make(map[*inFlightScope]time.Time),
}
}
// inFlightScope represents one tracked execution until its end is called.
type inFlightScope struct {
tracker *inFlightTracker
provider string
done sync.Once
}
// setProvider updates the provider attribution of an in-flight scope without
// changing the total count or its start time.
func (s *inFlightScope) setProvider(provider string) {
if s == nil || s.tracker == nil {
return
}
t := s.tracker
t.mu.Lock()
defer t.mu.Unlock()
if _, ok := t.inflightScopes[s]; !ok || s.provider == provider {
return
}
if next := t.byProvider[s.provider] - 1; next > 0 {
t.byProvider[s.provider] = next
} else {
delete(t.byProvider, s.provider)
}
s.provider = provider
t.byProvider[provider]++
}
// end releases the scope exactly once.
func (s *inFlightScope) end() {
if s == nil || s.tracker == nil {
return
}
s.done.Do(func() {
t := s.tracker
t.mu.Lock()
defer t.mu.Unlock()
startedAt, ok := t.inflightScopes[s]
if !ok {
return
}
delete(t.inflightScopes, s)
t.count--
if next := t.byProvider[s.provider] - 1; next > 0 {
t.byProvider[s.provider] = next
} else {
delete(t.byProvider, s.provider)
}
if t.count == 0 {
t.oldestStarted = time.Time{}
} else if startedAt.Equal(t.oldestStarted) {
t.recomputeOldestLocked()
}
})
}
func (t *inFlightTracker) recomputeOldestLocked() {
var oldest time.Time
for _, startedAt := range t.inflightScopes {
if oldest.IsZero() || startedAt.Before(oldest) {
oldest = startedAt
}
}
t.oldestStarted = oldest
}
// begin tracks a new execution. The returned scope must be ended exactly once.
func (t *inFlightTracker) begin(provider string) *inFlightScope {
if t == nil {
return nil
}
scope := &inFlightScope{tracker: t, provider: provider}
t.mu.Lock()
defer t.mu.Unlock()
now := time.Now()
t.inflightScopes[scope] = now
t.count++
t.byProvider[provider]++
if t.oldestStarted.IsZero() || now.Before(t.oldestStarted) {
t.oldestStarted = now
}
return scope
}
// snapshot copies the current in-flight state into an immutable view.
func (t *inFlightTracker) snapshot() InFlightSnapshot {
if t == nil {
return InFlightSnapshot{ByProvider: map[string]int64{}}
}
t.mu.Lock()
defer t.mu.Unlock()
byProvider := make(map[string]int64, len(t.byProvider))
for provider, count := range t.byProvider {
byProvider[provider] = count
}
return InFlightSnapshot{
InFlight: t.count,
OldestStartedAt: t.oldestStarted,
ByProvider: byProvider,
}
}
// InFlightSnapshot returns the current standalone in-flight execution counts.
// It reports an empty snapshot while Home mode is enabled, where the execution
// registry is the authoritative in-flight source instead.
func (m *Manager) InFlightSnapshot() InFlightSnapshot {
if m == nil || m.inFlightTracker == nil || m.HomeEnabled() {
return InFlightSnapshot{ByProvider: map[string]int64{}}
}
return m.inFlightTracker.snapshot()
}
// beginInFlight starts tracking one local execution. The returned scope must be
// ended exactly once; it is nil-safe and ends as a no-op in Home mode.
func (m *Manager) beginInFlight(provider string) *inFlightScope {
if m == nil || m.inFlightTracker == nil || m.HomeEnabled() {
return nil
}
return m.inFlightTracker.begin(provider)
}
// wrapInFlightStreamResult extends an in-flight scope until the returned
// stream finishes or ctx is canceled, whichever comes first.
func wrapInFlightStreamResult(result *cliproxyexecutor.StreamResult, scope *inFlightScope, ctx context.Context) *cliproxyexecutor.StreamResult {
if result == nil || scope == nil {
if scope != nil {
scope.end()
}
return result
}
out := make(chan cliproxyexecutor.StreamChunk)
go func() {
// Release the in-flight scope as soon as forwarding stops (stream end
// or cancellation). The output channel closes only after the inner
// channel is fully drained so callers keep the original happens-before
// between stream end and upstream result bookkeeping.
defer scope.end()
if result.Chunks == nil {
close(out)
return
}
forward := true
for forward {
select {
case <-ctx.Done():
forward = false
case chunk, ok := <-result.Chunks:
if !ok {
close(out)
return
}
select {
case <-ctx.Done():
forward = false
case out <- chunk:
}
}
}
// After cancellation, drain the inner channel on the same goroutine so
// the inner stream wrapper finishes its bookkeeping before out closes.
// The scope is released explicitly here before the drain can block;
// the deferred end above remains as the safety net for other exits.
scope.end()
for range result.Chunks {
}
close(out)
}()
return &cliproxyexecutor.StreamResult{Headers: result.Headers, Chunks: out}
}
// inFlightScopeContextKey carries the request in-flight scope to inner
// selection code so the provider attribution can be corrected after pick.
type inFlightScopeContextKey struct{}
// contextWithInFlightScope returns a context carrying the scope, or ctx when
// scope is nil.
func contextWithInFlightScope(ctx context.Context, scope *inFlightScope) context.Context {
if ctx == nil || scope == nil {
return ctx
}
return context.WithValue(ctx, inFlightScopeContextKey{}, scope)
}
// updateInFlightScopeProvider re-attributes the request scope to the actually
// selected provider after auth selection. It is a no-op without a scope.
func updateInFlightScopeProvider(ctx context.Context, provider string) {
if ctx == nil {
return
}
scope, _ := ctx.Value(inFlightScopeContextKey{}).(*inFlightScope)
scope.setProvider(provider)
}

View File

@@ -0,0 +1,409 @@
package auth
import (
"context"
"errors"
"net/http"
"sync"
"sync/atomic"
"testing"
"time"
internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
// inFlightBlockingExecutor blocks inside Execute until released so the test can
// observe the manager in-flight snapshot mid-request.
type inFlightBlockingExecutor struct {
id string
release chan struct{}
entered chan struct{}
releaseMu sync.Once
}
func newInFlightBlockingExecutor(id string) *inFlightBlockingExecutor {
return &inFlightBlockingExecutor{
id: id,
release: make(chan struct{}),
entered: make(chan struct{}, 4),
}
}
func (e *inFlightBlockingExecutor) Identifier() string { return e.id }
func (e *inFlightBlockingExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
e.entered <- struct{}{}
<-e.release
return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil
}
func (e *inFlightBlockingExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
return nil, errors.New("stream not supported")
}
func (e *inFlightBlockingExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
return auth, nil
}
func (e *inFlightBlockingExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
return cliproxyexecutor.Response{}, errors.New("not supported")
}
func (e *inFlightBlockingExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
return nil, errors.New("not supported")
}
// inFlightStreamExecutor returns a stream whose chunks channel stays open until
// the test closes it, letting the stream lifetime be observed via the snapshot.
type inFlightStreamExecutor struct {
id string
chunks chan cliproxyexecutor.StreamChunk
}
func (e *inFlightStreamExecutor) Identifier() string { return e.id }
func (e *inFlightStreamExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
return cliproxyexecutor.Response{}, errors.New("not supported")
}
func (e *inFlightStreamExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
return &cliproxyexecutor.StreamResult{Chunks: e.chunks}, nil
}
func (e *inFlightStreamExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
return auth, nil
}
func (e *inFlightStreamExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
return cliproxyexecutor.Response{}, errors.New("not supported")
}
func (e *inFlightStreamExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
return nil, errors.New("not supported")
}
func registerInFlightAuth(t *testing.T, m *Manager, authID, provider, model string) {
t.Helper()
auth := &Auth{ID: authID, Provider: provider, Status: StatusActive}
if _, errRegister := m.Register(context.Background(), auth); errRegister != nil {
t.Fatalf("Register auth: %v", errRegister)
}
registry.GetGlobalRegistry().RegisterClient(authID, provider, []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() {
registry.GetGlobalRegistry().UnregisterClient(authID)
})
}
func waitForInFlightCount(t *testing.T, m *Manager, want int) InFlightSnapshot {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
snapshot := m.InFlightSnapshot()
if snapshot.InFlight == int64(want) {
return snapshot
}
if time.Now().After(deadline) {
t.Fatalf("in-flight count = %d, want %d (snapshot: %+v)", snapshot.InFlight, want, snapshot)
}
time.Sleep(5 * time.Millisecond)
}
}
// TestManager_InFlightSnapshot_NonStreaming verifies that an in-flight
// non-streaming request is visible in the manager snapshot while it executes.
func TestManager_InFlightSnapshot_NonStreaming(t *testing.T) {
executor := newInFlightBlockingExecutor("inflight-provider")
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
model := "inflight-model"
registerInFlightAuth(t, m, "auth-inflight-1", executor.Identifier(), model)
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
done := make(chan error, 1)
go func() {
_, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{})
done <- errExec
}()
<-executor.entered
snapshot := waitForInFlightCount(t, m, 1)
if snapshot.ByProvider[executor.Identifier()] != 1 {
t.Fatalf("by_provider[%q] = %d, want 1 (snapshot: %+v)", executor.Identifier(), snapshot.ByProvider[executor.Identifier()], snapshot)
}
if snapshot.OldestStartedAt.IsZero() {
t.Fatal("expected oldest_started_at to be set while a request is in flight")
}
executor.releaseMu.Do(func() { close(executor.release) })
if errExec := <-done; errExec != nil {
t.Fatalf("Execute() error = %v", errExec)
}
waitForInFlightCount(t, m, 0)
}
// TestManager_InFlightSnapshot_Streaming verifies that a long-lived streaming
// request remains counted until the chunk channel closes.
func TestManager_InFlightSnapshot_Streaming(t *testing.T) {
chunks := make(chan cliproxyexecutor.StreamChunk)
executor := &inFlightStreamExecutor{id: "inflight-stream-provider", chunks: chunks}
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
model := "inflight-stream-model"
registerInFlightAuth(t, m, "auth-inflight-stream-1", executor.Identifier(), model)
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
type streamOutcome struct {
result *cliproxyexecutor.StreamResult
err error
}
outcome := make(chan streamOutcome, 1)
go func() {
result, errStream := m.ExecuteStream(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
outcome <- streamOutcome{result: result, err: errStream}
}()
// readStreamBootstrap blocks until the first payload chunk arrives.
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
streamRes := <-outcome
result, errStream := streamRes.result, streamRes.err
if errStream != nil {
t.Fatalf("ExecuteStream() error = %v", errStream)
}
waitForInFlightCount(t, m, 1)
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: hello\n\n")}
if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 {
t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload)
}
if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 {
t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot)
}
close(chunks)
for range result.Chunks {
}
waitForInFlightCount(t, m, 0)
}
// TestManager_InFlightSnapshot_DisabledInHomeMode verifies that the standalone
// in-flight tracker stays empty when Home mode is enabled, since the execution
// registry covers that mode.
func TestManager_InFlightSnapshot_DisabledInHomeMode(t *testing.T) {
executor := newInFlightBlockingExecutor("inflight-home-provider")
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
m.SetConfigSnapshot(&internalconfig.Config{Home: internalconfig.HomeConfig{Enabled: true}})
snapshot := m.InFlightSnapshot()
if snapshot.InFlight != 0 {
t.Fatalf("expected empty snapshot in home mode, got %+v", snapshot)
}
}
// inFlightRetryableStatusError is a retryable upstream error for retry tests.
type inFlightRetryableStatusError struct{}
func (inFlightRetryableStatusError) Error() string { return "upstream unavailable" }
func (inFlightRetryableStatusError) StatusCode() int { return http.StatusServiceUnavailable }
// inFlightFailThenBlockExecutor fails the first Execute call with a retryable
// error and blocks in subsequent calls until released.
type inFlightFailThenBlockExecutor struct {
id string
calls atomic.Int32
release chan struct{}
releaseOnce sync.Once
}
func newInFlightFailThenBlockExecutor(id string) *inFlightFailThenBlockExecutor {
return &inFlightFailThenBlockExecutor{id: id, release: make(chan struct{})}
}
func (e *inFlightFailThenBlockExecutor) Identifier() string { return e.id }
func (e *inFlightFailThenBlockExecutor) Execute(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
call := e.calls.Add(1)
if call == 1 {
cliproxyexecutor.MarkUpstreamAttempt(ctx)
return cliproxyexecutor.Response{}, inFlightRetryableStatusError{}
}
<-e.release
return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil
}
func (e *inFlightFailThenBlockExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
return nil, errors.New("stream not supported")
}
func (e *inFlightFailThenBlockExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
return auth, nil
}
func (e *inFlightFailThenBlockExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
return cliproxyexecutor.Response{}, errors.New("not supported")
}
func (e *inFlightFailThenBlockExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
return nil, errors.New("not supported")
}
// TestManager_InFlightSnapshot_CoversRetryWait verifies that a request waiting
// between retry rounds still appears in the snapshot instead of blinking to 0.
func TestManager_InFlightSnapshot_CoversRetryWait(t *testing.T) {
executor := newInFlightFailThenBlockExecutor("inflight-retry-provider")
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
m.SetRetryConfig(1, 20*time.Second, 0)
model := "inflight-retry-model"
if _, errRegister := m.Register(context.Background(), &Auth{
ID: "auth-inflight-retry-1",
Provider: executor.Identifier(),
Metadata: map[string]any{"request_retry": 1, "disable_cooling": true},
}); errRegister != nil {
t.Fatalf("Register auth: %v", errRegister)
}
registry.GetGlobalRegistry().RegisterClient("auth-inflight-retry-1", executor.Identifier(), []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() {
registry.GetGlobalRegistry().UnregisterClient("auth-inflight-retry-1")
})
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
done := make(chan error, 1)
go func() {
_, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{})
done <- errExec
}()
// Wait for the first call to record its retryable failure. The request then
// enters the retry wait window while the second call blocks on release.
deadline := time.Now().Add(5 * time.Second)
for executor.calls.Load() < 1 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if executor.calls.Load() < 1 {
t.Fatalf("expected the first upstream call to happen, calls = %d", executor.calls.Load())
}
// Allow the retry loop to record the failure and enter its wait or the
// blocking second call; either way the request is still active and must
// remain visible in the snapshot.
deadline = time.Now().Add(2 * time.Second)
for {
snapshot := m.InFlightSnapshot()
if snapshot.InFlight == 1 {
break
}
if time.Now().After(deadline) {
t.Fatalf("in_flight = %d during retry, want 1 (snapshot: %+v)", snapshot.InFlight, snapshot)
}
time.Sleep(5 * time.Millisecond)
}
executor.releaseOnce.Do(func() { close(executor.release) })
if errExec := <-done; errExec != nil {
t.Fatalf("Execute() error = %v", errExec)
}
waitForInFlightCount(t, m, 0)
}
// TestManager_InFlightSnapshot_StreamCancel verifies that canceling the request
// context releases the in-flight scope even while the upstream chunk channel
// stays open and silent after the first chunk was already forwarded.
func TestManager_InFlightSnapshot_StreamCancel(t *testing.T) {
chunks := make(chan cliproxyexecutor.StreamChunk)
executor := &inFlightStreamExecutor{id: "inflight-cancel-provider", chunks: chunks}
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
model := "inflight-cancel-model"
registerInFlightAuth(t, m, "auth-inflight-cancel-1", executor.Identifier(), model)
ctx, cancel := context.WithCancel(context.Background())
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
type streamOutcome struct {
result *cliproxyexecutor.StreamResult
err error
}
outcome := make(chan streamOutcome, 1)
go func() {
result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
outcome <- streamOutcome{result: result, err: errStream}
}()
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
res := <-outcome
if res.err != nil {
t.Fatalf("ExecuteStream() error = %v", res.err)
}
result := res.result
waitForInFlightCount(t, m, 1)
// Consume the forwarded bootstrap chunk so the wrapper is blocked waiting
// for the next upstream chunk (receive branch) when cancel happens.
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: second\n\n")}
if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 {
t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload)
}
if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 {
t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot)
}
// The upstream keeps its chunk channel open but silent; canceling the
// request context must release the scope without needing another chunk.
cancel()
waitForInFlightCount(t, m, 0)
// Close the source so the drain path and the output channel can finish
// instead of leaking a background goroutine.
close(chunks)
for range result.Chunks {
}
}
// TestManager_InFlightSnapshot_StreamCancelBeforeConsume verifies the send
// branch: cancellation while the first chunk is still queued for the caller
// also releases the in-flight scope.
func TestManager_InFlightSnapshot_StreamCancelBeforeConsume(t *testing.T) {
chunks := make(chan cliproxyexecutor.StreamChunk)
executor := &inFlightStreamExecutor{id: "inflight-cancel-early-provider", chunks: chunks}
m := NewManager(nil, nil, nil)
m.RegisterExecutor(executor)
model := "inflight-cancel-early-model"
registerInFlightAuth(t, m, "auth-inflight-cancel-early-1", executor.Identifier(), model)
ctx, cancel := context.WithCancel(context.Background())
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
type streamOutcome struct {
result *cliproxyexecutor.StreamResult
err error
}
outcome := make(chan streamOutcome, 1)
go func() {
result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
outcome <- streamOutcome{result: result, err: errStream}
}()
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
res := <-outcome
if res.err != nil {
t.Fatalf("ExecuteStream() error = %v", res.err)
}
result := res.result
waitForInFlightCount(t, m, 1)
// Cancel without consuming the queued bootstrap chunk; the send branch of
// the wrapper must observe the cancellation and release the scope.
cancel()
waitForInFlightCount(t, m, 0)
close(chunks)
for range result.Chunks {
}
}