mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-16 14:49:40 +08:00
feat(api): expose standalone in-flight request count on management API
- Add a lightweight per-request in-flight tracker on the auth manager that covers selection, upstream calls, retry waits, and the full stream lifetime; it stays disabled in Home mode where the execution registry is authoritative. - Register GET /v8/management/observability/requests/in-flight returning counts only (in_flight, oldest_started_at, by_provider) with no request bodies or credential identifiers. Closes: #7476 Co-authored-by: Dylan <1990016@gmail.com> Co-authored-by: lzt404 <2596933790@qq.com>
This commit is contained in:
40
internal/api/handlers/management/in_flight.go
Normal file
40
internal/api/handlers/management/in_flight.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package management
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// inFlightResponse is the read-only payload for the standalone in-flight
|
||||
// request count endpoint. It intentionally exposes counts only: no request
|
||||
// bodies, prompts, client addresses, or credential identifiers.
|
||||
type inFlightResponse struct {
|
||||
InFlight int64 `json:"in_flight"`
|
||||
OldestStartedAt *time.Time `json:"oldest_started_at,omitempty"`
|
||||
ByProvider map[string]int64 `json:"by_provider"`
|
||||
}
|
||||
|
||||
// GetInFlightRequests reports the current in-flight request count on a
|
||||
// standalone instance. In Home mode the snapshot stays empty because the
|
||||
// execution registry is the authoritative in-flight source there.
|
||||
func (h *Handler) GetInFlightRequests(c *gin.Context) {
|
||||
if h == nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "handler unavailable"})
|
||||
return
|
||||
}
|
||||
resp := inFlightResponse{ByProvider: map[string]int64{}}
|
||||
if h.authManager != nil {
|
||||
snapshot := h.authManager.InFlightSnapshot()
|
||||
resp.InFlight = snapshot.InFlight
|
||||
if !snapshot.OldestStartedAt.IsZero() {
|
||||
oldest := snapshot.OldestStartedAt.UTC()
|
||||
resp.OldestStartedAt = &oldest
|
||||
}
|
||||
if len(snapshot.ByProvider) > 0 {
|
||||
resp.ByProvider = snapshot.ByProvider
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
63
internal/api/handlers/management/in_flight_test.go
Normal file
63
internal/api/handlers/management/in_flight_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package management
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth"
|
||||
)
|
||||
|
||||
func TestGetInFlightRequestsReturnsSnapshot(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
manager := coreauth.NewManager(nil, nil, nil)
|
||||
h := &Handler{authManager: manager}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
ginCtx, _ := gin.CreateTestContext(rec)
|
||||
ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil)
|
||||
|
||||
h.GetInFlightRequests(ginCtx)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
InFlight int64 `json:"in_flight"`
|
||||
OldestStartedAt string `json:"oldest_started_at"`
|
||||
ByProvider map[string]int64 `json:"by_provider"`
|
||||
}
|
||||
if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil {
|
||||
t.Fatalf("unmarshal response: %v", errUnmarshal)
|
||||
}
|
||||
if payload.InFlight != 0 {
|
||||
t.Fatalf("in_flight = %d, want 0", payload.InFlight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetInFlightRequestsNilManager(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
h := &Handler{}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
ginCtx, _ := gin.CreateTestContext(rec)
|
||||
ginCtx.Request = httptest.NewRequest(http.MethodGet, "/v8/management/observability/requests/in-flight", nil)
|
||||
|
||||
h.GetInFlightRequests(ginCtx)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
InFlight int64 `json:"in_flight"`
|
||||
}
|
||||
if errUnmarshal := json.Unmarshal(rec.Body.Bytes(), &payload); errUnmarshal != nil {
|
||||
t.Fatalf("unmarshal response: %v", errUnmarshal)
|
||||
}
|
||||
if payload.InFlight != 0 {
|
||||
t.Fatalf("in_flight = %d, want 0", payload.InFlight)
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ func (s *Server) registerManagementV8Routes() {
|
||||
v8.GET("/observability/logs/requests/:id", s.mgmt.GetRequestLogByID)
|
||||
v8.GET("/observability/usage/api-keys", s.mgmt.GetAPIKeyUsage)
|
||||
v8.GET("/observability/usage/queue", s.mgmt.GetUsageQueue)
|
||||
v8.GET("/observability/requests/in-flight", s.mgmt.GetInFlightRequests)
|
||||
|
||||
v8.GET("/credentials", s.mgmt.ListAuthFiles)
|
||||
v8.POST("/credentials", s.mgmt.UploadAuthFileV8)
|
||||
|
||||
@@ -172,6 +172,7 @@ type resultPolicyHolder struct {
|
||||
|
||||
// Manager orchestrates auth lifecycle, selection, execution, and persistence.
|
||||
type Manager struct {
|
||||
inFlightTracker *inFlightTracker
|
||||
store Store
|
||||
cooldownStore CooldownStateStore
|
||||
pendingCooldownStateStore CooldownStateStore
|
||||
@@ -258,6 +259,7 @@ func NewManager(store Store, selector Selector, hook Hook) *Manager {
|
||||
hook = NoopHook{}
|
||||
}
|
||||
manager := &Manager{
|
||||
inFlightTracker: newInFlightTracker(),
|
||||
store: store,
|
||||
authLoadGate: semaphore.NewWeighted(math.MaxInt64),
|
||||
executors: make(map[string]ProviderExecutor),
|
||||
|
||||
@@ -200,10 +200,13 @@ func isOpenAIImageExecutionOptions(opts cliproxyexecutor.Options) bool {
|
||||
}
|
||||
|
||||
func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
scope := m.beginInFlight("")
|
||||
defer func() { scope.end() }()
|
||||
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
|
||||
req, opts = cliproxysession.Enrich(req, opts)
|
||||
ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers))
|
||||
ctx = coreusage.CaptureIdentity(ctx)
|
||||
ctx = contextWithInFlightScope(ctx, scope)
|
||||
normalized := m.normalizeProviders(providers)
|
||||
if len(normalized) == 0 {
|
||||
return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"}
|
||||
@@ -276,10 +279,13 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye
|
||||
|
||||
// It supports multiple providers for the same model and round-robins the starting provider per model.
|
||||
func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
scope := m.beginInFlight("")
|
||||
defer func() { scope.end() }()
|
||||
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
|
||||
req, opts = cliproxysession.Enrich(req, opts)
|
||||
ctx = WithCodexClientRequest(ctx, IsCodexClientRequest(ctx, opts.Headers))
|
||||
ctx = coreusage.CaptureIdentity(ctx)
|
||||
ctx = contextWithInFlightScope(ctx, scope)
|
||||
normalized := m.normalizeProviders(providers)
|
||||
if len(normalized) == 0 {
|
||||
return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"}
|
||||
@@ -338,11 +344,19 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip
|
||||
|
||||
// ExecuteStream performs a streaming execution using the configured selector and executor.
|
||||
// It supports multiple providers for the same model and round-robins the starting provider per model.
|
||||
func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (result *cliproxyexecutor.StreamResult, err error) {
|
||||
scope := m.beginInFlight("")
|
||||
defer func() {
|
||||
if scope != nil {
|
||||
result = wrapInFlightStreamResult(result, scope, ctx)
|
||||
scope = nil
|
||||
}
|
||||
}()
|
||||
ctx = cliproxyexecutor.WithRequestProxyURL(ctx, opts.ProxyURL)
|
||||
req, opts = cliproxysession.Enrich(req, opts)
|
||||
ctx = WithCodexClientRequest(ctx, isOpenAIImageExecutionOptions(opts) || IsCodexClientRequest(ctx, opts.Headers))
|
||||
ctx = coreusage.CaptureIdentity(ctx)
|
||||
ctx = contextWithInFlightScope(ctx, scope)
|
||||
if m.HomeEnabled() {
|
||||
if unlockSession := m.lockHomeWebsocketSession(ctx, opts); unlockSession != nil {
|
||||
defer unlockSession()
|
||||
@@ -658,6 +672,9 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req
|
||||
pickOpts = withHomeExcludedAuthIDs(pickOpts, tried)
|
||||
}
|
||||
auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
|
||||
if errPick == nil {
|
||||
updateInFlightScopeProvider(ctx, provider)
|
||||
}
|
||||
if errPick != nil {
|
||||
if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) {
|
||||
return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick)
|
||||
@@ -876,6 +893,9 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string,
|
||||
pickOpts = withHomeExcludedAuthIDs(pickOpts, tried)
|
||||
}
|
||||
auth, executor, provider, errPick := m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
|
||||
if errPick == nil {
|
||||
updateInFlightScopeProvider(ctx, provider)
|
||||
}
|
||||
if errPick != nil {
|
||||
if shouldReturnLastErrorOnPickFailure(homeMode, lastErr, errPick) {
|
||||
return cliproxyexecutor.Response{}, roundPickFailureError(lastErr, upstreamErr, errPick)
|
||||
@@ -1124,6 +1144,9 @@ func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string
|
||||
}
|
||||
} else {
|
||||
auth, executor, provider, errPick = m.pickNextMixed(ctx, providers, routeModel, pickOpts, tried)
|
||||
if errPick == nil {
|
||||
updateInFlightScopeProvider(ctx, provider)
|
||||
}
|
||||
}
|
||||
if errPick != nil {
|
||||
preferredErr := preferredExecutionAttemptError(lastErr, upstreamErr)
|
||||
|
||||
@@ -1473,6 +1473,7 @@ func shouldAttemptAntigravityCreditsFallback(m *Manager, lastErr error, provider
|
||||
}
|
||||
|
||||
func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, bool, error) {
|
||||
updateInFlightScopeProvider(ctx, "antigravity")
|
||||
if m != nil && m.HomeEnabled() {
|
||||
return cliproxyexecutor.Response{}, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable}
|
||||
}
|
||||
@@ -1541,6 +1542,7 @@ func (m *Manager) tryAntigravityCreditsExecute(ctx context.Context, req cliproxy
|
||||
}
|
||||
|
||||
func (m *Manager) tryAntigravityCreditsExecuteStream(ctx context.Context, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, bool, error) {
|
||||
updateInFlightScopeProvider(ctx, "antigravity")
|
||||
if m != nil && m.HomeEnabled() {
|
||||
return nil, false, &Error{Code: "home_fallback_unsupported", Message: "Home does not support Antigravity credits fallback", HTTPStatus: http.StatusServiceUnavailable}
|
||||
}
|
||||
|
||||
227
sdk/cliproxy/auth/conductor_inflight.go
Normal file
227
sdk/cliproxy/auth/conductor_inflight.go
Normal file
@@ -0,0 +1,227 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
// InFlightSnapshot is an immutable view of standalone in-flight executions.
|
||||
type InFlightSnapshot struct {
|
||||
InFlight int64
|
||||
OldestStartedAt time.Time
|
||||
ByProvider map[string]int64
|
||||
}
|
||||
|
||||
// inFlightTracker counts in-flight local executions for standalone instances.
|
||||
// It stays disabled while Home mode is active because the execution registry
|
||||
// already tracks Home-dispatched executions for that lifetime.
|
||||
type inFlightTracker struct {
|
||||
mu sync.Mutex
|
||||
count int64
|
||||
oldestStarted time.Time
|
||||
byProvider map[string]int64
|
||||
inflightScopes map[*inFlightScope]time.Time
|
||||
}
|
||||
|
||||
func newInFlightTracker() *inFlightTracker {
|
||||
return &inFlightTracker{
|
||||
byProvider: make(map[string]int64),
|
||||
inflightScopes: make(map[*inFlightScope]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// inFlightScope represents one tracked execution until its end is called.
|
||||
type inFlightScope struct {
|
||||
tracker *inFlightTracker
|
||||
provider string
|
||||
done sync.Once
|
||||
}
|
||||
|
||||
// setProvider updates the provider attribution of an in-flight scope without
|
||||
// changing the total count or its start time.
|
||||
func (s *inFlightScope) setProvider(provider string) {
|
||||
if s == nil || s.tracker == nil {
|
||||
return
|
||||
}
|
||||
t := s.tracker
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if _, ok := t.inflightScopes[s]; !ok || s.provider == provider {
|
||||
return
|
||||
}
|
||||
if next := t.byProvider[s.provider] - 1; next > 0 {
|
||||
t.byProvider[s.provider] = next
|
||||
} else {
|
||||
delete(t.byProvider, s.provider)
|
||||
}
|
||||
s.provider = provider
|
||||
t.byProvider[provider]++
|
||||
}
|
||||
|
||||
// end releases the scope exactly once.
|
||||
func (s *inFlightScope) end() {
|
||||
if s == nil || s.tracker == nil {
|
||||
return
|
||||
}
|
||||
s.done.Do(func() {
|
||||
t := s.tracker
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
startedAt, ok := t.inflightScopes[s]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(t.inflightScopes, s)
|
||||
t.count--
|
||||
if next := t.byProvider[s.provider] - 1; next > 0 {
|
||||
t.byProvider[s.provider] = next
|
||||
} else {
|
||||
delete(t.byProvider, s.provider)
|
||||
}
|
||||
if t.count == 0 {
|
||||
t.oldestStarted = time.Time{}
|
||||
} else if startedAt.Equal(t.oldestStarted) {
|
||||
t.recomputeOldestLocked()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *inFlightTracker) recomputeOldestLocked() {
|
||||
var oldest time.Time
|
||||
for _, startedAt := range t.inflightScopes {
|
||||
if oldest.IsZero() || startedAt.Before(oldest) {
|
||||
oldest = startedAt
|
||||
}
|
||||
}
|
||||
t.oldestStarted = oldest
|
||||
}
|
||||
|
||||
// begin tracks a new execution. The returned scope must be ended exactly once.
|
||||
func (t *inFlightTracker) begin(provider string) *inFlightScope {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
scope := &inFlightScope{tracker: t, provider: provider}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
now := time.Now()
|
||||
t.inflightScopes[scope] = now
|
||||
t.count++
|
||||
t.byProvider[provider]++
|
||||
if t.oldestStarted.IsZero() || now.Before(t.oldestStarted) {
|
||||
t.oldestStarted = now
|
||||
}
|
||||
return scope
|
||||
}
|
||||
|
||||
// snapshot copies the current in-flight state into an immutable view.
|
||||
func (t *inFlightTracker) snapshot() InFlightSnapshot {
|
||||
if t == nil {
|
||||
return InFlightSnapshot{ByProvider: map[string]int64{}}
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
byProvider := make(map[string]int64, len(t.byProvider))
|
||||
for provider, count := range t.byProvider {
|
||||
byProvider[provider] = count
|
||||
}
|
||||
return InFlightSnapshot{
|
||||
InFlight: t.count,
|
||||
OldestStartedAt: t.oldestStarted,
|
||||
ByProvider: byProvider,
|
||||
}
|
||||
}
|
||||
|
||||
// InFlightSnapshot returns the current standalone in-flight execution counts.
|
||||
// It reports an empty snapshot while Home mode is enabled, where the execution
|
||||
// registry is the authoritative in-flight source instead.
|
||||
func (m *Manager) InFlightSnapshot() InFlightSnapshot {
|
||||
if m == nil || m.inFlightTracker == nil || m.HomeEnabled() {
|
||||
return InFlightSnapshot{ByProvider: map[string]int64{}}
|
||||
}
|
||||
return m.inFlightTracker.snapshot()
|
||||
}
|
||||
|
||||
// beginInFlight starts tracking one local execution. The returned scope must be
|
||||
// ended exactly once; it is nil-safe and ends as a no-op in Home mode.
|
||||
func (m *Manager) beginInFlight(provider string) *inFlightScope {
|
||||
if m == nil || m.inFlightTracker == nil || m.HomeEnabled() {
|
||||
return nil
|
||||
}
|
||||
return m.inFlightTracker.begin(provider)
|
||||
}
|
||||
|
||||
// wrapInFlightStreamResult extends an in-flight scope until the returned
|
||||
// stream finishes or ctx is canceled, whichever comes first.
|
||||
func wrapInFlightStreamResult(result *cliproxyexecutor.StreamResult, scope *inFlightScope, ctx context.Context) *cliproxyexecutor.StreamResult {
|
||||
if result == nil || scope == nil {
|
||||
if scope != nil {
|
||||
scope.end()
|
||||
}
|
||||
return result
|
||||
}
|
||||
out := make(chan cliproxyexecutor.StreamChunk)
|
||||
go func() {
|
||||
// Release the in-flight scope as soon as forwarding stops (stream end
|
||||
// or cancellation). The output channel closes only after the inner
|
||||
// channel is fully drained so callers keep the original happens-before
|
||||
// between stream end and upstream result bookkeeping.
|
||||
defer scope.end()
|
||||
if result.Chunks == nil {
|
||||
close(out)
|
||||
return
|
||||
}
|
||||
forward := true
|
||||
for forward {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
forward = false
|
||||
case chunk, ok := <-result.Chunks:
|
||||
if !ok {
|
||||
close(out)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
forward = false
|
||||
case out <- chunk:
|
||||
}
|
||||
}
|
||||
}
|
||||
// After cancellation, drain the inner channel on the same goroutine so
|
||||
// the inner stream wrapper finishes its bookkeeping before out closes.
|
||||
// The scope is released explicitly here before the drain can block;
|
||||
// the deferred end above remains as the safety net for other exits.
|
||||
scope.end()
|
||||
for range result.Chunks {
|
||||
}
|
||||
close(out)
|
||||
}()
|
||||
return &cliproxyexecutor.StreamResult{Headers: result.Headers, Chunks: out}
|
||||
}
|
||||
|
||||
// inFlightScopeContextKey carries the request in-flight scope to inner
|
||||
// selection code so the provider attribution can be corrected after pick.
|
||||
type inFlightScopeContextKey struct{}
|
||||
|
||||
// contextWithInFlightScope returns a context carrying the scope, or ctx when
|
||||
// scope is nil.
|
||||
func contextWithInFlightScope(ctx context.Context, scope *inFlightScope) context.Context {
|
||||
if ctx == nil || scope == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, inFlightScopeContextKey{}, scope)
|
||||
}
|
||||
|
||||
// updateInFlightScopeProvider re-attributes the request scope to the actually
|
||||
// selected provider after auth selection. It is a no-op without a scope.
|
||||
func updateInFlightScopeProvider(ctx context.Context, provider string) {
|
||||
if ctx == nil {
|
||||
return
|
||||
}
|
||||
scope, _ := ctx.Value(inFlightScopeContextKey{}).(*inFlightScope)
|
||||
scope.setProvider(provider)
|
||||
}
|
||||
409
sdk/cliproxy/auth/conductor_inflight_test.go
Normal file
409
sdk/cliproxy/auth/conductor_inflight_test.go
Normal file
@@ -0,0 +1,409 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config"
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
// inFlightBlockingExecutor blocks inside Execute until released so the test can
|
||||
// observe the manager in-flight snapshot mid-request.
|
||||
type inFlightBlockingExecutor struct {
|
||||
id string
|
||||
release chan struct{}
|
||||
entered chan struct{}
|
||||
releaseMu sync.Once
|
||||
}
|
||||
|
||||
func newInFlightBlockingExecutor(id string) *inFlightBlockingExecutor {
|
||||
return &inFlightBlockingExecutor{
|
||||
id: id,
|
||||
release: make(chan struct{}),
|
||||
entered: make(chan struct{}, 4),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *inFlightBlockingExecutor) Identifier() string { return e.id }
|
||||
|
||||
func (e *inFlightBlockingExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
e.entered <- struct{}{}
|
||||
<-e.release
|
||||
return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil
|
||||
}
|
||||
|
||||
func (e *inFlightBlockingExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
return nil, errors.New("stream not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightBlockingExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (e *inFlightBlockingExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
return cliproxyexecutor.Response{}, errors.New("not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightBlockingExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("not supported")
|
||||
}
|
||||
|
||||
// inFlightStreamExecutor returns a stream whose chunks channel stays open until
|
||||
// the test closes it, letting the stream lifetime be observed via the snapshot.
|
||||
type inFlightStreamExecutor struct {
|
||||
id string
|
||||
chunks chan cliproxyexecutor.StreamChunk
|
||||
}
|
||||
|
||||
func (e *inFlightStreamExecutor) Identifier() string { return e.id }
|
||||
|
||||
func (e *inFlightStreamExecutor) Execute(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
return cliproxyexecutor.Response{}, errors.New("not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightStreamExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
return &cliproxyexecutor.StreamResult{Chunks: e.chunks}, nil
|
||||
}
|
||||
|
||||
func (e *inFlightStreamExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (e *inFlightStreamExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
return cliproxyexecutor.Response{}, errors.New("not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightStreamExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("not supported")
|
||||
}
|
||||
|
||||
func registerInFlightAuth(t *testing.T, m *Manager, authID, provider, model string) {
|
||||
t.Helper()
|
||||
auth := &Auth{ID: authID, Provider: provider, Status: StatusActive}
|
||||
if _, errRegister := m.Register(context.Background(), auth); errRegister != nil {
|
||||
t.Fatalf("Register auth: %v", errRegister)
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient(authID, provider, []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() {
|
||||
registry.GetGlobalRegistry().UnregisterClient(authID)
|
||||
})
|
||||
}
|
||||
|
||||
func waitForInFlightCount(t *testing.T, m *Manager, want int) InFlightSnapshot {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
snapshot := m.InFlightSnapshot()
|
||||
if snapshot.InFlight == int64(want) {
|
||||
return snapshot
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("in-flight count = %d, want %d (snapshot: %+v)", snapshot.InFlight, want, snapshot)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_NonStreaming verifies that an in-flight
|
||||
// non-streaming request is visible in the manager snapshot while it executes.
|
||||
func TestManager_InFlightSnapshot_NonStreaming(t *testing.T) {
|
||||
executor := newInFlightBlockingExecutor("inflight-provider")
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
|
||||
model := "inflight-model"
|
||||
registerInFlightAuth(t, m, "auth-inflight-1", executor.Identifier(), model)
|
||||
|
||||
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{})
|
||||
done <- errExec
|
||||
}()
|
||||
|
||||
<-executor.entered
|
||||
snapshot := waitForInFlightCount(t, m, 1)
|
||||
if snapshot.ByProvider[executor.Identifier()] != 1 {
|
||||
t.Fatalf("by_provider[%q] = %d, want 1 (snapshot: %+v)", executor.Identifier(), snapshot.ByProvider[executor.Identifier()], snapshot)
|
||||
}
|
||||
if snapshot.OldestStartedAt.IsZero() {
|
||||
t.Fatal("expected oldest_started_at to be set while a request is in flight")
|
||||
}
|
||||
|
||||
executor.releaseMu.Do(func() { close(executor.release) })
|
||||
if errExec := <-done; errExec != nil {
|
||||
t.Fatalf("Execute() error = %v", errExec)
|
||||
}
|
||||
waitForInFlightCount(t, m, 0)
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_Streaming verifies that a long-lived streaming
|
||||
// request remains counted until the chunk channel closes.
|
||||
func TestManager_InFlightSnapshot_Streaming(t *testing.T) {
|
||||
chunks := make(chan cliproxyexecutor.StreamChunk)
|
||||
executor := &inFlightStreamExecutor{id: "inflight-stream-provider", chunks: chunks}
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
|
||||
model := "inflight-stream-model"
|
||||
registerInFlightAuth(t, m, "auth-inflight-stream-1", executor.Identifier(), model)
|
||||
|
||||
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
|
||||
type streamOutcome struct {
|
||||
result *cliproxyexecutor.StreamResult
|
||||
err error
|
||||
}
|
||||
outcome := make(chan streamOutcome, 1)
|
||||
go func() {
|
||||
result, errStream := m.ExecuteStream(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
|
||||
outcome <- streamOutcome{result: result, err: errStream}
|
||||
}()
|
||||
// readStreamBootstrap blocks until the first payload chunk arrives.
|
||||
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
|
||||
streamRes := <-outcome
|
||||
result, errStream := streamRes.result, streamRes.err
|
||||
if errStream != nil {
|
||||
t.Fatalf("ExecuteStream() error = %v", errStream)
|
||||
}
|
||||
|
||||
waitForInFlightCount(t, m, 1)
|
||||
|
||||
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: hello\n\n")}
|
||||
if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 {
|
||||
t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload)
|
||||
}
|
||||
if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 {
|
||||
t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot)
|
||||
}
|
||||
|
||||
close(chunks)
|
||||
for range result.Chunks {
|
||||
}
|
||||
waitForInFlightCount(t, m, 0)
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_DisabledInHomeMode verifies that the standalone
|
||||
// in-flight tracker stays empty when Home mode is enabled, since the execution
|
||||
// registry covers that mode.
|
||||
func TestManager_InFlightSnapshot_DisabledInHomeMode(t *testing.T) {
|
||||
executor := newInFlightBlockingExecutor("inflight-home-provider")
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
m.SetConfigSnapshot(&internalconfig.Config{Home: internalconfig.HomeConfig{Enabled: true}})
|
||||
|
||||
snapshot := m.InFlightSnapshot()
|
||||
if snapshot.InFlight != 0 {
|
||||
t.Fatalf("expected empty snapshot in home mode, got %+v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
// inFlightRetryableStatusError is a retryable upstream error for retry tests.
|
||||
type inFlightRetryableStatusError struct{}
|
||||
|
||||
func (inFlightRetryableStatusError) Error() string { return "upstream unavailable" }
|
||||
func (inFlightRetryableStatusError) StatusCode() int { return http.StatusServiceUnavailable }
|
||||
|
||||
// inFlightFailThenBlockExecutor fails the first Execute call with a retryable
|
||||
// error and blocks in subsequent calls until released.
|
||||
type inFlightFailThenBlockExecutor struct {
|
||||
id string
|
||||
calls atomic.Int32
|
||||
release chan struct{}
|
||||
releaseOnce sync.Once
|
||||
}
|
||||
|
||||
func newInFlightFailThenBlockExecutor(id string) *inFlightFailThenBlockExecutor {
|
||||
return &inFlightFailThenBlockExecutor{id: id, release: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) Identifier() string { return e.id }
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) Execute(ctx context.Context, _ *Auth, _ cliproxyexecutor.Request, _ cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
call := e.calls.Add(1)
|
||||
if call == 1 {
|
||||
cliproxyexecutor.MarkUpstreamAttempt(ctx)
|
||||
return cliproxyexecutor.Response{}, inFlightRetryableStatusError{}
|
||||
}
|
||||
<-e.release
|
||||
return cliproxyexecutor.Response{Payload: []byte(`{"status":"ok"}`)}, nil
|
||||
}
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) ExecuteStream(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (*cliproxyexecutor.StreamResult, error) {
|
||||
return nil, errors.New("stream not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) Refresh(_ context.Context, auth *Auth) (*Auth, error) {
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) CountTokens(context.Context, *Auth, cliproxyexecutor.Request, cliproxyexecutor.Options) (cliproxyexecutor.Response, error) {
|
||||
return cliproxyexecutor.Response{}, errors.New("not supported")
|
||||
}
|
||||
|
||||
func (e *inFlightFailThenBlockExecutor) HttpRequest(context.Context, *Auth, *http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("not supported")
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_CoversRetryWait verifies that a request waiting
|
||||
// between retry rounds still appears in the snapshot instead of blinking to 0.
|
||||
func TestManager_InFlightSnapshot_CoversRetryWait(t *testing.T) {
|
||||
executor := newInFlightFailThenBlockExecutor("inflight-retry-provider")
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
m.SetRetryConfig(1, 20*time.Second, 0)
|
||||
|
||||
model := "inflight-retry-model"
|
||||
if _, errRegister := m.Register(context.Background(), &Auth{
|
||||
ID: "auth-inflight-retry-1",
|
||||
Provider: executor.Identifier(),
|
||||
Metadata: map[string]any{"request_retry": 1, "disable_cooling": true},
|
||||
}); errRegister != nil {
|
||||
t.Fatalf("Register auth: %v", errRegister)
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient("auth-inflight-retry-1", executor.Identifier(), []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() {
|
||||
registry.GetGlobalRegistry().UnregisterClient("auth-inflight-retry-1")
|
||||
})
|
||||
|
||||
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, errExec := m.Execute(context.Background(), []string{executor.Identifier()}, req, cliproxyexecutor.Options{})
|
||||
done <- errExec
|
||||
}()
|
||||
|
||||
// Wait for the first call to record its retryable failure. The request then
|
||||
// enters the retry wait window while the second call blocks on release.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for executor.calls.Load() < 1 && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if executor.calls.Load() < 1 {
|
||||
t.Fatalf("expected the first upstream call to happen, calls = %d", executor.calls.Load())
|
||||
}
|
||||
// Allow the retry loop to record the failure and enter its wait or the
|
||||
// blocking second call; either way the request is still active and must
|
||||
// remain visible in the snapshot.
|
||||
deadline = time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
snapshot := m.InFlightSnapshot()
|
||||
if snapshot.InFlight == 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("in_flight = %d during retry, want 1 (snapshot: %+v)", snapshot.InFlight, snapshot)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
executor.releaseOnce.Do(func() { close(executor.release) })
|
||||
if errExec := <-done; errExec != nil {
|
||||
t.Fatalf("Execute() error = %v", errExec)
|
||||
}
|
||||
waitForInFlightCount(t, m, 0)
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_StreamCancel verifies that canceling the request
|
||||
// context releases the in-flight scope even while the upstream chunk channel
|
||||
// stays open and silent after the first chunk was already forwarded.
|
||||
func TestManager_InFlightSnapshot_StreamCancel(t *testing.T) {
|
||||
chunks := make(chan cliproxyexecutor.StreamChunk)
|
||||
executor := &inFlightStreamExecutor{id: "inflight-cancel-provider", chunks: chunks}
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
|
||||
model := "inflight-cancel-model"
|
||||
registerInFlightAuth(t, m, "auth-inflight-cancel-1", executor.Identifier(), model)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
|
||||
|
||||
type streamOutcome struct {
|
||||
result *cliproxyexecutor.StreamResult
|
||||
err error
|
||||
}
|
||||
outcome := make(chan streamOutcome, 1)
|
||||
go func() {
|
||||
result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
|
||||
outcome <- streamOutcome{result: result, err: errStream}
|
||||
}()
|
||||
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
|
||||
res := <-outcome
|
||||
if res.err != nil {
|
||||
t.Fatalf("ExecuteStream() error = %v", res.err)
|
||||
}
|
||||
result := res.result
|
||||
|
||||
waitForInFlightCount(t, m, 1)
|
||||
|
||||
// Consume the forwarded bootstrap chunk so the wrapper is blocked waiting
|
||||
// for the next upstream chunk (receive branch) when cancel happens.
|
||||
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: second\n\n")}
|
||||
if chunk, ok := <-result.Chunks; !ok || len(chunk.Payload) == 0 {
|
||||
t.Fatalf("expected forwarded chunk, got ok=%v payload=%q", ok, chunk.Payload)
|
||||
}
|
||||
if snapshot := m.InFlightSnapshot(); snapshot.InFlight != 1 {
|
||||
t.Fatalf("expected stream to remain in flight, snapshot: %+v", snapshot)
|
||||
}
|
||||
|
||||
// The upstream keeps its chunk channel open but silent; canceling the
|
||||
// request context must release the scope without needing another chunk.
|
||||
cancel()
|
||||
waitForInFlightCount(t, m, 0)
|
||||
|
||||
// Close the source so the drain path and the output channel can finish
|
||||
// instead of leaking a background goroutine.
|
||||
close(chunks)
|
||||
for range result.Chunks {
|
||||
}
|
||||
}
|
||||
|
||||
// TestManager_InFlightSnapshot_StreamCancelBeforeConsume verifies the send
|
||||
// branch: cancellation while the first chunk is still queued for the caller
|
||||
// also releases the in-flight scope.
|
||||
func TestManager_InFlightSnapshot_StreamCancelBeforeConsume(t *testing.T) {
|
||||
chunks := make(chan cliproxyexecutor.StreamChunk)
|
||||
executor := &inFlightStreamExecutor{id: "inflight-cancel-early-provider", chunks: chunks}
|
||||
m := NewManager(nil, nil, nil)
|
||||
m.RegisterExecutor(executor)
|
||||
|
||||
model := "inflight-cancel-early-model"
|
||||
registerInFlightAuth(t, m, "auth-inflight-cancel-early-1", executor.Identifier(), model)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
req := cliproxyexecutor.Request{Model: model, Payload: []byte(`{"input":"hi"}`)}
|
||||
|
||||
type streamOutcome struct {
|
||||
result *cliproxyexecutor.StreamResult
|
||||
err error
|
||||
}
|
||||
outcome := make(chan streamOutcome, 1)
|
||||
go func() {
|
||||
result, errStream := m.ExecuteStream(ctx, []string{executor.Identifier()}, req, cliproxyexecutor.Options{Stream: true})
|
||||
outcome <- streamOutcome{result: result, err: errStream}
|
||||
}()
|
||||
chunks <- cliproxyexecutor.StreamChunk{Payload: []byte("data: bootstrap\n\n")}
|
||||
res := <-outcome
|
||||
if res.err != nil {
|
||||
t.Fatalf("ExecuteStream() error = %v", res.err)
|
||||
}
|
||||
result := res.result
|
||||
|
||||
waitForInFlightCount(t, m, 1)
|
||||
|
||||
// Cancel without consuming the queued bootstrap chunk; the send branch of
|
||||
// the wrapper must observe the cancellation and release the scope.
|
||||
cancel()
|
||||
waitForInFlightCount(t, m, 0)
|
||||
|
||||
close(chunks)
|
||||
for range result.Chunks {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user