Merge pull request #7377 from router-for-me/home

feat(auth): compensate weighted routing for sticky sessions
This commit is contained in:
Luis Pater
2026-11-09 17:34:44 +08:00
committed by GitHub
19 changed files with 2503 additions and 77 deletions

View File

@@ -170,6 +170,16 @@ routing:
strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first
# weighted-round-robin uses each credential's integer weight (default 1, maximum 1,000,000).
# Non-positive weights exclude the credential while this strategy is active.
# Sticky requests (including inherited and LCP bindings) also consume weighted credits
# without moving existing bindings. Compensation is scoped to each eligible credential set;
# priority tiers, transport preferences, and retry/cooldown subsets do not share credits.
# Pinned requests use the eligible pool before pinning, within the pinned credential
# priority tier and transport class. Returning to a pool resumes its retained credits.
# Settled credits are capped at +/-32 times the eligible pool total; ratios are approximate.
# In-flight offsets are preserved until the pool settles, regardless of commit order.
# Sticky requests keep their bindings but stop adding compensation when the full
# credit delta cannot fit the cap; new sessions keep earning credit to avoid starvation.
# Credits are process-local and reset on restart or selector replacement.
# For OAuth/file credentials, add a top-level numeric "weight" field to the auth JSON.
# Enable universal session-sticky routing for all clients.
# Explicit Claude Code, Codex, OpenCode, and pi session headers are preferred,

View File

@@ -64,6 +64,36 @@ The Home-owned revision fields
`credentials.concurrency.observation-barrier-revision`, and `plugins.auth-revision`
cannot be changed through these endpoints.
### Weighted routing with session affinity
Set `routing.strategy` to `weighted-round-robin` to use per-credential integer
`weight` values (default `1`, maximum `1000000`; non-positive values are excluded).
With `routing.session-affinity` enabled, sticky requests, parent inheritance,
Guardian selections, and LCP matches also consume weighted credits without moving
existing bindings. New or reselected sessions favor under-served credentials
within the same eligible credential set. For example, with weights `3:1`, four
sticky requests to the second credential can be compensated by twelve new
requests to the first when both remain eligible.
Different priority tiers, transport preferences, and retry/cooldown candidate
sets keep separate credits; returning to a set resumes its retained credits.
Pinned requests account against the eligible pool before pinning, within the
pinned credential's priority tier and transport class. A lower-priority binding
never shifts the request ratio among higher-priority credentials.
Settled credits are capped at ±32 times the eligible pool total, so request
ratios are approximate rather than guaranteed. While reservations are in flight,
committed balances can temporarily exceed the cap; their pending offsets remain
intact until the pool settles, including when the last reservation rolls back. Sticky requests still use their binding at
the cap, but stop adding compensation when their full credit delta cannot fit.
New sessions continue repaying debt and earning credit, so excessive sticky
traffic does not starve smaller-weight peers.
Credits are local to the CPA process and reset on restart, selector replacement,
or bounded pool-cache eviction. Concurrent selections see pending credit; local
selection/binding success confirms it, while failure releases it. In Home mode,
scheduling and credit accounting remain owned by Home.
### Codex multi-agent configuration migration
`client.codex.optimize-multi-agent-v2` is the sole runtime setting. Loading older

View File

@@ -1084,11 +1084,17 @@ func (m *Manager) pickHomeDispatchSelection(ctx context.Context, model string, o
}
dispatchHeaders := homeDispatchHeaders(ctx, opts.Headers)
if opts.Metadata != nil {
if nodeKind, ok := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string); ok && strings.TrimSpace(nodeKind) != "" {
nodeKind, _ := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string)
if isFork, _ := opts.Metadata[cliproxyexecutor.IsForkMetadataKey].(bool); isFork {
nodeKind = "fork"
}
if nodeKind = strings.TrimSpace(nodeKind); nodeKind != "" {
// Keep the caller headers unchanged when adding Home-only lineage metadata.
dispatchHeaders = dispatchHeaders.Clone()
if dispatchHeaders == nil {
dispatchHeaders = make(http.Header)
}
dispatchHeaders.Set("X-Node-Kind", strings.TrimSpace(nodeKind))
dispatchHeaders.Set("X-Node-Kind", nodeKind)
}
}
credentialPolicy := credentialPolicyFromContext(ctx)

View File

@@ -0,0 +1,96 @@
package auth
import (
"context"
"errors"
"fmt"
"net/http"
"testing"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
func TestManagerNonWeightedPinnedSelection(t *testing.T) {
for _, mixed := range []bool{false, true} {
for _, affinity := range []bool{false, true} {
for _, strategy := range []string{"round-robin", "fill-first"} {
t.Run(fmt.Sprintf("mixed=%t/affinity=%t/%s", mixed, affinity, strategy), func(t *testing.T) {
var selector Selector = &RoundRobinSelector{}
if strategy == "fill-first" {
selector = &FillFirstSelector{}
}
if affinity {
sticky := NewSessionAffinitySelector(selector)
t.Cleanup(sticky.Stop)
selector = sticky
}
manager := NewManager(nil, selector, nil)
const provider, model = "pinned-non-weighted", "pinned-model"
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
pinnedID := t.Name() + "-pinned"
for i := range 32 {
auth := &Auth{
ID: fmt.Sprintf("%s-other-%02d", t.Name(), i), Provider: provider, Status: StatusActive,
Attributes: map[string]string{AttributeAuthKind: AuthKindAPIKey, "priority": "10"},
}
if i == 0 {
auth.ID = pinnedID
auth.Attributes["priority"] = "0"
}
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
t.Fatal(errRegister)
}
}
// Normalize the legacy selection signatures for the same pinning checks.
pick := func(ctx context.Context, id string, tried map[string]struct{}) (*Auth, error) {
opts := cliproxyexecutor.Options{
Headers: http.Header{"Session_id": {"non-weighted-pinned"}},
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: id},
}
if mixed {
auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider}, model, opts, tried)
return auth, errPick
}
auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
return auth, errPick
}
ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
for range 3 {
if auth, errPick := pick(ctx, pinnedID, nil); errPick != nil || auth == nil || auth.ID != pinnedID {
t.Fatalf("lower-priority pin not enforced: auth=%v err=%v", auth, errPick)
}
}
for _, tc := range []struct {
name string
ctx context.Context
id string
tried map[string]struct{}
}{
{name: "missing pin", ctx: ctx, id: "missing"},
{name: "already tried", ctx: ctx, id: pinnedID, tried: map[string]struct{}{pinnedID: {}}},
{name: "ineligible kind", ctx: withRequiredAuthKind(t.Context(), AuthKindOAuth), id: pinnedID},
} {
auth, errPick := pick(tc.ctx, tc.id, tc.tried)
var authErr *Error
if auth != nil || !errors.As(errPick, &authErr) || authErr.Code != "auth_not_found" {
t.Fatalf("%s: auth=%v err=%v, want auth_not_found", tc.name, auth, errPick)
}
}
if sticky, ok := selector.(*SessionAffinitySelector); ok {
namespace := provider
if mixed {
namespace = "mixed"
}
id := CanonicalSessionID(http.Header{"Session_id": {"non-weighted-pinned"}}, nil, nil)
if binding, found := sticky.cache.Get(namespace + "::" + id + "::" + model); !found || binding != pinnedID {
t.Fatalf("pinning checks changed the sticky binding: %q, found=%t", binding, found)
}
}
})
}
}
}
}

View File

@@ -2258,7 +2258,11 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
targetKey := canonicalSchedulingProvider(provider)
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
var affinityCandidates []*Auth
trackPinnedAccounting := false
if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
_, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
}
var accountingCandidates, affinityCandidates []*Auth
if trackTransientAffinity {
affinityCandidates = make([]*Auth, 0, len(m.auths))
}
@@ -2266,7 +2270,7 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if candidate == nil || canonicalSchedulingProvider(executorKeyFromAuth(candidate)) != targetKey || candidate.Disabled {
continue
}
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if !eligibility.allows(candidate) {
@@ -2275,10 +2279,18 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
continue
}
_, used := tried[candidate.ID]
// Pinning restricts selection, not the eligible pool used for accounting.
if trackPinnedAccounting && !used {
accountingCandidates = append(accountingCandidates, candidate)
}
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if trackTransientAffinity {
affinityCandidates = append(affinityCandidates, candidate)
}
if _, used := tried[candidate.ID]; used {
if used {
continue
}
candidates = append(candidates, candidate)
@@ -2302,6 +2314,15 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
return nil, nil, errAvailable
}
}
var accountingAuths []*Auth
if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
_, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, provider, model, now, false, opts.Headers)
if errAccounting != nil {
m.mu.RUnlock()
return nil, nil, errAccounting
}
accountingAuths = fullAvailable
}
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
m.mu.RUnlock()
@@ -2313,6 +2334,9 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if !handled {
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
if accountingAuths != nil {
selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
}
selected, errPick = selector.Pick(selectorCtx, provider, selectionArgForSelector(selector, model), opts, selectorAuths)
if errPick != nil {
if isAuthNotFoundError(errPick) {
@@ -2678,7 +2702,11 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
registryRef := registry.GetGlobalRegistry()
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
var affinityCandidates []*Auth
trackPinnedAccounting := false
if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
_, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
}
var accountingCandidates, affinityCandidates []*Auth
if trackTransientAffinity {
affinityCandidates = make([]*Auth, 0, len(m.auths))
}
@@ -2686,7 +2714,7 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if candidate == nil || candidate.Disabled {
continue
}
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if !eligibility.allows(candidate) {
@@ -2705,10 +2733,18 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
continue
}
_, used := tried[candidate.ID]
// Pinning restricts selection, not the eligible pool used for accounting.
if trackPinnedAccounting && !used {
accountingCandidates = append(accountingCandidates, candidate)
}
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if trackTransientAffinity {
affinityCandidates = append(affinityCandidates, candidate)
}
if _, used := tried[candidate.ID]; used {
if used {
continue
}
candidates = append(candidates, candidate)
@@ -2732,6 +2768,15 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
return nil, nil, "", errAvailable
}
}
var accountingAuths []*Auth
if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
_, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, "mixed", model, now, false, opts.Headers)
if errAccounting != nil {
m.mu.RUnlock()
return nil, nil, "", errAccounting
}
accountingAuths = fullAvailable
}
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
m.mu.RUnlock()
@@ -2743,6 +2788,9 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if !handled {
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
if accountingAuths != nil {
selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
}
selected, errPick = selector.Pick(selectorCtx, "mixed", selectionArgForSelector(selector, model), opts, selectorAuths)
if errPick != nil {
if isAuthNotFoundError(errPick) {

View File

@@ -0,0 +1,161 @@
package auth
import (
"fmt"
"maps"
"net/http"
"testing"
"time"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
)
func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
for _, mixed := range []bool{false, true} {
for _, mode := range []string{"sticky", "cold", "no-session", "lcp"} {
t.Run(fmt.Sprintf("mixed=%t/%s", mixed, mode), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
manager := NewManager(nil, selector, nil)
const provider, model = "weighted-pinned", "weighted-pinned-model"
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
aID, bID := t.Name()+"-a", t.Name()+"-b"
triedID := t.Name() + "-tried"
excluded := []*Auth{
{ID: t.Name() + "-disabled", Disabled: true},
{ID: t.Name() + "-zero", Attributes: map[string]string{AttributeWeight: "0"}},
{ID: t.Name() + "-cooldown", ModelStates: map[string]*ModelState{
model: {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
}},
{ID: t.Name() + "-wrong-model"},
{ID: t.Name() + "-wrong-kind", Attributes: map[string]string{AttributeAuthKind: AuthKindOAuth}},
{ID: t.Name() + "-wrong-provider", Provider: "other-provider"},
{ID: t.Name() + "-no-executor", Provider: "no-executor"},
{ID: t.Name() + "-lower-priority", Attributes: map[string]string{"priority": "0"}},
{ID: triedID},
}
auths := append([]*Auth{
{ID: aID, Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
{ID: bID, Attributes: map[string]string{AttributeWeight: "1", "priority": "1"}},
}, excluded...)
for _, auth := range auths {
if auth.Provider == "" {
auth.Provider = provider
}
auth.Status = StatusActive
if auth.Attributes == nil {
auth.Attributes = make(map[string]string)
}
if _, ok := auth.Attributes[AttributeWeight]; !ok {
auth.Attributes[AttributeWeight] = "100"
}
if _, ok := auth.Attributes["priority"]; !ok {
auth.Attributes["priority"] = "2"
}
if _, ok := auth.Attributes[AttributeAuthKind]; !ok {
auth.Attributes[AttributeAuthKind] = AuthKindAPIKey
}
registeredModel := model
if auth.ID == t.Name()+"-wrong-model" {
registeredModel = "other-model"
}
registry.GetGlobalRegistry().RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: registeredModel}})
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
if _, errRegister := manager.Register(WithSkipPersist(ctx), auth); errRegister != nil {
t.Fatal(errRegister)
}
}
namespace := provider
if mixed {
namespace = "mixed"
}
stickyHeaders := http.Header{"Session_id": {"pinned-session"}}
id := CanonicalSessionID(stickyHeaders, nil, nil)
bindingKey := namespace + "::" + id + "::" + model
if mode == "sticky" {
selector.cache.Set(bindingKey, bID)
}
tried := map[string]struct{}{triedID: {}}
// Normalize the legacy signatures without changing their pinning behavior.
pick := func(opts cliproxyexecutor.Options) (*Auth, error) {
if mixed {
auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider, "no-executor"}, model, opts, tried)
return auth, errPick
}
auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
return auth, errPick
}
for i := range 4 {
opts := cliproxyexecutor.Options{Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: bID}}
switch mode {
case "sticky", "cold":
opts.Headers = stickyHeaders
case "lcp":
opts.SourceFormat = sdktranslator.FormatOpenAI
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"pinned conversation"}]}`)
opts.Metadata[cliproxyexecutor.CallerScopeMetadataKey] = "pinned-caller"
}
if auth, errPick := pick(opts); errPick != nil || auth == nil || auth.ID != bID {
t.Fatalf("pinned request %d: auth=%v err=%v, want %q", i, auth, errPick, bID)
}
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 {
t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state)
}
counts := map[string]int{}
for i := range 12 {
auth, errPick := pick(cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}})
if errPick != nil || auth == nil {
t.Fatalf("fresh request %d: auth=%v err=%v", i, auth, errPick)
}
counts[auth.ID]++
}
if counts[aID] != 12 || counts[bID] != 0 {
t.Fatalf("fresh distribution = %v, want a=12 b=0", counts)
}
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if mode == "sticky" || mode == "cold" {
if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID {
t.Fatalf("compensation changed pinned binding: %q, found=%t", binding, ok)
}
}
// A broader accounting pool must never make an invalid pin selectable.
for _, pinnedID := range []string{excluded[0].ID, excluded[1].ID, excluded[2].ID, excluded[3].ID, excluded[4].ID, excluded[5].ID, excluded[6].ID, triedID, "missing"} {
opts := cliproxyexecutor.Options{
Headers: stickyHeaders,
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: pinnedID},
}
if auth, errPick := pick(opts); errPick == nil || auth != nil {
t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick)
}
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest())
}
}
// A valid lower-priority pin still wins without charging another tier.
lowerID := t.Name() + "-lower-priority"
lowerOpts := cliproxyexecutor.Options{
Headers: http.Header{"Session_id": {"lower-pinned-session"}},
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: lowerID},
}
if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID {
t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick)
}
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID)
if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 {
t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState)
}
})
}
}
}

View File

@@ -0,0 +1,59 @@
package auth
import (
"context"
"net/http"
"testing"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
type sessionKindCaptureDispatcher struct {
fixtureHomeDispatcher
headers http.Header
sessionID string
parentSessionID string
}
func (d *sessionKindCaptureDispatcher) RPopAuthWithSessionHierarchy(_ context.Context, _ string, sessionID, parentSessionID string, headers http.Header, _ int, _ string, _ *int, _ []string, _ string) ([]byte, error) {
d.headers = headers.Clone()
d.sessionID = sessionID
d.parentSessionID = parentSessionID
return d.payload, nil
}
func TestHomeDispatchPreservesForkAndCompactionKind(t *testing.T) {
for _, tc := range []struct {
name string
body string
metadata map[string]any
want string
}{
{name: "body fork", body: `{"thread_id":"child","metadata":{"forked_from_thread_id":"parent"}}`, want: "fork"},
{name: "compaction", body: `{"session_id":"child","parent_session_id":"parent"}`, metadata: map[string]any{cliproxyexecutor.NodeKindMetadataKey: "compaction"}, want: "compaction"},
{name: "ordinary child", body: `{"session_id":"child","parent_session_id":"parent"}`},
} {
t.Run(tc.name, func(t *testing.T) {
dispatcher := &sessionKindCaptureDispatcher{fixtureHomeDispatcher: fixtureHomeDispatcher{payload: []byte(`{"auth":{"id":"kind-auth","provider":"kind-provider"}}`)}}
manager := newHomeSelectionTestManager(t, dispatcher)
manager.RegisterExecutor(schedulerTestExecutor{provider: "kind-provider"})
headers := http.Header{"User-Agent": {"test-client"}}
opts := cliproxyexecutor.Options{Headers: headers, OriginalRequest: []byte(tc.body), Metadata: tc.metadata}
opts.EnsureMetadata()
selection, errSelection := manager.pickHomeDispatchSelection(t.Context(), "model", opts)
if errSelection != nil {
t.Fatal(errSelection)
}
selection.End("test_complete")
if dispatcher.sessionID == "" || dispatcher.parentSessionID == "" || dispatcher.sessionID == dispatcher.parentSessionID {
t.Fatalf("lost session hierarchy: child=%q parent=%q", dispatcher.sessionID, dispatcher.parentSessionID)
}
if got := dispatcher.headers.Get("X-Node-Kind"); got != tc.want {
t.Fatalf("node kind=%q, want %q", got, tc.want)
}
if got := headers.Get("X-Node-Kind"); got != "" {
t.Fatalf("caller headers mutated: %q", got)
}
})
}
}

View File

@@ -214,3 +214,41 @@ func BenchmarkManagerPickNextAndMarkResult1000(b *testing.B) {
manager.MarkResult(ctx, Result{AuthID: auth.ID, Provider: "gemini", Model: model, Success: true})
}
}
func BenchmarkManagerPinnedLegacy1000(b *testing.B) {
for _, mixed := range []bool{false, true} {
b.Run(fmt.Sprintf("mixed=%t", mixed), func(b *testing.B) {
manager, providers, model := benchmarkManagerSetup(b, 1000, mixed, false)
selector := NewSessionAffinitySelector(&RoundRobinSelector{})
b.Cleanup(selector.Stop)
manager.SetSelector(selector)
ctx := context.Background()
opts := cliproxyexecutor.Options{Metadata: map[string]any{
cliproxyexecutor.PinnedAuthMetadataKey: "bench-gemini-0000",
}}
// Warm up index assignment and selection metadata outside the timed loop.
if mixed {
if _, _, _, errWarm := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil); errWarm != nil {
b.Fatal(errWarm)
}
} else if _, _, errWarm := manager.pickNextLegacy(ctx, providers[0], model, opts, nil); errWarm != nil {
b.Fatal(errWarm)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if mixed {
auth, exec, provider, errPick := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil)
if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil || provider != providers[0] {
b.Fatalf("pinned mixed pick failed: auth=%v exec=%v provider=%q err=%v", auth, exec, provider, errPick)
}
} else {
auth, exec, errPick := manager.pickNextLegacy(ctx, providers[0], model, opts, nil)
if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil {
b.Fatalf("pinned pick failed: auth=%v exec=%v err=%v", auth, exec, errPick)
}
}
}
})
}
}

View File

@@ -39,9 +39,10 @@ type RoundRobinSelector struct {
// WeightedRoundRobinSelector provides smooth weighted round-robin selection.
type WeightedRoundRobinSelector struct {
mu sync.Mutex
states map[string]*smoothWeightedState
maxKeys int
mu sync.Mutex
states map[[32]byte]*weightedPoolState
stateCount int
maxKeys int
}
type smoothWeightedState struct {
@@ -698,32 +699,12 @@ func (s *WeightedRoundRobinSelector) Pick(ctx context.Context, provider, model s
return nil, errAvailable
}
available = preferCodexWebsocketAuths(ctx, provider, available)
stateModel := weightedSelectorStateModel(ctx, model)
key := provider + ":" + canonicalModelLookupKey(stateModel)
s.mu.Lock()
defer s.mu.Unlock()
if s.states == nil {
s.states = make(map[string]*smoothWeightedState)
}
limit := s.maxKeys
if limit <= 0 {
limit = 4096
}
if _, ok := s.states[key]; !ok && len(s.states) >= limit {
s.states = make(map[string]*smoothWeightedState)
}
state := s.states[key]
if state == nil {
state = &smoothWeightedState{}
s.states[key] = state
}
weights := authWeightVector(available)
state.prepare(weights)
picked := pickSmoothWeightedAuth(available, state.current)
key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
picked, reservation := s.reserve(key, available, "")
if picked == nil {
return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
}
reservation.finish(true)
return picked, nil
}
@@ -796,29 +777,6 @@ func authWeightVector(auths []*Auth) map[string]int64 {
return weights
}
func pickSmoothWeightedAuth(auths []*Auth, current map[string]int64) *Auth {
var picked *Auth
var pickedCurrent int64
var totalWeight int64
for _, auth := range auths {
weight := authWeight(auth)
if auth == nil || weight <= 0 {
continue
}
current[auth.ID] = saturatingAddInt64(current[auth.ID], weight)
totalWeight = saturatingAddInt64(totalWeight, weight)
if picked == nil || current[auth.ID] > pickedCurrent {
picked = auth
pickedCurrent = current[auth.ID]
}
}
if picked == nil {
return nil
}
current[picked.ID] = saturatingAddInt64(current[picked.ID], -totalWeight)
return picked
}
func saturatingAddInt64(value, delta int64) int64 {
if delta > 0 && value > math.MaxInt64-delta {
return math.MaxInt64
@@ -1164,7 +1122,16 @@ func (s *SessionAffinitySelector) Trees() *cliproxysession.InMemorySessionTreeSt
// Note: The cache key includes provider, session ID, and model to handle cases where
// a session uses multiple models (e.g., gemini-2.5-pro and gemini-3-flash-preview)
// that may be supported by different auth credentials, and to avoid cross-provider conflicts.
func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) {
func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (selected *Auth, errPick error) {
if ctx == nil {
ctx = context.Background()
}
if errContext := ctx.Err(); errContext != nil {
return nil, errContext
}
var reservation *weightedReservation
// Local bindings complete inside Pick; failed selections release speculative credit.
defer func() { reservation.finish(selected != nil && errPick == nil) }()
entry := selectorLogEntry(ctx)
if opts.Metadata == nil {
opts.Metadata = make(map[string]any)
@@ -1176,7 +1143,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
// consulted when no header, body, or execution-session identity is present.
explicitID, explicitFallbackID := extractExplicitSessionIDs(opts.Headers, opts.OriginalRequest, opts.Metadata)
if explicitID == "" {
if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry); handled || errLCP != nil {
if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry, &reservation); handled || errLCP != nil {
return auth, errLCP
}
} else if opts.Metadata != nil {
@@ -1218,7 +1185,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, errAvailable
}
entry.Debugf("session-affinity: no session ID extracted, falling back to default selector | provider=%s model=%s", provider, model)
return s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
return s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
}
// A single availability pass serves both lookups: the bound credential is validated against
@@ -1268,7 +1235,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if auth.ID == cachedAuthID {
bind(auth.ID, true)
entry.Infof("session-affinity: cache hit | session=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), auth.ID, provider, model)
return auth, nil
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
if s.rebindOnNever {
@@ -1281,7 +1248,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, err
}
// Cached auth not available, reselect via fallback selector for even distribution
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1315,7 +1282,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if auth.ID == parentAuthID {
bind(auth.ID, true)
entry.Infof("session-affinity: guardian inherited parent binding | session=%s parent=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
return auth, nil
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
}
@@ -1332,7 +1299,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
} else {
entry.Infof("session-affinity: fallback cache hit | session=%s fallback=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
}
return auth, nil
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
}
@@ -1343,7 +1310,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, errBound
}
if !isSubagent && !isFork && s.shouldPreserveBindingWhenUnavailable(ctx, availabilityCandidates, model, cachedAuthID, now) {
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1360,7 +1327,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if err != nil {
return nil, err
}
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1376,7 +1343,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return auth, nil
}
func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry) (*Auth, bool, error) {
func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry, reservation **weightedReservation) (*Auth, bool, error) {
if s == nil || s.matcher == nil {
return nil, false, nil
}
@@ -1449,7 +1416,8 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
}
entry.Infof("session-affinity: LCP cache hit | session=%s prefix=%d auth=%s provider=%s model=%s", truncateSessionID(match.SessionID), match.PrefixLength, auth.ID, provider, model)
}
return auth, true, nil
picked, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, reservation)
return picked, true, errPick
}
if !match.IsFork && s.rebindOnNever {
if bindRes := s.matcher.BindFingerprintsWithContext(namespace, fingerprints, tailFingerprints, envDigest, minPrefixLength, match.AuthID); bindRes.SessionID != "" {
@@ -1488,7 +1456,7 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
return nil, true, errAvailable
}
fallbackAuths := highestPriorityAuths(available)
auth, errPick := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
auth, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, reservation)
if errPick != nil {
return nil, true, errPick
}

View File

@@ -205,16 +205,14 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) {
}
}
func TestPickSmoothWeightedAuth_SaturatesCorruptState(t *testing.T) {
func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) {
t.Parallel()
current := map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}
picked := pickSmoothWeightedAuth([]*Auth{{ID: "a"}, {ID: "b"}}, current)
if picked == nil {
t.Fatal("pickSmoothWeightedAuth() returned nil")
}
if current["a"] != math.MaxInt64-2 || current["b"] != math.MinInt64+1 {
t.Fatalf("current state = %#v, want saturated arithmetic", current)
state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64})
state.charge(0, false)
limit := maxSmoothWeightedCreditRounds * 2
if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit {
t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest())
}
}

View File

@@ -0,0 +1,358 @@
package auth
import (
"context"
"crypto/sha256"
"encoding/binary"
"math"
"sort"
"time"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
// weightedAccountingPoolKey carries the manager-validated pre-pin pool. It affects
// accounting only; the selector still receives pin-filtered selection candidates.
type weightedAccountingPoolKey struct{}
// weightedReservation separates speculative credit from committed, clamped credit.
// Releasing one request must never undo accounting committed by another request.
type weightedReservation struct {
selector *WeightedRoundRobinSelector
state *weightedPoolState
generation uint64
member int
total int64
bound bool
finished bool
}
// pickWithAffinityReservation normalizes weighted candidates and defers accounting
// until the local binding completes. Other fallback selectors keep their behavior.
func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, bound *Auth, reservation **weightedReservation) (*Auth, error) {
weighted, ok := s.fallback.(*WeightedRoundRobinSelector)
if !ok {
if bound != nil {
return bound, nil
}
return s.fallback.Pick(ctx, provider, model, opts, auths)
}
if accountingAuths, ok := ctx.Value(weightedAccountingPoolKey{}).([]*Auth); ok {
pinnedID := pinnedAuthIDFromMetadata(opts.Metadata)
if bound == nil {
for _, auth := range auths {
if auth != nil && auth.ID == pinnedID {
bound = auth
break
}
}
}
if bound == nil || bound.ID != pinnedID || authWeight(bound) <= 0 {
return nil, &Error{Code: "auth_unavailable", Message: "pinned auth is not available for weighted accounting"}
}
auths = positiveWeightAuths(accountingAuths)
}
var available []*Auth
authID := ""
if bound == nil {
var errAvailable error
available, errAvailable = getSelectorAvailableAuths(ctx, positiveWeightAuths(auths), provider, model, time.Now())
if errAvailable != nil {
return nil, errAvailable
}
available = preferCodexWebsocketAuths(ctx, provider, available)
} else {
authID = bound.ID
// A binding may bypass priority or transport preference, but its traffic
// must only charge peers that compete with it, never another priority tier.
priority := authPriority(bound)
for _, auth := range auths {
if authPriority(auth) == priority {
available = append(available, auth)
}
}
preferred := preferCodexWebsocketAuths(ctx, provider, available)
if len(preferred) < len(available) && !authWebsocketsEnabled(bound) {
// Keep an HTTP-bound session outside the preferred WebSocket pool.
nonWebsocket := available[:0]
for _, auth := range available {
if !authWebsocketsEnabled(auth) {
nonWebsocket = append(nonWebsocket, auth)
}
}
available = nonWebsocket
} else {
available = preferred
}
}
key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
picked, pending := weighted.reserve(key, available, authID)
if picked == nil {
return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
}
*reservation = pending
return picked, nil
}
// weightedPoolMember stores each ID once, with parallel accounting values in a
// single allocation. Members are sorted by ID; reservations retain only an index.
type weightedPoolMember struct {
id string
weight int64
current int64
pending int64
}
// weightedPoolState is separate from the scheduler's transient-subset accumulator:
// each selector pool has fixed membership and shares its weights across reservations.
type weightedPoolState struct {
route string
members []weightedPoolMember
inFlight uint64
generation uint64
next *weightedPoolState // Exact-identity collision chain for the digest key.
}
func sortedWeightedIDs(weights map[string]int64) []string {
ids := make([]string, 0, len(weights))
for id := range weights {
ids = append(ids, id)
}
sort.Strings(ids)
return ids
}
// weightedPoolStateKey isolates competing sets without retaining concatenated IDs.
// Weights stay out of the key so configuration changes invalidate reservations.
func weightedPoolStateKey(route string, weights map[string]int64) [32]byte {
return weightedPoolDigest(route, sortedWeightedIDs(weights))
}
func weightedPoolDigest(route string, ids []string) [32]byte {
size := 8 + len(route)
for _, id := range ids {
size += 8 + len(id)
}
identity := make([]byte, 0, size)
identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route)))
identity = append(identity, route...)
for _, id := range ids {
identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id)))
identity = append(identity, id...)
}
return sha256.Sum256(identity)
}
// matches verifies the full identity even on a digest hit. Hash collisions must
// never combine the quota or configured weights of unrelated candidate pools.
func (s *weightedPoolState) matches(route string, ids []string) bool {
if s.route != route || len(s.members) != len(ids) {
return false
}
for i, id := range ids {
if s.members[i].id != id {
return false
}
}
return true
}
func (s *weightedPoolState) prepare(weights map[string]int64) {
for _, member := range s.members {
if member.weight != weights[member.id] {
s.generation++
s.inFlight = 0
for i := range s.members {
s.members[i].weight = weights[s.members[i].id]
s.members[i].current = 0
s.members[i].pending = 0
}
return
}
}
}
// reserve selects and reserves credit atomically. A non-empty authID preserves a
// sticky binding; otherwise committed and pending credits both guide selection.
func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) {
weights := authWeightVector(available)
ids := sortedWeightedIDs(weights)
key := weightedPoolDigest(route, ids)
s.mu.Lock()
defer s.mu.Unlock()
if s.states == nil {
s.states = make(map[[32]byte]*weightedPoolState)
}
state := s.states[key]
for state != nil && !state.matches(route, ids) {
state = state.next
}
if state == nil {
limit := s.maxKeys
if limit <= 0 {
limit = 4096
}
if s.stateCount >= limit {
s.states = make(map[[32]byte]*weightedPoolState)
s.stateCount = 0
}
state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]}
for i, id := range ids {
state.members[i] = weightedPoolMember{id: id, weight: weights[id]}
}
s.states[key] = state
s.stateCount++
} else {
state.prepare(weights)
}
var picked *Auth
var pickedCurrent, total int64
pickedMember := -1
for _, auth := range available {
if auth == nil || weights[auth.ID] <= 0 {
continue
}
total = saturatingAddInt64(total, weights[auth.ID])
if authID != "" && auth.ID != authID {
continue
}
index := sort.SearchStrings(ids, auth.ID)
member := &state.members[index]
current := saturatingAddInt64(member.current, member.pending)
current = saturatingAddInt64(current, member.weight)
if picked == nil || current > pickedCurrent {
picked, pickedCurrent, pickedMember = auth, current, index
}
}
if picked == nil {
return nil, nil
}
// Bound traffic beyond the compensation window still keeps its credential,
// but must not create speculative credit that can starve concurrent picks.
if authID != "" && !state.boundChargeFits(pickedMember, total) {
return picked, nil
}
state.adjustPending(pickedMember, total, false)
state.inFlight++
return picked, &weightedReservation{
selector: s, state: state, generation: state.generation,
member: pickedMember, total: total, bound: authID != "",
}
}
// finish confirms or releases a reservation exactly once under the selector lock.
func (r *weightedReservation) finish(committed bool) {
if r == nil {
return
}
r.selector.mu.Lock()
defer r.selector.mu.Unlock()
if r.finished {
return
}
r.finished = true
if r.state.generation != r.generation {
return
}
r.state.adjustPending(r.member, r.total, true)
r.state.inFlight--
if committed {
r.state.charge(r.member, r.bound)
} else if r.state.inFlight == 0 {
// A final rollback can remove the offset to already committed credit.
r.state.boundCredits(r.total)
}
}
// adjustPending must be called with the selector mutex held. Pending deltas are
// not clamped, so releasing them cannot lose credit at the compensation bound.
func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) {
for i := range s.members {
member := &s.members[i]
delta := member.weight
if i == selected {
delta -= total
}
if release {
delta = -delta
}
member.pending = saturatingAddInt64(member.pending, delta)
}
}
// maxSmoothWeightedCreditRounds bounds compensation after a long sticky session.
const maxSmoothWeightedCreditRounds int64 = 32
// boundChargeFits admits the whole sticky delta or none. Both reservation and
// commit include outstanding deltas: committed credit alone can transiently
// exceed the cap while successful reservations settle in a different order.
func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool {
limit := int64(math.MaxInt64)
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
limit = totalWeight * maxSmoothWeightedCreditRounds
}
for i, member := range s.members {
delta := member.weight
if i == selected {
delta -= totalWeight
}
current := saturatingAddInt64(member.current, member.pending)
next := saturatingAddInt64(current, delta)
if next < -limit || next > limit {
return false
}
}
return true
}
func (s *weightedPoolState) charge(selected int, bound bool) {
var totalWeight int64
for _, member := range s.members {
totalWeight = saturatingAddInt64(totalWeight, member.weight)
}
if !bound || s.boundChargeFits(selected, totalWeight) {
// At the cap, forced traffic cannot erase a small peer's earnings.
// Pending repayments still count until they actually roll back.
for i := range s.members {
member := &s.members[i]
member.current = saturatingAddInt64(member.current, member.weight)
}
member := &s.members[selected]
member.current = saturatingAddInt64(member.current, -totalWeight)
}
if s.inFlight == 0 {
s.boundCredits(totalWeight)
}
}
// boundCredits runs only once the pool settles. Net pending deltas can be zero
// even with live reservations, so the in-flight count, not len(pending), decides
// when clipping cannot irreversibly discard another reservation's offset.
func (s *weightedPoolState) boundCredits(totalWeight int64) {
limit := int64(math.MaxInt64)
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
limit = totalWeight * maxSmoothWeightedCreditRounds
}
var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64
for _, member := range s.members {
minCredit = min(minCredit, member.current)
maxCredit = max(maxCredit, member.current)
}
// A common offset keeps leading credits distinct at the compensation bound.
// Independent clamps can repeatedly tie the leaders and starve the later
// candidate in deterministic selection order.
minOffset := saturatingAddInt64(maxCredit, -limit)
maxOffset := saturatingAddInt64(minCredit, limit)
var offset int64
if minOffset > 0 || minOffset > maxOffset {
offset = minOffset
} else if maxOffset < 0 {
offset = maxOffset
}
// If the full spread cannot fit, cap only the trailing debt after rebasing
// the leader to the upper bound. Do not flatten leading candidates' credits.
for i := range s.members {
member := &s.members[i]
member.current = max(-limit, saturatingAddInt64(member.current, -offset))
}
}

View File

@@ -0,0 +1,436 @@
package auth
import (
"context"
"errors"
"fmt"
"net/http"
"sync"
"testing"
"time"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
cliproxysession "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/session"
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
)
func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) {
for _, source := range []string{"sticky", "parent", "guardian", "fork"} {
t.Run(source, func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{
{ID: "a", Provider: "codex", Attributes: map[string]string{AttributeWeight: "3"}},
{ID: "b", Provider: "codex"},
}
parent := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
parentID := CanonicalSessionID(parent.Headers, nil, nil)
parentModel := "model"
if source == "guardian" {
parentModel = "parent-model"
}
key := "codex::" + parentID + "::" + parentModel
selector.cache.setSessionAliases("b", "codex", parentID, key)
for i := range 4 {
opts := parent
if source != "sticky" {
opts = cliproxyexecutor.Options{Headers: http.Header{
"Session_id": {fmt.Sprintf("child-%d", i)}, "X-Codex-Parent-Thread-Id": {"busy"},
}}
if source == "guardian" {
opts.Headers.Set("X-Openai-Subagent", "guardian")
}
if source == "fork" {
opts.OriginalRequest = []byte(`{"forked_from_thread_id":"busy"}`)
}
}
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
if errPick != nil || picked == nil || picked.ID != "b" {
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
}
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 {
t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
for i := range 12 {
// Requests without an identity and cold sessions share the same credits.
opts := cliproxyexecutor.Options{}
if i%2 == 0 {
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
}
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
if errPick != nil || picked == nil || picked.ID != "a" {
t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick)
}
}
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if binding, ok := selector.cache.Get(key); !ok || binding != "b" {
t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok)
}
})
}
}
func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing.T) {
for _, newSession := range []bool{false, true} {
t.Run(fmt.Sprintf("new-session=%t", newSession), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{
{ID: "a", Attributes: map[string]string{"priority": "0"}},
{ID: "b", Attributes: map[string]string{"priority": "1"}},
{ID: "c", Attributes: map[string]string{"priority": "1"}},
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"lower-priority"}}}
id := CanonicalSessionID(sticky.Headers, nil, nil)
key := "codex::" + id + "::model"
selector.cache.Set(key, "a")
counts := map[string]int{}
const warmup, requests = 100, 1000
for i := range requests {
picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
if errPick != nil || picked == nil || picked.ID != "a" {
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
}
opts := cliproxyexecutor.Options{}
if newSession {
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}
}
picked, errPick = selector.Pick(t.Context(), "codex", "model", opts, auths)
if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
}
if i >= warmup {
counts[picked.ID]++
}
}
if counts["b"] != (requests-warmup)/2 || counts["c"] != (requests-warmup)/2 {
t.Fatalf("changing pools starved equal-weight candidates after warmup: %v, want b=450 c=450", counts)
}
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok)
}
if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 {
t.Fatalf("pending credits leaked: %v", state.pendingForTest())
}
})
}
}
func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
counts := map[string]int{}
for i := range 40 {
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{
Headers: http.Header{"Session_id": {fmt.Sprintf("session-%d", i)}},
}, auths)
if errPick != nil || picked == nil {
t.Fatalf("pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) {
t.Fatalf("first selection charged more than once: %v", state.currentForTest())
}
counts[picked.ID]++
}
if counts["a"] != 30 || counts["b"] != 10 {
t.Fatalf("cold distribution = %v, want 30:10", counts)
}
}
func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"long"}}}
id := CanonicalSessionID(opts.Headers, nil, nil)
selector.cache.Set("codex::"+id+"::model", "b")
for range 1000 {
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
if errPick != nil || picked == nil || picked.ID != "b" {
t.Fatalf("long session: pick=%v err=%v", picked, errPick)
}
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
limit := maxSmoothWeightedCreditRounds * 4
// A bound charge is admitted as a whole, stopping within one delta of the cap.
wantCredit := limit / 3 * 3
if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit {
t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit)
}
for range wantCredit {
if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
t.Fatal(errPick)
}
}
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 {
t.Fatalf("compensation did not finish: %v", state.currentForTest())
}
}
func TestWeightedSessionAffinityLCPCredits(t *testing.T) {
for _, source := range []string{"hit", "fork", "compaction"} {
t.Run(source, func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
root := cliproxyexecutor.Options{
SourceFormat: sdktranslator.FormatOpenAI,
OriginalRequest: []byte(`{"messages":[{"role":"user","content":"step 1"},` +
`{"role":"assistant","content":"ack 1"},{"role":"user","content":"step 2"},` +
`{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"}]}`),
Metadata: map[string]any{cliproxyexecutor.CallerScopeMetadataKey: "caller"},
}
// Seed the matcher without consuming weighted credits.
turns := cliproxysession.ExtractCanonicalTurns(root.SourceFormat, root.OriginalRequest)
selector.matcher.Bind(lcpAffinityNamespace("openai", "model", root.Metadata), turns, "b")
opts := root
if source == "fork" {
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"step 1"},` +
`{"role":"assistant","content":"ack 1"},{"role":"user","content":"branch"}]}`)
} else if source == "compaction" {
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"<summary>Steps 1 and 2 completed</summary>"},` +
`{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"},` +
`{"role":"user","content":"step 4"}]}`)
}
picked, errPick := selector.Pick(t.Context(), "openai", "model", opts, auths)
if errPick != nil || picked == nil || picked.ID != "b" {
t.Fatalf("LCP %s: pick=%v err=%v", source, picked, errPick)
}
if source == "fork" && opts.Metadata[cliproxyexecutor.IsForkMetadataKey] != true {
t.Fatal("expected LCP fork")
}
if source == "compaction" && opts.Metadata[cliproxyexecutor.IsCompactionMetadataKey] != true {
t.Fatal("expected LCP compaction")
}
state := weightedStateForTest(t, weighted, "openai:model", "a", "b")
if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 {
t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
})
}
}
func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
key := "codex:model"
for range 1000 {
_, reservation := weighted.reserve(key, auths, "b")
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, key, "a", "b")
_, rejected := weighted.reserve(key, auths, "b")
_, accepted := weighted.reserve(key, auths, "a")
accepted.finish(true)
rejected.finish(false)
rejected.finish(true)
accepted.finish(false)
limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3
if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 {
t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
key := "codex:model"
_, old := weighted.reserve(key, auths, "")
auths[0].Attributes[AttributeWeight] = "1"
_, latest := weighted.reserve(key, auths, "")
old.finish(true)
state := weightedStateForTest(t, weighted, key, "a", "b")
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 {
t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
latest.finish(true)
if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
const requests = 40
selected := make(chan *Auth, requests)
reservations := make(chan *weightedReservation, requests)
var ready sync.WaitGroup
ready.Add(requests)
for range requests {
go func() {
defer ready.Done()
picked, reservation := weighted.reserve("codex:model", auths, "")
selected <- picked
reservations <- reservation
}()
}
ready.Wait()
close(selected)
close(reservations)
counts := map[string]int{}
for auth := range selected {
if auth == nil {
t.Fatal("concurrent reservation returned no auth")
}
counts[auth.ID]++
}
if counts["a"] != 30 || counts["b"] != 10 {
t.Fatalf("pending allocations ignored: %v", counts)
}
for reservation := range reservations {
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
func TestWeightedSessionAffinityFailureDoesNotCharge(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{
Fallback: weighted, RebindOn: SessionAffinityRebindOnNever,
})
t.Cleanup(selector.Stop)
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound"}}}
id := CanonicalSessionID(opts.Headers, nil, nil)
selector.cache.Set("codex::"+id+"::model", "missing")
if auth, errPick := selector.Pick(t.Context(), "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); errPick == nil || auth != nil {
t.Fatalf("expected unavailable binding: auth=%v err=%v", auth, errPick)
}
ctx, cancel := context.WithCancel(t.Context())
cancel()
if auth, errPick := selector.Pick(ctx, "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); !errors.Is(errPick, context.Canceled) || auth != nil {
t.Fatalf("expected cancellation: auth=%v err=%v", auth, errPick)
}
if len(weighted.states) != 0 {
t.Fatalf("failed selection changed weighted state: %v", weighted.states)
}
}
func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{
{ID: "a", Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
{ID: "b", Attributes: map[string]string{"priority": "1"}},
{ID: "lower", Attributes: map[string]string{AttributeWeight: "100"}},
{ID: "zero", Attributes: map[string]string{AttributeWeight: "0", "priority": "2"}},
{ID: "cooldown", Attributes: map[string]string{"priority": "2"}, ModelStates: map[string]*ModelState{
"upstream": {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
}},
}
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"alias"}}}
id := CanonicalSessionID(opts.Headers, nil, nil)
selector.cache.Set("codex::"+id+"::upstream", "b")
ctx := withWeightedSelectorStateModel(t.Context(), weighted, "requested(high)")
picked, errPick := selector.Pick(ctx, "codex", "upstream", opts, auths)
if errPick != nil || picked == nil || picked.ID != "b" {
t.Fatalf("sticky pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:requested", "a", "b")
if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 {
t.Fatalf("wrong model key or eligible pool: %v", weighted.states)
}
picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2])
if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest())
}
for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} {
if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil {
t.Fatal(errOther)
}
}
if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
t.Fatal("another provider or model changed credits")
}
}
func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"reselect"}}}
id := CanonicalSessionID(opts.Headers, nil, nil)
key := "codex::" + id + "::model"
selector.cache.Set(key, "removed")
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts,
[]*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}})
if errPick != nil || picked == nil || picked.ID != "a" {
t.Fatalf("reselection: pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
t.Fatalf("reselection did not bind: %q, found=%t", binding, ok)
}
}
func TestManagerWeightedSessionAffinityCompensation(t *testing.T) {
for _, mixed := range []bool{false, true} {
t.Run(fmt.Sprintf("mixed=%t", mixed), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
manager := NewManager(nil, selector, nil)
provider, model := "weighted-affinity", "weighted-model"
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
aID, bID := t.Name()+"-a", t.Name()+"-b"
for _, auth := range []*Auth{
{ID: aID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "3"}},
{ID: bID, Provider: provider, Status: StatusActive},
} {
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
t.Fatal(errRegister)
}
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
}
namespace := provider
if mixed {
namespace = "mixed"
}
parentHeaders := http.Header{"Session_id": {"manager-sticky"}}
id := CanonicalSessionID(parentHeaders, nil, nil)
selector.cache.Set(namespace+"::"+id+"::"+model, bID)
for i := range 16 {
opts := cliproxyexecutor.Options{Headers: parentHeaders}
want := bID
if i >= 4 {
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
want = aID
}
var auth *Auth
var errPick error
if mixed {
auth, _, _, errPick = manager.pickNextMixed(t.Context(), []string{provider}, model, opts, nil)
} else {
auth, _, errPick = manager.pickNext(t.Context(), provider, model, opts, nil)
}
if errPick != nil || auth == nil || auth.ID != want {
t.Fatalf("request %d: auth=%v err=%v, want %q", i, auth, errPick, want)
}
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 {
t.Fatalf("manager credits did not balance: %#v", state)
}
})
}
}

View File

@@ -0,0 +1,199 @@
package auth
import (
"fmt"
"maps"
"testing"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) {
for _, rollbackFirst := range []bool{false, true} {
t.Run(fmt.Sprintf("rollback-first=%t", rollbackFirst), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a"}, {ID: "b"}}
reservations := map[string][]*weightedReservation{}
t.Cleanup(func() {
for _, group := range reservations {
for _, reservation := range group {
reservation.finish(false)
}
}
})
for range 256 {
picked, reservation := weighted.reserve("codex:model", auths, "")
if picked == nil || reservation == nil {
t.Fatal("missing cold reservation")
}
reservations[picked.ID] = append(reservations[picked.ID], reservation)
}
ids := []string{"a", "b"}
if rollbackFirst {
ids = []string{"b", "a"}
}
for _, id := range ids {
for _, reservation := range reservations[id] {
reservation.finish(id == "a")
// Repeated finishing must not decrement the counter again.
reservation.finish(false)
}
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
limit := maxSmoothWeightedCreditRounds * 2
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) {
t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight)
}
})
}
}
func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) {
for _, first := range []string{"a", "b"} {
t.Run(first, func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
reservations := map[string][]*weightedReservation{}
t.Cleanup(func() {
for _, group := range reservations {
for _, reservation := range group {
reservation.finish(false)
}
}
})
for range 512 {
picked, reservation := weighted.reserve("codex:model", auths, "")
if picked == nil || reservation == nil {
t.Fatal("missing cold reservation")
}
reservations[picked.ID] = append(reservations[picked.ID], reservation)
}
if len(reservations["a"]) != 384 || len(reservations["b"]) != 128 {
t.Fatalf("reservations did not follow 3:1: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
}
last := "b"
if first == "b" {
last = "a"
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
for _, reservation := range reservations[first] {
reservation.finish(true)
}
// New arrivals must see the remaining offsets before the batch settles.
for _, phase := range []string{"pending", "settled"} {
if phase == "settled" {
for _, reservation := range reservations[last] {
reservation.finish(true)
}
}
counts := map[string]int{}
for range 64 {
picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
if errPick != nil || picked == nil {
t.Fatalf("Pick failed: auth=%v err=%v", picked, errPick)
}
counts[picked.ID]++
}
if counts["a"] != 48 || counts["b"] != 16 {
t.Fatalf("%s distribution = %v, want 48:16", phase, counts)
}
}
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) {
t.Fatalf("settled 3:1 batch drifted: %#v", state)
}
})
}
}
func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a"}, {ID: "b"}}
var old []*weightedReservation
t.Cleanup(func() {
for _, reservation := range old {
reservation.finish(false)
}
})
for range 256 {
_, reservation := weighted.reserve("codex:model", auths, "")
old = append(old, reservation)
}
auths[0].Attributes = map[string]string{AttributeWeight: "3"}
_, latest := weighted.reserve("codex:model", auths, "")
if latest == nil {
t.Fatal("missing new-generation reservation")
}
t.Cleanup(func() { latest.finish(false) })
for _, reservation := range old {
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatalf("old generation altered new accounting: %#v", state)
}
latest.finish(true)
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatalf("new generation failed to settle: %#v", state)
}
}
func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) {
for _, bound := range []bool{false, true} {
for _, first := range []string{"a", "b"} {
t.Run(fmt.Sprintf("bound=%t/first=%s", bound, first), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a"}, {ID: "b"}}
reservations := map[string][]*weightedReservation{}
t.Cleanup(func() {
for _, group := range reservations {
for _, reservation := range group {
reservation.finish(false)
}
}
})
for i := range 256 {
boundID := ""
if bound {
boundID = auths[i%2].ID
}
picked, reservation := weighted.reserve("codex:model", auths, boundID)
if picked == nil || reservation == nil {
t.Fatalf("reserve %d returned pick=%v reservation=%v", i, picked, reservation)
}
reservations[picked.ID] = append(reservations[picked.ID], reservation)
}
if len(reservations["a"]) != 128 || len(reservations["b"]) != 128 {
t.Fatalf("unbalanced reservations: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
// A zero net pending vector does not mean all reservations settled.
if state.inFlight != 256 || len(state.pendingForTest()) != 0 {
t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest())
}
last := "b"
if first == "b" {
last = "a"
}
for _, id := range []string{first, last} {
for _, reservation := range reservations[id] {
reservation.finish(true)
}
}
if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 {
t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
counts := map[string]int{}
for range 64 {
picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
if errPick != nil || picked == nil {
t.Fatalf("Pick: auth=%v err=%v", picked, errPick)
}
counts[picked.ID]++
}
if counts["a"] != 32 || counts["b"] != 32 {
t.Fatalf("post-commit distribution = %v, want a=32 b=32", counts)
}
})
}
}
}

View File

@@ -0,0 +1,71 @@
package auth
import (
"fmt"
"runtime"
"testing"
)
func BenchmarkWeightedPoolReserve1000(b *testing.B) {
for _, bound := range []bool{false, true} {
b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) {
auths := make([]*Auth, 1000)
for i := range auths {
auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
}
selector := &WeightedRoundRobinSelector{}
_, warmup := selector.reserve("codex:model", auths, "")
warmup.finish(true)
b.ReportAllocs()
b.ResetTimer()
for i := range b.N {
boundID := ""
if bound {
boundID = auths[i%len(auths)].ID
}
_, reservation := selector.reserve("codex:model", auths, boundID)
reservation.finish(true)
}
})
}
}
// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from
// total heap and allocation churn. Each retry subset omits a different credential.
func BenchmarkWeightedPoolRetainedMemory(b *testing.B) {
const members = 1000
auths := make([]*Auth, members)
for i := range auths {
auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
}
subset := make([]*Auth, 0, members-1)
var retained, heap, allocated int64
b.ResetTimer()
for range b.N {
selector := &WeightedRoundRobinSelector{}
_, warmup := selector.reserve("codex:model", auths, "")
warmup.finish(true)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
for excluded := range auths {
subset = append(subset[:0], auths[:excluded]...)
subset = append(subset, auths[excluded+1:]...)
_, reservation := selector.reserve("codex:model", subset, "")
reservation.finish(true)
}
runtime.GC()
runtime.ReadMemStats(&after)
if len(selector.states) != members+1 {
b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1)
}
runtime.KeepAlive(selector)
runtime.KeepAlive(auths)
retained += int64(after.HeapAlloc) - int64(before.HeapAlloc)
heap += int64(after.HeapAlloc)
allocated += int64(after.TotalAlloc - before.TotalAlloc)
}
b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op")
b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op")
b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op")
}

View File

@@ -0,0 +1,39 @@
package auth
import (
"fmt"
"testing"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
func TestWeightedSelectorsShareEquivalentModelKeys(t *testing.T) {
for _, firstAffinity := range []bool{false, true} {
for _, secondAffinity := range []bool{false, true} {
for _, routeOverride := range []bool{false, true} {
t.Run(fmt.Sprintf("affinity=%t/%t/route=%t", firstAffinity, secondAffinity, routeOverride), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
affinity := NewSessionAffinitySelector(weighted)
t.Cleanup(affinity.Stop)
auths := []*Auth{{ID: "a"}, {ID: "b"}}
for i, useAffinity := range []bool{firstAffinity, secondAffinity} {
ctx := t.Context()
model := []string{" Model(high) ", "model(low)"}[i]
if routeOverride {
ctx = withWeightedSelectorStateModel(ctx, weighted, model)
model = []string{"upstream-a", "upstream-b"}[i]
}
var selector Selector = weighted
if useAffinity {
selector = affinity
}
picked, errPick := selector.Pick(ctx, "codex", model, cliproxyexecutor.Options{}, auths)
if errPick != nil || picked == nil || picked.ID != auths[i].ID {
t.Fatalf("request %d: pick=%v err=%v, want auth %s", i, picked, errPick, auths[i].ID)
}
}
})
}
}
}
}

View File

@@ -0,0 +1,455 @@
package auth
import (
"fmt"
"maps"
"net/http"
"strconv"
"sync"
"testing"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
func TestWeightedOverloadExactCompensation(t *testing.T) {
for _, weights := range [][]int{{1, 1}, {3, 1}, {1, 3}, {1, 5, 1}, {2, 3, 5}, {1, 2, 3, 4}} {
for boundIndex := range weights {
t.Run(fmt.Sprintf("weights=%v/bound=%d", weights, boundIndex), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := make([]*Auth, len(weights))
total := 0
for i, weight := range weights {
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
total += weight
}
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"sticky"}}}
key := "codex::" + CanonicalSessionID(opts.Headers, nil, nil) + "::model"
selector.cache.Set(key, auths[boundIndex].ID)
counts := map[string]int{}
for i := range total * 4 {
request := opts
sticky := i < weights[boundIndex]*4
if !sticky {
request = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
}
picked, errPick := selector.Pick(t.Context(), "codex", "model", request, auths)
if errPick != nil || picked == nil {
t.Fatalf("selection failed: pick=%v err=%v", picked, errPick)
}
if sticky && picked.ID != auths[boundIndex].ID {
t.Fatal("balancing moved an existing binding")
}
counts[picked.ID]++
}
for i, weight := range weights {
if counts[auths[i].ID] != weight*4 {
t.Errorf("total allocation = %v, want weight vector %v times 4", counts, weights)
}
}
for _, state := range weighted.states {
for id, credit := range state.currentForTest() {
if credit != 0 {
t.Errorf("exact compensation left %q credit=%d", id, credit)
}
}
if len(state.pendingForTest()) != 0 {
t.Errorf("pending credit leaked: %v", state.pendingForTest())
}
}
})
}
}
}
func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) {
for _, weights := range [][]int{{1, 1, 1}, {1, 3, 1}, {1, 5, 1}, {1, 1, 5}, {1, 9, 1}, {1, 1, 9}, {1, 5, 2, 1}} {
t.Run(fmt.Sprint(weights), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := make([]*Auth, len(weights))
for i, weight := range weights {
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
selector.cache.Set(key, auths[0].ID)
counts := map[string]int{}
for i := range 3000 {
picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
if errPick != nil || picked == nil || picked.ID != auths[0].ID {
t.Fatalf("binding changed: pick=%v err=%v", picked, errPick)
}
fresh := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
picked, errPick = selector.Pick(t.Context(), "codex", "model", fresh, auths)
if errPick != nil || picked == nil {
t.Fatalf("fresh selection failed: pick=%v err=%v", picked, errPick)
}
if i >= 1000 {
counts[picked.ID]++
}
}
ids := make([]string, len(auths))
for i, auth := range auths {
ids[i] = auth.ID
}
state := weightedStateForTest(t, weighted, "codex:model", ids...)
t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest())
// Forced traffic makes global ratios infeasible; the remaining capacity
// should be shared by unbound peers rather than monopolized by one.
peerWeight := 0
for _, weight := range weights[1:] {
peerWeight += weight
}
for i, auth := range auths[1:] {
want := 2000 * weights[i+1] / peerWeight
if counts[auth.ID] < want-2 || counts[auth.ID] > want+2 {
t.Errorf("peer %q got %d fresh requests, want approximately %d; distribution=%v", auth.ID, counts[auth.ID], want, counts)
}
}
})
}
}
func TestWeightedOverloadFeasibleSustainedTraffic(t *testing.T) {
for _, weights := range [][]int{{3, 1}, {1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
t.Run(fmt.Sprint(weights), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := make([]*Auth, len(weights))
total := 0
for i, weight := range weights {
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
total += weight
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"feasible"}}}
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
selector.cache.Set(key, auths[0].ID)
counts := map[string]int{}
// Forced traffic is below A's target share, so exact aggregate ratios
// remain feasible without moving established bindings.
for i := range total * 2000 {
opts := cliproxyexecutor.Options{}
bound := i%(total*2) == 0
if bound {
opts = sticky
}
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
if errPick != nil || picked == nil || (bound && picked.ID != auths[0].ID) {
t.Fatalf("request %d: pick=%v err=%v", i, picked, errPick)
}
if i >= total*1000 {
counts[picked.ID]++
}
}
for i, weight := range weights {
if counts[auths[i].ID] != weight*1000 {
t.Errorf("feasible distribution drifted: counts=%v weights=%v", counts, weights)
}
}
t.Logf("feasible distribution: %v", counts)
})
}
}
func TestWeightedOverloadRecoveryAfterStickyTrafficStops(t *testing.T) {
for _, weights := range [][]int{{1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
t.Run(fmt.Sprint(weights), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := make([]*Auth, len(weights))
total := 0
for i, weight := range weights {
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
total += weight
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
selector.cache.Set(key, auths[0].ID)
for range 3000 {
if picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths); errPick != nil || picked == nil || picked.ID != auths[0].ID {
t.Fatalf("sticky selection failed: pick=%v err=%v", picked, errPick)
}
if _, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
t.Fatal(errPick)
}
}
counts := map[string]int{}
for i := range 1000 + total*100 {
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
if errPick != nil || picked == nil {
t.Fatalf("recovery selection failed: pick=%v err=%v", picked, errPick)
}
if i >= 1000 {
counts[picked.ID]++
}
}
for i, weight := range weights {
if counts[auths[i].ID] != weight*100 {
t.Errorf("allocation did not recover after sticky traffic stopped: counts=%v weights=%v", counts, weights)
}
}
t.Logf("recovered distribution: %v", counts)
})
}
}
func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}}
const key = "codex:model"
const inFlight = 200
reservations := make([]*weightedReservation, inFlight)
t.Cleanup(func() {
for _, reservation := range reservations {
reservation.finish(false)
}
})
for i := range reservations {
picked, reservation := weighted.reserve(key, auths, "a")
if picked == nil || picked.ID != "a" {
t.Fatal("pending sticky request lost its binding")
}
reservations[i] = reservation
}
counts := map[string]int{}
for i := range 3000 {
// Maintain a fixed backlog without sleeps or timing-dependent coordination.
reservations[i%inFlight].finish(true)
_, reservations[i%inFlight] = weighted.reserve(key, auths, "a")
picked, reservation := weighted.reserve(key, auths, "")
reservation.finish(true)
if i >= 1000 {
counts[picked.ID]++
}
}
if counts["c"] == 0 {
t.Fatalf("speculative sticky traffic starved c: %v", counts)
}
for _, reservation := range reservations {
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, key, "a", "b", "c")
if len(state.pendingForTest()) != 0 {
t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest())
}
}
func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
const key = "codex:model"
for range 1000 {
_, reservation := weighted.reserve(key, auths, "b")
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, key, "a", "b")
before := maps.Clone(state.currentForTest())
_, first := weighted.reserve(key, auths, "")
_, second := weighted.reserve(key, auths, "")
t.Cleanup(func() { first.finish(false); second.finish(false) })
// Two pending cold selections make room for a bound delta, but neither
// repayment is real yet. Its later commit must recheck the committed cap.
picked, bound := weighted.reserve(key, auths, "b")
if picked == nil || picked.ID != "b" || bound == nil {
t.Fatal("expected bound reservation funded by pending repayments")
}
t.Cleanup(func() { bound.finish(false) })
first.finish(false)
second.finish(false)
bound.finish(true)
bound.finish(true)
bound.finish(false)
if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest())
}
// An over-limit bound selection succeeds without reserving any credit.
picked, skipped := weighted.reserve(key, auths, "b")
if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
t.Fatal("saturated bound selection should preserve binding and all credits")
}
}
func TestWeightedOverloadMultipleBusyBindings(t *testing.T) {
for _, busyPerFresh := range []int{1, 3, 10} {
t.Run(fmt.Sprint(busyPerFresh), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{
{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}, {ID: "d"},
}
boundOpts := make([]cliproxyexecutor.Options, 2)
for i, id := range []string{"a", "d"} {
boundOpts[i].Headers = http.Header{"Session_id": {"busy-" + id}}
sessionID := CanonicalSessionID(boundOpts[i].Headers, nil, nil)
selector.cache.Set("codex::"+sessionID+"::model", id)
}
windowCounts := map[string]int{}
for i := range 1400 {
for j := range busyPerFresh {
index := (i + j) % len(boundOpts)
picked, errPick := selector.Pick(t.Context(), "codex", "model", boundOpts[index], auths)
if errPick != nil || picked == nil || picked.ID != []string{"a", "d"}[index] {
t.Fatalf("busy binding changed: auth=%v err=%v", picked, errPick)
}
}
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
if errPick != nil || picked == nil {
t.Fatalf("fresh pick failed: auth=%v err=%v", picked, errPick)
}
if i >= 1000 {
windowCounts[picked.ID]++
if (i+1)%100 == 0 {
if windowCounts["b"] == 0 || windowCounts["c"] == 0 {
t.Fatalf("busy=%d unbound peer starved in a 100-request window: %v", busyPerFresh, windowCounts)
}
clear(windowCounts)
}
}
}
})
}
}
func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) {
for _, credit := range []map[string]int64{
{"a": -222, "b": 111, "c": 111},
{"a": -112, "b": 222, "c": -110},
} {
state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit)
state.charge(0, true)
if !maps.Equal(state.currentForTest(), credit) {
t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit)
}
}
}
func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) {
for _, mixed := range []bool{false, true} {
for _, pinned := range []bool{false, true} {
t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
manager := NewManager(nil, selector, nil)
const provider, model = "sticky-overload", "sticky-overload-model"
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
aID, bID, cID := t.Name()+"-a", t.Name()+"-b", t.Name()+"-c"
for _, auth := range []*Auth{
{ID: aID, Provider: provider, Status: StatusActive},
{ID: bID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "5"}},
{ID: cID, Provider: provider, Status: StatusActive},
} {
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
t.Fatal(errRegister)
}
}
namespace := provider
if mixed {
namespace = "mixed"
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}, Metadata: map[string]any{}}
key := namespace + "::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::" + model
selector.cache.Set(key, aID)
if pinned {
sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = aID
}
counts := map[string]int{}
for i := range 1600 {
for _, bound := range []bool{true, false} {
opts := sticky
if !bound {
opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}}
}
var auth *Auth
var errPick error
if mixed {
auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
} else {
auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
}
if errPick != nil || auth == nil || (bound && auth.ID != aID) {
t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
}
if !bound && i >= 1000 {
counts[auth.ID]++
}
}
}
if counts[bID] != 500 || counts[cID] != 100 {
t.Fatalf("manager overload allocation = %v, want b=500 c=100", counts)
}
if binding, ok := selector.cache.Get(key); !ok || binding != aID {
t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok)
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID)
if len(state.pendingForTest()) != 0 {
t.Fatalf("manager left speculative credit: %v", state.pendingForTest())
}
})
}
}
}
func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
headers := http.Header{"Session_id": {"sticky-concurrent"}}
key := "codex::" + CanonicalSessionID(headers, nil, nil) + "::model"
selector.cache.Set(key, "b")
type selection struct {
id string
err error
sticky bool
}
results := make(chan selection, 64)
start := make(chan struct{})
var done sync.WaitGroup
for i := range 64 {
done.Add(1)
go func() {
defer done.Done()
isSticky := i < 8
opts := cliproxyexecutor.Options{Headers: headers}
if !isSticky {
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
}
<-start
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
id := ""
if picked != nil {
id = picked.ID
}
results <- selection{id: id, err: errPick, sticky: isSticky}
}()
}
close(start)
done.Wait()
close(results)
counts := map[string]int{}
for result := range results {
if result.err != nil || result.id == "" || (result.sticky && result.id != "b") {
t.Fatalf("invalid concurrent result: %+v", result)
}
counts[result.id]++
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
// All 64 calls share one pool. Accounting must match actual accepted picks,
// regardless of whether the final arrivals happened to be sticky requests.
if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 {
t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest())
}
if bound, ok := selector.cache.Get(key); !ok || bound != "b" {
t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok)
}
}

View File

@@ -0,0 +1,287 @@
package auth
import (
"fmt"
"maps"
"math/rand/v2"
"net/http"
"strconv"
"testing"
"time"
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
)
// weightedStateForTest checks that the requested accounting pool was created.
func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState {
t.Helper()
weights := make(map[string]int64, len(ids))
for _, id := range ids {
weights[id] = 1
}
state := selector.states[weightedPoolStateKey(route, weights)]
ids = sortedWeightedIDs(weights)
for state != nil && !state.matches(route, ids) {
state = state.next
}
if state == nil {
t.Fatalf("missing weighted state for %q and %v", route, ids)
}
return state
}
func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}, {ID: "c"}}
key := "codex:model"
_, pendingFull := weighted.reserve(key, auths, "b")
t.Cleanup(func() { pendingFull.finish(false) })
_, pendingSubset := weighted.reserve(key, auths[:2], "b")
t.Cleanup(func() { pendingSubset.finish(false) })
pendingSubset.finish(true)
subset := weightedStateForTest(t, weighted, key, "a", "b")
full := weightedStateForTest(t, weighted, key, "a", "b", "c")
if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() {
t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest())
}
if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) {
t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest())
}
pendingFull.finish(false)
for range 100 {
_, reservation := weighted.reserve(key, auths, "b")
reservation.finish(true)
}
// A returning pool resumes its debt, independent of full-pool traffic.
for range 3 {
picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "")
if picked == nil || picked.ID != "a" {
t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest())
}
reservation.finish(true)
}
if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 {
t.Fatalf("returning subset failed to balance: %#v", subset)
}
}
func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) {
shared := &WeightedRoundRobinSelector{}
auths := []*Auth{
{ID: "a", Attributes: map[string]string{AttributeWeight: "1"}},
{ID: "b", Attributes: map[string]string{AttributeWeight: "3"}},
{ID: "c", Attributes: map[string]string{AttributeWeight: "2"}},
{ID: "d", Attributes: map[string]string{AttributeWeight: "5"}},
}
var independent [16]WeightedRoundRobinSelector
rng := rand.New(rand.NewPCG(17, 29))
for step := range 5000 {
mask := 1 + rng.IntN(15)
var pool []*Auth
for i, auth := range auths {
if mask&(1<<i) != 0 {
pool = append(pool, auth)
}
}
// Membership and weights, not input order or other pools, define credit.
rng.Shuffle(len(pool), func(i, j int) { pool[i], pool[j] = pool[j], pool[i] })
boundID := ""
if rng.IntN(3) == 0 {
boundID = pool[rng.IntN(len(pool))].ID
}
picked, reservation := shared.reserve("route", pool, boundID)
want, independentReservation := independent[mask].reserve("route", pool, boundID)
if picked == nil || want == nil || picked.ID != want.ID {
t.Fatalf("step %d pool %d: shared=%v independent=%v", step, mask, picked, want)
}
committed := rng.IntN(4) != 0
reservation.finish(committed)
independentReservation.finish(committed)
poolKey := weightedPoolStateKey("route", authWeightVector(pool))
state, independentState := shared.states[poolKey], independent[mask].states[poolKey]
if !maps.Equal(state.currentForTest(), independentState.currentForTest()) || len(state.pendingForTest()) != 0 {
t.Fatalf("step %d pool %d: credits diverged or pending leaked", step, mask)
}
}
}
func TestWeightedPoolKeysAndCapacity(t *testing.T) {
seen := make(map[[32]byte]struct{})
for _, tc := range []struct {
route string
ids []string
}{
{route: "route", ids: []string{"a", "bc"}},
{route: "route", ids: []string{"ab", "c"}},
{route: "route1:a", ids: []string{"bc"}},
{route: "route", ids: []string{"a1:b", "c"}},
} {
t.Run(fmt.Sprint(tc), func(t *testing.T) {
weights := make(map[string]int64)
for _, id := range tc.ids {
weights[id] = 1
}
key := weightedPoolStateKey(tc.route, weights)
if _, duplicate := seen[key]; duplicate {
t.Fatal("different pool identities collided")
}
seen[key] = struct{}{}
for _, id := range tc.ids {
weights[id] = 3
}
if key != weightedPoolStateKey(tc.route, weights) {
t.Fatal("weight change changed pool identity instead of resetting credit")
}
})
}
weighted := &WeightedRoundRobinSelector{maxKeys: 2}
auths := []*Auth{{ID: "a"}, {ID: "b"}}
_, old := weighted.reserve("route", auths, "b")
t.Cleanup(func() { old.finish(false) })
for _, auth := range auths {
_, reservation := weighted.reserve("route", []*Auth{auth}, auth.ID)
reservation.finish(true)
}
_, current := weighted.reserve("route", auths, "a")
t.Cleanup(func() { current.finish(false) })
if len(weighted.states) > weighted.maxKeys || old.state == current.state {
t.Fatal("pool eviction failed to bound or replace state")
}
old.finish(true)
if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatal("finishing an evicted reservation affected the new pool")
}
current.finish(true)
}
func TestManagerWeightedPriorityPoolIsolation(t *testing.T) {
for _, mixed := range []bool{false, true} {
for _, pinned := range []bool{false, true} {
t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
manager := NewManager(nil, selector, nil)
const provider, model = "pool-isolation", "pool-model"
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
auths := []*Auth{
{ID: t.Name() + "-a", Attributes: map[string]string{"priority": "0"}},
{ID: t.Name() + "-b", Attributes: map[string]string{"priority": "1", AttributeWeight: "3"}},
{ID: t.Name() + "-c", Attributes: map[string]string{"priority": "1", AttributeWeight: "1"}},
}
for _, auth := range auths {
auth.Provider, auth.Status = provider, StatusActive
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
t.Fatal(errRegister)
}
}
namespace := provider
if mixed {
namespace = "mixed"
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"priority-bound"}}, Metadata: map[string]any{}}
id := CanonicalSessionID(sticky.Headers, nil, nil)
selector.cache.Set(namespace+"::"+id+"::"+model, auths[0].ID)
if pinned {
sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = auths[0].ID
}
counts := map[string]int{}
for i := range 1200 {
for _, bound := range []bool{true, false} {
opts := sticky
if !bound {
opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
}
var auth *Auth
var errPick error
if mixed {
auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
} else {
auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
}
if errPick != nil || auth == nil || (bound && auth.ID != auths[0].ID) || (!bound && auth.ID == auths[0].ID) {
t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
}
if !bound && i >= 200 {
counts[auth.ID]++
}
}
}
if counts[auths[1].ID] != 750 || counts[auths[2].ID] != 250 {
t.Fatalf("manager cross-priority distribution = %v, want 750:250", counts)
}
})
}
}
}
func TestWeightedSessionAffinityPoolIsolation(t *testing.T) {
for _, reason := range []string{"priority", "websocket", "retry-exclusion", "cooldown"} {
for _, weights := range [][2]int{{1, 1}, {3, 1}, {1, 3}, {5, 2}} {
t.Run(fmt.Sprintf("%s/%d:%d", reason, weights[0], weights[1]), func(t *testing.T) {
weighted := &WeightedRoundRobinSelector{}
selector := NewSessionAffinitySelector(weighted)
t.Cleanup(selector.Stop)
auths := []*Auth{
{ID: "a", Attributes: map[string]string{"priority": "0"}},
{ID: "b", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[0])}},
{ID: "c", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[1])}},
}
ctx := t.Context()
switch reason {
case "priority":
auths[1].Attributes["priority"] = "1"
auths[2].Attributes["priority"] = "1"
case "websocket":
ctx = cliproxyexecutor.WithDownstreamWebsocket(ctx)
auths[1].Attributes["websockets"] = "true"
auths[2].Attributes["websockets"] = "true"
}
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound-a"}}}
id := CanonicalSessionID(sticky.Headers, nil, nil)
key := "codex::" + id + "::model"
selector.cache.Set(key, "a")
const warmup, measured = 1000, 1400
counts := map[string]int{}
for i := range warmup + measured {
picked, errPick := selector.Pick(ctx, "codex", "model", sticky, auths)
if errPick != nil || picked == nil || picked.ID != "a" {
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
}
freshAuths := auths
if reason == "retry-exclusion" {
freshAuths = auths[1:]
} else if reason == "cooldown" {
auths[0].Unavailable = true
auths[0].NextRetryAfter = time.Now().Add(time.Hour)
}
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
picked, errPick = selector.Pick(ctx, "codex", "model", opts, freshAuths)
if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
}
if i >= warmup {
counts[picked.ID]++
}
auths[0].Unavailable = false
auths[0].NextRetryAfter = time.Time{}
}
wantB := measured * weights[0] / (weights[0] + weights[1])
if counts["b"] != wantB || counts["c"] != measured-wantB {
t.Fatalf("cross-pool traffic distorted cold selections: %v, want b=%d c=%d", counts, wantB, measured-wantB)
}
if bound, ok := selector.cache.Get(key); !ok || bound != "a" {
t.Fatalf("binding moved: %q, found=%t", bound, ok)
}
for _, state := range weighted.states {
if len(state.pendingForTest()) != 0 {
t.Fatalf("pending credits leaked: %v", state.pendingForTest())
}
}
})
}
}
}

View File

@@ -0,0 +1,88 @@
package auth
import (
"maps"
"testing"
)
// These snapshots keep assertions independent of the compact production layout.
func (s *weightedPoolState) currentForTest() map[string]int64 {
current := make(map[string]int64, len(s.members))
for _, member := range s.members {
current[member.id] = member.current
}
return current
}
func (s *weightedPoolState) pendingForTest() map[string]int64 {
pending := make(map[string]int64)
for _, member := range s.members {
if member.pending != 0 {
pending[member.id] = member.pending
}
}
return pending
}
func (s *weightedPoolState) hasCurrentForTest() bool {
for _, member := range s.members {
if member.current != 0 {
return true
}
}
return false
}
// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact
// identities, including a matching state behind a collision-chain head.
func TestWeightedPoolDigestCollisionIsolation(t *testing.T) {
for _, unrelated := range []*weightedPoolState{
weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}),
weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}),
weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}),
} {
t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) {
weights := map[string]int64{"a": 1, "b": 1}
key := weightedPoolStateKey("route", weights)
selector := &WeightedRoundRobinSelector{
states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2,
}
before := unrelated.currentForTest()
picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b")
if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated {
t.Fatal("digest collision reused an unrelated pool")
}
t.Cleanup(func() { first.finish(false) })
matching := first.state
// Put the nonmatching state first to exercise a full chain lookup.
matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated
_, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a")
if second == nil || second.state != matching || selector.stateCount != 2 {
t.Fatal("collision lookup failed to reuse the matching pool")
}
t.Cleanup(func() { second.finish(false) })
first.finish(true)
second.finish(true)
if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 {
t.Fatal("collision accounting changed the unrelated pool")
}
if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 {
t.Fatal("matching pool did not balance")
}
// Capacity still counts pools, not just distinct digest buckets.
_, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "")
third.finish(true)
if selector.stateCount != 1 || len(selector.states) != 1 {
t.Fatal("colliding pools bypassed the existing capacity limit")
}
})
}
}
func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState {
state := &weightedPoolState{route: route}
for _, id := range sortedWeightedIDs(weights) {
state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]})
}
return state
}

View File

@@ -0,0 +1,79 @@
package auth
import (
"maps"
"sort"
"testing"
)
func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
for _, tc := range []struct {
name string
current map[string]int64
weights map[string]int64
authID string
want map[string]int64
}{
{
name: "smaller pool preserves leading difference",
current: map[string]int64{"a": -96, "b": 96, "c": 95},
weights: map[string]int64{"b": 1, "c": 1},
authID: "b",
want: map[string]int64{"b": 63, "c": 64},
},
{
name: "negative credits shift together",
current: map[string]int64{"a": 96, "b": -64, "c": -63},
weights: map[string]int64{"b": 1, "c": 1},
authID: "b",
want: map[string]int64{"b": -64, "c": -61},
},
{
name: "excessive spread clips debt without tying leaders",
current: map[string]int64{"a": -96, "b": 96, "c": 95},
weights: map[string]int64{"a": 1, "b": 1, "c": 1},
authID: "a",
want: map[string]int64{"a": -96, "b": 96, "c": 95},
},
{
name: "bounded credits do not shift",
current: map[string]int64{"b": 0, "c": 0},
weights: map[string]int64{"b": 1, "c": 1},
authID: "b",
want: map[string]int64{"b": -1, "c": 1},
},
} {
t.Run(tc.name, func(t *testing.T) {
state := weightedPoolForTest("route", tc.weights, tc.current)
state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false)
if !maps.Equal(state.currentForTest(), tc.want) {
t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want)
}
})
}
}
func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) {
key := "codex:model"
weights := map[string]int64{"a": 1, "b": 1, "c": 1}
state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93})
weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1}
auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}}
_, rejected := weighted.reserve(key, auths, "a")
t.Cleanup(func() { rejected.finish(false) })
_, accepted := weighted.reserve(key, auths, "b")
t.Cleanup(func() { accepted.finish(false) })
accepted.finish(true)
want := map[string]int64{"a": -93, "b": 92, "c": 94}
if !maps.Equal(state.currentForTest(), want) {
t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want)
}
if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) {
t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest())
}
rejected.finish(false)
rejected.finish(true)
if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 {
t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}