From 0561334f06c5c2bf4f1ba85566a551d39bbff138 Mon Sep 17 00:00:00 2001
From: hkfires <10558748+hkfires@users.noreply.github.com>
Date: Mon, 9 Nov 2026 10:21:08 +0800
Subject: [PATCH 1/3] fix(home): preserve fork kind in home dispatch
Map IsFork metadata to X-Node-Kind: fork when dispatching to
Home so forked sessions keep their lineage kind alongside
explicit kinds such as compaction. Clone dispatch headers
before adding the Home-only header to avoid mutating caller
headers, and cover fork, compaction, and ordinary children
with session hierarchy tests.
---
sdk/cliproxy/auth/conductor_home.go | 10 +++-
sdk/cliproxy/auth/home_session_kind_test.go | 59 +++++++++++++++++++++
2 files changed, 67 insertions(+), 2 deletions(-)
create mode 100644 sdk/cliproxy/auth/home_session_kind_test.go
diff --git a/sdk/cliproxy/auth/conductor_home.go b/sdk/cliproxy/auth/conductor_home.go
index 44603aeaf..26e85115f 100644
--- a/sdk/cliproxy/auth/conductor_home.go
+++ b/sdk/cliproxy/auth/conductor_home.go
@@ -1084,11 +1084,17 @@ func (m *Manager) pickHomeDispatchSelection(ctx context.Context, model string, o
}
dispatchHeaders := homeDispatchHeaders(ctx, opts.Headers)
if opts.Metadata != nil {
- if nodeKind, ok := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string); ok && strings.TrimSpace(nodeKind) != "" {
+ nodeKind, _ := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string)
+ if isFork, _ := opts.Metadata[cliproxyexecutor.IsForkMetadataKey].(bool); isFork {
+ nodeKind = "fork"
+ }
+ if nodeKind = strings.TrimSpace(nodeKind); nodeKind != "" {
+ // Keep the caller headers unchanged when adding Home-only lineage metadata.
+ dispatchHeaders = dispatchHeaders.Clone()
if dispatchHeaders == nil {
dispatchHeaders = make(http.Header)
}
- dispatchHeaders.Set("X-Node-Kind", strings.TrimSpace(nodeKind))
+ dispatchHeaders.Set("X-Node-Kind", nodeKind)
}
}
credentialPolicy := credentialPolicyFromContext(ctx)
diff --git a/sdk/cliproxy/auth/home_session_kind_test.go b/sdk/cliproxy/auth/home_session_kind_test.go
new file mode 100644
index 000000000..76203a8bf
--- /dev/null
+++ b/sdk/cliproxy/auth/home_session_kind_test.go
@@ -0,0 +1,59 @@
+package auth
+
+import (
+ "context"
+ "net/http"
+ "testing"
+
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+type sessionKindCaptureDispatcher struct {
+ fixtureHomeDispatcher
+ headers http.Header
+ sessionID string
+ parentSessionID string
+}
+
+func (d *sessionKindCaptureDispatcher) RPopAuthWithSessionHierarchy(_ context.Context, _ string, sessionID, parentSessionID string, headers http.Header, _ int, _ string, _ *int, _ []string, _ string) ([]byte, error) {
+ d.headers = headers.Clone()
+ d.sessionID = sessionID
+ d.parentSessionID = parentSessionID
+ return d.payload, nil
+}
+
+func TestHomeDispatchPreservesForkAndCompactionKind(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ body string
+ metadata map[string]any
+ want string
+ }{
+ {name: "body fork", body: `{"thread_id":"child","metadata":{"forked_from_thread_id":"parent"}}`, want: "fork"},
+ {name: "compaction", body: `{"session_id":"child","parent_session_id":"parent"}`, metadata: map[string]any{cliproxyexecutor.NodeKindMetadataKey: "compaction"}, want: "compaction"},
+ {name: "ordinary child", body: `{"session_id":"child","parent_session_id":"parent"}`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dispatcher := &sessionKindCaptureDispatcher{fixtureHomeDispatcher: fixtureHomeDispatcher{payload: []byte(`{"auth":{"id":"kind-auth","provider":"kind-provider"}}`)}}
+ manager := newHomeSelectionTestManager(t, dispatcher)
+ manager.RegisterExecutor(schedulerTestExecutor{provider: "kind-provider"})
+ headers := http.Header{"User-Agent": {"test-client"}}
+ opts := cliproxyexecutor.Options{Headers: headers, OriginalRequest: []byte(tc.body), Metadata: tc.metadata}
+ opts.EnsureMetadata()
+ selection, errSelection := manager.pickHomeDispatchSelection(t.Context(), "model", opts)
+ if errSelection != nil {
+ t.Fatal(errSelection)
+ }
+ selection.End("test_complete")
+ if dispatcher.sessionID == "" || dispatcher.parentSessionID == "" || dispatcher.sessionID == dispatcher.parentSessionID {
+ t.Fatalf("lost session hierarchy: child=%q parent=%q", dispatcher.sessionID, dispatcher.parentSessionID)
+ }
+ if got := dispatcher.headers.Get("X-Node-Kind"); got != tc.want {
+ t.Fatalf("node kind=%q, want %q", got, tc.want)
+ }
+ if got := headers.Get("X-Node-Kind"); got != "" {
+ t.Fatalf("caller headers mutated: %q", got)
+ }
+ })
+ }
+}
From d20451504f473d303a877a8f4edef195f988da4e Mon Sep 17 00:00:00 2001
From: hkfires <10558748+hkfires@users.noreply.github.com>
Date: Mon, 9 Nov 2026 12:43:14 +0800
Subject: [PATCH 2/3] feat(auth): compensate weighted routing for sticky
sessions
Charge session-affinity hits, parent inheritance, LCP matches,
and pinned requests against the eligible weighted pool without
moving existing bindings, so new sessions favor under-served
credentials and request ratios stay approximately weighted.
Scope settled credits per eligible credential set, capped at
+/-32 pool rounds, with in-flight reservations that commit on
local selection success and release on failure.
---
config.example.yaml | 10 +
docs/management-api-v8.md | 30 ++
sdk/cliproxy/auth/conductor_pinned_test.go | 96 ++++
sdk/cliproxy/auth/conductor_selection.go | 60 ++-
.../auth/conductor_weighted_pinned_test.go | 161 +++++++
sdk/cliproxy/auth/scheduler_benchmark_test.go | 38 ++
sdk/cliproxy/auth/selector.go | 93 ++--
sdk/cliproxy/auth/selector_test.go | 14 +-
sdk/cliproxy/auth/selector_weighted.go | 292 +++++++++++
.../auth/selector_weighted_affinity_test.go | 436 +++++++++++++++++
.../selector_weighted_commit_order_test.go | 199 ++++++++
.../auth/selector_weighted_model_key_test.go | 39 ++
.../auth/selector_weighted_overload_test.go | 455 ++++++++++++++++++
.../auth/selector_weighted_pool_test.go | 283 +++++++++++
sdk/cliproxy/auth/selector_weighted_test.go | 81 ++++
15 files changed, 2213 insertions(+), 74 deletions(-)
create mode 100644 sdk/cliproxy/auth/conductor_pinned_test.go
create mode 100644 sdk/cliproxy/auth/conductor_weighted_pinned_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_affinity_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_commit_order_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_model_key_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_overload_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_pool_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_test.go
diff --git a/config.example.yaml b/config.example.yaml
index a3dc24045..404c40573 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -170,6 +170,16 @@ routing:
strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first
# weighted-round-robin uses each credential's integer weight (default 1, maximum 1,000,000).
# Non-positive weights exclude the credential while this strategy is active.
+ # Sticky requests (including inherited and LCP bindings) also consume weighted credits
+ # without moving existing bindings. Compensation is scoped to each eligible credential set;
+ # priority tiers, transport preferences, and retry/cooldown subsets do not share credits.
+ # Pinned requests use the eligible pool before pinning, within the pinned credential
+ # priority tier and transport class. Returning to a pool resumes its retained credits.
+ # Settled credits are capped at +/-32 times the eligible pool total; ratios are approximate.
+ # In-flight offsets are preserved until the pool settles, regardless of commit order.
+ # Sticky requests keep their bindings but stop adding compensation when the full
+ # credit delta cannot fit the cap; new sessions keep earning credit to avoid starvation.
+ # Credits are process-local and reset on restart or selector replacement.
# For OAuth/file credentials, add a top-level numeric "weight" field to the auth JSON.
# Enable universal session-sticky routing for all clients.
# Explicit Claude Code, Codex, OpenCode, and pi session headers are preferred,
diff --git a/docs/management-api-v8.md b/docs/management-api-v8.md
index 790e70add..5aba973c3 100644
--- a/docs/management-api-v8.md
+++ b/docs/management-api-v8.md
@@ -64,6 +64,36 @@ The Home-owned revision fields
`credentials.concurrency.observation-barrier-revision`, and `plugins.auth-revision`
cannot be changed through these endpoints.
+### Weighted routing with session affinity
+
+Set `routing.strategy` to `weighted-round-robin` to use per-credential integer
+`weight` values (default `1`, maximum `1000000`; non-positive values are excluded).
+With `routing.session-affinity` enabled, sticky requests, parent inheritance,
+Guardian selections, and LCP matches also consume weighted credits without moving
+existing bindings. New or reselected sessions favor under-served credentials
+within the same eligible credential set. For example, with weights `3:1`, four
+sticky requests to the second credential can be compensated by twelve new
+requests to the first when both remain eligible.
+
+Different priority tiers, transport preferences, and retry/cooldown candidate
+sets keep separate credits; returning to a set resumes its retained credits.
+Pinned requests account against the eligible pool before pinning, within the
+pinned credential's priority tier and transport class. A lower-priority binding
+never shifts the request ratio among higher-priority credentials.
+
+Settled credits are capped at ±32 times the eligible pool total, so request
+ratios are approximate rather than guaranteed. While reservations are in flight,
+committed balances can temporarily exceed the cap; their pending offsets remain
+intact until the pool settles, including when the last reservation rolls back. Sticky requests still use their binding at
+the cap, but stop adding compensation when their full credit delta cannot fit.
+New sessions continue repaying debt and earning credit, so excessive sticky
+traffic does not starve smaller-weight peers.
+
+Credits are local to the CPA process and reset on restart, selector replacement,
+or bounded pool-cache eviction. Concurrent selections see pending credit; local
+selection/binding success confirms it, while failure releases it. In Home mode,
+scheduling and credit accounting remain owned by Home.
+
### Codex multi-agent configuration migration
`client.codex.optimize-multi-agent-v2` is the sole runtime setting. Loading older
diff --git a/sdk/cliproxy/auth/conductor_pinned_test.go b/sdk/cliproxy/auth/conductor_pinned_test.go
new file mode 100644
index 000000000..0f183bc6e
--- /dev/null
+++ b/sdk/cliproxy/auth/conductor_pinned_test.go
@@ -0,0 +1,96 @@
+package auth
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "testing"
+
+ "github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+func TestManagerNonWeightedPinnedSelection(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ for _, affinity := range []bool{false, true} {
+ for _, strategy := range []string{"round-robin", "fill-first"} {
+ t.Run(fmt.Sprintf("mixed=%t/affinity=%t/%s", mixed, affinity, strategy), func(t *testing.T) {
+ var selector Selector = &RoundRobinSelector{}
+ if strategy == "fill-first" {
+ selector = &FillFirstSelector{}
+ }
+ if affinity {
+ sticky := NewSessionAffinitySelector(selector)
+ t.Cleanup(sticky.Stop)
+ selector = sticky
+ }
+ manager := NewManager(nil, selector, nil)
+ const provider, model = "pinned-non-weighted", "pinned-model"
+ manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
+ pinnedID := t.Name() + "-pinned"
+ for i := range 32 {
+ auth := &Auth{
+ ID: fmt.Sprintf("%s-other-%02d", t.Name(), i), Provider: provider, Status: StatusActive,
+ Attributes: map[string]string{AttributeAuthKind: AuthKindAPIKey, "priority": "10"},
+ }
+ if i == 0 {
+ auth.ID = pinnedID
+ auth.Attributes["priority"] = "0"
+ }
+ registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
+ t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
+ if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
+ t.Fatal(errRegister)
+ }
+ }
+ // Normalize the legacy selection signatures for the same pinning checks.
+ pick := func(ctx context.Context, id string, tried map[string]struct{}) (*Auth, error) {
+ opts := cliproxyexecutor.Options{
+ Headers: http.Header{"Session_id": {"non-weighted-pinned"}},
+ Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: id},
+ }
+ if mixed {
+ auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider}, model, opts, tried)
+ return auth, errPick
+ }
+ auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
+ return auth, errPick
+ }
+ ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
+ for range 3 {
+ if auth, errPick := pick(ctx, pinnedID, nil); errPick != nil || auth == nil || auth.ID != pinnedID {
+ t.Fatalf("lower-priority pin not enforced: auth=%v err=%v", auth, errPick)
+ }
+ }
+ for _, tc := range []struct {
+ name string
+ ctx context.Context
+ id string
+ tried map[string]struct{}
+ }{
+ {name: "missing pin", ctx: ctx, id: "missing"},
+ {name: "already tried", ctx: ctx, id: pinnedID, tried: map[string]struct{}{pinnedID: {}}},
+ {name: "ineligible kind", ctx: withRequiredAuthKind(t.Context(), AuthKindOAuth), id: pinnedID},
+ } {
+ auth, errPick := pick(tc.ctx, tc.id, tc.tried)
+ var authErr *Error
+ if auth != nil || !errors.As(errPick, &authErr) || authErr.Code != "auth_not_found" {
+ t.Fatalf("%s: auth=%v err=%v, want auth_not_found", tc.name, auth, errPick)
+ }
+ }
+ if sticky, ok := selector.(*SessionAffinitySelector); ok {
+ namespace := provider
+ if mixed {
+ namespace = "mixed"
+ }
+ id := CanonicalSessionID(http.Header{"Session_id": {"non-weighted-pinned"}}, nil, nil)
+ if binding, found := sticky.cache.Get(namespace + "::" + id + "::" + model); !found || binding != pinnedID {
+ t.Fatalf("pinning checks changed the sticky binding: %q, found=%t", binding, found)
+ }
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/sdk/cliproxy/auth/conductor_selection.go b/sdk/cliproxy/auth/conductor_selection.go
index 330a29ccd..ec12690e2 100644
--- a/sdk/cliproxy/auth/conductor_selection.go
+++ b/sdk/cliproxy/auth/conductor_selection.go
@@ -2258,7 +2258,11 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
targetKey := canonicalSchedulingProvider(provider)
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
- var affinityCandidates []*Auth
+ trackPinnedAccounting := false
+ if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
+ _, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
+ }
+ var accountingCandidates, affinityCandidates []*Auth
if trackTransientAffinity {
affinityCandidates = make([]*Auth, 0, len(m.auths))
}
@@ -2266,7 +2270,7 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if candidate == nil || canonicalSchedulingProvider(executorKeyFromAuth(candidate)) != targetKey || candidate.Disabled {
continue
}
- if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
+ if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if !eligibility.allows(candidate) {
@@ -2275,10 +2279,18 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
continue
}
+ _, used := tried[candidate.ID]
+ // Pinning restricts selection, not the eligible pool used for accounting.
+ if trackPinnedAccounting && !used {
+ accountingCandidates = append(accountingCandidates, candidate)
+ }
+ if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
+ continue
+ }
if trackTransientAffinity {
affinityCandidates = append(affinityCandidates, candidate)
}
- if _, used := tried[candidate.ID]; used {
+ if used {
continue
}
candidates = append(candidates, candidate)
@@ -2302,6 +2314,15 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
return nil, nil, errAvailable
}
}
+ var accountingAuths []*Auth
+ if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
+ _, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, provider, model, now, false, opts.Headers)
+ if errAccounting != nil {
+ m.mu.RUnlock()
+ return nil, nil, errAccounting
+ }
+ accountingAuths = fullAvailable
+ }
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
m.mu.RUnlock()
@@ -2313,6 +2334,9 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
if !handled {
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
+ if accountingAuths != nil {
+ selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
+ }
selected, errPick = selector.Pick(selectorCtx, provider, selectionArgForSelector(selector, model), opts, selectorAuths)
if errPick != nil {
if isAuthNotFoundError(errPick) {
@@ -2678,7 +2702,11 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
registryRef := registry.GetGlobalRegistry()
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
- var affinityCandidates []*Auth
+ trackPinnedAccounting := false
+ if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
+ _, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
+ }
+ var accountingCandidates, affinityCandidates []*Auth
if trackTransientAffinity {
affinityCandidates = make([]*Auth, 0, len(m.auths))
}
@@ -2686,7 +2714,7 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if candidate == nil || candidate.Disabled {
continue
}
- if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
+ if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
continue
}
if !eligibility.allows(candidate) {
@@ -2705,10 +2733,18 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
continue
}
+ _, used := tried[candidate.ID]
+ // Pinning restricts selection, not the eligible pool used for accounting.
+ if trackPinnedAccounting && !used {
+ accountingCandidates = append(accountingCandidates, candidate)
+ }
+ if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
+ continue
+ }
if trackTransientAffinity {
affinityCandidates = append(affinityCandidates, candidate)
}
- if _, used := tried[candidate.ID]; used {
+ if used {
continue
}
candidates = append(candidates, candidate)
@@ -2732,6 +2768,15 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
return nil, nil, "", errAvailable
}
}
+ var accountingAuths []*Auth
+ if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
+ _, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, "mixed", model, now, false, opts.Headers)
+ if errAccounting != nil {
+ m.mu.RUnlock()
+ return nil, nil, "", errAccounting
+ }
+ accountingAuths = fullAvailable
+ }
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
m.mu.RUnlock()
@@ -2743,6 +2788,9 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
if !handled {
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
+ if accountingAuths != nil {
+ selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
+ }
selected, errPick = selector.Pick(selectorCtx, "mixed", selectionArgForSelector(selector, model), opts, selectorAuths)
if errPick != nil {
if isAuthNotFoundError(errPick) {
diff --git a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go
new file mode 100644
index 000000000..6f52dd303
--- /dev/null
+++ b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go
@@ -0,0 +1,161 @@
+package auth
+
+import (
+ "fmt"
+ "maps"
+ "net/http"
+ "testing"
+ "time"
+
+ "github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+ sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
+)
+
+func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ for _, mode := range []string{"sticky", "cold", "no-session", "lcp"} {
+ t.Run(fmt.Sprintf("mixed=%t/%s", mixed, mode), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ manager := NewManager(nil, selector, nil)
+ const provider, model = "weighted-pinned", "weighted-pinned-model"
+ manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
+ ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
+ aID, bID := t.Name()+"-a", t.Name()+"-b"
+ triedID := t.Name() + "-tried"
+ excluded := []*Auth{
+ {ID: t.Name() + "-disabled", Disabled: true},
+ {ID: t.Name() + "-zero", Attributes: map[string]string{AttributeWeight: "0"}},
+ {ID: t.Name() + "-cooldown", ModelStates: map[string]*ModelState{
+ model: {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
+ }},
+ {ID: t.Name() + "-wrong-model"},
+ {ID: t.Name() + "-wrong-kind", Attributes: map[string]string{AttributeAuthKind: AuthKindOAuth}},
+ {ID: t.Name() + "-wrong-provider", Provider: "other-provider"},
+ {ID: t.Name() + "-no-executor", Provider: "no-executor"},
+ {ID: t.Name() + "-lower-priority", Attributes: map[string]string{"priority": "0"}},
+ {ID: triedID},
+ }
+ auths := append([]*Auth{
+ {ID: aID, Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
+ {ID: bID, Attributes: map[string]string{AttributeWeight: "1", "priority": "1"}},
+ }, excluded...)
+ for _, auth := range auths {
+ if auth.Provider == "" {
+ auth.Provider = provider
+ }
+ auth.Status = StatusActive
+ if auth.Attributes == nil {
+ auth.Attributes = make(map[string]string)
+ }
+ if _, ok := auth.Attributes[AttributeWeight]; !ok {
+ auth.Attributes[AttributeWeight] = "100"
+ }
+ if _, ok := auth.Attributes["priority"]; !ok {
+ auth.Attributes["priority"] = "2"
+ }
+ if _, ok := auth.Attributes[AttributeAuthKind]; !ok {
+ auth.Attributes[AttributeAuthKind] = AuthKindAPIKey
+ }
+ registeredModel := model
+ if auth.ID == t.Name()+"-wrong-model" {
+ registeredModel = "other-model"
+ }
+ registry.GetGlobalRegistry().RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: registeredModel}})
+ t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
+ if _, errRegister := manager.Register(WithSkipPersist(ctx), auth); errRegister != nil {
+ t.Fatal(errRegister)
+ }
+ }
+ namespace := provider
+ if mixed {
+ namespace = "mixed"
+ }
+ stickyHeaders := http.Header{"Session_id": {"pinned-session"}}
+ id := CanonicalSessionID(stickyHeaders, nil, nil)
+ bindingKey := namespace + "::" + id + "::" + model
+ if mode == "sticky" {
+ selector.cache.Set(bindingKey, bID)
+ }
+ tried := map[string]struct{}{triedID: {}}
+ // Normalize the legacy signatures without changing their pinning behavior.
+ pick := func(opts cliproxyexecutor.Options) (*Auth, error) {
+ if mixed {
+ auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider, "no-executor"}, model, opts, tried)
+ return auth, errPick
+ }
+ auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
+ return auth, errPick
+ }
+ for i := range 4 {
+ opts := cliproxyexecutor.Options{Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: bID}}
+ switch mode {
+ case "sticky", "cold":
+ opts.Headers = stickyHeaders
+ case "lcp":
+ opts.SourceFormat = sdktranslator.FormatOpenAI
+ opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"pinned conversation"}]}`)
+ opts.Metadata[cliproxyexecutor.CallerScopeMetadataKey] = "pinned-caller"
+ }
+ if auth, errPick := pick(opts); errPick != nil || auth == nil || auth.ID != bID {
+ t.Fatalf("pinned request %d: auth=%v err=%v, want %q", i, auth, errPick, bID)
+ }
+ }
+ state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
+ if state == nil || !maps.Equal(state.current, map[string]int64{aID: 12, bID: -12}) || len(state.pending) != 0 {
+ t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state)
+ }
+ counts := map[string]int{}
+ for i := range 12 {
+ auth, errPick := pick(cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}})
+ if errPick != nil || auth == nil {
+ t.Fatalf("fresh request %d: auth=%v err=%v", i, auth, errPick)
+ }
+ counts[auth.ID]++
+ }
+ if counts[aID] != 12 || counts[bID] != 0 {
+ t.Fatalf("fresh distribution = %v, want a=12 b=0", counts)
+ }
+ if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
+ t.Fatalf("compensation did not balance: current=%v pending=%v", state.current, state.pending)
+ }
+ if mode == "sticky" || mode == "cold" {
+ if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID {
+ t.Fatalf("compensation changed pinned binding: %q, found=%t", binding, ok)
+ }
+ }
+ // A broader accounting pool must never make an invalid pin selectable.
+ for _, pinnedID := range []string{excluded[0].ID, excluded[1].ID, excluded[2].ID, excluded[3].ID, excluded[4].ID, excluded[5].ID, excluded[6].ID, triedID, "missing"} {
+ opts := cliproxyexecutor.Options{
+ Headers: stickyHeaders,
+ Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: pinnedID},
+ }
+ if auth, errPick := pick(opts); errPick == nil || auth != nil {
+ t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick)
+ }
+ if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
+ t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.current, state.pending)
+ }
+ }
+ // A valid lower-priority pin still wins without charging another tier.
+ lowerID := t.Name() + "-lower-priority"
+ lowerOpts := cliproxyexecutor.Options{
+ Headers: http.Header{"Session_id": {"lower-pinned-session"}},
+ Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: lowerID},
+ }
+ if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID {
+ t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick)
+ }
+ if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
+ t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.current, state.pending)
+ }
+ lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID)
+ if !maps.Equal(lowerState.current, map[string]int64{lowerID: 0}) || len(lowerState.pending) != 0 {
+ t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState)
+ }
+ })
+ }
+ }
+}
diff --git a/sdk/cliproxy/auth/scheduler_benchmark_test.go b/sdk/cliproxy/auth/scheduler_benchmark_test.go
index 85a901435..91817fd60 100644
--- a/sdk/cliproxy/auth/scheduler_benchmark_test.go
+++ b/sdk/cliproxy/auth/scheduler_benchmark_test.go
@@ -214,3 +214,41 @@ func BenchmarkManagerPickNextAndMarkResult1000(b *testing.B) {
manager.MarkResult(ctx, Result{AuthID: auth.ID, Provider: "gemini", Model: model, Success: true})
}
}
+
+func BenchmarkManagerPinnedLegacy1000(b *testing.B) {
+ for _, mixed := range []bool{false, true} {
+ b.Run(fmt.Sprintf("mixed=%t", mixed), func(b *testing.B) {
+ manager, providers, model := benchmarkManagerSetup(b, 1000, mixed, false)
+ selector := NewSessionAffinitySelector(&RoundRobinSelector{})
+ b.Cleanup(selector.Stop)
+ manager.SetSelector(selector)
+ ctx := context.Background()
+ opts := cliproxyexecutor.Options{Metadata: map[string]any{
+ cliproxyexecutor.PinnedAuthMetadataKey: "bench-gemini-0000",
+ }}
+ // Warm up index assignment and selection metadata outside the timed loop.
+ if mixed {
+ if _, _, _, errWarm := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil); errWarm != nil {
+ b.Fatal(errWarm)
+ }
+ } else if _, _, errWarm := manager.pickNextLegacy(ctx, providers[0], model, opts, nil); errWarm != nil {
+ b.Fatal(errWarm)
+ }
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ if mixed {
+ auth, exec, provider, errPick := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil)
+ if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil || provider != providers[0] {
+ b.Fatalf("pinned mixed pick failed: auth=%v exec=%v provider=%q err=%v", auth, exec, provider, errPick)
+ }
+ } else {
+ auth, exec, errPick := manager.pickNextLegacy(ctx, providers[0], model, opts, nil)
+ if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil {
+ b.Fatalf("pinned pick failed: auth=%v exec=%v err=%v", auth, exec, errPick)
+ }
+ }
+ }
+ })
+ }
+}
diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go
index 323f13af0..54e16ebc9 100644
--- a/sdk/cliproxy/auth/selector.go
+++ b/sdk/cliproxy/auth/selector.go
@@ -45,8 +45,11 @@ type WeightedRoundRobinSelector struct {
}
type smoothWeightedState struct {
- current map[string]int64
- weights map[string]int64
+ current map[string]int64
+ weights map[string]int64
+ pending map[string]int64
+ inFlight uint64
+ generation uint64
}
type weightedSelectorStateModelKey struct{}
@@ -698,32 +701,12 @@ func (s *WeightedRoundRobinSelector) Pick(ctx context.Context, provider, model s
return nil, errAvailable
}
available = preferCodexWebsocketAuths(ctx, provider, available)
- stateModel := weightedSelectorStateModel(ctx, model)
- key := provider + ":" + canonicalModelLookupKey(stateModel)
-
- s.mu.Lock()
- defer s.mu.Unlock()
- if s.states == nil {
- s.states = make(map[string]*smoothWeightedState)
- }
- limit := s.maxKeys
- if limit <= 0 {
- limit = 4096
- }
- if _, ok := s.states[key]; !ok && len(s.states) >= limit {
- s.states = make(map[string]*smoothWeightedState)
- }
- state := s.states[key]
- if state == nil {
- state = &smoothWeightedState{}
- s.states[key] = state
- }
- weights := authWeightVector(available)
- state.prepare(weights)
- picked := pickSmoothWeightedAuth(available, state.current)
+ key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
+ picked, reservation := s.reserve(key, available, "")
if picked == nil {
return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
}
+ reservation.finish(true)
return picked, nil
}
@@ -740,6 +723,9 @@ const maxSmoothWeightedStateEntries = 1024
func (s *smoothWeightedState) prepare(weights map[string]int64) {
if s.current == nil || weightsConfigChanged(s.weights, weights) {
s.current = make(map[string]int64, len(weights))
+ s.pending = make(map[string]int64, len(weights))
+ s.inFlight = 0
+ s.generation++
}
if s.weights == nil {
s.weights = make(map[string]int64, len(weights))
@@ -796,29 +782,6 @@ func authWeightVector(auths []*Auth) map[string]int64 {
return weights
}
-func pickSmoothWeightedAuth(auths []*Auth, current map[string]int64) *Auth {
- var picked *Auth
- var pickedCurrent int64
- var totalWeight int64
- for _, auth := range auths {
- weight := authWeight(auth)
- if auth == nil || weight <= 0 {
- continue
- }
- current[auth.ID] = saturatingAddInt64(current[auth.ID], weight)
- totalWeight = saturatingAddInt64(totalWeight, weight)
- if picked == nil || current[auth.ID] > pickedCurrent {
- picked = auth
- pickedCurrent = current[auth.ID]
- }
- }
- if picked == nil {
- return nil
- }
- current[picked.ID] = saturatingAddInt64(current[picked.ID], -totalWeight)
- return picked
-}
-
func saturatingAddInt64(value, delta int64) int64 {
if delta > 0 && value > math.MaxInt64-delta {
return math.MaxInt64
@@ -1164,7 +1127,16 @@ func (s *SessionAffinitySelector) Trees() *cliproxysession.InMemorySessionTreeSt
// Note: The cache key includes provider, session ID, and model to handle cases where
// a session uses multiple models (e.g., gemini-2.5-pro and gemini-3-flash-preview)
// that may be supported by different auth credentials, and to avoid cross-provider conflicts.
-func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) {
+func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (selected *Auth, errPick error) {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ if errContext := ctx.Err(); errContext != nil {
+ return nil, errContext
+ }
+ var reservation *weightedReservation
+ // Local bindings complete inside Pick; failed selections release speculative credit.
+ defer func() { reservation.finish(selected != nil && errPick == nil) }()
entry := selectorLogEntry(ctx)
if opts.Metadata == nil {
opts.Metadata = make(map[string]any)
@@ -1176,7 +1148,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
// consulted when no header, body, or execution-session identity is present.
explicitID, explicitFallbackID := extractExplicitSessionIDs(opts.Headers, opts.OriginalRequest, opts.Metadata)
if explicitID == "" {
- if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry); handled || errLCP != nil {
+ if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry, &reservation); handled || errLCP != nil {
return auth, errLCP
}
} else if opts.Metadata != nil {
@@ -1218,7 +1190,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, errAvailable
}
entry.Debugf("session-affinity: no session ID extracted, falling back to default selector | provider=%s model=%s", provider, model)
- return s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
+ return s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
}
// A single availability pass serves both lookups: the bound credential is validated against
@@ -1268,7 +1240,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if auth.ID == cachedAuthID {
bind(auth.ID, true)
entry.Infof("session-affinity: cache hit | session=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), auth.ID, provider, model)
- return auth, nil
+ return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
if s.rebindOnNever {
@@ -1281,7 +1253,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, err
}
// Cached auth not available, reselect via fallback selector for even distribution
- auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
+ auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1315,7 +1287,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if auth.ID == parentAuthID {
bind(auth.ID, true)
entry.Infof("session-affinity: guardian inherited parent binding | session=%s parent=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
- return auth, nil
+ return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
}
@@ -1332,7 +1304,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
} else {
entry.Infof("session-affinity: fallback cache hit | session=%s fallback=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
}
- return auth, nil
+ return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
}
}
}
@@ -1343,7 +1315,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return nil, errBound
}
if !isSubagent && !isFork && s.shouldPreserveBindingWhenUnavailable(ctx, availabilityCandidates, model, cachedAuthID, now) {
- auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
+ auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1360,7 +1332,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
if err != nil {
return nil, err
}
- auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
+ auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
if err != nil {
return nil, err
}
@@ -1376,7 +1348,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
return auth, nil
}
-func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry) (*Auth, bool, error) {
+func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry, reservation **weightedReservation) (*Auth, bool, error) {
if s == nil || s.matcher == nil {
return nil, false, nil
}
@@ -1449,7 +1421,8 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
}
entry.Infof("session-affinity: LCP cache hit | session=%s prefix=%d auth=%s provider=%s model=%s", truncateSessionID(match.SessionID), match.PrefixLength, auth.ID, provider, model)
}
- return auth, true, nil
+ picked, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, reservation)
+ return picked, true, errPick
}
if !match.IsFork && s.rebindOnNever {
if bindRes := s.matcher.BindFingerprintsWithContext(namespace, fingerprints, tailFingerprints, envDigest, minPrefixLength, match.AuthID); bindRes.SessionID != "" {
@@ -1488,7 +1461,7 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
return nil, true, errAvailable
}
fallbackAuths := highestPriorityAuths(available)
- auth, errPick := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
+ auth, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, reservation)
if errPick != nil {
return nil, true, errPick
}
diff --git a/sdk/cliproxy/auth/selector_test.go b/sdk/cliproxy/auth/selector_test.go
index 18ed81f30..b6654cfcf 100644
--- a/sdk/cliproxy/auth/selector_test.go
+++ b/sdk/cliproxy/auth/selector_test.go
@@ -205,16 +205,14 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) {
}
}
-func TestPickSmoothWeightedAuth_SaturatesCorruptState(t *testing.T) {
+func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) {
t.Parallel()
- current := map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}
- picked := pickSmoothWeightedAuth([]*Auth{{ID: "a"}, {ID: "b"}}, current)
- if picked == nil {
- t.Fatal("pickSmoothWeightedAuth() returned nil")
- }
- if current["a"] != math.MaxInt64-2 || current["b"] != math.MinInt64+1 {
- t.Fatalf("current state = %#v, want saturated arithmetic", current)
+ state := &smoothWeightedState{current: map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}}
+ state.charge(map[string]int64{"a": 1, "b": 1}, "a", false)
+ limit := maxSmoothWeightedCreditRounds * 2
+ if state.current["a"] != limit || state.current["b"] != -limit {
+ t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.current)
}
}
diff --git a/sdk/cliproxy/auth/selector_weighted.go b/sdk/cliproxy/auth/selector_weighted.go
new file mode 100644
index 000000000..ed92b5bb0
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted.go
@@ -0,0 +1,292 @@
+package auth
+
+import (
+ "context"
+ "math"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+// weightedAccountingPoolKey carries the manager-validated pre-pin pool. It affects
+// accounting only; the selector still receives pin-filtered selection candidates.
+type weightedAccountingPoolKey struct{}
+
+// weightedReservation separates speculative credit from committed, clamped credit.
+// Releasing one request must never undo accounting committed by another request.
+type weightedReservation struct {
+ selector *WeightedRoundRobinSelector
+ state *smoothWeightedState
+ generation uint64
+ weights map[string]int64
+ authID string
+ total int64
+ bound bool
+ finished bool
+}
+
+// pickWithAffinityReservation normalizes weighted candidates and defers accounting
+// until the local binding completes. Other fallback selectors keep their behavior.
+func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, bound *Auth, reservation **weightedReservation) (*Auth, error) {
+ weighted, ok := s.fallback.(*WeightedRoundRobinSelector)
+ if !ok {
+ if bound != nil {
+ return bound, nil
+ }
+ return s.fallback.Pick(ctx, provider, model, opts, auths)
+ }
+ if accountingAuths, ok := ctx.Value(weightedAccountingPoolKey{}).([]*Auth); ok {
+ pinnedID := pinnedAuthIDFromMetadata(opts.Metadata)
+ if bound == nil {
+ for _, auth := range auths {
+ if auth != nil && auth.ID == pinnedID {
+ bound = auth
+ break
+ }
+ }
+ }
+ if bound == nil || bound.ID != pinnedID || authWeight(bound) <= 0 {
+ return nil, &Error{Code: "auth_unavailable", Message: "pinned auth is not available for weighted accounting"}
+ }
+ auths = positiveWeightAuths(accountingAuths)
+ }
+ var available []*Auth
+ authID := ""
+ if bound == nil {
+ var errAvailable error
+ available, errAvailable = getSelectorAvailableAuths(ctx, positiveWeightAuths(auths), provider, model, time.Now())
+ if errAvailable != nil {
+ return nil, errAvailable
+ }
+ available = preferCodexWebsocketAuths(ctx, provider, available)
+ } else {
+ authID = bound.ID
+ // A binding may bypass priority or transport preference, but its traffic
+ // must only charge peers that compete with it, never another priority tier.
+ priority := authPriority(bound)
+ for _, auth := range auths {
+ if authPriority(auth) == priority {
+ available = append(available, auth)
+ }
+ }
+ preferred := preferCodexWebsocketAuths(ctx, provider, available)
+ if len(preferred) < len(available) && !authWebsocketsEnabled(bound) {
+ // Keep an HTTP-bound session outside the preferred WebSocket pool.
+ nonWebsocket := available[:0]
+ for _, auth := range available {
+ if !authWebsocketsEnabled(auth) {
+ nonWebsocket = append(nonWebsocket, auth)
+ }
+ }
+ available = nonWebsocket
+ } else {
+ available = preferred
+ }
+ }
+ key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
+ picked, pending := weighted.reserve(key, available, authID)
+ if picked == nil {
+ return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
+ }
+ *reservation = pending
+ return picked, nil
+}
+
+// weightedPoolStateKey isolates credits for actual competing sets. Reusing a
+// provider/model accumulator across different sets grants credit for requests
+// those sets cannot serve, which can permanently starve their smaller weights.
+// Length-prefixed, sorted IDs avoid collisions and keep input order irrelevant.
+// Weights stay out of the key so prepare can invalidate pending reservations on
+// configuration changes. Returning to a prior set resumes that set's credits.
+func weightedPoolStateKey(route string, weights map[string]int64) string {
+ ids := make([]string, 0, len(weights))
+ for id := range weights {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ var key strings.Builder
+ key.WriteString(strconv.Itoa(len(route)))
+ key.WriteByte(':')
+ key.WriteString(route)
+ for _, id := range ids {
+ key.WriteString(strconv.Itoa(len(id)))
+ key.WriteByte(':')
+ key.WriteString(id)
+ }
+ return key.String()
+}
+
+// reserve selects and reserves credit atomically. A non-empty authID preserves a
+// sticky binding; otherwise committed and pending credits both guide selection.
+func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, authID string) (*Auth, *weightedReservation) {
+ weights := authWeightVector(available)
+ key = weightedPoolStateKey(key, weights)
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.states == nil {
+ s.states = make(map[string]*smoothWeightedState)
+ }
+ limit := s.maxKeys
+ if limit <= 0 {
+ limit = 4096
+ }
+ if _, ok := s.states[key]; !ok && len(s.states) >= limit {
+ s.states = make(map[string]*smoothWeightedState)
+ }
+ state := s.states[key]
+ if state == nil {
+ state = &smoothWeightedState{}
+ s.states[key] = state
+ }
+ state.prepare(weights)
+ var picked *Auth
+ var pickedCurrent, total int64
+ for _, auth := range available {
+ if auth == nil || weights[auth.ID] <= 0 {
+ continue
+ }
+ total = saturatingAddInt64(total, weights[auth.ID])
+ if authID != "" && auth.ID != authID {
+ continue
+ }
+ current := saturatingAddInt64(state.current[auth.ID], state.pending[auth.ID])
+ current = saturatingAddInt64(current, weights[auth.ID])
+ if picked == nil || current > pickedCurrent {
+ picked, pickedCurrent = auth, current
+ }
+ }
+ if picked == nil {
+ return nil, nil
+ }
+ // Bound traffic beyond the compensation window still keeps its credential,
+ // but must not create speculative credit that can starve concurrent picks.
+ if authID != "" && !state.boundChargeFits(weights, picked.ID, total) {
+ return picked, nil
+ }
+ state.adjustPending(weights, picked.ID, total, false)
+ state.inFlight++
+ return picked, &weightedReservation{
+ selector: s, state: state, generation: state.generation,
+ weights: weights, authID: picked.ID, total: total, bound: authID != "",
+ }
+}
+
+// finish confirms or releases a reservation exactly once under the selector lock.
+func (r *weightedReservation) finish(committed bool) {
+ if r == nil {
+ return
+ }
+ r.selector.mu.Lock()
+ defer r.selector.mu.Unlock()
+ if r.finished {
+ return
+ }
+ r.finished = true
+ if r.state.generation != r.generation {
+ return
+ }
+ r.state.adjustPending(r.weights, r.authID, r.total, true)
+ r.state.inFlight--
+ if committed {
+ r.state.charge(r.weights, r.authID, r.bound)
+ } else if r.state.inFlight == 0 {
+ // A final rollback can remove the offset to already committed credit.
+ r.state.boundCredits(r.weights, r.total)
+ }
+}
+
+// adjustPending must be called with the selector mutex held. Pending deltas are
+// not clamped, so releasing them cannot lose credit at the compensation bound.
+func (s *smoothWeightedState) adjustPending(weights map[string]int64, authID string, total int64, release bool) {
+ for id, weight := range weights {
+ delta := weight
+ if id == authID {
+ delta -= total
+ }
+ if release {
+ delta = -delta
+ }
+ s.pending[id] = saturatingAddInt64(s.pending[id], delta)
+ if s.pending[id] == 0 {
+ delete(s.pending, id)
+ }
+ }
+}
+
+// maxSmoothWeightedCreditRounds bounds compensation after a long sticky session.
+const maxSmoothWeightedCreditRounds int64 = 32
+
+// boundChargeFits admits the whole sticky delta or none. Both reservation and
+// commit include outstanding deltas: committed credit alone can transiently
+// exceed the cap while successful reservations settle in a different order.
+func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID string, totalWeight int64) bool {
+ limit := int64(math.MaxInt64)
+ if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
+ limit = totalWeight * maxSmoothWeightedCreditRounds
+ }
+ for id, weight := range weights {
+ delta := weight
+ if id == authID {
+ delta -= totalWeight
+ }
+ current := saturatingAddInt64(s.current[id], s.pending[id])
+ next := saturatingAddInt64(current, delta)
+ if next < -limit || next > limit {
+ return false
+ }
+ }
+ return true
+}
+
+func (s *smoothWeightedState) charge(weights map[string]int64, authID string, bound bool) {
+ var totalWeight int64
+ for _, weight := range weights {
+ totalWeight = saturatingAddInt64(totalWeight, weight)
+ }
+ if !bound || s.boundChargeFits(weights, authID, totalWeight) {
+ // At the cap, forced traffic cannot erase a small peer's earnings.
+ // Pending repayments still count until they actually roll back.
+ for id, weight := range weights {
+ s.current[id] = saturatingAddInt64(s.current[id], weight)
+ }
+ s.current[authID] = saturatingAddInt64(s.current[authID], -totalWeight)
+ }
+ if s.inFlight == 0 {
+ s.boundCredits(weights, totalWeight)
+ }
+}
+
+// boundCredits runs only once the pool settles. Net pending deltas can be zero
+// even with live reservations, so the in-flight count, not len(pending), decides
+// when clipping cannot irreversibly discard another reservation's offset.
+func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight int64) {
+ limit := int64(math.MaxInt64)
+ if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
+ limit = totalWeight * maxSmoothWeightedCreditRounds
+ }
+ var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64
+ for id := range weights {
+ minCredit = min(minCredit, s.current[id])
+ maxCredit = max(maxCredit, s.current[id])
+ }
+ // A common offset keeps leading credits distinct at the compensation bound.
+ // Independent clamps can repeatedly tie the leaders and starve the later
+ // candidate in deterministic selection order.
+ minOffset := saturatingAddInt64(maxCredit, -limit)
+ maxOffset := saturatingAddInt64(minCredit, limit)
+ var offset int64
+ if minOffset > 0 || minOffset > maxOffset {
+ offset = minOffset
+ } else if maxOffset < 0 {
+ offset = maxOffset
+ }
+ // If the full spread cannot fit, cap only the trailing debt after rebasing
+ // the leader to the upper bound. Do not flatten leading candidates' credits.
+ // Temporarily excluded credentials retain their credit for later selections.
+ for id := range weights {
+ s.current[id] = max(-limit, saturatingAddInt64(s.current[id], -offset))
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_affinity_test.go b/sdk/cliproxy/auth/selector_weighted_affinity_test.go
new file mode 100644
index 000000000..12f3c2fc7
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_affinity_test.go
@@ -0,0 +1,436 @@
+package auth
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+ cliproxysession "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/session"
+ sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
+)
+
+func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) {
+ for _, source := range []string{"sticky", "parent", "guardian", "fork"} {
+ t.Run(source, func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{
+ {ID: "a", Provider: "codex", Attributes: map[string]string{AttributeWeight: "3"}},
+ {ID: "b", Provider: "codex"},
+ }
+ parent := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
+ parentID := CanonicalSessionID(parent.Headers, nil, nil)
+ parentModel := "model"
+ if source == "guardian" {
+ parentModel = "parent-model"
+ }
+ key := "codex::" + parentID + "::" + parentModel
+ selector.cache.setSessionAliases("b", "codex", parentID, key)
+ for i := range 4 {
+ opts := parent
+ if source != "sticky" {
+ opts = cliproxyexecutor.Options{Headers: http.Header{
+ "Session_id": {fmt.Sprintf("child-%d", i)}, "X-Codex-Parent-Thread-Id": {"busy"},
+ }}
+ if source == "guardian" {
+ opts.Headers.Set("X-Openai-Subagent", "guardian")
+ }
+ if source == "fork" {
+ opts.OriginalRequest = []byte(`{"forked_from_thread_id":"busy"}`)
+ }
+ }
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
+ if errPick != nil || picked == nil || picked.ID != "b" {
+ t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ if state.current["a"] != 12 || state.current["b"] != -12 || len(state.pending) != 0 {
+ t.Fatalf("sticky credits: current=%v pending=%v", state.current, state.pending)
+ }
+ for i := range 12 {
+ // Requests without an identity and cold sessions share the same credits.
+ opts := cliproxyexecutor.Options{}
+ if i%2 == 0 {
+ opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
+ }
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
+ if errPick != nil || picked == nil || picked.ID != "a" {
+ t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick)
+ }
+ }
+ if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 {
+ t.Fatalf("compensated credits: current=%v pending=%v", state.current, state.pending)
+ }
+ if binding, ok := selector.cache.Get(key); !ok || binding != "b" {
+ t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok)
+ }
+ })
+ }
+}
+
+func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing.T) {
+ for _, newSession := range []bool{false, true} {
+ t.Run(fmt.Sprintf("new-session=%t", newSession), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{
+ {ID: "a", Attributes: map[string]string{"priority": "0"}},
+ {ID: "b", Attributes: map[string]string{"priority": "1"}},
+ {ID: "c", Attributes: map[string]string{"priority": "1"}},
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"lower-priority"}}}
+ id := CanonicalSessionID(sticky.Headers, nil, nil)
+ key := "codex::" + id + "::model"
+ selector.cache.Set(key, "a")
+ counts := map[string]int{}
+ const warmup, requests = 100, 1000
+ for i := range requests {
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
+ if errPick != nil || picked == nil || picked.ID != "a" {
+ t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ opts := cliproxyexecutor.Options{}
+ if newSession {
+ opts.Headers = http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}
+ }
+ picked, errPick = selector.Pick(t.Context(), "codex", "model", opts, auths)
+ if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
+ t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ if i >= warmup {
+ counts[picked.ID]++
+ }
+ }
+ if counts["b"] != (requests-warmup)/2 || counts["c"] != (requests-warmup)/2 {
+ t.Fatalf("changing pools starved equal-weight candidates after warmup: %v, want b=450 c=450", counts)
+ }
+ if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
+ t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok)
+ }
+ if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pending) != 0 {
+ t.Fatalf("pending credits leaked: %v", state.pending)
+ }
+ })
+ }
+}
+
+func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ counts := map[string]int{}
+ for i := range 40 {
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{
+ Headers: http.Header{"Session_id": {fmt.Sprintf("session-%d", i)}},
+ }, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("pick=%v err=%v", picked, errPick)
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ if i == 0 && (state.current["a"] != -1 || state.current["b"] != 1) {
+ t.Fatalf("first selection charged more than once: %v", state.current)
+ }
+ counts[picked.ID]++
+ }
+ if counts["a"] != 30 || counts["b"] != 10 {
+ t.Fatalf("cold distribution = %v, want 30:10", counts)
+ }
+}
+
+func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"long"}}}
+ id := CanonicalSessionID(opts.Headers, nil, nil)
+ selector.cache.Set("codex::"+id+"::model", "b")
+ for range 1000 {
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
+ if errPick != nil || picked == nil || picked.ID != "b" {
+ t.Fatalf("long session: pick=%v err=%v", picked, errPick)
+ }
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ limit := maxSmoothWeightedCreditRounds * 4
+ // A bound charge is admitted as a whole, stopping within one delta of the cap.
+ wantCredit := limit / 3 * 3
+ if state.current["a"] != wantCredit || state.current["b"] != -wantCredit {
+ t.Fatalf("bounded credits = %v, want +/- %d", state.current, wantCredit)
+ }
+ for range wantCredit {
+ if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
+ t.Fatal(errPick)
+ }
+ }
+ if state.current["a"] != 0 || state.current["b"] != 0 {
+ t.Fatalf("compensation did not finish: %v", state.current)
+ }
+}
+
+func TestWeightedSessionAffinityLCPCredits(t *testing.T) {
+ for _, source := range []string{"hit", "fork", "compaction"} {
+ t.Run(source, func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ root := cliproxyexecutor.Options{
+ SourceFormat: sdktranslator.FormatOpenAI,
+ OriginalRequest: []byte(`{"messages":[{"role":"user","content":"step 1"},` +
+ `{"role":"assistant","content":"ack 1"},{"role":"user","content":"step 2"},` +
+ `{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"}]}`),
+ Metadata: map[string]any{cliproxyexecutor.CallerScopeMetadataKey: "caller"},
+ }
+ // Seed the matcher without consuming weighted credits.
+ turns := cliproxysession.ExtractCanonicalTurns(root.SourceFormat, root.OriginalRequest)
+ selector.matcher.Bind(lcpAffinityNamespace("openai", "model", root.Metadata), turns, "b")
+ opts := root
+ if source == "fork" {
+ opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"step 1"},` +
+ `{"role":"assistant","content":"ack 1"},{"role":"user","content":"branch"}]}`)
+ } else if source == "compaction" {
+ opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"Steps 1 and 2 completed"},` +
+ `{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"},` +
+ `{"role":"user","content":"step 4"}]}`)
+ }
+ picked, errPick := selector.Pick(t.Context(), "openai", "model", opts, auths)
+ if errPick != nil || picked == nil || picked.ID != "b" {
+ t.Fatalf("LCP %s: pick=%v err=%v", source, picked, errPick)
+ }
+ if source == "fork" && opts.Metadata[cliproxyexecutor.IsForkMetadataKey] != true {
+ t.Fatal("expected LCP fork")
+ }
+ if source == "compaction" && opts.Metadata[cliproxyexecutor.IsCompactionMetadataKey] != true {
+ t.Fatal("expected LCP compaction")
+ }
+ state := weightedStateForTest(t, weighted, "openai:model", "a", "b")
+ if state.current["a"] != 3 || state.current["b"] != -3 || len(state.pending) != 0 {
+ t.Fatalf("LCP credits: current=%v pending=%v", state.current, state.pending)
+ }
+ })
+ }
+}
+
+func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ key := "codex:model"
+ for range 1000 {
+ _, reservation := weighted.reserve(key, auths, "b")
+ reservation.finish(true)
+ }
+ state := weightedStateForTest(t, weighted, key, "a", "b")
+ _, rejected := weighted.reserve(key, auths, "b")
+ _, accepted := weighted.reserve(key, auths, "a")
+ accepted.finish(true)
+ rejected.finish(false)
+ rejected.finish(true)
+ accepted.finish(false)
+ limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3
+ if state.current["a"] != limit-1 || state.current["b"] != -limit+1 || len(state.pending) != 0 {
+ t.Fatalf("rollback lost another charge: current=%v pending=%v", state.current, state.pending)
+ }
+}
+
+func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ key := "codex:model"
+ _, old := weighted.reserve(key, auths, "")
+ auths[0].Attributes[AttributeWeight] = "1"
+ _, latest := weighted.reserve(key, auths, "")
+ old.finish(true)
+ state := weightedStateForTest(t, weighted, key, "a", "b")
+ if state.current["a"] != 0 || state.current["b"] != 0 || state.pending["a"] != -1 || state.pending["b"] != 1 {
+ t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.current, state.pending)
+ }
+ latest.finish(true)
+ if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 {
+ t.Fatalf("new generation accounting: current=%v pending=%v", state.current, state.pending)
+ }
+}
+
+func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ const requests = 40
+ selected := make(chan *Auth, requests)
+ reservations := make(chan *weightedReservation, requests)
+ var ready sync.WaitGroup
+ ready.Add(requests)
+ for range requests {
+ go func() {
+ defer ready.Done()
+ picked, reservation := weighted.reserve("codex:model", auths, "")
+ selected <- picked
+ reservations <- reservation
+ }()
+ }
+ ready.Wait()
+ close(selected)
+ close(reservations)
+ counts := map[string]int{}
+ for auth := range selected {
+ if auth == nil {
+ t.Fatal("concurrent reservation returned no auth")
+ }
+ counts[auth.ID]++
+ }
+ if counts["a"] != 30 || counts["b"] != 10 {
+ t.Fatalf("pending allocations ignored: %v", counts)
+ }
+ for reservation := range reservations {
+ reservation.finish(true)
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 {
+ t.Fatalf("concurrent accounting: current=%v pending=%v", state.current, state.pending)
+ }
+}
+
+func TestWeightedSessionAffinityFailureDoesNotCharge(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{
+ Fallback: weighted, RebindOn: SessionAffinityRebindOnNever,
+ })
+ t.Cleanup(selector.Stop)
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound"}}}
+ id := CanonicalSessionID(opts.Headers, nil, nil)
+ selector.cache.Set("codex::"+id+"::model", "missing")
+ if auth, errPick := selector.Pick(t.Context(), "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); errPick == nil || auth != nil {
+ t.Fatalf("expected unavailable binding: auth=%v err=%v", auth, errPick)
+ }
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ if auth, errPick := selector.Pick(ctx, "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); !errors.Is(errPick, context.Canceled) || auth != nil {
+ t.Fatalf("expected cancellation: auth=%v err=%v", auth, errPick)
+ }
+ if len(weighted.states) != 0 {
+ t.Fatalf("failed selection changed weighted state: %v", weighted.states)
+ }
+}
+
+func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{
+ {ID: "a", Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
+ {ID: "b", Attributes: map[string]string{"priority": "1"}},
+ {ID: "lower", Attributes: map[string]string{AttributeWeight: "100"}},
+ {ID: "zero", Attributes: map[string]string{AttributeWeight: "0", "priority": "2"}},
+ {ID: "cooldown", Attributes: map[string]string{"priority": "2"}, ModelStates: map[string]*ModelState{
+ "upstream": {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
+ }},
+ }
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"alias"}}}
+ id := CanonicalSessionID(opts.Headers, nil, nil)
+ selector.cache.Set("codex::"+id+"::upstream", "b")
+ ctx := withWeightedSelectorStateModel(t.Context(), weighted, "requested(high)")
+ picked, errPick := selector.Pick(ctx, "codex", "upstream", opts, auths)
+ if errPick != nil || picked == nil || picked.ID != "b" {
+ t.Fatalf("sticky pick=%v err=%v", picked, errPick)
+ }
+ state := weightedStateForTest(t, weighted, "codex:requested", "a", "b")
+ if state == nil || len(weighted.states) != 1 || len(state.current) != 2 || state.current["a"] != 3 || state.current["b"] != -3 {
+ t.Fatalf("wrong model key or eligible pool: %v", weighted.states)
+ }
+ picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2])
+ if errPick != nil || picked == nil || picked.ID != "a" || state.current["a"] != 2 || state.current["b"] != -2 {
+ t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.current)
+ }
+ for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} {
+ if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil {
+ t.Fatal(errOther)
+ }
+ }
+ if state.current["a"] != 2 || state.current["b"] != -2 {
+ t.Fatal("another provider or model changed credits")
+ }
+}
+
+func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"reselect"}}}
+ id := CanonicalSessionID(opts.Headers, nil, nil)
+ key := "codex::" + id + "::model"
+ selector.cache.Set(key, "removed")
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts,
+ []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}})
+ if errPick != nil || picked == nil || picked.ID != "a" {
+ t.Fatalf("reselection: pick=%v err=%v", picked, errPick)
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 {
+ t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.current, state.pending)
+ }
+ if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
+ t.Fatalf("reselection did not bind: %q, found=%t", binding, ok)
+ }
+}
+
+func TestManagerWeightedSessionAffinityCompensation(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ t.Run(fmt.Sprintf("mixed=%t", mixed), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ manager := NewManager(nil, selector, nil)
+ provider, model := "weighted-affinity", "weighted-model"
+ manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
+ aID, bID := t.Name()+"-a", t.Name()+"-b"
+ for _, auth := range []*Auth{
+ {ID: aID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "3"}},
+ {ID: bID, Provider: provider, Status: StatusActive},
+ } {
+ if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
+ t.Fatal(errRegister)
+ }
+ registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
+ t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
+ }
+ namespace := provider
+ if mixed {
+ namespace = "mixed"
+ }
+ parentHeaders := http.Header{"Session_id": {"manager-sticky"}}
+ id := CanonicalSessionID(parentHeaders, nil, nil)
+ selector.cache.Set(namespace+"::"+id+"::"+model, bID)
+ for i := range 16 {
+ opts := cliproxyexecutor.Options{Headers: parentHeaders}
+ want := bID
+ if i >= 4 {
+ opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
+ want = aID
+ }
+ var auth *Auth
+ var errPick error
+ if mixed {
+ auth, _, _, errPick = manager.pickNextMixed(t.Context(), []string{provider}, model, opts, nil)
+ } else {
+ auth, _, errPick = manager.pickNext(t.Context(), provider, model, opts, nil)
+ }
+ if errPick != nil || auth == nil || auth.ID != want {
+ t.Fatalf("request %d: auth=%v err=%v, want %q", i, auth, errPick, want)
+ }
+ }
+ state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
+ if state == nil || state.current[aID] != 0 || state.current[bID] != 0 || len(state.pending) != 0 {
+ t.Fatalf("manager credits did not balance: %#v", state)
+ }
+ })
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go
new file mode 100644
index 000000000..22ecac2eb
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go
@@ -0,0 +1,199 @@
+package auth
+
+import (
+ "fmt"
+ "maps"
+ "testing"
+
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) {
+ for _, rollbackFirst := range []bool{false, true} {
+ t.Run(fmt.Sprintf("rollback-first=%t", rollbackFirst), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a"}, {ID: "b"}}
+ reservations := map[string][]*weightedReservation{}
+ t.Cleanup(func() {
+ for _, group := range reservations {
+ for _, reservation := range group {
+ reservation.finish(false)
+ }
+ }
+ })
+ for range 256 {
+ picked, reservation := weighted.reserve("codex:model", auths, "")
+ if picked == nil || reservation == nil {
+ t.Fatal("missing cold reservation")
+ }
+ reservations[picked.ID] = append(reservations[picked.ID], reservation)
+ }
+ ids := []string{"a", "b"}
+ if rollbackFirst {
+ ids = []string{"b", "a"}
+ }
+ for _, id := range ids {
+ for _, reservation := range reservations[id] {
+ reservation.finish(id == "a")
+ // Repeated finishing must not decrement the counter again.
+ reservation.finish(false)
+ }
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ limit := maxSmoothWeightedCreditRounds * 2
+ if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -limit, "b": limit}) {
+ t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.current, state.pending, state.inFlight)
+ }
+ })
+ }
+}
+
+func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) {
+ for _, first := range []string{"a", "b"} {
+ t.Run(first, func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ reservations := map[string][]*weightedReservation{}
+ t.Cleanup(func() {
+ for _, group := range reservations {
+ for _, reservation := range group {
+ reservation.finish(false)
+ }
+ }
+ })
+ for range 512 {
+ picked, reservation := weighted.reserve("codex:model", auths, "")
+ if picked == nil || reservation == nil {
+ t.Fatal("missing cold reservation")
+ }
+ reservations[picked.ID] = append(reservations[picked.ID], reservation)
+ }
+ if len(reservations["a"]) != 384 || len(reservations["b"]) != 128 {
+ t.Fatalf("reservations did not follow 3:1: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
+ }
+ last := "b"
+ if first == "b" {
+ last = "a"
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ for _, reservation := range reservations[first] {
+ reservation.finish(true)
+ }
+ // New arrivals must see the remaining offsets before the batch settles.
+ for _, phase := range []string{"pending", "settled"} {
+ if phase == "settled" {
+ for _, reservation := range reservations[last] {
+ reservation.finish(true)
+ }
+ }
+ counts := map[string]int{}
+ for range 64 {
+ picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("Pick failed: auth=%v err=%v", picked, errPick)
+ }
+ counts[picked.ID]++
+ }
+ if counts["a"] != 48 || counts["b"] != 16 {
+ t.Fatalf("%s distribution = %v, want 48:16", phase, counts)
+ }
+ }
+ if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) {
+ t.Fatalf("settled 3:1 batch drifted: %#v", state)
+ }
+ })
+ }
+}
+
+func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a"}, {ID: "b"}}
+ var old []*weightedReservation
+ t.Cleanup(func() {
+ for _, reservation := range old {
+ reservation.finish(false)
+ }
+ })
+ for range 256 {
+ _, reservation := weighted.reserve("codex:model", auths, "")
+ old = append(old, reservation)
+ }
+ auths[0].Attributes = map[string]string{AttributeWeight: "3"}
+ _, latest := weighted.reserve("codex:model", auths, "")
+ if latest == nil {
+ t.Fatal("missing new-generation reservation")
+ }
+ t.Cleanup(func() { latest.finish(false) })
+ for _, reservation := range old {
+ reservation.finish(true)
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ if state.inFlight != 1 || len(state.current) != 0 || !maps.Equal(state.pending, map[string]int64{"a": -1, "b": 1}) {
+ t.Fatalf("old generation altered new accounting: %#v", state)
+ }
+ latest.finish(true)
+ if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -1, "b": 1}) {
+ t.Fatalf("new generation failed to settle: %#v", state)
+ }
+}
+
+func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) {
+ for _, bound := range []bool{false, true} {
+ for _, first := range []string{"a", "b"} {
+ t.Run(fmt.Sprintf("bound=%t/first=%s", bound, first), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a"}, {ID: "b"}}
+ reservations := map[string][]*weightedReservation{}
+ t.Cleanup(func() {
+ for _, group := range reservations {
+ for _, reservation := range group {
+ reservation.finish(false)
+ }
+ }
+ })
+ for i := range 256 {
+ boundID := ""
+ if bound {
+ boundID = auths[i%2].ID
+ }
+ picked, reservation := weighted.reserve("codex:model", auths, boundID)
+ if picked == nil || reservation == nil {
+ t.Fatalf("reserve %d returned pick=%v reservation=%v", i, picked, reservation)
+ }
+ reservations[picked.ID] = append(reservations[picked.ID], reservation)
+ }
+ if len(reservations["a"]) != 128 || len(reservations["b"]) != 128 {
+ t.Fatalf("unbalanced reservations: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ // A zero net pending vector does not mean all reservations settled.
+ if state.inFlight != 256 || len(state.pending) != 0 {
+ t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pending)
+ }
+ last := "b"
+ if first == "b" {
+ last = "a"
+ }
+ for _, id := range []string{first, last} {
+ for _, reservation := range reservations[id] {
+ reservation.finish(true)
+ }
+ }
+ if !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) || len(state.pending) != 0 || state.inFlight != 0 {
+ t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.current, state.pending)
+ }
+ counts := map[string]int{}
+ for range 64 {
+ picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("Pick: auth=%v err=%v", picked, errPick)
+ }
+ counts[picked.ID]++
+ }
+ if counts["a"] != 32 || counts["b"] != 32 {
+ t.Fatalf("post-commit distribution = %v, want a=32 b=32", counts)
+ }
+ })
+ }
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_model_key_test.go b/sdk/cliproxy/auth/selector_weighted_model_key_test.go
new file mode 100644
index 000000000..2f988316b
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_model_key_test.go
@@ -0,0 +1,39 @@
+package auth
+
+import (
+ "fmt"
+ "testing"
+
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+func TestWeightedSelectorsShareEquivalentModelKeys(t *testing.T) {
+ for _, firstAffinity := range []bool{false, true} {
+ for _, secondAffinity := range []bool{false, true} {
+ for _, routeOverride := range []bool{false, true} {
+ t.Run(fmt.Sprintf("affinity=%t/%t/route=%t", firstAffinity, secondAffinity, routeOverride), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ affinity := NewSessionAffinitySelector(weighted)
+ t.Cleanup(affinity.Stop)
+ auths := []*Auth{{ID: "a"}, {ID: "b"}}
+ for i, useAffinity := range []bool{firstAffinity, secondAffinity} {
+ ctx := t.Context()
+ model := []string{" Model(high) ", "model(low)"}[i]
+ if routeOverride {
+ ctx = withWeightedSelectorStateModel(ctx, weighted, model)
+ model = []string{"upstream-a", "upstream-b"}[i]
+ }
+ var selector Selector = weighted
+ if useAffinity {
+ selector = affinity
+ }
+ picked, errPick := selector.Pick(ctx, "codex", model, cliproxyexecutor.Options{}, auths)
+ if errPick != nil || picked == nil || picked.ID != auths[i].ID {
+ t.Fatalf("request %d: pick=%v err=%v, want auth %s", i, picked, errPick, auths[i].ID)
+ }
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_overload_test.go b/sdk/cliproxy/auth/selector_weighted_overload_test.go
new file mode 100644
index 000000000..47e3da067
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_overload_test.go
@@ -0,0 +1,455 @@
+package auth
+
+import (
+ "fmt"
+ "maps"
+ "net/http"
+ "strconv"
+ "sync"
+ "testing"
+
+ "github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+func TestWeightedOverloadExactCompensation(t *testing.T) {
+ for _, weights := range [][]int{{1, 1}, {3, 1}, {1, 3}, {1, 5, 1}, {2, 3, 5}, {1, 2, 3, 4}} {
+ for boundIndex := range weights {
+ t.Run(fmt.Sprintf("weights=%v/bound=%d", weights, boundIndex), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := make([]*Auth, len(weights))
+ total := 0
+ for i, weight := range weights {
+ auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
+ total += weight
+ }
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"sticky"}}}
+ key := "codex::" + CanonicalSessionID(opts.Headers, nil, nil) + "::model"
+ selector.cache.Set(key, auths[boundIndex].ID)
+ counts := map[string]int{}
+ for i := range total * 4 {
+ request := opts
+ sticky := i < weights[boundIndex]*4
+ if !sticky {
+ request = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
+ }
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", request, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("selection failed: pick=%v err=%v", picked, errPick)
+ }
+ if sticky && picked.ID != auths[boundIndex].ID {
+ t.Fatal("balancing moved an existing binding")
+ }
+ counts[picked.ID]++
+ }
+ for i, weight := range weights {
+ if counts[auths[i].ID] != weight*4 {
+ t.Errorf("total allocation = %v, want weight vector %v times 4", counts, weights)
+ }
+ }
+ for _, state := range weighted.states {
+ for id, credit := range state.current {
+ if credit != 0 {
+ t.Errorf("exact compensation left %q credit=%d", id, credit)
+ }
+ }
+ if len(state.pending) != 0 {
+ t.Errorf("pending credit leaked: %v", state.pending)
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) {
+ for _, weights := range [][]int{{1, 1, 1}, {1, 3, 1}, {1, 5, 1}, {1, 1, 5}, {1, 9, 1}, {1, 1, 9}, {1, 5, 2, 1}} {
+ t.Run(fmt.Sprint(weights), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := make([]*Auth, len(weights))
+ for i, weight := range weights {
+ auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
+ key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
+ selector.cache.Set(key, auths[0].ID)
+ counts := map[string]int{}
+ for i := range 3000 {
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
+ if errPick != nil || picked == nil || picked.ID != auths[0].ID {
+ t.Fatalf("binding changed: pick=%v err=%v", picked, errPick)
+ }
+ fresh := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
+ picked, errPick = selector.Pick(t.Context(), "codex", "model", fresh, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("fresh selection failed: pick=%v err=%v", picked, errPick)
+ }
+ if i >= 1000 {
+ counts[picked.ID]++
+ }
+ }
+ ids := make([]string, len(auths))
+ for i, auth := range auths {
+ ids[i] = auth.ID
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", ids...)
+ t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.current)
+ // Forced traffic makes global ratios infeasible; the remaining capacity
+ // should be shared by unbound peers rather than monopolized by one.
+ peerWeight := 0
+ for _, weight := range weights[1:] {
+ peerWeight += weight
+ }
+ for i, auth := range auths[1:] {
+ want := 2000 * weights[i+1] / peerWeight
+ if counts[auth.ID] < want-2 || counts[auth.ID] > want+2 {
+ t.Errorf("peer %q got %d fresh requests, want approximately %d; distribution=%v", auth.ID, counts[auth.ID], want, counts)
+ }
+ }
+ })
+ }
+}
+
+func TestWeightedOverloadFeasibleSustainedTraffic(t *testing.T) {
+ for _, weights := range [][]int{{3, 1}, {1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
+ t.Run(fmt.Sprint(weights), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := make([]*Auth, len(weights))
+ total := 0
+ for i, weight := range weights {
+ auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
+ total += weight
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"feasible"}}}
+ key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
+ selector.cache.Set(key, auths[0].ID)
+ counts := map[string]int{}
+ // Forced traffic is below A's target share, so exact aggregate ratios
+ // remain feasible without moving established bindings.
+ for i := range total * 2000 {
+ opts := cliproxyexecutor.Options{}
+ bound := i%(total*2) == 0
+ if bound {
+ opts = sticky
+ }
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
+ if errPick != nil || picked == nil || (bound && picked.ID != auths[0].ID) {
+ t.Fatalf("request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ if i >= total*1000 {
+ counts[picked.ID]++
+ }
+ }
+ for i, weight := range weights {
+ if counts[auths[i].ID] != weight*1000 {
+ t.Errorf("feasible distribution drifted: counts=%v weights=%v", counts, weights)
+ }
+ }
+ t.Logf("feasible distribution: %v", counts)
+ })
+ }
+}
+
+func TestWeightedOverloadRecoveryAfterStickyTrafficStops(t *testing.T) {
+ for _, weights := range [][]int{{1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
+ t.Run(fmt.Sprint(weights), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := make([]*Auth, len(weights))
+ total := 0
+ for i, weight := range weights {
+ auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
+ total += weight
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
+ key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
+ selector.cache.Set(key, auths[0].ID)
+ for range 3000 {
+ if picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths); errPick != nil || picked == nil || picked.ID != auths[0].ID {
+ t.Fatalf("sticky selection failed: pick=%v err=%v", picked, errPick)
+ }
+ if _, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
+ t.Fatal(errPick)
+ }
+ }
+ counts := map[string]int{}
+ for i := range 1000 + total*100 {
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("recovery selection failed: pick=%v err=%v", picked, errPick)
+ }
+ if i >= 1000 {
+ counts[picked.ID]++
+ }
+ }
+ for i, weight := range weights {
+ if counts[auths[i].ID] != weight*100 {
+ t.Errorf("allocation did not recover after sticky traffic stopped: counts=%v weights=%v", counts, weights)
+ }
+ }
+ t.Logf("recovered distribution: %v", counts)
+ })
+ }
+}
+
+func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}}
+ const key = "codex:model"
+ const inFlight = 200
+ reservations := make([]*weightedReservation, inFlight)
+ t.Cleanup(func() {
+ for _, reservation := range reservations {
+ reservation.finish(false)
+ }
+ })
+ for i := range reservations {
+ picked, reservation := weighted.reserve(key, auths, "a")
+ if picked == nil || picked.ID != "a" {
+ t.Fatal("pending sticky request lost its binding")
+ }
+ reservations[i] = reservation
+ }
+ counts := map[string]int{}
+ for i := range 3000 {
+ // Maintain a fixed backlog without sleeps or timing-dependent coordination.
+ reservations[i%inFlight].finish(true)
+ _, reservations[i%inFlight] = weighted.reserve(key, auths, "a")
+ picked, reservation := weighted.reserve(key, auths, "")
+ reservation.finish(true)
+ if i >= 1000 {
+ counts[picked.ID]++
+ }
+ }
+ if counts["c"] == 0 {
+ t.Fatalf("speculative sticky traffic starved c: %v", counts)
+ }
+ for _, reservation := range reservations {
+ reservation.finish(true)
+ }
+ state := weightedStateForTest(t, weighted, key, "a", "b", "c")
+ if len(state.pending) != 0 {
+ t.Fatalf("settled reservations leaked credit: %v", state.pending)
+ }
+}
+
+func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ const key = "codex:model"
+ for range 1000 {
+ _, reservation := weighted.reserve(key, auths, "b")
+ reservation.finish(true)
+ }
+ state := weightedStateForTest(t, weighted, key, "a", "b")
+ before := maps.Clone(state.current)
+ _, first := weighted.reserve(key, auths, "")
+ _, second := weighted.reserve(key, auths, "")
+ t.Cleanup(func() { first.finish(false); second.finish(false) })
+ // Two pending cold selections make room for a bound delta, but neither
+ // repayment is real yet. Its later commit must recheck the committed cap.
+ picked, bound := weighted.reserve(key, auths, "b")
+ if picked == nil || picked.ID != "b" || bound == nil {
+ t.Fatal("expected bound reservation funded by pending repayments")
+ }
+ t.Cleanup(func() { bound.finish(false) })
+ first.finish(false)
+ second.finish(false)
+ bound.finish(true)
+ bound.finish(true)
+ bound.finish(false)
+ if !maps.Equal(state.current, before) || len(state.pending) != 0 {
+ t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.current, state.pending)
+ }
+ // An over-limit bound selection succeeds without reserving any credit.
+ picked, skipped := weighted.reserve(key, auths, "b")
+ if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.current, before) || len(state.pending) != 0 {
+ t.Fatal("saturated bound selection should preserve binding and all credits")
+ }
+}
+
+func TestWeightedOverloadMultipleBusyBindings(t *testing.T) {
+ for _, busyPerFresh := range []int{1, 3, 10} {
+ t.Run(fmt.Sprint(busyPerFresh), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{
+ {ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}, {ID: "d"},
+ }
+ boundOpts := make([]cliproxyexecutor.Options, 2)
+ for i, id := range []string{"a", "d"} {
+ boundOpts[i].Headers = http.Header{"Session_id": {"busy-" + id}}
+ sessionID := CanonicalSessionID(boundOpts[i].Headers, nil, nil)
+ selector.cache.Set("codex::"+sessionID+"::model", id)
+ }
+ windowCounts := map[string]int{}
+ for i := range 1400 {
+ for j := range busyPerFresh {
+ index := (i + j) % len(boundOpts)
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", boundOpts[index], auths)
+ if errPick != nil || picked == nil || picked.ID != []string{"a", "d"}[index] {
+ t.Fatalf("busy binding changed: auth=%v err=%v", picked, errPick)
+ }
+ }
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
+ if errPick != nil || picked == nil {
+ t.Fatalf("fresh pick failed: auth=%v err=%v", picked, errPick)
+ }
+ if i >= 1000 {
+ windowCounts[picked.ID]++
+ if (i+1)%100 == 0 {
+ if windowCounts["b"] == 0 || windowCounts["c"] == 0 {
+ t.Fatalf("busy=%d unbound peer starved in a 100-request window: %v", busyPerFresh, windowCounts)
+ }
+ clear(windowCounts)
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) {
+ for _, credit := range []map[string]int64{
+ {"a": -222, "b": 111, "c": 111},
+ {"a": -112, "b": 222, "c": -110},
+ } {
+ state := &smoothWeightedState{current: maps.Clone(credit)}
+ state.charge(map[string]int64{"a": 1, "b": 5, "c": 1}, "a", true)
+ if !maps.Equal(state.current, credit) {
+ t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.current, credit)
+ }
+ }
+}
+
+func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ for _, pinned := range []bool{false, true} {
+ t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ manager := NewManager(nil, selector, nil)
+ const provider, model = "sticky-overload", "sticky-overload-model"
+ manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
+ aID, bID, cID := t.Name()+"-a", t.Name()+"-b", t.Name()+"-c"
+ for _, auth := range []*Auth{
+ {ID: aID, Provider: provider, Status: StatusActive},
+ {ID: bID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "5"}},
+ {ID: cID, Provider: provider, Status: StatusActive},
+ } {
+ registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
+ t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
+ if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
+ t.Fatal(errRegister)
+ }
+ }
+ namespace := provider
+ if mixed {
+ namespace = "mixed"
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}, Metadata: map[string]any{}}
+ key := namespace + "::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::" + model
+ selector.cache.Set(key, aID)
+ if pinned {
+ sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = aID
+ }
+ counts := map[string]int{}
+ for i := range 1600 {
+ for _, bound := range []bool{true, false} {
+ opts := sticky
+ if !bound {
+ opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}}
+ }
+ var auth *Auth
+ var errPick error
+ if mixed {
+ auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
+ } else {
+ auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
+ }
+ if errPick != nil || auth == nil || (bound && auth.ID != aID) {
+ t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
+ }
+ if !bound && i >= 1000 {
+ counts[auth.ID]++
+ }
+ }
+ }
+ if counts[bID] != 500 || counts[cID] != 100 {
+ t.Fatalf("manager overload allocation = %v, want b=500 c=100", counts)
+ }
+ if binding, ok := selector.cache.Get(key); !ok || binding != aID {
+ t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok)
+ }
+ state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID)
+ if len(state.pending) != 0 {
+ t.Fatalf("manager left speculative credit: %v", state.pending)
+ }
+ })
+ }
+ }
+}
+
+func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
+ headers := http.Header{"Session_id": {"sticky-concurrent"}}
+ key := "codex::" + CanonicalSessionID(headers, nil, nil) + "::model"
+ selector.cache.Set(key, "b")
+ type selection struct {
+ id string
+ err error
+ sticky bool
+ }
+ results := make(chan selection, 64)
+ start := make(chan struct{})
+ var done sync.WaitGroup
+ for i := range 64 {
+ done.Add(1)
+ go func() {
+ defer done.Done()
+ isSticky := i < 8
+ opts := cliproxyexecutor.Options{Headers: headers}
+ if !isSticky {
+ opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
+ }
+ <-start
+ picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
+ id := ""
+ if picked != nil {
+ id = picked.ID
+ }
+ results <- selection{id: id, err: errPick, sticky: isSticky}
+ }()
+ }
+ close(start)
+ done.Wait()
+ close(results)
+ counts := map[string]int{}
+ for result := range results {
+ if result.err != nil || result.id == "" || (result.sticky && result.id != "b") {
+ t.Fatalf("invalid concurrent result: %+v", result)
+ }
+ counts[result.id]++
+ }
+ state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
+ // All 64 calls share one pool. Accounting must match actual accepted picks,
+ // regardless of whether the final arrivals happened to be sticky requests.
+ if state.current["a"] != int64(64*3-counts["a"]*4) || state.current["b"] != int64(64-counts["b"]*4) || len(state.pending) != 0 {
+ t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.current, state.pending)
+ }
+ if bound, ok := selector.cache.Get(key); !ok || bound != "b" {
+ t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok)
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_pool_test.go b/sdk/cliproxy/auth/selector_weighted_pool_test.go
new file mode 100644
index 000000000..96dae7581
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_pool_test.go
@@ -0,0 +1,283 @@
+package auth
+
+import (
+ "fmt"
+ "maps"
+ "math/rand/v2"
+ "net/http"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
+ cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
+)
+
+// weightedStateForTest checks that the requested accounting pool was created.
+func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *smoothWeightedState {
+ t.Helper()
+ weights := make(map[string]int64, len(ids))
+ for _, id := range ids {
+ weights[id] = 1
+ }
+ state := selector.states[weightedPoolStateKey(route, weights)]
+ if state == nil {
+ t.Fatalf("missing weighted state for %q and %v", route, ids)
+ }
+ return state
+}
+
+func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}, {ID: "c"}}
+ key := "codex:model"
+ _, pendingFull := weighted.reserve(key, auths, "b")
+ t.Cleanup(func() { pendingFull.finish(false) })
+ _, pendingSubset := weighted.reserve(key, auths[:2], "b")
+ t.Cleanup(func() { pendingSubset.finish(false) })
+ pendingSubset.finish(true)
+ subset := weightedStateForTest(t, weighted, key, "a", "b")
+ full := weightedStateForTest(t, weighted, key, "a", "b", "c")
+ if !maps.Equal(subset.current, map[string]int64{"a": 3, "b": -3}) || len(full.current) != 0 {
+ t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.current, full.current)
+ }
+ if !maps.Equal(full.pending, map[string]int64{"a": 3, "b": -4, "c": 1}) {
+ t.Fatalf("another pool consumed pending credit: %v", full.pending)
+ }
+ pendingFull.finish(false)
+ for range 100 {
+ _, reservation := weighted.reserve(key, auths, "b")
+ reservation.finish(true)
+ }
+ // A returning pool resumes its debt, independent of full-pool traffic.
+ for range 3 {
+ picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "")
+ if picked == nil || picked.ID != "a" {
+ t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.current)
+ }
+ reservation.finish(true)
+ }
+ if !maps.Equal(subset.current, map[string]int64{"a": 0, "b": 0}) || len(subset.pending) != 0 || len(full.pending) != 0 {
+ t.Fatalf("returning subset failed to balance: %#v", subset)
+ }
+}
+
+func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) {
+ shared := &WeightedRoundRobinSelector{}
+ auths := []*Auth{
+ {ID: "a", Attributes: map[string]string{AttributeWeight: "1"}},
+ {ID: "b", Attributes: map[string]string{AttributeWeight: "3"}},
+ {ID: "c", Attributes: map[string]string{AttributeWeight: "2"}},
+ {ID: "d", Attributes: map[string]string{AttributeWeight: "5"}},
+ }
+ var independent [16]WeightedRoundRobinSelector
+ rng := rand.New(rand.NewPCG(17, 29))
+ for step := range 5000 {
+ mask := 1 + rng.IntN(15)
+ var pool []*Auth
+ for i, auth := range auths {
+ if mask&(1< weighted.maxKeys || old.state == current.state {
+ t.Fatal("pool eviction failed to bound or replace state")
+ }
+ old.finish(true)
+ if len(current.state.current) != 0 || !maps.Equal(current.state.pending, map[string]int64{"a": -1, "b": 1}) {
+ t.Fatal("finishing an evicted reservation affected the new pool")
+ }
+ current.finish(true)
+}
+
+func TestManagerWeightedPriorityPoolIsolation(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ for _, pinned := range []bool{false, true} {
+ t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ manager := NewManager(nil, selector, nil)
+ const provider, model = "pool-isolation", "pool-model"
+ manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
+ auths := []*Auth{
+ {ID: t.Name() + "-a", Attributes: map[string]string{"priority": "0"}},
+ {ID: t.Name() + "-b", Attributes: map[string]string{"priority": "1", AttributeWeight: "3"}},
+ {ID: t.Name() + "-c", Attributes: map[string]string{"priority": "1", AttributeWeight: "1"}},
+ }
+ for _, auth := range auths {
+ auth.Provider, auth.Status = provider, StatusActive
+ registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
+ t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
+ if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
+ t.Fatal(errRegister)
+ }
+ }
+ namespace := provider
+ if mixed {
+ namespace = "mixed"
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"priority-bound"}}, Metadata: map[string]any{}}
+ id := CanonicalSessionID(sticky.Headers, nil, nil)
+ selector.cache.Set(namespace+"::"+id+"::"+model, auths[0].ID)
+ if pinned {
+ sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = auths[0].ID
+ }
+ counts := map[string]int{}
+ for i := range 1200 {
+ for _, bound := range []bool{true, false} {
+ opts := sticky
+ if !bound {
+ opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
+ }
+ var auth *Auth
+ var errPick error
+ if mixed {
+ auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
+ } else {
+ auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
+ }
+ if errPick != nil || auth == nil || (bound && auth.ID != auths[0].ID) || (!bound && auth.ID == auths[0].ID) {
+ t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
+ }
+ if !bound && i >= 200 {
+ counts[auth.ID]++
+ }
+ }
+ }
+ if counts[auths[1].ID] != 750 || counts[auths[2].ID] != 250 {
+ t.Fatalf("manager cross-priority distribution = %v, want 750:250", counts)
+ }
+ })
+ }
+ }
+}
+
+func TestWeightedSessionAffinityPoolIsolation(t *testing.T) {
+ for _, reason := range []string{"priority", "websocket", "retry-exclusion", "cooldown"} {
+ for _, weights := range [][2]int{{1, 1}, {3, 1}, {1, 3}, {5, 2}} {
+ t.Run(fmt.Sprintf("%s/%d:%d", reason, weights[0], weights[1]), func(t *testing.T) {
+ weighted := &WeightedRoundRobinSelector{}
+ selector := NewSessionAffinitySelector(weighted)
+ t.Cleanup(selector.Stop)
+ auths := []*Auth{
+ {ID: "a", Attributes: map[string]string{"priority": "0"}},
+ {ID: "b", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[0])}},
+ {ID: "c", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[1])}},
+ }
+ ctx := t.Context()
+ switch reason {
+ case "priority":
+ auths[1].Attributes["priority"] = "1"
+ auths[2].Attributes["priority"] = "1"
+ case "websocket":
+ ctx = cliproxyexecutor.WithDownstreamWebsocket(ctx)
+ auths[1].Attributes["websockets"] = "true"
+ auths[2].Attributes["websockets"] = "true"
+ }
+ sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound-a"}}}
+ id := CanonicalSessionID(sticky.Headers, nil, nil)
+ key := "codex::" + id + "::model"
+ selector.cache.Set(key, "a")
+ const warmup, measured = 1000, 1400
+ counts := map[string]int{}
+ for i := range warmup + measured {
+ picked, errPick := selector.Pick(ctx, "codex", "model", sticky, auths)
+ if errPick != nil || picked == nil || picked.ID != "a" {
+ t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ freshAuths := auths
+ if reason == "retry-exclusion" {
+ freshAuths = auths[1:]
+ } else if reason == "cooldown" {
+ auths[0].Unavailable = true
+ auths[0].NextRetryAfter = time.Now().Add(time.Hour)
+ }
+ opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
+ picked, errPick = selector.Pick(ctx, "codex", "model", opts, freshAuths)
+ if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
+ t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
+ }
+ if i >= warmup {
+ counts[picked.ID]++
+ }
+ auths[0].Unavailable = false
+ auths[0].NextRetryAfter = time.Time{}
+ }
+ wantB := measured * weights[0] / (weights[0] + weights[1])
+ if counts["b"] != wantB || counts["c"] != measured-wantB {
+ t.Fatalf("cross-pool traffic distorted cold selections: %v, want b=%d c=%d", counts, wantB, measured-wantB)
+ }
+ if bound, ok := selector.cache.Get(key); !ok || bound != "a" {
+ t.Fatalf("binding moved: %q, found=%t", bound, ok)
+ }
+ for _, state := range weighted.states {
+ if len(state.pending) != 0 {
+ t.Fatalf("pending credits leaked: %v", state.pending)
+ }
+ }
+ })
+ }
+ }
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_test.go b/sdk/cliproxy/auth/selector_weighted_test.go
new file mode 100644
index 000000000..3d9872085
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_test.go
@@ -0,0 +1,81 @@
+package auth
+
+import (
+ "maps"
+ "testing"
+)
+
+func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ current map[string]int64
+ weights map[string]int64
+ authID string
+ want map[string]int64
+ }{
+ {
+ name: "smaller pool preserves leading difference",
+ current: map[string]int64{"a": -96, "b": 96, "c": 95},
+ weights: map[string]int64{"b": 1, "c": 1},
+ authID: "b",
+ want: map[string]int64{"a": -96, "b": 63, "c": 64},
+ },
+ {
+ name: "negative credits shift together",
+ current: map[string]int64{"a": 96, "b": -64, "c": -63},
+ weights: map[string]int64{"b": 1, "c": 1},
+ authID: "b",
+ want: map[string]int64{"a": 96, "b": -64, "c": -61},
+ },
+ {
+ name: "excessive spread clips debt without tying leaders",
+ current: map[string]int64{"a": -96, "b": 96, "c": 95},
+ weights: map[string]int64{"a": 1, "b": 1, "c": 1},
+ authID: "a",
+ want: map[string]int64{"a": -96, "b": 96, "c": 95},
+ },
+ {
+ name: "bounded credits do not shift",
+ current: map[string]int64{"b": 0, "c": 0},
+ weights: map[string]int64{"b": 1, "c": 1},
+ authID: "b",
+ want: map[string]int64{"b": -1, "c": 1},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ state := &smoothWeightedState{current: tc.current}
+ state.charge(tc.weights, tc.authID, false)
+ if !maps.Equal(state.current, tc.want) {
+ t.Fatalf("rebased credits = %v, want %v", state.current, tc.want)
+ }
+ })
+ }
+}
+
+func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) {
+ key := "codex:model"
+ state := &smoothWeightedState{
+ current: map[string]int64{"a": -94, "b": 94, "c": 93},
+ weights: map[string]int64{"a": 1, "b": 1, "c": 1},
+ pending: make(map[string]int64),
+ }
+ weighted := &WeightedRoundRobinSelector{states: map[string]*smoothWeightedState{weightedPoolStateKey(key, state.weights): state}}
+ auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}}
+ _, rejected := weighted.reserve(key, auths, "a")
+ t.Cleanup(func() { rejected.finish(false) })
+ _, accepted := weighted.reserve(key, auths, "b")
+ t.Cleanup(func() { accepted.finish(false) })
+ accepted.finish(true)
+ want := map[string]int64{"a": -93, "b": 92, "c": 94}
+ if !maps.Equal(state.current, want) {
+ t.Fatalf("committed credits = %v, want %v", state.current, want)
+ }
+ if !maps.Equal(state.pending, map[string]int64{"a": -2, "b": 1, "c": 1}) {
+ t.Fatalf("commit changed another request's reservation: %v", state.pending)
+ }
+ rejected.finish(false)
+ rejected.finish(true)
+ if !maps.Equal(state.current, want) || len(state.pending) != 0 {
+ t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.current, state.pending)
+ }
+}
From e479ab12538cb027ea558bbfa070fef1ca9d187c Mon Sep 17 00:00:00 2001
From: hkfires <10558748+hkfires@users.noreply.github.com>
Date: Mon, 9 Nov 2026 13:48:44 +0800
Subject: [PATCH 3/3] perf(auth): compact weighted round-robin pools
Replace string pool keys and map-based credit state with SHA-256
digest keys and sorted slice members holding weight, current, and
pending credits in a single allocation. Reservations now retain
only the member index instead of a weights map and auth ID.
Verify full pool identity on digest hits and chain collisions so
unrelated candidate pools never share credits or weights. Update
tests to access pool credits through test helpers.
---
.../auth/conductor_weighted_pinned_test.go | 16 +-
sdk/cliproxy/auth/selector.go | 17 +-
sdk/cliproxy/auth/selector_test.go | 8 +-
sdk/cliproxy/auth/selector_weighted.go | 208 ++++++++++++------
.../auth/selector_weighted_affinity_test.go | 58 ++---
.../selector_weighted_commit_order_test.go | 18 +-
.../auth/selector_weighted_memory_test.go | 71 ++++++
.../auth/selector_weighted_overload_test.go | 36 +--
.../auth/selector_weighted_pool_test.go | 28 ++-
.../auth/selector_weighted_storage_test.go | 88 ++++++++
sdk/cliproxy/auth/selector_weighted_test.go | 34 ++-
11 files changed, 402 insertions(+), 180 deletions(-)
create mode 100644 sdk/cliproxy/auth/selector_weighted_memory_test.go
create mode 100644 sdk/cliproxy/auth/selector_weighted_storage_test.go
diff --git a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go
index 6f52dd303..8768a2a8a 100644
--- a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go
+++ b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go
@@ -104,7 +104,7 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
}
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
- if state == nil || !maps.Equal(state.current, map[string]int64{aID: 12, bID: -12}) || len(state.pending) != 0 {
+ if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 {
t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state)
}
counts := map[string]int{}
@@ -118,8 +118,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
if counts[aID] != 12 || counts[bID] != 0 {
t.Fatalf("fresh distribution = %v, want a=12 b=0", counts)
}
- if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
- t.Fatalf("compensation did not balance: current=%v pending=%v", state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if mode == "sticky" || mode == "cold" {
if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID {
@@ -135,8 +135,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
if auth, errPick := pick(opts); errPick == nil || auth != nil {
t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick)
}
- if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
- t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest())
}
}
// A valid lower-priority pin still wins without charging another tier.
@@ -148,11 +148,11 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID {
t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick)
}
- if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 {
- t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID)
- if !maps.Equal(lowerState.current, map[string]int64{lowerID: 0}) || len(lowerState.pending) != 0 {
+ if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 {
t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState)
}
})
diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go
index 54e16ebc9..af11116b9 100644
--- a/sdk/cliproxy/auth/selector.go
+++ b/sdk/cliproxy/auth/selector.go
@@ -39,17 +39,15 @@ type RoundRobinSelector struct {
// WeightedRoundRobinSelector provides smooth weighted round-robin selection.
type WeightedRoundRobinSelector struct {
- mu sync.Mutex
- states map[string]*smoothWeightedState
- maxKeys int
+ mu sync.Mutex
+ states map[[32]byte]*weightedPoolState
+ stateCount int
+ maxKeys int
}
type smoothWeightedState struct {
- current map[string]int64
- weights map[string]int64
- pending map[string]int64
- inFlight uint64
- generation uint64
+ current map[string]int64
+ weights map[string]int64
}
type weightedSelectorStateModelKey struct{}
@@ -723,9 +721,6 @@ const maxSmoothWeightedStateEntries = 1024
func (s *smoothWeightedState) prepare(weights map[string]int64) {
if s.current == nil || weightsConfigChanged(s.weights, weights) {
s.current = make(map[string]int64, len(weights))
- s.pending = make(map[string]int64, len(weights))
- s.inFlight = 0
- s.generation++
}
if s.weights == nil {
s.weights = make(map[string]int64, len(weights))
diff --git a/sdk/cliproxy/auth/selector_test.go b/sdk/cliproxy/auth/selector_test.go
index b6654cfcf..37678684b 100644
--- a/sdk/cliproxy/auth/selector_test.go
+++ b/sdk/cliproxy/auth/selector_test.go
@@ -208,11 +208,11 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) {
func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) {
t.Parallel()
- state := &smoothWeightedState{current: map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}}
- state.charge(map[string]int64{"a": 1, "b": 1}, "a", false)
+ state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64})
+ state.charge(0, false)
limit := maxSmoothWeightedCreditRounds * 2
- if state.current["a"] != limit || state.current["b"] != -limit {
- t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.current)
+ if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit {
+ t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest())
}
}
diff --git a/sdk/cliproxy/auth/selector_weighted.go b/sdk/cliproxy/auth/selector_weighted.go
index ed92b5bb0..57852f6b2 100644
--- a/sdk/cliproxy/auth/selector_weighted.go
+++ b/sdk/cliproxy/auth/selector_weighted.go
@@ -2,10 +2,10 @@ package auth
import (
"context"
+ "crypto/sha256"
+ "encoding/binary"
"math"
"sort"
- "strconv"
- "strings"
"time"
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
@@ -19,10 +19,9 @@ type weightedAccountingPoolKey struct{}
// Releasing one request must never undo accounting committed by another request.
type weightedReservation struct {
selector *WeightedRoundRobinSelector
- state *smoothWeightedState
+ state *weightedPoolState
generation uint64
- weights map[string]int64
- authID string
+ member int
total int64
bound bool
finished bool
@@ -95,55 +94,120 @@ func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Contex
return picked, nil
}
-// weightedPoolStateKey isolates credits for actual competing sets. Reusing a
-// provider/model accumulator across different sets grants credit for requests
-// those sets cannot serve, which can permanently starve their smaller weights.
-// Length-prefixed, sorted IDs avoid collisions and keep input order irrelevant.
-// Weights stay out of the key so prepare can invalidate pending reservations on
-// configuration changes. Returning to a prior set resumes that set's credits.
-func weightedPoolStateKey(route string, weights map[string]int64) string {
+// weightedPoolMember stores each ID once, with parallel accounting values in a
+// single allocation. Members are sorted by ID; reservations retain only an index.
+type weightedPoolMember struct {
+ id string
+ weight int64
+ current int64
+ pending int64
+}
+
+// weightedPoolState is separate from the scheduler's transient-subset accumulator:
+// each selector pool has fixed membership and shares its weights across reservations.
+type weightedPoolState struct {
+ route string
+ members []weightedPoolMember
+ inFlight uint64
+ generation uint64
+ next *weightedPoolState // Exact-identity collision chain for the digest key.
+}
+
+func sortedWeightedIDs(weights map[string]int64) []string {
ids := make([]string, 0, len(weights))
for id := range weights {
ids = append(ids, id)
}
sort.Strings(ids)
- var key strings.Builder
- key.WriteString(strconv.Itoa(len(route)))
- key.WriteByte(':')
- key.WriteString(route)
+ return ids
+}
+
+// weightedPoolStateKey isolates competing sets without retaining concatenated IDs.
+// Weights stay out of the key so configuration changes invalidate reservations.
+func weightedPoolStateKey(route string, weights map[string]int64) [32]byte {
+ return weightedPoolDigest(route, sortedWeightedIDs(weights))
+}
+
+func weightedPoolDigest(route string, ids []string) [32]byte {
+ size := 8 + len(route)
for _, id := range ids {
- key.WriteString(strconv.Itoa(len(id)))
- key.WriteByte(':')
- key.WriteString(id)
+ size += 8 + len(id)
+ }
+ identity := make([]byte, 0, size)
+ identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route)))
+ identity = append(identity, route...)
+ for _, id := range ids {
+ identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id)))
+ identity = append(identity, id...)
+ }
+ return sha256.Sum256(identity)
+}
+
+// matches verifies the full identity even on a digest hit. Hash collisions must
+// never combine the quota or configured weights of unrelated candidate pools.
+func (s *weightedPoolState) matches(route string, ids []string) bool {
+ if s.route != route || len(s.members) != len(ids) {
+ return false
+ }
+ for i, id := range ids {
+ if s.members[i].id != id {
+ return false
+ }
+ }
+ return true
+}
+
+func (s *weightedPoolState) prepare(weights map[string]int64) {
+ for _, member := range s.members {
+ if member.weight != weights[member.id] {
+ s.generation++
+ s.inFlight = 0
+ for i := range s.members {
+ s.members[i].weight = weights[s.members[i].id]
+ s.members[i].current = 0
+ s.members[i].pending = 0
+ }
+ return
+ }
}
- return key.String()
}
// reserve selects and reserves credit atomically. A non-empty authID preserves a
// sticky binding; otherwise committed and pending credits both guide selection.
-func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, authID string) (*Auth, *weightedReservation) {
+func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) {
weights := authWeightVector(available)
- key = weightedPoolStateKey(key, weights)
+ ids := sortedWeightedIDs(weights)
+ key := weightedPoolDigest(route, ids)
s.mu.Lock()
defer s.mu.Unlock()
if s.states == nil {
- s.states = make(map[string]*smoothWeightedState)
- }
- limit := s.maxKeys
- if limit <= 0 {
- limit = 4096
- }
- if _, ok := s.states[key]; !ok && len(s.states) >= limit {
- s.states = make(map[string]*smoothWeightedState)
+ s.states = make(map[[32]byte]*weightedPoolState)
}
state := s.states[key]
- if state == nil {
- state = &smoothWeightedState{}
- s.states[key] = state
+ for state != nil && !state.matches(route, ids) {
+ state = state.next
+ }
+ if state == nil {
+ limit := s.maxKeys
+ if limit <= 0 {
+ limit = 4096
+ }
+ if s.stateCount >= limit {
+ s.states = make(map[[32]byte]*weightedPoolState)
+ s.stateCount = 0
+ }
+ state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]}
+ for i, id := range ids {
+ state.members[i] = weightedPoolMember{id: id, weight: weights[id]}
+ }
+ s.states[key] = state
+ s.stateCount++
+ } else {
+ state.prepare(weights)
}
- state.prepare(weights)
var picked *Auth
var pickedCurrent, total int64
+ pickedMember := -1
for _, auth := range available {
if auth == nil || weights[auth.ID] <= 0 {
continue
@@ -152,10 +216,12 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth
if authID != "" && auth.ID != authID {
continue
}
- current := saturatingAddInt64(state.current[auth.ID], state.pending[auth.ID])
- current = saturatingAddInt64(current, weights[auth.ID])
+ index := sort.SearchStrings(ids, auth.ID)
+ member := &state.members[index]
+ current := saturatingAddInt64(member.current, member.pending)
+ current = saturatingAddInt64(current, member.weight)
if picked == nil || current > pickedCurrent {
- picked, pickedCurrent = auth, current
+ picked, pickedCurrent, pickedMember = auth, current, index
}
}
if picked == nil {
@@ -163,14 +229,14 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth
}
// Bound traffic beyond the compensation window still keeps its credential,
// but must not create speculative credit that can starve concurrent picks.
- if authID != "" && !state.boundChargeFits(weights, picked.ID, total) {
+ if authID != "" && !state.boundChargeFits(pickedMember, total) {
return picked, nil
}
- state.adjustPending(weights, picked.ID, total, false)
+ state.adjustPending(pickedMember, total, false)
state.inFlight++
return picked, &weightedReservation{
selector: s, state: state, generation: state.generation,
- weights: weights, authID: picked.ID, total: total, bound: authID != "",
+ member: pickedMember, total: total, bound: authID != "",
}
}
@@ -188,31 +254,29 @@ func (r *weightedReservation) finish(committed bool) {
if r.state.generation != r.generation {
return
}
- r.state.adjustPending(r.weights, r.authID, r.total, true)
+ r.state.adjustPending(r.member, r.total, true)
r.state.inFlight--
if committed {
- r.state.charge(r.weights, r.authID, r.bound)
+ r.state.charge(r.member, r.bound)
} else if r.state.inFlight == 0 {
// A final rollback can remove the offset to already committed credit.
- r.state.boundCredits(r.weights, r.total)
+ r.state.boundCredits(r.total)
}
}
// adjustPending must be called with the selector mutex held. Pending deltas are
// not clamped, so releasing them cannot lose credit at the compensation bound.
-func (s *smoothWeightedState) adjustPending(weights map[string]int64, authID string, total int64, release bool) {
- for id, weight := range weights {
- delta := weight
- if id == authID {
+func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) {
+ for i := range s.members {
+ member := &s.members[i]
+ delta := member.weight
+ if i == selected {
delta -= total
}
if release {
delta = -delta
}
- s.pending[id] = saturatingAddInt64(s.pending[id], delta)
- if s.pending[id] == 0 {
- delete(s.pending, id)
- }
+ member.pending = saturatingAddInt64(member.pending, delta)
}
}
@@ -222,17 +286,17 @@ const maxSmoothWeightedCreditRounds int64 = 32
// boundChargeFits admits the whole sticky delta or none. Both reservation and
// commit include outstanding deltas: committed credit alone can transiently
// exceed the cap while successful reservations settle in a different order.
-func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID string, totalWeight int64) bool {
+func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool {
limit := int64(math.MaxInt64)
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
limit = totalWeight * maxSmoothWeightedCreditRounds
}
- for id, weight := range weights {
- delta := weight
- if id == authID {
+ for i, member := range s.members {
+ delta := member.weight
+ if i == selected {
delta -= totalWeight
}
- current := saturatingAddInt64(s.current[id], s.pending[id])
+ current := saturatingAddInt64(member.current, member.pending)
next := saturatingAddInt64(current, delta)
if next < -limit || next > limit {
return false
@@ -241,36 +305,38 @@ func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID s
return true
}
-func (s *smoothWeightedState) charge(weights map[string]int64, authID string, bound bool) {
+func (s *weightedPoolState) charge(selected int, bound bool) {
var totalWeight int64
- for _, weight := range weights {
- totalWeight = saturatingAddInt64(totalWeight, weight)
+ for _, member := range s.members {
+ totalWeight = saturatingAddInt64(totalWeight, member.weight)
}
- if !bound || s.boundChargeFits(weights, authID, totalWeight) {
+ if !bound || s.boundChargeFits(selected, totalWeight) {
// At the cap, forced traffic cannot erase a small peer's earnings.
// Pending repayments still count until they actually roll back.
- for id, weight := range weights {
- s.current[id] = saturatingAddInt64(s.current[id], weight)
+ for i := range s.members {
+ member := &s.members[i]
+ member.current = saturatingAddInt64(member.current, member.weight)
}
- s.current[authID] = saturatingAddInt64(s.current[authID], -totalWeight)
+ member := &s.members[selected]
+ member.current = saturatingAddInt64(member.current, -totalWeight)
}
if s.inFlight == 0 {
- s.boundCredits(weights, totalWeight)
+ s.boundCredits(totalWeight)
}
}
// boundCredits runs only once the pool settles. Net pending deltas can be zero
// even with live reservations, so the in-flight count, not len(pending), decides
// when clipping cannot irreversibly discard another reservation's offset.
-func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight int64) {
+func (s *weightedPoolState) boundCredits(totalWeight int64) {
limit := int64(math.MaxInt64)
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
limit = totalWeight * maxSmoothWeightedCreditRounds
}
var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64
- for id := range weights {
- minCredit = min(minCredit, s.current[id])
- maxCredit = max(maxCredit, s.current[id])
+ for _, member := range s.members {
+ minCredit = min(minCredit, member.current)
+ maxCredit = max(maxCredit, member.current)
}
// A common offset keeps leading credits distinct at the compensation bound.
// Independent clamps can repeatedly tie the leaders and starve the later
@@ -285,8 +351,8 @@ func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight
}
// If the full spread cannot fit, cap only the trailing debt after rebasing
// the leader to the upper bound. Do not flatten leading candidates' credits.
- // Temporarily excluded credentials retain their credit for later selections.
- for id := range weights {
- s.current[id] = max(-limit, saturatingAddInt64(s.current[id], -offset))
+ for i := range s.members {
+ member := &s.members[i]
+ member.current = max(-limit, saturatingAddInt64(member.current, -offset))
}
}
diff --git a/sdk/cliproxy/auth/selector_weighted_affinity_test.go b/sdk/cliproxy/auth/selector_weighted_affinity_test.go
index 12f3c2fc7..5736e0c82 100644
--- a/sdk/cliproxy/auth/selector_weighted_affinity_test.go
+++ b/sdk/cliproxy/auth/selector_weighted_affinity_test.go
@@ -52,8 +52,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) {
}
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
- if state.current["a"] != 12 || state.current["b"] != -12 || len(state.pending) != 0 {
- t.Fatalf("sticky credits: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
for i := range 12 {
// Requests without an identity and cold sessions share the same credits.
@@ -66,8 +66,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) {
t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick)
}
}
- if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 {
- t.Fatalf("compensated credits: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if binding, ok := selector.cache.Get(key); !ok || binding != "b" {
t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok)
@@ -116,8 +116,8 @@ func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing.
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok)
}
- if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pending) != 0 {
- t.Fatalf("pending credits leaked: %v", state.pending)
+ if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 {
+ t.Fatalf("pending credits leaked: %v", state.pendingForTest())
}
})
}
@@ -137,8 +137,8 @@ func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) {
t.Fatalf("pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
- if i == 0 && (state.current["a"] != -1 || state.current["b"] != 1) {
- t.Fatalf("first selection charged more than once: %v", state.current)
+ if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) {
+ t.Fatalf("first selection charged more than once: %v", state.currentForTest())
}
counts[picked.ID]++
}
@@ -165,16 +165,16 @@ func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) {
limit := maxSmoothWeightedCreditRounds * 4
// A bound charge is admitted as a whole, stopping within one delta of the cap.
wantCredit := limit / 3 * 3
- if state.current["a"] != wantCredit || state.current["b"] != -wantCredit {
- t.Fatalf("bounded credits = %v, want +/- %d", state.current, wantCredit)
+ if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit {
+ t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit)
}
for range wantCredit {
if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
t.Fatal(errPick)
}
}
- if state.current["a"] != 0 || state.current["b"] != 0 {
- t.Fatalf("compensation did not finish: %v", state.current)
+ if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 {
+ t.Fatalf("compensation did not finish: %v", state.currentForTest())
}
}
@@ -215,8 +215,8 @@ func TestWeightedSessionAffinityLCPCredits(t *testing.T) {
t.Fatal("expected LCP compaction")
}
state := weightedStateForTest(t, weighted, "openai:model", "a", "b")
- if state.current["a"] != 3 || state.current["b"] != -3 || len(state.pending) != 0 {
- t.Fatalf("LCP credits: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
})
}
@@ -238,8 +238,8 @@ func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) {
rejected.finish(true)
accepted.finish(false)
limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3
- if state.current["a"] != limit-1 || state.current["b"] != -limit+1 || len(state.pending) != 0 {
- t.Fatalf("rollback lost another charge: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
@@ -252,12 +252,12 @@ func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) {
_, latest := weighted.reserve(key, auths, "")
old.finish(true)
state := weightedStateForTest(t, weighted, key, "a", "b")
- if state.current["a"] != 0 || state.current["b"] != 0 || state.pending["a"] != -1 || state.pending["b"] != 1 {
- t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 {
+ t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
latest.finish(true)
- if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 {
- t.Fatalf("new generation accounting: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
@@ -294,8 +294,8 @@ func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T)
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
- if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 {
- t.Fatalf("concurrent accounting: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}
@@ -343,19 +343,19 @@ func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) {
t.Fatalf("sticky pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:requested", "a", "b")
- if state == nil || len(weighted.states) != 1 || len(state.current) != 2 || state.current["a"] != 3 || state.current["b"] != -3 {
+ if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 {
t.Fatalf("wrong model key or eligible pool: %v", weighted.states)
}
picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2])
- if errPick != nil || picked == nil || picked.ID != "a" || state.current["a"] != 2 || state.current["b"] != -2 {
- t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.current)
+ if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
+ t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest())
}
for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} {
if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil {
t.Fatal(errOther)
}
}
- if state.current["a"] != 2 || state.current["b"] != -2 {
+ if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
t.Fatal("another provider or model changed credits")
}
}
@@ -374,8 +374,8 @@ func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) {
t.Fatalf("reselection: pick=%v err=%v", picked, errPick)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
- if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 {
- t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.current, state.pending)
+ if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
t.Fatalf("reselection did not bind: %q, found=%t", binding, ok)
@@ -428,7 +428,7 @@ func TestManagerWeightedSessionAffinityCompensation(t *testing.T) {
}
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
- if state == nil || state.current[aID] != 0 || state.current[bID] != 0 || len(state.pending) != 0 {
+ if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 {
t.Fatalf("manager credits did not balance: %#v", state)
}
})
diff --git a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go
index 22ecac2eb..58d18499a 100644
--- a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go
+++ b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go
@@ -41,8 +41,8 @@ func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) {
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
limit := maxSmoothWeightedCreditRounds * 2
- if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -limit, "b": limit}) {
- t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.current, state.pending, state.inFlight)
+ if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) {
+ t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight)
}
})
}
@@ -98,7 +98,7 @@ func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) {
t.Fatalf("%s distribution = %v, want 48:16", phase, counts)
}
}
- if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) {
+ if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) {
t.Fatalf("settled 3:1 batch drifted: %#v", state)
}
})
@@ -128,11 +128,11 @@ func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) {
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
- if state.inFlight != 1 || len(state.current) != 0 || !maps.Equal(state.pending, map[string]int64{"a": -1, "b": 1}) {
+ if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatalf("old generation altered new accounting: %#v", state)
}
latest.finish(true)
- if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -1, "b": 1}) {
+ if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatalf("new generation failed to settle: %#v", state)
}
}
@@ -167,8 +167,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) {
}
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
// A zero net pending vector does not mean all reservations settled.
- if state.inFlight != 256 || len(state.pending) != 0 {
- t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pending)
+ if state.inFlight != 256 || len(state.pendingForTest()) != 0 {
+ t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest())
}
last := "b"
if first == "b" {
@@ -179,8 +179,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) {
reservation.finish(true)
}
}
- if !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) || len(state.pending) != 0 || state.inFlight != 0 {
- t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 {
+ t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
counts := map[string]int{}
for range 64 {
diff --git a/sdk/cliproxy/auth/selector_weighted_memory_test.go b/sdk/cliproxy/auth/selector_weighted_memory_test.go
new file mode 100644
index 000000000..eec967e28
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_memory_test.go
@@ -0,0 +1,71 @@
+package auth
+
+import (
+ "fmt"
+ "runtime"
+ "testing"
+)
+
+func BenchmarkWeightedPoolReserve1000(b *testing.B) {
+ for _, bound := range []bool{false, true} {
+ b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) {
+ auths := make([]*Auth, 1000)
+ for i := range auths {
+ auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
+ }
+ selector := &WeightedRoundRobinSelector{}
+ _, warmup := selector.reserve("codex:model", auths, "")
+ warmup.finish(true)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := range b.N {
+ boundID := ""
+ if bound {
+ boundID = auths[i%len(auths)].ID
+ }
+ _, reservation := selector.reserve("codex:model", auths, boundID)
+ reservation.finish(true)
+ }
+ })
+ }
+}
+
+// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from
+// total heap and allocation churn. Each retry subset omits a different credential.
+func BenchmarkWeightedPoolRetainedMemory(b *testing.B) {
+ const members = 1000
+ auths := make([]*Auth, members)
+ for i := range auths {
+ auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
+ }
+ subset := make([]*Auth, 0, members-1)
+ var retained, heap, allocated int64
+ b.ResetTimer()
+ for range b.N {
+ selector := &WeightedRoundRobinSelector{}
+ _, warmup := selector.reserve("codex:model", auths, "")
+ warmup.finish(true)
+ runtime.GC()
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ for excluded := range auths {
+ subset = append(subset[:0], auths[:excluded]...)
+ subset = append(subset, auths[excluded+1:]...)
+ _, reservation := selector.reserve("codex:model", subset, "")
+ reservation.finish(true)
+ }
+ runtime.GC()
+ runtime.ReadMemStats(&after)
+ if len(selector.states) != members+1 {
+ b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1)
+ }
+ runtime.KeepAlive(selector)
+ runtime.KeepAlive(auths)
+ retained += int64(after.HeapAlloc) - int64(before.HeapAlloc)
+ heap += int64(after.HeapAlloc)
+ allocated += int64(after.TotalAlloc - before.TotalAlloc)
+ }
+ b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op")
+ b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op")
+ b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op")
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_overload_test.go b/sdk/cliproxy/auth/selector_weighted_overload_test.go
index 47e3da067..f7639a536 100644
--- a/sdk/cliproxy/auth/selector_weighted_overload_test.go
+++ b/sdk/cliproxy/auth/selector_weighted_overload_test.go
@@ -50,13 +50,13 @@ func TestWeightedOverloadExactCompensation(t *testing.T) {
}
}
for _, state := range weighted.states {
- for id, credit := range state.current {
+ for id, credit := range state.currentForTest() {
if credit != 0 {
t.Errorf("exact compensation left %q credit=%d", id, credit)
}
}
- if len(state.pending) != 0 {
- t.Errorf("pending credit leaked: %v", state.pending)
+ if len(state.pendingForTest()) != 0 {
+ t.Errorf("pending credit leaked: %v", state.pendingForTest())
}
}
})
@@ -97,7 +97,7 @@ func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T)
ids[i] = auth.ID
}
state := weightedStateForTest(t, weighted, "codex:model", ids...)
- t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.current)
+ t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest())
// Forced traffic makes global ratios infeasible; the remaining capacity
// should be shared by unbound peers rather than monopolized by one.
peerWeight := 0
@@ -235,8 +235,8 @@ func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, key, "a", "b", "c")
- if len(state.pending) != 0 {
- t.Fatalf("settled reservations leaked credit: %v", state.pending)
+ if len(state.pendingForTest()) != 0 {
+ t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest())
}
}
@@ -249,7 +249,7 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T)
reservation.finish(true)
}
state := weightedStateForTest(t, weighted, key, "a", "b")
- before := maps.Clone(state.current)
+ before := maps.Clone(state.currentForTest())
_, first := weighted.reserve(key, auths, "")
_, second := weighted.reserve(key, auths, "")
t.Cleanup(func() { first.finish(false); second.finish(false) })
@@ -265,12 +265,12 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T)
bound.finish(true)
bound.finish(true)
bound.finish(false)
- if !maps.Equal(state.current, before) || len(state.pending) != 0 {
- t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest())
}
// An over-limit bound selection succeeds without reserving any credit.
picked, skipped := weighted.reserve(key, auths, "b")
- if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.current, before) || len(state.pending) != 0 {
+ if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
t.Fatal("saturated bound selection should preserve binding and all credits")
}
}
@@ -322,10 +322,10 @@ func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) {
{"a": -222, "b": 111, "c": 111},
{"a": -112, "b": 222, "c": -110},
} {
- state := &smoothWeightedState{current: maps.Clone(credit)}
- state.charge(map[string]int64{"a": 1, "b": 5, "c": 1}, "a", true)
- if !maps.Equal(state.current, credit) {
- t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.current, credit)
+ state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit)
+ state.charge(0, true)
+ if !maps.Equal(state.currentForTest(), credit) {
+ t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit)
}
}
}
@@ -391,8 +391,8 @@ func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) {
t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok)
}
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID)
- if len(state.pending) != 0 {
- t.Fatalf("manager left speculative credit: %v", state.pending)
+ if len(state.pendingForTest()) != 0 {
+ t.Fatalf("manager left speculative credit: %v", state.pendingForTest())
}
})
}
@@ -446,8 +446,8 @@ func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) {
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
// All 64 calls share one pool. Accounting must match actual accepted picks,
// regardless of whether the final arrivals happened to be sticky requests.
- if state.current["a"] != int64(64*3-counts["a"]*4) || state.current["b"] != int64(64-counts["b"]*4) || len(state.pending) != 0 {
- t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.current, state.pending)
+ if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest())
}
if bound, ok := selector.cache.Get(key); !ok || bound != "b" {
t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok)
diff --git a/sdk/cliproxy/auth/selector_weighted_pool_test.go b/sdk/cliproxy/auth/selector_weighted_pool_test.go
index 96dae7581..5fb73ffa9 100644
--- a/sdk/cliproxy/auth/selector_weighted_pool_test.go
+++ b/sdk/cliproxy/auth/selector_weighted_pool_test.go
@@ -14,13 +14,17 @@ import (
)
// weightedStateForTest checks that the requested accounting pool was created.
-func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *smoothWeightedState {
+func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState {
t.Helper()
weights := make(map[string]int64, len(ids))
for _, id := range ids {
weights[id] = 1
}
state := selector.states[weightedPoolStateKey(route, weights)]
+ ids = sortedWeightedIDs(weights)
+ for state != nil && !state.matches(route, ids) {
+ state = state.next
+ }
if state == nil {
t.Fatalf("missing weighted state for %q and %v", route, ids)
}
@@ -38,11 +42,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) {
pendingSubset.finish(true)
subset := weightedStateForTest(t, weighted, key, "a", "b")
full := weightedStateForTest(t, weighted, key, "a", "b", "c")
- if !maps.Equal(subset.current, map[string]int64{"a": 3, "b": -3}) || len(full.current) != 0 {
- t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.current, full.current)
+ if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() {
+ t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest())
}
- if !maps.Equal(full.pending, map[string]int64{"a": 3, "b": -4, "c": 1}) {
- t.Fatalf("another pool consumed pending credit: %v", full.pending)
+ if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) {
+ t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest())
}
pendingFull.finish(false)
for range 100 {
@@ -53,11 +57,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) {
for range 3 {
picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "")
if picked == nil || picked.ID != "a" {
- t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.current)
+ t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest())
}
reservation.finish(true)
}
- if !maps.Equal(subset.current, map[string]int64{"a": 0, "b": 0}) || len(subset.pending) != 0 || len(full.pending) != 0 {
+ if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 {
t.Fatalf("returning subset failed to balance: %#v", subset)
}
}
@@ -96,14 +100,14 @@ func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) {
independentReservation.finish(committed)
poolKey := weightedPoolStateKey("route", authWeightVector(pool))
state, independentState := shared.states[poolKey], independent[mask].states[poolKey]
- if !maps.Equal(state.current, independentState.current) || len(state.pending) != 0 {
+ if !maps.Equal(state.currentForTest(), independentState.currentForTest()) || len(state.pendingForTest()) != 0 {
t.Fatalf("step %d pool %d: credits diverged or pending leaked", step, mask)
}
}
}
func TestWeightedPoolKeysAndCapacity(t *testing.T) {
- seen := make(map[string]struct{})
+ seen := make(map[[32]byte]struct{})
for _, tc := range []struct {
route string
ids []string
@@ -145,7 +149,7 @@ func TestWeightedPoolKeysAndCapacity(t *testing.T) {
t.Fatal("pool eviction failed to bound or replace state")
}
old.finish(true)
- if len(current.state.current) != 0 || !maps.Equal(current.state.pending, map[string]int64{"a": -1, "b": 1}) {
+ if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
t.Fatal("finishing an evicted reservation affected the new pool")
}
current.finish(true)
@@ -273,8 +277,8 @@ func TestWeightedSessionAffinityPoolIsolation(t *testing.T) {
t.Fatalf("binding moved: %q, found=%t", bound, ok)
}
for _, state := range weighted.states {
- if len(state.pending) != 0 {
- t.Fatalf("pending credits leaked: %v", state.pending)
+ if len(state.pendingForTest()) != 0 {
+ t.Fatalf("pending credits leaked: %v", state.pendingForTest())
}
}
})
diff --git a/sdk/cliproxy/auth/selector_weighted_storage_test.go b/sdk/cliproxy/auth/selector_weighted_storage_test.go
new file mode 100644
index 000000000..0d937b903
--- /dev/null
+++ b/sdk/cliproxy/auth/selector_weighted_storage_test.go
@@ -0,0 +1,88 @@
+package auth
+
+import (
+ "maps"
+ "testing"
+)
+
+// These snapshots keep assertions independent of the compact production layout.
+func (s *weightedPoolState) currentForTest() map[string]int64 {
+ current := make(map[string]int64, len(s.members))
+ for _, member := range s.members {
+ current[member.id] = member.current
+ }
+ return current
+}
+
+func (s *weightedPoolState) pendingForTest() map[string]int64 {
+ pending := make(map[string]int64)
+ for _, member := range s.members {
+ if member.pending != 0 {
+ pending[member.id] = member.pending
+ }
+ }
+ return pending
+}
+
+func (s *weightedPoolState) hasCurrentForTest() bool {
+ for _, member := range s.members {
+ if member.current != 0 {
+ return true
+ }
+ }
+ return false
+}
+
+// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact
+// identities, including a matching state behind a collision-chain head.
+func TestWeightedPoolDigestCollisionIsolation(t *testing.T) {
+ for _, unrelated := range []*weightedPoolState{
+ weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}),
+ weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}),
+ weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}),
+ } {
+ t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) {
+ weights := map[string]int64{"a": 1, "b": 1}
+ key := weightedPoolStateKey("route", weights)
+ selector := &WeightedRoundRobinSelector{
+ states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2,
+ }
+ before := unrelated.currentForTest()
+ picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b")
+ if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated {
+ t.Fatal("digest collision reused an unrelated pool")
+ }
+ t.Cleanup(func() { first.finish(false) })
+ matching := first.state
+ // Put the nonmatching state first to exercise a full chain lookup.
+ matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated
+ _, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a")
+ if second == nil || second.state != matching || selector.stateCount != 2 {
+ t.Fatal("collision lookup failed to reuse the matching pool")
+ }
+ t.Cleanup(func() { second.finish(false) })
+ first.finish(true)
+ second.finish(true)
+ if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 {
+ t.Fatal("collision accounting changed the unrelated pool")
+ }
+ if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 {
+ t.Fatal("matching pool did not balance")
+ }
+ // Capacity still counts pools, not just distinct digest buckets.
+ _, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "")
+ third.finish(true)
+ if selector.stateCount != 1 || len(selector.states) != 1 {
+ t.Fatal("colliding pools bypassed the existing capacity limit")
+ }
+ })
+ }
+}
+
+func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState {
+ state := &weightedPoolState{route: route}
+ for _, id := range sortedWeightedIDs(weights) {
+ state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]})
+ }
+ return state
+}
diff --git a/sdk/cliproxy/auth/selector_weighted_test.go b/sdk/cliproxy/auth/selector_weighted_test.go
index 3d9872085..31352e301 100644
--- a/sdk/cliproxy/auth/selector_weighted_test.go
+++ b/sdk/cliproxy/auth/selector_weighted_test.go
@@ -2,6 +2,7 @@ package auth
import (
"maps"
+ "sort"
"testing"
)
@@ -18,14 +19,14 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
current: map[string]int64{"a": -96, "b": 96, "c": 95},
weights: map[string]int64{"b": 1, "c": 1},
authID: "b",
- want: map[string]int64{"a": -96, "b": 63, "c": 64},
+ want: map[string]int64{"b": 63, "c": 64},
},
{
name: "negative credits shift together",
current: map[string]int64{"a": 96, "b": -64, "c": -63},
weights: map[string]int64{"b": 1, "c": 1},
authID: "b",
- want: map[string]int64{"a": 96, "b": -64, "c": -61},
+ want: map[string]int64{"b": -64, "c": -61},
},
{
name: "excessive spread clips debt without tying leaders",
@@ -43,10 +44,10 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
- state := &smoothWeightedState{current: tc.current}
- state.charge(tc.weights, tc.authID, false)
- if !maps.Equal(state.current, tc.want) {
- t.Fatalf("rebased credits = %v, want %v", state.current, tc.want)
+ state := weightedPoolForTest("route", tc.weights, tc.current)
+ state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false)
+ if !maps.Equal(state.currentForTest(), tc.want) {
+ t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want)
}
})
}
@@ -54,12 +55,9 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) {
key := "codex:model"
- state := &smoothWeightedState{
- current: map[string]int64{"a": -94, "b": 94, "c": 93},
- weights: map[string]int64{"a": 1, "b": 1, "c": 1},
- pending: make(map[string]int64),
- }
- weighted := &WeightedRoundRobinSelector{states: map[string]*smoothWeightedState{weightedPoolStateKey(key, state.weights): state}}
+ weights := map[string]int64{"a": 1, "b": 1, "c": 1}
+ state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93})
+ weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1}
auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}}
_, rejected := weighted.reserve(key, auths, "a")
t.Cleanup(func() { rejected.finish(false) })
@@ -67,15 +65,15 @@ func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) {
t.Cleanup(func() { accepted.finish(false) })
accepted.finish(true)
want := map[string]int64{"a": -93, "b": 92, "c": 94}
- if !maps.Equal(state.current, want) {
- t.Fatalf("committed credits = %v, want %v", state.current, want)
+ if !maps.Equal(state.currentForTest(), want) {
+ t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want)
}
- if !maps.Equal(state.pending, map[string]int64{"a": -2, "b": 1, "c": 1}) {
- t.Fatalf("commit changed another request's reservation: %v", state.pending)
+ if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) {
+ t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest())
}
rejected.finish(false)
rejected.finish(true)
- if !maps.Equal(state.current, want) || len(state.pending) != 0 {
- t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.current, state.pending)
+ if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 {
+ t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
}
}