mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-22 09:53:02 +08:00
Merge pull request #7377 from router-for-me/home
feat(auth): compensate weighted routing for sticky sessions
This commit is contained in:
@@ -170,6 +170,16 @@ routing:
|
||||
strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first
|
||||
# weighted-round-robin uses each credential's integer weight (default 1, maximum 1,000,000).
|
||||
# Non-positive weights exclude the credential while this strategy is active.
|
||||
# Sticky requests (including inherited and LCP bindings) also consume weighted credits
|
||||
# without moving existing bindings. Compensation is scoped to each eligible credential set;
|
||||
# priority tiers, transport preferences, and retry/cooldown subsets do not share credits.
|
||||
# Pinned requests use the eligible pool before pinning, within the pinned credential
|
||||
# priority tier and transport class. Returning to a pool resumes its retained credits.
|
||||
# Settled credits are capped at +/-32 times the eligible pool total; ratios are approximate.
|
||||
# In-flight offsets are preserved until the pool settles, regardless of commit order.
|
||||
# Sticky requests keep their bindings but stop adding compensation when the full
|
||||
# credit delta cannot fit the cap; new sessions keep earning credit to avoid starvation.
|
||||
# Credits are process-local and reset on restart or selector replacement.
|
||||
# For OAuth/file credentials, add a top-level numeric "weight" field to the auth JSON.
|
||||
# Enable universal session-sticky routing for all clients.
|
||||
# Explicit Claude Code, Codex, OpenCode, and pi session headers are preferred,
|
||||
|
||||
@@ -64,6 +64,36 @@ The Home-owned revision fields
|
||||
`credentials.concurrency.observation-barrier-revision`, and `plugins.auth-revision`
|
||||
cannot be changed through these endpoints.
|
||||
|
||||
### Weighted routing with session affinity
|
||||
|
||||
Set `routing.strategy` to `weighted-round-robin` to use per-credential integer
|
||||
`weight` values (default `1`, maximum `1000000`; non-positive values are excluded).
|
||||
With `routing.session-affinity` enabled, sticky requests, parent inheritance,
|
||||
Guardian selections, and LCP matches also consume weighted credits without moving
|
||||
existing bindings. New or reselected sessions favor under-served credentials
|
||||
within the same eligible credential set. For example, with weights `3:1`, four
|
||||
sticky requests to the second credential can be compensated by twelve new
|
||||
requests to the first when both remain eligible.
|
||||
|
||||
Different priority tiers, transport preferences, and retry/cooldown candidate
|
||||
sets keep separate credits; returning to a set resumes its retained credits.
|
||||
Pinned requests account against the eligible pool before pinning, within the
|
||||
pinned credential's priority tier and transport class. A lower-priority binding
|
||||
never shifts the request ratio among higher-priority credentials.
|
||||
|
||||
Settled credits are capped at ±32 times the eligible pool total, so request
|
||||
ratios are approximate rather than guaranteed. While reservations are in flight,
|
||||
committed balances can temporarily exceed the cap; their pending offsets remain
|
||||
intact until the pool settles, including when the last reservation rolls back. Sticky requests still use their binding at
|
||||
the cap, but stop adding compensation when their full credit delta cannot fit.
|
||||
New sessions continue repaying debt and earning credit, so excessive sticky
|
||||
traffic does not starve smaller-weight peers.
|
||||
|
||||
Credits are local to the CPA process and reset on restart, selector replacement,
|
||||
or bounded pool-cache eviction. Concurrent selections see pending credit; local
|
||||
selection/binding success confirms it, while failure releases it. In Home mode,
|
||||
scheduling and credit accounting remain owned by Home.
|
||||
|
||||
### Codex multi-agent configuration migration
|
||||
|
||||
`client.codex.optimize-multi-agent-v2` is the sole runtime setting. Loading older
|
||||
|
||||
@@ -1084,11 +1084,17 @@ func (m *Manager) pickHomeDispatchSelection(ctx context.Context, model string, o
|
||||
}
|
||||
dispatchHeaders := homeDispatchHeaders(ctx, opts.Headers)
|
||||
if opts.Metadata != nil {
|
||||
if nodeKind, ok := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string); ok && strings.TrimSpace(nodeKind) != "" {
|
||||
nodeKind, _ := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string)
|
||||
if isFork, _ := opts.Metadata[cliproxyexecutor.IsForkMetadataKey].(bool); isFork {
|
||||
nodeKind = "fork"
|
||||
}
|
||||
if nodeKind = strings.TrimSpace(nodeKind); nodeKind != "" {
|
||||
// Keep the caller headers unchanged when adding Home-only lineage metadata.
|
||||
dispatchHeaders = dispatchHeaders.Clone()
|
||||
if dispatchHeaders == nil {
|
||||
dispatchHeaders = make(http.Header)
|
||||
}
|
||||
dispatchHeaders.Set("X-Node-Kind", strings.TrimSpace(nodeKind))
|
||||
dispatchHeaders.Set("X-Node-Kind", nodeKind)
|
||||
}
|
||||
}
|
||||
credentialPolicy := credentialPolicyFromContext(ctx)
|
||||
|
||||
96
sdk/cliproxy/auth/conductor_pinned_test.go
Normal file
96
sdk/cliproxy/auth/conductor_pinned_test.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
func TestManagerNonWeightedPinnedSelection(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
for _, affinity := range []bool{false, true} {
|
||||
for _, strategy := range []string{"round-robin", "fill-first"} {
|
||||
t.Run(fmt.Sprintf("mixed=%t/affinity=%t/%s", mixed, affinity, strategy), func(t *testing.T) {
|
||||
var selector Selector = &RoundRobinSelector{}
|
||||
if strategy == "fill-first" {
|
||||
selector = &FillFirstSelector{}
|
||||
}
|
||||
if affinity {
|
||||
sticky := NewSessionAffinitySelector(selector)
|
||||
t.Cleanup(sticky.Stop)
|
||||
selector = sticky
|
||||
}
|
||||
manager := NewManager(nil, selector, nil)
|
||||
const provider, model = "pinned-non-weighted", "pinned-model"
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
|
||||
pinnedID := t.Name() + "-pinned"
|
||||
for i := range 32 {
|
||||
auth := &Auth{
|
||||
ID: fmt.Sprintf("%s-other-%02d", t.Name(), i), Provider: provider, Status: StatusActive,
|
||||
Attributes: map[string]string{AttributeAuthKind: AuthKindAPIKey, "priority": "10"},
|
||||
}
|
||||
if i == 0 {
|
||||
auth.ID = pinnedID
|
||||
auth.Attributes["priority"] = "0"
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
|
||||
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
|
||||
t.Fatal(errRegister)
|
||||
}
|
||||
}
|
||||
// Normalize the legacy selection signatures for the same pinning checks.
|
||||
pick := func(ctx context.Context, id string, tried map[string]struct{}) (*Auth, error) {
|
||||
opts := cliproxyexecutor.Options{
|
||||
Headers: http.Header{"Session_id": {"non-weighted-pinned"}},
|
||||
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: id},
|
||||
}
|
||||
if mixed {
|
||||
auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider}, model, opts, tried)
|
||||
return auth, errPick
|
||||
}
|
||||
auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
|
||||
return auth, errPick
|
||||
}
|
||||
ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
|
||||
for range 3 {
|
||||
if auth, errPick := pick(ctx, pinnedID, nil); errPick != nil || auth == nil || auth.ID != pinnedID {
|
||||
t.Fatalf("lower-priority pin not enforced: auth=%v err=%v", auth, errPick)
|
||||
}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
id string
|
||||
tried map[string]struct{}
|
||||
}{
|
||||
{name: "missing pin", ctx: ctx, id: "missing"},
|
||||
{name: "already tried", ctx: ctx, id: pinnedID, tried: map[string]struct{}{pinnedID: {}}},
|
||||
{name: "ineligible kind", ctx: withRequiredAuthKind(t.Context(), AuthKindOAuth), id: pinnedID},
|
||||
} {
|
||||
auth, errPick := pick(tc.ctx, tc.id, tc.tried)
|
||||
var authErr *Error
|
||||
if auth != nil || !errors.As(errPick, &authErr) || authErr.Code != "auth_not_found" {
|
||||
t.Fatalf("%s: auth=%v err=%v, want auth_not_found", tc.name, auth, errPick)
|
||||
}
|
||||
}
|
||||
if sticky, ok := selector.(*SessionAffinitySelector); ok {
|
||||
namespace := provider
|
||||
if mixed {
|
||||
namespace = "mixed"
|
||||
}
|
||||
id := CanonicalSessionID(http.Header{"Session_id": {"non-weighted-pinned"}}, nil, nil)
|
||||
if binding, found := sticky.cache.Get(namespace + "::" + id + "::" + model); !found || binding != pinnedID {
|
||||
t.Fatalf("pinning checks changed the sticky binding: %q, found=%t", binding, found)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2258,7 +2258,11 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
|
||||
targetKey := canonicalSchedulingProvider(provider)
|
||||
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
|
||||
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
|
||||
var affinityCandidates []*Auth
|
||||
trackPinnedAccounting := false
|
||||
if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
|
||||
_, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
|
||||
}
|
||||
var accountingCandidates, affinityCandidates []*Auth
|
||||
if trackTransientAffinity {
|
||||
affinityCandidates = make([]*Auth, 0, len(m.auths))
|
||||
}
|
||||
@@ -2266,7 +2270,7 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
|
||||
if candidate == nil || canonicalSchedulingProvider(executorKeyFromAuth(candidate)) != targetKey || candidate.Disabled {
|
||||
continue
|
||||
}
|
||||
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
continue
|
||||
}
|
||||
if !eligibility.allows(candidate) {
|
||||
@@ -2275,10 +2279,18 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
|
||||
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
|
||||
continue
|
||||
}
|
||||
_, used := tried[candidate.ID]
|
||||
// Pinning restricts selection, not the eligible pool used for accounting.
|
||||
if trackPinnedAccounting && !used {
|
||||
accountingCandidates = append(accountingCandidates, candidate)
|
||||
}
|
||||
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
continue
|
||||
}
|
||||
if trackTransientAffinity {
|
||||
affinityCandidates = append(affinityCandidates, candidate)
|
||||
}
|
||||
if _, used := tried[candidate.ID]; used {
|
||||
if used {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
@@ -2302,6 +2314,15 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
|
||||
return nil, nil, errAvailable
|
||||
}
|
||||
}
|
||||
var accountingAuths []*Auth
|
||||
if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
|
||||
_, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, provider, model, now, false, opts.Headers)
|
||||
if errAccounting != nil {
|
||||
m.mu.RUnlock()
|
||||
return nil, nil, errAccounting
|
||||
}
|
||||
accountingAuths = fullAvailable
|
||||
}
|
||||
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
|
||||
m.mu.RUnlock()
|
||||
|
||||
@@ -2313,6 +2334,9 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op
|
||||
if !handled {
|
||||
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
|
||||
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
|
||||
if accountingAuths != nil {
|
||||
selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
|
||||
}
|
||||
selected, errPick = selector.Pick(selectorCtx, provider, selectionArgForSelector(selector, model), opts, selectorAuths)
|
||||
if errPick != nil {
|
||||
if isAuthNotFoundError(errPick) {
|
||||
@@ -2678,7 +2702,11 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
|
||||
registryRef := registry.GetGlobalRegistry()
|
||||
affinitySelector, hasAffinity := selector.(*SessionAffinitySelector)
|
||||
trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever)
|
||||
var affinityCandidates []*Auth
|
||||
trackPinnedAccounting := false
|
||||
if hasAffinity && affinitySelector != nil && pinnedAuthID != "" {
|
||||
_, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector)
|
||||
}
|
||||
var accountingCandidates, affinityCandidates []*Auth
|
||||
if trackTransientAffinity {
|
||||
affinityCandidates = make([]*Auth, 0, len(m.auths))
|
||||
}
|
||||
@@ -2686,7 +2714,7 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
|
||||
if candidate == nil || candidate.Disabled {
|
||||
continue
|
||||
}
|
||||
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
continue
|
||||
}
|
||||
if !eligibility.allows(candidate) {
|
||||
@@ -2705,10 +2733,18 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
|
||||
if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) {
|
||||
continue
|
||||
}
|
||||
_, used := tried[candidate.ID]
|
||||
// Pinning restricts selection, not the eligible pool used for accounting.
|
||||
if trackPinnedAccounting && !used {
|
||||
accountingCandidates = append(accountingCandidates, candidate)
|
||||
}
|
||||
if pinnedAuthID != "" && candidate.ID != pinnedAuthID {
|
||||
continue
|
||||
}
|
||||
if trackTransientAffinity {
|
||||
affinityCandidates = append(affinityCandidates, candidate)
|
||||
}
|
||||
if _, used := tried[candidate.ID]; used {
|
||||
if used {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
@@ -2732,6 +2768,15 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
|
||||
return nil, nil, "", errAvailable
|
||||
}
|
||||
}
|
||||
var accountingAuths []*Auth
|
||||
if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil {
|
||||
_, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, "mixed", model, now, false, opts.Headers)
|
||||
if errAccounting != nil {
|
||||
m.mu.RUnlock()
|
||||
return nil, nil, "", errAccounting
|
||||
}
|
||||
accountingAuths = fullAvailable
|
||||
}
|
||||
transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers)
|
||||
m.mu.RUnlock()
|
||||
|
||||
@@ -2743,6 +2788,9 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m
|
||||
if !handled {
|
||||
selectorCtx := selectorContextForAvailableAuths(ctx, selector, model)
|
||||
selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs)
|
||||
if accountingAuths != nil {
|
||||
selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths)
|
||||
}
|
||||
selected, errPick = selector.Pick(selectorCtx, "mixed", selectionArgForSelector(selector, model), opts, selectorAuths)
|
||||
if errPick != nil {
|
||||
if isAuthNotFoundError(errPick) {
|
||||
|
||||
161
sdk/cliproxy/auth/conductor_weighted_pinned_test.go
Normal file
161
sdk/cliproxy/auth/conductor_weighted_pinned_test.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
|
||||
)
|
||||
|
||||
func TestManagerWeightedPinnedSessionCompensation(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
for _, mode := range []string{"sticky", "cold", "no-session", "lcp"} {
|
||||
t.Run(fmt.Sprintf("mixed=%t/%s", mixed, mode), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
manager := NewManager(nil, selector, nil)
|
||||
const provider, model = "weighted-pinned", "weighted-pinned-model"
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
|
||||
ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey)
|
||||
aID, bID := t.Name()+"-a", t.Name()+"-b"
|
||||
triedID := t.Name() + "-tried"
|
||||
excluded := []*Auth{
|
||||
{ID: t.Name() + "-disabled", Disabled: true},
|
||||
{ID: t.Name() + "-zero", Attributes: map[string]string{AttributeWeight: "0"}},
|
||||
{ID: t.Name() + "-cooldown", ModelStates: map[string]*ModelState{
|
||||
model: {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
|
||||
}},
|
||||
{ID: t.Name() + "-wrong-model"},
|
||||
{ID: t.Name() + "-wrong-kind", Attributes: map[string]string{AttributeAuthKind: AuthKindOAuth}},
|
||||
{ID: t.Name() + "-wrong-provider", Provider: "other-provider"},
|
||||
{ID: t.Name() + "-no-executor", Provider: "no-executor"},
|
||||
{ID: t.Name() + "-lower-priority", Attributes: map[string]string{"priority": "0"}},
|
||||
{ID: triedID},
|
||||
}
|
||||
auths := append([]*Auth{
|
||||
{ID: aID, Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
|
||||
{ID: bID, Attributes: map[string]string{AttributeWeight: "1", "priority": "1"}},
|
||||
}, excluded...)
|
||||
for _, auth := range auths {
|
||||
if auth.Provider == "" {
|
||||
auth.Provider = provider
|
||||
}
|
||||
auth.Status = StatusActive
|
||||
if auth.Attributes == nil {
|
||||
auth.Attributes = make(map[string]string)
|
||||
}
|
||||
if _, ok := auth.Attributes[AttributeWeight]; !ok {
|
||||
auth.Attributes[AttributeWeight] = "100"
|
||||
}
|
||||
if _, ok := auth.Attributes["priority"]; !ok {
|
||||
auth.Attributes["priority"] = "2"
|
||||
}
|
||||
if _, ok := auth.Attributes[AttributeAuthKind]; !ok {
|
||||
auth.Attributes[AttributeAuthKind] = AuthKindAPIKey
|
||||
}
|
||||
registeredModel := model
|
||||
if auth.ID == t.Name()+"-wrong-model" {
|
||||
registeredModel = "other-model"
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: registeredModel}})
|
||||
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
|
||||
if _, errRegister := manager.Register(WithSkipPersist(ctx), auth); errRegister != nil {
|
||||
t.Fatal(errRegister)
|
||||
}
|
||||
}
|
||||
namespace := provider
|
||||
if mixed {
|
||||
namespace = "mixed"
|
||||
}
|
||||
stickyHeaders := http.Header{"Session_id": {"pinned-session"}}
|
||||
id := CanonicalSessionID(stickyHeaders, nil, nil)
|
||||
bindingKey := namespace + "::" + id + "::" + model
|
||||
if mode == "sticky" {
|
||||
selector.cache.Set(bindingKey, bID)
|
||||
}
|
||||
tried := map[string]struct{}{triedID: {}}
|
||||
// Normalize the legacy signatures without changing their pinning behavior.
|
||||
pick := func(opts cliproxyexecutor.Options) (*Auth, error) {
|
||||
if mixed {
|
||||
auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider, "no-executor"}, model, opts, tried)
|
||||
return auth, errPick
|
||||
}
|
||||
auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried)
|
||||
return auth, errPick
|
||||
}
|
||||
for i := range 4 {
|
||||
opts := cliproxyexecutor.Options{Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: bID}}
|
||||
switch mode {
|
||||
case "sticky", "cold":
|
||||
opts.Headers = stickyHeaders
|
||||
case "lcp":
|
||||
opts.SourceFormat = sdktranslator.FormatOpenAI
|
||||
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"pinned conversation"}]}`)
|
||||
opts.Metadata[cliproxyexecutor.CallerScopeMetadataKey] = "pinned-caller"
|
||||
}
|
||||
if auth, errPick := pick(opts); errPick != nil || auth == nil || auth.ID != bID {
|
||||
t.Fatalf("pinned request %d: auth=%v err=%v, want %q", i, auth, errPick, bID)
|
||||
}
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
|
||||
if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state)
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range 12 {
|
||||
auth, errPick := pick(cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}})
|
||||
if errPick != nil || auth == nil {
|
||||
t.Fatalf("fresh request %d: auth=%v err=%v", i, auth, errPick)
|
||||
}
|
||||
counts[auth.ID]++
|
||||
}
|
||||
if counts[aID] != 12 || counts[bID] != 0 {
|
||||
t.Fatalf("fresh distribution = %v, want a=12 b=0", counts)
|
||||
}
|
||||
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
if mode == "sticky" || mode == "cold" {
|
||||
if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID {
|
||||
t.Fatalf("compensation changed pinned binding: %q, found=%t", binding, ok)
|
||||
}
|
||||
}
|
||||
// A broader accounting pool must never make an invalid pin selectable.
|
||||
for _, pinnedID := range []string{excluded[0].ID, excluded[1].ID, excluded[2].ID, excluded[3].ID, excluded[4].ID, excluded[5].ID, excluded[6].ID, triedID, "missing"} {
|
||||
opts := cliproxyexecutor.Options{
|
||||
Headers: stickyHeaders,
|
||||
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: pinnedID},
|
||||
}
|
||||
if auth, errPick := pick(opts); errPick == nil || auth != nil {
|
||||
t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick)
|
||||
}
|
||||
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
}
|
||||
// A valid lower-priority pin still wins without charging another tier.
|
||||
lowerID := t.Name() + "-lower-priority"
|
||||
lowerOpts := cliproxyexecutor.Options{
|
||||
Headers: http.Header{"Session_id": {"lower-pinned-session"}},
|
||||
Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: lowerID},
|
||||
}
|
||||
if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID {
|
||||
t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick)
|
||||
}
|
||||
if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID)
|
||||
if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 {
|
||||
t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
59
sdk/cliproxy/auth/home_session_kind_test.go
Normal file
59
sdk/cliproxy/auth/home_session_kind_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
type sessionKindCaptureDispatcher struct {
|
||||
fixtureHomeDispatcher
|
||||
headers http.Header
|
||||
sessionID string
|
||||
parentSessionID string
|
||||
}
|
||||
|
||||
func (d *sessionKindCaptureDispatcher) RPopAuthWithSessionHierarchy(_ context.Context, _ string, sessionID, parentSessionID string, headers http.Header, _ int, _ string, _ *int, _ []string, _ string) ([]byte, error) {
|
||||
d.headers = headers.Clone()
|
||||
d.sessionID = sessionID
|
||||
d.parentSessionID = parentSessionID
|
||||
return d.payload, nil
|
||||
}
|
||||
|
||||
func TestHomeDispatchPreservesForkAndCompactionKind(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
metadata map[string]any
|
||||
want string
|
||||
}{
|
||||
{name: "body fork", body: `{"thread_id":"child","metadata":{"forked_from_thread_id":"parent"}}`, want: "fork"},
|
||||
{name: "compaction", body: `{"session_id":"child","parent_session_id":"parent"}`, metadata: map[string]any{cliproxyexecutor.NodeKindMetadataKey: "compaction"}, want: "compaction"},
|
||||
{name: "ordinary child", body: `{"session_id":"child","parent_session_id":"parent"}`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dispatcher := &sessionKindCaptureDispatcher{fixtureHomeDispatcher: fixtureHomeDispatcher{payload: []byte(`{"auth":{"id":"kind-auth","provider":"kind-provider"}}`)}}
|
||||
manager := newHomeSelectionTestManager(t, dispatcher)
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: "kind-provider"})
|
||||
headers := http.Header{"User-Agent": {"test-client"}}
|
||||
opts := cliproxyexecutor.Options{Headers: headers, OriginalRequest: []byte(tc.body), Metadata: tc.metadata}
|
||||
opts.EnsureMetadata()
|
||||
selection, errSelection := manager.pickHomeDispatchSelection(t.Context(), "model", opts)
|
||||
if errSelection != nil {
|
||||
t.Fatal(errSelection)
|
||||
}
|
||||
selection.End("test_complete")
|
||||
if dispatcher.sessionID == "" || dispatcher.parentSessionID == "" || dispatcher.sessionID == dispatcher.parentSessionID {
|
||||
t.Fatalf("lost session hierarchy: child=%q parent=%q", dispatcher.sessionID, dispatcher.parentSessionID)
|
||||
}
|
||||
if got := dispatcher.headers.Get("X-Node-Kind"); got != tc.want {
|
||||
t.Fatalf("node kind=%q, want %q", got, tc.want)
|
||||
}
|
||||
if got := headers.Get("X-Node-Kind"); got != "" {
|
||||
t.Fatalf("caller headers mutated: %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -214,3 +214,41 @@ func BenchmarkManagerPickNextAndMarkResult1000(b *testing.B) {
|
||||
manager.MarkResult(ctx, Result{AuthID: auth.ID, Provider: "gemini", Model: model, Success: true})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkManagerPinnedLegacy1000(b *testing.B) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
b.Run(fmt.Sprintf("mixed=%t", mixed), func(b *testing.B) {
|
||||
manager, providers, model := benchmarkManagerSetup(b, 1000, mixed, false)
|
||||
selector := NewSessionAffinitySelector(&RoundRobinSelector{})
|
||||
b.Cleanup(selector.Stop)
|
||||
manager.SetSelector(selector)
|
||||
ctx := context.Background()
|
||||
opts := cliproxyexecutor.Options{Metadata: map[string]any{
|
||||
cliproxyexecutor.PinnedAuthMetadataKey: "bench-gemini-0000",
|
||||
}}
|
||||
// Warm up index assignment and selection metadata outside the timed loop.
|
||||
if mixed {
|
||||
if _, _, _, errWarm := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil); errWarm != nil {
|
||||
b.Fatal(errWarm)
|
||||
}
|
||||
} else if _, _, errWarm := manager.pickNextLegacy(ctx, providers[0], model, opts, nil); errWarm != nil {
|
||||
b.Fatal(errWarm)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if mixed {
|
||||
auth, exec, provider, errPick := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil)
|
||||
if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil || provider != providers[0] {
|
||||
b.Fatalf("pinned mixed pick failed: auth=%v exec=%v provider=%q err=%v", auth, exec, provider, errPick)
|
||||
}
|
||||
} else {
|
||||
auth, exec, errPick := manager.pickNextLegacy(ctx, providers[0], model, opts, nil)
|
||||
if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil {
|
||||
b.Fatalf("pinned pick failed: auth=%v exec=%v err=%v", auth, exec, errPick)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,9 +39,10 @@ type RoundRobinSelector struct {
|
||||
|
||||
// WeightedRoundRobinSelector provides smooth weighted round-robin selection.
|
||||
type WeightedRoundRobinSelector struct {
|
||||
mu sync.Mutex
|
||||
states map[string]*smoothWeightedState
|
||||
maxKeys int
|
||||
mu sync.Mutex
|
||||
states map[[32]byte]*weightedPoolState
|
||||
stateCount int
|
||||
maxKeys int
|
||||
}
|
||||
|
||||
type smoothWeightedState struct {
|
||||
@@ -698,32 +699,12 @@ func (s *WeightedRoundRobinSelector) Pick(ctx context.Context, provider, model s
|
||||
return nil, errAvailable
|
||||
}
|
||||
available = preferCodexWebsocketAuths(ctx, provider, available)
|
||||
stateModel := weightedSelectorStateModel(ctx, model)
|
||||
key := provider + ":" + canonicalModelLookupKey(stateModel)
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.states == nil {
|
||||
s.states = make(map[string]*smoothWeightedState)
|
||||
}
|
||||
limit := s.maxKeys
|
||||
if limit <= 0 {
|
||||
limit = 4096
|
||||
}
|
||||
if _, ok := s.states[key]; !ok && len(s.states) >= limit {
|
||||
s.states = make(map[string]*smoothWeightedState)
|
||||
}
|
||||
state := s.states[key]
|
||||
if state == nil {
|
||||
state = &smoothWeightedState{}
|
||||
s.states[key] = state
|
||||
}
|
||||
weights := authWeightVector(available)
|
||||
state.prepare(weights)
|
||||
picked := pickSmoothWeightedAuth(available, state.current)
|
||||
key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
|
||||
picked, reservation := s.reserve(key, available, "")
|
||||
if picked == nil {
|
||||
return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
|
||||
}
|
||||
reservation.finish(true)
|
||||
return picked, nil
|
||||
}
|
||||
|
||||
@@ -796,29 +777,6 @@ func authWeightVector(auths []*Auth) map[string]int64 {
|
||||
return weights
|
||||
}
|
||||
|
||||
func pickSmoothWeightedAuth(auths []*Auth, current map[string]int64) *Auth {
|
||||
var picked *Auth
|
||||
var pickedCurrent int64
|
||||
var totalWeight int64
|
||||
for _, auth := range auths {
|
||||
weight := authWeight(auth)
|
||||
if auth == nil || weight <= 0 {
|
||||
continue
|
||||
}
|
||||
current[auth.ID] = saturatingAddInt64(current[auth.ID], weight)
|
||||
totalWeight = saturatingAddInt64(totalWeight, weight)
|
||||
if picked == nil || current[auth.ID] > pickedCurrent {
|
||||
picked = auth
|
||||
pickedCurrent = current[auth.ID]
|
||||
}
|
||||
}
|
||||
if picked == nil {
|
||||
return nil
|
||||
}
|
||||
current[picked.ID] = saturatingAddInt64(current[picked.ID], -totalWeight)
|
||||
return picked
|
||||
}
|
||||
|
||||
func saturatingAddInt64(value, delta int64) int64 {
|
||||
if delta > 0 && value > math.MaxInt64-delta {
|
||||
return math.MaxInt64
|
||||
@@ -1164,7 +1122,16 @@ func (s *SessionAffinitySelector) Trees() *cliproxysession.InMemorySessionTreeSt
|
||||
// Note: The cache key includes provider, session ID, and model to handle cases where
|
||||
// a session uses multiple models (e.g., gemini-2.5-pro and gemini-3-flash-preview)
|
||||
// that may be supported by different auth credentials, and to avoid cross-provider conflicts.
|
||||
func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) {
|
||||
func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (selected *Auth, errPick error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if errContext := ctx.Err(); errContext != nil {
|
||||
return nil, errContext
|
||||
}
|
||||
var reservation *weightedReservation
|
||||
// Local bindings complete inside Pick; failed selections release speculative credit.
|
||||
defer func() { reservation.finish(selected != nil && errPick == nil) }()
|
||||
entry := selectorLogEntry(ctx)
|
||||
if opts.Metadata == nil {
|
||||
opts.Metadata = make(map[string]any)
|
||||
@@ -1176,7 +1143,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
// consulted when no header, body, or execution-session identity is present.
|
||||
explicitID, explicitFallbackID := extractExplicitSessionIDs(opts.Headers, opts.OriginalRequest, opts.Metadata)
|
||||
if explicitID == "" {
|
||||
if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry); handled || errLCP != nil {
|
||||
if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry, &reservation); handled || errLCP != nil {
|
||||
return auth, errLCP
|
||||
}
|
||||
} else if opts.Metadata != nil {
|
||||
@@ -1218,7 +1185,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
return nil, errAvailable
|
||||
}
|
||||
entry.Debugf("session-affinity: no session ID extracted, falling back to default selector | provider=%s model=%s", provider, model)
|
||||
return s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
|
||||
return s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
|
||||
}
|
||||
|
||||
// A single availability pass serves both lookups: the bound credential is validated against
|
||||
@@ -1268,7 +1235,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
if auth.ID == cachedAuthID {
|
||||
bind(auth.ID, true)
|
||||
entry.Infof("session-affinity: cache hit | session=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), auth.ID, provider, model)
|
||||
return auth, nil
|
||||
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
|
||||
}
|
||||
}
|
||||
if s.rebindOnNever {
|
||||
@@ -1281,7 +1248,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
return nil, err
|
||||
}
|
||||
// Cached auth not available, reselect via fallback selector for even distribution
|
||||
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
|
||||
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1315,7 +1282,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
if auth.ID == parentAuthID {
|
||||
bind(auth.ID, true)
|
||||
entry.Infof("session-affinity: guardian inherited parent binding | session=%s parent=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
|
||||
return auth, nil
|
||||
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1332,7 +1299,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
} else {
|
||||
entry.Infof("session-affinity: fallback cache hit | session=%s fallback=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model)
|
||||
}
|
||||
return auth, nil
|
||||
return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1343,7 +1310,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
return nil, errBound
|
||||
}
|
||||
if !isSubagent && !isFork && s.shouldPreserveBindingWhenUnavailable(ctx, availabilityCandidates, model, cachedAuthID, now) {
|
||||
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
|
||||
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1360,7 +1327,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
|
||||
auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1376,7 +1343,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry) (*Auth, bool, error) {
|
||||
func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry, reservation **weightedReservation) (*Auth, bool, error) {
|
||||
if s == nil || s.matcher == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -1449,7 +1416,8 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
|
||||
}
|
||||
entry.Infof("session-affinity: LCP cache hit | session=%s prefix=%d auth=%s provider=%s model=%s", truncateSessionID(match.SessionID), match.PrefixLength, auth.ID, provider, model)
|
||||
}
|
||||
return auth, true, nil
|
||||
picked, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, reservation)
|
||||
return picked, true, errPick
|
||||
}
|
||||
if !match.IsFork && s.rebindOnNever {
|
||||
if bindRes := s.matcher.BindFingerprintsWithContext(namespace, fingerprints, tailFingerprints, envDigest, minPrefixLength, match.AuthID); bindRes.SessionID != "" {
|
||||
@@ -1488,7 +1456,7 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s
|
||||
return nil, true, errAvailable
|
||||
}
|
||||
fallbackAuths := highestPriorityAuths(available)
|
||||
auth, errPick := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths)
|
||||
auth, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, reservation)
|
||||
if errPick != nil {
|
||||
return nil, true, errPick
|
||||
}
|
||||
|
||||
@@ -205,16 +205,14 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickSmoothWeightedAuth_SaturatesCorruptState(t *testing.T) {
|
||||
func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
current := map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}
|
||||
picked := pickSmoothWeightedAuth([]*Auth{{ID: "a"}, {ID: "b"}}, current)
|
||||
if picked == nil {
|
||||
t.Fatal("pickSmoothWeightedAuth() returned nil")
|
||||
}
|
||||
if current["a"] != math.MaxInt64-2 || current["b"] != math.MinInt64+1 {
|
||||
t.Fatalf("current state = %#v, want saturated arithmetic", current)
|
||||
state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64})
|
||||
state.charge(0, false)
|
||||
limit := maxSmoothWeightedCreditRounds * 2
|
||||
if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit {
|
||||
t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
358
sdk/cliproxy/auth/selector_weighted.go
Normal file
358
sdk/cliproxy/auth/selector_weighted.go
Normal file
@@ -0,0 +1,358 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
// weightedAccountingPoolKey carries the manager-validated pre-pin pool. It affects
|
||||
// accounting only; the selector still receives pin-filtered selection candidates.
|
||||
type weightedAccountingPoolKey struct{}
|
||||
|
||||
// weightedReservation separates speculative credit from committed, clamped credit.
|
||||
// Releasing one request must never undo accounting committed by another request.
|
||||
type weightedReservation struct {
|
||||
selector *WeightedRoundRobinSelector
|
||||
state *weightedPoolState
|
||||
generation uint64
|
||||
member int
|
||||
total int64
|
||||
bound bool
|
||||
finished bool
|
||||
}
|
||||
|
||||
// pickWithAffinityReservation normalizes weighted candidates and defers accounting
|
||||
// until the local binding completes. Other fallback selectors keep their behavior.
|
||||
func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, bound *Auth, reservation **weightedReservation) (*Auth, error) {
|
||||
weighted, ok := s.fallback.(*WeightedRoundRobinSelector)
|
||||
if !ok {
|
||||
if bound != nil {
|
||||
return bound, nil
|
||||
}
|
||||
return s.fallback.Pick(ctx, provider, model, opts, auths)
|
||||
}
|
||||
if accountingAuths, ok := ctx.Value(weightedAccountingPoolKey{}).([]*Auth); ok {
|
||||
pinnedID := pinnedAuthIDFromMetadata(opts.Metadata)
|
||||
if bound == nil {
|
||||
for _, auth := range auths {
|
||||
if auth != nil && auth.ID == pinnedID {
|
||||
bound = auth
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if bound == nil || bound.ID != pinnedID || authWeight(bound) <= 0 {
|
||||
return nil, &Error{Code: "auth_unavailable", Message: "pinned auth is not available for weighted accounting"}
|
||||
}
|
||||
auths = positiveWeightAuths(accountingAuths)
|
||||
}
|
||||
var available []*Auth
|
||||
authID := ""
|
||||
if bound == nil {
|
||||
var errAvailable error
|
||||
available, errAvailable = getSelectorAvailableAuths(ctx, positiveWeightAuths(auths), provider, model, time.Now())
|
||||
if errAvailable != nil {
|
||||
return nil, errAvailable
|
||||
}
|
||||
available = preferCodexWebsocketAuths(ctx, provider, available)
|
||||
} else {
|
||||
authID = bound.ID
|
||||
// A binding may bypass priority or transport preference, but its traffic
|
||||
// must only charge peers that compete with it, never another priority tier.
|
||||
priority := authPriority(bound)
|
||||
for _, auth := range auths {
|
||||
if authPriority(auth) == priority {
|
||||
available = append(available, auth)
|
||||
}
|
||||
}
|
||||
preferred := preferCodexWebsocketAuths(ctx, provider, available)
|
||||
if len(preferred) < len(available) && !authWebsocketsEnabled(bound) {
|
||||
// Keep an HTTP-bound session outside the preferred WebSocket pool.
|
||||
nonWebsocket := available[:0]
|
||||
for _, auth := range available {
|
||||
if !authWebsocketsEnabled(auth) {
|
||||
nonWebsocket = append(nonWebsocket, auth)
|
||||
}
|
||||
}
|
||||
available = nonWebsocket
|
||||
} else {
|
||||
available = preferred
|
||||
}
|
||||
}
|
||||
key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model))
|
||||
picked, pending := weighted.reserve(key, available, authID)
|
||||
if picked == nil {
|
||||
return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"}
|
||||
}
|
||||
*reservation = pending
|
||||
return picked, nil
|
||||
}
|
||||
|
||||
// weightedPoolMember stores each ID once, with parallel accounting values in a
|
||||
// single allocation. Members are sorted by ID; reservations retain only an index.
|
||||
type weightedPoolMember struct {
|
||||
id string
|
||||
weight int64
|
||||
current int64
|
||||
pending int64
|
||||
}
|
||||
|
||||
// weightedPoolState is separate from the scheduler's transient-subset accumulator:
|
||||
// each selector pool has fixed membership and shares its weights across reservations.
|
||||
type weightedPoolState struct {
|
||||
route string
|
||||
members []weightedPoolMember
|
||||
inFlight uint64
|
||||
generation uint64
|
||||
next *weightedPoolState // Exact-identity collision chain for the digest key.
|
||||
}
|
||||
|
||||
func sortedWeightedIDs(weights map[string]int64) []string {
|
||||
ids := make([]string, 0, len(weights))
|
||||
for id := range weights {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// weightedPoolStateKey isolates competing sets without retaining concatenated IDs.
|
||||
// Weights stay out of the key so configuration changes invalidate reservations.
|
||||
func weightedPoolStateKey(route string, weights map[string]int64) [32]byte {
|
||||
return weightedPoolDigest(route, sortedWeightedIDs(weights))
|
||||
}
|
||||
|
||||
func weightedPoolDigest(route string, ids []string) [32]byte {
|
||||
size := 8 + len(route)
|
||||
for _, id := range ids {
|
||||
size += 8 + len(id)
|
||||
}
|
||||
identity := make([]byte, 0, size)
|
||||
identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route)))
|
||||
identity = append(identity, route...)
|
||||
for _, id := range ids {
|
||||
identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id)))
|
||||
identity = append(identity, id...)
|
||||
}
|
||||
return sha256.Sum256(identity)
|
||||
}
|
||||
|
||||
// matches verifies the full identity even on a digest hit. Hash collisions must
|
||||
// never combine the quota or configured weights of unrelated candidate pools.
|
||||
func (s *weightedPoolState) matches(route string, ids []string) bool {
|
||||
if s.route != route || len(s.members) != len(ids) {
|
||||
return false
|
||||
}
|
||||
for i, id := range ids {
|
||||
if s.members[i].id != id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *weightedPoolState) prepare(weights map[string]int64) {
|
||||
for _, member := range s.members {
|
||||
if member.weight != weights[member.id] {
|
||||
s.generation++
|
||||
s.inFlight = 0
|
||||
for i := range s.members {
|
||||
s.members[i].weight = weights[s.members[i].id]
|
||||
s.members[i].current = 0
|
||||
s.members[i].pending = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reserve selects and reserves credit atomically. A non-empty authID preserves a
|
||||
// sticky binding; otherwise committed and pending credits both guide selection.
|
||||
func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) {
|
||||
weights := authWeightVector(available)
|
||||
ids := sortedWeightedIDs(weights)
|
||||
key := weightedPoolDigest(route, ids)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.states == nil {
|
||||
s.states = make(map[[32]byte]*weightedPoolState)
|
||||
}
|
||||
state := s.states[key]
|
||||
for state != nil && !state.matches(route, ids) {
|
||||
state = state.next
|
||||
}
|
||||
if state == nil {
|
||||
limit := s.maxKeys
|
||||
if limit <= 0 {
|
||||
limit = 4096
|
||||
}
|
||||
if s.stateCount >= limit {
|
||||
s.states = make(map[[32]byte]*weightedPoolState)
|
||||
s.stateCount = 0
|
||||
}
|
||||
state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]}
|
||||
for i, id := range ids {
|
||||
state.members[i] = weightedPoolMember{id: id, weight: weights[id]}
|
||||
}
|
||||
s.states[key] = state
|
||||
s.stateCount++
|
||||
} else {
|
||||
state.prepare(weights)
|
||||
}
|
||||
var picked *Auth
|
||||
var pickedCurrent, total int64
|
||||
pickedMember := -1
|
||||
for _, auth := range available {
|
||||
if auth == nil || weights[auth.ID] <= 0 {
|
||||
continue
|
||||
}
|
||||
total = saturatingAddInt64(total, weights[auth.ID])
|
||||
if authID != "" && auth.ID != authID {
|
||||
continue
|
||||
}
|
||||
index := sort.SearchStrings(ids, auth.ID)
|
||||
member := &state.members[index]
|
||||
current := saturatingAddInt64(member.current, member.pending)
|
||||
current = saturatingAddInt64(current, member.weight)
|
||||
if picked == nil || current > pickedCurrent {
|
||||
picked, pickedCurrent, pickedMember = auth, current, index
|
||||
}
|
||||
}
|
||||
if picked == nil {
|
||||
return nil, nil
|
||||
}
|
||||
// Bound traffic beyond the compensation window still keeps its credential,
|
||||
// but must not create speculative credit that can starve concurrent picks.
|
||||
if authID != "" && !state.boundChargeFits(pickedMember, total) {
|
||||
return picked, nil
|
||||
}
|
||||
state.adjustPending(pickedMember, total, false)
|
||||
state.inFlight++
|
||||
return picked, &weightedReservation{
|
||||
selector: s, state: state, generation: state.generation,
|
||||
member: pickedMember, total: total, bound: authID != "",
|
||||
}
|
||||
}
|
||||
|
||||
// finish confirms or releases a reservation exactly once under the selector lock.
|
||||
func (r *weightedReservation) finish(committed bool) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.selector.mu.Lock()
|
||||
defer r.selector.mu.Unlock()
|
||||
if r.finished {
|
||||
return
|
||||
}
|
||||
r.finished = true
|
||||
if r.state.generation != r.generation {
|
||||
return
|
||||
}
|
||||
r.state.adjustPending(r.member, r.total, true)
|
||||
r.state.inFlight--
|
||||
if committed {
|
||||
r.state.charge(r.member, r.bound)
|
||||
} else if r.state.inFlight == 0 {
|
||||
// A final rollback can remove the offset to already committed credit.
|
||||
r.state.boundCredits(r.total)
|
||||
}
|
||||
}
|
||||
|
||||
// adjustPending must be called with the selector mutex held. Pending deltas are
|
||||
// not clamped, so releasing them cannot lose credit at the compensation bound.
|
||||
func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) {
|
||||
for i := range s.members {
|
||||
member := &s.members[i]
|
||||
delta := member.weight
|
||||
if i == selected {
|
||||
delta -= total
|
||||
}
|
||||
if release {
|
||||
delta = -delta
|
||||
}
|
||||
member.pending = saturatingAddInt64(member.pending, delta)
|
||||
}
|
||||
}
|
||||
|
||||
// maxSmoothWeightedCreditRounds bounds compensation after a long sticky session.
|
||||
const maxSmoothWeightedCreditRounds int64 = 32
|
||||
|
||||
// boundChargeFits admits the whole sticky delta or none. Both reservation and
|
||||
// commit include outstanding deltas: committed credit alone can transiently
|
||||
// exceed the cap while successful reservations settle in a different order.
|
||||
func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool {
|
||||
limit := int64(math.MaxInt64)
|
||||
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
|
||||
limit = totalWeight * maxSmoothWeightedCreditRounds
|
||||
}
|
||||
for i, member := range s.members {
|
||||
delta := member.weight
|
||||
if i == selected {
|
||||
delta -= totalWeight
|
||||
}
|
||||
current := saturatingAddInt64(member.current, member.pending)
|
||||
next := saturatingAddInt64(current, delta)
|
||||
if next < -limit || next > limit {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *weightedPoolState) charge(selected int, bound bool) {
|
||||
var totalWeight int64
|
||||
for _, member := range s.members {
|
||||
totalWeight = saturatingAddInt64(totalWeight, member.weight)
|
||||
}
|
||||
if !bound || s.boundChargeFits(selected, totalWeight) {
|
||||
// At the cap, forced traffic cannot erase a small peer's earnings.
|
||||
// Pending repayments still count until they actually roll back.
|
||||
for i := range s.members {
|
||||
member := &s.members[i]
|
||||
member.current = saturatingAddInt64(member.current, member.weight)
|
||||
}
|
||||
member := &s.members[selected]
|
||||
member.current = saturatingAddInt64(member.current, -totalWeight)
|
||||
}
|
||||
if s.inFlight == 0 {
|
||||
s.boundCredits(totalWeight)
|
||||
}
|
||||
}
|
||||
|
||||
// boundCredits runs only once the pool settles. Net pending deltas can be zero
|
||||
// even with live reservations, so the in-flight count, not len(pending), decides
|
||||
// when clipping cannot irreversibly discard another reservation's offset.
|
||||
func (s *weightedPoolState) boundCredits(totalWeight int64) {
|
||||
limit := int64(math.MaxInt64)
|
||||
if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds {
|
||||
limit = totalWeight * maxSmoothWeightedCreditRounds
|
||||
}
|
||||
var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64
|
||||
for _, member := range s.members {
|
||||
minCredit = min(minCredit, member.current)
|
||||
maxCredit = max(maxCredit, member.current)
|
||||
}
|
||||
// A common offset keeps leading credits distinct at the compensation bound.
|
||||
// Independent clamps can repeatedly tie the leaders and starve the later
|
||||
// candidate in deterministic selection order.
|
||||
minOffset := saturatingAddInt64(maxCredit, -limit)
|
||||
maxOffset := saturatingAddInt64(minCredit, limit)
|
||||
var offset int64
|
||||
if minOffset > 0 || minOffset > maxOffset {
|
||||
offset = minOffset
|
||||
} else if maxOffset < 0 {
|
||||
offset = maxOffset
|
||||
}
|
||||
// If the full spread cannot fit, cap only the trailing debt after rebasing
|
||||
// the leader to the upper bound. Do not flatten leading candidates' credits.
|
||||
for i := range s.members {
|
||||
member := &s.members[i]
|
||||
member.current = max(-limit, saturatingAddInt64(member.current, -offset))
|
||||
}
|
||||
}
|
||||
436
sdk/cliproxy/auth/selector_weighted_affinity_test.go
Normal file
436
sdk/cliproxy/auth/selector_weighted_affinity_test.go
Normal file
@@ -0,0 +1,436 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
cliproxysession "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/session"
|
||||
sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator"
|
||||
)
|
||||
|
||||
func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) {
|
||||
for _, source := range []string{"sticky", "parent", "guardian", "fork"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{
|
||||
{ID: "a", Provider: "codex", Attributes: map[string]string{AttributeWeight: "3"}},
|
||||
{ID: "b", Provider: "codex"},
|
||||
}
|
||||
parent := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
|
||||
parentID := CanonicalSessionID(parent.Headers, nil, nil)
|
||||
parentModel := "model"
|
||||
if source == "guardian" {
|
||||
parentModel = "parent-model"
|
||||
}
|
||||
key := "codex::" + parentID + "::" + parentModel
|
||||
selector.cache.setSessionAliases("b", "codex", parentID, key)
|
||||
for i := range 4 {
|
||||
opts := parent
|
||||
if source != "sticky" {
|
||||
opts = cliproxyexecutor.Options{Headers: http.Header{
|
||||
"Session_id": {fmt.Sprintf("child-%d", i)}, "X-Codex-Parent-Thread-Id": {"busy"},
|
||||
}}
|
||||
if source == "guardian" {
|
||||
opts.Headers.Set("X-Openai-Subagent", "guardian")
|
||||
}
|
||||
if source == "fork" {
|
||||
opts.OriginalRequest = []byte(`{"forked_from_thread_id":"busy"}`)
|
||||
}
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "b" {
|
||||
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
for i := range 12 {
|
||||
// Requests without an identity and cold sessions share the same credits.
|
||||
opts := cliproxyexecutor.Options{}
|
||||
if i%2 == 0 {
|
||||
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "a" {
|
||||
t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
}
|
||||
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
if binding, ok := selector.cache.Get(key); !ok || binding != "b" {
|
||||
t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing.T) {
|
||||
for _, newSession := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("new-session=%t", newSession), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{
|
||||
{ID: "a", Attributes: map[string]string{"priority": "0"}},
|
||||
{ID: "b", Attributes: map[string]string{"priority": "1"}},
|
||||
{ID: "c", Attributes: map[string]string{"priority": "1"}},
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"lower-priority"}}}
|
||||
id := CanonicalSessionID(sticky.Headers, nil, nil)
|
||||
key := "codex::" + id + "::model"
|
||||
selector.cache.Set(key, "a")
|
||||
counts := map[string]int{}
|
||||
const warmup, requests = 100, 1000
|
||||
for i := range requests {
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "a" {
|
||||
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
opts := cliproxyexecutor.Options{}
|
||||
if newSession {
|
||||
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}
|
||||
}
|
||||
picked, errPick = selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
|
||||
t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
if i >= warmup {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
}
|
||||
if counts["b"] != (requests-warmup)/2 || counts["c"] != (requests-warmup)/2 {
|
||||
t.Fatalf("changing pools starved equal-weight candidates after warmup: %v, want b=450 c=450", counts)
|
||||
}
|
||||
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
|
||||
t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok)
|
||||
}
|
||||
if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("pending credits leaked: %v", state.pendingForTest())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
counts := map[string]int{}
|
||||
for i := range 40 {
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{
|
||||
Headers: http.Header{"Session_id": {fmt.Sprintf("session-%d", i)}},
|
||||
}, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) {
|
||||
t.Fatalf("first selection charged more than once: %v", state.currentForTest())
|
||||
}
|
||||
counts[picked.ID]++
|
||||
}
|
||||
if counts["a"] != 30 || counts["b"] != 10 {
|
||||
t.Fatalf("cold distribution = %v, want 30:10", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"long"}}}
|
||||
id := CanonicalSessionID(opts.Headers, nil, nil)
|
||||
selector.cache.Set("codex::"+id+"::model", "b")
|
||||
for range 1000 {
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "b" {
|
||||
t.Fatalf("long session: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
limit := maxSmoothWeightedCreditRounds * 4
|
||||
// A bound charge is admitted as a whole, stopping within one delta of the cap.
|
||||
wantCredit := limit / 3 * 3
|
||||
if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit {
|
||||
t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit)
|
||||
}
|
||||
for range wantCredit {
|
||||
if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
|
||||
t.Fatal(errPick)
|
||||
}
|
||||
}
|
||||
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 {
|
||||
t.Fatalf("compensation did not finish: %v", state.currentForTest())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityLCPCredits(t *testing.T) {
|
||||
for _, source := range []string{"hit", "fork", "compaction"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
root := cliproxyexecutor.Options{
|
||||
SourceFormat: sdktranslator.FormatOpenAI,
|
||||
OriginalRequest: []byte(`{"messages":[{"role":"user","content":"step 1"},` +
|
||||
`{"role":"assistant","content":"ack 1"},{"role":"user","content":"step 2"},` +
|
||||
`{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"}]}`),
|
||||
Metadata: map[string]any{cliproxyexecutor.CallerScopeMetadataKey: "caller"},
|
||||
}
|
||||
// Seed the matcher without consuming weighted credits.
|
||||
turns := cliproxysession.ExtractCanonicalTurns(root.SourceFormat, root.OriginalRequest)
|
||||
selector.matcher.Bind(lcpAffinityNamespace("openai", "model", root.Metadata), turns, "b")
|
||||
opts := root
|
||||
if source == "fork" {
|
||||
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"step 1"},` +
|
||||
`{"role":"assistant","content":"ack 1"},{"role":"user","content":"branch"}]}`)
|
||||
} else if source == "compaction" {
|
||||
opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"<summary>Steps 1 and 2 completed</summary>"},` +
|
||||
`{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"},` +
|
||||
`{"role":"user","content":"step 4"}]}`)
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "openai", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "b" {
|
||||
t.Fatalf("LCP %s: pick=%v err=%v", source, picked, errPick)
|
||||
}
|
||||
if source == "fork" && opts.Metadata[cliproxyexecutor.IsForkMetadataKey] != true {
|
||||
t.Fatal("expected LCP fork")
|
||||
}
|
||||
if source == "compaction" && opts.Metadata[cliproxyexecutor.IsCompactionMetadataKey] != true {
|
||||
t.Fatal("expected LCP compaction")
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "openai:model", "a", "b")
|
||||
if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
key := "codex:model"
|
||||
for range 1000 {
|
||||
_, reservation := weighted.reserve(key, auths, "b")
|
||||
reservation.finish(true)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, key, "a", "b")
|
||||
_, rejected := weighted.reserve(key, auths, "b")
|
||||
_, accepted := weighted.reserve(key, auths, "a")
|
||||
accepted.finish(true)
|
||||
rejected.finish(false)
|
||||
rejected.finish(true)
|
||||
accepted.finish(false)
|
||||
limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3
|
||||
if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
key := "codex:model"
|
||||
_, old := weighted.reserve(key, auths, "")
|
||||
auths[0].Attributes[AttributeWeight] = "1"
|
||||
_, latest := weighted.reserve(key, auths, "")
|
||||
old.finish(true)
|
||||
state := weightedStateForTest(t, weighted, key, "a", "b")
|
||||
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 {
|
||||
t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
latest.finish(true)
|
||||
if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
const requests = 40
|
||||
selected := make(chan *Auth, requests)
|
||||
reservations := make(chan *weightedReservation, requests)
|
||||
var ready sync.WaitGroup
|
||||
ready.Add(requests)
|
||||
for range requests {
|
||||
go func() {
|
||||
defer ready.Done()
|
||||
picked, reservation := weighted.reserve("codex:model", auths, "")
|
||||
selected <- picked
|
||||
reservations <- reservation
|
||||
}()
|
||||
}
|
||||
ready.Wait()
|
||||
close(selected)
|
||||
close(reservations)
|
||||
counts := map[string]int{}
|
||||
for auth := range selected {
|
||||
if auth == nil {
|
||||
t.Fatal("concurrent reservation returned no auth")
|
||||
}
|
||||
counts[auth.ID]++
|
||||
}
|
||||
if counts["a"] != 30 || counts["b"] != 10 {
|
||||
t.Fatalf("pending allocations ignored: %v", counts)
|
||||
}
|
||||
for reservation := range reservations {
|
||||
reservation.finish(true)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityFailureDoesNotCharge(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{
|
||||
Fallback: weighted, RebindOn: SessionAffinityRebindOnNever,
|
||||
})
|
||||
t.Cleanup(selector.Stop)
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound"}}}
|
||||
id := CanonicalSessionID(opts.Headers, nil, nil)
|
||||
selector.cache.Set("codex::"+id+"::model", "missing")
|
||||
if auth, errPick := selector.Pick(t.Context(), "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); errPick == nil || auth != nil {
|
||||
t.Fatalf("expected unavailable binding: auth=%v err=%v", auth, errPick)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
if auth, errPick := selector.Pick(ctx, "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); !errors.Is(errPick, context.Canceled) || auth != nil {
|
||||
t.Fatalf("expected cancellation: auth=%v err=%v", auth, errPick)
|
||||
}
|
||||
if len(weighted.states) != 0 {
|
||||
t.Fatalf("failed selection changed weighted state: %v", weighted.states)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{
|
||||
{ID: "a", Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}},
|
||||
{ID: "b", Attributes: map[string]string{"priority": "1"}},
|
||||
{ID: "lower", Attributes: map[string]string{AttributeWeight: "100"}},
|
||||
{ID: "zero", Attributes: map[string]string{AttributeWeight: "0", "priority": "2"}},
|
||||
{ID: "cooldown", Attributes: map[string]string{"priority": "2"}, ModelStates: map[string]*ModelState{
|
||||
"upstream": {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)},
|
||||
}},
|
||||
}
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"alias"}}}
|
||||
id := CanonicalSessionID(opts.Headers, nil, nil)
|
||||
selector.cache.Set("codex::"+id+"::upstream", "b")
|
||||
ctx := withWeightedSelectorStateModel(t.Context(), weighted, "requested(high)")
|
||||
picked, errPick := selector.Pick(ctx, "codex", "upstream", opts, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "b" {
|
||||
t.Fatalf("sticky pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:requested", "a", "b")
|
||||
if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 {
|
||||
t.Fatalf("wrong model key or eligible pool: %v", weighted.states)
|
||||
}
|
||||
picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2])
|
||||
if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
|
||||
t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest())
|
||||
}
|
||||
for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} {
|
||||
if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil {
|
||||
t.Fatal(errOther)
|
||||
}
|
||||
}
|
||||
if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 {
|
||||
t.Fatal("another provider or model changed credits")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"reselect"}}}
|
||||
id := CanonicalSessionID(opts.Headers, nil, nil)
|
||||
key := "codex::" + id + "::model"
|
||||
selector.cache.Set(key, "removed")
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts,
|
||||
[]*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}})
|
||||
if errPick != nil || picked == nil || picked.ID != "a" {
|
||||
t.Fatalf("reselection: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
if binding, ok := selector.cache.Get(key); !ok || binding != "a" {
|
||||
t.Fatalf("reselection did not bind: %q, found=%t", binding, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerWeightedSessionAffinityCompensation(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("mixed=%t", mixed), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
manager := NewManager(nil, selector, nil)
|
||||
provider, model := "weighted-affinity", "weighted-model"
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
|
||||
aID, bID := t.Name()+"-a", t.Name()+"-b"
|
||||
for _, auth := range []*Auth{
|
||||
{ID: aID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "3"}},
|
||||
{ID: bID, Provider: provider, Status: StatusActive},
|
||||
} {
|
||||
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
|
||||
t.Fatal(errRegister)
|
||||
}
|
||||
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
|
||||
}
|
||||
namespace := provider
|
||||
if mixed {
|
||||
namespace = "mixed"
|
||||
}
|
||||
parentHeaders := http.Header{"Session_id": {"manager-sticky"}}
|
||||
id := CanonicalSessionID(parentHeaders, nil, nil)
|
||||
selector.cache.Set(namespace+"::"+id+"::"+model, bID)
|
||||
for i := range 16 {
|
||||
opts := cliproxyexecutor.Options{Headers: parentHeaders}
|
||||
want := bID
|
||||
if i >= 4 {
|
||||
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
|
||||
want = aID
|
||||
}
|
||||
var auth *Auth
|
||||
var errPick error
|
||||
if mixed {
|
||||
auth, _, _, errPick = manager.pickNextMixed(t.Context(), []string{provider}, model, opts, nil)
|
||||
} else {
|
||||
auth, _, errPick = manager.pickNext(t.Context(), provider, model, opts, nil)
|
||||
}
|
||||
if errPick != nil || auth == nil || auth.ID != want {
|
||||
t.Fatalf("request %d: auth=%v err=%v, want %q", i, auth, errPick, want)
|
||||
}
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID)
|
||||
if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("manager credits did not balance: %#v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
199
sdk/cliproxy/auth/selector_weighted_commit_order_test.go
Normal file
199
sdk/cliproxy/auth/selector_weighted_commit_order_test.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"testing"
|
||||
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) {
|
||||
for _, rollbackFirst := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("rollback-first=%t", rollbackFirst), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}}
|
||||
reservations := map[string][]*weightedReservation{}
|
||||
t.Cleanup(func() {
|
||||
for _, group := range reservations {
|
||||
for _, reservation := range group {
|
||||
reservation.finish(false)
|
||||
}
|
||||
}
|
||||
})
|
||||
for range 256 {
|
||||
picked, reservation := weighted.reserve("codex:model", auths, "")
|
||||
if picked == nil || reservation == nil {
|
||||
t.Fatal("missing cold reservation")
|
||||
}
|
||||
reservations[picked.ID] = append(reservations[picked.ID], reservation)
|
||||
}
|
||||
ids := []string{"a", "b"}
|
||||
if rollbackFirst {
|
||||
ids = []string{"b", "a"}
|
||||
}
|
||||
for _, id := range ids {
|
||||
for _, reservation := range reservations[id] {
|
||||
reservation.finish(id == "a")
|
||||
// Repeated finishing must not decrement the counter again.
|
||||
reservation.finish(false)
|
||||
}
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
limit := maxSmoothWeightedCreditRounds * 2
|
||||
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) {
|
||||
t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) {
|
||||
for _, first := range []string{"a", "b"} {
|
||||
t.Run(first, func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
reservations := map[string][]*weightedReservation{}
|
||||
t.Cleanup(func() {
|
||||
for _, group := range reservations {
|
||||
for _, reservation := range group {
|
||||
reservation.finish(false)
|
||||
}
|
||||
}
|
||||
})
|
||||
for range 512 {
|
||||
picked, reservation := weighted.reserve("codex:model", auths, "")
|
||||
if picked == nil || reservation == nil {
|
||||
t.Fatal("missing cold reservation")
|
||||
}
|
||||
reservations[picked.ID] = append(reservations[picked.ID], reservation)
|
||||
}
|
||||
if len(reservations["a"]) != 384 || len(reservations["b"]) != 128 {
|
||||
t.Fatalf("reservations did not follow 3:1: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
|
||||
}
|
||||
last := "b"
|
||||
if first == "b" {
|
||||
last = "a"
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
for _, reservation := range reservations[first] {
|
||||
reservation.finish(true)
|
||||
}
|
||||
// New arrivals must see the remaining offsets before the batch settles.
|
||||
for _, phase := range []string{"pending", "settled"} {
|
||||
if phase == "settled" {
|
||||
for _, reservation := range reservations[last] {
|
||||
reservation.finish(true)
|
||||
}
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for range 64 {
|
||||
picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("Pick failed: auth=%v err=%v", picked, errPick)
|
||||
}
|
||||
counts[picked.ID]++
|
||||
}
|
||||
if counts["a"] != 48 || counts["b"] != 16 {
|
||||
t.Fatalf("%s distribution = %v, want 48:16", phase, counts)
|
||||
}
|
||||
}
|
||||
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) {
|
||||
t.Fatalf("settled 3:1 batch drifted: %#v", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}}
|
||||
var old []*weightedReservation
|
||||
t.Cleanup(func() {
|
||||
for _, reservation := range old {
|
||||
reservation.finish(false)
|
||||
}
|
||||
})
|
||||
for range 256 {
|
||||
_, reservation := weighted.reserve("codex:model", auths, "")
|
||||
old = append(old, reservation)
|
||||
}
|
||||
auths[0].Attributes = map[string]string{AttributeWeight: "3"}
|
||||
_, latest := weighted.reserve("codex:model", auths, "")
|
||||
if latest == nil {
|
||||
t.Fatal("missing new-generation reservation")
|
||||
}
|
||||
t.Cleanup(func() { latest.finish(false) })
|
||||
for _, reservation := range old {
|
||||
reservation.finish(true)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
|
||||
t.Fatalf("old generation altered new accounting: %#v", state)
|
||||
}
|
||||
latest.finish(true)
|
||||
if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) {
|
||||
t.Fatalf("new generation failed to settle: %#v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) {
|
||||
for _, bound := range []bool{false, true} {
|
||||
for _, first := range []string{"a", "b"} {
|
||||
t.Run(fmt.Sprintf("bound=%t/first=%s", bound, first), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}}
|
||||
reservations := map[string][]*weightedReservation{}
|
||||
t.Cleanup(func() {
|
||||
for _, group := range reservations {
|
||||
for _, reservation := range group {
|
||||
reservation.finish(false)
|
||||
}
|
||||
}
|
||||
})
|
||||
for i := range 256 {
|
||||
boundID := ""
|
||||
if bound {
|
||||
boundID = auths[i%2].ID
|
||||
}
|
||||
picked, reservation := weighted.reserve("codex:model", auths, boundID)
|
||||
if picked == nil || reservation == nil {
|
||||
t.Fatalf("reserve %d returned pick=%v reservation=%v", i, picked, reservation)
|
||||
}
|
||||
reservations[picked.ID] = append(reservations[picked.ID], reservation)
|
||||
}
|
||||
if len(reservations["a"]) != 128 || len(reservations["b"]) != 128 {
|
||||
t.Fatalf("unbalanced reservations: a=%d b=%d", len(reservations["a"]), len(reservations["b"]))
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
// A zero net pending vector does not mean all reservations settled.
|
||||
if state.inFlight != 256 || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest())
|
||||
}
|
||||
last := "b"
|
||||
if first == "b" {
|
||||
last = "a"
|
||||
}
|
||||
for _, id := range []string{first, last} {
|
||||
for _, reservation := range reservations[id] {
|
||||
reservation.finish(true)
|
||||
}
|
||||
}
|
||||
if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 {
|
||||
t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for range 64 {
|
||||
picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("Pick: auth=%v err=%v", picked, errPick)
|
||||
}
|
||||
counts[picked.ID]++
|
||||
}
|
||||
if counts["a"] != 32 || counts["b"] != 32 {
|
||||
t.Fatalf("post-commit distribution = %v, want a=32 b=32", counts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
71
sdk/cliproxy/auth/selector_weighted_memory_test.go
Normal file
71
sdk/cliproxy/auth/selector_weighted_memory_test.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkWeightedPoolReserve1000(b *testing.B) {
|
||||
for _, bound := range []bool{false, true} {
|
||||
b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) {
|
||||
auths := make([]*Auth, 1000)
|
||||
for i := range auths {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
|
||||
}
|
||||
selector := &WeightedRoundRobinSelector{}
|
||||
_, warmup := selector.reserve("codex:model", auths, "")
|
||||
warmup.finish(true)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
boundID := ""
|
||||
if bound {
|
||||
boundID = auths[i%len(auths)].ID
|
||||
}
|
||||
_, reservation := selector.reserve("codex:model", auths, boundID)
|
||||
reservation.finish(true)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from
|
||||
// total heap and allocation churn. Each retry subset omits a different credential.
|
||||
func BenchmarkWeightedPoolRetainedMemory(b *testing.B) {
|
||||
const members = 1000
|
||||
auths := make([]*Auth, members)
|
||||
for i := range auths {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)}
|
||||
}
|
||||
subset := make([]*Auth, 0, members-1)
|
||||
var retained, heap, allocated int64
|
||||
b.ResetTimer()
|
||||
for range b.N {
|
||||
selector := &WeightedRoundRobinSelector{}
|
||||
_, warmup := selector.reserve("codex:model", auths, "")
|
||||
warmup.finish(true)
|
||||
runtime.GC()
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
for excluded := range auths {
|
||||
subset = append(subset[:0], auths[:excluded]...)
|
||||
subset = append(subset, auths[excluded+1:]...)
|
||||
_, reservation := selector.reserve("codex:model", subset, "")
|
||||
reservation.finish(true)
|
||||
}
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&after)
|
||||
if len(selector.states) != members+1 {
|
||||
b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1)
|
||||
}
|
||||
runtime.KeepAlive(selector)
|
||||
runtime.KeepAlive(auths)
|
||||
retained += int64(after.HeapAlloc) - int64(before.HeapAlloc)
|
||||
heap += int64(after.HeapAlloc)
|
||||
allocated += int64(after.TotalAlloc - before.TotalAlloc)
|
||||
}
|
||||
b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op")
|
||||
b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op")
|
||||
b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op")
|
||||
}
|
||||
39
sdk/cliproxy/auth/selector_weighted_model_key_test.go
Normal file
39
sdk/cliproxy/auth/selector_weighted_model_key_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
func TestWeightedSelectorsShareEquivalentModelKeys(t *testing.T) {
|
||||
for _, firstAffinity := range []bool{false, true} {
|
||||
for _, secondAffinity := range []bool{false, true} {
|
||||
for _, routeOverride := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("affinity=%t/%t/route=%t", firstAffinity, secondAffinity, routeOverride), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
affinity := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(affinity.Stop)
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}}
|
||||
for i, useAffinity := range []bool{firstAffinity, secondAffinity} {
|
||||
ctx := t.Context()
|
||||
model := []string{" Model(high) ", "model(low)"}[i]
|
||||
if routeOverride {
|
||||
ctx = withWeightedSelectorStateModel(ctx, weighted, model)
|
||||
model = []string{"upstream-a", "upstream-b"}[i]
|
||||
}
|
||||
var selector Selector = weighted
|
||||
if useAffinity {
|
||||
selector = affinity
|
||||
}
|
||||
picked, errPick := selector.Pick(ctx, "codex", model, cliproxyexecutor.Options{}, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != auths[i].ID {
|
||||
t.Fatalf("request %d: pick=%v err=%v, want auth %s", i, picked, errPick, auths[i].ID)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
455
sdk/cliproxy/auth/selector_weighted_overload_test.go
Normal file
455
sdk/cliproxy/auth/selector_weighted_overload_test.go
Normal file
@@ -0,0 +1,455 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
func TestWeightedOverloadExactCompensation(t *testing.T) {
|
||||
for _, weights := range [][]int{{1, 1}, {3, 1}, {1, 3}, {1, 5, 1}, {2, 3, 5}, {1, 2, 3, 4}} {
|
||||
for boundIndex := range weights {
|
||||
t.Run(fmt.Sprintf("weights=%v/bound=%d", weights, boundIndex), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := make([]*Auth, len(weights))
|
||||
total := 0
|
||||
for i, weight := range weights {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
|
||||
total += weight
|
||||
}
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"sticky"}}}
|
||||
key := "codex::" + CanonicalSessionID(opts.Headers, nil, nil) + "::model"
|
||||
selector.cache.Set(key, auths[boundIndex].ID)
|
||||
counts := map[string]int{}
|
||||
for i := range total * 4 {
|
||||
request := opts
|
||||
sticky := i < weights[boundIndex]*4
|
||||
if !sticky {
|
||||
request = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", request, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("selection failed: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
if sticky && picked.ID != auths[boundIndex].ID {
|
||||
t.Fatal("balancing moved an existing binding")
|
||||
}
|
||||
counts[picked.ID]++
|
||||
}
|
||||
for i, weight := range weights {
|
||||
if counts[auths[i].ID] != weight*4 {
|
||||
t.Errorf("total allocation = %v, want weight vector %v times 4", counts, weights)
|
||||
}
|
||||
}
|
||||
for _, state := range weighted.states {
|
||||
for id, credit := range state.currentForTest() {
|
||||
if credit != 0 {
|
||||
t.Errorf("exact compensation left %q credit=%d", id, credit)
|
||||
}
|
||||
}
|
||||
if len(state.pendingForTest()) != 0 {
|
||||
t.Errorf("pending credit leaked: %v", state.pendingForTest())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) {
|
||||
for _, weights := range [][]int{{1, 1, 1}, {1, 3, 1}, {1, 5, 1}, {1, 1, 5}, {1, 9, 1}, {1, 1, 9}, {1, 5, 2, 1}} {
|
||||
t.Run(fmt.Sprint(weights), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := make([]*Auth, len(weights))
|
||||
for i, weight := range weights {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
|
||||
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
|
||||
selector.cache.Set(key, auths[0].ID)
|
||||
counts := map[string]int{}
|
||||
for i := range 3000 {
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != auths[0].ID {
|
||||
t.Fatalf("binding changed: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
fresh := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
|
||||
picked, errPick = selector.Pick(t.Context(), "codex", "model", fresh, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("fresh selection failed: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
if i >= 1000 {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
}
|
||||
ids := make([]string, len(auths))
|
||||
for i, auth := range auths {
|
||||
ids[i] = auth.ID
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", ids...)
|
||||
t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest())
|
||||
// Forced traffic makes global ratios infeasible; the remaining capacity
|
||||
// should be shared by unbound peers rather than monopolized by one.
|
||||
peerWeight := 0
|
||||
for _, weight := range weights[1:] {
|
||||
peerWeight += weight
|
||||
}
|
||||
for i, auth := range auths[1:] {
|
||||
want := 2000 * weights[i+1] / peerWeight
|
||||
if counts[auth.ID] < want-2 || counts[auth.ID] > want+2 {
|
||||
t.Errorf("peer %q got %d fresh requests, want approximately %d; distribution=%v", auth.ID, counts[auth.ID], want, counts)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadFeasibleSustainedTraffic(t *testing.T) {
|
||||
for _, weights := range [][]int{{3, 1}, {1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
|
||||
t.Run(fmt.Sprint(weights), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := make([]*Auth, len(weights))
|
||||
total := 0
|
||||
for i, weight := range weights {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
|
||||
total += weight
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"feasible"}}}
|
||||
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
|
||||
selector.cache.Set(key, auths[0].ID)
|
||||
counts := map[string]int{}
|
||||
// Forced traffic is below A's target share, so exact aggregate ratios
|
||||
// remain feasible without moving established bindings.
|
||||
for i := range total * 2000 {
|
||||
opts := cliproxyexecutor.Options{}
|
||||
bound := i%(total*2) == 0
|
||||
if bound {
|
||||
opts = sticky
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
if errPick != nil || picked == nil || (bound && picked.ID != auths[0].ID) {
|
||||
t.Fatalf("request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
if i >= total*1000 {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
}
|
||||
for i, weight := range weights {
|
||||
if counts[auths[i].ID] != weight*1000 {
|
||||
t.Errorf("feasible distribution drifted: counts=%v weights=%v", counts, weights)
|
||||
}
|
||||
}
|
||||
t.Logf("feasible distribution: %v", counts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadRecoveryAfterStickyTrafficStops(t *testing.T) {
|
||||
for _, weights := range [][]int{{1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} {
|
||||
t.Run(fmt.Sprint(weights), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := make([]*Auth, len(weights))
|
||||
total := 0
|
||||
for i, weight := range weights {
|
||||
auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}}
|
||||
total += weight
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}}
|
||||
key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model"
|
||||
selector.cache.Set(key, auths[0].ID)
|
||||
for range 3000 {
|
||||
if picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths); errPick != nil || picked == nil || picked.ID != auths[0].ID {
|
||||
t.Fatalf("sticky selection failed: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
if _, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil {
|
||||
t.Fatal(errPick)
|
||||
}
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range 1000 + total*100 {
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("recovery selection failed: pick=%v err=%v", picked, errPick)
|
||||
}
|
||||
if i >= 1000 {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
}
|
||||
for i, weight := range weights {
|
||||
if counts[auths[i].ID] != weight*100 {
|
||||
t.Errorf("allocation did not recover after sticky traffic stopped: counts=%v weights=%v", counts, weights)
|
||||
}
|
||||
}
|
||||
t.Logf("recovered distribution: %v", counts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}}
|
||||
const key = "codex:model"
|
||||
const inFlight = 200
|
||||
reservations := make([]*weightedReservation, inFlight)
|
||||
t.Cleanup(func() {
|
||||
for _, reservation := range reservations {
|
||||
reservation.finish(false)
|
||||
}
|
||||
})
|
||||
for i := range reservations {
|
||||
picked, reservation := weighted.reserve(key, auths, "a")
|
||||
if picked == nil || picked.ID != "a" {
|
||||
t.Fatal("pending sticky request lost its binding")
|
||||
}
|
||||
reservations[i] = reservation
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range 3000 {
|
||||
// Maintain a fixed backlog without sleeps or timing-dependent coordination.
|
||||
reservations[i%inFlight].finish(true)
|
||||
_, reservations[i%inFlight] = weighted.reserve(key, auths, "a")
|
||||
picked, reservation := weighted.reserve(key, auths, "")
|
||||
reservation.finish(true)
|
||||
if i >= 1000 {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
}
|
||||
if counts["c"] == 0 {
|
||||
t.Fatalf("speculative sticky traffic starved c: %v", counts)
|
||||
}
|
||||
for _, reservation := range reservations {
|
||||
reservation.finish(true)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, key, "a", "b", "c")
|
||||
if len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
const key = "codex:model"
|
||||
for range 1000 {
|
||||
_, reservation := weighted.reserve(key, auths, "b")
|
||||
reservation.finish(true)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, key, "a", "b")
|
||||
before := maps.Clone(state.currentForTest())
|
||||
_, first := weighted.reserve(key, auths, "")
|
||||
_, second := weighted.reserve(key, auths, "")
|
||||
t.Cleanup(func() { first.finish(false); second.finish(false) })
|
||||
// Two pending cold selections make room for a bound delta, but neither
|
||||
// repayment is real yet. Its later commit must recheck the committed cap.
|
||||
picked, bound := weighted.reserve(key, auths, "b")
|
||||
if picked == nil || picked.ID != "b" || bound == nil {
|
||||
t.Fatal("expected bound reservation funded by pending repayments")
|
||||
}
|
||||
t.Cleanup(func() { bound.finish(false) })
|
||||
first.finish(false)
|
||||
second.finish(false)
|
||||
bound.finish(true)
|
||||
bound.finish(true)
|
||||
bound.finish(false)
|
||||
if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
// An over-limit bound selection succeeds without reserving any credit.
|
||||
picked, skipped := weighted.reserve(key, auths, "b")
|
||||
if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatal("saturated bound selection should preserve binding and all credits")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadMultipleBusyBindings(t *testing.T) {
|
||||
for _, busyPerFresh := range []int{1, 3, 10} {
|
||||
t.Run(fmt.Sprint(busyPerFresh), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{
|
||||
{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}, {ID: "d"},
|
||||
}
|
||||
boundOpts := make([]cliproxyexecutor.Options, 2)
|
||||
for i, id := range []string{"a", "d"} {
|
||||
boundOpts[i].Headers = http.Header{"Session_id": {"busy-" + id}}
|
||||
sessionID := CanonicalSessionID(boundOpts[i].Headers, nil, nil)
|
||||
selector.cache.Set("codex::"+sessionID+"::model", id)
|
||||
}
|
||||
windowCounts := map[string]int{}
|
||||
for i := range 1400 {
|
||||
for j := range busyPerFresh {
|
||||
index := (i + j) % len(boundOpts)
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", boundOpts[index], auths)
|
||||
if errPick != nil || picked == nil || picked.ID != []string{"a", "d"}[index] {
|
||||
t.Fatalf("busy binding changed: auth=%v err=%v", picked, errPick)
|
||||
}
|
||||
}
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths)
|
||||
if errPick != nil || picked == nil {
|
||||
t.Fatalf("fresh pick failed: auth=%v err=%v", picked, errPick)
|
||||
}
|
||||
if i >= 1000 {
|
||||
windowCounts[picked.ID]++
|
||||
if (i+1)%100 == 0 {
|
||||
if windowCounts["b"] == 0 || windowCounts["c"] == 0 {
|
||||
t.Fatalf("busy=%d unbound peer starved in a 100-request window: %v", busyPerFresh, windowCounts)
|
||||
}
|
||||
clear(windowCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) {
|
||||
for _, credit := range []map[string]int64{
|
||||
{"a": -222, "b": 111, "c": 111},
|
||||
{"a": -112, "b": 222, "c": -110},
|
||||
} {
|
||||
state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit)
|
||||
state.charge(0, true)
|
||||
if !maps.Equal(state.currentForTest(), credit) {
|
||||
t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
for _, pinned := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
manager := NewManager(nil, selector, nil)
|
||||
const provider, model = "sticky-overload", "sticky-overload-model"
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
|
||||
aID, bID, cID := t.Name()+"-a", t.Name()+"-b", t.Name()+"-c"
|
||||
for _, auth := range []*Auth{
|
||||
{ID: aID, Provider: provider, Status: StatusActive},
|
||||
{ID: bID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "5"}},
|
||||
{ID: cID, Provider: provider, Status: StatusActive},
|
||||
} {
|
||||
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
|
||||
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
|
||||
t.Fatal(errRegister)
|
||||
}
|
||||
}
|
||||
namespace := provider
|
||||
if mixed {
|
||||
namespace = "mixed"
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}, Metadata: map[string]any{}}
|
||||
key := namespace + "::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::" + model
|
||||
selector.cache.Set(key, aID)
|
||||
if pinned {
|
||||
sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = aID
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range 1600 {
|
||||
for _, bound := range []bool{true, false} {
|
||||
opts := sticky
|
||||
if !bound {
|
||||
opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}}
|
||||
}
|
||||
var auth *Auth
|
||||
var errPick error
|
||||
if mixed {
|
||||
auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
|
||||
} else {
|
||||
auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
|
||||
}
|
||||
if errPick != nil || auth == nil || (bound && auth.ID != aID) {
|
||||
t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
|
||||
}
|
||||
if !bound && i >= 1000 {
|
||||
counts[auth.ID]++
|
||||
}
|
||||
}
|
||||
}
|
||||
if counts[bID] != 500 || counts[cID] != 100 {
|
||||
t.Fatalf("manager overload allocation = %v, want b=500 c=100", counts)
|
||||
}
|
||||
if binding, ok := selector.cache.Get(key); !ok || binding != aID {
|
||||
t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok)
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID)
|
||||
if len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("manager left speculative credit: %v", state.pendingForTest())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}
|
||||
headers := http.Header{"Session_id": {"sticky-concurrent"}}
|
||||
key := "codex::" + CanonicalSessionID(headers, nil, nil) + "::model"
|
||||
selector.cache.Set(key, "b")
|
||||
type selection struct {
|
||||
id string
|
||||
err error
|
||||
sticky bool
|
||||
}
|
||||
results := make(chan selection, 64)
|
||||
start := make(chan struct{})
|
||||
var done sync.WaitGroup
|
||||
for i := range 64 {
|
||||
done.Add(1)
|
||||
go func() {
|
||||
defer done.Done()
|
||||
isSticky := i < 8
|
||||
opts := cliproxyexecutor.Options{Headers: headers}
|
||||
if !isSticky {
|
||||
opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}
|
||||
}
|
||||
<-start
|
||||
picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths)
|
||||
id := ""
|
||||
if picked != nil {
|
||||
id = picked.ID
|
||||
}
|
||||
results <- selection{id: id, err: errPick, sticky: isSticky}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
done.Wait()
|
||||
close(results)
|
||||
counts := map[string]int{}
|
||||
for result := range results {
|
||||
if result.err != nil || result.id == "" || (result.sticky && result.id != "b") {
|
||||
t.Fatalf("invalid concurrent result: %+v", result)
|
||||
}
|
||||
counts[result.id]++
|
||||
}
|
||||
state := weightedStateForTest(t, weighted, "codex:model", "a", "b")
|
||||
// All 64 calls share one pool. Accounting must match actual accepted picks,
|
||||
// regardless of whether the final arrivals happened to be sticky requests.
|
||||
if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
if bound, ok := selector.cache.Get(key); !ok || bound != "b" {
|
||||
t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok)
|
||||
}
|
||||
}
|
||||
287
sdk/cliproxy/auth/selector_weighted_pool_test.go
Normal file
287
sdk/cliproxy/auth/selector_weighted_pool_test.go
Normal file
@@ -0,0 +1,287 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/router-for-me/CLIProxyAPI/v8/internal/registry"
|
||||
cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor"
|
||||
)
|
||||
|
||||
// weightedStateForTest checks that the requested accounting pool was created.
|
||||
func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState {
|
||||
t.Helper()
|
||||
weights := make(map[string]int64, len(ids))
|
||||
for _, id := range ids {
|
||||
weights[id] = 1
|
||||
}
|
||||
state := selector.states[weightedPoolStateKey(route, weights)]
|
||||
ids = sortedWeightedIDs(weights)
|
||||
for state != nil && !state.matches(route, ids) {
|
||||
state = state.next
|
||||
}
|
||||
if state == nil {
|
||||
t.Fatalf("missing weighted state for %q and %v", route, ids)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}, {ID: "c"}}
|
||||
key := "codex:model"
|
||||
_, pendingFull := weighted.reserve(key, auths, "b")
|
||||
t.Cleanup(func() { pendingFull.finish(false) })
|
||||
_, pendingSubset := weighted.reserve(key, auths[:2], "b")
|
||||
t.Cleanup(func() { pendingSubset.finish(false) })
|
||||
pendingSubset.finish(true)
|
||||
subset := weightedStateForTest(t, weighted, key, "a", "b")
|
||||
full := weightedStateForTest(t, weighted, key, "a", "b", "c")
|
||||
if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() {
|
||||
t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest())
|
||||
}
|
||||
if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) {
|
||||
t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest())
|
||||
}
|
||||
pendingFull.finish(false)
|
||||
for range 100 {
|
||||
_, reservation := weighted.reserve(key, auths, "b")
|
||||
reservation.finish(true)
|
||||
}
|
||||
// A returning pool resumes its debt, independent of full-pool traffic.
|
||||
for range 3 {
|
||||
picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "")
|
||||
if picked == nil || picked.ID != "a" {
|
||||
t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest())
|
||||
}
|
||||
reservation.finish(true)
|
||||
}
|
||||
if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 {
|
||||
t.Fatalf("returning subset failed to balance: %#v", subset)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) {
|
||||
shared := &WeightedRoundRobinSelector{}
|
||||
auths := []*Auth{
|
||||
{ID: "a", Attributes: map[string]string{AttributeWeight: "1"}},
|
||||
{ID: "b", Attributes: map[string]string{AttributeWeight: "3"}},
|
||||
{ID: "c", Attributes: map[string]string{AttributeWeight: "2"}},
|
||||
{ID: "d", Attributes: map[string]string{AttributeWeight: "5"}},
|
||||
}
|
||||
var independent [16]WeightedRoundRobinSelector
|
||||
rng := rand.New(rand.NewPCG(17, 29))
|
||||
for step := range 5000 {
|
||||
mask := 1 + rng.IntN(15)
|
||||
var pool []*Auth
|
||||
for i, auth := range auths {
|
||||
if mask&(1<<i) != 0 {
|
||||
pool = append(pool, auth)
|
||||
}
|
||||
}
|
||||
// Membership and weights, not input order or other pools, define credit.
|
||||
rng.Shuffle(len(pool), func(i, j int) { pool[i], pool[j] = pool[j], pool[i] })
|
||||
boundID := ""
|
||||
if rng.IntN(3) == 0 {
|
||||
boundID = pool[rng.IntN(len(pool))].ID
|
||||
}
|
||||
picked, reservation := shared.reserve("route", pool, boundID)
|
||||
want, independentReservation := independent[mask].reserve("route", pool, boundID)
|
||||
if picked == nil || want == nil || picked.ID != want.ID {
|
||||
t.Fatalf("step %d pool %d: shared=%v independent=%v", step, mask, picked, want)
|
||||
}
|
||||
committed := rng.IntN(4) != 0
|
||||
reservation.finish(committed)
|
||||
independentReservation.finish(committed)
|
||||
poolKey := weightedPoolStateKey("route", authWeightVector(pool))
|
||||
state, independentState := shared.states[poolKey], independent[mask].states[poolKey]
|
||||
if !maps.Equal(state.currentForTest(), independentState.currentForTest()) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("step %d pool %d: credits diverged or pending leaked", step, mask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedPoolKeysAndCapacity(t *testing.T) {
|
||||
seen := make(map[[32]byte]struct{})
|
||||
for _, tc := range []struct {
|
||||
route string
|
||||
ids []string
|
||||
}{
|
||||
{route: "route", ids: []string{"a", "bc"}},
|
||||
{route: "route", ids: []string{"ab", "c"}},
|
||||
{route: "route1:a", ids: []string{"bc"}},
|
||||
{route: "route", ids: []string{"a1:b", "c"}},
|
||||
} {
|
||||
t.Run(fmt.Sprint(tc), func(t *testing.T) {
|
||||
weights := make(map[string]int64)
|
||||
for _, id := range tc.ids {
|
||||
weights[id] = 1
|
||||
}
|
||||
key := weightedPoolStateKey(tc.route, weights)
|
||||
if _, duplicate := seen[key]; duplicate {
|
||||
t.Fatal("different pool identities collided")
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
for _, id := range tc.ids {
|
||||
weights[id] = 3
|
||||
}
|
||||
if key != weightedPoolStateKey(tc.route, weights) {
|
||||
t.Fatal("weight change changed pool identity instead of resetting credit")
|
||||
}
|
||||
})
|
||||
}
|
||||
weighted := &WeightedRoundRobinSelector{maxKeys: 2}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}}
|
||||
_, old := weighted.reserve("route", auths, "b")
|
||||
t.Cleanup(func() { old.finish(false) })
|
||||
for _, auth := range auths {
|
||||
_, reservation := weighted.reserve("route", []*Auth{auth}, auth.ID)
|
||||
reservation.finish(true)
|
||||
}
|
||||
_, current := weighted.reserve("route", auths, "a")
|
||||
t.Cleanup(func() { current.finish(false) })
|
||||
if len(weighted.states) > weighted.maxKeys || old.state == current.state {
|
||||
t.Fatal("pool eviction failed to bound or replace state")
|
||||
}
|
||||
old.finish(true)
|
||||
if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) {
|
||||
t.Fatal("finishing an evicted reservation affected the new pool")
|
||||
}
|
||||
current.finish(true)
|
||||
}
|
||||
|
||||
func TestManagerWeightedPriorityPoolIsolation(t *testing.T) {
|
||||
for _, mixed := range []bool{false, true} {
|
||||
for _, pinned := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
manager := NewManager(nil, selector, nil)
|
||||
const provider, model = "pool-isolation", "pool-model"
|
||||
manager.RegisterExecutor(schedulerTestExecutor{provider: provider})
|
||||
auths := []*Auth{
|
||||
{ID: t.Name() + "-a", Attributes: map[string]string{"priority": "0"}},
|
||||
{ID: t.Name() + "-b", Attributes: map[string]string{"priority": "1", AttributeWeight: "3"}},
|
||||
{ID: t.Name() + "-c", Attributes: map[string]string{"priority": "1", AttributeWeight: "1"}},
|
||||
}
|
||||
for _, auth := range auths {
|
||||
auth.Provider, auth.Status = provider, StatusActive
|
||||
registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}})
|
||||
t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) })
|
||||
if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil {
|
||||
t.Fatal(errRegister)
|
||||
}
|
||||
}
|
||||
namespace := provider
|
||||
if mixed {
|
||||
namespace = "mixed"
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"priority-bound"}}, Metadata: map[string]any{}}
|
||||
id := CanonicalSessionID(sticky.Headers, nil, nil)
|
||||
selector.cache.Set(namespace+"::"+id+"::"+model, auths[0].ID)
|
||||
if pinned {
|
||||
sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = auths[0].ID
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for i := range 1200 {
|
||||
for _, bound := range []bool{true, false} {
|
||||
opts := sticky
|
||||
if !bound {
|
||||
opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
|
||||
}
|
||||
var auth *Auth
|
||||
var errPick error
|
||||
if mixed {
|
||||
auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil)
|
||||
} else {
|
||||
auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil)
|
||||
}
|
||||
if errPick != nil || auth == nil || (bound && auth.ID != auths[0].ID) || (!bound && auth.ID == auths[0].ID) {
|
||||
t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick)
|
||||
}
|
||||
if !bound && i >= 200 {
|
||||
counts[auth.ID]++
|
||||
}
|
||||
}
|
||||
}
|
||||
if counts[auths[1].ID] != 750 || counts[auths[2].ID] != 250 {
|
||||
t.Fatalf("manager cross-priority distribution = %v, want 750:250", counts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedSessionAffinityPoolIsolation(t *testing.T) {
|
||||
for _, reason := range []string{"priority", "websocket", "retry-exclusion", "cooldown"} {
|
||||
for _, weights := range [][2]int{{1, 1}, {3, 1}, {1, 3}, {5, 2}} {
|
||||
t.Run(fmt.Sprintf("%s/%d:%d", reason, weights[0], weights[1]), func(t *testing.T) {
|
||||
weighted := &WeightedRoundRobinSelector{}
|
||||
selector := NewSessionAffinitySelector(weighted)
|
||||
t.Cleanup(selector.Stop)
|
||||
auths := []*Auth{
|
||||
{ID: "a", Attributes: map[string]string{"priority": "0"}},
|
||||
{ID: "b", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[0])}},
|
||||
{ID: "c", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[1])}},
|
||||
}
|
||||
ctx := t.Context()
|
||||
switch reason {
|
||||
case "priority":
|
||||
auths[1].Attributes["priority"] = "1"
|
||||
auths[2].Attributes["priority"] = "1"
|
||||
case "websocket":
|
||||
ctx = cliproxyexecutor.WithDownstreamWebsocket(ctx)
|
||||
auths[1].Attributes["websockets"] = "true"
|
||||
auths[2].Attributes["websockets"] = "true"
|
||||
}
|
||||
sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound-a"}}}
|
||||
id := CanonicalSessionID(sticky.Headers, nil, nil)
|
||||
key := "codex::" + id + "::model"
|
||||
selector.cache.Set(key, "a")
|
||||
const warmup, measured = 1000, 1400
|
||||
counts := map[string]int{}
|
||||
for i := range warmup + measured {
|
||||
picked, errPick := selector.Pick(ctx, "codex", "model", sticky, auths)
|
||||
if errPick != nil || picked == nil || picked.ID != "a" {
|
||||
t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
freshAuths := auths
|
||||
if reason == "retry-exclusion" {
|
||||
freshAuths = auths[1:]
|
||||
} else if reason == "cooldown" {
|
||||
auths[0].Unavailable = true
|
||||
auths[0].NextRetryAfter = time.Now().Add(time.Hour)
|
||||
}
|
||||
opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}
|
||||
picked, errPick = selector.Pick(ctx, "codex", "model", opts, freshAuths)
|
||||
if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") {
|
||||
t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick)
|
||||
}
|
||||
if i >= warmup {
|
||||
counts[picked.ID]++
|
||||
}
|
||||
auths[0].Unavailable = false
|
||||
auths[0].NextRetryAfter = time.Time{}
|
||||
}
|
||||
wantB := measured * weights[0] / (weights[0] + weights[1])
|
||||
if counts["b"] != wantB || counts["c"] != measured-wantB {
|
||||
t.Fatalf("cross-pool traffic distorted cold selections: %v, want b=%d c=%d", counts, wantB, measured-wantB)
|
||||
}
|
||||
if bound, ok := selector.cache.Get(key); !ok || bound != "a" {
|
||||
t.Fatalf("binding moved: %q, found=%t", bound, ok)
|
||||
}
|
||||
for _, state := range weighted.states {
|
||||
if len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("pending credits leaked: %v", state.pendingForTest())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
88
sdk/cliproxy/auth/selector_weighted_storage_test.go
Normal file
88
sdk/cliproxy/auth/selector_weighted_storage_test.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// These snapshots keep assertions independent of the compact production layout.
|
||||
func (s *weightedPoolState) currentForTest() map[string]int64 {
|
||||
current := make(map[string]int64, len(s.members))
|
||||
for _, member := range s.members {
|
||||
current[member.id] = member.current
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func (s *weightedPoolState) pendingForTest() map[string]int64 {
|
||||
pending := make(map[string]int64)
|
||||
for _, member := range s.members {
|
||||
if member.pending != 0 {
|
||||
pending[member.id] = member.pending
|
||||
}
|
||||
}
|
||||
return pending
|
||||
}
|
||||
|
||||
func (s *weightedPoolState) hasCurrentForTest() bool {
|
||||
for _, member := range s.members {
|
||||
if member.current != 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact
|
||||
// identities, including a matching state behind a collision-chain head.
|
||||
func TestWeightedPoolDigestCollisionIsolation(t *testing.T) {
|
||||
for _, unrelated := range []*weightedPoolState{
|
||||
weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}),
|
||||
weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}),
|
||||
weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}),
|
||||
} {
|
||||
t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) {
|
||||
weights := map[string]int64{"a": 1, "b": 1}
|
||||
key := weightedPoolStateKey("route", weights)
|
||||
selector := &WeightedRoundRobinSelector{
|
||||
states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2,
|
||||
}
|
||||
before := unrelated.currentForTest()
|
||||
picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b")
|
||||
if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated {
|
||||
t.Fatal("digest collision reused an unrelated pool")
|
||||
}
|
||||
t.Cleanup(func() { first.finish(false) })
|
||||
matching := first.state
|
||||
// Put the nonmatching state first to exercise a full chain lookup.
|
||||
matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated
|
||||
_, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a")
|
||||
if second == nil || second.state != matching || selector.stateCount != 2 {
|
||||
t.Fatal("collision lookup failed to reuse the matching pool")
|
||||
}
|
||||
t.Cleanup(func() { second.finish(false) })
|
||||
first.finish(true)
|
||||
second.finish(true)
|
||||
if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 {
|
||||
t.Fatal("collision accounting changed the unrelated pool")
|
||||
}
|
||||
if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 {
|
||||
t.Fatal("matching pool did not balance")
|
||||
}
|
||||
// Capacity still counts pools, not just distinct digest buckets.
|
||||
_, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "")
|
||||
third.finish(true)
|
||||
if selector.stateCount != 1 || len(selector.states) != 1 {
|
||||
t.Fatal("colliding pools bypassed the existing capacity limit")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState {
|
||||
state := &weightedPoolState{route: route}
|
||||
for _, id := range sortedWeightedIDs(weights) {
|
||||
state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]})
|
||||
}
|
||||
return state
|
||||
}
|
||||
79
sdk/cliproxy/auth/selector_weighted_test.go
Normal file
79
sdk/cliproxy/auth/selector_weighted_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
current map[string]int64
|
||||
weights map[string]int64
|
||||
authID string
|
||||
want map[string]int64
|
||||
}{
|
||||
{
|
||||
name: "smaller pool preserves leading difference",
|
||||
current: map[string]int64{"a": -96, "b": 96, "c": 95},
|
||||
weights: map[string]int64{"b": 1, "c": 1},
|
||||
authID: "b",
|
||||
want: map[string]int64{"b": 63, "c": 64},
|
||||
},
|
||||
{
|
||||
name: "negative credits shift together",
|
||||
current: map[string]int64{"a": 96, "b": -64, "c": -63},
|
||||
weights: map[string]int64{"b": 1, "c": 1},
|
||||
authID: "b",
|
||||
want: map[string]int64{"b": -64, "c": -61},
|
||||
},
|
||||
{
|
||||
name: "excessive spread clips debt without tying leaders",
|
||||
current: map[string]int64{"a": -96, "b": 96, "c": 95},
|
||||
weights: map[string]int64{"a": 1, "b": 1, "c": 1},
|
||||
authID: "a",
|
||||
want: map[string]int64{"a": -96, "b": 96, "c": 95},
|
||||
},
|
||||
{
|
||||
name: "bounded credits do not shift",
|
||||
current: map[string]int64{"b": 0, "c": 0},
|
||||
weights: map[string]int64{"b": 1, "c": 1},
|
||||
authID: "b",
|
||||
want: map[string]int64{"b": -1, "c": 1},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
state := weightedPoolForTest("route", tc.weights, tc.current)
|
||||
state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false)
|
||||
if !maps.Equal(state.currentForTest(), tc.want) {
|
||||
t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) {
|
||||
key := "codex:model"
|
||||
weights := map[string]int64{"a": 1, "b": 1, "c": 1}
|
||||
state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93})
|
||||
weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1}
|
||||
auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}}
|
||||
_, rejected := weighted.reserve(key, auths, "a")
|
||||
t.Cleanup(func() { rejected.finish(false) })
|
||||
_, accepted := weighted.reserve(key, auths, "b")
|
||||
t.Cleanup(func() { accepted.finish(false) })
|
||||
accepted.finish(true)
|
||||
want := map[string]int64{"a": -93, "b": 92, "c": 94}
|
||||
if !maps.Equal(state.currentForTest(), want) {
|
||||
t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want)
|
||||
}
|
||||
if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) {
|
||||
t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest())
|
||||
}
|
||||
rejected.finish(false)
|
||||
rejected.finish(true)
|
||||
if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 {
|
||||
t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user