From e479ab12538cb027ea558bbfa070fef1ca9d187c Mon Sep 17 00:00:00 2001 From: hkfires <10558748+hkfires@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:48:44 +0800 Subject: [PATCH] perf(auth): compact weighted round-robin pools Replace string pool keys and map-based credit state with SHA-256 digest keys and sorted slice members holding weight, current, and pending credits in a single allocation. Reservations now retain only the member index instead of a weights map and auth ID. Verify full pool identity on digest hits and chain collisions so unrelated candidate pools never share credits or weights. Update tests to access pool credits through test helpers. --- .../auth/conductor_weighted_pinned_test.go | 16 +- sdk/cliproxy/auth/selector.go | 17 +- sdk/cliproxy/auth/selector_test.go | 8 +- sdk/cliproxy/auth/selector_weighted.go | 208 ++++++++++++------ .../auth/selector_weighted_affinity_test.go | 58 ++--- .../selector_weighted_commit_order_test.go | 18 +- .../auth/selector_weighted_memory_test.go | 71 ++++++ .../auth/selector_weighted_overload_test.go | 36 +-- .../auth/selector_weighted_pool_test.go | 28 ++- .../auth/selector_weighted_storage_test.go | 88 ++++++++ sdk/cliproxy/auth/selector_weighted_test.go | 34 ++- 11 files changed, 402 insertions(+), 180 deletions(-) create mode 100644 sdk/cliproxy/auth/selector_weighted_memory_test.go create mode 100644 sdk/cliproxy/auth/selector_weighted_storage_test.go diff --git a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go index 6f52dd3038..8768a2a8a6 100644 --- a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go +++ b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go @@ -104,7 +104,7 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { } } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) - if state == nil || !maps.Equal(state.current, map[string]int64{aID: 12, bID: -12}) || len(state.pending) != 0 { + if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 { t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state) } counts := map[string]int{} @@ -118,8 +118,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if counts[aID] != 12 || counts[bID] != 0 { t.Fatalf("fresh distribution = %v, want a=12 b=0", counts) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("compensation did not balance: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if mode == "sticky" || mode == "cold" { if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID { @@ -135,8 +135,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if auth, errPick := pick(opts); errPick == nil || auth != nil { t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest()) } } // A valid lower-priority pin still wins without charging another tier. @@ -148,11 +148,11 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID { t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID) - if !maps.Equal(lowerState.current, map[string]int64{lowerID: 0}) || len(lowerState.pending) != 0 { + if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 { t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState) } }) diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go index 54e16ebc94..af11116b99 100644 --- a/sdk/cliproxy/auth/selector.go +++ b/sdk/cliproxy/auth/selector.go @@ -39,17 +39,15 @@ type RoundRobinSelector struct { // WeightedRoundRobinSelector provides smooth weighted round-robin selection. type WeightedRoundRobinSelector struct { - mu sync.Mutex - states map[string]*smoothWeightedState - maxKeys int + mu sync.Mutex + states map[[32]byte]*weightedPoolState + stateCount int + maxKeys int } type smoothWeightedState struct { - current map[string]int64 - weights map[string]int64 - pending map[string]int64 - inFlight uint64 - generation uint64 + current map[string]int64 + weights map[string]int64 } type weightedSelectorStateModelKey struct{} @@ -723,9 +721,6 @@ const maxSmoothWeightedStateEntries = 1024 func (s *smoothWeightedState) prepare(weights map[string]int64) { if s.current == nil || weightsConfigChanged(s.weights, weights) { s.current = make(map[string]int64, len(weights)) - s.pending = make(map[string]int64, len(weights)) - s.inFlight = 0 - s.generation++ } if s.weights == nil { s.weights = make(map[string]int64, len(weights)) diff --git a/sdk/cliproxy/auth/selector_test.go b/sdk/cliproxy/auth/selector_test.go index b6654cfcff..37678684b2 100644 --- a/sdk/cliproxy/auth/selector_test.go +++ b/sdk/cliproxy/auth/selector_test.go @@ -208,11 +208,11 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) { func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) { t.Parallel() - state := &smoothWeightedState{current: map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}} - state.charge(map[string]int64{"a": 1, "b": 1}, "a", false) + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}) + state.charge(0, false) limit := maxSmoothWeightedCreditRounds * 2 - if state.current["a"] != limit || state.current["b"] != -limit { - t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.current) + if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit { + t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest()) } } diff --git a/sdk/cliproxy/auth/selector_weighted.go b/sdk/cliproxy/auth/selector_weighted.go index ed92b5bb0d..57852f6b2e 100644 --- a/sdk/cliproxy/auth/selector_weighted.go +++ b/sdk/cliproxy/auth/selector_weighted.go @@ -2,10 +2,10 @@ package auth import ( "context" + "crypto/sha256" + "encoding/binary" "math" "sort" - "strconv" - "strings" "time" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" @@ -19,10 +19,9 @@ type weightedAccountingPoolKey struct{} // Releasing one request must never undo accounting committed by another request. type weightedReservation struct { selector *WeightedRoundRobinSelector - state *smoothWeightedState + state *weightedPoolState generation uint64 - weights map[string]int64 - authID string + member int total int64 bound bool finished bool @@ -95,55 +94,120 @@ func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Contex return picked, nil } -// weightedPoolStateKey isolates credits for actual competing sets. Reusing a -// provider/model accumulator across different sets grants credit for requests -// those sets cannot serve, which can permanently starve their smaller weights. -// Length-prefixed, sorted IDs avoid collisions and keep input order irrelevant. -// Weights stay out of the key so prepare can invalidate pending reservations on -// configuration changes. Returning to a prior set resumes that set's credits. -func weightedPoolStateKey(route string, weights map[string]int64) string { +// weightedPoolMember stores each ID once, with parallel accounting values in a +// single allocation. Members are sorted by ID; reservations retain only an index. +type weightedPoolMember struct { + id string + weight int64 + current int64 + pending int64 +} + +// weightedPoolState is separate from the scheduler's transient-subset accumulator: +// each selector pool has fixed membership and shares its weights across reservations. +type weightedPoolState struct { + route string + members []weightedPoolMember + inFlight uint64 + generation uint64 + next *weightedPoolState // Exact-identity collision chain for the digest key. +} + +func sortedWeightedIDs(weights map[string]int64) []string { ids := make([]string, 0, len(weights)) for id := range weights { ids = append(ids, id) } sort.Strings(ids) - var key strings.Builder - key.WriteString(strconv.Itoa(len(route))) - key.WriteByte(':') - key.WriteString(route) + return ids +} + +// weightedPoolStateKey isolates competing sets without retaining concatenated IDs. +// Weights stay out of the key so configuration changes invalidate reservations. +func weightedPoolStateKey(route string, weights map[string]int64) [32]byte { + return weightedPoolDigest(route, sortedWeightedIDs(weights)) +} + +func weightedPoolDigest(route string, ids []string) [32]byte { + size := 8 + len(route) for _, id := range ids { - key.WriteString(strconv.Itoa(len(id))) - key.WriteByte(':') - key.WriteString(id) + size += 8 + len(id) + } + identity := make([]byte, 0, size) + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route))) + identity = append(identity, route...) + for _, id := range ids { + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id))) + identity = append(identity, id...) + } + return sha256.Sum256(identity) +} + +// matches verifies the full identity even on a digest hit. Hash collisions must +// never combine the quota or configured weights of unrelated candidate pools. +func (s *weightedPoolState) matches(route string, ids []string) bool { + if s.route != route || len(s.members) != len(ids) { + return false + } + for i, id := range ids { + if s.members[i].id != id { + return false + } + } + return true +} + +func (s *weightedPoolState) prepare(weights map[string]int64) { + for _, member := range s.members { + if member.weight != weights[member.id] { + s.generation++ + s.inFlight = 0 + for i := range s.members { + s.members[i].weight = weights[s.members[i].id] + s.members[i].current = 0 + s.members[i].pending = 0 + } + return + } } - return key.String() } // reserve selects and reserves credit atomically. A non-empty authID preserves a // sticky binding; otherwise committed and pending credits both guide selection. -func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, authID string) (*Auth, *weightedReservation) { +func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) { weights := authWeightVector(available) - key = weightedPoolStateKey(key, weights) + ids := sortedWeightedIDs(weights) + key := weightedPoolDigest(route, ids) s.mu.Lock() defer s.mu.Unlock() if s.states == nil { - s.states = make(map[string]*smoothWeightedState) - } - limit := s.maxKeys - if limit <= 0 { - limit = 4096 - } - if _, ok := s.states[key]; !ok && len(s.states) >= limit { - s.states = make(map[string]*smoothWeightedState) + s.states = make(map[[32]byte]*weightedPoolState) } state := s.states[key] - if state == nil { - state = &smoothWeightedState{} - s.states[key] = state + for state != nil && !state.matches(route, ids) { + state = state.next + } + if state == nil { + limit := s.maxKeys + if limit <= 0 { + limit = 4096 + } + if s.stateCount >= limit { + s.states = make(map[[32]byte]*weightedPoolState) + s.stateCount = 0 + } + state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]} + for i, id := range ids { + state.members[i] = weightedPoolMember{id: id, weight: weights[id]} + } + s.states[key] = state + s.stateCount++ + } else { + state.prepare(weights) } - state.prepare(weights) var picked *Auth var pickedCurrent, total int64 + pickedMember := -1 for _, auth := range available { if auth == nil || weights[auth.ID] <= 0 { continue @@ -152,10 +216,12 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth if authID != "" && auth.ID != authID { continue } - current := saturatingAddInt64(state.current[auth.ID], state.pending[auth.ID]) - current = saturatingAddInt64(current, weights[auth.ID]) + index := sort.SearchStrings(ids, auth.ID) + member := &state.members[index] + current := saturatingAddInt64(member.current, member.pending) + current = saturatingAddInt64(current, member.weight) if picked == nil || current > pickedCurrent { - picked, pickedCurrent = auth, current + picked, pickedCurrent, pickedMember = auth, current, index } } if picked == nil { @@ -163,14 +229,14 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth } // Bound traffic beyond the compensation window still keeps its credential, // but must not create speculative credit that can starve concurrent picks. - if authID != "" && !state.boundChargeFits(weights, picked.ID, total) { + if authID != "" && !state.boundChargeFits(pickedMember, total) { return picked, nil } - state.adjustPending(weights, picked.ID, total, false) + state.adjustPending(pickedMember, total, false) state.inFlight++ return picked, &weightedReservation{ selector: s, state: state, generation: state.generation, - weights: weights, authID: picked.ID, total: total, bound: authID != "", + member: pickedMember, total: total, bound: authID != "", } } @@ -188,31 +254,29 @@ func (r *weightedReservation) finish(committed bool) { if r.state.generation != r.generation { return } - r.state.adjustPending(r.weights, r.authID, r.total, true) + r.state.adjustPending(r.member, r.total, true) r.state.inFlight-- if committed { - r.state.charge(r.weights, r.authID, r.bound) + r.state.charge(r.member, r.bound) } else if r.state.inFlight == 0 { // A final rollback can remove the offset to already committed credit. - r.state.boundCredits(r.weights, r.total) + r.state.boundCredits(r.total) } } // adjustPending must be called with the selector mutex held. Pending deltas are // not clamped, so releasing them cannot lose credit at the compensation bound. -func (s *smoothWeightedState) adjustPending(weights map[string]int64, authID string, total int64, release bool) { - for id, weight := range weights { - delta := weight - if id == authID { +func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) { + for i := range s.members { + member := &s.members[i] + delta := member.weight + if i == selected { delta -= total } if release { delta = -delta } - s.pending[id] = saturatingAddInt64(s.pending[id], delta) - if s.pending[id] == 0 { - delete(s.pending, id) - } + member.pending = saturatingAddInt64(member.pending, delta) } } @@ -222,17 +286,17 @@ const maxSmoothWeightedCreditRounds int64 = 32 // boundChargeFits admits the whole sticky delta or none. Both reservation and // commit include outstanding deltas: committed credit alone can transiently // exceed the cap while successful reservations settle in a different order. -func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID string, totalWeight int64) bool { +func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool { limit := int64(math.MaxInt64) if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { limit = totalWeight * maxSmoothWeightedCreditRounds } - for id, weight := range weights { - delta := weight - if id == authID { + for i, member := range s.members { + delta := member.weight + if i == selected { delta -= totalWeight } - current := saturatingAddInt64(s.current[id], s.pending[id]) + current := saturatingAddInt64(member.current, member.pending) next := saturatingAddInt64(current, delta) if next < -limit || next > limit { return false @@ -241,36 +305,38 @@ func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID s return true } -func (s *smoothWeightedState) charge(weights map[string]int64, authID string, bound bool) { +func (s *weightedPoolState) charge(selected int, bound bool) { var totalWeight int64 - for _, weight := range weights { - totalWeight = saturatingAddInt64(totalWeight, weight) + for _, member := range s.members { + totalWeight = saturatingAddInt64(totalWeight, member.weight) } - if !bound || s.boundChargeFits(weights, authID, totalWeight) { + if !bound || s.boundChargeFits(selected, totalWeight) { // At the cap, forced traffic cannot erase a small peer's earnings. // Pending repayments still count until they actually roll back. - for id, weight := range weights { - s.current[id] = saturatingAddInt64(s.current[id], weight) + for i := range s.members { + member := &s.members[i] + member.current = saturatingAddInt64(member.current, member.weight) } - s.current[authID] = saturatingAddInt64(s.current[authID], -totalWeight) + member := &s.members[selected] + member.current = saturatingAddInt64(member.current, -totalWeight) } if s.inFlight == 0 { - s.boundCredits(weights, totalWeight) + s.boundCredits(totalWeight) } } // boundCredits runs only once the pool settles. Net pending deltas can be zero // even with live reservations, so the in-flight count, not len(pending), decides // when clipping cannot irreversibly discard another reservation's offset. -func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight int64) { +func (s *weightedPoolState) boundCredits(totalWeight int64) { limit := int64(math.MaxInt64) if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { limit = totalWeight * maxSmoothWeightedCreditRounds } var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64 - for id := range weights { - minCredit = min(minCredit, s.current[id]) - maxCredit = max(maxCredit, s.current[id]) + for _, member := range s.members { + minCredit = min(minCredit, member.current) + maxCredit = max(maxCredit, member.current) } // A common offset keeps leading credits distinct at the compensation bound. // Independent clamps can repeatedly tie the leaders and starve the later @@ -285,8 +351,8 @@ func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight } // If the full spread cannot fit, cap only the trailing debt after rebasing // the leader to the upper bound. Do not flatten leading candidates' credits. - // Temporarily excluded credentials retain their credit for later selections. - for id := range weights { - s.current[id] = max(-limit, saturatingAddInt64(s.current[id], -offset)) + for i := range s.members { + member := &s.members[i] + member.current = max(-limit, saturatingAddInt64(member.current, -offset)) } } diff --git a/sdk/cliproxy/auth/selector_weighted_affinity_test.go b/sdk/cliproxy/auth/selector_weighted_affinity_test.go index 12f3c2fc7c..5736e0c821 100644 --- a/sdk/cliproxy/auth/selector_weighted_affinity_test.go +++ b/sdk/cliproxy/auth/selector_weighted_affinity_test.go @@ -52,8 +52,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) { } } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != 12 || state.current["b"] != -12 || len(state.pending) != 0 { - t.Fatalf("sticky credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 { + t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } for i := range 12 { // Requests without an identity and cold sessions share the same credits. @@ -66,8 +66,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) { t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick) } } - if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 { - t.Fatalf("compensated credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if binding, ok := selector.cache.Get(key); !ok || binding != "b" { t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok) @@ -116,8 +116,8 @@ func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing. if binding, ok := selector.cache.Get(key); !ok || binding != "a" { t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok) } - if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pending) != 0 { - t.Fatalf("pending credits leaked: %v", state.pending) + if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) } }) } @@ -137,8 +137,8 @@ func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) { t.Fatalf("pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if i == 0 && (state.current["a"] != -1 || state.current["b"] != 1) { - t.Fatalf("first selection charged more than once: %v", state.current) + if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) { + t.Fatalf("first selection charged more than once: %v", state.currentForTest()) } counts[picked.ID]++ } @@ -165,16 +165,16 @@ func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) { limit := maxSmoothWeightedCreditRounds * 4 // A bound charge is admitted as a whole, stopping within one delta of the cap. wantCredit := limit / 3 * 3 - if state.current["a"] != wantCredit || state.current["b"] != -wantCredit { - t.Fatalf("bounded credits = %v, want +/- %d", state.current, wantCredit) + if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit { + t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit) } for range wantCredit { if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil { t.Fatal(errPick) } } - if state.current["a"] != 0 || state.current["b"] != 0 { - t.Fatalf("compensation did not finish: %v", state.current) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 { + t.Fatalf("compensation did not finish: %v", state.currentForTest()) } } @@ -215,8 +215,8 @@ func TestWeightedSessionAffinityLCPCredits(t *testing.T) { t.Fatal("expected LCP compaction") } state := weightedStateForTest(t, weighted, "openai:model", "a", "b") - if state.current["a"] != 3 || state.current["b"] != -3 || len(state.pending) != 0 { - t.Fatalf("LCP credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 { + t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } }) } @@ -238,8 +238,8 @@ func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) { rejected.finish(true) accepted.finish(false) limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3 - if state.current["a"] != limit-1 || state.current["b"] != -limit+1 || len(state.pending) != 0 { - t.Fatalf("rollback lost another charge: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -252,12 +252,12 @@ func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) { _, latest := weighted.reserve(key, auths, "") old.finish(true) state := weightedStateForTest(t, weighted, key, "a", "b") - if state.current["a"] != 0 || state.current["b"] != 0 || state.pending["a"] != -1 || state.pending["b"] != 1 { - t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 { + t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } latest.finish(true) - if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 { - t.Fatalf("new generation accounting: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -294,8 +294,8 @@ func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) reservation.finish(true) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 { - t.Fatalf("concurrent accounting: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -343,19 +343,19 @@ func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) { t.Fatalf("sticky pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:requested", "a", "b") - if state == nil || len(weighted.states) != 1 || len(state.current) != 2 || state.current["a"] != 3 || state.current["b"] != -3 { + if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 { t.Fatalf("wrong model key or eligible pool: %v", weighted.states) } picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2]) - if errPick != nil || picked == nil || picked.ID != "a" || state.current["a"] != 2 || state.current["b"] != -2 { - t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.current) + if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { + t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest()) } for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} { if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil { t.Fatal(errOther) } } - if state.current["a"] != 2 || state.current["b"] != -2 { + if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { t.Fatal("another provider or model changed credits") } } @@ -374,8 +374,8 @@ func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) { t.Fatalf("reselection: pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 { - t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if binding, ok := selector.cache.Get(key); !ok || binding != "a" { t.Fatalf("reselection did not bind: %q, found=%t", binding, ok) @@ -428,7 +428,7 @@ func TestManagerWeightedSessionAffinityCompensation(t *testing.T) { } } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) - if state == nil || state.current[aID] != 0 || state.current[bID] != 0 || len(state.pending) != 0 { + if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 { t.Fatalf("manager credits did not balance: %#v", state) } }) diff --git a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go index 22ecac2eb5..58d18499a9 100644 --- a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go +++ b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go @@ -41,8 +41,8 @@ func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) { } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") limit := maxSmoothWeightedCreditRounds * 2 - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -limit, "b": limit}) { - t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.current, state.pending, state.inFlight) + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) { + t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight) } }) } @@ -98,7 +98,7 @@ func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) { t.Fatalf("%s distribution = %v, want 48:16", phase, counts) } } - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) { + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) { t.Fatalf("settled 3:1 batch drifted: %#v", state) } }) @@ -128,11 +128,11 @@ func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) { reservation.finish(true) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.inFlight != 1 || len(state.current) != 0 || !maps.Equal(state.pending, map[string]int64{"a": -1, "b": 1}) { + if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatalf("old generation altered new accounting: %#v", state) } latest.finish(true) - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -1, "b": 1}) { + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatalf("new generation failed to settle: %#v", state) } } @@ -167,8 +167,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) { } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") // A zero net pending vector does not mean all reservations settled. - if state.inFlight != 256 || len(state.pending) != 0 { - t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pending) + if state.inFlight != 256 || len(state.pendingForTest()) != 0 { + t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest()) } last := "b" if first == "b" { @@ -179,8 +179,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) { reservation.finish(true) } } - if !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) || len(state.pending) != 0 || state.inFlight != 0 { - t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 { + t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } counts := map[string]int{} for range 64 { diff --git a/sdk/cliproxy/auth/selector_weighted_memory_test.go b/sdk/cliproxy/auth/selector_weighted_memory_test.go new file mode 100644 index 0000000000..eec967e288 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_memory_test.go @@ -0,0 +1,71 @@ +package auth + +import ( + "fmt" + "runtime" + "testing" +) + +func BenchmarkWeightedPoolReserve1000(b *testing.B) { + for _, bound := range []bool{false, true} { + b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) { + auths := make([]*Auth, 1000) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + boundID := "" + if bound { + boundID = auths[i%len(auths)].ID + } + _, reservation := selector.reserve("codex:model", auths, boundID) + reservation.finish(true) + } + }) + } +} + +// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from +// total heap and allocation churn. Each retry subset omits a different credential. +func BenchmarkWeightedPoolRetainedMemory(b *testing.B) { + const members = 1000 + auths := make([]*Auth, members) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + subset := make([]*Auth, 0, members-1) + var retained, heap, allocated int64 + b.ResetTimer() + for range b.N { + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for excluded := range auths { + subset = append(subset[:0], auths[:excluded]...) + subset = append(subset, auths[excluded+1:]...) + _, reservation := selector.reserve("codex:model", subset, "") + reservation.finish(true) + } + runtime.GC() + runtime.ReadMemStats(&after) + if len(selector.states) != members+1 { + b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1) + } + runtime.KeepAlive(selector) + runtime.KeepAlive(auths) + retained += int64(after.HeapAlloc) - int64(before.HeapAlloc) + heap += int64(after.HeapAlloc) + allocated += int64(after.TotalAlloc - before.TotalAlloc) + } + b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op") + b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op") + b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op") +} diff --git a/sdk/cliproxy/auth/selector_weighted_overload_test.go b/sdk/cliproxy/auth/selector_weighted_overload_test.go index 47e3da0672..f7639a5368 100644 --- a/sdk/cliproxy/auth/selector_weighted_overload_test.go +++ b/sdk/cliproxy/auth/selector_weighted_overload_test.go @@ -50,13 +50,13 @@ func TestWeightedOverloadExactCompensation(t *testing.T) { } } for _, state := range weighted.states { - for id, credit := range state.current { + for id, credit := range state.currentForTest() { if credit != 0 { t.Errorf("exact compensation left %q credit=%d", id, credit) } } - if len(state.pending) != 0 { - t.Errorf("pending credit leaked: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Errorf("pending credit leaked: %v", state.pendingForTest()) } } }) @@ -97,7 +97,7 @@ func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) ids[i] = auth.ID } state := weightedStateForTest(t, weighted, "codex:model", ids...) - t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.current) + t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest()) // Forced traffic makes global ratios infeasible; the remaining capacity // should be shared by unbound peers rather than monopolized by one. peerWeight := 0 @@ -235,8 +235,8 @@ func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T reservation.finish(true) } state := weightedStateForTest(t, weighted, key, "a", "b", "c") - if len(state.pending) != 0 { - t.Fatalf("settled reservations leaked credit: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest()) } } @@ -249,7 +249,7 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) reservation.finish(true) } state := weightedStateForTest(t, weighted, key, "a", "b") - before := maps.Clone(state.current) + before := maps.Clone(state.currentForTest()) _, first := weighted.reserve(key, auths, "") _, second := weighted.reserve(key, auths, "") t.Cleanup(func() { first.finish(false); second.finish(false) }) @@ -265,12 +265,12 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) bound.finish(true) bound.finish(true) bound.finish(false) - if !maps.Equal(state.current, before) || len(state.pending) != 0 { - t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.current, state.pending) + if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { + t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest()) } // An over-limit bound selection succeeds without reserving any credit. picked, skipped := weighted.reserve(key, auths, "b") - if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.current, before) || len(state.pending) != 0 { + if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { t.Fatal("saturated bound selection should preserve binding and all credits") } } @@ -322,10 +322,10 @@ func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) { {"a": -222, "b": 111, "c": 111}, {"a": -112, "b": 222, "c": -110}, } { - state := &smoothWeightedState{current: maps.Clone(credit)} - state.charge(map[string]int64{"a": 1, "b": 5, "c": 1}, "a", true) - if !maps.Equal(state.current, credit) { - t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.current, credit) + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit) + state.charge(0, true) + if !maps.Equal(state.currentForTest(), credit) { + t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit) } } } @@ -391,8 +391,8 @@ func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) { t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok) } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID) - if len(state.pending) != 0 { - t.Fatalf("manager left speculative credit: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("manager left speculative credit: %v", state.pendingForTest()) } }) } @@ -446,8 +446,8 @@ func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) { state := weightedStateForTest(t, weighted, "codex:model", "a", "b") // All 64 calls share one pool. Accounting must match actual accepted picks, // regardless of whether the final arrivals happened to be sticky requests. - if state.current["a"] != int64(64*3-counts["a"]*4) || state.current["b"] != int64(64-counts["b"]*4) || len(state.pending) != 0 { - t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.current, state.pending) + if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest()) } if bound, ok := selector.cache.Get(key); !ok || bound != "b" { t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok) diff --git a/sdk/cliproxy/auth/selector_weighted_pool_test.go b/sdk/cliproxy/auth/selector_weighted_pool_test.go index 96dae75816..5fb73ffa9c 100644 --- a/sdk/cliproxy/auth/selector_weighted_pool_test.go +++ b/sdk/cliproxy/auth/selector_weighted_pool_test.go @@ -14,13 +14,17 @@ import ( ) // weightedStateForTest checks that the requested accounting pool was created. -func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *smoothWeightedState { +func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState { t.Helper() weights := make(map[string]int64, len(ids)) for _, id := range ids { weights[id] = 1 } state := selector.states[weightedPoolStateKey(route, weights)] + ids = sortedWeightedIDs(weights) + for state != nil && !state.matches(route, ids) { + state = state.next + } if state == nil { t.Fatalf("missing weighted state for %q and %v", route, ids) } @@ -38,11 +42,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) { pendingSubset.finish(true) subset := weightedStateForTest(t, weighted, key, "a", "b") full := weightedStateForTest(t, weighted, key, "a", "b", "c") - if !maps.Equal(subset.current, map[string]int64{"a": 3, "b": -3}) || len(full.current) != 0 { - t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.current, full.current) + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() { + t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest()) } - if !maps.Equal(full.pending, map[string]int64{"a": 3, "b": -4, "c": 1}) { - t.Fatalf("another pool consumed pending credit: %v", full.pending) + if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) { + t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest()) } pendingFull.finish(false) for range 100 { @@ -53,11 +57,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) { for range 3 { picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "") if picked == nil || picked.ID != "a" { - t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.current) + t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest()) } reservation.finish(true) } - if !maps.Equal(subset.current, map[string]int64{"a": 0, "b": 0}) || len(subset.pending) != 0 || len(full.pending) != 0 { + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 { t.Fatalf("returning subset failed to balance: %#v", subset) } } @@ -96,14 +100,14 @@ func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) { independentReservation.finish(committed) poolKey := weightedPoolStateKey("route", authWeightVector(pool)) state, independentState := shared.states[poolKey], independent[mask].states[poolKey] - if !maps.Equal(state.current, independentState.current) || len(state.pending) != 0 { + if !maps.Equal(state.currentForTest(), independentState.currentForTest()) || len(state.pendingForTest()) != 0 { t.Fatalf("step %d pool %d: credits diverged or pending leaked", step, mask) } } } func TestWeightedPoolKeysAndCapacity(t *testing.T) { - seen := make(map[string]struct{}) + seen := make(map[[32]byte]struct{}) for _, tc := range []struct { route string ids []string @@ -145,7 +149,7 @@ func TestWeightedPoolKeysAndCapacity(t *testing.T) { t.Fatal("pool eviction failed to bound or replace state") } old.finish(true) - if len(current.state.current) != 0 || !maps.Equal(current.state.pending, map[string]int64{"a": -1, "b": 1}) { + if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatal("finishing an evicted reservation affected the new pool") } current.finish(true) @@ -273,8 +277,8 @@ func TestWeightedSessionAffinityPoolIsolation(t *testing.T) { t.Fatalf("binding moved: %q, found=%t", bound, ok) } for _, state := range weighted.states { - if len(state.pending) != 0 { - t.Fatalf("pending credits leaked: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) } } }) diff --git a/sdk/cliproxy/auth/selector_weighted_storage_test.go b/sdk/cliproxy/auth/selector_weighted_storage_test.go new file mode 100644 index 0000000000..0d937b9038 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_storage_test.go @@ -0,0 +1,88 @@ +package auth + +import ( + "maps" + "testing" +) + +// These snapshots keep assertions independent of the compact production layout. +func (s *weightedPoolState) currentForTest() map[string]int64 { + current := make(map[string]int64, len(s.members)) + for _, member := range s.members { + current[member.id] = member.current + } + return current +} + +func (s *weightedPoolState) pendingForTest() map[string]int64 { + pending := make(map[string]int64) + for _, member := range s.members { + if member.pending != 0 { + pending[member.id] = member.pending + } + } + return pending +} + +func (s *weightedPoolState) hasCurrentForTest() bool { + for _, member := range s.members { + if member.current != 0 { + return true + } + } + return false +} + +// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact +// identities, including a matching state behind a collision-chain head. +func TestWeightedPoolDigestCollisionIsolation(t *testing.T) { + for _, unrelated := range []*weightedPoolState{ + weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}), + } { + t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) { + weights := map[string]int64{"a": 1, "b": 1} + key := weightedPoolStateKey("route", weights) + selector := &WeightedRoundRobinSelector{ + states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2, + } + before := unrelated.currentForTest() + picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b") + if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated { + t.Fatal("digest collision reused an unrelated pool") + } + t.Cleanup(func() { first.finish(false) }) + matching := first.state + // Put the nonmatching state first to exercise a full chain lookup. + matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated + _, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a") + if second == nil || second.state != matching || selector.stateCount != 2 { + t.Fatal("collision lookup failed to reuse the matching pool") + } + t.Cleanup(func() { second.finish(false) }) + first.finish(true) + second.finish(true) + if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 { + t.Fatal("collision accounting changed the unrelated pool") + } + if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 { + t.Fatal("matching pool did not balance") + } + // Capacity still counts pools, not just distinct digest buckets. + _, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "") + third.finish(true) + if selector.stateCount != 1 || len(selector.states) != 1 { + t.Fatal("colliding pools bypassed the existing capacity limit") + } + }) + } +} + +func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState { + state := &weightedPoolState{route: route} + for _, id := range sortedWeightedIDs(weights) { + state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]}) + } + return state +} diff --git a/sdk/cliproxy/auth/selector_weighted_test.go b/sdk/cliproxy/auth/selector_weighted_test.go index 3d9872085f..31352e3017 100644 --- a/sdk/cliproxy/auth/selector_weighted_test.go +++ b/sdk/cliproxy/auth/selector_weighted_test.go @@ -2,6 +2,7 @@ package auth import ( "maps" + "sort" "testing" ) @@ -18,14 +19,14 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { current: map[string]int64{"a": -96, "b": 96, "c": 95}, weights: map[string]int64{"b": 1, "c": 1}, authID: "b", - want: map[string]int64{"a": -96, "b": 63, "c": 64}, + want: map[string]int64{"b": 63, "c": 64}, }, { name: "negative credits shift together", current: map[string]int64{"a": 96, "b": -64, "c": -63}, weights: map[string]int64{"b": 1, "c": 1}, authID: "b", - want: map[string]int64{"a": 96, "b": -64, "c": -61}, + want: map[string]int64{"b": -64, "c": -61}, }, { name: "excessive spread clips debt without tying leaders", @@ -43,10 +44,10 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - state := &smoothWeightedState{current: tc.current} - state.charge(tc.weights, tc.authID, false) - if !maps.Equal(state.current, tc.want) { - t.Fatalf("rebased credits = %v, want %v", state.current, tc.want) + state := weightedPoolForTest("route", tc.weights, tc.current) + state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false) + if !maps.Equal(state.currentForTest(), tc.want) { + t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want) } }) } @@ -54,12 +55,9 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) { key := "codex:model" - state := &smoothWeightedState{ - current: map[string]int64{"a": -94, "b": 94, "c": 93}, - weights: map[string]int64{"a": 1, "b": 1, "c": 1}, - pending: make(map[string]int64), - } - weighted := &WeightedRoundRobinSelector{states: map[string]*smoothWeightedState{weightedPoolStateKey(key, state.weights): state}} + weights := map[string]int64{"a": 1, "b": 1, "c": 1} + state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93}) + weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1} auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}} _, rejected := weighted.reserve(key, auths, "a") t.Cleanup(func() { rejected.finish(false) }) @@ -67,15 +65,15 @@ func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) { t.Cleanup(func() { accepted.finish(false) }) accepted.finish(true) want := map[string]int64{"a": -93, "b": 92, "c": 94} - if !maps.Equal(state.current, want) { - t.Fatalf("committed credits = %v, want %v", state.current, want) + if !maps.Equal(state.currentForTest(), want) { + t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want) } - if !maps.Equal(state.pending, map[string]int64{"a": -2, "b": 1, "c": 1}) { - t.Fatalf("commit changed another request's reservation: %v", state.pending) + if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) { + t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest()) } rejected.finish(false) rejected.finish(true) - if !maps.Equal(state.current, want) || len(state.pending) != 0 { - t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } }