diff --git a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go index 6f52dd3038..8768a2a8a6 100644 --- a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go +++ b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go @@ -104,7 +104,7 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { } } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) - if state == nil || !maps.Equal(state.current, map[string]int64{aID: 12, bID: -12}) || len(state.pending) != 0 { + if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 { t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state) } counts := map[string]int{} @@ -118,8 +118,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if counts[aID] != 12 || counts[bID] != 0 { t.Fatalf("fresh distribution = %v, want a=12 b=0", counts) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("compensation did not balance: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if mode == "sticky" || mode == "cold" { if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID { @@ -135,8 +135,8 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if auth, errPick := pick(opts); errPick == nil || auth != nil { t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest()) } } // A valid lower-priority pin still wins without charging another tier. @@ -148,11 +148,11 @@ func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID { t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick) } - if !maps.Equal(state.current, map[string]int64{aID: 0, bID: 0}) || len(state.pending) != 0 { - t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID) - if !maps.Equal(lowerState.current, map[string]int64{lowerID: 0}) || len(lowerState.pending) != 0 { + if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 { t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState) } }) diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go index 54e16ebc94..af11116b99 100644 --- a/sdk/cliproxy/auth/selector.go +++ b/sdk/cliproxy/auth/selector.go @@ -39,17 +39,15 @@ type RoundRobinSelector struct { // WeightedRoundRobinSelector provides smooth weighted round-robin selection. type WeightedRoundRobinSelector struct { - mu sync.Mutex - states map[string]*smoothWeightedState - maxKeys int + mu sync.Mutex + states map[[32]byte]*weightedPoolState + stateCount int + maxKeys int } type smoothWeightedState struct { - current map[string]int64 - weights map[string]int64 - pending map[string]int64 - inFlight uint64 - generation uint64 + current map[string]int64 + weights map[string]int64 } type weightedSelectorStateModelKey struct{} @@ -723,9 +721,6 @@ const maxSmoothWeightedStateEntries = 1024 func (s *smoothWeightedState) prepare(weights map[string]int64) { if s.current == nil || weightsConfigChanged(s.weights, weights) { s.current = make(map[string]int64, len(weights)) - s.pending = make(map[string]int64, len(weights)) - s.inFlight = 0 - s.generation++ } if s.weights == nil { s.weights = make(map[string]int64, len(weights)) diff --git a/sdk/cliproxy/auth/selector_test.go b/sdk/cliproxy/auth/selector_test.go index b6654cfcff..37678684b2 100644 --- a/sdk/cliproxy/auth/selector_test.go +++ b/sdk/cliproxy/auth/selector_test.go @@ -208,11 +208,11 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) { func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) { t.Parallel() - state := &smoothWeightedState{current: map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}} - state.charge(map[string]int64{"a": 1, "b": 1}, "a", false) + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}) + state.charge(0, false) limit := maxSmoothWeightedCreditRounds * 2 - if state.current["a"] != limit || state.current["b"] != -limit { - t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.current) + if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit { + t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest()) } } diff --git a/sdk/cliproxy/auth/selector_weighted.go b/sdk/cliproxy/auth/selector_weighted.go index ed92b5bb0d..57852f6b2e 100644 --- a/sdk/cliproxy/auth/selector_weighted.go +++ b/sdk/cliproxy/auth/selector_weighted.go @@ -2,10 +2,10 @@ package auth import ( "context" + "crypto/sha256" + "encoding/binary" "math" "sort" - "strconv" - "strings" "time" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" @@ -19,10 +19,9 @@ type weightedAccountingPoolKey struct{} // Releasing one request must never undo accounting committed by another request. type weightedReservation struct { selector *WeightedRoundRobinSelector - state *smoothWeightedState + state *weightedPoolState generation uint64 - weights map[string]int64 - authID string + member int total int64 bound bool finished bool @@ -95,55 +94,120 @@ func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Contex return picked, nil } -// weightedPoolStateKey isolates credits for actual competing sets. Reusing a -// provider/model accumulator across different sets grants credit for requests -// those sets cannot serve, which can permanently starve their smaller weights. -// Length-prefixed, sorted IDs avoid collisions and keep input order irrelevant. -// Weights stay out of the key so prepare can invalidate pending reservations on -// configuration changes. Returning to a prior set resumes that set's credits. -func weightedPoolStateKey(route string, weights map[string]int64) string { +// weightedPoolMember stores each ID once, with parallel accounting values in a +// single allocation. Members are sorted by ID; reservations retain only an index. +type weightedPoolMember struct { + id string + weight int64 + current int64 + pending int64 +} + +// weightedPoolState is separate from the scheduler's transient-subset accumulator: +// each selector pool has fixed membership and shares its weights across reservations. +type weightedPoolState struct { + route string + members []weightedPoolMember + inFlight uint64 + generation uint64 + next *weightedPoolState // Exact-identity collision chain for the digest key. +} + +func sortedWeightedIDs(weights map[string]int64) []string { ids := make([]string, 0, len(weights)) for id := range weights { ids = append(ids, id) } sort.Strings(ids) - var key strings.Builder - key.WriteString(strconv.Itoa(len(route))) - key.WriteByte(':') - key.WriteString(route) + return ids +} + +// weightedPoolStateKey isolates competing sets without retaining concatenated IDs. +// Weights stay out of the key so configuration changes invalidate reservations. +func weightedPoolStateKey(route string, weights map[string]int64) [32]byte { + return weightedPoolDigest(route, sortedWeightedIDs(weights)) +} + +func weightedPoolDigest(route string, ids []string) [32]byte { + size := 8 + len(route) for _, id := range ids { - key.WriteString(strconv.Itoa(len(id))) - key.WriteByte(':') - key.WriteString(id) + size += 8 + len(id) + } + identity := make([]byte, 0, size) + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route))) + identity = append(identity, route...) + for _, id := range ids { + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id))) + identity = append(identity, id...) + } + return sha256.Sum256(identity) +} + +// matches verifies the full identity even on a digest hit. Hash collisions must +// never combine the quota or configured weights of unrelated candidate pools. +func (s *weightedPoolState) matches(route string, ids []string) bool { + if s.route != route || len(s.members) != len(ids) { + return false + } + for i, id := range ids { + if s.members[i].id != id { + return false + } + } + return true +} + +func (s *weightedPoolState) prepare(weights map[string]int64) { + for _, member := range s.members { + if member.weight != weights[member.id] { + s.generation++ + s.inFlight = 0 + for i := range s.members { + s.members[i].weight = weights[s.members[i].id] + s.members[i].current = 0 + s.members[i].pending = 0 + } + return + } } - return key.String() } // reserve selects and reserves credit atomically. A non-empty authID preserves a // sticky binding; otherwise committed and pending credits both guide selection. -func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, authID string) (*Auth, *weightedReservation) { +func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) { weights := authWeightVector(available) - key = weightedPoolStateKey(key, weights) + ids := sortedWeightedIDs(weights) + key := weightedPoolDigest(route, ids) s.mu.Lock() defer s.mu.Unlock() if s.states == nil { - s.states = make(map[string]*smoothWeightedState) - } - limit := s.maxKeys - if limit <= 0 { - limit = 4096 - } - if _, ok := s.states[key]; !ok && len(s.states) >= limit { - s.states = make(map[string]*smoothWeightedState) + s.states = make(map[[32]byte]*weightedPoolState) } state := s.states[key] - if state == nil { - state = &smoothWeightedState{} - s.states[key] = state + for state != nil && !state.matches(route, ids) { + state = state.next + } + if state == nil { + limit := s.maxKeys + if limit <= 0 { + limit = 4096 + } + if s.stateCount >= limit { + s.states = make(map[[32]byte]*weightedPoolState) + s.stateCount = 0 + } + state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]} + for i, id := range ids { + state.members[i] = weightedPoolMember{id: id, weight: weights[id]} + } + s.states[key] = state + s.stateCount++ + } else { + state.prepare(weights) } - state.prepare(weights) var picked *Auth var pickedCurrent, total int64 + pickedMember := -1 for _, auth := range available { if auth == nil || weights[auth.ID] <= 0 { continue @@ -152,10 +216,12 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth if authID != "" && auth.ID != authID { continue } - current := saturatingAddInt64(state.current[auth.ID], state.pending[auth.ID]) - current = saturatingAddInt64(current, weights[auth.ID]) + index := sort.SearchStrings(ids, auth.ID) + member := &state.members[index] + current := saturatingAddInt64(member.current, member.pending) + current = saturatingAddInt64(current, member.weight) if picked == nil || current > pickedCurrent { - picked, pickedCurrent = auth, current + picked, pickedCurrent, pickedMember = auth, current, index } } if picked == nil { @@ -163,14 +229,14 @@ func (s *WeightedRoundRobinSelector) reserve(key string, available []*Auth, auth } // Bound traffic beyond the compensation window still keeps its credential, // but must not create speculative credit that can starve concurrent picks. - if authID != "" && !state.boundChargeFits(weights, picked.ID, total) { + if authID != "" && !state.boundChargeFits(pickedMember, total) { return picked, nil } - state.adjustPending(weights, picked.ID, total, false) + state.adjustPending(pickedMember, total, false) state.inFlight++ return picked, &weightedReservation{ selector: s, state: state, generation: state.generation, - weights: weights, authID: picked.ID, total: total, bound: authID != "", + member: pickedMember, total: total, bound: authID != "", } } @@ -188,31 +254,29 @@ func (r *weightedReservation) finish(committed bool) { if r.state.generation != r.generation { return } - r.state.adjustPending(r.weights, r.authID, r.total, true) + r.state.adjustPending(r.member, r.total, true) r.state.inFlight-- if committed { - r.state.charge(r.weights, r.authID, r.bound) + r.state.charge(r.member, r.bound) } else if r.state.inFlight == 0 { // A final rollback can remove the offset to already committed credit. - r.state.boundCredits(r.weights, r.total) + r.state.boundCredits(r.total) } } // adjustPending must be called with the selector mutex held. Pending deltas are // not clamped, so releasing them cannot lose credit at the compensation bound. -func (s *smoothWeightedState) adjustPending(weights map[string]int64, authID string, total int64, release bool) { - for id, weight := range weights { - delta := weight - if id == authID { +func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) { + for i := range s.members { + member := &s.members[i] + delta := member.weight + if i == selected { delta -= total } if release { delta = -delta } - s.pending[id] = saturatingAddInt64(s.pending[id], delta) - if s.pending[id] == 0 { - delete(s.pending, id) - } + member.pending = saturatingAddInt64(member.pending, delta) } } @@ -222,17 +286,17 @@ const maxSmoothWeightedCreditRounds int64 = 32 // boundChargeFits admits the whole sticky delta or none. Both reservation and // commit include outstanding deltas: committed credit alone can transiently // exceed the cap while successful reservations settle in a different order. -func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID string, totalWeight int64) bool { +func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool { limit := int64(math.MaxInt64) if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { limit = totalWeight * maxSmoothWeightedCreditRounds } - for id, weight := range weights { - delta := weight - if id == authID { + for i, member := range s.members { + delta := member.weight + if i == selected { delta -= totalWeight } - current := saturatingAddInt64(s.current[id], s.pending[id]) + current := saturatingAddInt64(member.current, member.pending) next := saturatingAddInt64(current, delta) if next < -limit || next > limit { return false @@ -241,36 +305,38 @@ func (s *smoothWeightedState) boundChargeFits(weights map[string]int64, authID s return true } -func (s *smoothWeightedState) charge(weights map[string]int64, authID string, bound bool) { +func (s *weightedPoolState) charge(selected int, bound bool) { var totalWeight int64 - for _, weight := range weights { - totalWeight = saturatingAddInt64(totalWeight, weight) + for _, member := range s.members { + totalWeight = saturatingAddInt64(totalWeight, member.weight) } - if !bound || s.boundChargeFits(weights, authID, totalWeight) { + if !bound || s.boundChargeFits(selected, totalWeight) { // At the cap, forced traffic cannot erase a small peer's earnings. // Pending repayments still count until they actually roll back. - for id, weight := range weights { - s.current[id] = saturatingAddInt64(s.current[id], weight) + for i := range s.members { + member := &s.members[i] + member.current = saturatingAddInt64(member.current, member.weight) } - s.current[authID] = saturatingAddInt64(s.current[authID], -totalWeight) + member := &s.members[selected] + member.current = saturatingAddInt64(member.current, -totalWeight) } if s.inFlight == 0 { - s.boundCredits(weights, totalWeight) + s.boundCredits(totalWeight) } } // boundCredits runs only once the pool settles. Net pending deltas can be zero // even with live reservations, so the in-flight count, not len(pending), decides // when clipping cannot irreversibly discard another reservation's offset. -func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight int64) { +func (s *weightedPoolState) boundCredits(totalWeight int64) { limit := int64(math.MaxInt64) if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { limit = totalWeight * maxSmoothWeightedCreditRounds } var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64 - for id := range weights { - minCredit = min(minCredit, s.current[id]) - maxCredit = max(maxCredit, s.current[id]) + for _, member := range s.members { + minCredit = min(minCredit, member.current) + maxCredit = max(maxCredit, member.current) } // A common offset keeps leading credits distinct at the compensation bound. // Independent clamps can repeatedly tie the leaders and starve the later @@ -285,8 +351,8 @@ func (s *smoothWeightedState) boundCredits(weights map[string]int64, totalWeight } // If the full spread cannot fit, cap only the trailing debt after rebasing // the leader to the upper bound. Do not flatten leading candidates' credits. - // Temporarily excluded credentials retain their credit for later selections. - for id := range weights { - s.current[id] = max(-limit, saturatingAddInt64(s.current[id], -offset)) + for i := range s.members { + member := &s.members[i] + member.current = max(-limit, saturatingAddInt64(member.current, -offset)) } } diff --git a/sdk/cliproxy/auth/selector_weighted_affinity_test.go b/sdk/cliproxy/auth/selector_weighted_affinity_test.go index 12f3c2fc7c..5736e0c821 100644 --- a/sdk/cliproxy/auth/selector_weighted_affinity_test.go +++ b/sdk/cliproxy/auth/selector_weighted_affinity_test.go @@ -52,8 +52,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) { } } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != 12 || state.current["b"] != -12 || len(state.pending) != 0 { - t.Fatalf("sticky credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 { + t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } for i := range 12 { // Requests without an identity and cold sessions share the same credits. @@ -66,8 +66,8 @@ func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) { t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick) } } - if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 { - t.Fatalf("compensated credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if binding, ok := selector.cache.Get(key); !ok || binding != "b" { t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok) @@ -116,8 +116,8 @@ func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing. if binding, ok := selector.cache.Get(key); !ok || binding != "a" { t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok) } - if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pending) != 0 { - t.Fatalf("pending credits leaked: %v", state.pending) + if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) } }) } @@ -137,8 +137,8 @@ func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) { t.Fatalf("pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if i == 0 && (state.current["a"] != -1 || state.current["b"] != 1) { - t.Fatalf("first selection charged more than once: %v", state.current) + if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) { + t.Fatalf("first selection charged more than once: %v", state.currentForTest()) } counts[picked.ID]++ } @@ -165,16 +165,16 @@ func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) { limit := maxSmoothWeightedCreditRounds * 4 // A bound charge is admitted as a whole, stopping within one delta of the cap. wantCredit := limit / 3 * 3 - if state.current["a"] != wantCredit || state.current["b"] != -wantCredit { - t.Fatalf("bounded credits = %v, want +/- %d", state.current, wantCredit) + if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit { + t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit) } for range wantCredit { if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil { t.Fatal(errPick) } } - if state.current["a"] != 0 || state.current["b"] != 0 { - t.Fatalf("compensation did not finish: %v", state.current) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 { + t.Fatalf("compensation did not finish: %v", state.currentForTest()) } } @@ -215,8 +215,8 @@ func TestWeightedSessionAffinityLCPCredits(t *testing.T) { t.Fatal("expected LCP compaction") } state := weightedStateForTest(t, weighted, "openai:model", "a", "b") - if state.current["a"] != 3 || state.current["b"] != -3 || len(state.pending) != 0 { - t.Fatalf("LCP credits: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 { + t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } }) } @@ -238,8 +238,8 @@ func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) { rejected.finish(true) accepted.finish(false) limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3 - if state.current["a"] != limit-1 || state.current["b"] != -limit+1 || len(state.pending) != 0 { - t.Fatalf("rollback lost another charge: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -252,12 +252,12 @@ func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) { _, latest := weighted.reserve(key, auths, "") old.finish(true) state := weightedStateForTest(t, weighted, key, "a", "b") - if state.current["a"] != 0 || state.current["b"] != 0 || state.pending["a"] != -1 || state.pending["b"] != 1 { - t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 { + t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } latest.finish(true) - if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 { - t.Fatalf("new generation accounting: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -294,8 +294,8 @@ func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) reservation.finish(true) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != 0 || state.current["b"] != 0 || len(state.pending) != 0 { - t.Fatalf("concurrent accounting: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } } @@ -343,19 +343,19 @@ func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) { t.Fatalf("sticky pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:requested", "a", "b") - if state == nil || len(weighted.states) != 1 || len(state.current) != 2 || state.current["a"] != 3 || state.current["b"] != -3 { + if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 { t.Fatalf("wrong model key or eligible pool: %v", weighted.states) } picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2]) - if errPick != nil || picked == nil || picked.ID != "a" || state.current["a"] != 2 || state.current["b"] != -2 { - t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.current) + if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { + t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest()) } for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} { if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil { t.Fatal(errOther) } } - if state.current["a"] != 2 || state.current["b"] != -2 { + if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { t.Fatal("another provider or model changed credits") } } @@ -374,8 +374,8 @@ func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) { t.Fatalf("reselection: pick=%v err=%v", picked, errPick) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.current["a"] != -1 || state.current["b"] != 1 || len(state.pending) != 0 { - t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.current, state.pending) + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } if binding, ok := selector.cache.Get(key); !ok || binding != "a" { t.Fatalf("reselection did not bind: %q, found=%t", binding, ok) @@ -428,7 +428,7 @@ func TestManagerWeightedSessionAffinityCompensation(t *testing.T) { } } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) - if state == nil || state.current[aID] != 0 || state.current[bID] != 0 || len(state.pending) != 0 { + if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 { t.Fatalf("manager credits did not balance: %#v", state) } }) diff --git a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go index 22ecac2eb5..58d18499a9 100644 --- a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go +++ b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go @@ -41,8 +41,8 @@ func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) { } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") limit := maxSmoothWeightedCreditRounds * 2 - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -limit, "b": limit}) { - t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.current, state.pending, state.inFlight) + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) { + t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight) } }) } @@ -98,7 +98,7 @@ func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) { t.Fatalf("%s distribution = %v, want 48:16", phase, counts) } } - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) { + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) { t.Fatalf("settled 3:1 batch drifted: %#v", state) } }) @@ -128,11 +128,11 @@ func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) { reservation.finish(true) } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") - if state.inFlight != 1 || len(state.current) != 0 || !maps.Equal(state.pending, map[string]int64{"a": -1, "b": 1}) { + if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatalf("old generation altered new accounting: %#v", state) } latest.finish(true) - if state.inFlight != 0 || len(state.pending) != 0 || !maps.Equal(state.current, map[string]int64{"a": -1, "b": 1}) { + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatalf("new generation failed to settle: %#v", state) } } @@ -167,8 +167,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) { } state := weightedStateForTest(t, weighted, "codex:model", "a", "b") // A zero net pending vector does not mean all reservations settled. - if state.inFlight != 256 || len(state.pending) != 0 { - t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pending) + if state.inFlight != 256 || len(state.pendingForTest()) != 0 { + t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest()) } last := "b" if first == "b" { @@ -179,8 +179,8 @@ func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) { reservation.finish(true) } } - if !maps.Equal(state.current, map[string]int64{"a": 0, "b": 0}) || len(state.pending) != 0 || state.inFlight != 0 { - t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 { + t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } counts := map[string]int{} for range 64 { diff --git a/sdk/cliproxy/auth/selector_weighted_memory_test.go b/sdk/cliproxy/auth/selector_weighted_memory_test.go new file mode 100644 index 0000000000..eec967e288 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_memory_test.go @@ -0,0 +1,71 @@ +package auth + +import ( + "fmt" + "runtime" + "testing" +) + +func BenchmarkWeightedPoolReserve1000(b *testing.B) { + for _, bound := range []bool{false, true} { + b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) { + auths := make([]*Auth, 1000) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + boundID := "" + if bound { + boundID = auths[i%len(auths)].ID + } + _, reservation := selector.reserve("codex:model", auths, boundID) + reservation.finish(true) + } + }) + } +} + +// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from +// total heap and allocation churn. Each retry subset omits a different credential. +func BenchmarkWeightedPoolRetainedMemory(b *testing.B) { + const members = 1000 + auths := make([]*Auth, members) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + subset := make([]*Auth, 0, members-1) + var retained, heap, allocated int64 + b.ResetTimer() + for range b.N { + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for excluded := range auths { + subset = append(subset[:0], auths[:excluded]...) + subset = append(subset, auths[excluded+1:]...) + _, reservation := selector.reserve("codex:model", subset, "") + reservation.finish(true) + } + runtime.GC() + runtime.ReadMemStats(&after) + if len(selector.states) != members+1 { + b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1) + } + runtime.KeepAlive(selector) + runtime.KeepAlive(auths) + retained += int64(after.HeapAlloc) - int64(before.HeapAlloc) + heap += int64(after.HeapAlloc) + allocated += int64(after.TotalAlloc - before.TotalAlloc) + } + b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op") + b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op") + b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op") +} diff --git a/sdk/cliproxy/auth/selector_weighted_overload_test.go b/sdk/cliproxy/auth/selector_weighted_overload_test.go index 47e3da0672..f7639a5368 100644 --- a/sdk/cliproxy/auth/selector_weighted_overload_test.go +++ b/sdk/cliproxy/auth/selector_weighted_overload_test.go @@ -50,13 +50,13 @@ func TestWeightedOverloadExactCompensation(t *testing.T) { } } for _, state := range weighted.states { - for id, credit := range state.current { + for id, credit := range state.currentForTest() { if credit != 0 { t.Errorf("exact compensation left %q credit=%d", id, credit) } } - if len(state.pending) != 0 { - t.Errorf("pending credit leaked: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Errorf("pending credit leaked: %v", state.pendingForTest()) } } }) @@ -97,7 +97,7 @@ func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) ids[i] = auth.ID } state := weightedStateForTest(t, weighted, "codex:model", ids...) - t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.current) + t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest()) // Forced traffic makes global ratios infeasible; the remaining capacity // should be shared by unbound peers rather than monopolized by one. peerWeight := 0 @@ -235,8 +235,8 @@ func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T reservation.finish(true) } state := weightedStateForTest(t, weighted, key, "a", "b", "c") - if len(state.pending) != 0 { - t.Fatalf("settled reservations leaked credit: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest()) } } @@ -249,7 +249,7 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) reservation.finish(true) } state := weightedStateForTest(t, weighted, key, "a", "b") - before := maps.Clone(state.current) + before := maps.Clone(state.currentForTest()) _, first := weighted.reserve(key, auths, "") _, second := weighted.reserve(key, auths, "") t.Cleanup(func() { first.finish(false); second.finish(false) }) @@ -265,12 +265,12 @@ func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) bound.finish(true) bound.finish(true) bound.finish(false) - if !maps.Equal(state.current, before) || len(state.pending) != 0 { - t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.current, state.pending) + if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { + t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest()) } // An over-limit bound selection succeeds without reserving any credit. picked, skipped := weighted.reserve(key, auths, "b") - if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.current, before) || len(state.pending) != 0 { + if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { t.Fatal("saturated bound selection should preserve binding and all credits") } } @@ -322,10 +322,10 @@ func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) { {"a": -222, "b": 111, "c": 111}, {"a": -112, "b": 222, "c": -110}, } { - state := &smoothWeightedState{current: maps.Clone(credit)} - state.charge(map[string]int64{"a": 1, "b": 5, "c": 1}, "a", true) - if !maps.Equal(state.current, credit) { - t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.current, credit) + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit) + state.charge(0, true) + if !maps.Equal(state.currentForTest(), credit) { + t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit) } } } @@ -391,8 +391,8 @@ func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) { t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok) } state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID) - if len(state.pending) != 0 { - t.Fatalf("manager left speculative credit: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("manager left speculative credit: %v", state.pendingForTest()) } }) } @@ -446,8 +446,8 @@ func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) { state := weightedStateForTest(t, weighted, "codex:model", "a", "b") // All 64 calls share one pool. Accounting must match actual accepted picks, // regardless of whether the final arrivals happened to be sticky requests. - if state.current["a"] != int64(64*3-counts["a"]*4) || state.current["b"] != int64(64-counts["b"]*4) || len(state.pending) != 0 { - t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.current, state.pending) + if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest()) } if bound, ok := selector.cache.Get(key); !ok || bound != "b" { t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok) diff --git a/sdk/cliproxy/auth/selector_weighted_pool_test.go b/sdk/cliproxy/auth/selector_weighted_pool_test.go index 96dae75816..5fb73ffa9c 100644 --- a/sdk/cliproxy/auth/selector_weighted_pool_test.go +++ b/sdk/cliproxy/auth/selector_weighted_pool_test.go @@ -14,13 +14,17 @@ import ( ) // weightedStateForTest checks that the requested accounting pool was created. -func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *smoothWeightedState { +func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState { t.Helper() weights := make(map[string]int64, len(ids)) for _, id := range ids { weights[id] = 1 } state := selector.states[weightedPoolStateKey(route, weights)] + ids = sortedWeightedIDs(weights) + for state != nil && !state.matches(route, ids) { + state = state.next + } if state == nil { t.Fatalf("missing weighted state for %q and %v", route, ids) } @@ -38,11 +42,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) { pendingSubset.finish(true) subset := weightedStateForTest(t, weighted, key, "a", "b") full := weightedStateForTest(t, weighted, key, "a", "b", "c") - if !maps.Equal(subset.current, map[string]int64{"a": 3, "b": -3}) || len(full.current) != 0 { - t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.current, full.current) + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() { + t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest()) } - if !maps.Equal(full.pending, map[string]int64{"a": 3, "b": -4, "c": 1}) { - t.Fatalf("another pool consumed pending credit: %v", full.pending) + if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) { + t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest()) } pendingFull.finish(false) for range 100 { @@ -53,11 +57,11 @@ func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) { for range 3 { picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "") if picked == nil || picked.ID != "a" { - t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.current) + t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest()) } reservation.finish(true) } - if !maps.Equal(subset.current, map[string]int64{"a": 0, "b": 0}) || len(subset.pending) != 0 || len(full.pending) != 0 { + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 { t.Fatalf("returning subset failed to balance: %#v", subset) } } @@ -96,14 +100,14 @@ func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) { independentReservation.finish(committed) poolKey := weightedPoolStateKey("route", authWeightVector(pool)) state, independentState := shared.states[poolKey], independent[mask].states[poolKey] - if !maps.Equal(state.current, independentState.current) || len(state.pending) != 0 { + if !maps.Equal(state.currentForTest(), independentState.currentForTest()) || len(state.pendingForTest()) != 0 { t.Fatalf("step %d pool %d: credits diverged or pending leaked", step, mask) } } } func TestWeightedPoolKeysAndCapacity(t *testing.T) { - seen := make(map[string]struct{}) + seen := make(map[[32]byte]struct{}) for _, tc := range []struct { route string ids []string @@ -145,7 +149,7 @@ func TestWeightedPoolKeysAndCapacity(t *testing.T) { t.Fatal("pool eviction failed to bound or replace state") } old.finish(true) - if len(current.state.current) != 0 || !maps.Equal(current.state.pending, map[string]int64{"a": -1, "b": 1}) { + if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { t.Fatal("finishing an evicted reservation affected the new pool") } current.finish(true) @@ -273,8 +277,8 @@ func TestWeightedSessionAffinityPoolIsolation(t *testing.T) { t.Fatalf("binding moved: %q, found=%t", bound, ok) } for _, state := range weighted.states { - if len(state.pending) != 0 { - t.Fatalf("pending credits leaked: %v", state.pending) + if len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) } } }) diff --git a/sdk/cliproxy/auth/selector_weighted_storage_test.go b/sdk/cliproxy/auth/selector_weighted_storage_test.go new file mode 100644 index 0000000000..0d937b9038 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_storage_test.go @@ -0,0 +1,88 @@ +package auth + +import ( + "maps" + "testing" +) + +// These snapshots keep assertions independent of the compact production layout. +func (s *weightedPoolState) currentForTest() map[string]int64 { + current := make(map[string]int64, len(s.members)) + for _, member := range s.members { + current[member.id] = member.current + } + return current +} + +func (s *weightedPoolState) pendingForTest() map[string]int64 { + pending := make(map[string]int64) + for _, member := range s.members { + if member.pending != 0 { + pending[member.id] = member.pending + } + } + return pending +} + +func (s *weightedPoolState) hasCurrentForTest() bool { + for _, member := range s.members { + if member.current != 0 { + return true + } + } + return false +} + +// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact +// identities, including a matching state behind a collision-chain head. +func TestWeightedPoolDigestCollisionIsolation(t *testing.T) { + for _, unrelated := range []*weightedPoolState{ + weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}), + } { + t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) { + weights := map[string]int64{"a": 1, "b": 1} + key := weightedPoolStateKey("route", weights) + selector := &WeightedRoundRobinSelector{ + states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2, + } + before := unrelated.currentForTest() + picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b") + if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated { + t.Fatal("digest collision reused an unrelated pool") + } + t.Cleanup(func() { first.finish(false) }) + matching := first.state + // Put the nonmatching state first to exercise a full chain lookup. + matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated + _, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a") + if second == nil || second.state != matching || selector.stateCount != 2 { + t.Fatal("collision lookup failed to reuse the matching pool") + } + t.Cleanup(func() { second.finish(false) }) + first.finish(true) + second.finish(true) + if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 { + t.Fatal("collision accounting changed the unrelated pool") + } + if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 { + t.Fatal("matching pool did not balance") + } + // Capacity still counts pools, not just distinct digest buckets. + _, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "") + third.finish(true) + if selector.stateCount != 1 || len(selector.states) != 1 { + t.Fatal("colliding pools bypassed the existing capacity limit") + } + }) + } +} + +func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState { + state := &weightedPoolState{route: route} + for _, id := range sortedWeightedIDs(weights) { + state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]}) + } + return state +} diff --git a/sdk/cliproxy/auth/selector_weighted_test.go b/sdk/cliproxy/auth/selector_weighted_test.go index 3d9872085f..31352e3017 100644 --- a/sdk/cliproxy/auth/selector_weighted_test.go +++ b/sdk/cliproxy/auth/selector_weighted_test.go @@ -2,6 +2,7 @@ package auth import ( "maps" + "sort" "testing" ) @@ -18,14 +19,14 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { current: map[string]int64{"a": -96, "b": 96, "c": 95}, weights: map[string]int64{"b": 1, "c": 1}, authID: "b", - want: map[string]int64{"a": -96, "b": 63, "c": 64}, + want: map[string]int64{"b": 63, "c": 64}, }, { name: "negative credits shift together", current: map[string]int64{"a": 96, "b": -64, "c": -63}, weights: map[string]int64{"b": 1, "c": 1}, authID: "b", - want: map[string]int64{"a": 96, "b": -64, "c": -61}, + want: map[string]int64{"b": -64, "c": -61}, }, { name: "excessive spread clips debt without tying leaders", @@ -43,10 +44,10 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - state := &smoothWeightedState{current: tc.current} - state.charge(tc.weights, tc.authID, false) - if !maps.Equal(state.current, tc.want) { - t.Fatalf("rebased credits = %v, want %v", state.current, tc.want) + state := weightedPoolForTest("route", tc.weights, tc.current) + state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false) + if !maps.Equal(state.currentForTest(), tc.want) { + t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want) } }) } @@ -54,12 +55,9 @@ func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) { key := "codex:model" - state := &smoothWeightedState{ - current: map[string]int64{"a": -94, "b": 94, "c": 93}, - weights: map[string]int64{"a": 1, "b": 1, "c": 1}, - pending: make(map[string]int64), - } - weighted := &WeightedRoundRobinSelector{states: map[string]*smoothWeightedState{weightedPoolStateKey(key, state.weights): state}} + weights := map[string]int64{"a": 1, "b": 1, "c": 1} + state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93}) + weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1} auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}} _, rejected := weighted.reserve(key, auths, "a") t.Cleanup(func() { rejected.finish(false) }) @@ -67,15 +65,15 @@ func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) { t.Cleanup(func() { accepted.finish(false) }) accepted.finish(true) want := map[string]int64{"a": -93, "b": 92, "c": 94} - if !maps.Equal(state.current, want) { - t.Fatalf("committed credits = %v, want %v", state.current, want) + if !maps.Equal(state.currentForTest(), want) { + t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want) } - if !maps.Equal(state.pending, map[string]int64{"a": -2, "b": 1, "c": 1}) { - t.Fatalf("commit changed another request's reservation: %v", state.pending) + if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) { + t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest()) } rejected.finish(false) rejected.finish(true) - if !maps.Equal(state.current, want) || len(state.pending) != 0 { - t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.current, state.pending) + if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) } }