diff --git a/config.example.yaml b/config.example.yaml index a3dc24045..404c40573 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,16 @@ routing: strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first # weighted-round-robin uses each credential's integer weight (default 1, maximum 1,000,000). # Non-positive weights exclude the credential while this strategy is active. + # Sticky requests (including inherited and LCP bindings) also consume weighted credits + # without moving existing bindings. Compensation is scoped to each eligible credential set; + # priority tiers, transport preferences, and retry/cooldown subsets do not share credits. + # Pinned requests use the eligible pool before pinning, within the pinned credential + # priority tier and transport class. Returning to a pool resumes its retained credits. + # Settled credits are capped at +/-32 times the eligible pool total; ratios are approximate. + # In-flight offsets are preserved until the pool settles, regardless of commit order. + # Sticky requests keep their bindings but stop adding compensation when the full + # credit delta cannot fit the cap; new sessions keep earning credit to avoid starvation. + # Credits are process-local and reset on restart or selector replacement. # For OAuth/file credentials, add a top-level numeric "weight" field to the auth JSON. # Enable universal session-sticky routing for all clients. # Explicit Claude Code, Codex, OpenCode, and pi session headers are preferred, diff --git a/docs/management-api-v8.md b/docs/management-api-v8.md index 790e70add..5aba973c3 100644 --- a/docs/management-api-v8.md +++ b/docs/management-api-v8.md @@ -64,6 +64,36 @@ The Home-owned revision fields `credentials.concurrency.observation-barrier-revision`, and `plugins.auth-revision` cannot be changed through these endpoints. +### Weighted routing with session affinity + +Set `routing.strategy` to `weighted-round-robin` to use per-credential integer +`weight` values (default `1`, maximum `1000000`; non-positive values are excluded). +With `routing.session-affinity` enabled, sticky requests, parent inheritance, +Guardian selections, and LCP matches also consume weighted credits without moving +existing bindings. New or reselected sessions favor under-served credentials +within the same eligible credential set. For example, with weights `3:1`, four +sticky requests to the second credential can be compensated by twelve new +requests to the first when both remain eligible. + +Different priority tiers, transport preferences, and retry/cooldown candidate +sets keep separate credits; returning to a set resumes its retained credits. +Pinned requests account against the eligible pool before pinning, within the +pinned credential's priority tier and transport class. A lower-priority binding +never shifts the request ratio among higher-priority credentials. + +Settled credits are capped at ±32 times the eligible pool total, so request +ratios are approximate rather than guaranteed. While reservations are in flight, +committed balances can temporarily exceed the cap; their pending offsets remain +intact until the pool settles, including when the last reservation rolls back. Sticky requests still use their binding at +the cap, but stop adding compensation when their full credit delta cannot fit. +New sessions continue repaying debt and earning credit, so excessive sticky +traffic does not starve smaller-weight peers. + +Credits are local to the CPA process and reset on restart, selector replacement, +or bounded pool-cache eviction. Concurrent selections see pending credit; local +selection/binding success confirms it, while failure releases it. In Home mode, +scheduling and credit accounting remain owned by Home. + ### Codex multi-agent configuration migration `client.codex.optimize-multi-agent-v2` is the sole runtime setting. Loading older diff --git a/sdk/cliproxy/auth/conductor_home.go b/sdk/cliproxy/auth/conductor_home.go index 44603aeaf..26e85115f 100644 --- a/sdk/cliproxy/auth/conductor_home.go +++ b/sdk/cliproxy/auth/conductor_home.go @@ -1084,11 +1084,17 @@ func (m *Manager) pickHomeDispatchSelection(ctx context.Context, model string, o } dispatchHeaders := homeDispatchHeaders(ctx, opts.Headers) if opts.Metadata != nil { - if nodeKind, ok := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string); ok && strings.TrimSpace(nodeKind) != "" { + nodeKind, _ := opts.Metadata[cliproxyexecutor.NodeKindMetadataKey].(string) + if isFork, _ := opts.Metadata[cliproxyexecutor.IsForkMetadataKey].(bool); isFork { + nodeKind = "fork" + } + if nodeKind = strings.TrimSpace(nodeKind); nodeKind != "" { + // Keep the caller headers unchanged when adding Home-only lineage metadata. + dispatchHeaders = dispatchHeaders.Clone() if dispatchHeaders == nil { dispatchHeaders = make(http.Header) } - dispatchHeaders.Set("X-Node-Kind", strings.TrimSpace(nodeKind)) + dispatchHeaders.Set("X-Node-Kind", nodeKind) } } credentialPolicy := credentialPolicyFromContext(ctx) diff --git a/sdk/cliproxy/auth/conductor_pinned_test.go b/sdk/cliproxy/auth/conductor_pinned_test.go new file mode 100644 index 000000000..0f183bc6e --- /dev/null +++ b/sdk/cliproxy/auth/conductor_pinned_test.go @@ -0,0 +1,96 @@ +package auth + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +func TestManagerNonWeightedPinnedSelection(t *testing.T) { + for _, mixed := range []bool{false, true} { + for _, affinity := range []bool{false, true} { + for _, strategy := range []string{"round-robin", "fill-first"} { + t.Run(fmt.Sprintf("mixed=%t/affinity=%t/%s", mixed, affinity, strategy), func(t *testing.T) { + var selector Selector = &RoundRobinSelector{} + if strategy == "fill-first" { + selector = &FillFirstSelector{} + } + if affinity { + sticky := NewSessionAffinitySelector(selector) + t.Cleanup(sticky.Stop) + selector = sticky + } + manager := NewManager(nil, selector, nil) + const provider, model = "pinned-non-weighted", "pinned-model" + manager.RegisterExecutor(schedulerTestExecutor{provider: provider}) + pinnedID := t.Name() + "-pinned" + for i := range 32 { + auth := &Auth{ + ID: fmt.Sprintf("%s-other-%02d", t.Name(), i), Provider: provider, Status: StatusActive, + Attributes: map[string]string{AttributeAuthKind: AuthKindAPIKey, "priority": "10"}, + } + if i == 0 { + auth.ID = pinnedID + auth.Attributes["priority"] = "0" + } + registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) }) + if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil { + t.Fatal(errRegister) + } + } + // Normalize the legacy selection signatures for the same pinning checks. + pick := func(ctx context.Context, id string, tried map[string]struct{}) (*Auth, error) { + opts := cliproxyexecutor.Options{ + Headers: http.Header{"Session_id": {"non-weighted-pinned"}}, + Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: id}, + } + if mixed { + auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider}, model, opts, tried) + return auth, errPick + } + auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried) + return auth, errPick + } + ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey) + for range 3 { + if auth, errPick := pick(ctx, pinnedID, nil); errPick != nil || auth == nil || auth.ID != pinnedID { + t.Fatalf("lower-priority pin not enforced: auth=%v err=%v", auth, errPick) + } + } + for _, tc := range []struct { + name string + ctx context.Context + id string + tried map[string]struct{} + }{ + {name: "missing pin", ctx: ctx, id: "missing"}, + {name: "already tried", ctx: ctx, id: pinnedID, tried: map[string]struct{}{pinnedID: {}}}, + {name: "ineligible kind", ctx: withRequiredAuthKind(t.Context(), AuthKindOAuth), id: pinnedID}, + } { + auth, errPick := pick(tc.ctx, tc.id, tc.tried) + var authErr *Error + if auth != nil || !errors.As(errPick, &authErr) || authErr.Code != "auth_not_found" { + t.Fatalf("%s: auth=%v err=%v, want auth_not_found", tc.name, auth, errPick) + } + } + if sticky, ok := selector.(*SessionAffinitySelector); ok { + namespace := provider + if mixed { + namespace = "mixed" + } + id := CanonicalSessionID(http.Header{"Session_id": {"non-weighted-pinned"}}, nil, nil) + if binding, found := sticky.cache.Get(namespace + "::" + id + "::" + model); !found || binding != pinnedID { + t.Fatalf("pinning checks changed the sticky binding: %q, found=%t", binding, found) + } + } + }) + } + } + } +} diff --git a/sdk/cliproxy/auth/conductor_selection.go b/sdk/cliproxy/auth/conductor_selection.go index 330a29ccd..ec12690e2 100644 --- a/sdk/cliproxy/auth/conductor_selection.go +++ b/sdk/cliproxy/auth/conductor_selection.go @@ -2258,7 +2258,11 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op targetKey := canonicalSchedulingProvider(provider) affinitySelector, hasAffinity := selector.(*SessionAffinitySelector) trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever) - var affinityCandidates []*Auth + trackPinnedAccounting := false + if hasAffinity && affinitySelector != nil && pinnedAuthID != "" { + _, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector) + } + var accountingCandidates, affinityCandidates []*Auth if trackTransientAffinity { affinityCandidates = make([]*Auth, 0, len(m.auths)) } @@ -2266,7 +2270,7 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op if candidate == nil || canonicalSchedulingProvider(executorKeyFromAuth(candidate)) != targetKey || candidate.Disabled { continue } - if pinnedAuthID != "" && candidate.ID != pinnedAuthID { + if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID { continue } if !eligibility.allows(candidate) { @@ -2275,10 +2279,18 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) { continue } + _, used := tried[candidate.ID] + // Pinning restricts selection, not the eligible pool used for accounting. + if trackPinnedAccounting && !used { + accountingCandidates = append(accountingCandidates, candidate) + } + if pinnedAuthID != "" && candidate.ID != pinnedAuthID { + continue + } if trackTransientAffinity { affinityCandidates = append(affinityCandidates, candidate) } - if _, used := tried[candidate.ID]; used { + if used { continue } candidates = append(candidates, candidate) @@ -2302,6 +2314,15 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op return nil, nil, errAvailable } } + var accountingAuths []*Auth + if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil { + _, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, provider, model, now, false, opts.Headers) + if errAccounting != nil { + m.mu.RUnlock() + return nil, nil, errAccounting + } + accountingAuths = fullAvailable + } transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers) m.mu.RUnlock() @@ -2313,6 +2334,9 @@ func (m *Manager) pickNextLegacy(ctx context.Context, provider, model string, op if !handled { selectorCtx := selectorContextForAvailableAuths(ctx, selector, model) selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs) + if accountingAuths != nil { + selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths) + } selected, errPick = selector.Pick(selectorCtx, provider, selectionArgForSelector(selector, model), opts, selectorAuths) if errPick != nil { if isAuthNotFoundError(errPick) { @@ -2678,7 +2702,11 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m registryRef := registry.GetGlobalRegistry() affinitySelector, hasAffinity := selector.(*SessionAffinitySelector) trackTransientAffinity := hasAffinity && affinitySelector != nil && (affinitySelector.rebindOnQuotaOnly || affinitySelector.rebindOnNever) - var affinityCandidates []*Auth + trackPinnedAccounting := false + if hasAffinity && affinitySelector != nil && pinnedAuthID != "" { + _, trackPinnedAccounting = affinitySelector.fallback.(*WeightedRoundRobinSelector) + } + var accountingCandidates, affinityCandidates []*Auth if trackTransientAffinity { affinityCandidates = make([]*Auth, 0, len(m.auths)) } @@ -2686,7 +2714,7 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m if candidate == nil || candidate.Disabled { continue } - if pinnedAuthID != "" && candidate.ID != pinnedAuthID { + if !trackPinnedAccounting && pinnedAuthID != "" && candidate.ID != pinnedAuthID { continue } if !eligibility.allows(candidate) { @@ -2705,10 +2733,18 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m if modelKey != "" && !m.authSupportsRouteModel(registryRef, candidate, model) { continue } + _, used := tried[candidate.ID] + // Pinning restricts selection, not the eligible pool used for accounting. + if trackPinnedAccounting && !used { + accountingCandidates = append(accountingCandidates, candidate) + } + if pinnedAuthID != "" && candidate.ID != pinnedAuthID { + continue + } if trackTransientAffinity { affinityCandidates = append(affinityCandidates, candidate) } - if _, used := tried[candidate.ID]; used { + if used { continue } candidates = append(candidates, candidate) @@ -2732,6 +2768,15 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m return nil, nil, "", errAvailable } } + var accountingAuths []*Auth + if trackPinnedAccounting && len(candidates) > 0 && errAvailable == nil { + _, fullAvailable, errAccounting := m.availableAuthsForSelector(ctx, selector, accountingCandidates, "mixed", model, now, false, opts.Headers) + if errAccounting != nil { + m.mu.RUnlock() + return nil, nil, "", errAccounting + } + accountingAuths = fullAvailable + } transientAuthIDs := m.transientlyUnavailableSessionAffinityAuthIDs(ctx, selector, affinityCandidates, model, now, opts.Headers) m.mu.RUnlock() @@ -2743,6 +2788,9 @@ func (m *Manager) pickNextMixedLegacy(ctx context.Context, providers []string, m if !handled { selectorCtx := selectorContextForAvailableAuths(ctx, selector, model) selectorCtx = withSessionAffinityTransientAuthIDs(selectorCtx, transientAuthIDs) + if accountingAuths != nil { + selectorCtx = context.WithValue(selectorCtx, weightedAccountingPoolKey{}, accountingAuths) + } selected, errPick = selector.Pick(selectorCtx, "mixed", selectionArgForSelector(selector, model), opts, selectorAuths) if errPick != nil { if isAuthNotFoundError(errPick) { diff --git a/sdk/cliproxy/auth/conductor_weighted_pinned_test.go b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go new file mode 100644 index 000000000..8768a2a8a --- /dev/null +++ b/sdk/cliproxy/auth/conductor_weighted_pinned_test.go @@ -0,0 +1,161 @@ +package auth + +import ( + "fmt" + "maps" + "net/http" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator" +) + +func TestManagerWeightedPinnedSessionCompensation(t *testing.T) { + for _, mixed := range []bool{false, true} { + for _, mode := range []string{"sticky", "cold", "no-session", "lcp"} { + t.Run(fmt.Sprintf("mixed=%t/%s", mixed, mode), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + manager := NewManager(nil, selector, nil) + const provider, model = "weighted-pinned", "weighted-pinned-model" + manager.RegisterExecutor(schedulerTestExecutor{provider: provider}) + ctx := withRequiredAuthKind(t.Context(), AuthKindAPIKey) + aID, bID := t.Name()+"-a", t.Name()+"-b" + triedID := t.Name() + "-tried" + excluded := []*Auth{ + {ID: t.Name() + "-disabled", Disabled: true}, + {ID: t.Name() + "-zero", Attributes: map[string]string{AttributeWeight: "0"}}, + {ID: t.Name() + "-cooldown", ModelStates: map[string]*ModelState{ + model: {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)}, + }}, + {ID: t.Name() + "-wrong-model"}, + {ID: t.Name() + "-wrong-kind", Attributes: map[string]string{AttributeAuthKind: AuthKindOAuth}}, + {ID: t.Name() + "-wrong-provider", Provider: "other-provider"}, + {ID: t.Name() + "-no-executor", Provider: "no-executor"}, + {ID: t.Name() + "-lower-priority", Attributes: map[string]string{"priority": "0"}}, + {ID: triedID}, + } + auths := append([]*Auth{ + {ID: aID, Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}}, + {ID: bID, Attributes: map[string]string{AttributeWeight: "1", "priority": "1"}}, + }, excluded...) + for _, auth := range auths { + if auth.Provider == "" { + auth.Provider = provider + } + auth.Status = StatusActive + if auth.Attributes == nil { + auth.Attributes = make(map[string]string) + } + if _, ok := auth.Attributes[AttributeWeight]; !ok { + auth.Attributes[AttributeWeight] = "100" + } + if _, ok := auth.Attributes["priority"]; !ok { + auth.Attributes["priority"] = "2" + } + if _, ok := auth.Attributes[AttributeAuthKind]; !ok { + auth.Attributes[AttributeAuthKind] = AuthKindAPIKey + } + registeredModel := model + if auth.ID == t.Name()+"-wrong-model" { + registeredModel = "other-model" + } + registry.GetGlobalRegistry().RegisterClient(auth.ID, auth.Provider, []*registry.ModelInfo{{ID: registeredModel}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) }) + if _, errRegister := manager.Register(WithSkipPersist(ctx), auth); errRegister != nil { + t.Fatal(errRegister) + } + } + namespace := provider + if mixed { + namespace = "mixed" + } + stickyHeaders := http.Header{"Session_id": {"pinned-session"}} + id := CanonicalSessionID(stickyHeaders, nil, nil) + bindingKey := namespace + "::" + id + "::" + model + if mode == "sticky" { + selector.cache.Set(bindingKey, bID) + } + tried := map[string]struct{}{triedID: {}} + // Normalize the legacy signatures without changing their pinning behavior. + pick := func(opts cliproxyexecutor.Options) (*Auth, error) { + if mixed { + auth, _, _, errPick := manager.pickNextMixedLegacy(ctx, []string{provider, "no-executor"}, model, opts, tried) + return auth, errPick + } + auth, _, errPick := manager.pickNextLegacy(ctx, provider, model, opts, tried) + return auth, errPick + } + for i := range 4 { + opts := cliproxyexecutor.Options{Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: bID}} + switch mode { + case "sticky", "cold": + opts.Headers = stickyHeaders + case "lcp": + opts.SourceFormat = sdktranslator.FormatOpenAI + opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"pinned conversation"}]}`) + opts.Metadata[cliproxyexecutor.CallerScopeMetadataKey] = "pinned-caller" + } + if auth, errPick := pick(opts); errPick != nil || auth == nil || auth.ID != bID { + t.Fatalf("pinned request %d: auth=%v err=%v, want %q", i, auth, errPick, bID) + } + } + state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) + if state == nil || !maps.Equal(state.currentForTest(), map[string]int64{aID: 12, bID: -12}) || len(state.pendingForTest()) != 0 { + t.Fatalf("pinned traffic did not consume the eligible pool's credit: %#v", state) + } + counts := map[string]int{} + for i := range 12 { + auth, errPick := pick(cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}}) + if errPick != nil || auth == nil { + t.Fatalf("fresh request %d: auth=%v err=%v", i, auth, errPick) + } + counts[auth.ID]++ + } + if counts[aID] != 12 || counts[bID] != 0 { + t.Fatalf("fresh distribution = %v, want a=12 b=0", counts) + } + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("compensation did not balance: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + if mode == "sticky" || mode == "cold" { + if binding, ok := selector.cache.Get(bindingKey); !ok || binding != bID { + t.Fatalf("compensation changed pinned binding: %q, found=%t", binding, ok) + } + } + // A broader accounting pool must never make an invalid pin selectable. + for _, pinnedID := range []string{excluded[0].ID, excluded[1].ID, excluded[2].ID, excluded[3].ID, excluded[4].ID, excluded[5].ID, excluded[6].ID, triedID, "missing"} { + opts := cliproxyexecutor.Options{ + Headers: stickyHeaders, + Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: pinnedID}, + } + if auth, errPick := pick(opts); errPick == nil || auth != nil { + t.Fatalf("invalid pin %q escaped pinning checks: auth=%v err=%v", pinnedID, auth, errPick) + } + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("invalid pin %q changed accounting: current=%v pending=%v", pinnedID, state.currentForTest(), state.pendingForTest()) + } + } + // A valid lower-priority pin still wins without charging another tier. + lowerID := t.Name() + "-lower-priority" + lowerOpts := cliproxyexecutor.Options{ + Headers: http.Header{"Session_id": {"lower-pinned-session"}}, + Metadata: map[string]any{cliproxyexecutor.PinnedAuthMetadataKey: lowerID}, + } + if auth, errPick := pick(lowerOpts); errPick != nil || auth == nil || auth.ID != lowerID { + t.Fatalf("lower-priority pin was not enforced: auth=%v err=%v", auth, errPick) + } + if !maps.Equal(state.currentForTest(), map[string]int64{aID: 0, bID: 0}) || len(state.pendingForTest()) != 0 { + t.Fatalf("lower-priority pin changed higher-tier credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + lowerState := weightedStateForTest(t, weighted, namespace+":"+model, lowerID) + if !maps.Equal(lowerState.currentForTest(), map[string]int64{lowerID: 0}) || len(lowerState.pendingForTest()) != 0 { + t.Fatalf("lower-priority pin accounted the wrong pool: %#v", lowerState) + } + }) + } + } +} diff --git a/sdk/cliproxy/auth/home_session_kind_test.go b/sdk/cliproxy/auth/home_session_kind_test.go new file mode 100644 index 000000000..76203a8bf --- /dev/null +++ b/sdk/cliproxy/auth/home_session_kind_test.go @@ -0,0 +1,59 @@ +package auth + +import ( + "context" + "net/http" + "testing" + + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +type sessionKindCaptureDispatcher struct { + fixtureHomeDispatcher + headers http.Header + sessionID string + parentSessionID string +} + +func (d *sessionKindCaptureDispatcher) RPopAuthWithSessionHierarchy(_ context.Context, _ string, sessionID, parentSessionID string, headers http.Header, _ int, _ string, _ *int, _ []string, _ string) ([]byte, error) { + d.headers = headers.Clone() + d.sessionID = sessionID + d.parentSessionID = parentSessionID + return d.payload, nil +} + +func TestHomeDispatchPreservesForkAndCompactionKind(t *testing.T) { + for _, tc := range []struct { + name string + body string + metadata map[string]any + want string + }{ + {name: "body fork", body: `{"thread_id":"child","metadata":{"forked_from_thread_id":"parent"}}`, want: "fork"}, + {name: "compaction", body: `{"session_id":"child","parent_session_id":"parent"}`, metadata: map[string]any{cliproxyexecutor.NodeKindMetadataKey: "compaction"}, want: "compaction"}, + {name: "ordinary child", body: `{"session_id":"child","parent_session_id":"parent"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + dispatcher := &sessionKindCaptureDispatcher{fixtureHomeDispatcher: fixtureHomeDispatcher{payload: []byte(`{"auth":{"id":"kind-auth","provider":"kind-provider"}}`)}} + manager := newHomeSelectionTestManager(t, dispatcher) + manager.RegisterExecutor(schedulerTestExecutor{provider: "kind-provider"}) + headers := http.Header{"User-Agent": {"test-client"}} + opts := cliproxyexecutor.Options{Headers: headers, OriginalRequest: []byte(tc.body), Metadata: tc.metadata} + opts.EnsureMetadata() + selection, errSelection := manager.pickHomeDispatchSelection(t.Context(), "model", opts) + if errSelection != nil { + t.Fatal(errSelection) + } + selection.End("test_complete") + if dispatcher.sessionID == "" || dispatcher.parentSessionID == "" || dispatcher.sessionID == dispatcher.parentSessionID { + t.Fatalf("lost session hierarchy: child=%q parent=%q", dispatcher.sessionID, dispatcher.parentSessionID) + } + if got := dispatcher.headers.Get("X-Node-Kind"); got != tc.want { + t.Fatalf("node kind=%q, want %q", got, tc.want) + } + if got := headers.Get("X-Node-Kind"); got != "" { + t.Fatalf("caller headers mutated: %q", got) + } + }) + } +} diff --git a/sdk/cliproxy/auth/scheduler_benchmark_test.go b/sdk/cliproxy/auth/scheduler_benchmark_test.go index 85a901435..91817fd60 100644 --- a/sdk/cliproxy/auth/scheduler_benchmark_test.go +++ b/sdk/cliproxy/auth/scheduler_benchmark_test.go @@ -214,3 +214,41 @@ func BenchmarkManagerPickNextAndMarkResult1000(b *testing.B) { manager.MarkResult(ctx, Result{AuthID: auth.ID, Provider: "gemini", Model: model, Success: true}) } } + +func BenchmarkManagerPinnedLegacy1000(b *testing.B) { + for _, mixed := range []bool{false, true} { + b.Run(fmt.Sprintf("mixed=%t", mixed), func(b *testing.B) { + manager, providers, model := benchmarkManagerSetup(b, 1000, mixed, false) + selector := NewSessionAffinitySelector(&RoundRobinSelector{}) + b.Cleanup(selector.Stop) + manager.SetSelector(selector) + ctx := context.Background() + opts := cliproxyexecutor.Options{Metadata: map[string]any{ + cliproxyexecutor.PinnedAuthMetadataKey: "bench-gemini-0000", + }} + // Warm up index assignment and selection metadata outside the timed loop. + if mixed { + if _, _, _, errWarm := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil); errWarm != nil { + b.Fatal(errWarm) + } + } else if _, _, errWarm := manager.pickNextLegacy(ctx, providers[0], model, opts, nil); errWarm != nil { + b.Fatal(errWarm) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if mixed { + auth, exec, provider, errPick := manager.pickNextMixedLegacy(ctx, providers, model, opts, nil) + if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil || provider != providers[0] { + b.Fatalf("pinned mixed pick failed: auth=%v exec=%v provider=%q err=%v", auth, exec, provider, errPick) + } + } else { + auth, exec, errPick := manager.pickNextLegacy(ctx, providers[0], model, opts, nil) + if errPick != nil || auth == nil || auth.ID != "bench-gemini-0000" || exec == nil { + b.Fatalf("pinned pick failed: auth=%v exec=%v err=%v", auth, exec, errPick) + } + } + } + }) + } +} diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go index 323f13af0..af11116b9 100644 --- a/sdk/cliproxy/auth/selector.go +++ b/sdk/cliproxy/auth/selector.go @@ -39,9 +39,10 @@ type RoundRobinSelector struct { // WeightedRoundRobinSelector provides smooth weighted round-robin selection. type WeightedRoundRobinSelector struct { - mu sync.Mutex - states map[string]*smoothWeightedState - maxKeys int + mu sync.Mutex + states map[[32]byte]*weightedPoolState + stateCount int + maxKeys int } type smoothWeightedState struct { @@ -698,32 +699,12 @@ func (s *WeightedRoundRobinSelector) Pick(ctx context.Context, provider, model s return nil, errAvailable } available = preferCodexWebsocketAuths(ctx, provider, available) - stateModel := weightedSelectorStateModel(ctx, model) - key := provider + ":" + canonicalModelLookupKey(stateModel) - - s.mu.Lock() - defer s.mu.Unlock() - if s.states == nil { - s.states = make(map[string]*smoothWeightedState) - } - limit := s.maxKeys - if limit <= 0 { - limit = 4096 - } - if _, ok := s.states[key]; !ok && len(s.states) >= limit { - s.states = make(map[string]*smoothWeightedState) - } - state := s.states[key] - if state == nil { - state = &smoothWeightedState{} - s.states[key] = state - } - weights := authWeightVector(available) - state.prepare(weights) - picked := pickSmoothWeightedAuth(available, state.current) + key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model)) + picked, reservation := s.reserve(key, available, "") if picked == nil { return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"} } + reservation.finish(true) return picked, nil } @@ -796,29 +777,6 @@ func authWeightVector(auths []*Auth) map[string]int64 { return weights } -func pickSmoothWeightedAuth(auths []*Auth, current map[string]int64) *Auth { - var picked *Auth - var pickedCurrent int64 - var totalWeight int64 - for _, auth := range auths { - weight := authWeight(auth) - if auth == nil || weight <= 0 { - continue - } - current[auth.ID] = saturatingAddInt64(current[auth.ID], weight) - totalWeight = saturatingAddInt64(totalWeight, weight) - if picked == nil || current[auth.ID] > pickedCurrent { - picked = auth - pickedCurrent = current[auth.ID] - } - } - if picked == nil { - return nil - } - current[picked.ID] = saturatingAddInt64(current[picked.ID], -totalWeight) - return picked -} - func saturatingAddInt64(value, delta int64) int64 { if delta > 0 && value > math.MaxInt64-delta { return math.MaxInt64 @@ -1164,7 +1122,16 @@ func (s *SessionAffinitySelector) Trees() *cliproxysession.InMemorySessionTreeSt // Note: The cache key includes provider, session ID, and model to handle cases where // a session uses multiple models (e.g., gemini-2.5-pro and gemini-3-flash-preview) // that may be supported by different auth credentials, and to avoid cross-provider conflicts. -func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) { +func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (selected *Auth, errPick error) { + if ctx == nil { + ctx = context.Background() + } + if errContext := ctx.Err(); errContext != nil { + return nil, errContext + } + var reservation *weightedReservation + // Local bindings complete inside Pick; failed selections release speculative credit. + defer func() { reservation.finish(selected != nil && errPick == nil) }() entry := selectorLogEntry(ctx) if opts.Metadata == nil { opts.Metadata = make(map[string]any) @@ -1176,7 +1143,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri // consulted when no header, body, or execution-session identity is present. explicitID, explicitFallbackID := extractExplicitSessionIDs(opts.Headers, opts.OriginalRequest, opts.Metadata) if explicitID == "" { - if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry); handled || errLCP != nil { + if auth, handled, errLCP := s.pickLCP(ctx, provider, model, opts, auths, entry, &reservation); handled || errLCP != nil { return auth, errLCP } } else if opts.Metadata != nil { @@ -1218,7 +1185,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri return nil, errAvailable } entry.Debugf("session-affinity: no session ID extracted, falling back to default selector | provider=%s model=%s", provider, model) - return s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) + return s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation) } // A single availability pass serves both lookups: the bound credential is validated against @@ -1268,7 +1235,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri if auth.ID == cachedAuthID { bind(auth.ID, true) entry.Infof("session-affinity: cache hit | session=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), auth.ID, provider, model) - return auth, nil + return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation) } } if s.rebindOnNever { @@ -1281,7 +1248,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri return nil, err } // Cached auth not available, reselect via fallback selector for even distribution - auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) + auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation) if err != nil { return nil, err } @@ -1315,7 +1282,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri if auth.ID == parentAuthID { bind(auth.ID, true) entry.Infof("session-affinity: guardian inherited parent binding | session=%s parent=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model) - return auth, nil + return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation) } } } @@ -1332,7 +1299,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri } else { entry.Infof("session-affinity: fallback cache hit | session=%s fallback=%s auth=%s provider=%s model=%s", truncateSessionID(primaryID), truncateSessionID(fallbackID), auth.ID, provider, model) } - return auth, nil + return s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, &reservation) } } } @@ -1343,7 +1310,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri return nil, errBound } if !isSubagent && !isFork && s.shouldPreserveBindingWhenUnavailable(ctx, availabilityCandidates, model, cachedAuthID, now) { - auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) + auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation) if err != nil { return nil, err } @@ -1360,7 +1327,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri if err != nil { return nil, err } - auth, err := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) + auth, err := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, &reservation) if err != nil { return nil, err } @@ -1376,7 +1343,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri return auth, nil } -func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry) (*Auth, bool, error) { +func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, entry *log.Entry, reservation **weightedReservation) (*Auth, bool, error) { if s == nil || s.matcher == nil { return nil, false, nil } @@ -1449,7 +1416,8 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s } entry.Infof("session-affinity: LCP cache hit | session=%s prefix=%d auth=%s provider=%s model=%s", truncateSessionID(match.SessionID), match.PrefixLength, auth.ID, provider, model) } - return auth, true, nil + picked, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, available, auth, reservation) + return picked, true, errPick } if !match.IsFork && s.rebindOnNever { if bindRes := s.matcher.BindFingerprintsWithContext(namespace, fingerprints, tailFingerprints, envDigest, minPrefixLength, match.AuthID); bindRes.SessionID != "" { @@ -1488,7 +1456,7 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s return nil, true, errAvailable } fallbackAuths := highestPriorityAuths(available) - auth, errPick := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) + auth, errPick := s.pickWithAffinityReservation(ctx, provider, model, opts, fallbackAuths, nil, reservation) if errPick != nil { return nil, true, errPick } diff --git a/sdk/cliproxy/auth/selector_test.go b/sdk/cliproxy/auth/selector_test.go index 18ed81f30..37678684b 100644 --- a/sdk/cliproxy/auth/selector_test.go +++ b/sdk/cliproxy/auth/selector_test.go @@ -205,16 +205,14 @@ func TestAuthWeight_InvalidAndOverflowValuesAreExcluded(t *testing.T) { } } -func TestPickSmoothWeightedAuth_SaturatesCorruptState(t *testing.T) { +func TestSmoothWeightedStateCharge_SaturatesAndClampsCorruptState(t *testing.T) { t.Parallel() - current := map[string]int64{"a": math.MaxInt64, "b": math.MinInt64} - picked := pickSmoothWeightedAuth([]*Auth{{ID: "a"}, {ID: "b"}}, current) - if picked == nil { - t.Fatal("pickSmoothWeightedAuth() returned nil") - } - if current["a"] != math.MaxInt64-2 || current["b"] != math.MinInt64+1 { - t.Fatalf("current state = %#v, want saturated arithmetic", current) + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": math.MaxInt64, "b": math.MinInt64}) + state.charge(0, false) + limit := maxSmoothWeightedCreditRounds * 2 + if state.currentForTest()["a"] != limit || state.currentForTest()["b"] != -limit { + t.Fatalf("current state = %#v, want saturated and clamped arithmetic", state.currentForTest()) } } diff --git a/sdk/cliproxy/auth/selector_weighted.go b/sdk/cliproxy/auth/selector_weighted.go new file mode 100644 index 000000000..57852f6b2 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted.go @@ -0,0 +1,358 @@ +package auth + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "math" + "sort" + "time" + + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +// weightedAccountingPoolKey carries the manager-validated pre-pin pool. It affects +// accounting only; the selector still receives pin-filtered selection candidates. +type weightedAccountingPoolKey struct{} + +// weightedReservation separates speculative credit from committed, clamped credit. +// Releasing one request must never undo accounting committed by another request. +type weightedReservation struct { + selector *WeightedRoundRobinSelector + state *weightedPoolState + generation uint64 + member int + total int64 + bound bool + finished bool +} + +// pickWithAffinityReservation normalizes weighted candidates and defers accounting +// until the local binding completes. Other fallback selectors keep their behavior. +func (s *SessionAffinitySelector) pickWithAffinityReservation(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth, bound *Auth, reservation **weightedReservation) (*Auth, error) { + weighted, ok := s.fallback.(*WeightedRoundRobinSelector) + if !ok { + if bound != nil { + return bound, nil + } + return s.fallback.Pick(ctx, provider, model, opts, auths) + } + if accountingAuths, ok := ctx.Value(weightedAccountingPoolKey{}).([]*Auth); ok { + pinnedID := pinnedAuthIDFromMetadata(opts.Metadata) + if bound == nil { + for _, auth := range auths { + if auth != nil && auth.ID == pinnedID { + bound = auth + break + } + } + } + if bound == nil || bound.ID != pinnedID || authWeight(bound) <= 0 { + return nil, &Error{Code: "auth_unavailable", Message: "pinned auth is not available for weighted accounting"} + } + auths = positiveWeightAuths(accountingAuths) + } + var available []*Auth + authID := "" + if bound == nil { + var errAvailable error + available, errAvailable = getSelectorAvailableAuths(ctx, positiveWeightAuths(auths), provider, model, time.Now()) + if errAvailable != nil { + return nil, errAvailable + } + available = preferCodexWebsocketAuths(ctx, provider, available) + } else { + authID = bound.ID + // A binding may bypass priority or transport preference, but its traffic + // must only charge peers that compete with it, never another priority tier. + priority := authPriority(bound) + for _, auth := range auths { + if authPriority(auth) == priority { + available = append(available, auth) + } + } + preferred := preferCodexWebsocketAuths(ctx, provider, available) + if len(preferred) < len(available) && !authWebsocketsEnabled(bound) { + // Keep an HTTP-bound session outside the preferred WebSocket pool. + nonWebsocket := available[:0] + for _, auth := range available { + if !authWebsocketsEnabled(auth) { + nonWebsocket = append(nonWebsocket, auth) + } + } + available = nonWebsocket + } else { + available = preferred + } + } + key := provider + ":" + canonicalModelLookupKey(weightedSelectorStateModel(ctx, model)) + picked, pending := weighted.reserve(key, available, authID) + if picked == nil { + return nil, &Error{Code: "auth_unavailable", Message: "no auth available with positive weight"} + } + *reservation = pending + return picked, nil +} + +// weightedPoolMember stores each ID once, with parallel accounting values in a +// single allocation. Members are sorted by ID; reservations retain only an index. +type weightedPoolMember struct { + id string + weight int64 + current int64 + pending int64 +} + +// weightedPoolState is separate from the scheduler's transient-subset accumulator: +// each selector pool has fixed membership and shares its weights across reservations. +type weightedPoolState struct { + route string + members []weightedPoolMember + inFlight uint64 + generation uint64 + next *weightedPoolState // Exact-identity collision chain for the digest key. +} + +func sortedWeightedIDs(weights map[string]int64) []string { + ids := make([]string, 0, len(weights)) + for id := range weights { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// weightedPoolStateKey isolates competing sets without retaining concatenated IDs. +// Weights stay out of the key so configuration changes invalidate reservations. +func weightedPoolStateKey(route string, weights map[string]int64) [32]byte { + return weightedPoolDigest(route, sortedWeightedIDs(weights)) +} + +func weightedPoolDigest(route string, ids []string) [32]byte { + size := 8 + len(route) + for _, id := range ids { + size += 8 + len(id) + } + identity := make([]byte, 0, size) + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(route))) + identity = append(identity, route...) + for _, id := range ids { + identity = binary.LittleEndian.AppendUint64(identity, uint64(len(id))) + identity = append(identity, id...) + } + return sha256.Sum256(identity) +} + +// matches verifies the full identity even on a digest hit. Hash collisions must +// never combine the quota or configured weights of unrelated candidate pools. +func (s *weightedPoolState) matches(route string, ids []string) bool { + if s.route != route || len(s.members) != len(ids) { + return false + } + for i, id := range ids { + if s.members[i].id != id { + return false + } + } + return true +} + +func (s *weightedPoolState) prepare(weights map[string]int64) { + for _, member := range s.members { + if member.weight != weights[member.id] { + s.generation++ + s.inFlight = 0 + for i := range s.members { + s.members[i].weight = weights[s.members[i].id] + s.members[i].current = 0 + s.members[i].pending = 0 + } + return + } + } +} + +// reserve selects and reserves credit atomically. A non-empty authID preserves a +// sticky binding; otherwise committed and pending credits both guide selection. +func (s *WeightedRoundRobinSelector) reserve(route string, available []*Auth, authID string) (*Auth, *weightedReservation) { + weights := authWeightVector(available) + ids := sortedWeightedIDs(weights) + key := weightedPoolDigest(route, ids) + s.mu.Lock() + defer s.mu.Unlock() + if s.states == nil { + s.states = make(map[[32]byte]*weightedPoolState) + } + state := s.states[key] + for state != nil && !state.matches(route, ids) { + state = state.next + } + if state == nil { + limit := s.maxKeys + if limit <= 0 { + limit = 4096 + } + if s.stateCount >= limit { + s.states = make(map[[32]byte]*weightedPoolState) + s.stateCount = 0 + } + state = &weightedPoolState{route: route, members: make([]weightedPoolMember, len(ids)), next: s.states[key]} + for i, id := range ids { + state.members[i] = weightedPoolMember{id: id, weight: weights[id]} + } + s.states[key] = state + s.stateCount++ + } else { + state.prepare(weights) + } + var picked *Auth + var pickedCurrent, total int64 + pickedMember := -1 + for _, auth := range available { + if auth == nil || weights[auth.ID] <= 0 { + continue + } + total = saturatingAddInt64(total, weights[auth.ID]) + if authID != "" && auth.ID != authID { + continue + } + index := sort.SearchStrings(ids, auth.ID) + member := &state.members[index] + current := saturatingAddInt64(member.current, member.pending) + current = saturatingAddInt64(current, member.weight) + if picked == nil || current > pickedCurrent { + picked, pickedCurrent, pickedMember = auth, current, index + } + } + if picked == nil { + return nil, nil + } + // Bound traffic beyond the compensation window still keeps its credential, + // but must not create speculative credit that can starve concurrent picks. + if authID != "" && !state.boundChargeFits(pickedMember, total) { + return picked, nil + } + state.adjustPending(pickedMember, total, false) + state.inFlight++ + return picked, &weightedReservation{ + selector: s, state: state, generation: state.generation, + member: pickedMember, total: total, bound: authID != "", + } +} + +// finish confirms or releases a reservation exactly once under the selector lock. +func (r *weightedReservation) finish(committed bool) { + if r == nil { + return + } + r.selector.mu.Lock() + defer r.selector.mu.Unlock() + if r.finished { + return + } + r.finished = true + if r.state.generation != r.generation { + return + } + r.state.adjustPending(r.member, r.total, true) + r.state.inFlight-- + if committed { + r.state.charge(r.member, r.bound) + } else if r.state.inFlight == 0 { + // A final rollback can remove the offset to already committed credit. + r.state.boundCredits(r.total) + } +} + +// adjustPending must be called with the selector mutex held. Pending deltas are +// not clamped, so releasing them cannot lose credit at the compensation bound. +func (s *weightedPoolState) adjustPending(selected int, total int64, release bool) { + for i := range s.members { + member := &s.members[i] + delta := member.weight + if i == selected { + delta -= total + } + if release { + delta = -delta + } + member.pending = saturatingAddInt64(member.pending, delta) + } +} + +// maxSmoothWeightedCreditRounds bounds compensation after a long sticky session. +const maxSmoothWeightedCreditRounds int64 = 32 + +// boundChargeFits admits the whole sticky delta or none. Both reservation and +// commit include outstanding deltas: committed credit alone can transiently +// exceed the cap while successful reservations settle in a different order. +func (s *weightedPoolState) boundChargeFits(selected int, totalWeight int64) bool { + limit := int64(math.MaxInt64) + if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { + limit = totalWeight * maxSmoothWeightedCreditRounds + } + for i, member := range s.members { + delta := member.weight + if i == selected { + delta -= totalWeight + } + current := saturatingAddInt64(member.current, member.pending) + next := saturatingAddInt64(current, delta) + if next < -limit || next > limit { + return false + } + } + return true +} + +func (s *weightedPoolState) charge(selected int, bound bool) { + var totalWeight int64 + for _, member := range s.members { + totalWeight = saturatingAddInt64(totalWeight, member.weight) + } + if !bound || s.boundChargeFits(selected, totalWeight) { + // At the cap, forced traffic cannot erase a small peer's earnings. + // Pending repayments still count until they actually roll back. + for i := range s.members { + member := &s.members[i] + member.current = saturatingAddInt64(member.current, member.weight) + } + member := &s.members[selected] + member.current = saturatingAddInt64(member.current, -totalWeight) + } + if s.inFlight == 0 { + s.boundCredits(totalWeight) + } +} + +// boundCredits runs only once the pool settles. Net pending deltas can be zero +// even with live reservations, so the in-flight count, not len(pending), decides +// when clipping cannot irreversibly discard another reservation's offset. +func (s *weightedPoolState) boundCredits(totalWeight int64) { + limit := int64(math.MaxInt64) + if totalWeight <= math.MaxInt64/maxSmoothWeightedCreditRounds { + limit = totalWeight * maxSmoothWeightedCreditRounds + } + var minCredit, maxCredit int64 = math.MaxInt64, math.MinInt64 + for _, member := range s.members { + minCredit = min(minCredit, member.current) + maxCredit = max(maxCredit, member.current) + } + // A common offset keeps leading credits distinct at the compensation bound. + // Independent clamps can repeatedly tie the leaders and starve the later + // candidate in deterministic selection order. + minOffset := saturatingAddInt64(maxCredit, -limit) + maxOffset := saturatingAddInt64(minCredit, limit) + var offset int64 + if minOffset > 0 || minOffset > maxOffset { + offset = minOffset + } else if maxOffset < 0 { + offset = maxOffset + } + // If the full spread cannot fit, cap only the trailing debt after rebasing + // the leader to the upper bound. Do not flatten leading candidates' credits. + for i := range s.members { + member := &s.members[i] + member.current = max(-limit, saturatingAddInt64(member.current, -offset)) + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_affinity_test.go b/sdk/cliproxy/auth/selector_weighted_affinity_test.go new file mode 100644 index 000000000..5736e0c82 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_affinity_test.go @@ -0,0 +1,436 @@ +package auth + +import ( + "context" + "errors" + "fmt" + "net/http" + "sync" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + cliproxysession "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/session" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator" +) + +func TestWeightedSessionAffinityCompensatesStickyRequests(t *testing.T) { + for _, source := range []string{"sticky", "parent", "guardian", "fork"} { + t.Run(source, func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{ + {ID: "a", Provider: "codex", Attributes: map[string]string{AttributeWeight: "3"}}, + {ID: "b", Provider: "codex"}, + } + parent := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}} + parentID := CanonicalSessionID(parent.Headers, nil, nil) + parentModel := "model" + if source == "guardian" { + parentModel = "parent-model" + } + key := "codex::" + parentID + "::" + parentModel + selector.cache.setSessionAliases("b", "codex", parentID, key) + for i := range 4 { + opts := parent + if source != "sticky" { + opts = cliproxyexecutor.Options{Headers: http.Header{ + "Session_id": {fmt.Sprintf("child-%d", i)}, "X-Codex-Parent-Thread-Id": {"busy"}, + }} + if source == "guardian" { + opts.Headers.Set("X-Openai-Subagent", "guardian") + } + if source == "fork" { + opts.OriginalRequest = []byte(`{"forked_from_thread_id":"busy"}`) + } + } + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths) + if errPick != nil || picked == nil || picked.ID != "b" { + t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick) + } + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + if state.currentForTest()["a"] != 12 || state.currentForTest()["b"] != -12 || len(state.pendingForTest()) != 0 { + t.Fatalf("sticky credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + for i := range 12 { + // Requests without an identity and cold sessions share the same credits. + opts := cliproxyexecutor.Options{} + if i%2 == 0 { + opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}} + } + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths) + if errPick != nil || picked == nil || picked.ID != "a" { + t.Fatalf("compensation %d: pick=%v err=%v", i, picked, errPick) + } + } + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("compensated credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + if binding, ok := selector.cache.Get(key); !ok || binding != "b" { + t.Fatalf("balancing moved the parent binding: %q, found=%t", binding, ok) + } + }) + } +} + +func TestWeightedSessionAffinityChangingPoolsDoNotStarveEqualWeights(t *testing.T) { + for _, newSession := range []bool{false, true} { + t.Run(fmt.Sprintf("new-session=%t", newSession), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{ + {ID: "a", Attributes: map[string]string{"priority": "0"}}, + {ID: "b", Attributes: map[string]string{"priority": "1"}}, + {ID: "c", Attributes: map[string]string{"priority": "1"}}, + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"lower-priority"}}} + id := CanonicalSessionID(sticky.Headers, nil, nil) + key := "codex::" + id + "::model" + selector.cache.Set(key, "a") + counts := map[string]int{} + const warmup, requests = 100, 1000 + for i := range requests { + picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths) + if errPick != nil || picked == nil || picked.ID != "a" { + t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick) + } + opts := cliproxyexecutor.Options{} + if newSession { + opts.Headers = http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}} + } + picked, errPick = selector.Pick(t.Context(), "codex", "model", opts, auths) + if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") { + t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick) + } + if i >= warmup { + counts[picked.ID]++ + } + } + if counts["b"] != (requests-warmup)/2 || counts["c"] != (requests-warmup)/2 { + t.Fatalf("changing pools starved equal-weight candidates after warmup: %v, want b=450 c=450", counts) + } + if binding, ok := selector.cache.Get(key); !ok || binding != "a" { + t.Fatalf("balancing moved the sticky binding: %q, found=%t", binding, ok) + } + if state := weightedStateForTest(t, weighted, "codex:model", "b", "c"); len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) + } + }) + } +} + +func TestWeightedSessionAffinityColdSelectionsCountOnce(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + counts := map[string]int{} + for i := range 40 { + picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{ + Headers: http.Header{"Session_id": {fmt.Sprintf("session-%d", i)}}, + }, auths) + if errPick != nil || picked == nil { + t.Fatalf("pick=%v err=%v", picked, errPick) + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + if i == 0 && (state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1) { + t.Fatalf("first selection charged more than once: %v", state.currentForTest()) + } + counts[picked.ID]++ + } + if counts["a"] != 30 || counts["b"] != 10 { + t.Fatalf("cold distribution = %v, want 30:10", counts) + } +} + +func TestWeightedSessionAffinityBoundsCompensation(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"long"}}} + id := CanonicalSessionID(opts.Headers, nil, nil) + selector.cache.Set("codex::"+id+"::model", "b") + for range 1000 { + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths) + if errPick != nil || picked == nil || picked.ID != "b" { + t.Fatalf("long session: pick=%v err=%v", picked, errPick) + } + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + limit := maxSmoothWeightedCreditRounds * 4 + // A bound charge is admitted as a whole, stopping within one delta of the cap. + wantCredit := limit / 3 * 3 + if state.currentForTest()["a"] != wantCredit || state.currentForTest()["b"] != -wantCredit { + t.Fatalf("bounded credits = %v, want +/- %d", state.currentForTest(), wantCredit) + } + for range wantCredit { + if _, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil { + t.Fatal(errPick) + } + } + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 { + t.Fatalf("compensation did not finish: %v", state.currentForTest()) + } +} + +func TestWeightedSessionAffinityLCPCredits(t *testing.T) { + for _, source := range []string{"hit", "fork", "compaction"} { + t.Run(source, func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + root := cliproxyexecutor.Options{ + SourceFormat: sdktranslator.FormatOpenAI, + OriginalRequest: []byte(`{"messages":[{"role":"user","content":"step 1"},` + + `{"role":"assistant","content":"ack 1"},{"role":"user","content":"step 2"},` + + `{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"}]}`), + Metadata: map[string]any{cliproxyexecutor.CallerScopeMetadataKey: "caller"}, + } + // Seed the matcher without consuming weighted credits. + turns := cliproxysession.ExtractCanonicalTurns(root.SourceFormat, root.OriginalRequest) + selector.matcher.Bind(lcpAffinityNamespace("openai", "model", root.Metadata), turns, "b") + opts := root + if source == "fork" { + opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"step 1"},` + + `{"role":"assistant","content":"ack 1"},{"role":"user","content":"branch"}]}`) + } else if source == "compaction" { + opts.OriginalRequest = []byte(`{"messages":[{"role":"user","content":"Steps 1 and 2 completed"},` + + `{"role":"assistant","content":"ack 2"},{"role":"user","content":"step 3"},` + + `{"role":"user","content":"step 4"}]}`) + } + picked, errPick := selector.Pick(t.Context(), "openai", "model", opts, auths) + if errPick != nil || picked == nil || picked.ID != "b" { + t.Fatalf("LCP %s: pick=%v err=%v", source, picked, errPick) + } + if source == "fork" && opts.Metadata[cliproxyexecutor.IsForkMetadataKey] != true { + t.Fatal("expected LCP fork") + } + if source == "compaction" && opts.Metadata[cliproxyexecutor.IsCompactionMetadataKey] != true { + t.Fatal("expected LCP compaction") + } + state := weightedStateForTest(t, weighted, "openai:model", "a", "b") + if state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 || len(state.pendingForTest()) != 0 { + t.Fatalf("LCP credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + }) + } +} + +func TestWeightedReservationRollbackPreservesConcurrentCharges(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + key := "codex:model" + for range 1000 { + _, reservation := weighted.reserve(key, auths, "b") + reservation.finish(true) + } + state := weightedStateForTest(t, weighted, key, "a", "b") + _, rejected := weighted.reserve(key, auths, "b") + _, accepted := weighted.reserve(key, auths, "a") + accepted.finish(true) + rejected.finish(false) + rejected.finish(true) + accepted.finish(false) + limit := maxSmoothWeightedCreditRounds * 4 / 3 * 3 + if state.currentForTest()["a"] != limit-1 || state.currentForTest()["b"] != -limit+1 || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback lost another charge: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } +} + +func TestWeightedReservationResetInvalidatesOldGeneration(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + key := "codex:model" + _, old := weighted.reserve(key, auths, "") + auths[0].Attributes[AttributeWeight] = "1" + _, latest := weighted.reserve(key, auths, "") + old.finish(true) + state := weightedStateForTest(t, weighted, key, "a", "b") + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || state.pendingForTest()["a"] != -1 || state.pendingForTest()["b"] != 1 { + t.Fatalf("stale reservation changed new generation: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + latest.finish(true) + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("new generation accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } +} + +func TestWeightedReservationConcurrentSelectionsSeePendingCredits(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + const requests = 40 + selected := make(chan *Auth, requests) + reservations := make(chan *weightedReservation, requests) + var ready sync.WaitGroup + ready.Add(requests) + for range requests { + go func() { + defer ready.Done() + picked, reservation := weighted.reserve("codex:model", auths, "") + selected <- picked + reservations <- reservation + }() + } + ready.Wait() + close(selected) + close(reservations) + counts := map[string]int{} + for auth := range selected { + if auth == nil { + t.Fatal("concurrent reservation returned no auth") + } + counts[auth.ID]++ + } + if counts["a"] != 30 || counts["b"] != 10 { + t.Fatalf("pending allocations ignored: %v", counts) + } + for reservation := range reservations { + reservation.finish(true) + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + if state.currentForTest()["a"] != 0 || state.currentForTest()["b"] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } +} + +func TestWeightedSessionAffinityFailureDoesNotCharge(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelectorWithConfig(SessionAffinityConfig{ + Fallback: weighted, RebindOn: SessionAffinityRebindOnNever, + }) + t.Cleanup(selector.Stop) + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound"}}} + id := CanonicalSessionID(opts.Headers, nil, nil) + selector.cache.Set("codex::"+id+"::model", "missing") + if auth, errPick := selector.Pick(t.Context(), "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); errPick == nil || auth != nil { + t.Fatalf("expected unavailable binding: auth=%v err=%v", auth, errPick) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if auth, errPick := selector.Pick(ctx, "codex", "model", opts, []*Auth{{ID: "a"}, {ID: "b"}}); !errors.Is(errPick, context.Canceled) || auth != nil { + t.Fatalf("expected cancellation: auth=%v err=%v", auth, errPick) + } + if len(weighted.states) != 0 { + t.Fatalf("failed selection changed weighted state: %v", weighted.states) + } +} + +func TestWeightedSessionAffinityEligiblePoolAndModelIsolation(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{ + {ID: "a", Attributes: map[string]string{AttributeWeight: "3", "priority": "1"}}, + {ID: "b", Attributes: map[string]string{"priority": "1"}}, + {ID: "lower", Attributes: map[string]string{AttributeWeight: "100"}}, + {ID: "zero", Attributes: map[string]string{AttributeWeight: "0", "priority": "2"}}, + {ID: "cooldown", Attributes: map[string]string{"priority": "2"}, ModelStates: map[string]*ModelState{ + "upstream": {Unavailable: true, NextRetryAfter: time.Now().Add(time.Hour)}, + }}, + } + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"alias"}}} + id := CanonicalSessionID(opts.Headers, nil, nil) + selector.cache.Set("codex::"+id+"::upstream", "b") + ctx := withWeightedSelectorStateModel(t.Context(), weighted, "requested(high)") + picked, errPick := selector.Pick(ctx, "codex", "upstream", opts, auths) + if errPick != nil || picked == nil || picked.ID != "b" { + t.Fatalf("sticky pick=%v err=%v", picked, errPick) + } + state := weightedStateForTest(t, weighted, "codex:requested", "a", "b") + if state == nil || len(weighted.states) != 1 || len(state.currentForTest()) != 2 || state.currentForTest()["a"] != 3 || state.currentForTest()["b"] != -3 { + t.Fatalf("wrong model key or eligible pool: %v", weighted.states) + } + picked, errPick = weighted.Pick(ctx, "codex", "", cliproxyexecutor.Options{}, auths[:2]) + if errPick != nil || picked == nil || picked.ID != "a" || state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { + t.Fatalf("fallback did not share credits: pick=%v err=%v credits=%v", picked, errPick, state.currentForTest()) + } + for _, pair := range [][2]string{{"codex", "other-model"}, {"other-provider", "requested"}} { + if _, errOther := selector.Pick(t.Context(), pair[0], pair[1], cliproxyexecutor.Options{}, auths[:2]); errOther != nil { + t.Fatal(errOther) + } + } + if state.currentForTest()["a"] != 2 || state.currentForTest()["b"] != -2 { + t.Fatal("another provider or model changed credits") + } +} + +func TestWeightedSessionAffinityReselectionCountsOnce(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"reselect"}}} + id := CanonicalSessionID(opts.Headers, nil, nil) + key := "codex::" + id + "::model" + selector.cache.Set(key, "removed") + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, + []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}}) + if errPick != nil || picked == nil || picked.ID != "a" { + t.Fatalf("reselection: pick=%v err=%v", picked, errPick) + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + if state.currentForTest()["a"] != -1 || state.currentForTest()["b"] != 1 || len(state.pendingForTest()) != 0 { + t.Fatalf("reselection counted incorrectly: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + if binding, ok := selector.cache.Get(key); !ok || binding != "a" { + t.Fatalf("reselection did not bind: %q, found=%t", binding, ok) + } +} + +func TestManagerWeightedSessionAffinityCompensation(t *testing.T) { + for _, mixed := range []bool{false, true} { + t.Run(fmt.Sprintf("mixed=%t", mixed), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + manager := NewManager(nil, selector, nil) + provider, model := "weighted-affinity", "weighted-model" + manager.RegisterExecutor(schedulerTestExecutor{provider: provider}) + aID, bID := t.Name()+"-a", t.Name()+"-b" + for _, auth := range []*Auth{ + {ID: aID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "3"}}, + {ID: bID, Provider: provider, Status: StatusActive}, + } { + if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil { + t.Fatal(errRegister) + } + registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) }) + } + namespace := provider + if mixed { + namespace = "mixed" + } + parentHeaders := http.Header{"Session_id": {"manager-sticky"}} + id := CanonicalSessionID(parentHeaders, nil, nil) + selector.cache.Set(namespace+"::"+id+"::"+model, bID) + for i := range 16 { + opts := cliproxyexecutor.Options{Headers: parentHeaders} + want := bID + if i >= 4 { + opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}} + want = aID + } + var auth *Auth + var errPick error + if mixed { + auth, _, _, errPick = manager.pickNextMixed(t.Context(), []string{provider}, model, opts, nil) + } else { + auth, _, errPick = manager.pickNext(t.Context(), provider, model, opts, nil) + } + if errPick != nil || auth == nil || auth.ID != want { + t.Fatalf("request %d: auth=%v err=%v, want %q", i, auth, errPick, want) + } + } + state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID) + if state == nil || state.currentForTest()[aID] != 0 || state.currentForTest()[bID] != 0 || len(state.pendingForTest()) != 0 { + t.Fatalf("manager credits did not balance: %#v", state) + } + }) + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_commit_order_test.go b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go new file mode 100644 index 000000000..58d18499a --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_commit_order_test.go @@ -0,0 +1,199 @@ +package auth + +import ( + "fmt" + "maps" + "testing" + + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +func TestWeightedReservationFinalRollbackBoundsSettledCredits(t *testing.T) { + for _, rollbackFirst := range []bool{false, true} { + t.Run(fmt.Sprintf("rollback-first=%t", rollbackFirst), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a"}, {ID: "b"}} + reservations := map[string][]*weightedReservation{} + t.Cleanup(func() { + for _, group := range reservations { + for _, reservation := range group { + reservation.finish(false) + } + } + }) + for range 256 { + picked, reservation := weighted.reserve("codex:model", auths, "") + if picked == nil || reservation == nil { + t.Fatal("missing cold reservation") + } + reservations[picked.ID] = append(reservations[picked.ID], reservation) + } + ids := []string{"a", "b"} + if rollbackFirst { + ids = []string{"b", "a"} + } + for _, id := range ids { + for _, reservation := range reservations[id] { + reservation.finish(id == "a") + // Repeated finishing must not decrement the counter again. + reservation.finish(false) + } + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + limit := maxSmoothWeightedCreditRounds * 2 + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -limit, "b": limit}) { + t.Fatalf("settled credit did not reach its bound: current=%v pending=%v in-flight=%d", state.currentForTest(), state.pendingForTest(), state.inFlight) + } + }) + } +} + +func TestWeightedReservationUnequalWeightsOutOfOrderCommits(t *testing.T) { + for _, first := range []string{"a", "b"} { + t.Run(first, func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + reservations := map[string][]*weightedReservation{} + t.Cleanup(func() { + for _, group := range reservations { + for _, reservation := range group { + reservation.finish(false) + } + } + }) + for range 512 { + picked, reservation := weighted.reserve("codex:model", auths, "") + if picked == nil || reservation == nil { + t.Fatal("missing cold reservation") + } + reservations[picked.ID] = append(reservations[picked.ID], reservation) + } + if len(reservations["a"]) != 384 || len(reservations["b"]) != 128 { + t.Fatalf("reservations did not follow 3:1: a=%d b=%d", len(reservations["a"]), len(reservations["b"])) + } + last := "b" + if first == "b" { + last = "a" + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + for _, reservation := range reservations[first] { + reservation.finish(true) + } + // New arrivals must see the remaining offsets before the batch settles. + for _, phase := range []string{"pending", "settled"} { + if phase == "settled" { + for _, reservation := range reservations[last] { + reservation.finish(true) + } + } + counts := map[string]int{} + for range 64 { + picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths) + if errPick != nil || picked == nil { + t.Fatalf("Pick failed: auth=%v err=%v", picked, errPick) + } + counts[picked.ID]++ + } + if counts["a"] != 48 || counts["b"] != 16 { + t.Fatalf("%s distribution = %v, want 48:16", phase, counts) + } + } + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) { + t.Fatalf("settled 3:1 batch drifted: %#v", state) + } + }) + } +} + +func TestWeightedReservationResetDoesNotReleaseNewGeneration(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a"}, {ID: "b"}} + var old []*weightedReservation + t.Cleanup(func() { + for _, reservation := range old { + reservation.finish(false) + } + }) + for range 256 { + _, reservation := weighted.reserve("codex:model", auths, "") + old = append(old, reservation) + } + auths[0].Attributes = map[string]string{AttributeWeight: "3"} + _, latest := weighted.reserve("codex:model", auths, "") + if latest == nil { + t.Fatal("missing new-generation reservation") + } + t.Cleanup(func() { latest.finish(false) }) + for _, reservation := range old { + reservation.finish(true) + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + if state.inFlight != 1 || state.hasCurrentForTest() || !maps.Equal(state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { + t.Fatalf("old generation altered new accounting: %#v", state) + } + latest.finish(true) + if state.inFlight != 0 || len(state.pendingForTest()) != 0 || !maps.Equal(state.currentForTest(), map[string]int64{"a": -1, "b": 1}) { + t.Fatalf("new generation failed to settle: %#v", state) + } +} + +func TestWeightedReservationOutOfOrderBalancedCommits(t *testing.T) { + for _, bound := range []bool{false, true} { + for _, first := range []string{"a", "b"} { + t.Run(fmt.Sprintf("bound=%t/first=%s", bound, first), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a"}, {ID: "b"}} + reservations := map[string][]*weightedReservation{} + t.Cleanup(func() { + for _, group := range reservations { + for _, reservation := range group { + reservation.finish(false) + } + } + }) + for i := range 256 { + boundID := "" + if bound { + boundID = auths[i%2].ID + } + picked, reservation := weighted.reserve("codex:model", auths, boundID) + if picked == nil || reservation == nil { + t.Fatalf("reserve %d returned pick=%v reservation=%v", i, picked, reservation) + } + reservations[picked.ID] = append(reservations[picked.ID], reservation) + } + if len(reservations["a"]) != 128 || len(reservations["b"]) != 128 { + t.Fatalf("unbalanced reservations: a=%d b=%d", len(reservations["a"]), len(reservations["b"])) + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + // A zero net pending vector does not mean all reservations settled. + if state.inFlight != 256 || len(state.pendingForTest()) != 0 { + t.Fatalf("in-flight=%d pending=%v, want 256 live balanced reservations", state.inFlight, state.pendingForTest()) + } + last := "b" + if first == "b" { + last = "a" + } + for _, id := range []string{first, last} { + for _, reservation := range reservations[id] { + reservation.finish(true) + } + } + if !maps.Equal(state.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(state.pendingForTest()) != 0 || state.inFlight != 0 { + t.Errorf("commit order distorted balanced credit: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } + counts := map[string]int{} + for range 64 { + picked, errPick := weighted.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths) + if errPick != nil || picked == nil { + t.Fatalf("Pick: auth=%v err=%v", picked, errPick) + } + counts[picked.ID]++ + } + if counts["a"] != 32 || counts["b"] != 32 { + t.Fatalf("post-commit distribution = %v, want a=32 b=32", counts) + } + }) + } + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_memory_test.go b/sdk/cliproxy/auth/selector_weighted_memory_test.go new file mode 100644 index 000000000..eec967e28 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_memory_test.go @@ -0,0 +1,71 @@ +package auth + +import ( + "fmt" + "runtime" + "testing" +) + +func BenchmarkWeightedPoolReserve1000(b *testing.B) { + for _, bound := range []bool{false, true} { + b.Run(fmt.Sprintf("bound=%t", bound), func(b *testing.B) { + auths := make([]*Auth, 1000) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + boundID := "" + if bound { + boundID = auths[i%len(auths)].ID + } + _, reservation := selector.reserve("codex:model", auths, boundID) + reservation.finish(true) + } + }) + } +} + +// BenchmarkWeightedPoolRetainedMemory measures live cache growth separately from +// total heap and allocation churn. Each retry subset omits a different credential. +func BenchmarkWeightedPoolRetainedMemory(b *testing.B) { + const members = 1000 + auths := make([]*Auth, members) + for i := range auths { + auths[i] = &Auth{ID: fmt.Sprintf("%036d", i)} + } + subset := make([]*Auth, 0, members-1) + var retained, heap, allocated int64 + b.ResetTimer() + for range b.N { + selector := &WeightedRoundRobinSelector{} + _, warmup := selector.reserve("codex:model", auths, "") + warmup.finish(true) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for excluded := range auths { + subset = append(subset[:0], auths[:excluded]...) + subset = append(subset, auths[excluded+1:]...) + _, reservation := selector.reserve("codex:model", subset, "") + reservation.finish(true) + } + runtime.GC() + runtime.ReadMemStats(&after) + if len(selector.states) != members+1 { + b.Fatalf("cached pools = %d, want %d", len(selector.states), members+1) + } + runtime.KeepAlive(selector) + runtime.KeepAlive(auths) + retained += int64(after.HeapAlloc) - int64(before.HeapAlloc) + heap += int64(after.HeapAlloc) + allocated += int64(after.TotalAlloc - before.TotalAlloc) + } + b.ReportMetric(float64(retained)/float64(b.N), "retained-B/op") + b.ReportMetric(float64(heap)/float64(b.N), "heap-total-B/op") + b.ReportMetric(float64(allocated)/float64(b.N), "alloc-total-B/op") +} diff --git a/sdk/cliproxy/auth/selector_weighted_model_key_test.go b/sdk/cliproxy/auth/selector_weighted_model_key_test.go new file mode 100644 index 000000000..2f988316b --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_model_key_test.go @@ -0,0 +1,39 @@ +package auth + +import ( + "fmt" + "testing" + + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +func TestWeightedSelectorsShareEquivalentModelKeys(t *testing.T) { + for _, firstAffinity := range []bool{false, true} { + for _, secondAffinity := range []bool{false, true} { + for _, routeOverride := range []bool{false, true} { + t.Run(fmt.Sprintf("affinity=%t/%t/route=%t", firstAffinity, secondAffinity, routeOverride), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + affinity := NewSessionAffinitySelector(weighted) + t.Cleanup(affinity.Stop) + auths := []*Auth{{ID: "a"}, {ID: "b"}} + for i, useAffinity := range []bool{firstAffinity, secondAffinity} { + ctx := t.Context() + model := []string{" Model(high) ", "model(low)"}[i] + if routeOverride { + ctx = withWeightedSelectorStateModel(ctx, weighted, model) + model = []string{"upstream-a", "upstream-b"}[i] + } + var selector Selector = weighted + if useAffinity { + selector = affinity + } + picked, errPick := selector.Pick(ctx, "codex", model, cliproxyexecutor.Options{}, auths) + if errPick != nil || picked == nil || picked.ID != auths[i].ID { + t.Fatalf("request %d: pick=%v err=%v, want auth %s", i, picked, errPick, auths[i].ID) + } + } + }) + } + } + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_overload_test.go b/sdk/cliproxy/auth/selector_weighted_overload_test.go new file mode 100644 index 000000000..f7639a536 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_overload_test.go @@ -0,0 +1,455 @@ +package auth + +import ( + "fmt" + "maps" + "net/http" + "strconv" + "sync" + "testing" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +func TestWeightedOverloadExactCompensation(t *testing.T) { + for _, weights := range [][]int{{1, 1}, {3, 1}, {1, 3}, {1, 5, 1}, {2, 3, 5}, {1, 2, 3, 4}} { + for boundIndex := range weights { + t.Run(fmt.Sprintf("weights=%v/bound=%d", weights, boundIndex), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := make([]*Auth, len(weights)) + total := 0 + for i, weight := range weights { + auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}} + total += weight + } + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"sticky"}}} + key := "codex::" + CanonicalSessionID(opts.Headers, nil, nil) + "::model" + selector.cache.Set(key, auths[boundIndex].ID) + counts := map[string]int{} + for i := range total * 4 { + request := opts + sticky := i < weights[boundIndex]*4 + if !sticky { + request = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}} + } + picked, errPick := selector.Pick(t.Context(), "codex", "model", request, auths) + if errPick != nil || picked == nil { + t.Fatalf("selection failed: pick=%v err=%v", picked, errPick) + } + if sticky && picked.ID != auths[boundIndex].ID { + t.Fatal("balancing moved an existing binding") + } + counts[picked.ID]++ + } + for i, weight := range weights { + if counts[auths[i].ID] != weight*4 { + t.Errorf("total allocation = %v, want weight vector %v times 4", counts, weights) + } + } + for _, state := range weighted.states { + for id, credit := range state.currentForTest() { + if credit != 0 { + t.Errorf("exact compensation left %q credit=%d", id, credit) + } + } + if len(state.pendingForTest()) != 0 { + t.Errorf("pending credit leaked: %v", state.pendingForTest()) + } + } + }) + } + } +} + +func TestWeightedOverloadSustainedStickyTrafficDoesNotStarveOthers(t *testing.T) { + for _, weights := range [][]int{{1, 1, 1}, {1, 3, 1}, {1, 5, 1}, {1, 1, 5}, {1, 9, 1}, {1, 1, 9}, {1, 5, 2, 1}} { + t.Run(fmt.Sprint(weights), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := make([]*Auth, len(weights)) + for i, weight := range weights { + auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}} + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}} + key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model" + selector.cache.Set(key, auths[0].ID) + counts := map[string]int{} + for i := range 3000 { + picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths) + if errPick != nil || picked == nil || picked.ID != auths[0].ID { + t.Fatalf("binding changed: pick=%v err=%v", picked, errPick) + } + fresh := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}} + picked, errPick = selector.Pick(t.Context(), "codex", "model", fresh, auths) + if errPick != nil || picked == nil { + t.Fatalf("fresh selection failed: pick=%v err=%v", picked, errPick) + } + if i >= 1000 { + counts[picked.ID]++ + } + } + ids := make([]string, len(auths)) + for i, auth := range auths { + ids[i] = auth.ID + } + state := weightedStateForTest(t, weighted, "codex:model", ids...) + t.Logf("fresh distribution after warmup: %v; committed=%v", counts, state.currentForTest()) + // Forced traffic makes global ratios infeasible; the remaining capacity + // should be shared by unbound peers rather than monopolized by one. + peerWeight := 0 + for _, weight := range weights[1:] { + peerWeight += weight + } + for i, auth := range auths[1:] { + want := 2000 * weights[i+1] / peerWeight + if counts[auth.ID] < want-2 || counts[auth.ID] > want+2 { + t.Errorf("peer %q got %d fresh requests, want approximately %d; distribution=%v", auth.ID, counts[auth.ID], want, counts) + } + } + }) + } +} + +func TestWeightedOverloadFeasibleSustainedTraffic(t *testing.T) { + for _, weights := range [][]int{{3, 1}, {1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} { + t.Run(fmt.Sprint(weights), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := make([]*Auth, len(weights)) + total := 0 + for i, weight := range weights { + auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}} + total += weight + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"feasible"}}} + key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model" + selector.cache.Set(key, auths[0].ID) + counts := map[string]int{} + // Forced traffic is below A's target share, so exact aggregate ratios + // remain feasible without moving established bindings. + for i := range total * 2000 { + opts := cliproxyexecutor.Options{} + bound := i%(total*2) == 0 + if bound { + opts = sticky + } + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths) + if errPick != nil || picked == nil || (bound && picked.ID != auths[0].ID) { + t.Fatalf("request %d: pick=%v err=%v", i, picked, errPick) + } + if i >= total*1000 { + counts[picked.ID]++ + } + } + for i, weight := range weights { + if counts[auths[i].ID] != weight*1000 { + t.Errorf("feasible distribution drifted: counts=%v weights=%v", counts, weights) + } + } + t.Logf("feasible distribution: %v", counts) + }) + } +} + +func TestWeightedOverloadRecoveryAfterStickyTrafficStops(t *testing.T) { + for _, weights := range [][]int{{1, 5, 1}, {1, 9, 1}, {1, 5, 2, 1}} { + t.Run(fmt.Sprint(weights), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := make([]*Auth, len(weights)) + total := 0 + for i, weight := range weights { + auths[i] = &Auth{ID: fmt.Sprintf("auth-%d", i), Attributes: map[string]string{AttributeWeight: strconv.Itoa(weight)}} + total += weight + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}} + key := "codex::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::model" + selector.cache.Set(key, auths[0].ID) + for range 3000 { + if picked, errPick := selector.Pick(t.Context(), "codex", "model", sticky, auths); errPick != nil || picked == nil || picked.ID != auths[0].ID { + t.Fatalf("sticky selection failed: pick=%v err=%v", picked, errPick) + } + if _, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths); errPick != nil { + t.Fatal(errPick) + } + } + counts := map[string]int{} + for i := range 1000 + total*100 { + picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths) + if errPick != nil || picked == nil { + t.Fatalf("recovery selection failed: pick=%v err=%v", picked, errPick) + } + if i >= 1000 { + counts[picked.ID]++ + } + } + for i, weight := range weights { + if counts[auths[i].ID] != weight*100 { + t.Errorf("allocation did not recover after sticky traffic stopped: counts=%v weights=%v", counts, weights) + } + } + t.Logf("recovered distribution: %v", counts) + }) + } +} + +func TestWeightedOverloadPendingStickyReservationsDoNotStarveOthers(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}} + const key = "codex:model" + const inFlight = 200 + reservations := make([]*weightedReservation, inFlight) + t.Cleanup(func() { + for _, reservation := range reservations { + reservation.finish(false) + } + }) + for i := range reservations { + picked, reservation := weighted.reserve(key, auths, "a") + if picked == nil || picked.ID != "a" { + t.Fatal("pending sticky request lost its binding") + } + reservations[i] = reservation + } + counts := map[string]int{} + for i := range 3000 { + // Maintain a fixed backlog without sleeps or timing-dependent coordination. + reservations[i%inFlight].finish(true) + _, reservations[i%inFlight] = weighted.reserve(key, auths, "a") + picked, reservation := weighted.reserve(key, auths, "") + reservation.finish(true) + if i >= 1000 { + counts[picked.ID]++ + } + } + if counts["c"] == 0 { + t.Fatalf("speculative sticky traffic starved c: %v", counts) + } + for _, reservation := range reservations { + reservation.finish(true) + } + state := weightedStateForTest(t, weighted, key, "a", "b", "c") + if len(state.pendingForTest()) != 0 { + t.Fatalf("settled reservations leaked credit: %v", state.pendingForTest()) + } +} + +func TestWeightedOverloadCommitAfterSpeculativeRepaymentRollsBack(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + const key = "codex:model" + for range 1000 { + _, reservation := weighted.reserve(key, auths, "b") + reservation.finish(true) + } + state := weightedStateForTest(t, weighted, key, "a", "b") + before := maps.Clone(state.currentForTest()) + _, first := weighted.reserve(key, auths, "") + _, second := weighted.reserve(key, auths, "") + t.Cleanup(func() { first.finish(false); second.finish(false) }) + // Two pending cold selections make room for a bound delta, but neither + // repayment is real yet. Its later commit must recheck the committed cap. + picked, bound := weighted.reserve(key, auths, "b") + if picked == nil || picked.ID != "b" || bound == nil { + t.Fatal("expected bound reservation funded by pending repayments") + } + t.Cleanup(func() { bound.finish(false) }) + first.finish(false) + second.finish(false) + bound.finish(true) + bound.finish(true) + bound.finish(false) + if !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { + t.Fatalf("late commit crossed the cap or leaked pending credit: before=%v current=%v pending=%v", before, state.currentForTest(), state.pendingForTest()) + } + // An over-limit bound selection succeeds without reserving any credit. + picked, skipped := weighted.reserve(key, auths, "b") + if picked == nil || picked.ID != "b" || skipped != nil || !maps.Equal(state.currentForTest(), before) || len(state.pendingForTest()) != 0 { + t.Fatal("saturated bound selection should preserve binding and all credits") + } +} + +func TestWeightedOverloadMultipleBusyBindings(t *testing.T) { + for _, busyPerFresh := range []int{1, 3, 10} { + t.Run(fmt.Sprint(busyPerFresh), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{ + {ID: "a"}, {ID: "b", Attributes: map[string]string{AttributeWeight: "5"}}, {ID: "c"}, {ID: "d"}, + } + boundOpts := make([]cliproxyexecutor.Options, 2) + for i, id := range []string{"a", "d"} { + boundOpts[i].Headers = http.Header{"Session_id": {"busy-" + id}} + sessionID := CanonicalSessionID(boundOpts[i].Headers, nil, nil) + selector.cache.Set("codex::"+sessionID+"::model", id) + } + windowCounts := map[string]int{} + for i := range 1400 { + for j := range busyPerFresh { + index := (i + j) % len(boundOpts) + picked, errPick := selector.Pick(t.Context(), "codex", "model", boundOpts[index], auths) + if errPick != nil || picked == nil || picked.ID != []string{"a", "d"}[index] { + t.Fatalf("busy binding changed: auth=%v err=%v", picked, errPick) + } + } + picked, errPick := selector.Pick(t.Context(), "codex", "model", cliproxyexecutor.Options{}, auths) + if errPick != nil || picked == nil { + t.Fatalf("fresh pick failed: auth=%v err=%v", picked, errPick) + } + if i >= 1000 { + windowCounts[picked.ID]++ + if (i+1)%100 == 0 { + if windowCounts["b"] == 0 || windowCounts["c"] == 0 { + t.Fatalf("busy=%d unbound peer starved in a 100-request window: %v", busyPerFresh, windowCounts) + } + clear(windowCounts) + } + } + } + }) + } +} + +func TestWeightedOverloadSaturatedChargeLeavesPeersUnchanged(t *testing.T) { + for _, credit := range []map[string]int64{ + {"a": -222, "b": 111, "c": 111}, + {"a": -112, "b": 222, "c": -110}, + } { + state := weightedPoolForTest("route", map[string]int64{"a": 1, "b": 5, "c": 1}, credit) + state.charge(0, true) + if !maps.Equal(state.currentForTest(), credit) { + t.Fatalf("saturated sticky charge changed peers: got=%v want=%v", state.currentForTest(), credit) + } + } +} + +func TestManagerWeightedOverloadStickyAndPinnedSessions(t *testing.T) { + for _, mixed := range []bool{false, true} { + for _, pinned := range []bool{false, true} { + t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + manager := NewManager(nil, selector, nil) + const provider, model = "sticky-overload", "sticky-overload-model" + manager.RegisterExecutor(schedulerTestExecutor{provider: provider}) + aID, bID, cID := t.Name()+"-a", t.Name()+"-b", t.Name()+"-c" + for _, auth := range []*Auth{ + {ID: aID, Provider: provider, Status: StatusActive}, + {ID: bID, Provider: provider, Status: StatusActive, Attributes: map[string]string{AttributeWeight: "5"}}, + {ID: cID, Provider: provider, Status: StatusActive}, + } { + registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) }) + if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil { + t.Fatal(errRegister) + } + } + namespace := provider + if mixed { + namespace = "mixed" + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"busy"}}, Metadata: map[string]any{}} + key := namespace + "::" + CanonicalSessionID(sticky.Headers, nil, nil) + "::" + model + selector.cache.Set(key, aID) + if pinned { + sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = aID + } + counts := map[string]int{} + for i := range 1600 { + for _, bound := range []bool{true, false} { + opts := sticky + if !bound { + opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}}} + } + var auth *Auth + var errPick error + if mixed { + auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil) + } else { + auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil) + } + if errPick != nil || auth == nil || (bound && auth.ID != aID) { + t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick) + } + if !bound && i >= 1000 { + counts[auth.ID]++ + } + } + } + if counts[bID] != 500 || counts[cID] != 100 { + t.Fatalf("manager overload allocation = %v, want b=500 c=100", counts) + } + if binding, ok := selector.cache.Get(key); !ok || binding != aID { + t.Fatalf("manager moved busy binding: %q, found=%t", binding, ok) + } + state := weightedStateForTest(t, weighted, namespace+":"+model, aID, bID, cID) + if len(state.pendingForTest()) != 0 { + t.Fatalf("manager left speculative credit: %v", state.pendingForTest()) + } + }) + } + } +} + +func TestWeightedOverloadConcurrentStickyAccounting(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}} + headers := http.Header{"Session_id": {"sticky-concurrent"}} + key := "codex::" + CanonicalSessionID(headers, nil, nil) + "::model" + selector.cache.Set(key, "b") + type selection struct { + id string + err error + sticky bool + } + results := make(chan selection, 64) + start := make(chan struct{}) + var done sync.WaitGroup + for i := range 64 { + done.Add(1) + go func() { + defer done.Done() + isSticky := i < 8 + opts := cliproxyexecutor.Options{Headers: headers} + if !isSticky { + opts.Headers = http.Header{"Session_id": {fmt.Sprintf("new-%d", i)}} + } + <-start + picked, errPick := selector.Pick(t.Context(), "codex", "model", opts, auths) + id := "" + if picked != nil { + id = picked.ID + } + results <- selection{id: id, err: errPick, sticky: isSticky} + }() + } + close(start) + done.Wait() + close(results) + counts := map[string]int{} + for result := range results { + if result.err != nil || result.id == "" || (result.sticky && result.id != "b") { + t.Fatalf("invalid concurrent result: %+v", result) + } + counts[result.id]++ + } + state := weightedStateForTest(t, weighted, "codex:model", "a", "b") + // All 64 calls share one pool. Accounting must match actual accepted picks, + // regardless of whether the final arrivals happened to be sticky requests. + if state.currentForTest()["a"] != int64(64*3-counts["a"]*4) || state.currentForTest()["b"] != int64(64-counts["b"]*4) || len(state.pendingForTest()) != 0 { + t.Fatalf("concurrent accounting drifted: counts=%v committed=%v pending=%v", counts, state.currentForTest(), state.pendingForTest()) + } + if bound, ok := selector.cache.Get(key); !ok || bound != "b" { + t.Fatalf("concurrency moved sticky binding: %q, found=%t", bound, ok) + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_pool_test.go b/sdk/cliproxy/auth/selector_weighted_pool_test.go new file mode 100644 index 000000000..5fb73ffa9 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_pool_test.go @@ -0,0 +1,287 @@ +package auth + +import ( + "fmt" + "maps" + "math/rand/v2" + "net/http" + "strconv" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +// weightedStateForTest checks that the requested accounting pool was created. +func weightedStateForTest(t *testing.T, selector *WeightedRoundRobinSelector, route string, ids ...string) *weightedPoolState { + t.Helper() + weights := make(map[string]int64, len(ids)) + for _, id := range ids { + weights[id] = 1 + } + state := selector.states[weightedPoolStateKey(route, weights)] + ids = sortedWeightedIDs(weights) + for state != nil && !state.matches(route, ids) { + state = state.next + } + if state == nil { + t.Fatalf("missing weighted state for %q and %v", route, ids) + } + return state +} + +func TestWeightedPoolsResumeCreditsAndIsolatePendingReservations(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + auths := []*Auth{{ID: "a", Attributes: map[string]string{AttributeWeight: "3"}}, {ID: "b"}, {ID: "c"}} + key := "codex:model" + _, pendingFull := weighted.reserve(key, auths, "b") + t.Cleanup(func() { pendingFull.finish(false) }) + _, pendingSubset := weighted.reserve(key, auths[:2], "b") + t.Cleanup(func() { pendingSubset.finish(false) }) + pendingSubset.finish(true) + subset := weightedStateForTest(t, weighted, key, "a", "b") + full := weightedStateForTest(t, weighted, key, "a", "b", "c") + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 3, "b": -3}) || full.hasCurrentForTest() { + t.Fatalf("commit crossed pool boundary: subset=%v full=%v", subset.currentForTest(), full.currentForTest()) + } + if !maps.Equal(full.pendingForTest(), map[string]int64{"a": 3, "b": -4, "c": 1}) { + t.Fatalf("another pool consumed pending credit: %v", full.pendingForTest()) + } + pendingFull.finish(false) + for range 100 { + _, reservation := weighted.reserve(key, auths, "b") + reservation.finish(true) + } + // A returning pool resumes its debt, independent of full-pool traffic. + for range 3 { + picked, reservation := weighted.reserve(key, []*Auth{auths[1], auths[0]}, "") + if picked == nil || picked.ID != "a" { + t.Fatalf("returning subset lost credit: pick=%v credits=%v", picked, subset.currentForTest()) + } + reservation.finish(true) + } + if !maps.Equal(subset.currentForTest(), map[string]int64{"a": 0, "b": 0}) || len(subset.pendingForTest()) != 0 || len(full.pendingForTest()) != 0 { + t.Fatalf("returning subset failed to balance: %#v", subset) + } +} + +func TestWeightedPoolInterleavingMatchesIndependentSelectors(t *testing.T) { + shared := &WeightedRoundRobinSelector{} + auths := []*Auth{ + {ID: "a", Attributes: map[string]string{AttributeWeight: "1"}}, + {ID: "b", Attributes: map[string]string{AttributeWeight: "3"}}, + {ID: "c", Attributes: map[string]string{AttributeWeight: "2"}}, + {ID: "d", Attributes: map[string]string{AttributeWeight: "5"}}, + } + var independent [16]WeightedRoundRobinSelector + rng := rand.New(rand.NewPCG(17, 29)) + for step := range 5000 { + mask := 1 + rng.IntN(15) + var pool []*Auth + for i, auth := range auths { + if mask&(1< weighted.maxKeys || old.state == current.state { + t.Fatal("pool eviction failed to bound or replace state") + } + old.finish(true) + if current.state.hasCurrentForTest() || !maps.Equal(current.state.pendingForTest(), map[string]int64{"a": -1, "b": 1}) { + t.Fatal("finishing an evicted reservation affected the new pool") + } + current.finish(true) +} + +func TestManagerWeightedPriorityPoolIsolation(t *testing.T) { + for _, mixed := range []bool{false, true} { + for _, pinned := range []bool{false, true} { + t.Run(fmt.Sprintf("mixed=%t/pinned=%t", mixed, pinned), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + manager := NewManager(nil, selector, nil) + const provider, model = "pool-isolation", "pool-model" + manager.RegisterExecutor(schedulerTestExecutor{provider: provider}) + auths := []*Auth{ + {ID: t.Name() + "-a", Attributes: map[string]string{"priority": "0"}}, + {ID: t.Name() + "-b", Attributes: map[string]string{"priority": "1", AttributeWeight: "3"}}, + {ID: t.Name() + "-c", Attributes: map[string]string{"priority": "1", AttributeWeight: "1"}}, + } + for _, auth := range auths { + auth.Provider, auth.Status = provider, StatusActive + registry.GetGlobalRegistry().RegisterClient(auth.ID, provider, []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(auth.ID) }) + if _, errRegister := manager.Register(WithSkipPersist(t.Context()), auth); errRegister != nil { + t.Fatal(errRegister) + } + } + namespace := provider + if mixed { + namespace = "mixed" + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"priority-bound"}}, Metadata: map[string]any{}} + id := CanonicalSessionID(sticky.Headers, nil, nil) + selector.cache.Set(namespace+"::"+id+"::"+model, auths[0].ID) + if pinned { + sticky.Metadata[cliproxyexecutor.PinnedAuthMetadataKey] = auths[0].ID + } + counts := map[string]int{} + for i := range 1200 { + for _, bound := range []bool{true, false} { + opts := sticky + if !bound { + opts = cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}} + } + var auth *Auth + var errPick error + if mixed { + auth, _, _, errPick = manager.pickNextMixedLegacy(t.Context(), []string{provider}, model, opts, nil) + } else { + auth, _, errPick = manager.pickNextLegacy(t.Context(), provider, model, opts, nil) + } + if errPick != nil || auth == nil || (bound && auth.ID != auths[0].ID) || (!bound && auth.ID == auths[0].ID) { + t.Fatalf("request %d bound=%t: auth=%v err=%v", i, bound, auth, errPick) + } + if !bound && i >= 200 { + counts[auth.ID]++ + } + } + } + if counts[auths[1].ID] != 750 || counts[auths[2].ID] != 250 { + t.Fatalf("manager cross-priority distribution = %v, want 750:250", counts) + } + }) + } + } +} + +func TestWeightedSessionAffinityPoolIsolation(t *testing.T) { + for _, reason := range []string{"priority", "websocket", "retry-exclusion", "cooldown"} { + for _, weights := range [][2]int{{1, 1}, {3, 1}, {1, 3}, {5, 2}} { + t.Run(fmt.Sprintf("%s/%d:%d", reason, weights[0], weights[1]), func(t *testing.T) { + weighted := &WeightedRoundRobinSelector{} + selector := NewSessionAffinitySelector(weighted) + t.Cleanup(selector.Stop) + auths := []*Auth{ + {ID: "a", Attributes: map[string]string{"priority": "0"}}, + {ID: "b", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[0])}}, + {ID: "c", Attributes: map[string]string{AttributeWeight: strconv.Itoa(weights[1])}}, + } + ctx := t.Context() + switch reason { + case "priority": + auths[1].Attributes["priority"] = "1" + auths[2].Attributes["priority"] = "1" + case "websocket": + ctx = cliproxyexecutor.WithDownstreamWebsocket(ctx) + auths[1].Attributes["websockets"] = "true" + auths[2].Attributes["websockets"] = "true" + } + sticky := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {"bound-a"}}} + id := CanonicalSessionID(sticky.Headers, nil, nil) + key := "codex::" + id + "::model" + selector.cache.Set(key, "a") + const warmup, measured = 1000, 1400 + counts := map[string]int{} + for i := range warmup + measured { + picked, errPick := selector.Pick(ctx, "codex", "model", sticky, auths) + if errPick != nil || picked == nil || picked.ID != "a" { + t.Fatalf("sticky request %d: pick=%v err=%v", i, picked, errPick) + } + freshAuths := auths + if reason == "retry-exclusion" { + freshAuths = auths[1:] + } else if reason == "cooldown" { + auths[0].Unavailable = true + auths[0].NextRetryAfter = time.Now().Add(time.Hour) + } + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": {fmt.Sprintf("fresh-%d", i)}}} + picked, errPick = selector.Pick(ctx, "codex", "model", opts, freshAuths) + if errPick != nil || picked == nil || (picked.ID != "b" && picked.ID != "c") { + t.Fatalf("fresh request %d: pick=%v err=%v", i, picked, errPick) + } + if i >= warmup { + counts[picked.ID]++ + } + auths[0].Unavailable = false + auths[0].NextRetryAfter = time.Time{} + } + wantB := measured * weights[0] / (weights[0] + weights[1]) + if counts["b"] != wantB || counts["c"] != measured-wantB { + t.Fatalf("cross-pool traffic distorted cold selections: %v, want b=%d c=%d", counts, wantB, measured-wantB) + } + if bound, ok := selector.cache.Get(key); !ok || bound != "a" { + t.Fatalf("binding moved: %q, found=%t", bound, ok) + } + for _, state := range weighted.states { + if len(state.pendingForTest()) != 0 { + t.Fatalf("pending credits leaked: %v", state.pendingForTest()) + } + } + }) + } + } +} diff --git a/sdk/cliproxy/auth/selector_weighted_storage_test.go b/sdk/cliproxy/auth/selector_weighted_storage_test.go new file mode 100644 index 000000000..0d937b903 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_storage_test.go @@ -0,0 +1,88 @@ +package auth + +import ( + "maps" + "testing" +) + +// These snapshots keep assertions independent of the compact production layout. +func (s *weightedPoolState) currentForTest() map[string]int64 { + current := make(map[string]int64, len(s.members)) + for _, member := range s.members { + current[member.id] = member.current + } + return current +} + +func (s *weightedPoolState) pendingForTest() map[string]int64 { + pending := make(map[string]int64) + for _, member := range s.members { + if member.pending != 0 { + pending[member.id] = member.pending + } + } + return pending +} + +func (s *weightedPoolState) hasCurrentForTest() bool { + for _, member := range s.members { + if member.current != 0 { + return true + } + } + return false +} + +// TestWeightedPoolDigestCollisionIsolation forces digest hits for different exact +// identities, including a matching state behind a collision-chain head. +func TestWeightedPoolDigestCollisionIsolation(t *testing.T) { + for _, unrelated := range []*weightedPoolState{ + weightedPoolForTest("other-route", map[string]int64{"a": 1, "b": 1}, map[string]int64{"a": 7, "b": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1, "c": 1}, map[string]int64{"a": 7, "c": -7}), + weightedPoolForTest("route", map[string]int64{"a": 1}, map[string]int64{"a": 7}), + } { + t.Run(unrelated.route+"/"+unrelated.members[len(unrelated.members)-1].id, func(t *testing.T) { + weights := map[string]int64{"a": 1, "b": 1} + key := weightedPoolStateKey("route", weights) + selector := &WeightedRoundRobinSelector{ + states: map[[32]byte]*weightedPoolState{key: unrelated}, stateCount: 1, maxKeys: 2, + } + before := unrelated.currentForTest() + picked, first := selector.reserve("route", []*Auth{{ID: "b"}, {ID: "a"}}, "b") + if picked == nil || picked.ID != "b" || first == nil || first.state == unrelated { + t.Fatal("digest collision reused an unrelated pool") + } + t.Cleanup(func() { first.finish(false) }) + matching := first.state + // Put the nonmatching state first to exercise a full chain lookup. + matching.next, unrelated.next, selector.states[key] = nil, matching, unrelated + _, second := selector.reserve("route", []*Auth{{ID: "a"}, {ID: "b"}}, "a") + if second == nil || second.state != matching || selector.stateCount != 2 { + t.Fatal("collision lookup failed to reuse the matching pool") + } + t.Cleanup(func() { second.finish(false) }) + first.finish(true) + second.finish(true) + if !maps.Equal(before, unrelated.currentForTest()) || unrelated.inFlight != 0 || len(unrelated.pendingForTest()) != 0 { + t.Fatal("collision accounting changed the unrelated pool") + } + if matching.hasCurrentForTest() || matching.inFlight != 0 || len(matching.pendingForTest()) != 0 { + t.Fatal("matching pool did not balance") + } + // Capacity still counts pools, not just distinct digest buckets. + _, third := selector.reserve("third-route", []*Auth{{ID: "a"}}, "") + third.finish(true) + if selector.stateCount != 1 || len(selector.states) != 1 { + t.Fatal("colliding pools bypassed the existing capacity limit") + } + }) + } +} + +func weightedPoolForTest(route string, weights, current map[string]int64) *weightedPoolState { + state := &weightedPoolState{route: route} + for _, id := range sortedWeightedIDs(weights) { + state.members = append(state.members, weightedPoolMember{id: id, weight: weights[id], current: current[id]}) + } + return state +} diff --git a/sdk/cliproxy/auth/selector_weighted_test.go b/sdk/cliproxy/auth/selector_weighted_test.go new file mode 100644 index 000000000..31352e301 --- /dev/null +++ b/sdk/cliproxy/auth/selector_weighted_test.go @@ -0,0 +1,79 @@ +package auth + +import ( + "maps" + "sort" + "testing" +) + +func TestSmoothWeightedStateChargeRebasesBeforeBounding(t *testing.T) { + for _, tc := range []struct { + name string + current map[string]int64 + weights map[string]int64 + authID string + want map[string]int64 + }{ + { + name: "smaller pool preserves leading difference", + current: map[string]int64{"a": -96, "b": 96, "c": 95}, + weights: map[string]int64{"b": 1, "c": 1}, + authID: "b", + want: map[string]int64{"b": 63, "c": 64}, + }, + { + name: "negative credits shift together", + current: map[string]int64{"a": 96, "b": -64, "c": -63}, + weights: map[string]int64{"b": 1, "c": 1}, + authID: "b", + want: map[string]int64{"b": -64, "c": -61}, + }, + { + name: "excessive spread clips debt without tying leaders", + current: map[string]int64{"a": -96, "b": 96, "c": 95}, + weights: map[string]int64{"a": 1, "b": 1, "c": 1}, + authID: "a", + want: map[string]int64{"a": -96, "b": 96, "c": 95}, + }, + { + name: "bounded credits do not shift", + current: map[string]int64{"b": 0, "c": 0}, + weights: map[string]int64{"b": 1, "c": 1}, + authID: "b", + want: map[string]int64{"b": -1, "c": 1}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + state := weightedPoolForTest("route", tc.weights, tc.current) + state.charge(sort.Search(len(state.members), func(i int) bool { return state.members[i].id >= tc.authID }), false) + if !maps.Equal(state.currentForTest(), tc.want) { + t.Fatalf("rebased credits = %v, want %v", state.currentForTest(), tc.want) + } + }) + } +} + +func TestWeightedReservationRollbackPreservesConcurrentCredits(t *testing.T) { + key := "codex:model" + weights := map[string]int64{"a": 1, "b": 1, "c": 1} + state := weightedPoolForTest(key, weights, map[string]int64{"a": -94, "b": 94, "c": 93}) + weighted := &WeightedRoundRobinSelector{states: map[[32]byte]*weightedPoolState{weightedPoolStateKey(key, weights): state}, stateCount: 1} + auths := []*Auth{{ID: "a"}, {ID: "b"}, {ID: "c"}} + _, rejected := weighted.reserve(key, auths, "a") + t.Cleanup(func() { rejected.finish(false) }) + _, accepted := weighted.reserve(key, auths, "b") + t.Cleanup(func() { accepted.finish(false) }) + accepted.finish(true) + want := map[string]int64{"a": -93, "b": 92, "c": 94} + if !maps.Equal(state.currentForTest(), want) { + t.Fatalf("committed credits = %v, want %v", state.currentForTest(), want) + } + if !maps.Equal(state.pendingForTest(), map[string]int64{"a": -2, "b": 1, "c": 1}) { + t.Fatalf("commit changed another request's reservation: %v", state.pendingForTest()) + } + rejected.finish(false) + rejected.finish(true) + if !maps.Equal(state.currentForTest(), want) || len(state.pendingForTest()) != 0 { + t.Fatalf("rollback changed committed credits: current=%v pending=%v", state.currentForTest(), state.pendingForTest()) + } +}