package executor import ( "context" "errors" "io" "net/http" "strings" "testing" "github.com/gin-gonic/gin" "github.com/router-for-me/CLIProxyAPI/v7/internal/config" cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/executor" sdktranslator "github.com/router-for-me/CLIProxyAPI/v7/sdk/translator" "github.com/tidwall/gjson" ) // midSystemLegacyPayload is a caller body pairing a legacy model with a // mid-conversation role=system turn. The turn ends the array, so the shape is // rejected by the model rather than by Anthropic's ordering rule. func midSystemLegacyPayload(model string) []byte { return []byte(`{"model":"` + model + `","max_tokens":32,` + `"system":[{"type":"text","text":"Top rule"}],` + `"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]},` + `{"role":"system","content":[{"type":"text","text":"Mid rule"}]}],` + `"metadata":{"user_id":"{\"device_id\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"account_uuid\":\"\",\"session_id\":\"11111111-2222-4333-8444-555555555555\"}"}}`) } // midSystemUpstream intercepts the transport instead of standing up a test // server, so the executor keeps the default https://api.anthropic.com base URL. // The guard only fires on Anthropic's first-party origin, which a httptest // server address would not satisfy. type midSystemUpstream struct { body []byte called bool headers http.Header } func (u *midSystemUpstream) context(t *testing.T, headers http.Header) context.Context { t.Helper() gin.SetMode(gin.TestMode) ginCtx, _ := gin.CreateTestContext(nil) ginCtx.Request = httptest_NewRequest() ginCtx.Request.Header = headers.Clone() if ginCtx.Request.Header == nil { ginCtx.Request.Header = make(http.Header) } transport := roundTripperFunc(func(req *http.Request) (*http.Response, error) { payload, errRead := io.ReadAll(req.Body) if errRead != nil { t.Fatal(errRead) } u.body = payload u.called = true u.headers = req.Header.Clone() contentType := "application/json" responseBody := `{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}` if strings.Contains(req.URL.Path, "count_tokens") { responseBody = `{"input_tokens":18}` } else if gjson.GetBytes(payload, "stream").Bool() { contentType = "text/event-stream" // A translated caller aggregates the stream back into one message, so // the stub has to complete the block and report a stop reason. responseBody = strings.Join([]string{ `event: message_start` + "\n" + `data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[],"stop_reason":null,"usage":{"input_tokens":1,"output_tokens":0}}}`, `event: content_block_start` + "\n" + `data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`, `event: content_block_delta` + "\n" + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`, `event: content_block_stop` + "\n" + `data: {"type":"content_block_stop","index":0}`, `event: message_delta` + "\n" + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}`, `event: message_stop` + "\n" + `data: {"type":"message_stop"}`, }, "\n\n") + "\n\n" } return &http.Response{ StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{contentType}}, Body: io.NopCloser(strings.NewReader(responseBody)), Request: req, }, nil }) ctx := context.WithValue(context.Background(), "gin", ginCtx) return context.WithValue(ctx, "cliproxy.roundtripper", http.RoundTripper(transport)) } func httptest_NewRequest() *http.Request { req, _ := http.NewRequest(http.MethodPost, "http://example.invalid/", nil) return req } func midSystemAuth() *cliproxyauth.Auth { // No base_url, so the executor keeps Anthropic's first-party origin. return &cliproxyauth.Auth{Attributes: map[string]string{"api_key": "key-123", "cloak_mode": "always"}} } func midSystemConfig() *config.Config { return &config.Config{ClaudeKey: []config.ClaudeKey{{APIKey: "key-123"}}} } func assertMidSystemRejected(t *testing.T, err error, upstream *midSystemUpstream) { t.Helper() if err == nil { t.Fatal("error = nil, want the legacy pairing rejected") } if upstream.called { t.Fatalf("upstream must not be called for a guaranteed rejection; got %s", upstream.body) } var statusCoder interface{ StatusCode() int } if !errors.As(err, &statusCoder) || statusCoder.StatusCode() != http.StatusBadRequest { t.Fatalf("error = %v, want a 400 status error", err) } var scoped interface{ IsRequestScoped() bool } if !errors.As(err, &scoped) || !scoped.IsRequestScoped() { t.Fatalf("error = %v, want a request-scoped error so no credential is retried", err) } if !strings.Contains(err.Error(), "role 'system' is not supported on this model") { t.Fatalf("error = %v, want Anthropic's wording preserved", err) } } // Every executor path that can send the pairing to Anthropic must answer it // locally instead of spending an upstream call on a guaranteed 400. func TestClaudeExecutor_LegacyMidSystemMessageRejectedOnEveryUpstreamPath(t *testing.T) { for _, test := range []struct { name string model string send func(t *testing.T, ex *ClaudeExecutor, ctx context.Context, model string) error }{ {name: "execute", model: "claude-haiku-4-5-20251001", send: sendMidSystemExecute}, {name: "execute stream", model: "claude-haiku-4-5-20251001", send: sendMidSystemStream}, {name: "count tokens", model: "claude-haiku-4-5-20251001", send: sendMidSystemCountTokens}, {name: "execute legacy sonnet", model: "claude-sonnet-4-6", send: sendMidSystemExecute}, } { t.Run(test.name, func(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(midSystemConfig()) err := test.send(t, ex, upstream.context(t, nil), test.model) assertMidSystemRejected(t, err, upstream) }) } } func sendMidSystemExecute(t *testing.T, ex *ClaudeExecutor, ctx context.Context, model string) error { t.Helper() _, err := ex.Execute(ctx, midSystemAuth(), cliproxyexecutor.Request{ Model: model, Payload: midSystemLegacyPayload(model), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) return err } func sendMidSystemStream(t *testing.T, ex *ClaudeExecutor, ctx context.Context, model string) error { t.Helper() result, err := ex.ExecuteStream(ctx, midSystemAuth(), cliproxyexecutor.Request{ Model: model, Payload: midSystemLegacyPayload(model), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) if err != nil { return err } for chunk := range result.Chunks { if chunk.Err != nil { return chunk.Err } } return nil } func sendMidSystemCountTokens(t *testing.T, ex *ClaudeExecutor, ctx context.Context, model string) error { t.Helper() _, err := ex.CountTokens(ctx, midSystemAuth(), cliproxyexecutor.Request{ Model: model, Payload: midSystemLegacyPayload(model), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) return err } // Payload rules run long after translation and can rewrite model and messages, // so the guard has to read the finished body rather than an intermediate one. func TestClaudeExecutor_PayloadOverrideCannotSmuggleLegacyMidSystemMessage(t *testing.T) { upstream := &midSystemUpstream{} cfg := midSystemConfig() cfg.Payload.Override = []config.PayloadRule{{ Models: []config.PayloadModelRule{{Name: "*"}}, Params: map[string]any{"model": "claude-haiku-4-5-20251001"}, }} ex := NewClaudeExecutor(cfg) // The caller addresses a model that accepts the turn; only the payload rule // turns it into the rejected pairing. _, err := ex.Execute(upstream.context(t, nil), midSystemAuth(), cliproxyexecutor.Request{ Model: "claude-sonnet-5", Payload: midSystemLegacyPayload("claude-sonnet-5"), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) assertMidSystemRejected(t, err, upstream) } // A caller may already have the exact role=system turn that cloaking would // otherwise insert. The message-count proof must keep that turn caller-owned, // so a later legacy model rewrite is rejected instead of silently consuming it. func TestClaudeExecutor_PayloadOverrideDoesNotClaimMatchingCallerTurn(t *testing.T) { upstream := &midSystemUpstream{} cfg := midSystemConfig() cfg.Payload.Override = []config.PayloadRule{{ Models: []config.PayloadModelRule{{Name: "*"}}, Params: map[string]any{"model": "claude-haiku-4-5-20251001"}, }} ex := NewClaudeExecutor(cfg) payload := []byte(`{"model":"claude-sonnet-5","max_tokens":32,` + `"system":[{"type":"text","text":"Same rule"}],` + `"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]},` + `{"role":"system","content":[{"type":"text","text":"Same rule"}]}]}`) _, err := ex.Execute(upstream.context(t, nil), midSystemAuth(), cliproxyexecutor.Request{ Model: "claude-sonnet-5", Payload: payload, }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) assertMidSystemRejected(t, err, upstream) } // Cloaking relocates a caller's system prompt into a role=system turn for models // that accept one. A payload rule can then rewrite the model to one that does // not. Because the caller never wrote that turn, CPA must reconcile its own // placement through the legacy reminder path instead of returning 400. func TestClaudeExecutor_PayloadOverrideReconcilesRelocatedSystemPrompt(t *testing.T) { for _, test := range []struct { name string send func(t *testing.T, ex *ClaudeExecutor, ctx context.Context, payload []byte) error }{ {name: "execute", send: func(t *testing.T, ex *ClaudeExecutor, ctx context.Context, payload []byte) error { _, err := ex.Execute(ctx, midSystemAuth(), cliproxyexecutor.Request{ Model: "claude-sonnet-5", Payload: payload, }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) return err }}, {name: "execute stream", send: func(t *testing.T, ex *ClaudeExecutor, ctx context.Context, payload []byte) error { result, err := ex.ExecuteStream(ctx, midSystemAuth(), cliproxyexecutor.Request{ Model: "claude-sonnet-5", Payload: payload, }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}) if err != nil { return err } for chunk := range result.Chunks { if chunk.Err != nil { return chunk.Err } } return nil }}, } { t.Run(test.name, func(t *testing.T) { upstream := &midSystemUpstream{} cfg := midSystemConfig() cfg.Payload.Override = []config.PayloadRule{{ Models: []config.PayloadModelRule{{Name: "*"}}, Params: map[string]any{"model": "claude-haiku-4-5-20251001"}, }} ex := NewClaudeExecutor(cfg) // Only a top-level system prompt: the caller never writes a // role=system turn. payload := []byte(`{"model":"claude-sonnet-5","max_tokens":32,` + `"system":[{"type":"text","text":"Caller top"}],` + `"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) if err := test.send(t, ex, upstream.context(t, nil), payload); err != nil { t.Fatalf("request error = %v, want CPA's inserted turn reconciled", err) } if !upstream.called { t.Fatal("expected reconciled request to reach upstream") } if got := gjson.GetBytes(upstream.body, "model").String(); got != "claude-haiku-4-5-20251001" { t.Fatalf("upstream model = %q, want payload override preserved", got) } if gjson.GetBytes(upstream.body, `messages.#(role=="system")`).Exists() { t.Fatalf("reconciled body still carries role=system; body=%s", upstream.body) } if !strings.Contains(gjson.GetBytes(upstream.body, "messages.0.content").Raw, "") || !strings.Contains(gjson.GetBytes(upstream.body, "messages.0.content").Raw, "Caller top") { t.Fatalf("caller system prompt was not replayed as a legacy reminder; body=%s", upstream.body) } }) } } // A confirmed native caller owns its wire. It gates the turn on the model // itself, so CPA forwards the body untouched and lets the upstream answer. func TestClaudeExecutor_ConfirmedNativeLegacyMidSystemMessageForwarded(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(midSystemConfig()) headers := claudeNativeHelperHeaders("claude-code-20250219,"+claudeNativeHelperCoreBetas, "gzip", false) if _, err := ex.Execute(upstream.context(t, headers), midSystemAuth(), cliproxyexecutor.Request{ Model: "claude-haiku-4-5-20251001", Payload: midSystemLegacyPayload("claude-haiku-4-5-20251001"), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude, Headers: headers}); err != nil { t.Fatalf("Execute() error = %v, want the native body forwarded", err) } if !upstream.called { t.Fatal("expected the native request to reach the upstream") } if !gjson.GetBytes(upstream.body, `messages.#(role=="system")`).Exists() { t.Fatalf("confirmed native caller lost its role=system turn; body=%s", upstream.body) } } // The rejection was measured against api.anthropic.com. A third-party gateway // may map the same model ID onto something that accepts the turn, and answering // locally would also stop failover to another credential or base URL. func TestClaudeExecutor_LegacyMidSystemMessageForwardedToThirdPartyGateway(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(&config.Config{ClaudeKey: []config.ClaudeKey{{ APIKey: "key-123", BaseURL: "https://gateway.example", }}}) auth := &cliproxyauth.Auth{Attributes: map[string]string{ "api_key": "key-123", "base_url": "https://gateway.example", }} if _, err := ex.Execute(upstream.context(t, nil), auth, cliproxyexecutor.Request{ Model: "claude-haiku-4-5-20251001", Payload: midSystemLegacyPayload("claude-haiku-4-5-20251001"), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}); err != nil { t.Fatalf("Execute() error = %v, want a third-party gateway to decide for itself", err) } if !upstream.called { t.Fatal("expected the request to reach the third-party gateway") } } // A model outside claudeLegacySystemReminderModels stays optimistic, matching // how checkSystemInstructions treats unknown and future IDs. func TestClaudeExecutor_SupportedModelMidSystemMessageForwarded(t *testing.T) { for _, model := range []string{"claude-sonnet-5", "claude-sonnet-9"} { t.Run(model, func(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(midSystemConfig()) if _, err := ex.Execute(upstream.context(t, nil), midSystemAuth(), cliproxyexecutor.Request{ Model: model, Payload: midSystemLegacyPayload(model), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}); err != nil { t.Fatalf("Execute() error = %v, want the request forwarded", err) } if !upstream.called { t.Fatal("expected the request to reach the upstream") } }) } } // The opt-in rescues the pairing by folding the turn into the system slot, so // the guard must run after it rather than rejecting the request outright. func TestClaudeExecutor_LegacyMidSystemMessageOptInStillRebuilds(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(midSystemConfig()) auth := midSystemAuth() auth.Attributes["rebuild_mid_system_message"] = "true" if _, err := ex.Execute(upstream.context(t, nil), auth, cliproxyexecutor.Request{ Model: "claude-haiku-4-5-20251001", Payload: midSystemLegacyPayload("claude-haiku-4-5-20251001"), }, cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatClaude}); err != nil { t.Fatalf("Execute() error = %v, want the opt-in rebuild to rescue the request", err) } if !upstream.called { t.Fatal("expected the rebuilt request to reach the upstream") } if gjson.GetBytes(upstream.body, `messages.#(role=="system")`).Exists() { t.Fatalf("opt-in rebuild left a role=system turn; body=%s", upstream.body) } } // The pairing must never originate inside CPA. A non-Claude caller reaches the // Claude executor through a translator, and every translator hoists system // content into the top-level system field, so no translated body can carry a // role=system turn to a legacy model. This pins that guarantee: the guard is for // callers that speak Claude natively, never for a translated request. func TestTranslatedRequestNeverPairsLegacyModelWithMidSystemMessage(t *testing.T) { const legacyModel = "claude-haiku-4-5-20251001" for _, test := range []struct { name string format sdktranslator.Format payload string }{ {name: "openai chat with a mid conversation system message", format: sdktranslator.FormatOpenAI, payload: `{"model":"` + legacyModel + `","messages":[{"role":"system","content":"Top rule"},{"role":"user","content":"hi"},{"role":"system","content":"Mid rule"},{"role":"assistant","content":"ok"},{"role":"user","content":"go"}]}`}, {name: "openai chat ending on a system message", format: sdktranslator.FormatOpenAI, payload: `{"model":"` + legacyModel + `","messages":[{"role":"user","content":"hi"},{"role":"system","content":"Mid rule"}]}`}, {name: "gemini with a system instruction", format: sdktranslator.FormatGemini, payload: `{"contents":[{"role":"user","parts":[{"text":"hi"}]}],"systemInstruction":{"parts":[{"text":"Top rule"}]}}`}, {name: "openai responses with instructions", format: sdktranslator.FormatOpenAIResponse, payload: `{"model":"` + legacyModel + `","instructions":"Top rule","input":[{"role":"user","content":[{"type":"input_text","text":"hi"}]}]}`}, {name: "interactions with a system instruction", format: sdktranslator.FormatInteractions, payload: `{"model":"` + legacyModel + `","system_instruction":"Top rule","input":[{"type":"user_input","content":[{"type":"text","text":"hi"}]}]}`}, } { t.Run(test.name, func(t *testing.T) { upstream := &midSystemUpstream{} ex := NewClaudeExecutor(midSystemConfig()) if _, err := ex.Execute(upstream.context(t, nil), midSystemAuth(), cliproxyexecutor.Request{ Model: legacyModel, Payload: []byte(test.payload), }, cliproxyexecutor.Options{SourceFormat: test.format}); err != nil { t.Fatalf("Execute() error = %v, want the translated request forwarded", err) } if !upstream.called { t.Fatal("expected the translated request to reach the upstream") } // Without these the subject under test could drift away: a body that // no longer addresses the legacy model, or that lost the caller's // turns, would satisfy the role assertions for the wrong reason. if got := gjson.GetBytes(upstream.body, "model").String(); got != legacyModel { t.Fatalf("upstream model = %q, want the legacy model %q under test", got, legacyModel) } if got := len(gjson.GetBytes(upstream.body, "messages").Array()); got == 0 { t.Fatalf("upstream messages are empty, so the role assertions prove nothing; body=%s", upstream.body) } for _, role := range gjson.GetBytes(upstream.body, "messages.#.role").Array() { if strings.EqualFold(role.String(), "system") { t.Fatalf("translated body carries a system role; body=%s", upstream.body) } } }) } } func TestClaudePayloadHasMidSystemMessage(t *testing.T) { for _, test := range []struct { name string payload string want bool }{ {name: "mid conversation turn", want: true, payload: `{"messages":[{"role":"user","content":"a"},{"role":"system","content":"s"}]}`}, {name: "role casing is ignored", want: true, payload: `{"messages":[{"role":"SySTeM","content":"s"}]}`}, {name: "surrounding whitespace is ignored", want: true, payload: `{"messages":[{"role":" system ","content":"s"}]}`}, {name: "only user and assistant turns", payload: `{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"}]}`}, {name: "top level system field is not a turn", payload: `{"system":[{"type":"text","text":"s"}],"messages":[{"role":"user","content":"a"}]}`}, {name: "messages missing", payload: `{"model":"claude-haiku-4-5"}`}, {name: "messages is not an array", payload: `{"messages":"system"}`}, {name: "messages holds a bare string", payload: `{"messages":["system"]}`}, {name: "system appears only in content", payload: `{"messages":[{"role":"user","content":"role: system"}]}`}, } { t.Run(test.name, func(t *testing.T) { if got := claudePayloadHasMidSystemMessage([]byte(test.payload)); got != test.want { t.Fatalf("claudePayloadHasMidSystemMessage = %v, want %v", got, test.want) } }) } } func TestValidateClaudeMidSystemMessageModel(t *testing.T) { const turn = `,"messages":[{"role":"user","content":"a"},{"role":"system","content":"s"}]}` for _, test := range []struct { name string payload string confirmed bool thirdParty bool wantError bool }{ {name: "legacy model is rejected", wantError: true, payload: `{"model":"claude-haiku-4-5-20251001"` + turn}, {name: "vendor prefixed legacy model is rejected", wantError: true, payload: `{"model":"anthropic/claude-sonnet-4-6"` + turn}, {name: "model casing is ignored", wantError: true, payload: `{"model":"Claude-Haiku-4-5-20251001"` + turn}, {name: "confirmed native keeps the passthrough", confirmed: true, payload: `{"model":"claude-haiku-4-5-20251001"` + turn}, {name: "third party gateway decides for itself", thirdParty: true, payload: `{"model":"claude-haiku-4-5-20251001"` + turn}, {name: "supported model is forwarded", payload: `{"model":"claude-sonnet-5"` + turn}, {name: "unknown model stays optimistic", payload: `{"model":"claude-sonnet-9"` + turn}, {name: "legacy model without the turn is forwarded", payload: `{"model":"claude-haiku-4-5-20251001","messages":[{"role":"user","content":"a"}]}`}, } { t.Run(test.name, func(t *testing.T) { err := validateClaudeMidSystemMessageModel([]byte(test.payload), test.confirmed, !test.thirdParty) if test.wantError != (err != nil) { t.Fatalf("validateClaudeMidSystemMessageModel error = %v, want error %v", err, test.wantError) } if err == nil { return } if !strings.Contains(err.Error(), gjson.Get(test.payload, "model").String()) { t.Fatalf("error = %v, want the offending model named", err) } }) } }