mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-11-19 16:29:45 +08:00
Responses reasoning emitted only in response.output_item.done was ignored until a later token or terminal event. Count non-empty summary and content text in the shared classifier while excluding empty and encrypted-only items. Add deterministic coverage for Meta Responses and Claude downstreams, classification boundaries, and preservation of an earlier token time. Refresh the bootstrap-buffering comment to match the shared helper. Refs #6664
414 lines
14 KiB
Go
414 lines
14 KiB
Go
package helps
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
)
|
|
|
|
func TestIsResponsesTokenEvent_Classification(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
payload string
|
|
want bool
|
|
}{
|
|
{
|
|
name: "empty payload",
|
|
payload: "",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "whitespace only",
|
|
payload: " \n\t ",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "codex rate limits metadata",
|
|
payload: `{"type":"codex.rate_limits","rate_limits":{"plan_type":"pro"}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "codex response metadata",
|
|
payload: `{"type":"codex.response.metadata","etag":"W/\"123\""}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "responsesapi websocket timing",
|
|
payload: `{"type":"responsesapi.websocket_timing","timing":{"duration_ms":100}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response created",
|
|
payload: `{"type":"response.created","response":{"id":"resp_123","status":"in_progress"}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response in progress",
|
|
payload: `{"type":"response.in_progress","response":{"id":"resp_123","tools":[{"type":"function"}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response output item added with encrypted content only",
|
|
payload: `{"type":"response.output_item.added","item":{"type":"reasoning","encrypted_content":"gAAAAAB..."}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response content part added",
|
|
payload: `{"type":"response.content_part.added","part":{"type":"text","text":""}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response reasoning summary part added",
|
|
payload: `{"type":"response.reasoning_summary_part.added","part":{"type":"summary_text","text":""}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response reasoning summary text delta empty",
|
|
payload: `{"type":"response.reasoning_summary_text.delta","delta":""}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response reasoning summary text delta non-empty",
|
|
payload: `{"type":"response.reasoning_summary_text.delta","delta":"**Inspecting**"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response reasoning delta non-empty",
|
|
payload: `{"type":"response.reasoning.delta","delta":"Analyzing requirements..."}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response reasoning text delta non-empty",
|
|
payload: `{"type":"response.reasoning_text.delta","delta":"Step 1: Check code"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response output text delta non-empty",
|
|
payload: `{"type":"response.output_text.delta","delta":"Hello world"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response text delta non-empty",
|
|
payload: `{"type":"response.text.delta","delta":"Direct text chunk"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response function call arguments delta non-empty",
|
|
payload: `{"type":"response.function_call_arguments.delta","delta":"{\"query\":\"test\"}"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response custom_tool_call_input delta non-empty",
|
|
payload: `{"type":"response.custom_tool_call_input.delta","delta":"{\"param\":1}"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response code interpreter call code delta non-empty",
|
|
payload: `{"type":"response.code_interpreter_call_code.delta","delta":"import math\n"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response mcp call arguments delta non-empty",
|
|
payload: `{"type":"response.mcp_call_arguments.delta","delta":"{\"tool\":\"lookup\"}"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response shell call command added with non-empty command",
|
|
payload: `{"type":"response.shell_call_command.added","command":"ls -la"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response shell call command added with empty command",
|
|
payload: `{"type":"response.shell_call_command.added","command":""}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response shell call command delta non-empty",
|
|
payload: `{"type":"response.shell_call_command.delta","delta":"ls -la\n"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response shell call output content delta is tool execution output, not model token",
|
|
payload: `{"type":"response.shell_call_output_content.delta","delta":{"stdout":"output text\n","stderr":""}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response shell call output content done is tool execution output, not model token",
|
|
payload: `{"type":"response.shell_call_output_content.done","output":[]}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response refusal delta non-empty",
|
|
payload: `{"type":"response.refusal.delta","delta":"I cannot fulfill this request"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response audio transcript delta non-empty",
|
|
payload: `{"type":"response.audio.transcript.delta","delta":"Spoken text"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response audio delta non-empty",
|
|
payload: `{"type":"response.audio.delta","delta":"UklGRi..."}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response image generation call partial image non-empty",
|
|
payload: `{"type":"response.image_generation_call.partial_image","partial_image_b64":"iVBORw0KGgo..."}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response web search call in progress",
|
|
payload: `{"type":"response.web_search_call.in_progress"}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response file search call searching",
|
|
payload: `{"type":"response.file_search_call.searching"}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response code interpreter call interpreting",
|
|
payload: `{"type":"response.code_interpreter_call.interpreting"}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response mcp call in progress",
|
|
payload: `{"type":"response.mcp_call.in_progress"}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response output item done function call empty args",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"function_call","name":"lookup","arguments":""}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response output item done function call non-empty args",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"function_call","name":"lookup","arguments":"{\"q\":1}"}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response output item done message empty",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"message","content":[]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "response output item done message with text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"message","content":[{"type":"text","text":"hello"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response completed fallback",
|
|
payload: `{"type":"response.completed","response":{"id":"resp_123","status":"completed"}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response done fallback",
|
|
payload: `{"type":"response.done","response":{"id":"resp_123"}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response incomplete fallback",
|
|
payload: `{"type":"response.incomplete","response":{"id":"resp_123","status":"incomplete"}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "response failed fallback",
|
|
payload: `{"type":"response.failed","response":{"id":"resp_123","status":"failed"}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "generic error fallback",
|
|
payload: `{"type":"error","error":{"message":"overloaded","code":"rate_limit_exceeded"}}`,
|
|
want: true,
|
|
},
|
|
// SSE line format verification
|
|
{
|
|
name: "SSE data line with output text delta",
|
|
payload: `data: {"type":"response.output_text.delta","delta":"Hello SSE"}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "SSE data line with response created",
|
|
payload: `data: {"type":"response.created","response":{"id":"resp_sse"}}`,
|
|
want: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := IsResponsesTokenEvent([]byte(tt.payload)); got != tt.want {
|
|
t.Errorf("IsResponsesTokenEvent(%s) = %v, want %v", tt.name, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestObserveResponsesTokenEvent_Behavior(t *testing.T) {
|
|
ctx := context.Background()
|
|
reporter := NewUsageReporter(ctx, "codex", "gpt-5.6-luna", nil)
|
|
reporter.StartResponseTTFT()
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
// 1. Initial state: TTFT not set
|
|
if reporter.IsTTFTSet() {
|
|
t.Fatalf("expected IsTTFTSet() to be false initially")
|
|
}
|
|
|
|
// 2. Metadata event records first packet fallback, but does not mark effective TTFT
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"codex.rate_limits","rate_limits":{"plan_type":"pro"}}`))
|
|
if reporter.IsTTFTSet() {
|
|
t.Fatalf("metadata event must not trigger IsTTFTSet()")
|
|
}
|
|
if !reporter.IsFirstPacketSet() {
|
|
t.Fatalf("expected IsFirstPacketSet() == true")
|
|
}
|
|
|
|
// 3. Response created event does not mark effective TTFT
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.created","response":{"id":"resp_1"}}`))
|
|
if reporter.IsTTFTSet() {
|
|
t.Fatalf("response.created must not trigger IsTTFTSet()")
|
|
}
|
|
|
|
// 4. Meaningful token delta triggers effective TTFT
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.output_text.delta","delta":"First word"}`))
|
|
if !reporter.IsTTFTSet() {
|
|
t.Fatalf("response.output_text.delta must trigger IsTTFTSet()")
|
|
}
|
|
tokenTTFT := reporter.ttftDuration()
|
|
if tokenTTFT <= 0 {
|
|
t.Fatalf("expected token TTFT > 0, got %v", tokenTTFT)
|
|
}
|
|
|
|
// 5. Subsequent calls should be fast-path no-ops and not alter TTFT
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.output_text.delta","delta":"Second word"}`))
|
|
if reporter.ttftDuration() != tokenTTFT {
|
|
t.Fatalf("subsequent ObserveResponsesTokenEvent must not modify already set TTFT")
|
|
}
|
|
}
|
|
|
|
func TestObserveResponsesTokenEvent_FirstPacketFallback(t *testing.T) {
|
|
ctx := context.Background()
|
|
reporter := NewUsageReporter(ctx, "codex", "gpt-5.6-luna", nil)
|
|
reporter.StartResponseTTFT()
|
|
|
|
// Only send metadata events, no token deltas
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"codex.rate_limits","rate_limits":{"plan_type":"pro"}}`))
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.created","response":{"id":"resp_1"}}`))
|
|
|
|
if reporter.IsTTFTSet() {
|
|
t.Fatalf("expected IsTTFTSet() to be false when no tokens were received")
|
|
}
|
|
|
|
// First packet fallback must be returned
|
|
if !reporter.IsFirstPacketSet() {
|
|
t.Fatalf("expected IsFirstPacketSet() == true")
|
|
}
|
|
}
|
|
|
|
func TestIsResponsesTokenEvent_CompletedReasoning(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
payload string
|
|
want bool
|
|
}{
|
|
{
|
|
name: "summary text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"type":"summary_text","text":"thinking"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "content text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","content":[{"type":"reasoning_text","text":"thinking"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "later summary part",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":""},{"text":"thinking"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "empty summary with content text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":""}],"content":[{"text":"thinking"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "whitespace text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":" "}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "SSE text",
|
|
payload: `data: {"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":"thinking"}]}}`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "empty arrays",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[],"content":[]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "empty text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":""}],"content":[{"text":""}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "missing text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"type":"summary_text"}],"content":[{}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "null text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":null}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "missing parts",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning"}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "encrypted only",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[],"encrypted_content":"opaque"}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "malformed numeric text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":123}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "malformed object text",
|
|
payload: `{"type":"response.output_item.done","item":{"type":"reasoning","content":[{"text":{"value":"thinking"}}]}}`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "added stays metadata",
|
|
payload: `{"type":"response.output_item.added","item":{"type":"reasoning","summary":[{"text":"thinking"}]}}`,
|
|
want: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := IsResponsesTokenEvent([]byte(tt.payload)); got != tt.want {
|
|
t.Fatalf("IsResponsesTokenEvent = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestObserveResponsesTokenEvent_CompletedReasoningKeepsFirstTokenTime(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
reporter := NewUsageReporter(context.Background(), "meta", "muse-spark-1.3", nil)
|
|
reporter.StartResponseTTFT()
|
|
time.Sleep(100 * time.Millisecond)
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.reasoning_summary_text.delta","delta":"thinking"}`))
|
|
time.Sleep(time.Second)
|
|
ObserveResponsesTokenEvent(reporter, []byte(`{"type":"response.output_item.done","item":{"type":"reasoning","summary":[{"text":"thinking"}]}}`))
|
|
if got := reporter.ttftDuration(); got != 100*time.Millisecond {
|
|
t.Fatalf("TTFT = %v, want 100ms", got)
|
|
}
|
|
})
|
|
}
|