mirror of
https://github.com/router-for-me/CLIProxyAPI.git
synced 2026-09-03 06:35:00 +08:00
fix(antigravity): attach sibling tool images to the nearest functionResponse (#5075)
Responses→Gemini emits functionResponse plus sibling inline_data, but fixCLIToolResponse previously dropped those images when regrouping tool turns. Cloud Code Assist only sees tool images nested in functionResponse.parts with an explicit mimeType, and parallel tool results must stay bound to the nearest preceding functionResponse instead of last-wins. Closes #5070
This commit is contained in:
@@ -639,6 +639,74 @@ type FunctionCallGroup struct {
|
||||
CallNames []string // ordered function call names for backfilling empty response names
|
||||
}
|
||||
|
||||
func normalizeAntigravityInlineDataPart(part gjson.Result) ([]byte, bool) {
|
||||
inline := part.Get("inlineData")
|
||||
if !inline.Exists() {
|
||||
inline = part.Get("inline_data")
|
||||
}
|
||||
if !inline.Exists() {
|
||||
return nil, false
|
||||
}
|
||||
data := inline.Get("data").String()
|
||||
if data == "" {
|
||||
return nil, false
|
||||
}
|
||||
mimeType := inline.Get("mimeType").String()
|
||||
if mimeType == "" {
|
||||
mimeType = inline.Get("mime_type").String()
|
||||
}
|
||||
if mimeType == "" {
|
||||
// Cloud Code Assist ignores inlineData without mimeType.
|
||||
mimeType = "image/png"
|
||||
}
|
||||
out := []byte(`{"inlineData":{"mimeType":"","data":""}}`)
|
||||
out, _ = sjson.SetBytes(out, "inlineData.mimeType", mimeType)
|
||||
out, _ = sjson.SetBytes(out, "inlineData.data", data)
|
||||
return out, true
|
||||
}
|
||||
|
||||
func attachInlineDataToFunctionResponse(response gjson.Result, images [][]byte) gjson.Result {
|
||||
if len(images) == 0 {
|
||||
return response
|
||||
}
|
||||
target := []byte(response.Raw)
|
||||
for _, img := range images {
|
||||
target, _ = sjson.SetRawBytes(target, "functionResponse.parts.-1", img)
|
||||
}
|
||||
return gjson.ParseBytes(target)
|
||||
}
|
||||
|
||||
// collectFunctionResponsesWithSiblingInlineData keeps functionResponse parts and
|
||||
// moves sibling inline_data/inlineData onto the nearest preceding functionResponse.
|
||||
// Leading images before the first functionResponse attach to that first response.
|
||||
func collectFunctionResponsesWithSiblingInlineData(parts gjson.Result) []gjson.Result {
|
||||
responses := make([]gjson.Result, 0)
|
||||
leadingImages := make([][]byte, 0)
|
||||
current := -1
|
||||
parts.ForEach(func(_, part gjson.Result) bool {
|
||||
if part.Get("functionResponse").Exists() {
|
||||
responses = append(responses, part)
|
||||
current = len(responses) - 1
|
||||
if len(leadingImages) > 0 {
|
||||
responses[current] = attachInlineDataToFunctionResponse(responses[current], leadingImages)
|
||||
leadingImages = nil
|
||||
}
|
||||
return true
|
||||
}
|
||||
imagePart, ok := normalizeAntigravityInlineDataPart(part)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if current >= 0 {
|
||||
responses[current] = attachInlineDataToFunctionResponse(responses[current], [][]byte{imagePart})
|
||||
return true
|
||||
}
|
||||
leadingImages = append(leadingImages, imagePart)
|
||||
return true
|
||||
})
|
||||
return responses
|
||||
}
|
||||
|
||||
// parseFunctionResponseRaw attempts to normalize a function response part into a JSON object string.
|
||||
// Falls back to a minimal "functionResponse" object when parsing fails.
|
||||
// fallbackName is used when the response's own name is empty.
|
||||
@@ -749,14 +817,8 @@ func fixCLIToolResponse(input []byte) ([]byte, error) {
|
||||
role := value.Get("role").String()
|
||||
parts := value.Get("parts")
|
||||
|
||||
// Check if this content has function responses
|
||||
var responsePartsInThisContent []gjson.Result
|
||||
parts.ForEach(func(_, part gjson.Result) bool {
|
||||
if part.Get("functionResponse").Exists() {
|
||||
responsePartsInThisContent = append(responsePartsInThisContent, part)
|
||||
}
|
||||
return true
|
||||
})
|
||||
// Collect function responses and attach sibling inlineData to the nearest one.
|
||||
responsePartsInThisContent := collectFunctionResponsesWithSiblingInlineData(parts)
|
||||
|
||||
// If this content has function responses, collect them
|
||||
if len(responsePartsInThisContent) > 0 {
|
||||
|
||||
@@ -954,3 +954,190 @@ func TestSanitizeAntigravityClaudeGeminiRequestSignatures_LargeNumberDoesNotHalt
|
||||
t.Fatalf("expected hidden thoughtSignature to be stripped despite large number, got %s", fcSig.Raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixCLIToolResponse_AttachesSiblingInlineDataToNearestFunctionResponse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
parts string
|
||||
want []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}
|
||||
}{
|
||||
{
|
||||
name: "snake_case sibling after single response",
|
||||
parts: `{"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}},` +
|
||||
`{"inline_data":{"mime_type":"image/png","data":"QUJD"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{{id: "call_1", mime: "image/png", data: "QUJD"}},
|
||||
},
|
||||
{
|
||||
name: "camelCase sibling after single response",
|
||||
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` +
|
||||
`{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{{id: "call_1", mime: "image/webp", data: "NEW"}},
|
||||
},
|
||||
{
|
||||
name: "append sibling onto existing functionResponse.parts",
|
||||
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1","parts":[{"inlineData":{"mimeType":"image/gif","data":"OLD"}}]}},` +
|
||||
`{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{
|
||||
{id: "call_1", mime: "image/gif", data: "OLD"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "interleaved siblings attach to nearest response",
|
||||
parts: `{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` +
|
||||
`{"inline_data":{"mime_type":"image/png","data":"AAA"}},` +
|
||||
`{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}},` +
|
||||
`{"inline_data":{"mime_type":"image/jpeg","data":"BBB"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{
|
||||
{id: "call_a", mime: "image/png", data: "AAA"},
|
||||
{id: "call_b", mime: "image/jpeg", data: "BBB"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "leading sibling attaches to first response",
|
||||
parts: `{"inline_data":{"mime_type":"image/png","data":"LEAD"}},` +
|
||||
`{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` +
|
||||
`{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{
|
||||
{id: "call_a", mime: "image/png", data: "LEAD"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing mimeType defaults to image/png",
|
||||
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` +
|
||||
`{"inlineData":{"data":"QUJD"}}`,
|
||||
want: []struct {
|
||||
id string
|
||||
mime string
|
||||
data string
|
||||
}{{id: "call_1", mime: "image/png", data: "QUJD"}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
modelParts := `{"functionCall":{"name":"read","id":"call_1"}}`
|
||||
if tt.name == "interleaved siblings attach to nearest response" || tt.name == "leading sibling attaches to first response" {
|
||||
modelParts = `{"functionCall":{"name":"read","id":"call_a"}},{"functionCall":{"name":"read","id":"call_b"}}`
|
||||
}
|
||||
input := `{"request":{"contents":[` +
|
||||
`{"role":"model","parts":[` + modelParts + `]},` +
|
||||
`{"role":"user","parts":[` + tt.parts + `]}` +
|
||||
`]}}`
|
||||
result, err := fixCLIToolResponse([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("fixCLIToolResponse failed: %v", err)
|
||||
}
|
||||
contents := gjson.GetBytes(result, "request.contents").Array()
|
||||
if len(contents) != 2 {
|
||||
t.Fatalf("contents = %d, want 2. Output: %s", len(contents), result)
|
||||
}
|
||||
funcParts := contents[1].Get("parts").Array()
|
||||
gotByID := map[string][]gjson.Result{}
|
||||
for _, part := range funcParts {
|
||||
fr := part.Get("functionResponse")
|
||||
gotByID[fr.Get("id").String()] = fr.Get("parts").Array()
|
||||
}
|
||||
for _, want := range tt.want {
|
||||
images := gotByID[want.id]
|
||||
found := false
|
||||
for _, img := range images {
|
||||
if img.Get("inlineData.data").String() == want.data && img.Get("inlineData.mimeType").String() == want.mime {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("id=%s missing inlineData mime=%s data=%s. Output: %s", want.id, want.mime, want.data, result)
|
||||
}
|
||||
}
|
||||
if tt.name == "interleaved siblings attach to nearest response" {
|
||||
if len(gotByID["call_a"]) != 1 || len(gotByID["call_b"]) != 1 {
|
||||
t.Fatalf("nearest attribution failed: A=%d B=%d. Output: %s", len(gotByID["call_a"]), len(gotByID["call_b"]), result)
|
||||
}
|
||||
}
|
||||
if tt.name == "leading sibling attaches to first response" {
|
||||
if len(gotByID["call_b"]) != 0 {
|
||||
t.Fatalf("leading image leaked onto call_b. Output: %s", result)
|
||||
}
|
||||
}
|
||||
if tt.name == "append sibling onto existing functionResponse.parts" {
|
||||
images := gotByID["call_1"]
|
||||
if len(images) != 2 {
|
||||
t.Fatalf("existing+sibling parts = %d, want 2. Output: %s", len(images), result)
|
||||
}
|
||||
if images[1].Get("inlineData.data").String() != "NEW" {
|
||||
t.Fatalf("appended sibling data = %q, want NEW. Output: %s", images[1].Get("inlineData.data").String(), result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertGeminiRequestToAntigravity_PreservesSiblingToolImageOnUserRole(t *testing.T) {
|
||||
input := []byte(`{
|
||||
"contents": [
|
||||
{"role":"user","parts":[{"text":"read file"}]},
|
||||
{"role":"model","parts":[{"functionCall":{"name":"read","args":{},"id":"call_1"}}]},
|
||||
{"role":"user","parts":[
|
||||
{"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}},
|
||||
{"inline_data":{"mime_type":"image/png","data":"QUJD"}}
|
||||
]}
|
||||
]
|
||||
}`)
|
||||
out := ConvertGeminiRequestToAntigravity("gemini-3-flash", input, false)
|
||||
contents := gjson.GetBytes(out, "request.contents").Array()
|
||||
if len(contents) != 3 {
|
||||
t.Fatalf("contents = %d, want 3. Output: %s", len(contents), out)
|
||||
}
|
||||
funcContent := contents[2]
|
||||
if got := funcContent.Get("role").String(); got != "user" {
|
||||
t.Fatalf("role = %q, want user after Antigravity normalization. Output: %s", got, out)
|
||||
}
|
||||
funcResp := funcContent.Get("parts.0.functionResponse")
|
||||
if !funcResp.Exists() {
|
||||
t.Fatalf("functionResponse missing. Output: %s", out)
|
||||
}
|
||||
if got := funcResp.Get("id").String(); got != "call_1" {
|
||||
t.Fatalf("id = %q, want call_1", got)
|
||||
}
|
||||
if got := funcResp.Get("response.result").String(); got != "Read image file [image/png]" {
|
||||
t.Fatalf("result = %q", got)
|
||||
}
|
||||
inlineData := funcResp.Get("parts.0.inlineData")
|
||||
if !inlineData.Exists() {
|
||||
t.Fatalf("functionResponse.parts.0.inlineData missing. Output: %s", out)
|
||||
}
|
||||
if got := inlineData.Get("mimeType").String(); got != "image/png" {
|
||||
t.Fatalf("mimeType = %q, want image/png", got)
|
||||
}
|
||||
if got := inlineData.Get("data").String(); got != "QUJD" {
|
||||
t.Fatalf("data = %q, want QUJD", got)
|
||||
}
|
||||
if funcContent.Get("parts.1.inline_data").Exists() || funcContent.Get("parts.1.inlineData").Exists() {
|
||||
t.Fatalf("sibling inline data should be absorbed into functionResponse.parts. Output: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,3 +265,93 @@ func TestConvertOpenAIResponsesRequestToAntigravity_GeminiReasoningUsesNativeTho
|
||||
t.Fatalf("parts[0].thoughtSignature = %q, want preserved Gemini signature. Output: %s", got, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertOpenAIResponsesRequestToAntigravity_PreservesToolResultImage(t *testing.T) {
|
||||
inputJSON := `{
|
||||
"model": "gemini-3-flash",
|
||||
"input": [
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "请帮我读取分析这张图片"}]},
|
||||
{"type": "function_call", "id": "fc_read", "call_id": "call_read_1", "name": "read", "arguments": "{\"path\":\"/path/to/image.png\"}"},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_read_1",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "Read image file [image/png]"},
|
||||
{"type": "input_image", "detail": "auto", "image_url": "data:image/png;base64,QUJD"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`
|
||||
out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false)
|
||||
contents := gjson.GetBytes(out, "request.contents").Array()
|
||||
if len(contents) != 3 {
|
||||
t.Fatalf("expected 3 contents, got %d. Output: %s", len(contents), out)
|
||||
}
|
||||
funcContent := contents[2]
|
||||
if got := funcContent.Get("role").String(); got != "user" {
|
||||
t.Fatalf("role = %q, want user. Output: %s", got, out)
|
||||
}
|
||||
funcResp := funcContent.Get("parts.0.functionResponse")
|
||||
if !funcResp.Exists() {
|
||||
t.Fatalf("functionResponse should exist. Output: %s", out)
|
||||
}
|
||||
if got := funcResp.Get("id").String(); got != "call_read_1" {
|
||||
t.Fatalf("id = %q, want call_read_1", got)
|
||||
}
|
||||
if got := funcResp.Get("name").String(); got != "read" {
|
||||
t.Fatalf("name = %q, want read", got)
|
||||
}
|
||||
inlineData := funcResp.Get("parts.0.inlineData")
|
||||
if !inlineData.Exists() {
|
||||
t.Fatalf("expected functionResponse.parts.0.inlineData to exist, got: %s", out)
|
||||
}
|
||||
if got := inlineData.Get("mimeType").String(); got != "image/png" {
|
||||
t.Errorf("expected mimeType image/png, got %q", got)
|
||||
}
|
||||
if got := inlineData.Get("data").String(); got != "QUJD" {
|
||||
t.Errorf("expected data QUJD, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertOpenAIResponsesRequestToAntigravity_AttachesParallelToolImagesToNearestResponse(t *testing.T) {
|
||||
inputJSON := `{
|
||||
"model": "gemini-3-flash",
|
||||
"input": [
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "read both"}]},
|
||||
{"type": "function_call", "id": "fc_a", "call_id": "call_a", "name": "read", "arguments": "{\"path\":\"/tmp/a.png\"}"},
|
||||
{"type": "function_call", "id": "fc_b", "call_id": "call_b", "name": "read", "arguments": "{\"path\":\"/tmp/b.png\"}"},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_a",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "file A"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,AAA"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_b",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "file B"},
|
||||
{"type": "input_image", "image_url": "data:image/jpeg;base64,BBB"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`
|
||||
out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false)
|
||||
parts := gjson.GetBytes(out, "request.contents.2.parts").Array()
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("function parts = %d, want 2. Output: %s", len(parts), out)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, part := range parts {
|
||||
fr := part.Get("functionResponse")
|
||||
got[fr.Get("id").String()] = fr.Get("parts.0.inlineData.data").String()
|
||||
}
|
||||
if got["call_a"] != "AAA" {
|
||||
t.Fatalf("call_a image = %q, want AAA. Output: %s", got["call_a"], out)
|
||||
}
|
||||
if got["call_b"] != "BBB" {
|
||||
t.Fatalf("call_b image = %q, want BBB. Output: %s", got["call_b"], out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user