fix(antigravity): attach sibling tool images to the nearest functionResponse (#5075)

Responses→Gemini emits functionResponse plus sibling inline_data, but
fixCLIToolResponse previously dropped those images when regrouping tool
turns. Cloud Code Assist only sees tool images nested in
functionResponse.parts with an explicit mimeType, and parallel tool
results must stay bound to the nearest preceding functionResponse
instead of last-wins.

Closes #5070
This commit is contained in:
sususu98
2026-08-19 18:58:04 +08:00
committed by GitHub
parent 62f5a2798c
commit 79ef3618d6
3 changed files with 347 additions and 8 deletions

View File

@@ -639,6 +639,74 @@ type FunctionCallGroup struct {
CallNames []string // ordered function call names for backfilling empty response names
}
func normalizeAntigravityInlineDataPart(part gjson.Result) ([]byte, bool) {
inline := part.Get("inlineData")
if !inline.Exists() {
inline = part.Get("inline_data")
}
if !inline.Exists() {
return nil, false
}
data := inline.Get("data").String()
if data == "" {
return nil, false
}
mimeType := inline.Get("mimeType").String()
if mimeType == "" {
mimeType = inline.Get("mime_type").String()
}
if mimeType == "" {
// Cloud Code Assist ignores inlineData without mimeType.
mimeType = "image/png"
}
out := []byte(`{"inlineData":{"mimeType":"","data":""}}`)
out, _ = sjson.SetBytes(out, "inlineData.mimeType", mimeType)
out, _ = sjson.SetBytes(out, "inlineData.data", data)
return out, true
}
func attachInlineDataToFunctionResponse(response gjson.Result, images [][]byte) gjson.Result {
if len(images) == 0 {
return response
}
target := []byte(response.Raw)
for _, img := range images {
target, _ = sjson.SetRawBytes(target, "functionResponse.parts.-1", img)
}
return gjson.ParseBytes(target)
}
// collectFunctionResponsesWithSiblingInlineData keeps functionResponse parts and
// moves sibling inline_data/inlineData onto the nearest preceding functionResponse.
// Leading images before the first functionResponse attach to that first response.
func collectFunctionResponsesWithSiblingInlineData(parts gjson.Result) []gjson.Result {
responses := make([]gjson.Result, 0)
leadingImages := make([][]byte, 0)
current := -1
parts.ForEach(func(_, part gjson.Result) bool {
if part.Get("functionResponse").Exists() {
responses = append(responses, part)
current = len(responses) - 1
if len(leadingImages) > 0 {
responses[current] = attachInlineDataToFunctionResponse(responses[current], leadingImages)
leadingImages = nil
}
return true
}
imagePart, ok := normalizeAntigravityInlineDataPart(part)
if !ok {
return true
}
if current >= 0 {
responses[current] = attachInlineDataToFunctionResponse(responses[current], [][]byte{imagePart})
return true
}
leadingImages = append(leadingImages, imagePart)
return true
})
return responses
}
// parseFunctionResponseRaw attempts to normalize a function response part into a JSON object string.
// Falls back to a minimal "functionResponse" object when parsing fails.
// fallbackName is used when the response's own name is empty.
@@ -749,14 +817,8 @@ func fixCLIToolResponse(input []byte) ([]byte, error) {
role := value.Get("role").String()
parts := value.Get("parts")
// Check if this content has function responses
var responsePartsInThisContent []gjson.Result
parts.ForEach(func(_, part gjson.Result) bool {
if part.Get("functionResponse").Exists() {
responsePartsInThisContent = append(responsePartsInThisContent, part)
}
return true
})
// Collect function responses and attach sibling inlineData to the nearest one.
responsePartsInThisContent := collectFunctionResponsesWithSiblingInlineData(parts)
// If this content has function responses, collect them
if len(responsePartsInThisContent) > 0 {

View File

@@ -954,3 +954,190 @@ func TestSanitizeAntigravityClaudeGeminiRequestSignatures_LargeNumberDoesNotHalt
t.Fatalf("expected hidden thoughtSignature to be stripped despite large number, got %s", fcSig.Raw)
}
}
func TestFixCLIToolResponse_AttachesSiblingInlineDataToNearestFunctionResponse(t *testing.T) {
tests := []struct {
name string
parts string
want []struct {
id string
mime string
data string
}
}{
{
name: "snake_case sibling after single response",
parts: `{"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}},` +
`{"inline_data":{"mime_type":"image/png","data":"QUJD"}}`,
want: []struct {
id string
mime string
data string
}{{id: "call_1", mime: "image/png", data: "QUJD"}},
},
{
name: "camelCase sibling after single response",
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` +
`{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`,
want: []struct {
id string
mime string
data string
}{{id: "call_1", mime: "image/webp", data: "NEW"}},
},
{
name: "append sibling onto existing functionResponse.parts",
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1","parts":[{"inlineData":{"mimeType":"image/gif","data":"OLD"}}]}},` +
`{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`,
want: []struct {
id string
mime string
data string
}{
{id: "call_1", mime: "image/gif", data: "OLD"},
},
},
{
name: "interleaved siblings attach to nearest response",
parts: `{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` +
`{"inline_data":{"mime_type":"image/png","data":"AAA"}},` +
`{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}},` +
`{"inline_data":{"mime_type":"image/jpeg","data":"BBB"}}`,
want: []struct {
id string
mime string
data string
}{
{id: "call_a", mime: "image/png", data: "AAA"},
{id: "call_b", mime: "image/jpeg", data: "BBB"},
},
},
{
name: "leading sibling attaches to first response",
parts: `{"inline_data":{"mime_type":"image/png","data":"LEAD"}},` +
`{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` +
`{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}}`,
want: []struct {
id string
mime string
data string
}{
{id: "call_a", mime: "image/png", data: "LEAD"},
},
},
{
name: "missing mimeType defaults to image/png",
parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` +
`{"inlineData":{"data":"QUJD"}}`,
want: []struct {
id string
mime string
data string
}{{id: "call_1", mime: "image/png", data: "QUJD"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
modelParts := `{"functionCall":{"name":"read","id":"call_1"}}`
if tt.name == "interleaved siblings attach to nearest response" || tt.name == "leading sibling attaches to first response" {
modelParts = `{"functionCall":{"name":"read","id":"call_a"}},{"functionCall":{"name":"read","id":"call_b"}}`
}
input := `{"request":{"contents":[` +
`{"role":"model","parts":[` + modelParts + `]},` +
`{"role":"user","parts":[` + tt.parts + `]}` +
`]}}`
result, err := fixCLIToolResponse([]byte(input))
if err != nil {
t.Fatalf("fixCLIToolResponse failed: %v", err)
}
contents := gjson.GetBytes(result, "request.contents").Array()
if len(contents) != 2 {
t.Fatalf("contents = %d, want 2. Output: %s", len(contents), result)
}
funcParts := contents[1].Get("parts").Array()
gotByID := map[string][]gjson.Result{}
for _, part := range funcParts {
fr := part.Get("functionResponse")
gotByID[fr.Get("id").String()] = fr.Get("parts").Array()
}
for _, want := range tt.want {
images := gotByID[want.id]
found := false
for _, img := range images {
if img.Get("inlineData.data").String() == want.data && img.Get("inlineData.mimeType").String() == want.mime {
found = true
break
}
}
if !found {
t.Fatalf("id=%s missing inlineData mime=%s data=%s. Output: %s", want.id, want.mime, want.data, result)
}
}
if tt.name == "interleaved siblings attach to nearest response" {
if len(gotByID["call_a"]) != 1 || len(gotByID["call_b"]) != 1 {
t.Fatalf("nearest attribution failed: A=%d B=%d. Output: %s", len(gotByID["call_a"]), len(gotByID["call_b"]), result)
}
}
if tt.name == "leading sibling attaches to first response" {
if len(gotByID["call_b"]) != 0 {
t.Fatalf("leading image leaked onto call_b. Output: %s", result)
}
}
if tt.name == "append sibling onto existing functionResponse.parts" {
images := gotByID["call_1"]
if len(images) != 2 {
t.Fatalf("existing+sibling parts = %d, want 2. Output: %s", len(images), result)
}
if images[1].Get("inlineData.data").String() != "NEW" {
t.Fatalf("appended sibling data = %q, want NEW. Output: %s", images[1].Get("inlineData.data").String(), result)
}
}
})
}
}
func TestConvertGeminiRequestToAntigravity_PreservesSiblingToolImageOnUserRole(t *testing.T) {
input := []byte(`{
"contents": [
{"role":"user","parts":[{"text":"read file"}]},
{"role":"model","parts":[{"functionCall":{"name":"read","args":{},"id":"call_1"}}]},
{"role":"user","parts":[
{"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}},
{"inline_data":{"mime_type":"image/png","data":"QUJD"}}
]}
]
}`)
out := ConvertGeminiRequestToAntigravity("gemini-3-flash", input, false)
contents := gjson.GetBytes(out, "request.contents").Array()
if len(contents) != 3 {
t.Fatalf("contents = %d, want 3. Output: %s", len(contents), out)
}
funcContent := contents[2]
if got := funcContent.Get("role").String(); got != "user" {
t.Fatalf("role = %q, want user after Antigravity normalization. Output: %s", got, out)
}
funcResp := funcContent.Get("parts.0.functionResponse")
if !funcResp.Exists() {
t.Fatalf("functionResponse missing. Output: %s", out)
}
if got := funcResp.Get("id").String(); got != "call_1" {
t.Fatalf("id = %q, want call_1", got)
}
if got := funcResp.Get("response.result").String(); got != "Read image file [image/png]" {
t.Fatalf("result = %q", got)
}
inlineData := funcResp.Get("parts.0.inlineData")
if !inlineData.Exists() {
t.Fatalf("functionResponse.parts.0.inlineData missing. Output: %s", out)
}
if got := inlineData.Get("mimeType").String(); got != "image/png" {
t.Fatalf("mimeType = %q, want image/png", got)
}
if got := inlineData.Get("data").String(); got != "QUJD" {
t.Fatalf("data = %q, want QUJD", got)
}
if funcContent.Get("parts.1.inline_data").Exists() || funcContent.Get("parts.1.inlineData").Exists() {
t.Fatalf("sibling inline data should be absorbed into functionResponse.parts. Output: %s", out)
}
}

View File

@@ -265,3 +265,93 @@ func TestConvertOpenAIResponsesRequestToAntigravity_GeminiReasoningUsesNativeTho
t.Fatalf("parts[0].thoughtSignature = %q, want preserved Gemini signature. Output: %s", got, out)
}
}
func TestConvertOpenAIResponsesRequestToAntigravity_PreservesToolResultImage(t *testing.T) {
inputJSON := `{
"model": "gemini-3-flash",
"input": [
{"role": "user", "content": [{"type": "input_text", "text": "请帮我读取分析这张图片"}]},
{"type": "function_call", "id": "fc_read", "call_id": "call_read_1", "name": "read", "arguments": "{\"path\":\"/path/to/image.png\"}"},
{
"type": "function_call_output",
"call_id": "call_read_1",
"output": [
{"type": "input_text", "text": "Read image file [image/png]"},
{"type": "input_image", "detail": "auto", "image_url": "data:image/png;base64,QUJD"}
]
}
]
}`
out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false)
contents := gjson.GetBytes(out, "request.contents").Array()
if len(contents) != 3 {
t.Fatalf("expected 3 contents, got %d. Output: %s", len(contents), out)
}
funcContent := contents[2]
if got := funcContent.Get("role").String(); got != "user" {
t.Fatalf("role = %q, want user. Output: %s", got, out)
}
funcResp := funcContent.Get("parts.0.functionResponse")
if !funcResp.Exists() {
t.Fatalf("functionResponse should exist. Output: %s", out)
}
if got := funcResp.Get("id").String(); got != "call_read_1" {
t.Fatalf("id = %q, want call_read_1", got)
}
if got := funcResp.Get("name").String(); got != "read" {
t.Fatalf("name = %q, want read", got)
}
inlineData := funcResp.Get("parts.0.inlineData")
if !inlineData.Exists() {
t.Fatalf("expected functionResponse.parts.0.inlineData to exist, got: %s", out)
}
if got := inlineData.Get("mimeType").String(); got != "image/png" {
t.Errorf("expected mimeType image/png, got %q", got)
}
if got := inlineData.Get("data").String(); got != "QUJD" {
t.Errorf("expected data QUJD, got %q", got)
}
}
func TestConvertOpenAIResponsesRequestToAntigravity_AttachesParallelToolImagesToNearestResponse(t *testing.T) {
inputJSON := `{
"model": "gemini-3-flash",
"input": [
{"role": "user", "content": [{"type": "input_text", "text": "read both"}]},
{"type": "function_call", "id": "fc_a", "call_id": "call_a", "name": "read", "arguments": "{\"path\":\"/tmp/a.png\"}"},
{"type": "function_call", "id": "fc_b", "call_id": "call_b", "name": "read", "arguments": "{\"path\":\"/tmp/b.png\"}"},
{
"type": "function_call_output",
"call_id": "call_a",
"output": [
{"type": "input_text", "text": "file A"},
{"type": "input_image", "image_url": "data:image/png;base64,AAA"}
]
},
{
"type": "function_call_output",
"call_id": "call_b",
"output": [
{"type": "input_text", "text": "file B"},
{"type": "input_image", "image_url": "data:image/jpeg;base64,BBB"}
]
}
]
}`
out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false)
parts := gjson.GetBytes(out, "request.contents.2.parts").Array()
if len(parts) != 2 {
t.Fatalf("function parts = %d, want 2. Output: %s", len(parts), out)
}
got := map[string]string{}
for _, part := range parts {
fr := part.Get("functionResponse")
got[fr.Get("id").String()] = fr.Get("parts.0.inlineData.data").String()
}
if got["call_a"] != "AAA" {
t.Fatalf("call_a image = %q, want AAA. Output: %s", got["call_a"], out)
}
if got["call_b"] != "BBB" {
t.Fatalf("call_b image = %q, want BBB. Output: %s", got["call_b"], out)
}
}