diff --git a/internal/translator/antigravity/gemini/antigravity_gemini_request.go b/internal/translator/antigravity/gemini/antigravity_gemini_request.go index 6664fb34e..1952a60a2 100644 --- a/internal/translator/antigravity/gemini/antigravity_gemini_request.go +++ b/internal/translator/antigravity/gemini/antigravity_gemini_request.go @@ -639,6 +639,74 @@ type FunctionCallGroup struct { CallNames []string // ordered function call names for backfilling empty response names } +func normalizeAntigravityInlineDataPart(part gjson.Result) ([]byte, bool) { + inline := part.Get("inlineData") + if !inline.Exists() { + inline = part.Get("inline_data") + } + if !inline.Exists() { + return nil, false + } + data := inline.Get("data").String() + if data == "" { + return nil, false + } + mimeType := inline.Get("mimeType").String() + if mimeType == "" { + mimeType = inline.Get("mime_type").String() + } + if mimeType == "" { + // Cloud Code Assist ignores inlineData without mimeType. + mimeType = "image/png" + } + out := []byte(`{"inlineData":{"mimeType":"","data":""}}`) + out, _ = sjson.SetBytes(out, "inlineData.mimeType", mimeType) + out, _ = sjson.SetBytes(out, "inlineData.data", data) + return out, true +} + +func attachInlineDataToFunctionResponse(response gjson.Result, images [][]byte) gjson.Result { + if len(images) == 0 { + return response + } + target := []byte(response.Raw) + for _, img := range images { + target, _ = sjson.SetRawBytes(target, "functionResponse.parts.-1", img) + } + return gjson.ParseBytes(target) +} + +// collectFunctionResponsesWithSiblingInlineData keeps functionResponse parts and +// moves sibling inline_data/inlineData onto the nearest preceding functionResponse. +// Leading images before the first functionResponse attach to that first response. +func collectFunctionResponsesWithSiblingInlineData(parts gjson.Result) []gjson.Result { + responses := make([]gjson.Result, 0) + leadingImages := make([][]byte, 0) + current := -1 + parts.ForEach(func(_, part gjson.Result) bool { + if part.Get("functionResponse").Exists() { + responses = append(responses, part) + current = len(responses) - 1 + if len(leadingImages) > 0 { + responses[current] = attachInlineDataToFunctionResponse(responses[current], leadingImages) + leadingImages = nil + } + return true + } + imagePart, ok := normalizeAntigravityInlineDataPart(part) + if !ok { + return true + } + if current >= 0 { + responses[current] = attachInlineDataToFunctionResponse(responses[current], [][]byte{imagePart}) + return true + } + leadingImages = append(leadingImages, imagePart) + return true + }) + return responses +} + // parseFunctionResponseRaw attempts to normalize a function response part into a JSON object string. // Falls back to a minimal "functionResponse" object when parsing fails. // fallbackName is used when the response's own name is empty. @@ -749,14 +817,8 @@ func fixCLIToolResponse(input []byte) ([]byte, error) { role := value.Get("role").String() parts := value.Get("parts") - // Check if this content has function responses - var responsePartsInThisContent []gjson.Result - parts.ForEach(func(_, part gjson.Result) bool { - if part.Get("functionResponse").Exists() { - responsePartsInThisContent = append(responsePartsInThisContent, part) - } - return true - }) + // Collect function responses and attach sibling inlineData to the nearest one. + responsePartsInThisContent := collectFunctionResponsesWithSiblingInlineData(parts) // If this content has function responses, collect them if len(responsePartsInThisContent) > 0 { diff --git a/internal/translator/antigravity/gemini/antigravity_gemini_request_test.go b/internal/translator/antigravity/gemini/antigravity_gemini_request_test.go index 56f8332ae..227087acd 100644 --- a/internal/translator/antigravity/gemini/antigravity_gemini_request_test.go +++ b/internal/translator/antigravity/gemini/antigravity_gemini_request_test.go @@ -954,3 +954,190 @@ func TestSanitizeAntigravityClaudeGeminiRequestSignatures_LargeNumberDoesNotHalt t.Fatalf("expected hidden thoughtSignature to be stripped despite large number, got %s", fcSig.Raw) } } + +func TestFixCLIToolResponse_AttachesSiblingInlineDataToNearestFunctionResponse(t *testing.T) { + tests := []struct { + name string + parts string + want []struct { + id string + mime string + data string + } + }{ + { + name: "snake_case sibling after single response", + parts: `{"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}},` + + `{"inline_data":{"mime_type":"image/png","data":"QUJD"}}`, + want: []struct { + id string + mime string + data string + }{{id: "call_1", mime: "image/png", data: "QUJD"}}, + }, + { + name: "camelCase sibling after single response", + parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` + + `{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`, + want: []struct { + id string + mime string + data string + }{{id: "call_1", mime: "image/webp", data: "NEW"}}, + }, + { + name: "append sibling onto existing functionResponse.parts", + parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1","parts":[{"inlineData":{"mimeType":"image/gif","data":"OLD"}}]}},` + + `{"inlineData":{"mimeType":"image/webp","data":"NEW"}}`, + want: []struct { + id string + mime string + data string + }{ + {id: "call_1", mime: "image/gif", data: "OLD"}, + }, + }, + { + name: "interleaved siblings attach to nearest response", + parts: `{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` + + `{"inline_data":{"mime_type":"image/png","data":"AAA"}},` + + `{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}},` + + `{"inline_data":{"mime_type":"image/jpeg","data":"BBB"}}`, + want: []struct { + id string + mime string + data string + }{ + {id: "call_a", mime: "image/png", data: "AAA"}, + {id: "call_b", mime: "image/jpeg", data: "BBB"}, + }, + }, + { + name: "leading sibling attaches to first response", + parts: `{"inline_data":{"mime_type":"image/png","data":"LEAD"}},` + + `{"functionResponse":{"name":"read","response":{"result":"A"},"id":"call_a"}},` + + `{"functionResponse":{"name":"read","response":{"result":"B"},"id":"call_b"}}`, + want: []struct { + id string + mime string + data string + }{ + {id: "call_a", mime: "image/png", data: "LEAD"}, + }, + }, + { + name: "missing mimeType defaults to image/png", + parts: `{"functionResponse":{"name":"read","response":{"result":"ok"},"id":"call_1"}},` + + `{"inlineData":{"data":"QUJD"}}`, + want: []struct { + id string + mime string + data string + }{{id: "call_1", mime: "image/png", data: "QUJD"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + modelParts := `{"functionCall":{"name":"read","id":"call_1"}}` + if tt.name == "interleaved siblings attach to nearest response" || tt.name == "leading sibling attaches to first response" { + modelParts = `{"functionCall":{"name":"read","id":"call_a"}},{"functionCall":{"name":"read","id":"call_b"}}` + } + input := `{"request":{"contents":[` + + `{"role":"model","parts":[` + modelParts + `]},` + + `{"role":"user","parts":[` + tt.parts + `]}` + + `]}}` + result, err := fixCLIToolResponse([]byte(input)) + if err != nil { + t.Fatalf("fixCLIToolResponse failed: %v", err) + } + contents := gjson.GetBytes(result, "request.contents").Array() + if len(contents) != 2 { + t.Fatalf("contents = %d, want 2. Output: %s", len(contents), result) + } + funcParts := contents[1].Get("parts").Array() + gotByID := map[string][]gjson.Result{} + for _, part := range funcParts { + fr := part.Get("functionResponse") + gotByID[fr.Get("id").String()] = fr.Get("parts").Array() + } + for _, want := range tt.want { + images := gotByID[want.id] + found := false + for _, img := range images { + if img.Get("inlineData.data").String() == want.data && img.Get("inlineData.mimeType").String() == want.mime { + found = true + break + } + } + if !found { + t.Fatalf("id=%s missing inlineData mime=%s data=%s. Output: %s", want.id, want.mime, want.data, result) + } + } + if tt.name == "interleaved siblings attach to nearest response" { + if len(gotByID["call_a"]) != 1 || len(gotByID["call_b"]) != 1 { + t.Fatalf("nearest attribution failed: A=%d B=%d. Output: %s", len(gotByID["call_a"]), len(gotByID["call_b"]), result) + } + } + if tt.name == "leading sibling attaches to first response" { + if len(gotByID["call_b"]) != 0 { + t.Fatalf("leading image leaked onto call_b. Output: %s", result) + } + } + if tt.name == "append sibling onto existing functionResponse.parts" { + images := gotByID["call_1"] + if len(images) != 2 { + t.Fatalf("existing+sibling parts = %d, want 2. Output: %s", len(images), result) + } + if images[1].Get("inlineData.data").String() != "NEW" { + t.Fatalf("appended sibling data = %q, want NEW. Output: %s", images[1].Get("inlineData.data").String(), result) + } + } + }) + } +} + +func TestConvertGeminiRequestToAntigravity_PreservesSiblingToolImageOnUserRole(t *testing.T) { + input := []byte(`{ + "contents": [ + {"role":"user","parts":[{"text":"read file"}]}, + {"role":"model","parts":[{"functionCall":{"name":"read","args":{},"id":"call_1"}}]}, + {"role":"user","parts":[ + {"functionResponse":{"name":"read","response":{"result":"Read image file [image/png]"},"id":"call_1"}}, + {"inline_data":{"mime_type":"image/png","data":"QUJD"}} + ]} + ] + }`) + out := ConvertGeminiRequestToAntigravity("gemini-3-flash", input, false) + contents := gjson.GetBytes(out, "request.contents").Array() + if len(contents) != 3 { + t.Fatalf("contents = %d, want 3. Output: %s", len(contents), out) + } + funcContent := contents[2] + if got := funcContent.Get("role").String(); got != "user" { + t.Fatalf("role = %q, want user after Antigravity normalization. Output: %s", got, out) + } + funcResp := funcContent.Get("parts.0.functionResponse") + if !funcResp.Exists() { + t.Fatalf("functionResponse missing. Output: %s", out) + } + if got := funcResp.Get("id").String(); got != "call_1" { + t.Fatalf("id = %q, want call_1", got) + } + if got := funcResp.Get("response.result").String(); got != "Read image file [image/png]" { + t.Fatalf("result = %q", got) + } + inlineData := funcResp.Get("parts.0.inlineData") + if !inlineData.Exists() { + t.Fatalf("functionResponse.parts.0.inlineData missing. Output: %s", out) + } + if got := inlineData.Get("mimeType").String(); got != "image/png" { + t.Fatalf("mimeType = %q, want image/png", got) + } + if got := inlineData.Get("data").String(); got != "QUJD" { + t.Fatalf("data = %q, want QUJD", got) + } + if funcContent.Get("parts.1.inline_data").Exists() || funcContent.Get("parts.1.inlineData").Exists() { + t.Fatalf("sibling inline data should be absorbed into functionResponse.parts. Output: %s", out) + } +} diff --git a/internal/translator/antigravity/openai/responses/antigravity_openai-responses_request_test.go b/internal/translator/antigravity/openai/responses/antigravity_openai-responses_request_test.go index 7dfad0eb7..f283ceba9 100644 --- a/internal/translator/antigravity/openai/responses/antigravity_openai-responses_request_test.go +++ b/internal/translator/antigravity/openai/responses/antigravity_openai-responses_request_test.go @@ -265,3 +265,93 @@ func TestConvertOpenAIResponsesRequestToAntigravity_GeminiReasoningUsesNativeTho t.Fatalf("parts[0].thoughtSignature = %q, want preserved Gemini signature. Output: %s", got, out) } } + +func TestConvertOpenAIResponsesRequestToAntigravity_PreservesToolResultImage(t *testing.T) { + inputJSON := `{ + "model": "gemini-3-flash", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "请帮我读取分析这张图片"}]}, + {"type": "function_call", "id": "fc_read", "call_id": "call_read_1", "name": "read", "arguments": "{\"path\":\"/path/to/image.png\"}"}, + { + "type": "function_call_output", + "call_id": "call_read_1", + "output": [ + {"type": "input_text", "text": "Read image file [image/png]"}, + {"type": "input_image", "detail": "auto", "image_url": "data:image/png;base64,QUJD"} + ] + } + ] + }` + out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false) + contents := gjson.GetBytes(out, "request.contents").Array() + if len(contents) != 3 { + t.Fatalf("expected 3 contents, got %d. Output: %s", len(contents), out) + } + funcContent := contents[2] + if got := funcContent.Get("role").String(); got != "user" { + t.Fatalf("role = %q, want user. Output: %s", got, out) + } + funcResp := funcContent.Get("parts.0.functionResponse") + if !funcResp.Exists() { + t.Fatalf("functionResponse should exist. Output: %s", out) + } + if got := funcResp.Get("id").String(); got != "call_read_1" { + t.Fatalf("id = %q, want call_read_1", got) + } + if got := funcResp.Get("name").String(); got != "read" { + t.Fatalf("name = %q, want read", got) + } + inlineData := funcResp.Get("parts.0.inlineData") + if !inlineData.Exists() { + t.Fatalf("expected functionResponse.parts.0.inlineData to exist, got: %s", out) + } + if got := inlineData.Get("mimeType").String(); got != "image/png" { + t.Errorf("expected mimeType image/png, got %q", got) + } + if got := inlineData.Get("data").String(); got != "QUJD" { + t.Errorf("expected data QUJD, got %q", got) + } +} + +func TestConvertOpenAIResponsesRequestToAntigravity_AttachesParallelToolImagesToNearestResponse(t *testing.T) { + inputJSON := `{ + "model": "gemini-3-flash", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "read both"}]}, + {"type": "function_call", "id": "fc_a", "call_id": "call_a", "name": "read", "arguments": "{\"path\":\"/tmp/a.png\"}"}, + {"type": "function_call", "id": "fc_b", "call_id": "call_b", "name": "read", "arguments": "{\"path\":\"/tmp/b.png\"}"}, + { + "type": "function_call_output", + "call_id": "call_a", + "output": [ + {"type": "input_text", "text": "file A"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAA"} + ] + }, + { + "type": "function_call_output", + "call_id": "call_b", + "output": [ + {"type": "input_text", "text": "file B"}, + {"type": "input_image", "image_url": "data:image/jpeg;base64,BBB"} + ] + } + ] + }` + out := ConvertOpenAIResponsesRequestToAntigravity("gemini-3-flash", []byte(inputJSON), false) + parts := gjson.GetBytes(out, "request.contents.2.parts").Array() + if len(parts) != 2 { + t.Fatalf("function parts = %d, want 2. Output: %s", len(parts), out) + } + got := map[string]string{} + for _, part := range parts { + fr := part.Get("functionResponse") + got[fr.Get("id").String()] = fr.Get("parts.0.inlineData.data").String() + } + if got["call_a"] != "AAA" { + t.Fatalf("call_a image = %q, want AAA. Output: %s", got["call_a"], out) + } + if got["call_b"] != "BBB" { + t.Fatalf("call_b image = %q, want BBB. Output: %s", got["call_b"], out) + } +}