diff --git a/internal/translator/openai/openai/responses/openai_openai-responses_request.go b/internal/translator/openai/openai/responses/openai_openai-responses_request.go index 370933401..4522b5ced 100644 --- a/internal/translator/openai/openai/responses/openai_openai-responses_request.go +++ b/internal/translator/openai/openai/responses/openai_openai-responses_request.go @@ -176,8 +176,8 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu imageURL := contentItem.Get("image_url").String() contentPart := []byte(`{"type":"image_url","image_url":{"url":""}}`) contentPart, _ = sjson.SetBytes(contentPart, "image_url.url", imageURL) - if detail := contentItem.Get("detail"); detail.Exists() { - contentPart, _ = sjson.SetBytes(contentPart, "image_url.detail", detail.String()) + if detail, ok := normalizeChatImageDetail(contentItem.Get("detail")); ok && detail != "" { + contentPart, _ = sjson.SetBytes(contentPart, "image_url.detail", detail) } contentItems = append(contentItems, contentPart) } @@ -245,7 +245,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu } if output := item.Get("output"); output.Exists() { - toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String()) + toolMessage = setFunctionCallOutputContent(toolMessage, output) } appendMessage(toolMessage) @@ -344,6 +344,136 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu return out } +func setFunctionCallOutputContent(toolMessage []byte, output gjson.Result) []byte { + structuredContent := output + if output.Type == gjson.String { + if !gjson.Valid(output.String()) { + toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String()) + return toolMessage + } + structuredContent = gjson.Parse(output.String()) + } + + if hasChatToolOutputImagePart(structuredContent) { + contentItems := make([][]byte, 0, len(structuredContent.Array())) + for _, item := range structuredContent.Array() { + contentItems = append(contentItems, chatToolOutputContentPart(item)) + } + return translatorcommon.SetRawArrayItems(toolMessage, "content", contentItems) + } + + toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String()) + return toolMessage +} + +func chatToolOutputContentPart(item gjson.Result) []byte { + itemType := item.Get("type").String() + switch itemType { + case "text", "input_text", "output_text": + part := []byte(`{"type":"text","text":""}`) + part, _ = sjson.SetBytes(part, "text", item.Get("text").String()) + return part + case "image_url", "input_image": + imageURL, detail, ok := chatToolOutputImageFields(item) + if !ok { + return chatToolOutputFallbackPart(item) + } + part := []byte(`{"type":"image_url","image_url":{"url":""}}`) + part, _ = sjson.SetBytes(part, "image_url.url", imageURL) + if detail != "" { + part, _ = sjson.SetBytes(part, "image_url.detail", detail) + } + return part + default: + return chatToolOutputFallbackPart(item) + } +} + +func hasChatToolOutputImagePart(content gjson.Result) bool { + if !content.IsArray() { + return false + } + + hasImage := false + for _, item := range content.Array() { + itemType := item.Get("type") + if itemType.Type != gjson.String { + continue + } + switch itemType.String() { + case "text", "input_text", "output_text": + if item.Get("text").Type != gjson.String { + return false + } + case "image_url", "input_image": + if _, _, ok := chatToolOutputImageFields(item); !ok { + return false + } + hasImage = true + } + } + return hasImage +} + +func chatToolOutputImageFields(item gjson.Result) (imageURL, detail string, ok bool) { + var imageURLValue gjson.Result + var detailValue gjson.Result + switch item.Get("type").String() { + case "image_url": + imageURLValue = item.Get("image_url.url") + detailValue = item.Get("image_url.detail") + case "input_image": + imageURLValue = item.Get("image_url") + detailValue = item.Get("detail") + default: + return "", "", false + } + + if imageURLValue.Type != gjson.String { + return "", "", false + } + imageURL = strings.TrimSpace(imageURLValue.String()) + if imageURL == "" { + return "", "", false + } + + detail, ok = normalizeChatImageDetail(detailValue) + if !ok { + return "", "", false + } + return imageURL, detail, true +} + +func normalizeChatImageDetail(detailValue gjson.Result) (string, bool) { + if !detailValue.Exists() { + return "", true + } + if detailValue.Type != gjson.String { + return "", false + } + + normalizedDetail := strings.ToLower(strings.TrimSpace(detailValue.String())) + switch normalizedDetail { + case "auto", "low", "high": + return normalizedDetail, true + case "original": + // Chat Completions does not support Codex's original detail value. + return "high", true + default: + return "", true + } +} + +func chatToolOutputFallbackPart(item gjson.Result) []byte { + text := item.Raw + if item.Type == gjson.String || text == "" { + text = item.String() + } + part := []byte(`{"type":"text","text":""}`) + part, _ = sjson.SetBytes(part, "text", text) + return part +} + func collectOpenAIResponsesReasoningContent(item gjson.Result) string { var reasoningText strings.Builder if summary := item.Get("summary"); summary.Exists() && summary.IsArray() { diff --git a/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go b/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go index c8c0342f2..c94ced4f3 100644 --- a/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go +++ b/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go @@ -3,6 +3,7 @@ package responses import ( "bytes" "encoding/json" + "fmt" "testing" "github.com/tidwall/gjson" @@ -123,6 +124,140 @@ func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_DefersMessageUntil } } +func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_UnwrapsStringifiedToolOutputImages(t *testing.T) { + tests := []struct { + name string + output string + imageIndex int + expectedURL string + expectedText string + detail string + }{ + { + name: "Codex input image", + output: `[{"type":"input_text","text":"Captured screenshot."},{"detail":"original","image_url":"data:image/png;base64,AA==","type":"input_image"}]`, + imageIndex: 1, + expectedURL: "data:image/png;base64,AA==", + expectedText: "Captured screenshot.", + detail: "high", + }, + { + name: "OpenAI image URL", + output: `[{"type":"image_url","image_url":{"url":"https://example.com/generated.png","detail":"high"}}]`, + imageIndex: 0, + expectedURL: "https://example.com/generated.png", + detail: "high", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + raw := []byte(fmt.Sprintf(`{ + "input": [ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_image","output":%q} + ] + }`, tt.output)) + + out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false) + content := gjson.GetBytes(out, "messages.1.content") + if !content.IsArray() { + t.Fatalf("expected tool content array, got %s; output=%s", content.Raw, out) + } + parts := content.Array() + if len(parts) <= tt.imageIndex { + t.Fatalf("expected image part at index %d, got %s", tt.imageIndex, content.Raw) + } + imagePart := parts[tt.imageIndex] + if got := imagePart.Get("type").String(); got != "image_url" { + t.Fatalf("image type = %q, want image_url; part=%s", got, imagePart.Raw) + } + if got := imagePart.Get("image_url.url").String(); got != tt.expectedURL { + t.Fatalf("image URL = %q, want %q; part=%s", got, tt.expectedURL, imagePart.Raw) + } + if got := imagePart.Get("image_url.detail").String(); got != tt.detail { + t.Fatalf("image detail = %q, want %q; part=%s", got, tt.detail, imagePart.Raw) + } + if tt.expectedText != "" { + if got := parts[0].Get("type").String(); got != "text" { + t.Fatalf("text type = %q, want text; part=%s", got, parts[0].Raw) + } + if got := parts[0].Get("text").String(); got != tt.expectedText { + t.Fatalf("text = %q, want %q; part=%s", got, tt.expectedText, parts[0].Raw) + } + } + }) + } +} + +func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_ConvertsStructuredToolOutputImages(t *testing.T) { + raw := []byte(`{ + "input": [ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + { + "type":"function_call_output", + "call_id":"call_image", + "output":[ + {"type":"input_text","text":"Captured screenshot."}, + {"type":"input_image","image_url":"data:image/png;base64,AA==","detail":"original"} + ] + } + ] + }`) + + out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false) + content := gjson.GetBytes(out, "messages.1.content") + if !content.IsArray() { + t.Fatalf("expected tool content array, got %s; output=%s", content.Raw, out) + } + if got := content.Get("1.type").String(); got != "image_url" { + t.Fatalf("image type = %q, want image_url; output=%s", got, out) + } + if got := content.Get("1.image_url.url").String(); got != "data:image/png;base64,AA==" { + t.Fatalf("image URL = %q, want data URL; output=%s", got, out) + } + if got := content.Get("1.image_url.detail").String(); got != "high" { + t.Fatalf("image detail = %q, want high; output=%s", got, out) + } +} + +func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_KeepsNonImageToolOutputStrings(t *testing.T) { + tests := []struct { + name string + output string + }{ + {name: "plain text", output: "plain output"}, + {name: "JSON object", output: `{"status":"ok"}`}, + {name: "text-only array", output: `[{"type":"input_text","text":"still text"}]`}, + {name: "invalid image array", output: `[{"type":"input_image","detail":"low"}]`}, + {name: "image array with trailing text", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA=="}] trailing`}, + {name: "truncated image array", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA=="}`}, + {name: "non-string image URL", output: `[{"type":"input_image","image_url":123}]`}, + {name: "non-string image detail", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA==","detail":123}]`}, + {name: "non-string text in image array", output: `[{"type":"input_text","text":123},{"type":"input_image","image_url":"data:image/png;base64,AA=="}]`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + raw := []byte(fmt.Sprintf(`{ + "input": [ + {"type":"function_call","call_id":"call_output","name":"inspect","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_output","output":%q} + ] + }`, tt.output)) + + out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false) + content := gjson.GetBytes(out, "messages.1.content") + if content.Type != gjson.String { + t.Fatalf("expected tool content string, got %s; output=%s", content.Raw, out) + } + if got := content.String(); got != tt.output { + t.Fatalf("tool content = %q, want %q; output=%s", got, tt.output, out) + } + }) + } +} + func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_AttachesReasoningToAssistantMessage(t *testing.T) { raw := []byte(`{ "input": [ @@ -446,30 +581,49 @@ func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_PreservesParallelT } } -func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_PreservesInputImageDetail(t *testing.T) { - raw := []byte(`{ - "input": [ - { - "role": "user", - "content": [ +func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_NormalizesInputImageDetail(t *testing.T) { + tests := []struct { + name string + detailJSON string + expectedDetail string + }{ + {name: "standard high", detailJSON: `"high"`, expectedDetail: "high"}, + {name: "Codex original", detailJSON: `"original"`, expectedDetail: "high"}, + {name: "unsupported value", detailJSON: `"medium"`}, + {name: "non-string value", detailJSON: `123`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + raw := []byte(fmt.Sprintf(`{ + "input": [ { - "type": "input_image", - "image_url": "https://example.com/image.png", - "detail": "high" + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": "https://example.com/image.png", + "detail": %s + } + ] } ] + }`, tt.detailJSON)) + + out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("gpt-5.4", raw, false) + if got := gjson.GetBytes(out, "messages.0.content.0.image_url.url").String(); got != "https://example.com/image.png" { + t.Fatalf("image URL = %q, want https://example.com/image.png; output=%s", got, out) } - ] - }`) - t.Logf("input json:\n%s", prettyJSONForTest(raw)) - - out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("gpt-5.4", raw, false) - t.Logf("output json:\n%s", prettyJSONForTest(out)) - - if got := gjson.GetBytes(out, "messages.0.content.0.image_url.url").String(); got != "https://example.com/image.png" { - t.Fatalf("messages.0.content.0.image_url.url = %q, want https://example.com/image.png; output=%s", got, out) - } - if got := gjson.GetBytes(out, "messages.0.content.0.image_url.detail").String(); got != "high" { - t.Fatalf("messages.0.content.0.image_url.detail = %q, want high; output=%s", got, out) + detail := gjson.GetBytes(out, "messages.0.content.0.image_url.detail") + if tt.expectedDetail == "" { + if detail.Exists() { + t.Fatalf("image detail should be omitted, got %q; output=%s", detail.String(), out) + } + return + } + if got := detail.String(); got != tt.expectedDetail { + t.Fatalf("image detail = %q, want %q; output=%s", got, tt.expectedDetail, out) + } + }) } }