feat(translator): enhance OpenAI response handling for structured tool outputs

- Added logic to parse and normalize structured tool outputs, including text and image content. - Introduced `setFunctionCallOutputContent` and related helper methods for consistent response processing. - Enhanced handling of image details with normalization (`normalizeChatImageDetail`) and fallback mechanisms. - Updated tests to cover various tool output scenarios, ensuring robustness and accuracy in conversions. Closes: #4699
2026-09-09 17:42:08 +08:00 · 2026-08-01 02:08:52 +08:00
parent 198a26737c
commit 4d498ec7f7
2 changed files with 308 additions and 24 deletions
--- a/internal/translator/openai/openai/responses/openai_openai-responses_request.go
+++ b/internal/translator/openai/openai/responses/openai_openai-responses_request.go
@@ -176,8 +176,8 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu
 							imageURL := contentItem.Get("image_url").String()
 							contentPart := []byte(`{"type":"image_url","image_url":{"url":""}}`)
 							contentPart, _ = sjson.SetBytes(contentPart, "image_url.url", imageURL)
-							if detail := contentItem.Get("detail"); detail.Exists() {
-								contentPart, _ = sjson.SetBytes(contentPart, "image_url.detail", detail.String())
+							if detail, ok := normalizeChatImageDetail(contentItem.Get("detail")); ok && detail != "" {
+								contentPart, _ = sjson.SetBytes(contentPart, "image_url.detail", detail)
 							}
 							contentItems = append(contentItems, contentPart)
 						}
@@ -245,7 +245,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu
 				}

 				if output := item.Get("output"); output.Exists() {
-					toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String())
+					toolMessage = setFunctionCallOutputContent(toolMessage, output)
 				}

 				appendMessage(toolMessage)
@@ -344,6 +344,136 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu
 	return out
 }

+func setFunctionCallOutputContent(toolMessage []byte, output gjson.Result) []byte {
+	structuredContent := output
+	if output.Type == gjson.String {
+		if !gjson.Valid(output.String()) {
+			toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String())
+			return toolMessage
+		}
+		structuredContent = gjson.Parse(output.String())
+	}
+
+	if hasChatToolOutputImagePart(structuredContent) {
+		contentItems := make([][]byte, 0, len(structuredContent.Array()))
+		for _, item := range structuredContent.Array() {
+			contentItems = append(contentItems, chatToolOutputContentPart(item))
+		}
+		return translatorcommon.SetRawArrayItems(toolMessage, "content", contentItems)
+	}
+
+	toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String())
+	return toolMessage
+}
+
+func chatToolOutputContentPart(item gjson.Result) []byte {
+	itemType := item.Get("type").String()
+	switch itemType {
+	case "text", "input_text", "output_text":
+		part := []byte(`{"type":"text","text":""}`)
+		part, _ = sjson.SetBytes(part, "text", item.Get("text").String())
+		return part
+	case "image_url", "input_image":
+		imageURL, detail, ok := chatToolOutputImageFields(item)
+		if !ok {
+			return chatToolOutputFallbackPart(item)
+		}
+		part := []byte(`{"type":"image_url","image_url":{"url":""}}`)
+		part, _ = sjson.SetBytes(part, "image_url.url", imageURL)
+		if detail != "" {
+			part, _ = sjson.SetBytes(part, "image_url.detail", detail)
+		}
+		return part
+	default:
+		return chatToolOutputFallbackPart(item)
+	}
+}
+
+func hasChatToolOutputImagePart(content gjson.Result) bool {
+	if !content.IsArray() {
+		return false
+	}
+
+	hasImage := false
+	for _, item := range content.Array() {
+		itemType := item.Get("type")
+		if itemType.Type != gjson.String {
+			continue
+		}
+		switch itemType.String() {
+		case "text", "input_text", "output_text":
+			if item.Get("text").Type != gjson.String {
+				return false
+			}
+		case "image_url", "input_image":
+			if _, _, ok := chatToolOutputImageFields(item); !ok {
+				return false
+			}
+			hasImage = true
+		}
+	}
+	return hasImage
+}
+
+func chatToolOutputImageFields(item gjson.Result) (imageURL, detail string, ok bool) {
+	var imageURLValue gjson.Result
+	var detailValue gjson.Result
+	switch item.Get("type").String() {
+	case "image_url":
+		imageURLValue = item.Get("image_url.url")
+		detailValue = item.Get("image_url.detail")
+	case "input_image":
+		imageURLValue = item.Get("image_url")
+		detailValue = item.Get("detail")
+	default:
+		return "", "", false
+	}
+
+	if imageURLValue.Type != gjson.String {
+		return "", "", false
+	}
+	imageURL = strings.TrimSpace(imageURLValue.String())
+	if imageURL == "" {
+		return "", "", false
+	}
+
+	detail, ok = normalizeChatImageDetail(detailValue)
+	if !ok {
+		return "", "", false
+	}
+	return imageURL, detail, true
+}
+
+func normalizeChatImageDetail(detailValue gjson.Result) (string, bool) {
+	if !detailValue.Exists() {
+		return "", true
+	}
+	if detailValue.Type != gjson.String {
+		return "", false
+	}
+
+	normalizedDetail := strings.ToLower(strings.TrimSpace(detailValue.String()))
+	switch normalizedDetail {
+	case "auto", "low", "high":
+		return normalizedDetail, true
+	case "original":
+		// Chat Completions does not support Codex's original detail value.
+		return "high", true
+	default:
+		return "", true
+	}
+}
+
+func chatToolOutputFallbackPart(item gjson.Result) []byte {
+	text := item.Raw
+	if item.Type == gjson.String || text == "" {
+		text = item.String()
+	}
+	part := []byte(`{"type":"text","text":""}`)
+	part, _ = sjson.SetBytes(part, "text", text)
+	return part
+}
+
 func collectOpenAIResponsesReasoningContent(item gjson.Result) string {
 	var reasoningText strings.Builder
 	if summary := item.Get("summary"); summary.Exists() && summary.IsArray() {
--- a/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go
+++ b/internal/translator/openai/openai/responses/openai_openai-responses_request_test.go
@@ -3,6 +3,7 @@ package responses
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"testing"

 	"github.com/tidwall/gjson"
@@ -123,6 +124,140 @@ func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_DefersMessageUntil
 	}
 }

+func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_UnwrapsStringifiedToolOutputImages(t *testing.T) {
+	tests := []struct {
+		name         string
+		output       string
+		imageIndex   int
+		expectedURL  string
+		expectedText string
+		detail       string
+	}{
+		{
+			name:         "Codex input image",
+			output:       `[{"type":"input_text","text":"Captured screenshot."},{"detail":"original","image_url":"data:image/png;base64,AA==","type":"input_image"}]`,
+			imageIndex:   1,
+			expectedURL:  "data:image/png;base64,AA==",
+			expectedText: "Captured screenshot.",
+			detail:       "high",
+		},
+		{
+			name:        "OpenAI image URL",
+			output:      `[{"type":"image_url","image_url":{"url":"https://example.com/generated.png","detail":"high"}}]`,
+			imageIndex:  0,
+			expectedURL: "https://example.com/generated.png",
+			detail:      "high",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			raw := []byte(fmt.Sprintf(`{
+				"input": [
+					{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
+					{"type":"function_call_output","call_id":"call_image","output":%q}
+				]
+			}`, tt.output))
+
+			out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false)
+			content := gjson.GetBytes(out, "messages.1.content")
+			if !content.IsArray() {
+				t.Fatalf("expected tool content array, got %s; output=%s", content.Raw, out)
+			}
+			parts := content.Array()
+			if len(parts) <= tt.imageIndex {
+				t.Fatalf("expected image part at index %d, got %s", tt.imageIndex, content.Raw)
+			}
+			imagePart := parts[tt.imageIndex]
+			if got := imagePart.Get("type").String(); got != "image_url" {
+				t.Fatalf("image type = %q, want image_url; part=%s", got, imagePart.Raw)
+			}
+			if got := imagePart.Get("image_url.url").String(); got != tt.expectedURL {
+				t.Fatalf("image URL = %q, want %q; part=%s", got, tt.expectedURL, imagePart.Raw)
+			}
+			if got := imagePart.Get("image_url.detail").String(); got != tt.detail {
+				t.Fatalf("image detail = %q, want %q; part=%s", got, tt.detail, imagePart.Raw)
+			}
+			if tt.expectedText != "" {
+				if got := parts[0].Get("type").String(); got != "text" {
+					t.Fatalf("text type = %q, want text; part=%s", got, parts[0].Raw)
+				}
+				if got := parts[0].Get("text").String(); got != tt.expectedText {
+					t.Fatalf("text = %q, want %q; part=%s", got, tt.expectedText, parts[0].Raw)
+				}
+			}
+		})
+	}
+}
+
+func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_ConvertsStructuredToolOutputImages(t *testing.T) {
+	raw := []byte(`{
+		"input": [
+			{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
+			{
+				"type":"function_call_output",
+				"call_id":"call_image",
+				"output":[
+					{"type":"input_text","text":"Captured screenshot."},
+					{"type":"input_image","image_url":"data:image/png;base64,AA==","detail":"original"}
+				]
+			}
+		]
+	}`)
+
+	out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false)
+	content := gjson.GetBytes(out, "messages.1.content")
+	if !content.IsArray() {
+		t.Fatalf("expected tool content array, got %s; output=%s", content.Raw, out)
+	}
+	if got := content.Get("1.type").String(); got != "image_url" {
+		t.Fatalf("image type = %q, want image_url; output=%s", got, out)
+	}
+	if got := content.Get("1.image_url.url").String(); got != "data:image/png;base64,AA==" {
+		t.Fatalf("image URL = %q, want data URL; output=%s", got, out)
+	}
+	if got := content.Get("1.image_url.detail").String(); got != "high" {
+		t.Fatalf("image detail = %q, want high; output=%s", got, out)
+	}
+}
+
+func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_KeepsNonImageToolOutputStrings(t *testing.T) {
+	tests := []struct {
+		name   string
+		output string
+	}{
+		{name: "plain text", output: "plain output"},
+		{name: "JSON object", output: `{"status":"ok"}`},
+		{name: "text-only array", output: `[{"type":"input_text","text":"still text"}]`},
+		{name: "invalid image array", output: `[{"type":"input_image","detail":"low"}]`},
+		{name: "image array with trailing text", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA=="}] trailing`},
+		{name: "truncated image array", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA=="}`},
+		{name: "non-string image URL", output: `[{"type":"input_image","image_url":123}]`},
+		{name: "non-string image detail", output: `[{"type":"input_image","image_url":"data:image/png;base64,AA==","detail":123}]`},
+		{name: "non-string text in image array", output: `[{"type":"input_text","text":123},{"type":"input_image","image_url":"data:image/png;base64,AA=="}]`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			raw := []byte(fmt.Sprintf(`{
+				"input": [
+					{"type":"function_call","call_id":"call_output","name":"inspect","arguments":"{}"},
+					{"type":"function_call_output","call_id":"call_output","output":%q}
+				]
+			}`, tt.output))
+
+			out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("k3", raw, false)
+			content := gjson.GetBytes(out, "messages.1.content")
+			if content.Type != gjson.String {
+				t.Fatalf("expected tool content string, got %s; output=%s", content.Raw, out)
+			}
+			if got := content.String(); got != tt.output {
+				t.Fatalf("tool content = %q, want %q; output=%s", got, tt.output, out)
+			}
+		})
+	}
+}
+
 func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_AttachesReasoningToAssistantMessage(t *testing.T) {
 	raw := []byte(`{
 		"input": [
@@ -446,30 +581,49 @@ func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_PreservesParallelT
 	}
 }

-func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_PreservesInputImageDetail(t *testing.T) {
-	raw := []byte(`{
-		"input": [
-			{
-				"role": "user",
-				"content": [
+func TestConvertOpenAIResponsesRequestToOpenAIChatCompletions_NormalizesInputImageDetail(t *testing.T) {
+	tests := []struct {
+		name           string
+		detailJSON     string
+		expectedDetail string
+	}{
+		{name: "standard high", detailJSON: `"high"`, expectedDetail: "high"},
+		{name: "Codex original", detailJSON: `"original"`, expectedDetail: "high"},
+		{name: "unsupported value", detailJSON: `"medium"`},
+		{name: "non-string value", detailJSON: `123`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			raw := []byte(fmt.Sprintf(`{
+				"input": [
 					{
-						"type": "input_image",
-						"image_url": "https://example.com/image.png",
-						"detail": "high"
+						"role": "user",
+						"content": [
+							{
+								"type": "input_image",
+								"image_url": "https://example.com/image.png",
+								"detail": %s
+							}
+						]
 					}
 				]
+			}`, tt.detailJSON))
+
+			out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("gpt-5.4", raw, false)
+			if got := gjson.GetBytes(out, "messages.0.content.0.image_url.url").String(); got != "https://example.com/image.png" {
+				t.Fatalf("image URL = %q, want https://example.com/image.png; output=%s", got, out)
 			}
-		]
-	}`)
-	t.Logf("input json:\n%s", prettyJSONForTest(raw))
-
-	out := ConvertOpenAIResponsesRequestToOpenAIChatCompletions("gpt-5.4", raw, false)
-	t.Logf("output json:\n%s", prettyJSONForTest(out))
-
-	if got := gjson.GetBytes(out, "messages.0.content.0.image_url.url").String(); got != "https://example.com/image.png" {
-		t.Fatalf("messages.0.content.0.image_url.url = %q, want https://example.com/image.png; output=%s", got, out)
-	}
-	if got := gjson.GetBytes(out, "messages.0.content.0.image_url.detail").String(); got != "high" {
-		t.Fatalf("messages.0.content.0.image_url.detail = %q, want high; output=%s", got, out)
+			detail := gjson.GetBytes(out, "messages.0.content.0.image_url.detail")
+			if tt.expectedDetail == "" {
+				if detail.Exists() {
+					t.Fatalf("image detail should be omitted, got %q; output=%s", detail.String(), out)
+				}
+				return
+			}
+			if got := detail.String(); got != tt.expectedDetail {
+				t.Fatalf("image detail = %q, want %q; output=%s", got, tt.expectedDetail, out)
+			}
+		})
 	}
 }