feat(translator): enhance Claude tool response handling with structured content parsing

- Added `applyResponsesToolResultContent` to process and structure tool response content. - Introduced conversion logic for text, image, and file parts in tool responses. - Updated tests to validate correct handling of data URL images in tool response outputs. Closes: #4116
2026-09-26 19:01:37 +08:00 · 2026-07-07 15:38:35 +08:00
parent ab6ed392f2
commit dc77bf4db3
2 changed files with 155 additions and 2 deletions
--- a/internal/translator/claude/openai/responses/claude_openai-responses_request.go
+++ b/internal/translator/claude/openai/responses/claude_openai-responses_request.go
@@ -403,10 +403,10 @@ func ConvertOpenAIResponsesRequestToClaude(modelName string, inputRawJSON []byte
 				callID := item.Get("call_id").String()
 				callID = util.SanitizeClaudeToolID(callID)
 				flushPendingToolUseFor(callID)
-				outputStr := item.Get("output").String()
+				output := item.Get("output")
 				toolResult := []byte(`{"type":"tool_result","tool_use_id":"","content":""}`)
 				toolResult, _ = sjson.SetBytes(toolResult, "tool_use_id", callID)
-				toolResult, _ = sjson.SetBytes(toolResult, "content", outputStr)
+				toolResult = applyResponsesToolResultContent(toolResult, output)

 				usr := []byte(`{"role":"user","content":[]}`)
 				usr, _ = sjson.SetRawBytes(usr, "content.-1", toolResult)
@@ -505,6 +505,111 @@ func responsesReasoningSummaryText(item gjson.Result) string {
 	return builder.String()
 }

+func applyResponsesToolResultContent(toolResult []byte, output gjson.Result) []byte {
+	if output.Exists() && output.IsArray() {
+		var partsJSON []string
+		hasImage := false
+		hasFile := false
+		output.ForEach(func(_, part gjson.Result) bool {
+			if partJSON := convertResponsesContentPartToClaude(part); len(partJSON) > 0 {
+				partsJSON = append(partsJSON, string(partJSON))
+				partType := gjson.ParseBytes(partJSON).Get("type").String()
+				if partType == "image" {
+					hasImage = true
+				}
+				if partType == "document" {
+					hasFile = true
+				}
+			}
+			return true
+		})
+		if len(partsJSON) == 0 {
+			toolResult, _ = sjson.SetBytes(toolResult, "content", output.Raw)
+			return toolResult
+		}
+		if len(partsJSON) == 1 && !hasImage && !hasFile {
+			textPart := gjson.Parse(partsJSON[0])
+			if textPart.Get("type").String() == "text" {
+				toolResult, _ = sjson.SetBytes(toolResult, "content", textPart.Get("text").String())
+				return toolResult
+			}
+		}
+		contentJSON := []byte("[]")
+		for _, partJSON := range partsJSON {
+			contentJSON, _ = sjson.SetRawBytes(contentJSON, "-1", []byte(partJSON))
+		}
+		toolResult, _ = sjson.DeleteBytes(toolResult, "content")
+		toolResult, _ = sjson.SetRawBytes(toolResult, "content", contentJSON)
+		return toolResult
+	}
+	toolResult, _ = sjson.SetBytes(toolResult, "content", output.String())
+	return toolResult
+}
+
+func convertResponsesContentPartToClaude(part gjson.Result) []byte {
+	ptype := part.Get("type").String()
+	switch ptype {
+	case "input_text", "output_text":
+		if t := part.Get("text"); t.Exists() {
+			contentPart := []byte(`{"type":"text","text":""}`)
+			contentPart, _ = sjson.SetBytes(contentPart, "text", t.String())
+			return contentPart
+		}
+	case "input_image":
+		url := part.Get("image_url").String()
+		if url == "" {
+			url = part.Get("url").String()
+		}
+		if url == "" {
+			return nil
+		}
+		if strings.HasPrefix(url, "data:") {
+			trimmed := strings.TrimPrefix(url, "data:")
+			mediaAndData := strings.SplitN(trimmed, ";base64,", 2)
+			mediaType := "application/octet-stream"
+			data := ""
+			if len(mediaAndData) == 2 {
+				if mediaAndData[0] != "" {
+					mediaType = mediaAndData[0]
+				}
+				data = mediaAndData[1]
+			}
+			if data == "" {
+				return nil
+			}
+			contentPart := []byte(`{"type":"image","source":{"type":"base64","media_type":"","data":""}}`)
+			contentPart, _ = sjson.SetBytes(contentPart, "source.media_type", mediaType)
+			contentPart, _ = sjson.SetBytes(contentPart, "source.data", data)
+			return contentPart
+		}
+		contentPart := []byte(`{"type":"image","source":{"type":"url","url":""}}`)
+		contentPart, _ = sjson.SetBytes(contentPart, "source.url", url)
+		return contentPart
+	case "input_file":
+		fileData := part.Get("file_data").String()
+		if fileData == "" {
+			return nil
+		}
+		mediaType := "application/octet-stream"
+		data := fileData
+		if strings.HasPrefix(fileData, "data:") {
+			trimmed := strings.TrimPrefix(fileData, "data:")
+			mediaAndData := strings.SplitN(trimmed, ";base64,", 2)
+			if len(mediaAndData) == 2 {
+				if mediaAndData[0] != "" {
+					mediaType = mediaAndData[0]
+				}
+				data = mediaAndData[1]
+			}
+		}
+		contentPart := []byte(`{"type":"document","source":{"type":"base64","media_type":"","data":""}}`)
+		contentPart, _ = sjson.SetBytes(contentPart, "source.media_type", mediaType)
+		contentPart, _ = sjson.SetBytes(contentPart, "source.data", data)
+		return contentPart
+	}
+	return nil
+}
+
 func convertResponsesToolToClaudeTools(tool gjson.Result, toolNameMap map[string]string) [][]byte {
 	toolType := strings.TrimSpace(tool.Get("type").String())
 	switch toolType {
--- a/internal/translator/claude/openai/responses/claude_openai-responses_request_test.go
+++ b/internal/translator/claude/openai/responses/claude_openai-responses_request_test.go
@@ -2,6 +2,7 @@ package responses

 import (
 	"encoding/base64"
+	"strings"
 	"testing"

 	sigcompat "github.com/router-for-me/CLIProxyAPI/v7/internal/signature"
@@ -156,6 +157,53 @@ func TestConvertOpenAIResponsesRequestToClaude_DropsIncompatibleReasoningSignatu
 	}
 }

+func TestConvertOpenAIResponsesRequestToClaude_FunctionCallOutputPreservesInputImage(t *testing.T) {
+	const imageB64 = "iVBORw0KGgo="
+	dataURL := "data:image/png;base64," + imageB64
+	raw := []byte(`{
+		"model":"claude-test",
+		"input":[
+			{
+				"type":"function_call",
+				"call_id":"call_view_image_1",
+				"name":"view_image",
+				"arguments":"{}"
+			},
+			{
+				"type":"function_call_output",
+				"call_id":"call_view_image_1",
+				"output":[
+					{
+						"type":"input_image",
+						"image_url":"` + dataURL + `",
+						"detail":"high"
+					}
+				]
+			}
+		]
+	}`)
+
+	out := ConvertOpenAIResponsesRequestToClaude("claude-test", raw, false)
+	root := gjson.ParseBytes(out)
+
+	toolResult := root.Get("messages.1.content.0")
+	if got := toolResult.Get("type").String(); got != "tool_result" {
+		t.Fatalf("tool_result type = %q, want tool_result. Output: %s", got, string(out))
+	}
+	if got := toolResult.Get("content.0.type").String(); got != "image" {
+		t.Fatalf("tool_result content block type = %q, want image. Output: %s", got, string(out))
+	}
+	if got := toolResult.Get("content.0.source.media_type").String(); got != "image/png" {
+		t.Fatalf("image media_type = %q, want image/png. Output: %s", got, string(out))
+	}
+	if got := toolResult.Get("content.0.source.data").String(); got != imageB64 {
+		t.Fatalf("image data = %q, want raw base64 without data URL prefix", got)
+	}
+	if strings.Contains(toolResult.Get("content").Raw, "data:image") {
+		t.Fatalf("tool_result content must not embed data URL as text. Output: %s", string(out))
+	}
+}
+
 func TestConvertOpenAIResponsesRequestToClaude_KeepsToolUseAdjacentToToolResult(t *testing.T) {
 	raw := []byte(`{
 		"model":"claude-test",