From 4cd17293a3fd8ec52b396632fc68439cf2f47e48 Mon Sep 17 00:00:00 2001 From: Luis Pater Date: Fri, 11 Sep 2026 02:52:04 +0800 Subject: [PATCH] fix(translator): relay claude tool result images as user messages for openai - Extract image parts from Claude tool results and replay them in a user message following tool messages. - Merge relayed images into existing user message content when present within the same turn. - Insert a placeholder text for image-only tool results to keep tool message content non-empty. Closes: #5707 --- .../openai/claude/openai_claude_request.go | 89 ++++---- .../claude/openai_claude_request_test.go | 201 ++++++++++++++++-- 2 files changed, 231 insertions(+), 59 deletions(-) diff --git a/internal/translator/openai/claude/openai_claude_request.go b/internal/translator/openai/claude/openai_claude_request.go index 5fcb9575f..942c6e047 100644 --- a/internal/translator/openai/claude/openai_claude_request.go +++ b/internal/translator/openai/claude/openai_claude_request.go @@ -178,7 +178,8 @@ func convertClaudeRequestToOpenAI(modelName string, inputRawJSON []byte, stream contentItems := make([][]byte, 0) var reasoningParts []string // Accumulate thinking text for reasoning_content var toolCalls []interface{} - toolResults := make([][]byte, 0) // Collect tool_result messages to emit after the main message + toolResults := make([][]byte, 0) // Collect tool_result messages to emit after the main message + relayedToolImages := make([][]byte, 0) // Images pulled out of tool_result content for user-message relay contentResult.ForEach(func(_, part gjson.Result) bool { partType := part.Get("type").String() @@ -231,12 +232,9 @@ func convertClaudeRequestToOpenAI(modelName string, inputRawJSON []byte, stream // Collect tool_result to emit after the main message (ensures tool results follow tool_calls) toolResultJSON := []byte(`{"role":"tool","tool_call_id":"","content":""}`) toolResultJSON, _ = sjson.SetBytes(toolResultJSON, "tool_call_id", part.Get("tool_use_id").String()) - toolResultContent, toolResultContentRaw := convertClaudeToolResultContent(part.Get("content")) - if toolResultContentRaw { - toolResultJSON, _ = sjson.SetRawBytes(toolResultJSON, "content", []byte(toolResultContent)) - } else { - toolResultJSON, _ = sjson.SetBytes(toolResultJSON, "content", toolResultContent) - } + toolResultContent, toolResultImages := convertClaudeToolResultContent(part.Get("content")) + toolResultJSON, _ = sjson.SetBytes(toolResultJSON, "content", toolResultContent) + relayedToolImages = append(relayedToolImages, toolResultImages...) toolResults = append(toolResults, toolResultJSON) } return true @@ -264,6 +262,25 @@ func convertClaudeRequestToOpenAI(modelName string, inputRawJSON []byte, stream // then emit any queued system reminders, then emit the current message's content. messageItems = append(messageItems, toolResults...) + // OpenAI tool messages cannot carry image parts, so images returned by a tool are + // replayed as a user message directly after the tool results. + if len(relayedToolImages) > 0 { + relayItems := make([][]byte, 0, len(relayedToolImages)+1) + noticeJSON := []byte(`{"type":"text","text":""}`) + noticeJSON, _ = sjson.SetBytes(noticeJSON, "text", toolResultImageRelayNotice) + relayItems = append(relayItems, noticeJSON) + relayItems = append(relayItems, relayedToolImages...) + + if role == "user" && hasContent { + // Merge into the current user message so the request keeps a single user turn. + contentItems = append(relayItems, contentItems...) + } else { + relayJSON := []byte(`{"role":"user"}`) + relayJSON, _ = sjson.SetRawBytes(relayJSON, "content", translatorcommon.JoinRawArray(relayItems)) + messageItems = append(messageItems, relayJSON) + } + } + if len(pendingSystemReminders) > 0 { messageItems = append(messageItems, pendingSystemReminders...) pendingSystemReminders = nil @@ -502,38 +519,34 @@ func convertClaudeContentPart(part gjson.Result) (string, bool) { } } -func convertClaudeToolResultContent(content gjson.Result) (string, bool) { +// toolResultImagePlaceholder keeps the OpenAI tool message non-empty when a Claude +// tool_result carried nothing but images. +const toolResultImagePlaceholder = "[Tool returned image content; the images follow in the next user message.]" + +// toolResultImageRelayNotice labels the user message that carries relayed tool images. +const toolResultImageRelayNotice = "Images returned by the preceding tool call(s):" + +func convertClaudeToolResultContent(content gjson.Result) (string, [][]byte) { if !content.Exists() { - return "", false + return "", nil } if content.Type == gjson.String { - return content.String(), false + return content.String(), nil } if content.IsArray() { var parts []string - contentItems := make([][]byte, 0, 4) - hasImagePart := false + var images [][]byte content.ForEach(func(_, item gjson.Result) bool { switch { case item.Type == gjson.String: - text := item.String() - parts = append(parts, text) - textContent := []byte(`{"type":"text","text":""}`) - textContent, _ = sjson.SetBytes(textContent, "text", text) - contentItems = append(contentItems, textContent) + parts = append(parts, item.String()) case item.IsObject() && item.Get("type").String() == "text": - text := item.Get("text").String() - parts = append(parts, text) - textContent := []byte(`{"type":"text","text":""}`) - textContent, _ = sjson.SetBytes(textContent, "text", text) - contentItems = append(contentItems, textContent) + parts = append(parts, item.Get("text").String()) case item.IsObject() && item.Get("type").String() == "image": - contentItem, ok := convertClaudeContentPart(item) - if ok { - contentItems = append(contentItems, []byte(contentItem)) - hasImagePart = true + if contentItem, ok := convertClaudeContentPart(item); ok { + images = append(images, []byte(contentItem)) } else { parts = append(parts, item.Raw) } @@ -545,29 +558,27 @@ func convertClaudeToolResultContent(content gjson.Result) (string, bool) { return true }) - if hasImagePart { - return string(translatorcommon.JoinRawArray(contentItems)), true - } - joined := strings.Join(parts, "\n\n") - if strings.TrimSpace(joined) != "" { - return joined, false + if strings.TrimSpace(joined) == "" { + if len(images) > 0 { + return toolResultImagePlaceholder, images + } + return content.Raw, nil } - return content.Raw, false + return joined, images } if content.IsObject() { if content.Get("type").String() == "image" { - contentItem, ok := convertClaudeContentPart(content) - if ok { - return string(translatorcommon.JoinRawArray([][]byte{[]byte(contentItem)})), true + if contentItem, ok := convertClaudeContentPart(content); ok { + return toolResultImagePlaceholder, [][]byte{[]byte(contentItem)} } } if text := content.Get("text"); text.Exists() && text.Type == gjson.String { - return text.String(), false + return text.String(), nil } - return content.Raw, false + return content.Raw, nil } - return content.Raw, false + return content.Raw, nil } diff --git a/internal/translator/openai/claude/openai_claude_request_test.go b/internal/translator/openai/claude/openai_claude_request_test.go index 7f32df47c..1a5c97ae1 100644 --- a/internal/translator/openai/claude/openai_claude_request_test.go +++ b/internal/translator/openai/claude/openai_claude_request_test.go @@ -733,24 +733,37 @@ func TestConvertClaudeRequestToOpenAI_ToolResultTextAndImageContent(t *testing.T resultJSON := gjson.ParseBytes(result) messages := resultJSON.Get("messages").Array() - if len(messages) != 2 { - t.Fatalf("Expected 2 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) + if len(messages) != 3 { + t.Fatalf("Expected 3 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) } - toolContent := messages[1].Get("content") - if !toolContent.IsArray() { - t.Fatalf("Expected tool content array, got %s", toolContent.Raw) + // The tool message keeps only the textual payload; OpenAI drops image parts there. + if got := messages[1].Get("role").String(); got != "tool" { + t.Fatalf("Expected second message role %q, got %q", "tool", got) } - if got := toolContent.Get("0.type").String(); got != "text" { - t.Fatalf("Expected first tool content type %q, got %q", "text", got) + if messages[1].Get("content").IsArray() { + t.Fatalf("Tool content must not be an array, got %s", messages[1].Get("content").Raw) } - if got := toolContent.Get("0.text").String(); got != "tool ok" { - t.Fatalf("Expected first tool content text %q, got %q", "tool ok", got) + if got := messages[1].Get("content").String(); got != "tool ok" { + t.Fatalf("Expected tool content %q, got %q", "tool ok", got) } - if got := toolContent.Get("1.type").String(); got != "image_url" { - t.Fatalf("Expected second tool content type %q, got %q", "image_url", got) + + // The image is relayed as a user message directly after the tool result. + relay := messages[2] + if got := relay.Get("role").String(); got != "user" { + t.Fatalf("Expected relay message role %q, got %q", "user", got) } - if got := toolContent.Get("1.image_url.url").String(); got != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==" { + relayContent := relay.Get("content") + if !relayContent.IsArray() { + t.Fatalf("Expected relay content array, got %s", relayContent.Raw) + } + if got := relayContent.Get("0.text").String(); got != toolResultImageRelayNotice { + t.Fatalf("Expected relay notice %q, got %q", toolResultImageRelayNotice, got) + } + if got := relayContent.Get("1.type").String(); got != "image_url" { + t.Fatalf("Expected relay content type %q, got %q", "image_url", got) + } + if got := relayContent.Get("1.image_url.url").String(); got != "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==" { t.Fatalf("Unexpected image_url: %q", got) } } @@ -788,22 +801,170 @@ func TestConvertClaudeRequestToOpenAI_ToolResultURLImageOnly(t *testing.T) { resultJSON := gjson.ParseBytes(result) messages := resultJSON.Get("messages").Array() - if len(messages) != 2 { - t.Fatalf("Expected 2 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) + if len(messages) != 3 { + t.Fatalf("Expected 3 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) } - toolContent := messages[1].Get("content") - if !toolContent.IsArray() { - t.Fatalf("Expected tool content array, got %s", toolContent.Raw) + // An image-only tool_result still needs a non-empty text payload on the tool message. + if got := messages[1].Get("content").String(); got != toolResultImagePlaceholder { + t.Fatalf("Expected tool content %q, got %q", toolResultImagePlaceholder, got) } - if got := toolContent.Get("0.type").String(); got != "image_url" { - t.Fatalf("Expected tool content type %q, got %q", "image_url", got) + + relayContent := messages[2].Get("content") + if got := messages[2].Get("role").String(); got != "user" { + t.Fatalf("Expected relay message role %q, got %q", "user", got) } - if got := toolContent.Get("0.image_url.url").String(); got != "https://example.com/tool.png" { + if got := relayContent.Get("1.type").String(); got != "image_url" { + t.Fatalf("Expected relay content type %q, got %q", "image_url", got) + } + if got := relayContent.Get("1.image_url.url").String(); got != "https://example.com/tool.png" { t.Fatalf("Unexpected image_url: %q", got) } } +func TestConvertClaudeRequestToOpenAI_ToolResultImageMergesIntoUserText(t *testing.T) { + inputJSON := `{ + "model": "claude-3-opus", + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "call_1", "name": "screenshot", "input": {}} + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUg==" + } + } + ] + }, + {"type": "text", "text": "What color?"} + ] + } + ] + }` + + result := ConvertClaudeRequestToOpenAI("test-model", []byte(inputJSON), false) + resultJSON := gjson.ParseBytes(result) + messages := resultJSON.Get("messages").Array() + + // The relayed image joins the user text instead of adding a second user turn. + if len(messages) != 3 { + t.Fatalf("Expected 3 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) + } + if got := messages[2].Get("role").String(); got != "user" { + t.Fatalf("Expected third message role %q, got %q", "user", got) + } + + content := messages[2].Get("content") + if got := len(content.Array()); got != 3 { + t.Fatalf("Expected 3 user content parts, got %d: %s", got, content.Raw) + } + if got := content.Get("0.text").String(); got != toolResultImageRelayNotice { + t.Fatalf("Expected relay notice %q, got %q", toolResultImageRelayNotice, got) + } + if got := content.Get("1.type").String(); got != "image_url" { + t.Fatalf("Expected second part type %q, got %q", "image_url", got) + } + if got := content.Get("2.text").String(); got != "What color?" { + t.Fatalf("Expected trailing user text %q, got %q", "What color?", got) + } +} + +func TestConvertClaudeRequestToOpenAI_MultipleToolResultsWithImages(t *testing.T) { + inputJSON := `{ + "model": "claude-3-opus", + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "call_1", "name": "shot1", "input": {}}, + {"type": "tool_use", "id": "call_2", "name": "shot2", "input": {}} + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [ + {"type": "text", "text": "result 1"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "img1" + } + } + ] + }, + { + "type": "tool_result", + "tool_use_id": "call_2", + "content": { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/2.png" + } + } + } + ] + } + ] + }` + + result := ConvertClaudeRequestToOpenAI("test-model", []byte(inputJSON), false) + resultJSON := gjson.ParseBytes(result) + messages := resultJSON.Get("messages").Array() + + // Expected: assistant(2 tool calls), tool(call_1), tool(call_2), user(relay 2 images) + if len(messages) != 4 { + t.Fatalf("Expected 4 messages, got %d. Messages: %s", len(messages), resultJSON.Get("messages").Raw) + } + if got := messages[1].Get("role").String(); got != "tool" || messages[1].Get("tool_call_id").String() != "call_1" { + t.Fatalf("Expected tool 1 message, got: %s", messages[1].Raw) + } + if got := messages[1].Get("content").String(); got != "result 1" { + t.Fatalf("Expected tool 1 content 'result 1', got %q", got) + } + if got := messages[2].Get("role").String(); got != "tool" || messages[2].Get("tool_call_id").String() != "call_2" { + t.Fatalf("Expected tool 2 message, got: %s", messages[2].Raw) + } + if got := messages[2].Get("content").String(); got != toolResultImagePlaceholder { + t.Fatalf("Expected tool 2 placeholder, got %q", got) + } + if got := messages[3].Get("role").String(); got != "user" { + t.Fatalf("Expected user relay message, got: %s", messages[3].Raw) + } + relayContent := messages[3].Get("content").Array() + if len(relayContent) != 3 { + t.Fatalf("Expected 3 parts in relay (notice + 2 images), got %d: %s", len(relayContent), messages[3].Get("content").Raw) + } + if got := relayContent[0].Get("text").String(); got != toolResultImageRelayNotice { + t.Fatalf("Expected notice %q, got %q", toolResultImageRelayNotice, got) + } + if got := relayContent[1].Get("image_url.url").String(); got != "data:image/png;base64,img1" { + t.Fatalf("Expected image 1 url, got %q", got) + } + if got := relayContent[2].Get("image_url.url").String(); got != "https://example.com/2.png" { + t.Fatalf("Expected image 2 url, got %q", got) + } +} + func TestConvertClaudeRequestToOpenAI_AssistantTextToolUseTextOrder(t *testing.T) { inputJSON := `{ "model": "claude-3-opus",