lywsvip/prompt-optimizer
1
0
Fork 0
mirror of https://github.com/linshenkx/prompt-optimizer.git synced 2026-05-06 21:50:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(image): preserve multi-image inputs across runtime and storage

Browse Source 
- include ordered OpenRouter multi-image parts in provider payloads
- persist multi-image session inputs as asset references instead of inline payloads
- restore ordered inputs from storage and cover the persistence path with tests
This commit is contained in:
linshen
2026-04-05 22:20:51 +08:00
parent 90b957c983
commit d8d252eed6
4 changed files with 365 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/core/src/services/image/adapters/openrouter.ts
View File

@@ -165,13 +165,23 @@ export class OpenRouterImageAdapter extends AbstractImageProviderAdapter {
}
]
// 如果有输入图像，添加到消息中
if (request.inputImage) {
const imageContent = `data:${request.inputImage.mimeType || 'image/png'};base64,${request.inputImage.b64}`
const inputImages =
Array.isArray(request.inputImages) && request.inputImages.length > 0
? request.inputImages
: request.inputImage
? [request.inputImage]
: []
// 如果有输入图像，添加到消息中
if (inputImages.length > 0) {
messages[0].content = [
{ type: 'text', text: request.prompt },
{ type: 'image_url', image_url: { url: imageContent } }
...inputImages.map((inputImage) => ({
type: 'image_url',
image_url: {
url: `data:${inputImage.mimeType || 'image/png'};base64,${inputImage.b64}`
}
}))
]
}

71
packages/core/tests/unit/image/openrouter-adapter.test.ts
View File

@@ -1,15 +1,20 @@
import { describe, it, expect, beforeEach } from 'vitest'
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { OpenRouterImageAdapter } from '../../../src/services/image/adapters/openrouter'
import type { ImageModelConfig } from '../../../src/services/image/types'
import type { ImageModelConfig, ImageRequest } from '../../../src/services/image/types'
import { IMAGE_ERROR_CODES } from '../../../src/constants/error-codes'
describe('OpenRouterImageAdapter', () => {
let adapter: OpenRouterImageAdapter
const realFetch = global.fetch
beforeEach(() => {
adapter = new OpenRouterImageAdapter()
})
afterEach(() => {
global.fetch = realFetch
})
describe('Provider Information', () => {
it('should provide correct provider information', () => {
const provider = adapter.getProvider()
@@ -180,4 +185,66 @@ describe('OpenRouterImageAdapter', () => {
expect(base64Data).toBe('iVBORw0KGgoAAAANSUhEUgAA')
})
})
describe('Image Generation', () => {
it('should serialize multiple input images into OpenRouter chat content parts', async () => {
const config: ImageModelConfig = {
id: 'test-openrouter-multi',
name: 'Test OpenRouter Multi',
providerId: 'openrouter',
modelId: adapter.getModels()[0].id,
enabled: true,
connectionConfig: {
apiKey: 'test-api-key',
baseURL: 'https://openrouter.ai/api/v1'
},
provider: adapter.getProvider(),
model: adapter.getModels()[0]
}
const request: ImageRequest = {
prompt: 'combine image 1 and image 2 into one result',
configId: config.id,
count: 1,
inputImages: [
{ b64: 'AAAA', mimeType: 'image/png' },
{ b64: 'BBBB', mimeType: 'image/jpeg' }
]
}
global.fetch = vi.fn().mockResolvedValue({
ok: true,
json: () => Promise.resolve({
choices: [
{
finish_reason: 'stop',
message: {
content: 'done',
images: [
{
image_url: {
url: 'data:image/png;base64,CCCC'
}
}
]
}
}
]
})
}) as typeof fetch
await adapter.generate(request, config)
expect(fetch).toHaveBeenCalledTimes(1)
const fetchOptions = vi.mocked(fetch).mock.calls[0]?.[1]
const payload = JSON.parse(String(fetchOptions?.body))
expect(payload.messages).toHaveLength(1)
expect(payload.messages[0].content).toEqual([
{ type: 'text', text: 'combine image 1 and image 2 into one result' },
{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
{ type: 'image_url', image_url: { url: 'data:image/jpeg;base64,BBBB' } }
])
})
})
})

211
packages/ui/src/stores/session/useImageMultiImageSession.ts
View File

@@ -2,7 +2,17 @@ import { defineStore } from 'pinia'
import { ref } from 'vue'
import { getPiniaServices } from '../../plugins/pinia'
import { isValidVariableName, sanitizeVariableRecord } from '../../types/variable'
import type { ImageResult, IImageStorageService, ImageInputRef } from '@prompt-optimizer/core'
import {
isImageRef,
createImageRef,
type ImageResult,
type IImageStorageService,
type ImageInputRef,
} from '@prompt-optimizer/core'
import {
normalizeImageSourceToPayload,
persistImagePayloadAsAssetId,
} from '../../utils/image-asset-storage'
import {
IMAGE_MULTIIMAGE_SESSION_KEY,
computeStableImageId,
@@ -14,6 +24,8 @@ import {
type PersistedEvaluationResults,
} from '../../types/evaluation'
type ImageResultItem = ImageResult['images'][number]
export type TestPanelVersionValue = 'workspace' | 'previous' | 0 | number
export type TestVariantId = 'a' | 'b' | 'c' | 'd'
export type TestColumnCount = 2 | 3 | 4
@@ -139,21 +151,6 @@ const parseTestVariants = (value: unknown): TestVariantConfig[] => {
)
}
const parseTestVariantResults = (
value: unknown,
): Record<TestVariantId, ImageResult | null> => {
const defaults = createDefaultState().testVariantResults
if (!value || typeof value !== 'object') return defaults
const record = value as Record<string, unknown>
return {
a: (record.a as ImageResult | null) ?? defaults.a,
b: (record.b as ImageResult | null) ?? defaults.b,
c: (record.c as ImageResult | null) ?? defaults.c,
d: (record.d as ImageResult | null) ?? defaults.d,
}
}
const parseTestVariantFingerprints = (value: unknown): Record<TestVariantId, string> => {
const defaults = createDefaultState().testVariantLastRunFingerprint
if (!value || typeof value !== 'object') return defaults
@@ -190,6 +187,129 @@ const saveInputImage = async (
return stableId
}
const prepareForSave = async (
result: ImageResult | null,
storageService: IImageStorageService,
): Promise<ImageResult | null> => {
if (!result || !result.images || result.images.length === 0) {
return result
}
const processedImages: ImageResultItem[] = []
for (const img of result.images) {
if (isImageRef(img)) {
processedImages.push(img)
continue
}
const payload = img.b64
? {
b64: img.b64,
mimeType: img.mimeType || 'image/png',
}
: img.url
? await normalizeImageSourceToPayload(img.url)
: null
if (!payload) {
processedImages.push(img)
continue
}
const imageId = await persistImagePayloadAsAssetId({
payload,
storageService,
sourceType: 'generated',
metadata: {
prompt: result.metadata?.prompt,
modelId: result.metadata?.modelId,
configId: result.metadata?.configId,
},
})
if (imageId) {
processedImages.push(createImageRef(imageId))
continue
}
processedImages.push(img)
}
return {
...result,
images: processedImages,
}
}
const loadFromRef = async (
result: ImageResult | null,
storageService: IImageStorageService,
): Promise<ImageResult | null> => {
if (!result || !result.images || result.images.length === 0) {
return result
}
const loadedImages: ImageResultItem[] = []
for (const img of result.images) {
if (isImageRef(img)) {
try {
const fullImageData = await storageService.getImage(img.id)
if (fullImageData) {
loadedImages.push({
b64: fullImageData.data,
mimeType: fullImageData.metadata.mimeType,
})
} else {
console.warn(`[ImageMultiImageSession] 图像 ${img.id} 未找到`)
loadedImages.push(img)
}
} catch (error) {
console.error(`[ImageMultiImageSession] 加载图像 ${img.id} 失败:`, error)
loadedImages.push(img)
}
} else {
if (img.url && !img.b64) {
try {
const payload = await normalizeImageSourceToPayload(img.url)
if (payload?.b64) {
try {
await persistImagePayloadAsAssetId({
payload,
storageService,
sourceType: 'generated',
metadata: {
prompt: result.metadata?.prompt,
modelId: result.metadata?.modelId,
configId: result.metadata?.configId,
},
})
} catch (error) {
console.warn('[ImageMultiImageSession] 恢复 legacy url 图像时写入存储失败:', error)
}
loadedImages.push({
b64: payload.b64,
mimeType: payload.mimeType,
})
continue
}
} catch (error) {
console.warn('[ImageMultiImageSession] 恢复 legacy url 图像失败:', error)
}
}
loadedImages.push(img)
}
}
return {
...result,
images: loadedImages,
}
}
export const useImageMultiImageSession = defineStore('imageMultiImageSession', () => {
const originalPrompt = ref('')
const optimizedPrompt = ref('')
@@ -419,6 +539,20 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', (
assetId: persistedImages[index]?.assetId || image.assetId,
}))
const baseVariantResults = {
a: testVariantResults.value.a ?? originalImageResult.value,
b: testVariantResults.value.b ?? optimizedImageResult.value,
c: testVariantResults.value.c,
d: testVariantResults.value.d,
}
const variantResultsToSave = {
a: await prepareForSave(baseVariantResults.a, imageStorageService),
b: await prepareForSave(baseVariantResults.b, imageStorageService),
c: await prepareForSave(baseVariantResults.c, imageStorageService),
d: await prepareForSave(baseVariantResults.d, imageStorageService),
}
await $services.preferenceService.set(IMAGE_MULTIIMAGE_SESSION_KEY, {
originalPrompt: originalPrompt.value,
optimizedPrompt: optimizedPrompt.value,
@@ -427,11 +561,11 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', (
versionId: versionId.value,
temporaryVariables: sanitizeVariableRecord(temporaryVariables.value),
inputImages: persistedImages,
originalImageResult: originalImageResult.value,
optimizedImageResult: optimizedImageResult.value,
originalImageResult: variantResultsToSave.a,
optimizedImageResult: variantResultsToSave.b,
layout: layout.value,
testVariants: testVariants.value,
testVariantResults: testVariantResults.value,
testVariantResults: variantResultsToSave,
testVariantLastRunFingerprint: testVariantLastRunFingerprint.value,
evaluationResults: evaluationResults.value,
isCompareMode: isCompareMode.value,
@@ -486,6 +620,39 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', (
}),
)
const rawVariantResults = parsed.testVariantResults
let variantResultsLoaded: Record<TestVariantId, ImageResult | null> | null = null
if (rawVariantResults && typeof rawVariantResults === 'object') {
const record = rawVariantResults as Record<string, unknown>
const pick = (id: TestVariantId): ImageResult | null => {
const one = record[id]
if (!one || typeof one !== 'object') return null
return one as ImageResult
}
variantResultsLoaded = {
a: pick('a'),
b: pick('b'),
c: pick('c'),
d: pick('d'),
}
}
if (!variantResultsLoaded) {
variantResultsLoaded = {
a: (parsed.originalImageResult as ImageResult | null) ?? null,
b: (parsed.optimizedImageResult as ImageResult | null) ?? null,
c: null,
d: null,
}
}
const loadedVariantResults = {
a: await loadFromRef(variantResultsLoaded.a, imageStorageService),
b: await loadFromRef(variantResultsLoaded.b, imageStorageService),
c: await loadFromRef(variantResultsLoaded.c, imageStorageService),
d: await loadFromRef(variantResultsLoaded.d, imageStorageService),
}
originalPrompt.value = typeof parsed.originalPrompt === 'string' ? parsed.originalPrompt : ''
optimizedPrompt.value = typeof parsed.optimizedPrompt === 'string' ? parsed.optimizedPrompt : ''
reasoning.value = typeof parsed.reasoning === 'string' ? parsed.reasoning : ''
@@ -493,11 +660,11 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', (
versionId.value = typeof parsed.versionId === 'string' ? parsed.versionId : ''
temporaryVariables.value = sanitizeVariableRecord(parsed.temporaryVariables)
inputImages.value = restoredImages
originalImageResult.value = (parsed.originalImageResult as ImageResult | null) ?? null
optimizedImageResult.value = (parsed.optimizedImageResult as ImageResult | null) ?? null
originalImageResult.value = loadedVariantResults.a
optimizedImageResult.value = loadedVariantResults.b
layout.value = parseLayout(parsed.layout)
testVariants.value = parseTestVariants(parsed.testVariants)
testVariantResults.value = parseTestVariantResults(parsed.testVariantResults)
testVariantResults.value = loadedVariantResults
testVariantLastRunFingerprint.value = parseTestVariantFingerprints(
parsed.testVariantLastRunFingerprint,
)

93
packages/ui/tests/unit/stores/session/image-multiimage-session-persistence.spec.ts
View File

@@ -1,6 +1,7 @@
import { describe, it, expect, vi } from 'vitest'
import { createTestPinia } from '../../../utils/pinia-test-helpers'
import { useImageMultiImageSession } from '../../../../src/stores/session/useImageMultiImageSession'
import { IMAGE_MULTIIMAGE_SESSION_KEY } from '../../../../src/stores/session/imageStorageMaintenance'
describe('Session store (image-multiimage) persistence', () => {
it('persists the ordered multi-image list and restores it with the same order', async () => {
@@ -150,4 +151,96 @@ describe('Session store (image-multiimage) persistence', () => {
expect(restored.testVariantLastRunFingerprint.a).toBe('fingerprint-a')
expect(restored.isCompareMode).toBe(false)
})
it('stores generated multi-image results as image refs and restores their payloads from image storage', async () => {
const savedSnapshots = new Map<string, unknown>()
const imageMap = new Map<string, { data: string; metadata: { mimeType: string } }>()
const imageStorageService = {
saveImage: vi.fn(async (data: any) => {
imageMap.set(data.metadata.id, {
data: data.data,
metadata: { mimeType: data.metadata.mimeType },
})
return data.metadata.id
}),
getImage: vi.fn(async (id: string) => imageMap.get(id) ?? null),
getMetadata: vi.fn(async (id: string) =>
imageMap.has(id)
? {
id,
mimeType: imageMap.get(id)?.metadata.mimeType || 'image/png',
sizeBytes: 4,
createdAt: Date.now(),
accessedAt: Date.now(),
source: 'generated',
}
: null,
),
listAllMetadata: vi.fn(async () => []),
deleteImages: vi.fn(async () => {}),
}
const { pinia } = createTestPinia({
preferenceService: {
get: async <T,>(key: string, defaultValue: T) =>
(savedSnapshots.has(key) ? savedSnapshots.get(key) : defaultValue) as T,
set: async (key: string, value: unknown) => {
savedSnapshots.set(key, value)
},
delete: async () => {},
keys: async () => [],
clear: async () => {},
getAll: async () => ({}),
exportData: async () => ({}),
importData: async () => {},
getDataType: async () => 'preference',
validateData: async () => true,
} as any,
imageStorageService: imageStorageService as any,
})
const store = useImageMultiImageSession(pinia)
store.updateOriginalImageResult({
images: [{ b64: 'ORIGINAL', mimeType: 'image/png' }],
metadata: { providerId: 'provider', modelId: 'model-original', configId: 'image-model-original' },
})
store.updateOptimizedImageResult({
images: [{ b64: 'OPTIMIZED', mimeType: 'image/jpeg' }],
metadata: { providerId: 'provider', modelId: 'model-optimized', configId: 'image-model-optimized' },
})
store.updateTestVariantResult('c', {
images: [{ b64: 'VARIANTC', mimeType: 'image/png' }],
metadata: { providerId: 'provider', modelId: 'model-c', configId: 'image-model-c' },
})
await store.saveSession()
const snapshot = savedSnapshots.get(IMAGE_MULTIIMAGE_SESSION_KEY) as Record<string, any>
expect(snapshot.originalImageResult.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) })
expect(snapshot.optimizedImageResult.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) })
expect(snapshot.testVariantResults.a.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) })
expect(snapshot.testVariantResults.b.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) })
expect(snapshot.testVariantResults.c.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) })
expect(JSON.stringify(snapshot)).not.toContain('ORIGINAL')
expect(JSON.stringify(snapshot)).not.toContain('OPTIMIZED')
expect(JSON.stringify(snapshot)).not.toContain('VARIANTC')
const restored = useImageMultiImageSession(pinia)
restored.reset()
await restored.restoreSession()
expect(restored.originalImageResult).toMatchObject({
images: [{ b64: 'ORIGINAL', mimeType: 'image/png' }],
})
expect(restored.optimizedImageResult).toMatchObject({
images: [{ b64: 'OPTIMIZED', mimeType: 'image/jpeg' }],
})
expect(restored.testVariantResults.c).toMatchObject({
images: [{ b64: 'VARIANTC', mimeType: 'image/png' }],
})
expect(imageStorageService.saveImage).toHaveBeenCalledTimes(3)
expect(imageStorageService.getImage).toHaveBeenCalled()
})
})