diff --git a/packages/core/src/services/image/adapters/openrouter.ts b/packages/core/src/services/image/adapters/openrouter.ts index b2e2a009..e0b630c3 100644 --- a/packages/core/src/services/image/adapters/openrouter.ts +++ b/packages/core/src/services/image/adapters/openrouter.ts @@ -165,13 +165,23 @@ export class OpenRouterImageAdapter extends AbstractImageProviderAdapter { } ] - // 如果有输入图像，添加到消息中 - if (request.inputImage) { - const imageContent = `data:${request.inputImage.mimeType || 'image/png'};base64,${request.inputImage.b64}` + const inputImages = + Array.isArray(request.inputImages) && request.inputImages.length > 0 + ? request.inputImages + : request.inputImage + ? [request.inputImage] + : [] + // 如果有输入图像，添加到消息中 + if (inputImages.length > 0) { messages[0].content = [ { type: 'text', text: request.prompt }, - { type: 'image_url', image_url: { url: imageContent } } + ...inputImages.map((inputImage) => ({ + type: 'image_url', + image_url: { + url: `data:${inputImage.mimeType || 'image/png'};base64,${inputImage.b64}` + } + })) ] } diff --git a/packages/core/tests/unit/image/openrouter-adapter.test.ts b/packages/core/tests/unit/image/openrouter-adapter.test.ts index ba472779..88bf7930 100644 --- a/packages/core/tests/unit/image/openrouter-adapter.test.ts +++ b/packages/core/tests/unit/image/openrouter-adapter.test.ts @@ -1,15 +1,20 @@ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { OpenRouterImageAdapter } from '../../../src/services/image/adapters/openrouter' -import type { ImageModelConfig } from '../../../src/services/image/types' +import type { ImageModelConfig, ImageRequest } from '../../../src/services/image/types' import { IMAGE_ERROR_CODES } from '../../../src/constants/error-codes' describe('OpenRouterImageAdapter', () => { let adapter: OpenRouterImageAdapter + const realFetch = global.fetch beforeEach(() => { adapter = new OpenRouterImageAdapter() }) + afterEach(() => { + global.fetch = realFetch + }) + describe('Provider Information', () => { it('should provide correct provider information', () => { const provider = adapter.getProvider() @@ -180,4 +185,66 @@ describe('OpenRouterImageAdapter', () => { expect(base64Data).toBe('iVBORw0KGgoAAAANSUhEUgAA') }) }) + + describe('Image Generation', () => { + it('should serialize multiple input images into OpenRouter chat content parts', async () => { + const config: ImageModelConfig = { + id: 'test-openrouter-multi', + name: 'Test OpenRouter Multi', + providerId: 'openrouter', + modelId: adapter.getModels()[0].id, + enabled: true, + connectionConfig: { + apiKey: 'test-api-key', + baseURL: 'https://openrouter.ai/api/v1' + }, + provider: adapter.getProvider(), + model: adapter.getModels()[0] + } + + const request: ImageRequest = { + prompt: 'combine image 1 and image 2 into one result', + configId: config.id, + count: 1, + inputImages: [ + { b64: 'AAAA', mimeType: 'image/png' }, + { b64: 'BBBB', mimeType: 'image/jpeg' } + ] + } + + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ + choices: [ + { + finish_reason: 'stop', + message: { + content: 'done', + images: [ + { + image_url: { + url: 'data:image/png;base64,CCCC' + } + } + ] + } + } + ] + }) + }) as typeof fetch + + await adapter.generate(request, config) + + expect(fetch).toHaveBeenCalledTimes(1) + const fetchOptions = vi.mocked(fetch).mock.calls[0]?.[1] + const payload = JSON.parse(String(fetchOptions?.body)) + + expect(payload.messages).toHaveLength(1) + expect(payload.messages[0].content).toEqual([ + { type: 'text', text: 'combine image 1 and image 2 into one result' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,BBBB' } } + ]) + }) + }) }) diff --git a/packages/ui/src/stores/session/useImageMultiImageSession.ts b/packages/ui/src/stores/session/useImageMultiImageSession.ts index 0ce505dc..51e57448 100644 --- a/packages/ui/src/stores/session/useImageMultiImageSession.ts +++ b/packages/ui/src/stores/session/useImageMultiImageSession.ts @@ -2,7 +2,17 @@ import { defineStore } from 'pinia' import { ref } from 'vue' import { getPiniaServices } from '../../plugins/pinia' import { isValidVariableName, sanitizeVariableRecord } from '../../types/variable' -import type { ImageResult, IImageStorageService, ImageInputRef } from '@prompt-optimizer/core' +import { + isImageRef, + createImageRef, + type ImageResult, + type IImageStorageService, + type ImageInputRef, +} from '@prompt-optimizer/core' +import { + normalizeImageSourceToPayload, + persistImagePayloadAsAssetId, +} from '../../utils/image-asset-storage' import { IMAGE_MULTIIMAGE_SESSION_KEY, computeStableImageId, @@ -14,6 +24,8 @@ import { type PersistedEvaluationResults, } from '../../types/evaluation' +type ImageResultItem = ImageResult['images'][number] + export type TestPanelVersionValue = 'workspace' | 'previous' | 0 | number export type TestVariantId = 'a' | 'b' | 'c' | 'd' export type TestColumnCount = 2 | 3 | 4 @@ -139,21 +151,6 @@ const parseTestVariants = (value: unknown): TestVariantConfig[] => { ) } -const parseTestVariantResults = ( - value: unknown, -): Record => { - const defaults = createDefaultState().testVariantResults - if (!value || typeof value !== 'object') return defaults - - const record = value as Record - return { - a: (record.a as ImageResult | null) ?? defaults.a, - b: (record.b as ImageResult | null) ?? defaults.b, - c: (record.c as ImageResult | null) ?? defaults.c, - d: (record.d as ImageResult | null) ?? defaults.d, - } -} - const parseTestVariantFingerprints = (value: unknown): Record => { const defaults = createDefaultState().testVariantLastRunFingerprint if (!value || typeof value !== 'object') return defaults @@ -190,6 +187,129 @@ const saveInputImage = async ( return stableId } +const prepareForSave = async ( + result: ImageResult | null, + storageService: IImageStorageService, +): Promise => { + if (!result || !result.images || result.images.length === 0) { + return result + } + + const processedImages: ImageResultItem[] = [] + + for (const img of result.images) { + if (isImageRef(img)) { + processedImages.push(img) + continue + } + + const payload = img.b64 + ? { + b64: img.b64, + mimeType: img.mimeType || 'image/png', + } + : img.url + ? await normalizeImageSourceToPayload(img.url) + : null + + if (!payload) { + processedImages.push(img) + continue + } + + const imageId = await persistImagePayloadAsAssetId({ + payload, + storageService, + sourceType: 'generated', + metadata: { + prompt: result.metadata?.prompt, + modelId: result.metadata?.modelId, + configId: result.metadata?.configId, + }, + }) + + if (imageId) { + processedImages.push(createImageRef(imageId)) + continue + } + + processedImages.push(img) + } + + return { + ...result, + images: processedImages, + } +} + +const loadFromRef = async ( + result: ImageResult | null, + storageService: IImageStorageService, +): Promise => { + if (!result || !result.images || result.images.length === 0) { + return result + } + + const loadedImages: ImageResultItem[] = [] + + for (const img of result.images) { + if (isImageRef(img)) { + try { + const fullImageData = await storageService.getImage(img.id) + if (fullImageData) { + loadedImages.push({ + b64: fullImageData.data, + mimeType: fullImageData.metadata.mimeType, + }) + } else { + console.warn(`[ImageMultiImageSession] 图像 ${img.id} 未找到`) + loadedImages.push(img) + } + } catch (error) { + console.error(`[ImageMultiImageSession] 加载图像 ${img.id} 失败:`, error) + loadedImages.push(img) + } + } else { + if (img.url && !img.b64) { + try { + const payload = await normalizeImageSourceToPayload(img.url) + if (payload?.b64) { + try { + await persistImagePayloadAsAssetId({ + payload, + storageService, + sourceType: 'generated', + metadata: { + prompt: result.metadata?.prompt, + modelId: result.metadata?.modelId, + configId: result.metadata?.configId, + }, + }) + } catch (error) { + console.warn('[ImageMultiImageSession] 恢复 legacy url 图像时写入存储失败:', error) + } + + loadedImages.push({ + b64: payload.b64, + mimeType: payload.mimeType, + }) + continue + } + } catch (error) { + console.warn('[ImageMultiImageSession] 恢复 legacy url 图像失败:', error) + } + } + + loadedImages.push(img) + } + } + + return { + ...result, + images: loadedImages, + } +} + export const useImageMultiImageSession = defineStore('imageMultiImageSession', () => { const originalPrompt = ref('') const optimizedPrompt = ref('') @@ -419,6 +539,20 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', ( assetId: persistedImages[index]?.assetId || image.assetId, })) + const baseVariantResults = { + a: testVariantResults.value.a ?? originalImageResult.value, + b: testVariantResults.value.b ?? optimizedImageResult.value, + c: testVariantResults.value.c, + d: testVariantResults.value.d, + } + + const variantResultsToSave = { + a: await prepareForSave(baseVariantResults.a, imageStorageService), + b: await prepareForSave(baseVariantResults.b, imageStorageService), + c: await prepareForSave(baseVariantResults.c, imageStorageService), + d: await prepareForSave(baseVariantResults.d, imageStorageService), + } + await $services.preferenceService.set(IMAGE_MULTIIMAGE_SESSION_KEY, { originalPrompt: originalPrompt.value, optimizedPrompt: optimizedPrompt.value, @@ -427,11 +561,11 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', ( versionId: versionId.value, temporaryVariables: sanitizeVariableRecord(temporaryVariables.value), inputImages: persistedImages, - originalImageResult: originalImageResult.value, - optimizedImageResult: optimizedImageResult.value, + originalImageResult: variantResultsToSave.a, + optimizedImageResult: variantResultsToSave.b, layout: layout.value, testVariants: testVariants.value, - testVariantResults: testVariantResults.value, + testVariantResults: variantResultsToSave, testVariantLastRunFingerprint: testVariantLastRunFingerprint.value, evaluationResults: evaluationResults.value, isCompareMode: isCompareMode.value, @@ -486,6 +620,39 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', ( }), ) + const rawVariantResults = parsed.testVariantResults + let variantResultsLoaded: Record | null = null + if (rawVariantResults && typeof rawVariantResults === 'object') { + const record = rawVariantResults as Record + const pick = (id: TestVariantId): ImageResult | null => { + const one = record[id] + if (!one || typeof one !== 'object') return null + return one as ImageResult + } + variantResultsLoaded = { + a: pick('a'), + b: pick('b'), + c: pick('c'), + d: pick('d'), + } + } + + if (!variantResultsLoaded) { + variantResultsLoaded = { + a: (parsed.originalImageResult as ImageResult | null) ?? null, + b: (parsed.optimizedImageResult as ImageResult | null) ?? null, + c: null, + d: null, + } + } + + const loadedVariantResults = { + a: await loadFromRef(variantResultsLoaded.a, imageStorageService), + b: await loadFromRef(variantResultsLoaded.b, imageStorageService), + c: await loadFromRef(variantResultsLoaded.c, imageStorageService), + d: await loadFromRef(variantResultsLoaded.d, imageStorageService), + } + originalPrompt.value = typeof parsed.originalPrompt === 'string' ? parsed.originalPrompt : '' optimizedPrompt.value = typeof parsed.optimizedPrompt === 'string' ? parsed.optimizedPrompt : '' reasoning.value = typeof parsed.reasoning === 'string' ? parsed.reasoning : '' @@ -493,11 +660,11 @@ export const useImageMultiImageSession = defineStore('imageMultiImageSession', ( versionId.value = typeof parsed.versionId === 'string' ? parsed.versionId : '' temporaryVariables.value = sanitizeVariableRecord(parsed.temporaryVariables) inputImages.value = restoredImages - originalImageResult.value = (parsed.originalImageResult as ImageResult | null) ?? null - optimizedImageResult.value = (parsed.optimizedImageResult as ImageResult | null) ?? null + originalImageResult.value = loadedVariantResults.a + optimizedImageResult.value = loadedVariantResults.b layout.value = parseLayout(parsed.layout) testVariants.value = parseTestVariants(parsed.testVariants) - testVariantResults.value = parseTestVariantResults(parsed.testVariantResults) + testVariantResults.value = loadedVariantResults testVariantLastRunFingerprint.value = parseTestVariantFingerprints( parsed.testVariantLastRunFingerprint, ) diff --git a/packages/ui/tests/unit/stores/session/image-multiimage-session-persistence.spec.ts b/packages/ui/tests/unit/stores/session/image-multiimage-session-persistence.spec.ts index cd5c2ed0..5e607869 100644 --- a/packages/ui/tests/unit/stores/session/image-multiimage-session-persistence.spec.ts +++ b/packages/ui/tests/unit/stores/session/image-multiimage-session-persistence.spec.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi } from 'vitest' import { createTestPinia } from '../../../utils/pinia-test-helpers' import { useImageMultiImageSession } from '../../../../src/stores/session/useImageMultiImageSession' +import { IMAGE_MULTIIMAGE_SESSION_KEY } from '../../../../src/stores/session/imageStorageMaintenance' describe('Session store (image-multiimage) persistence', () => { it('persists the ordered multi-image list and restores it with the same order', async () => { @@ -150,4 +151,96 @@ describe('Session store (image-multiimage) persistence', () => { expect(restored.testVariantLastRunFingerprint.a).toBe('fingerprint-a') expect(restored.isCompareMode).toBe(false) }) + + it('stores generated multi-image results as image refs and restores their payloads from image storage', async () => { + const savedSnapshots = new Map() + const imageMap = new Map() + + const imageStorageService = { + saveImage: vi.fn(async (data: any) => { + imageMap.set(data.metadata.id, { + data: data.data, + metadata: { mimeType: data.metadata.mimeType }, + }) + return data.metadata.id + }), + getImage: vi.fn(async (id: string) => imageMap.get(id) ?? null), + getMetadata: vi.fn(async (id: string) => + imageMap.has(id) + ? { + id, + mimeType: imageMap.get(id)?.metadata.mimeType || 'image/png', + sizeBytes: 4, + createdAt: Date.now(), + accessedAt: Date.now(), + source: 'generated', + } + : null, + ), + listAllMetadata: vi.fn(async () => []), + deleteImages: vi.fn(async () => {}), + } + + const { pinia } = createTestPinia({ + preferenceService: { + get: async (key: string, defaultValue: T) => + (savedSnapshots.has(key) ? savedSnapshots.get(key) : defaultValue) as T, + set: async (key: string, value: unknown) => { + savedSnapshots.set(key, value) + }, + delete: async () => {}, + keys: async () => [], + clear: async () => {}, + getAll: async () => ({}), + exportData: async () => ({}), + importData: async () => {}, + getDataType: async () => 'preference', + validateData: async () => true, + } as any, + imageStorageService: imageStorageService as any, + }) + + const store = useImageMultiImageSession(pinia) + store.updateOriginalImageResult({ + images: [{ b64: 'ORIGINAL', mimeType: 'image/png' }], + metadata: { providerId: 'provider', modelId: 'model-original', configId: 'image-model-original' }, + }) + store.updateOptimizedImageResult({ + images: [{ b64: 'OPTIMIZED', mimeType: 'image/jpeg' }], + metadata: { providerId: 'provider', modelId: 'model-optimized', configId: 'image-model-optimized' }, + }) + store.updateTestVariantResult('c', { + images: [{ b64: 'VARIANTC', mimeType: 'image/png' }], + metadata: { providerId: 'provider', modelId: 'model-c', configId: 'image-model-c' }, + }) + + await store.saveSession() + + const snapshot = savedSnapshots.get(IMAGE_MULTIIMAGE_SESSION_KEY) as Record + + expect(snapshot.originalImageResult.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) }) + expect(snapshot.optimizedImageResult.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) }) + expect(snapshot.testVariantResults.a.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) }) + expect(snapshot.testVariantResults.b.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) }) + expect(snapshot.testVariantResults.c.images[0]).toMatchObject({ _type: 'image-ref', id: expect.any(String) }) + expect(JSON.stringify(snapshot)).not.toContain('ORIGINAL') + expect(JSON.stringify(snapshot)).not.toContain('OPTIMIZED') + expect(JSON.stringify(snapshot)).not.toContain('VARIANTC') + + const restored = useImageMultiImageSession(pinia) + restored.reset() + await restored.restoreSession() + + expect(restored.originalImageResult).toMatchObject({ + images: [{ b64: 'ORIGINAL', mimeType: 'image/png' }], + }) + expect(restored.optimizedImageResult).toMatchObject({ + images: [{ b64: 'OPTIMIZED', mimeType: 'image/jpeg' }], + }) + expect(restored.testVariantResults.c).toMatchObject({ + images: [{ b64: 'VARIANTC', mimeType: 'image/png' }], + }) + expect(imageStorageService.saveImage).toHaveBeenCalledTimes(3) + expect(imageStorageService.getImage).toHaveBeenCalled() + }) })