diff --git a/config.yaml b/config.yaml index bea76ef..917fb57 100644 --- a/config.yaml +++ b/config.yaml @@ -25,10 +25,32 @@ vision: enabled: true # mode 选项: 'ocr' 或 'api' # 'ocr': [默认模式] 彻底免 Key，零配置，完全依赖本机的 CPU 识图，提取文本、报错日志、代码段后发给大模型。 - # 'api': 需要配置下方的 baseUrl 和 apiKey，把图发给外部视觉模型（如 Gemini、OpenRouter），能“看到”画面内容和色彩。 + # 'api': 需要配置下方的 providers，把图发给外部视觉模型（如 Gemini、OpenRouter），能"看到"画面内容和色彩。 mode: 'ocr' - + # ---------- 以下选项仅在 mode: 'api' 时才生效 ---------- + + # 是否在所有 API 都失败时兜底使用本地 OCR（默认: true） + # fallback_to_ocr: true + + # API 提供者列表（按顺序尝试，第一个成功即返回，失败则自动尝试下一个） + # providers: + # - name: "openrouter-free" # 名称（仅用于日志显示） + # base_url: "https://openrouter.ai/api/v1/chat/completions" + # api_key: "sk-or-v1-..." + # model: "meta-llama/llama-3.2-11b-vision-instruct:free" + # + # - name: "gemini-backup" + # base_url: "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" + # api_key: "AIza..." + # model: "gemini-2.0-flash" + # + # - name: "openai-premium" + # base_url: "https://api.openai.com/v1/chat/completions" + # api_key: "sk-..." + # model: "gpt-4o-mini" + + # ---------- 兼容旧版单 API 写法（不推荐，建议改用 providers） ---------- # base_url: "https://openrouter.ai/api/v1/chat/completions" # api_key: "sk-or-v1-..." # model: "meta-llama/llama-3.2-11b-vision-instruct:free" diff --git a/src/config.ts b/src/config.ts index 63d8ae6..95358fd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,6 @@ import { readFileSync, existsSync } from 'fs'; import { parse as parseYaml } from 'yaml'; -import type { AppConfig } from './types.js'; +import type { AppConfig, VisionProvider } from './types.js'; let config: AppConfig; @@ -32,9 +32,30 @@ export function getConfig(): AppConfig { if (yaml.fingerprint.user_agent) config.fingerprint.userAgent = yaml.fingerprint.user_agent; } if (yaml.vision) { + // Parse providers array + let providers: VisionProvider[] = []; + if (Array.isArray(yaml.vision.providers)) { + providers = yaml.vision.providers.map((p: any) => ({ + name: p.name || '', + baseUrl: p.base_url || 'https://api.openai.com/v1/chat/completions', + apiKey: p.api_key || '', + model: p.model || 'gpt-4o-mini', + })); + } else if (yaml.vision.base_url && yaml.vision.api_key) { + // Backward compat: single provider from legacy fields + providers = [{ + name: 'default', + baseUrl: yaml.vision.base_url, + apiKey: yaml.vision.api_key, + model: yaml.vision.model || 'gpt-4o-mini', + }]; + } + config.vision = { - enabled: yaml.vision.enabled !== false, // default to true if vision section exists in some way + enabled: yaml.vision.enabled !== false, mode: yaml.vision.mode || 'ocr', + providers, + fallbackToOcr: yaml.vision.fallback_to_ocr !== false, // default true baseUrl: yaml.vision.base_url || 'https://api.openai.com/v1/chat/completions', apiKey: yaml.vision.api_key || '', model: yaml.vision.model || 'gpt-4o-mini', diff --git a/src/types.ts b/src/types.ts index 595bc83..9f57b2d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -118,6 +118,11 @@ export interface AppConfig { vision?: { enabled: boolean; mode: 'ocr' | 'api'; + /** Multiple API providers to try in order; used when mode is 'api' */ + providers: VisionProvider[]; + /** If all API providers fail, fall back to local OCR (default: true) */ + fallbackToOcr: boolean; + // Legacy single-provider fields kept for backward compat baseUrl: string; apiKey: string; model: string; @@ -126,3 +131,11 @@ export interface AppConfig { userAgent: string; }; } + +export interface VisionProvider { + name?: string; + baseUrl: string; + apiKey: string; + model: string; +} + diff --git a/src/vision.ts b/src/vision.ts index b685395..86d7002 100644 --- a/src/vision.ts +++ b/src/vision.ts @@ -1,5 +1,5 @@ import { getConfig } from './config.js'; -import type { AnthropicMessage, AnthropicContentBlock } from './types.js'; +import type { AnthropicMessage, AnthropicContentBlock, VisionProvider } from './types.js'; import { getProxyFetchOptions } from './proxy-agent.js'; import { createWorker } from 'tesseract.js'; import crypto from 'crypto'; @@ -51,8 +51,8 @@ export async function applyVisionInterceptor(messages: AnthropicMessage[]): Prom console.log(`[Vision] 启用纯本地 OCR 模式，正在处理 ${imagesToAnalyze.length} 张图片... (无需 API Key)`); descriptions = await processWithLocalOCR(imagesToAnalyze); } else { - console.log(`[Vision] 启用外部 API 模式，正在处理 ${imagesToAnalyze.length} 张图片...`); - descriptions = await callVisionAPI(imagesToAnalyze); + // API mode: try providers in order with fallback + descriptions = await processWithAPIFallback(imagesToAnalyze); } // Add descriptions as a simulated system text block @@ -74,6 +74,71 @@ export async function applyVisionInterceptor(messages: AnthropicMessage[]): Prom } } +/** + * Try each API provider in order. If all fail and fallbackToOcr is enabled, + * fall back to local OCR as the last resort. + */ +async function processWithAPIFallback(imagesToAnalyze: AnthropicContentBlock[]): Promise { + const visionConfig = getConfig().vision!; + const providers = visionConfig.providers; + const errors: string[] = []; + + // If we have providers, try them in order + if (providers.length > 0) { + for (let i = 0; i < providers.length; i++) { + const provider = providers[i]; + const providerLabel = provider.name || `Provider #${i + 1} (${provider.model})`; + try { + console.log(`[Vision] 尝试 API ${providerLabel}，正在处理 ${imagesToAnalyze.length} 张图片...`); + const result = await callVisionAPIWithProvider(imagesToAnalyze, provider); + console.log(`[Vision] ✅ ${providerLabel} 处理成功`); + return result; + } catch (err) { + const errMsg = (err as Error).message; + console.warn(`[Vision] ❌ ${providerLabel} 失败: ${errMsg}`); + errors.push(`${providerLabel}: ${errMsg}`); + // Continue to next provider + } + } + } else if (visionConfig.baseUrl && visionConfig.apiKey) { + // Legacy fallback: single provider from top-level fields + const legacyProvider: VisionProvider = { + name: 'default', + baseUrl: visionConfig.baseUrl, + apiKey: visionConfig.apiKey, + model: visionConfig.model, + }; + try { + console.log(`[Vision] 启用外部 API 模式，正在处理 ${imagesToAnalyze.length} 张图片...`); + const result = await callVisionAPIWithProvider(imagesToAnalyze, legacyProvider); + return result; + } catch (err) { + const errMsg = (err as Error).message; + console.warn(`[Vision] ❌ API 调用失败: ${errMsg}`); + errors.push(`default: ${errMsg}`); + } + } + + // All API providers failed — try OCR fallback + if (visionConfig.fallbackToOcr) { + console.log(`[Vision] 所有 API 均失败 (${errors.length} 个错误)，兜底使用本地 OCR...`); + try { + return await processWithLocalOCR(imagesToAnalyze); + } catch (ocrErr) { + throw new Error( + `All ${errors.length} API provider(s) failed AND local OCR fallback also failed. ` + + `API errors: [${errors.join(' | ')}]. OCR error: ${(ocrErr as Error).message}` + ); + } + } + + // fallbackToOcr is disabled and all providers failed + throw new Error( + `All ${errors.length} API provider(s) failed and fallback_to_ocr is disabled. ` + + `Errors: [${errors.join(' | ')}]` + ); +} + async function processWithLocalOCR(imageBlocks: AnthropicContentBlock[]): Promise { let combinedText = ''; const imagesToProcess: { index: number, source: string, hash: string }[] = []; @@ -123,11 +188,15 @@ async function processWithLocalOCR(imageBlocks: AnthropicContentBlock[]): Promis return combinedText; } -async function callVisionAPI(imageBlocks: AnthropicContentBlock[]): Promise { - const config = getConfig().vision!; +/** + * Call a specific Vision API provider for image analysis. + * Processes images individually for per-image caching. + * Throws on failure so the caller can try the next provider. + */ +async function callVisionAPIWithProvider(imageBlocks: AnthropicContentBlock[], provider: VisionProvider): Promise { let combinedText = ''; + let hasAnyFailure = false; - // We will process images individually to be able to cache them separately for (let i = 0; i < imageBlocks.length; i++) { const img = imageBlocks[i]; let url = ''; @@ -155,35 +224,31 @@ async function callVisionAPI(imageBlocks: AnthropicContentBlock[]): Promise