From 90a4ef66ea47230e38d7fe57bdc439001c756fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B5=B7?= <7836246@qq.com> Date: Fri, 13 Mar 2026 11:29:22 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=9B=BE=E7=89=87=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=20API=20Provider=20=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E5=B0=9D=E8=AF=95=20+=20=E5=85=9C=E5=BA=95=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=20OCR=20(#27)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - vision 配置新增 `providers` 数组，按顺序尝试多个外部视觉 API - 新增 `fallback_to_ocr` 选项（默认 true），所有 API 失败时自动降级为本地 OCR - 新增 `VisionProvider` 接口，每个 provider 可独立配置 name/base_url/api_key/model - 完全兼容旧版单 API 写法（base_url + api_key），自动转换为单元素 providers - config.yaml 更新多 provider 配置示例及中文说明 --- config.yaml | 26 ++++++++++- src/config.ts | 25 +++++++++- src/types.ts | 13 ++++++ src/vision.ts | 123 ++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 154 insertions(+), 33 deletions(-) diff --git a/config.yaml b/config.yaml index bea76ef..917fb57 100644 --- a/config.yaml +++ b/config.yaml @@ -25,10 +25,32 @@ vision: enabled: true # mode 选项: 'ocr' 或 'api' # 'ocr': [默认模式] 彻底免 Key，零配置，完全依赖本机的 CPU 识图，提取文本、报错日志、代码段后发给大模型。 - # 'api': 需要配置下方的 baseUrl 和 apiKey，把图发给外部视觉模型（如 Gemini、OpenRouter），能“看到”画面内容和色彩。 + # 'api': 需要配置下方的 providers，把图发给外部视觉模型（如 Gemini、OpenRouter），能"看到"画面内容和色彩。 mode: 'ocr' - + # ---------- 以下选项仅在 mode: 'api' 时才生效 ---------- + + # 是否在所有 API 都失败时兜底使用本地 OCR（默认: true） + # fallback_to_ocr: true + + # API 提供者列表（按顺序尝试，第一个成功即返回，失败则自动尝试下一个） + # providers: + # - name: "openrouter-free" # 名称（仅用于日志显示） + # base_url: "https://openrouter.ai/api/v1/chat/completions" + # api_key: "sk-or-v1-..." + # model: "meta-llama/llama-3.2-11b-vision-instruct:free" + # + # - name: "gemini-backup" + # base_url: "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" + # api_key: "AIza..." + # model: "gemini-2.0-flash" + # + # - name: "openai-premium" + # base_url: "https://api.openai.com/v1/chat/completions" + # api_key: "sk-..." + # model: "gpt-4o-mini" + + # ---------- 兼容旧版单 API 写法（不推荐，建议改用 providers） ---------- # base_url: "https://openrouter.ai/api/v1/chat/completions" # api_key: "sk-or-v1-..." # model: "meta-llama/llama-3.2-11b-vision-instruct:free" diff --git a/src/config.ts b/src/config.ts index 63d8ae6..95358fd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,6 @@ import { readFileSync, existsSync } from 'fs'; import { parse as parseYaml } from 'yaml'; -import type { AppConfig } from './types.js'; +import type { AppConfig, VisionProvider } from './types.js'; let config: AppConfig; @@ -32,9 +32,30 @@ export function getConfig(): AppConfig { if (yaml.fingerprint.user_agent) config.fingerprint.userAgent = yaml.fingerprint.user_agent; } if (yaml.vision) { + // Parse providers array + let providers: VisionProvider[] = []; + if (Array.isArray(yaml.vision.providers)) { + providers = yaml.vision.providers.map((p: any) => ({ + name: p.name || '', + baseUrl: p.base_url || 'https://api.openai.com/v1/chat/completions', + apiKey: p.api_key || '', + model: p.model || 'gpt-4o-mini', + })); + } else if (yaml.vision.base_url && yaml.vision.api_key) { + // Backward compat: single provider from legacy fields + providers = [{ + name: 'default', + baseUrl: yaml.vision.base_url, + apiKey: yaml.vision.api_key, + model: yaml.vision.model || 'gpt-4o-mini', + }]; + } + config.vision = { - enabled: yaml.vision.enabled !== false, // default to true if vision section exists in some way + enabled: yaml.vision.enabled !== false, mode: yaml.vision.mode || 'ocr', + providers, + fallbackToOcr: yaml.vision.fallback_to_ocr !== false, // default true baseUrl: yaml.vision.base_url || 'https://api.openai.com/v1/chat/completions', apiKey: yaml.vision.api_key || '', model: yaml.vision.model || 'gpt-4o-mini', diff --git a/src/types.ts b/src/types.ts index 595bc83..9f57b2d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -118,6 +118,11 @@ export interface AppConfig { vision?: { enabled: boolean; mode: 'ocr' | 'api'; + /** Multiple API providers to try in order; used when mode is 'api' */ + providers: VisionProvider[]; + /** If all API providers fail, fall back to local OCR (default: true) */ + fallbackToOcr: boolean; + // Legacy single-provider fields kept for backward compat baseUrl: string; apiKey: string; model: string; @@ -126,3 +131,11 @@ export interface AppConfig { userAgent: string; }; } + +export interface VisionProvider { + name?: string; + baseUrl: string; + apiKey: string; + model: string; +} + diff --git a/src/vision.ts b/src/vision.ts index b685395..86d7002 100644 --- a/src/vision.ts +++ b/src/vision.ts @@ -1,5 +1,5 @@ import { getConfig } from './config.js'; -import type { AnthropicMessage, AnthropicContentBlock } from './types.js'; +import type { AnthropicMessage, AnthropicContentBlock, VisionProvider } from './types.js'; import { getProxyFetchOptions } from './proxy-agent.js'; import { createWorker } from 'tesseract.js'; import crypto from 'crypto'; @@ -51,8 +51,8 @@ export async function applyVisionInterceptor(messages: AnthropicMessage[]): Prom console.log(`[Vision] 启用纯本地 OCR 模式，正在处理 ${imagesToAnalyze.length} 张图片... (无需 API Key)`); descriptions = await processWithLocalOCR(imagesToAnalyze); } else { - console.log(`[Vision] 启用外部 API 模式，正在处理 ${imagesToAnalyze.length} 张图片...`); - descriptions = await callVisionAPI(imagesToAnalyze); + // API mode: try providers in order with fallback + descriptions = await processWithAPIFallback(imagesToAnalyze); } // Add descriptions as a simulated system text block @@ -74,6 +74,71 @@ export async function applyVisionInterceptor(messages: AnthropicMessage[]): Prom } } +/** + * Try each API provider in order. If all fail and fallbackToOcr is enabled, + * fall back to local OCR as the last resort. + */ +async function processWithAPIFallback(imagesToAnalyze: AnthropicContentBlock[]): Promise { + const visionConfig = getConfig().vision!; + const providers = visionConfig.providers; + const errors: string[] = []; + + // If we have providers, try them in order + if (providers.length > 0) { + for (let i = 0; i < providers.length; i++) { + const provider = providers[i]; + const providerLabel = provider.name || `Provider #${i + 1} (${provider.model})`; + try { + console.log(`[Vision] 尝试 API ${providerLabel}，正在处理 ${imagesToAnalyze.length} 张图片...`); + const result = await callVisionAPIWithProvider(imagesToAnalyze, provider); + console.log(`[Vision] ✅ ${providerLabel} 处理成功`); + return result; + } catch (err) { + const errMsg = (err as Error).message; + console.warn(`[Vision] ❌ ${providerLabel} 失败: ${errMsg}`); + errors.push(`${providerLabel}: ${errMsg}`); + // Continue to next provider + } + } + } else if (visionConfig.baseUrl && visionConfig.apiKey) { + // Legacy fallback: single provider from top-level fields + const legacyProvider: VisionProvider = { + name: 'default', + baseUrl: visionConfig.baseUrl, + apiKey: visionConfig.apiKey, + model: visionConfig.model, + }; + try { + console.log(`[Vision] 启用外部 API 模式，正在处理 ${imagesToAnalyze.length} 张图片...`); + const result = await callVisionAPIWithProvider(imagesToAnalyze, legacyProvider); + return result; + } catch (err) { + const errMsg = (err as Error).message; + console.warn(`[Vision] ❌ API 调用失败: ${errMsg}`); + errors.push(`default: ${errMsg}`); + } + } + + // All API providers failed — try OCR fallback + if (visionConfig.fallbackToOcr) { + console.log(`[Vision] 所有 API 均失败 (${errors.length} 个错误)，兜底使用本地 OCR...`); + try { + return await processWithLocalOCR(imagesToAnalyze); + } catch (ocrErr) { + throw new Error( + `All ${errors.length} API provider(s) failed AND local OCR fallback also failed. ` + + `API errors: [${errors.join(' | ')}]. OCR error: ${(ocrErr as Error).message}` + ); + } + } + + // fallbackToOcr is disabled and all providers failed + throw new Error( + `All ${errors.length} API provider(s) failed and fallback_to_ocr is disabled. ` + + `Errors: [${errors.join(' | ')}]` + ); +} + async function processWithLocalOCR(imageBlocks: AnthropicContentBlock[]): Promise { let combinedText = ''; const imagesToProcess: { index: number, source: string, hash: string }[] = []; @@ -123,11 +188,15 @@ async function processWithLocalOCR(imageBlocks: AnthropicContentBlock[]): Promis return combinedText; } -async function callVisionAPI(imageBlocks: AnthropicContentBlock[]): Promise { - const config = getConfig().vision!; +/** + * Call a specific Vision API provider for image analysis. + * Processes images individually for per-image caching. + * Throws on failure so the caller can try the next provider. + */ +async function callVisionAPIWithProvider(imageBlocks: AnthropicContentBlock[], provider: VisionProvider): Promise { let combinedText = ''; + let hasAnyFailure = false; - // We will process images individually to be able to cache them separately for (let i = 0; i < imageBlocks.length; i++) { const img = imageBlocks[i]; let url = ''; @@ -155,35 +224,31 @@ async function callVisionAPI(imageBlocks: AnthropicContentBlock[]): Promise