diff --git a/src/config/index.ts b/src/config/index.ts index 1775785..8e9826a 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -16,6 +16,7 @@ export interface Config { logDir: string; logToFile: boolean; logConsole: boolean; + maxLengthBreak: number; } let globalConfig: Config | null = null; @@ -102,21 +103,22 @@ export function getConfig(): Config { configDir, modelDir, logDir, - + logLevel: getString('--log-level', 'MT_LOG_LEVEL', 'warn'), host: getString('--host', 'MT_HOST', '0.0.0.0'), port: getString('--port', 'MT_PORT', '8989'), - + enableWebUI: getBool('--ui', 'MT_ENABLE_UI', true), enableOfflineMode: getBool('--offline', 'MT_OFFLINE', false), - + workerIdleTimeout: getInt('--worker-idle-timeout', 'MT_WORKER_IDLE_TIMEOUT', 60), workersPerLanguage: getInt('--workers-per-language', 'MT_WORKERS_PER_LANGUAGE', 1), - + maxLengthBreak: getInt('--max-length-break', 'MT_MAX_LENGTH_BREAK', 128), + apiToken: getString('--api-token', 'MT_API_TOKEN', ''), - + logToFile: getBool('--log-to-file', 'MT_LOG_TO_FILE', false), - logConsole: getBool('--log-console', 'MT_LOG_CONSOLE', true), // Use --no-log-console to disable + logConsole: getBool('--log-console', 'MT_LOG_CONSOLE', true), }; return globalConfig; diff --git a/src/handlers/language.ts b/src/handlers/language.ts index 5cf62d5..6cb4cc6 100644 --- a/src/handlers/language.ts +++ b/src/handlers/language.ts @@ -1,5 +1,6 @@ import { Request, Response } from 'express'; import { getSupportedLanguages, getLanguagePairs } from '../models/index.js'; +import { detectLanguage, detectLanguageWithConfidence } from '../services/detector.js'; export async function handleLanguages(req: Request, res: Response) { try { @@ -14,3 +15,24 @@ export async function handleLanguages(req: Request, res: Response) { res.status(500).json({ error: `Failed to get languages: ${error}` }); } } + +export async function handleDetectLanguage(req: Request, res: Response) { + try { + const { text, minConfidence } = req.body; + + if (!text || typeof text !== 'string') { + res.status(400).json({ error: 'Text is required' }); + return; + } + + if (minConfidence !== undefined) { + const result = await detectLanguageWithConfidence(text, minConfidence); + res.json(result); + } else { + const language = await detectLanguage(text); + res.json({ language }); + } + } catch (error) { + res.status(500).json({ error: `Language detection failed: ${error}` }); + } +} diff --git a/src/routes/index.ts b/src/routes/index.ts index b4db951..d0827ef 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -6,6 +6,7 @@ import { handleHeartbeat, handleLBHeartbeat, handleLanguages, + handleDetectLanguage, handleTranslate, handleTranslateBatch, handleDeeplTranslate, @@ -27,6 +28,7 @@ export function setupRoutes(app: Express, apiToken: string) { const authMiddleware = auth(apiToken); app.get('/languages', authMiddleware, handleLanguages); + app.post('/detect', authMiddleware, handleDetectLanguage); app.post('/translate', authMiddleware, handleTranslate); app.post('/translate/batch', authMiddleware, handleTranslateBatch); diff --git a/src/services/detector.ts b/src/services/detector.ts new file mode 100644 index 0000000..c237e79 --- /dev/null +++ b/src/services/detector.ts @@ -0,0 +1,307 @@ +import { createRequire } from 'module'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import * as logger from '../logger/index.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export interface TextSegment { + text: string; + language: string; + start: number; + end: number; + confidence: number; +} + +const DEFAULT_CONFIDENCE_THRESHOLD = 0.5; +const MAXIMUM_LANGUAGES_IN_ONE_TEXT = 2; + +let cldModule: any = null; +let initPromise: Promise | null = null; + +async function initCLD(): Promise { + if (cldModule) return; + + if (initPromise) { + return initPromise; + } + + initPromise = (async () => { + try { + logger.debug('Initializing CLD language detector'); + + const require = createRequire(import.meta.url); + const cldWorkerPath = path.resolve(__dirname, '../lib/cld/cld-worker.js'); + + (global as any).Module = { + print: (msg: string) => logger.debug(`[CLD]: ${msg}`), + printErr: (msg: string) => logger.error(`[CLD Error]: ${msg}`), + locateFile: (filePath: string) => { + if (filePath.endsWith('.wasm')) { + return path.resolve(__dirname, '../lib/cld', filePath); + } + return filePath; + }, + noExitRuntime: true + }; + + const cldModule_ = require(cldWorkerPath); + + await new Promise((resolve) => { + if (cldModule_.onRuntimeInitialized) { + cldModule_.onRuntimeInitialized = () => { + logger.debug('CLD runtime initialized'); + resolve(); + }; + } else { + resolve(); + } + }); + + cldModule = cldModule_; + logger.debug('CLD language detector initialized'); + } catch (error) { + logger.error(`Failed to initialize CLD: ${error}`); + throw error; + } + })(); + + return initPromise; +} + +function detectLanguageWithCLD(text: string, isHTML: boolean = false) { + if (!cldModule) { + throw new Error('CLD module not initialized'); + } + + const result = cldModule.LanguageInfo.O(text, !isHTML); + + const languages = Array(3).fill(0).map((_, i) => { + const lang = result.U(i); + return { + languageCode: lang.P(), + percent: lang.da() + }; + }).filter(l => l.languageCode !== 'un' || l.percent > 0); + + const output = { + language: result.P(), + confident: result.ba(), + languages, + percentScore: languages[0]?.percent || 0 + }; + + cldModule.destroy(result); + + return output; +} + +function bcp47Normalize(code: string): string { + switch (code) { + case 'zh': + return 'zh-Hans'; + default: + return code.toLowerCase(); + } +} + +export async function detectLanguage(text: string): Promise { + if (!text) { + return ''; + } + + await initCLD(); + + try { + const result = detectLanguageWithCLD(text); + return bcp47Normalize(result.language); + } catch (error) { + logger.error(`Language detection failed: ${error}`); + return ''; + } +} + +export async function detectLanguageWithConfidence( + text: string, + minConfidence: number = DEFAULT_CONFIDENCE_THRESHOLD +): Promise<{ language: string; confidence: number }> { + if (!text) { + return { language: '', confidence: 0 }; + } + + await initCLD(); + + try { + const result = detectLanguageWithCLD(text); + const confidence = result.percentScore / 100; + + if (confidence < minConfidence) { + return { language: '', confidence }; + } + + return { + language: bcp47Normalize(result.language), + confidence + }; + } catch (error) { + logger.error(`Language detection with confidence failed: ${error}`); + return { language: '', confidence: 0 }; + } +} + +function hasMixedScripts(text: string): boolean { + let hasCJK = false; + let hasLatin = false; + + for (const char of text) { + const code = char.charCodeAt(0); + + if ( + (code >= 0x4e00 && code <= 0x9fff) || + (code >= 0x3040 && code <= 0x309f) || + (code >= 0x30a0 && code <= 0x30ff) || + (code >= 0xac00 && code <= 0xd7af) + ) { + hasCJK = true; + } else if ((code >= 0x0041 && code <= 0x005a) || (code >= 0x0061 && code <= 0x007a)) { + hasLatin = true; + } + + if (hasCJK && hasLatin) { + return true; + } + } + + return false; +} + +export async function detectMultipleLanguages(text: string): Promise { + return detectMultipleLanguagesWithThreshold(text, DEFAULT_CONFIDENCE_THRESHOLD); +} + +export async function detectMultipleLanguagesWithThreshold( + text: string, + threshold: number +): Promise { + if (!text) { + return []; + } + + await initCLD(); + + const fallbackLang = await detectLanguage(text); + const effectiveFallback = fallbackLang || 'en'; + + if (!hasMixedScripts(text)) { + logger.debug(`DetectMultipleLanguages: no mixed scripts, using single language: ${effectiveFallback}`); + return [{ + text, + language: effectiveFallback, + start: 0, + end: text.length, + confidence: 1.0 + }]; + } + + logger.debug(`DetectMultipleLanguages: mixed scripts detected, fallback=${effectiveFallback}, threshold=${threshold.toFixed(2)}`); + + const segments: TextSegment[] = []; + + const segmenterAny = new (Intl as any).Segmenter(undefined, { granularity: 'sentence' }); + const sentenceSegments = Array.from(segmenterAny.segment(text)) as Array<{segment: string, index: number}>; + + for (const { segment, index } of sentenceSegments) { + try { + const result = detectLanguageWithCLD(segment); + const detectedLang = bcp47Normalize(result.language); + const confidence = result.percentScore / 100; + + segments.push({ + text: segment, + language: confidence >= threshold ? detectedLang : effectiveFallback, + start: index, + end: index + segment.length, + confidence + }); + } catch (error) { + logger.warn(`Failed to detect language for segment: ${error}`); + segments.push({ + text: segment, + language: effectiveFallback, + start: index, + end: index + segment.length, + confidence: 0 + }); + } + } + + const mergedSegments = mergeAdjacentSegments(segments, text); + const limitedSegments = limitLanguages(mergedSegments, text, MAXIMUM_LANGUAGES_IN_ONE_TEXT); + + logger.debug(`DetectMultipleLanguages: ${sentenceSegments.length} sentences -> ${mergedSegments.length} merged -> ${limitedSegments.length} final segments`); + + return limitedSegments; +} + +function mergeAdjacentSegments(segments: TextSegment[], originalText: string): TextSegment[] { + if (segments.length <= 1) { + return segments; + } + + const merged: TextSegment[] = []; + let current = segments[0]; + + for (let i = 1; i < segments.length; i++) { + const next = segments[i]; + if (current.language === next.language) { + current.text = originalText.substring(current.start, next.end); + current.end = next.end; + if (next.confidence > current.confidence) { + current.confidence = next.confidence; + } + } else { + merged.push(current); + current = next; + } + } + merged.push(current); + + return merged; +} + +function limitLanguages( + segments: TextSegment[], + originalText: string, + maxLangs: number +): TextSegment[] { + if (segments.length <= 1) { + return segments; + } + + const langBytes = new Map(); + for (const seg of segments) { + langBytes.set(seg.language, (langBytes.get(seg.language) || 0) + (seg.end - seg.start)); + } + + if (langBytes.size <= maxLangs) { + return segments; + } + + const sorted = Array.from(langBytes.entries()) + .sort((a, b) => b[1] - a[1]); + + const keepLangs = new Set(sorted.slice(0, maxLangs).map(([lang]) => lang)); + const primaryLang = sorted[0][0]; + + for (const seg of segments) { + if (!keepLangs.has(seg.language)) { + seg.language = primaryLang; + } + } + + const result = mergeAdjacentSegments(segments, originalText); + logger.debug(`limitLanguages: reduced to ${maxLangs} languages, ${result.length} segments`); + + return result; +} diff --git a/src/services/engine.ts b/src/services/engine.ts index d714ede..0c7bbce 100644 --- a/src/services/engine.ts +++ b/src/services/engine.ts @@ -6,6 +6,7 @@ import loadBergamot from '../lib/bergamot/bergamot-translator.js'; import wasmPath from '../lib/bergamot/bergamot-translator.wasm' with { type: 'file' }; import * as logger from '../logger/index.js'; import * as models from '../models/index.js'; +import { detectLanguage, detectMultipleLanguages } from './detector.js'; interface EngineInfo { engine: TranslationEngine; @@ -190,11 +191,102 @@ export async function translateWithPivot( `TranslateWithPivot: ${fromLang} -> ${toLang}, text length: ${text.length}, isHTML: ${isHTML}` ); - if (fromLang === toLang) { + if (fromLang !== 'auto' && fromLang === toLang) { return text; } - return translateSegment(fromLang, toLang, text, isHTML); + if (fromLang !== 'auto' && text.length <= 128) { + return translateSegment(fromLang, toLang, text, isHTML); + } + + const config = getConfig(); + const segments = await detectMultipleLanguages(text); + + if (segments.length <= 1) { + let effectiveFromLang: string; + if (segments.length === 1) { + effectiveFromLang = segments[0].language; + } else if (fromLang === 'auto') { + const detected = await detectLanguage(text); + if (!detected) { + throw new Error('Failed to detect source language'); + } + effectiveFromLang = detected; + } else { + effectiveFromLang = fromLang; + } + + if (effectiveFromLang === toLang) { + return text; + } + + if (text.length > config.maxLengthBreak && !isHTML) { + return translateLongText(effectiveFromLang, toLang, text); + } + + return translateSegment(effectiveFromLang, toLang, text, isHTML); + } + + logger.debug(`Detected ${segments.length} language segments`); + let result = ''; + let lastEnd = 0; + + for (const seg of segments) { + if (seg.start > lastEnd) { + result += text.substring(lastEnd, seg.start); + } + + if (seg.language === toLang) { + result += seg.text; + } else { + try { + const translated = await translateSegment(seg.language, toLang, seg.text, isHTML); + result += translated; + } catch (error) { + logger.error(`Failed to translate segment: ${error}`); + result += seg.text; + } + } + lastEnd = seg.end; + } + + if (lastEnd < text.length) { + result += text.substring(lastEnd); + } + + return result; +} + +async function translateLongText( + fromLang: string, + toLang: string, + text: string +): Promise { + logger.debug(`Splitting long text (${text.length} chars) into sentences`); + + const segmenterAny = new (Intl as any).Segmenter(undefined, { granularity: 'sentence' }); + const sentences = Array.from(segmenterAny.segment(text)) as Array<{segment: string, index: number}>; + + logger.debug(`Split into ${sentences.length} sentences`); + + const results: string[] = []; + + for (let i = 0; i < sentences.length; i++) { + const { segment } = sentences[i]; + try { + const translated = await translateSegment(fromLang, toLang, segment, false); + results.push(translated); + + if ((i + 1) % 10 === 0) { + logger.debug(`Translated ${i + 1}/${sentences.length} sentences`); + } + } catch (error) { + logger.error(`Failed to translate sentence ${i + 1}: ${error}`); + results.push(segment); + } + } + + return results.join(''); } export function cleanupAllEngines() { diff --git a/src/services/index.ts b/src/services/index.ts index e35ce04..cc4a485 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1 +1,2 @@ export * from './engine.js'; +export * from './detector.js';