mirror of
https://github.com/xxnuo/MTranServer.git
synced 2026-09-03 06:35:20 +08:00
feat: auto 自动语言检测
This commit is contained in:
xxnuo
3
go.mod
3
go.mod
@@ -79,9 +79,11 @@ require (
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pemistahl/lingua-go v1.4.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.56.0 // indirect
|
||||
github.com/shopspring/decimal v1.3.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.3.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
@@ -90,6 +92,7 @@ require (
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.23.0 // indirect
|
||||
golang.org/x/crypto v0.43.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/net v0.46.0 // indirect
|
||||
golang.org/x/oauth2 v0.27.0 // indirect
|
||||
|
||||
6
go.sum
6
go.sum
@@ -193,6 +193,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=
|
||||
github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
@@ -202,6 +204,8 @@ github.com/quic-go/quic-go v0.56.0 h1:q/TW+OLismmXAehgFLczhCDTYB3bFmua4D9lsNBWxv
|
||||
github.com/quic-go/quic-go v0.56.0/go.mod h1:9gx5KsFQtw2oZ6GZTyh+7YEvOxWCL9WZAepnHxgAo6c=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -241,6 +245,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
|
||||
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 h1:Fq7F/w7MAa1KJ5bt2aJ62ihqp9HDcRuyILskkpIAurw=
|
||||
golang.org/x/exp v0.0.0-20221106115401-f9659909a136/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
|
||||
@@ -46,7 +46,7 @@ func HandleTranslate(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// 使用 TranslateWithPivot 处理可能需要中转的翻译
|
||||
// 使用 TranslateWithPivot 处理可能需要中转的翻译（支持 auto 模式）
|
||||
ctx, cancel := context.WithTimeout(c.Request.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -99,7 +99,7 @@ func HandleTranslateBatch(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// 批量翻译，使用 TranslateWithPivot 处理可能需要中转的翻译
|
||||
// 批量翻译，使用 TranslateWithPivot 处理可能需要中转的翻译（支持 auto 模式）
|
||||
results := make([]string, len(req.Texts))
|
||||
ctx, cancel := context.WithTimeout(c.Request.Context(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
110
internal/services/detector.go
Normal file
110
internal/services/detector.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/pemistahl/lingua-go"
|
||||
)
|
||||
|
||||
var (
|
||||
detector lingua.LanguageDetector
|
||||
detectorOnce sync.Once
|
||||
)
|
||||
|
||||
// initDetector 初始化语言检测器（懒加载）
|
||||
func initDetector() {
|
||||
detectorOnce.Do(func() {
|
||||
detector = lingua.NewLanguageDetectorBuilder().
|
||||
FromAllLanguages().
|
||||
WithPreloadedLanguageModels().
|
||||
Build()
|
||||
})
|
||||
}
|
||||
|
||||
// linguaToBCP47 将 lingua 语言代码转换为 BCP 47 格式
|
||||
func linguaToBCP47(lang lingua.Language) string {
|
||||
// 特殊处理中文
|
||||
switch lang {
|
||||
case lingua.Chinese:
|
||||
// 默认返回简体中文
|
||||
return "zh-Hans"
|
||||
default:
|
||||
// 其他语言直接返回 ISO 639-1 代码（小写）
|
||||
code := lang.IsoCode639_1()
|
||||
return strings.ToLower(code.String())
|
||||
}
|
||||
}
|
||||
|
||||
// DetectLanguage 检测文本语言并返回 BCP 47 格式的语言代码
|
||||
// 如果检测失败或无法确定，返回空字符串
|
||||
func DetectLanguage(text string) string {
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 初始化检测器
|
||||
initDetector()
|
||||
|
||||
// 检测语言
|
||||
lang, exists := detector.DetectLanguageOf(text)
|
||||
if !exists {
|
||||
return ""
|
||||
}
|
||||
|
||||
return linguaToBCP47(lang)
|
||||
}
|
||||
|
||||
// DetectLanguageWithConfidence 检测文本语言并返回置信度
|
||||
// 返回 BCP 47 格式的语言代码和置信度（0.0-1.0）
|
||||
func DetectLanguageWithConfidence(text string, minConfidence float64) (string, float64) {
|
||||
if text == "" {
|
||||
return "", 0.0
|
||||
}
|
||||
|
||||
// 初始化检测器
|
||||
initDetector()
|
||||
|
||||
// 获取所有语言的置信度
|
||||
confidenceValues := detector.ComputeLanguageConfidenceValues(text)
|
||||
if len(confidenceValues) == 0 {
|
||||
return "", 0.0
|
||||
}
|
||||
|
||||
// 获取置信度最高的语言
|
||||
topResult := confidenceValues[0]
|
||||
confidence := topResult.Value()
|
||||
|
||||
// 如果置信度低于阈值，返回空
|
||||
if confidence < minConfidence {
|
||||
return "", confidence
|
||||
}
|
||||
|
||||
return linguaToBCP47(topResult.Language()), confidence
|
||||
}
|
||||
|
||||
// NormalizeLanguageCode 标准化语言代码为 BCP 47 格式
|
||||
// 支持多种输入格式：zh, zh-CN, zh_CN, Chinese 等
|
||||
func NormalizeLanguageCode(code string) string {
|
||||
if code == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 转为小写并替换下划线为连字符
|
||||
code = strings.ToLower(strings.ReplaceAll(code, "_", "-"))
|
||||
|
||||
// 特殊处理中文
|
||||
switch code {
|
||||
case "zh", "zh-cn", "zh-hans", "chinese", "cmn":
|
||||
return "zh-Hans"
|
||||
case "zh-tw", "zh-hk", "zh-hant":
|
||||
return "zh-Hant"
|
||||
}
|
||||
|
||||
// 如果是标准的 BCP 47 格式（如 en-US），提取主语言代码
|
||||
parts := strings.Split(code, "-")
|
||||
mainCode := parts[0]
|
||||
|
||||
// 返回主语言代码
|
||||
return mainCode
|
||||
}
|
||||
@@ -203,7 +203,24 @@ func GetOrCreateEngine(fromLang, toLang string) (*manager.Manager, error) {
|
||||
|
||||
// TranslateWithPivot 处理可能需要中转的翻译
|
||||
// 如果需要中转（如 zh-Hans -> ja），会自动创建两个引擎并执行两步翻译
|
||||
// 支持 auto 模式：自动检测源语言
|
||||
func TranslateWithPivot(ctx context.Context, fromLang, toLang, text string, isHTML bool) (string, error) {
|
||||
// 处理 auto 模式：自动检测源语言
|
||||
if fromLang == "auto" {
|
||||
detected := DetectLanguage(text)
|
||||
if detected == "" {
|
||||
return "", fmt.Errorf("failed to detect source language")
|
||||
}
|
||||
log.Printf("Auto-detected source language: %s", detected)
|
||||
fromLang = detected
|
||||
}
|
||||
|
||||
// 如果源语言和目标语言相同，直接返回原文
|
||||
if fromLang == toLang {
|
||||
log.Printf("Source and target languages are the same (%s), returning original text", fromLang)
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// 如果不需要中转，直接翻译
|
||||
if !needsPivotTranslation(fromLang, toLang) {
|
||||
m, err := getOrCreateSingleEngine(fromLang, toLang)
|
||||
|
||||
Reference in New Issue
Block a user