mirror of
https://github.com/xxnuo/MTranServer.git
synced 2026-09-03 06:35:20 +08:00
88010e9aa7
The CLD2 WASM module has a stack size of only 8KB (TOTAL_STACK=8192), leading to stack overflow crashes when processing long texts. Error Manifestation: - Abort when processing 282-character mixed English and Chinese text - Error message: `RuntimeError: Aborted()` - Crash location: `detectLanguageWithCLD` calling CLD2 - **Makefile**: Increase TOTAL_STACK from 8192 (8KB) to 65536 (64KB) - An 8-fold increase is sufficient for language detection of complex texts. - The WASM module size remains unchanged at 1.1MB. - Input sanitization: Remove null bytes and control characters - UTF-8 byte boundary truncation: 512-byte limit - Enhanced error logging: Record crash context - API request limit: 10MB JSON parsing limit - `detectLanguageWithLength` method accepts explicit length - WebIDL interface definition completed - Temporarily disabled; using the original interface with input sanitization fix: detectMultipleLanguages fix: detectMultipleLanguages
76 lines
2.4 KiB
Makefile
76 lines
2.4 KiB
Makefile
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
PYTHON2 ?= uv run python
|
|
|
|
EMSCRIPTEN_ROOT := $(shell if which emcc >/dev/null 2>&1; \
|
|
then dirname `which emcc`; \
|
|
else echo /usr/lib/emscripten; \
|
|
fi)
|
|
|
|
EMCC ?= $(EMSCRIPTEN_ROOT)/emcc
|
|
|
|
WEBIDL ?= $(PYTHON2) $(EMSCRIPTEN_ROOT)/tools/webidl_binder.py
|
|
|
|
# A 2MB heap is to analyze most web pages. For the outliers, we need to either
|
|
# allow for heap growth, or allocate an unreasonable amount of memory at the
|
|
# outset.
|
|
# Unfortunately, once the heap has been enlarged, there is no shrinking, so
|
|
# analyzing one 20MB web page gives us a 30-40MB heap for the life of the
|
|
# worker.
|
|
FLAGS=-s -O3 -s INLINING_LIMIT=1 -s NO_FILESYSTEM=1 -s NO_EXIT_RUNTIME=1 -s INVOKE_RUN=0 \
|
|
-s TOTAL_STACK=65536 -s TOTAL_MEMORY=2097152 -s ALLOW_MEMORY_GROWTH=1 \
|
|
-s MODULARIZE=1 -s EXPORT_NAME=loadCLD2 \
|
|
--closure 1
|
|
|
|
export EMCC_CLOSURE_ARGS = --language_in UNSTABLE --language_out ES5_STRICT
|
|
|
|
SOURCES= \
|
|
internal/cldutil.cc \
|
|
internal/cldutil_shared.cc \
|
|
internal/compact_lang_det.cc \
|
|
internal/compact_lang_det_hint_code.cc \
|
|
internal/compact_lang_det_impl.cc \
|
|
internal/debug_empty.cc \
|
|
internal/fixunicodevalue.cc \
|
|
internal/generated_entities.cc \
|
|
internal/generated_language.cc \
|
|
internal/generated_ulscript.cc \
|
|
internal/getonescriptspan.cc \
|
|
internal/lang_script.cc \
|
|
internal/offsetmap.cc \
|
|
internal/scoreonescriptspan.cc \
|
|
internal/tote.cc \
|
|
internal/utf8statetable.cc \
|
|
internal/cld_generated_cjk_uni_prop_80.cc \
|
|
internal/cld2_generated_cjk_compatible.cc \
|
|
internal/cld_generated_cjk_delta_bi_4.cc \
|
|
internal/generated_distinct_bi_0.cc \
|
|
internal/cld2_generated_quadchrome0122_16.cc \
|
|
internal/cld2_generated_deltaoctachrome0122.cc \
|
|
internal/cld2_generated_distinctoctachrome0122.cc \
|
|
internal/cld_generated_score_quad_octa_0122_2.cc \
|
|
cldapp.cc \
|
|
$(NULL)
|
|
|
|
OBJECTS=$(SOURCES:.cc=.o)
|
|
|
|
default: all
|
|
|
|
%.o: %.cc Makefile
|
|
$(EMCC) -Os -I. -Wno-c++11-narrowing -c -o $@ $<
|
|
|
|
cldapp.o: cld.cpp
|
|
|
|
%.cpp %.js: %.idl
|
|
$(WEBIDL) $< $*
|
|
|
|
all: cld2.js cld2.wasm
|
|
|
|
cld2.js: $(OBJECTS) post.js cld.js
|
|
$(EMCC) $(FLAGS) -I. -o cld2.js $(OBJECTS) --post-js cld.js --post-js post.js
|
|
|
|
clean:
|
|
rm -f $(OBJECTS) cld.cpp cld.js before.js
|