diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index f3c6ef1..3bdd1f0 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -236,7 +236,30 @@ https://api.resend.com/emails - 通知去重：回到 D1 - 导入预览 token：回到 D1 - Logo 搜索缓存：仅保留进程内短缓存 -### 3. Logo 搜索是 Lite 版 + +### 3. Wallos 导入能力 + +当前 Worker 分支支持： + +- Wallos JSON 导入 +- Wallos SQLite 数据库导入（`.db` / `.sqlite` / `.sqlite3`） +- Wallos ZIP 备份导入 + +并且会尽量对齐 Wallos 自己的运行时语义： + +- 对 `auto_renew = true` 且 `next_payment` 已落后的订阅，会先按 Wallos 的周期推进逻辑修正日期，再判定状态 +- ZIP 预览阶段会先处理其中可匹配的 Logo 资产 + +关于 ZIP Logo： + +- **启用 R2 时** + - 预览阶段会先把匹配到的 Logo 写入 R2 临时对象 + - 确认导入时直接复用这些对象，不重复上传 +- **未启用 R2 时** + - 仍可导入 ZIP 中的数据库内容 + - 但会忽略 ZIP 中的 Logo，并在预览中给出 warning + +### 4. Logo 搜索是 Lite 版 Worker Lite 的 Logo 搜索只保留： @@ -256,7 +279,7 @@ Worker Lite 的 Logo 搜索只保留： - 搜索结果质量不如 Docker / main 分支 - 偶尔会混入不够理想的候选图 -### 4. Cron 已拆成 Lite 版职责 +### 5. Cron 已拆成 Lite 版职责 当前默认触发器是： @@ -273,7 +296,7 @@ Worker Lite 的 Logo 搜索只保留： - 只在规则命中的那一分钟发送提醒 - 相比旧的 `*/5` 版本，提醒更实时，但不再做 5 分钟补偿命中 -### 5. 错误提示会明确说明 Worker 限制 +### 6. 错误提示会明确说明 Worker 限制 前端遇到： @@ -341,10 +364,9 @@ http://127.0.0.1:8787 - Telegram - Resend 邮件 - AI 文本 / 图片识别 -- Wallos JSON 导入 +- Wallos JSON / SQLite / ZIP 导入 ### 不支持 - 本地 OCR -- Wallos SQLite / ZIP 导入 - 原生 SMTP diff --git a/README.md b/README.md index 59d476c..6a5d8b3 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ - **通知能力**：Webhook、Resend 邮件、PushPlus、Telegram Bot - **Logo 能力**：上传（R2）、远程引用、网络搜索 - **AI 识别**：支持文本 / 图片识别后自动填充订阅信息 -- **Wallos 导入**：当前分支支持 JSON 导入 +- **Wallos 导入**：支持 JSON、SQLite 数据库与 ZIP 备份导入；自动续订语义会按 Wallos 规则对齐 - **登录体验**：支持“记住我”、可配置的登录保留时长、默认密码修改提醒，以及登录失败限流保护 ## Lite 版说明 @@ -115,7 +115,7 @@ npm test 常用仓库 Variables： - `WORKER_NAME_PREFIX` -- `ENABLE_R2`（默认关闭） +- `ENABLE_R2`（默认关闭；开启后可持久化 ZIP 导入的 Logo） ## 工作流 diff --git a/apps/api/src/services/logo.service.ts b/apps/api/src/services/logo.service.ts index 914b916..a0fa12c 100644 --- a/apps/api/src/services/logo.service.ts +++ b/apps/api/src/services/logo.service.ts @@ -611,17 +611,30 @@ function isLocalLogoUrl(url?: string | null) { return Boolean(url && url.startsWith('/static/logos/')) } -function createLogoUrl(key: string) { +export function createLogoUrl(key: string) { return `/static/logos/${encodeURIComponent(key)}` } -async function writeLogoBuffer(buffer: Buffer | Uint8Array, contentType: string, logoSource: string) { +export function extractLogoStorageKey(logoUrl?: string | null) { + if (!isLocalLogoUrl(logoUrl)) return null + return decodeURIComponent(String(logoUrl).slice('/static/logos/'.length)) +} + +async function writeLogoBuffer( + buffer: Buffer | Uint8Array, + contentType: string, + logoSource: string, + options?: { + key?: string + } +) { const bucket = getWorkerLogoBucket() if (!bucket) { throw new Error('对象存储未启用，无法保存 Logo 文件') } - const filename = `logos/${Date.now()}-${crypto.randomBytes(6).toString('hex')}${extensionMap[contentType] ?? '.png'}` + const filename = + options?.key?.trim() || `logos/${Date.now()}-${crypto.randomBytes(6).toString('hex')}${extensionMap[contentType] ?? '.png'}` await bucket.put(filename, buffer, { httpMetadata: { contentType @@ -644,6 +657,26 @@ export async function saveImportedLogoBuffer(buffer: Buffer, contentType: string return writeLogoBuffer(buffer, contentType, logoSource) } +export async function saveImportedLogoBufferToKey( + buffer: Buffer, + contentType: string, + key: string, + logoSource = 'wallos-zip' +) { + if (!allowedTypes.has(contentType)) { + throw new Error('不支持的 Logo 图片类型') + } + if (!buffer.length) { + throw new Error('Logo 图片内容为空') + } + if (!key.trim()) { + throw new Error('Logo 存储 key 不能为空') + } + return writeLogoBuffer(buffer, contentType, logoSource, { + key + }) +} + export async function saveUploadedLogo(input: LogoUploadInput) { if (!allowedTypes.has(input.contentType)) { throw new Error('仅支持 PNG、JPG、WEBP、SVG 图片') @@ -814,3 +847,15 @@ export async function deleteLocalLogoFromLibrary(filename: string) { deleted: true } } + +export async function deleteLogoStorageObject(filename: string) { + const bucket = getWorkerLogoBucket() + if (!bucket) { + return + } + const safeName = filename.trim() + if (!safeName) { + return + } + await bucket.delete(safeName) +} diff --git a/apps/api/src/services/settings.service.ts b/apps/api/src/services/settings.service.ts index 5fd3c32..8db4588 100644 --- a/apps/api/src/services/settings.service.ts +++ b/apps/api/src/services/settings.service.ts @@ -64,7 +64,7 @@ function buildStorageCapabilities() { kvEnabled: Boolean(getWorkerCache()), r2Enabled: Boolean(getWorkerLogoBucket()), logoStorageEnabled: Boolean(getWorkerLogoBucket()), - wallosImportMode: 'json-only' + wallosImportMode: 'json-db-zip' }) } diff --git a/apps/api/src/services/wallos-import.service.ts b/apps/api/src/services/wallos-import.service.ts index 52bf3d5..ca1bf2d 100644 --- a/apps/api/src/services/wallos-import.service.ts +++ b/apps/api/src/services/wallos-import.service.ts @@ -1,3 +1,8 @@ +import crypto from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' +import AdmZip, { type IZipEntry } from 'adm-zip' +import type { Database as SqlJsDatabase, SqlJsStatic } from 'sql.js' import type { BillingIntervalUnit, SubscriptionStatus, @@ -9,14 +14,66 @@ import type { WallosImportTagDto } from '@subtracker/shared' import { prisma } from '../db' -import { isWorkerRuntime } from '../runtime' +import { getWorkerLogoBucket, isWorkerRuntime } from '../runtime' import { addInterval } from '../utils/date' import { getAppSettings } from './settings.service' import { appendSubscriptionOrders } from './subscription-order.service' +import { deleteLogoStorageObject, saveImportedLogoBufferToKey } from './logo.service' import { deleteImportPreview, getImportPreview, storeImportPreview } from './worker-lite-state.service' -const IMPORT_TOKEN_TTL_SECONDS = 15 * 60 -const IMPORT_TOKEN_TTL_MS = IMPORT_TOKEN_TTL_SECONDS * 1000 +const REQUIRED_TABLES = ['subscriptions', 'categories', 'currencies', 'cycles', 'frequencies'] as const +const IMPORT_TOKEN_TTL_MS = 15 * 60 * 1000 +const SQL_JS_CDN_BASE = 'https://cdn.jsdelivr.net/npm/sql.js@1.14.1/dist/' + +type SqlDatabase = SqlJsDatabase +type ImportFileType = 'json' | 'db' | 'zip' + +type WallosSubscriptionRow = { + id: number + name: string + logo: string | null + price: number | null + next_payment: string | null + cycle: number | null + frequency: number | null + notes: string | null + notify: number | null + url: string | null + inactive: number | null + notify_days_before: number | null + cancellation_date: string | null + start_date: string | number | null + auto_renew: number | null + currency_code: string | null + category_id: number | null + category_name: string | null + cycle_days: number | null + frequency_name: number | null + category_sort_order: number | null +} + +type WallosJsonRow = Record + +type ZipLogoAsset = { + filename: string + buffer: Buffer + contentType: string +} + +type ZipLogoManifestEntry = { + logoRef: string + r2Key: string + logoUrl: string + contentType: string + uploaded: boolean +} + +type StoredImportPreviewState = { + preview: WallosImportInspectResultDto + logoManifest: Record +} + +let sqlJsPromise: Promise | null = null const IMPORT_TAG_COLORS = [ '#3b82f6', '#8b5cf6', @@ -30,17 +87,6 @@ const IMPORT_TAG_COLORS = [ '#6366f1' ] -type WallosJsonRow = Record - -function decodeBase64(value: string) { - const binary = atob(value.trim()) - const bytes = new Uint8Array(binary.length) - for (let index = 0; index < binary.length; index += 1) { - bytes[index] = binary.charCodeAt(index) - } - return new TextDecoder().decode(bytes) -} - function getImportedTagColor(name: string) { let hash = 0 for (const char of name) { @@ -55,6 +101,80 @@ function createImportId() { return `c${timestamp}${random}`.slice(0, 25) } +function resolveSqlJsLocateFile(file: string) { + const cwd = process.cwd() + const localCandidates = [ + path.resolve(cwd, 'node_modules/sql.js/dist', file), + path.resolve(cwd, '../../node_modules/sql.js/dist', file) + ] + + for (const candidate of localCandidates) { + if (fs.existsSync(candidate)) { + return candidate + } + } + + return `${SQL_JS_CDN_BASE}${file}` +} + +function getSqlJs() { + if (!sqlJsPromise) { + sqlJsPromise = (async () => { + if (typeof globalThis.location === 'undefined') { + Object.defineProperty(globalThis, 'location', { + value: { href: SQL_JS_CDN_BASE }, + configurable: true + }) + } else if (!globalThis.location?.href) { + Object.defineProperty(globalThis.location, 'href', { + value: SQL_JS_CDN_BASE, + configurable: true + }) + } + + const sqlJsModule = await import('sql.js') + const initSqlJs = sqlJsModule.default + + return initSqlJs({ + locateFile: resolveSqlJsLocateFile + }) + })() + } + + return sqlJsPromise +} + +function openDatabase(buffer: Buffer) { + return getSqlJs().then((SQL: SqlJsStatic) => new SQL.Database(new Uint8Array(buffer))) +} + +function queryRows>(db: SqlDatabase, sql: string): T[] { + const statement = db.prepare(sql) + const rows: T[] = [] + + try { + while (statement.step()) { + rows.push(statement.getAsObject() as T) + } + return rows + } finally { + statement.free() + } +} + +function extractTableNames(db: SqlDatabase) { + return queryRows<{ name: string }>( + db, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).map((item) => String(item.name)) +} + +function extractTableColumnNames(db: SqlDatabase, tableName: string) { + return new Set( + queryRows<{ name: string }>(db, `PRAGMA table_info(${tableName})`).map((item) => String(item.name)) + ) +} + function normalizeWallosTagName(name: string | null | undefined) { const value = String(name ?? '').trim() if (!value) return null @@ -84,79 +204,25 @@ function startOfUtcDay(value: Date | string = new Date()) { return date } -function parsePriceString(input: unknown) { - const text = String(input ?? '').trim() - if (!text) { - return { amount: 0, currency: 'CNY', warning: '价格为空，已回退为 0 CNY' } +function detectImportFileType(input: WallosImportInspectInput, buffer: Buffer): ImportFileType { + const filename = input.filename.toLowerCase() + const trimmed = buffer.toString('utf8', 0, Math.min(buffer.length, 80)).trimStart() + + if (filename.endsWith('.zip') || buffer.subarray(0, 2).toString('hex') === '504b') { + return 'zip' } - - const normalized = text.replace(/,/g, '') - const amountMatch = normalized.match(/-?\d+(?:\.\d+)?/) - const amount = amountMatch ? Number(amountMatch[0]) : 0 - - let currency = 'CNY' - if (/¥|yuan|cny|rmb/i.test(normalized)) currency = 'CNY' - else if (/\$|usd|dollar/i.test(normalized)) currency = 'USD' - else if (/€|eur/i.test(normalized)) currency = 'EUR' - else if (/£|gbp/i.test(normalized)) currency = 'GBP' - else if (/jpy|yen|￥/i.test(normalized)) currency = 'JPY' - - let warning: string | null = null - if (!amountMatch) { - warning = `价格 "${text}" 无法完整解析，已回退为 0 ${currency}` - } else if (/\$/.test(normalized) && !/usd|dollar/i.test(normalized)) { - warning = `价格 "${text}" 的币种符号存在歧义，已默认按 USD 导入` - } else if (/¥/.test(normalized) && !/yuan|cny|rmb/i.test(normalized)) { - warning = `价格 "${text}" 的币种符号存在歧义，已默认按 CNY 导入` - } else if (!/[a-z￥¥€£$]/i.test(normalized)) { - warning = `价格 "${text}" 未包含明确币种，已默认按 CNY 导入` - } - - return { - amount: Number.isFinite(amount) ? amount : 0, - currency, - warning + if (filename.endsWith('.json') || trimmed.startsWith('[') || trimmed.startsWith('{')) { + return 'json' } + return 'db' } -function normalizeWallosWebsiteUrl(input: unknown) { - const raw = String(input ?? '') - .replace(/&/g, '&') - .trim() - if (!raw) { - return { websiteUrl: null, warning: null as string | null } - } - - const tryParse = (value: string) => { - try { - const parsed = new URL(value) - if (!['http:', 'https:'].includes(parsed.protocol) || !parsed.hostname) return null - return parsed.toString() - } catch { - return null - } - } - - const direct = tryParse(raw) - if (direct) { - return { websiteUrl: direct, warning: null as string | null } - } - - const looksLikeDomain = /^[a-z0-9.-]+\.[a-z]{2,}(?::\d+)?(?:[/?#].*)?$/i.test(raw) - if (looksLikeDomain) { - const withHttps = tryParse(`https://${raw}`) - if (withHttps) { - return { - websiteUrl: withHttps, - warning: `网址 "${raw}" 缺少协议，已自动补全为 ${withHttps}` - } - } - } - - return { - websiteUrl: null, - warning: `网址 "${raw}" 无法识别为合法链接，已忽略` +function decodeImportBuffer(input: WallosImportInspectInput) { + const buffer = Buffer.from(input.base64, 'base64') + if (!buffer.length) { + throw new Error('导入文件内容为空') } + return buffer } export function mapWallosBillingInterval(days: number | null | undefined, frequency: number | null | undefined) { @@ -270,6 +336,165 @@ export function resolveWallosNotifyDays(input: { return { webhookEnabled: true, notifyDaysBefore: Math.max(input.notifyDaysBefore, 0) } } +function buildTagCollection() { + const map = new Map() + + return { + add(name: string | null, sourceId?: number | null, sortOrder?: number | null) { + const normalized = normalizeWallosTagName(name) + if (!normalized || map.has(normalized)) return + map.set(normalized, { + sourceId: Number(sourceId ?? 0), + name: normalized, + sortOrder: Number(sortOrder ?? 0) + }) + }, + toArray() { + return Array.from(map.values()).sort((a, b) => a.sortOrder - b.sortOrder || a.name.localeCompare(b.name, 'zh-CN')) + } + } +} + +function ensureUniqueWarnings(warnings: string[]) { + return Array.from(new Set(warnings)) +} + +function inferContentTypeFromFilename(filename: string) { + const lower = filename.toLowerCase() + if (lower.endsWith('.png')) return 'image/png' + if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg' + if (lower.endsWith('.webp')) return 'image/webp' + if (lower.endsWith('.svg')) return 'image/svg+xml' + return '' +} + +function normalizeZipLogoName(filename: string) { + return path.basename(filename).toLowerCase() +} + +function scoreZipDatabasePath(entryName: string) { + const normalized = entryName.replace(/\\/g, '/').toLowerCase() + let score = 0 + + if (normalized.endsWith('/wallos.db') || normalized === 'wallos.db') score += 100 + if (normalized.includes('/db/')) score += 40 + if (normalized.includes('wallos')) score += 30 + if (normalized.endsWith('.db')) score += 20 + if (normalized.endsWith('.sqlite') || normalized.endsWith('.sqlite3')) score += 18 + if (normalized.includes('__macosx/')) score -= 100 + + return score +} + +function extractZipImport(buffer: Buffer) { + const zip = new AdmZip(buffer) + const entries = zip.getEntries() + const dbEntry = entries + .filter((entry: IZipEntry) => !entry.isDirectory) + .filter((entry: IZipEntry) => /\.(db|sqlite|sqlite3)$/i.test(entry.entryName)) + .sort((a, b) => scoreZipDatabasePath(b.entryName) - scoreZipDatabasePath(a.entryName))[0] + + if (!dbEntry) { + throw new Error('ZIP 中未找到 db/wallos.db') + } + + const zipLogos = new Map() + + for (const entry of entries) { + if (entry.isDirectory) continue + const filename = path.basename(entry.entryName) + const contentType = inferContentTypeFromFilename(filename) + if (!contentType) continue + if (entry.entryName === dbEntry.entryName) continue + + zipLogos.set(normalizeZipLogoName(filename), { + filename, + buffer: entry.getData(), + contentType + }) + } + + return { + dbBuffer: dbEntry.getData(), + zipLogos + } +} + +function parsePriceString(input: unknown) { + const text = String(input ?? '').trim() + if (!text) { + return { amount: 0, currency: 'CNY', warning: '价格为空，已回退为 0 CNY' } + } + + const normalized = text.replace(/,/g, '') + const amountMatch = normalized.match(/-?\d+(?:\.\d+)?/) + const amount = amountMatch ? Number(amountMatch[0]) : 0 + + let currency = 'CNY' + if (/¥|yuan|cny|rmb/i.test(normalized)) currency = 'CNY' + else if (/\$|usd|dollar/i.test(normalized)) currency = 'USD' + else if (/€|eur/i.test(normalized)) currency = 'EUR' + else if (/£|gbp/i.test(normalized)) currency = 'GBP' + else if (/jpy|yen|￥/i.test(normalized)) currency = 'JPY' + + let warning: string | null = null + if (!amountMatch) { + warning = `价格 "${text}" 无法完整解析，已回退为 0 ${currency}` + } else if (/\$/.test(normalized) && !/usd|dollar/i.test(normalized)) { + warning = `价格 "${text}" 的币种符号存在歧义，已默认按 USD 导入` + } else if (/¥/.test(normalized) && !/yuan|cny|rmb/i.test(normalized)) { + warning = `价格 "${text}" 的币种符号存在歧义，已默认按 CNY 导入` + } else if (!/[a-z￥¥€£$]/i.test(normalized)) { + warning = `价格 "${text}" 未包含明确币种，已默认按 CNY 导入` + } + + return { + amount: Number.isFinite(amount) ? amount : 0, + currency, + warning + } +} + +function normalizeWallosWebsiteUrl(input: unknown) { + const raw = String(input ?? '') + .replace(/&/g, '&') + .trim() + if (!raw) { + return { websiteUrl: null, warning: null as string | null } + } + + const tryParse = (value: string) => { + try { + const parsed = new URL(value) + if (!['http:', 'https:'].includes(parsed.protocol) || !parsed.hostname) return null + return parsed.toString() + } catch { + return null + } + } + + const direct = tryParse(raw) + if (direct) { + return { websiteUrl: direct, warning: null as string | null } + } + + const looksLikeDomain = /^[a-z0-9.-]+\.[a-z]{2,}(?::\d+)?(?:[/?#].*)?$/i.test(raw) + if (looksLikeDomain) { + const withHttps = tryParse(`https://${raw}`) + if (withHttps) { + return { + websiteUrl: withHttps, + warning: `网址 "${raw}" 缺少协议，已自动补全为 ${withHttps}` + } + } + } + + return { + websiteUrl: null, + warning: `网址 "${raw}" 无法识别为合法链接，已忽略` + } +} + function parsePaymentCycle(input: unknown) { const text = String(input ?? '').trim() const normalized = text.toLowerCase() @@ -352,43 +577,6 @@ function mapJsonStatus(row: WallosJsonRow) { return 'active' as SubscriptionStatus } -function mapJsonWebhookEnabled(row: WallosJsonRow) { - return String(row.Notifications ?? '').trim().toLowerCase() === 'enabled' -} - -function mapJsonAutoRenew(row: WallosJsonRow) { - return String(row.Renewal ?? '').trim().toLowerCase() === 'automatic' -} - -function buildTagCollection() { - const map = new Map() - - return { - add(name: string | null, sourceId?: number | null, sortOrder?: number | null) { - const normalized = normalizeWallosTagName(name) - if (!normalized || map.has(normalized)) return - map.set(normalized, { - sourceId: Number(sourceId ?? 0), - name: normalized, - sortOrder: Number(sortOrder ?? 0) - }) - }, - toArray() { - return Array.from(map.values()).sort((a, b) => a.sortOrder - b.sortOrder || a.name.localeCompare(b.name, 'zh-CN')) - } - } -} - -function ensureUniqueWarnings(warnings: string[]) { - return Array.from(new Set(warnings)) -} - -function pushRowWarning(warnings: string[], rowWarnings: string[], prefix: string, warning: string | null | undefined) { - if (!warning) return - warnings.push(`${prefix} ${warning}`) - rowWarnings.push(warning) -} - function resolveJsonEffectiveNextPayment(row: WallosJsonRow) { const nextPayment = parseDate(row['Next Payment']) if (!nextPayment) return null @@ -406,6 +594,47 @@ function resolveJsonEffectiveNextPayment(row: WallosJsonRow) { }) } +function resolveDbEffectiveNextPayment( + row: WallosSubscriptionRow, + mappedInterval: { billingIntervalCount: number; billingIntervalUnit: BillingIntervalUnit } +) { + return resolveWallosEffectiveNextPayment({ + nextPayment: row.next_payment, + autoRenew: Boolean(row.auto_renew), + billingIntervalCount: mappedInterval.billingIntervalCount, + billingIntervalUnit: mappedInterval.billingIntervalUnit, + inactive: row.inactive, + cancellationDate: row.cancellation_date + }) +} + +function pushRowWarning(warnings: string[], rowWarnings: string[], prefix: string, warning: string | null | undefined) { + if (!warning) return + warnings.push(`${prefix} ${warning}`) + rowWarnings.push(warning) +} + +function buildJsonStartDateWarning() { + return 'Wallos JSON 不包含 start_date，已使用 Next Payment 代填开始日期' +} + +function buildJsonDerivedWarnings(row: WallosJsonRow) { + const warnings: string[] = [] + const price = parsePriceString(row.Price) + if (price.warning) warnings.push(price.warning) + const cycle = parsePaymentCycle(row['Payment Cycle']) + if (cycle.warning) warnings.push(cycle.warning) + warnings.push(buildJsonStartDateWarning()) + const normalizedUrl = normalizeWallosWebsiteUrl(row.URL) + if (normalizedUrl.warning) warnings.push(normalizedUrl.warning) + return { + price, + cycle, + normalizedUrl, + warnings + } +} + function buildJsonPreview( rows: WallosJsonRow[], settings: { defaultNotifyDays: number; baseCurrency: string } @@ -424,16 +653,13 @@ function buildJsonPreview( return } - const price = parsePriceString(row.Price) - const cycle = parsePaymentCycle(row['Payment Cycle']) const tagName = normalizeWallosTagName(String(row.Category ?? '')) const rowWarnings: string[] = [] - const normalizedUrl = normalizeWallosWebsiteUrl(row.URL) + const derived = buildJsonDerivedWarnings(row) - pushRowWarning(warnings, rowWarnings, `json#${index + 1}`, price.warning) - pushRowWarning(warnings, rowWarnings, `json#${index + 1}`, cycle.warning) - pushRowWarning(warnings, rowWarnings, `json#${index + 1}`, 'Wallos JSON 不包含 start_date，已使用 Next Payment 代填开始日期') - pushRowWarning(warnings, rowWarnings, `json#${index + 1}`, normalizedUrl.warning) + derived.warnings.forEach((warning) => { + pushRowWarning(warnings, rowWarnings, `json#${index + 1}`, warning) + }) if (tagName) { tags.add(tagName, index + 1, index + 1) @@ -442,19 +668,19 @@ function buildJsonPreview( previewSubscriptions.push({ sourceId: index + 1, name, - amount: price.amount, - currency: price.currency, + amount: derived.price.amount, + currency: derived.price.currency, status: mapJsonStatus(row), autoRenew: mapJsonAutoRenew(row), - billingIntervalCount: cycle.billingIntervalCount, - billingIntervalUnit: cycle.billingIntervalUnit, + billingIntervalCount: derived.cycle.billingIntervalCount, + billingIntervalUnit: derived.cycle.billingIntervalUnit, startDate: nextPayment, nextRenewalDate: resolveJsonEffectiveNextPayment(row) ?? nextPayment, notifyDaysBefore: settings.defaultNotifyDays, webhookEnabled: mapJsonWebhookEnabled(row), notes: String(row.Notes ?? ''), description: '', - websiteUrl: normalizedUrl.websiteUrl, + websiteUrl: derived.normalizedUrl.websiteUrl, tagNames: tagName ? [tagName] : [], logoRef: null, logoImportStatus: 'none', @@ -484,14 +710,281 @@ function buildJsonPreview( } } -export async function inspectWallosImportFile(input: WallosImportInspectInput): Promise { - if (!/\.json$/i.test(input.filename) && !String(input.contentType || '').includes('json')) { - throw new Error('Cloudflare Worker 版本目前仅支持 Wallos JSON 导入') +function mapJsonWebhookEnabled(row: WallosJsonRow) { + return String(row.Notifications ?? '').trim().toLowerCase() === 'enabled' +} + +function mapJsonAutoRenew(row: WallosJsonRow) { + return String(row.Renewal ?? '').trim().toLowerCase() === 'automatic' +} + +function buildDbPreview( + rows: WallosSubscriptionRow[], + settings: { defaultNotifyDays: number; baseCurrency: string }, + globalNotifyDays: number, + fileType: ImportFileType, + zipLogos = new Map() +): Omit { + const warnings: string[] = [] + let skippedSubscriptions = 0 + let zipLogoMatched = 0 + let zipLogoMissing = 0 + const previewSubscriptions: WallosImportSubscriptionPreviewDto[] = [] + const tags = buildTagCollection() + + for (const row of rows) { + if (!row.name || row.price === null || row.price === undefined || !row.next_payment) { + skippedSubscriptions += 1 + warnings.push(`subscription#${row.id} 缺少关键字段，已跳过`) + continue + } + + const mappedInterval = mapWallosBillingInterval(row.cycle_days, row.frequency_name) + const effectiveNextRenewalDate = resolveDbEffectiveNextPayment(row, mappedInterval) + const mappedStatus = mapWallosSubscriptionStatus({ + inactive: row.inactive, + cancellationDate: row.cancellation_date, + nextPayment: row.next_payment, + autoRenew: Boolean(row.auto_renew), + billingIntervalCount: mappedInterval.billingIntervalCount, + billingIntervalUnit: mappedInterval.billingIntervalUnit + }) + const notifyConfig = resolveWallosNotifyDays({ + notify: row.notify, + notifyDaysBefore: row.notify_days_before, + globalNotifyDays + }) + const normalizedTag = normalizeWallosTagName(row.category_name) + const rowWarnings: string[] = [] + const normalizedUrl = normalizeWallosWebsiteUrl(row.url) + + if (mappedInterval.warning) { + warnings.push(`subscription#${row.id} ${mappedInterval.warning}`) + rowWarnings.push(mappedInterval.warning) + } + pushRowWarning(warnings, rowWarnings, `subscription#${row.id}`, normalizedUrl.warning) + + if (normalizedTag) { + tags.add(normalizedTag, row.category_id, row.category_sort_order) + } + + let logoImportStatus: WallosImportSubscriptionPreviewDto['logoImportStatus'] = 'none' + const effectiveLogoRef = fileType === 'zip' && row.logo ? String(row.logo) : null + + if (fileType === 'zip' && row.logo) { + const normalizedLogoName = normalizeZipLogoName(String(row.logo)) + if (zipLogos.has(normalizedLogoName)) { + logoImportStatus = 'ready-from-zip' + zipLogoMatched += 1 + } else { + logoImportStatus = 'pending-file-match' + zipLogoMissing += 1 + warnings.push(`subscription#${row.id} 存在 Logo 文件引用，当前包内未匹配到图片`) + rowWarnings.push('Logo 文件需后续通过目录或 zip 包补齐') + } + } + + previewSubscriptions.push({ + sourceId: Number(row.id), + name: String(row.name), + amount: Number(row.price), + currency: String(row.currency_code || settings.baseCurrency || 'CNY').toUpperCase(), + status: mappedStatus, + autoRenew: Boolean(row.auto_renew), + billingIntervalCount: mappedInterval.billingIntervalCount, + billingIntervalUnit: mappedInterval.billingIntervalUnit, + startDate: parseDate(row.start_date) ?? parseDate(row.next_payment) ?? new Date().toISOString().slice(0, 10), + nextRenewalDate: effectiveNextRenewalDate ?? parseDate(row.next_payment) ?? new Date().toISOString().slice(0, 10), + notifyDaysBefore: notifyConfig.notifyDaysBefore, + webhookEnabled: notifyConfig.webhookEnabled, + notes: String(row.notes || ''), + description: '', + websiteUrl: normalizedUrl.websiteUrl, + tagNames: normalizedTag ? [normalizedTag] : [], + logoRef: effectiveLogoRef, + logoImportStatus, + warnings: rowWarnings + }) } + const usedTags = tags.toArray() + + return { + isWallos: true, + summary: { + fileType, + subscriptionsTotal: rows.length, + tagsTotal: usedTags.length, + usedTagsTotal: usedTags.length, + supportedSubscriptions: previewSubscriptions.length, + skippedSubscriptions, + globalNotifyDays, + zipLogoMatched, + zipLogoMissing + }, + tags: usedTags, + usedTags, + subscriptionsPreview: previewSubscriptions, + warnings: ensureUniqueWarnings(warnings) + } +} + +async function buildDbPreviewFromBuffer( + buffer: Buffer, + fileType: ImportFileType, + zipLogos: Map, + options?: { + defaultNotifyDays?: number + baseCurrency?: string + } +) { + const settings = + options?.defaultNotifyDays !== undefined || options?.baseCurrency + ? { + defaultNotifyDays: options?.defaultNotifyDays ?? 3, + baseCurrency: options?.baseCurrency ?? 'CNY' + } + : await getAppSettings() + + const db = await openDatabase(buffer) + + try { + const tables = new Set(extractTableNames(db)) + const missingTables = REQUIRED_TABLES.filter((table) => !tables.has(table)) + if (missingTables.length > 0) { + throw new Error(`缺少 Wallos 关键表：${missingTables.join(', ')}`) + } + + const globalNotifyRow = queryRows<{ days: number | null }>(db, 'SELECT days FROM notification_settings LIMIT 1')[0] + const globalNotifyDays = globalNotifyRow?.days ?? settings.defaultNotifyDays ?? 3 + const subscriptionColumns = extractTableColumnNames(db, 'subscriptions') + const selectSubscriptionColumn = (columnName: string, fallbackSql = 'NULL') => + subscriptionColumns.has(columnName) ? `s.${columnName}` : `${fallbackSql} AS ${columnName}` + + const rows = queryRows( + db, + ` + SELECT + s.id, + s.name, + s.logo, + s.price, + s.next_payment, + s.cycle, + s.frequency, + s.notes, + s.notify, + s.url, + s.inactive, + ${selectSubscriptionColumn('notify_days_before', '0')}, + ${selectSubscriptionColumn('cancellation_date')}, + ${selectSubscriptionColumn('start_date')}, + ${selectSubscriptionColumn('auto_renew', '1')}, + c.code AS currency_code, + cat.id AS category_id, + cat.name AS category_name, + cy.days AS cycle_days, + f.name AS frequency_name, + cat."order" AS category_sort_order + FROM subscriptions s + LEFT JOIN currencies c ON c.id = s.currency_id + LEFT JOIN categories cat ON cat.id = s.category_id + LEFT JOIN cycles cy ON cy.id = s.cycle + LEFT JOIN frequencies f ON f.id = s.frequency + ORDER BY s.id + ` + ) + + return buildDbPreview(rows, settings, globalNotifyDays, fileType, zipLogos) + } finally { + db.close() + } +} + +function buildTemporaryLogoKey(importToken: string, filename: string) { + const ext = path.extname(filename).toLowerCase() || '.png' + const base = path.basename(filename, ext).replace(/[^a-z0-9._-]+/gi, '-').replace(/^-+|-+$/g, '') || 'logo' + return `logos/imports/wallos/${importToken}/${base}-${crypto.randomBytes(4).toString('hex')}${ext}` +} + +async function uploadZipLogoManifest( + importToken: string, + preview: Omit, + zipLogos: Map +) { + const manifest: Record = {} + if (!zipLogos.size) { + return manifest + } + + if (!getWorkerLogoBucket()) { + if (preview.summary.zipLogoMatched > 0) { + preview.warnings = ensureUniqueWarnings([ + ...preview.warnings, + `当前 Worker 未启用 R2，已忽略 ${preview.summary.zipLogoMatched} 个 ZIP Logo` + ]) + preview.subscriptionsPreview = preview.subscriptionsPreview.map((item) => + item.logoImportStatus === 'ready-from-zip' + ? { + ...item, + logoRef: null, + logoImportStatus: 'none' + } + : item + ) + } + return manifest + } + + const readyRefs = new Set( + preview.subscriptionsPreview + .filter((item) => item.logoImportStatus === 'ready-from-zip' && item.logoRef) + .map((item) => normalizeZipLogoName(String(item.logoRef))) + ) + + for (const logoRef of readyRefs) { + const asset = zipLogos.get(logoRef) + if (!asset) continue + const r2Key = buildTemporaryLogoKey(importToken, asset.filename) + const stored = await saveImportedLogoBufferToKey(asset.buffer, asset.contentType, r2Key, 'wallos-zip') + manifest[logoRef] = { + logoRef: asset.filename, + r2Key, + logoUrl: stored.logoUrl, + contentType: asset.contentType, + uploaded: true + } + } + + return manifest +} + +async function inspectZipImport( + buffer: Buffer, + importToken: string, + options?: { + defaultNotifyDays?: number + baseCurrency?: string + } +) { + const extracted = extractZipImport(buffer) + const preview = await buildDbPreviewFromBuffer(extracted.dbBuffer, 'zip', extracted.zipLogos, options) + return { + preview, + logoManifest: await uploadZipLogoManifest(importToken, preview, extracted.zipLogos) + } +} + +async function inspectJsonImport( + buffer: Buffer, + options?: { + defaultNotifyDays?: number + baseCurrency?: string + } +) { let parsed: unknown try { - parsed = JSON.parse(decodeBase64(input.base64)) + parsed = JSON.parse(buffer.toString('utf8')) } catch { throw new Error('JSON 解析失败') } @@ -500,26 +993,79 @@ export async function inspectWallosImportFile(input: WallosImportInspectInput): throw new Error('Wallos JSON 导出内容必须是数组') } - const settings = await getAppSettings() - const token = crypto.randomUUID().replaceAll('-', '') - const preview: WallosImportInspectResultDto = { - ...buildJsonPreview(parsed as WallosJsonRow[], settings), - importToken: token + const settings = + options?.defaultNotifyDays !== undefined || options?.baseCurrency + ? { + defaultNotifyDays: options?.defaultNotifyDays ?? 3, + baseCurrency: options?.baseCurrency ?? 'CNY' + } + : await getAppSettings() + return { + preview: buildJsonPreview(parsed as WallosJsonRow[], settings), + logoManifest: {} as Record + } +} + +async function inspectDbImport(buffer: Buffer, options?: { defaultNotifyDays?: number; baseCurrency?: string }) { + const preview = await buildDbPreviewFromBuffer(buffer, 'db', new Map(), options) + return { + preview, + logoManifest: {} as Record + } +} + +async function inspectImportBuffer( + input: WallosImportInspectInput, + importToken: string, + options?: { defaultNotifyDays?: number; baseCurrency?: string } +) { + const buffer = decodeImportBuffer(input) + const fileType = detectImportFileType(input, buffer) + + switch (fileType) { + case 'json': + return inspectJsonImport(buffer, options) + case 'zip': + return inspectZipImport(buffer, importToken, options) + case 'db': + default: + return inspectDbImport(buffer, options) + } +} + +async function cleanupStoredImportAssets(state: StoredImportPreviewState | null) { + if (!state) return + const entries = Object.values(state.logoManifest ?? {}) + if (!entries.length) return + await Promise.all(entries.map((item) => deleteLogoStorageObject(item.r2Key))) +} + +export async function inspectWallosImportFile(input: WallosImportInspectInput): Promise { + const importToken = crypto.randomBytes(24).toString('hex') + const { preview, logoManifest } = await inspectImportBuffer(input, importToken) + const storedState: StoredImportPreviewState = { + preview: { + ...preview, + importToken + }, + logoManifest } - await storeImportPreview(token, preview, IMPORT_TOKEN_TTL_MS) + await storeImportPreview(importToken, storedState, IMPORT_TOKEN_TTL_MS) - return preview + return storedState.preview } export async function commitWallosImport(input: WallosImportCommitInput): Promise { - const preview = await getImportPreview(input.importToken) - if (!preview) { - await deleteImportPreview(input.importToken) + const state = await getImportPreview(input.importToken, { + onExpired: cleanupStoredImportAssets + }) + if (!state) { throw new Error('导入令牌不存在或已失效，请重新生成预览') } - await deleteImportPreview(input.importToken) + const preview = state.preview + const logoManifest = state.logoManifest ?? {} const existingTags = await prisma.tag.findMany({ where: { @@ -532,8 +1078,9 @@ export async function commitWallosImport(input: WallosImportCommitInput): Promis const tagIdByName = new Map(existingTags.map((item) => [item.name, item.id])) let importedTags = 0 let importedSubscriptions = 0 - const missingTags = preview.usedTags.filter((tag) => !tagIdByName.has(tag.name)) + let importedLogos = 0 + const missingTags = preview.usedTags.filter((tag) => !tagIdByName.has(tag.name)) if (missingTags.length > 0) { if (isWorkerRuntime()) { for (const tag of missingTags) { @@ -585,6 +1132,9 @@ export async function commitWallosImport(input: WallosImportCommitInput): Promis webhookEnabled: boolean notes: string websiteUrl: string | null + logoUrl: string | null + logoSource: string | null + logoFetchedAt: Date | null status: SubscriptionStatus }> = [] const subscriptionTagRows: Array<{ subscriptionId: string; tagId: string }> = [] @@ -594,6 +1144,13 @@ export async function commitWallosImport(input: WallosImportCommitInput): Promis .map((name) => tagIdByName.get(name)) .filter((value): value is string => Boolean(value)) + const normalizedLogoRef = item.logoRef ? normalizeZipLogoName(item.logoRef) : null + const manifestEntry = normalizedLogoRef ? logoManifest[normalizedLogoRef] : null + const hasImportedLogo = Boolean(manifestEntry?.uploaded && manifestEntry.logoUrl) + if (hasImportedLogo) { + importedLogos += 1 + } + const subscriptionId = createImportId() createdSubscriptionIds.push(subscriptionId) subscriptionRows.push({ @@ -611,6 +1168,9 @@ export async function commitWallosImport(input: WallosImportCommitInput): Promis webhookEnabled: item.webhookEnabled, notes: item.notes, websiteUrl: item.websiteUrl ?? null, + logoUrl: manifestEntry?.logoUrl ?? null, + logoSource: manifestEntry?.uploaded ? 'wallos-zip' : null, + logoFetchedAt: manifestEntry?.uploaded ? new Date() : null, status: item.status }) tagIds.forEach((tagId) => { @@ -651,23 +1211,20 @@ export async function commitWallosImport(input: WallosImportCommitInput): Promis } await appendSubscriptionOrders(createdSubscriptionIds) + await deleteImportPreview(input.importToken) return { importedTags, importedSubscriptions, skippedSubscriptions: preview.summary.skippedSubscriptions, - importedLogos: 0, + importedLogos, warnings: preview.warnings } } export async function previewWallosImportFromBase64(input: WallosImportInspectInput) { - const settings = await getAppSettings() - const parsed = JSON.parse(decodeBase64(input.base64)) - if (!Array.isArray(parsed)) { - throw new Error('Wallos JSON 导出内容必须是数组') - } - return buildJsonPreview(parsed as WallosJsonRow[], settings) + const result = await inspectImportBuffer(input, crypto.randomBytes(12).toString('hex')) + return result.preview } export async function previewWallosImportFromBase64ForTest( @@ -677,12 +1234,6 @@ export async function previewWallosImportFromBase64ForTest( baseCurrency?: string } ) { - const parsed = JSON.parse(decodeBase64(input.base64)) - if (!Array.isArray(parsed)) { - throw new Error('Wallos JSON 导出内容必须是数组') - } - return buildJsonPreview(parsed as WallosJsonRow[], { - defaultNotifyDays: options?.defaultNotifyDays ?? 3, - baseCurrency: options?.baseCurrency ?? 'CNY' - }) + const result = await inspectImportBuffer(input, 'test-import-token', options) + return result.preview } diff --git a/apps/api/src/services/worker-lite-state.service.ts b/apps/api/src/services/worker-lite-state.service.ts index 217e7a9..9ec6db6 100644 --- a/apps/api/src/services/worker-lite-state.service.ts +++ b/apps/api/src/services/worker-lite-state.service.ts @@ -1,5 +1,5 @@ import { Prisma } from '@prisma/client' -import type { WallosImportInspectResultDto, WebhookEventType } from '@subtracker/shared' +import type { WebhookEventType } from '@subtracker/shared' import { prisma } from '../db' import { getRuntimeD1Database, isWorkerRuntime } from '../runtime' @@ -137,7 +137,7 @@ export async function releaseNotificationDelivery(params: { ) } -export async function storeImportPreview(token: string, preview: WallosImportInspectResultDto, ttlMs: number) { +export async function storeImportPreview(token: string, preview: T, ttlMs: number) { const expiresAt = new Date(Date.now() + ttlMs) if (!getD1()) { @@ -164,15 +164,24 @@ export async function storeImportPreview(token: string, preview: WallosImportIns ) } -export async function getImportPreview(token: string) { +export async function getImportPreview( + token: string, + options?: { + onExpired?: (payload: T | null) => Promise | void + } +) { if (!getD1()) { const row = await prisma.importPreview.findUnique({ where: { token } }) if (!row) return null if (row.expiresAt.getTime() <= Date.now()) { - await prisma.importPreview.deleteMany({ where: { token } }) + const payload = row.previewJson as unknown as T + await deleteImportPreview(token, { + payload, + onDelete: options?.onExpired + }) return null } - return row.previewJson as unknown as WallosImportInspectResultDto + return row.previewJson as unknown as T } const row = await d1First( @@ -185,14 +194,28 @@ export async function getImportPreview(token: string) { if (!row) return null if (new Date(row.expiresAt).getTime() <= Date.now()) { - await deleteImportPreview(token) + const payload = parseJsonValue(row.previewJson) + await deleteImportPreview(token, { + payload, + onDelete: options?.onExpired + }) return null } - return parseJsonValue(row.previewJson) + return parseJsonValue(row.previewJson) } -export async function deleteImportPreview(token: string) { +export async function deleteImportPreview( + token: string, + options?: { + payload?: T | null + onDelete?: (payload: T | null) => Promise | void + } +) { + if (options?.onDelete) { + await options.onDelete(options.payload ?? null) + } + if (!getD1()) { await prisma.importPreview.deleteMany({ where: { token } }) return diff --git a/apps/api/src/types/adm-zip.d.ts b/apps/api/src/types/adm-zip.d.ts index f0ec403..56ef525 100644 --- a/apps/api/src/types/adm-zip.d.ts +++ b/apps/api/src/types/adm-zip.d.ts @@ -6,7 +6,9 @@ declare module 'adm-zip' { } export default class AdmZip { - constructor(input?: Buffer | string) + constructor(input?: Buffer | Uint8Array | ArrayBuffer | string) + addFile(entryName: string, content: Buffer | Uint8Array | ArrayBuffer): void getEntries(): IZipEntry[] + toBuffer(): Buffer } } diff --git a/apps/api/tests/unit/wallos-import-commit.test.ts b/apps/api/tests/unit/wallos-import-commit.test.ts index 2a134e3..d1b7753 100644 --- a/apps/api/tests/unit/wallos-import-commit.test.ts +++ b/apps/api/tests/unit/wallos-import-commit.test.ts @@ -4,13 +4,16 @@ const { prismaMock, appendSubscriptionOrders } = vi.hoisted(() => ({ prismaMock: { tag: { findMany: vi.fn(), - createMany: vi.fn() + createMany: vi.fn(), + create: vi.fn() }, subscription: { - createMany: vi.fn() + createMany: vi.fn(), + create: vi.fn() }, subscriptionTag: { - createMany: vi.fn() + createMany: vi.fn(), + create: vi.fn() } }, appendSubscriptionOrders: vi.fn(async () => undefined) @@ -45,83 +48,90 @@ describe('commitWallosImport', () => { vi.setSystemTime(new Date('2026-04-25T00:00:00.000Z')) prismaMock.tag.findMany.mockReset() prismaMock.tag.createMany.mockReset() + prismaMock.tag.create.mockReset() prismaMock.subscription.createMany.mockReset() + prismaMock.subscription.create.mockReset() prismaMock.subscriptionTag.createMany.mockReset() + prismaMock.subscriptionTag.create.mockReset() appendSubscriptionOrders.mockClear() previewState.getImportPreview.mockReset() previewState.deleteImportPreview.mockReset() - previewState.getImportPreview.mockResolvedValue({ - importToken: 'token-1', - isWallos: true, - summary: { - fileType: 'json', - subscriptionsTotal: 2, - tagsTotal: 2, - usedTagsTotal: 2, - supportedSubscriptions: 2, - skippedSubscriptions: 0, - globalNotifyDays: 3, - zipLogoMatched: 0, - zipLogoMissing: 0 - }, - usedTags: [ - { sourceId: 1, name: 'Video', sortOrder: 1 }, - { sourceId: 2, name: 'Music', sortOrder: 2 } - ], - tags: [], - subscriptionsPreview: [ - { - sourceId: 1, - name: 'Netflix', - amount: 10, - currency: 'USD', - status: 'active', - autoRenew: true, - billingIntervalCount: 1, - billingIntervalUnit: 'year', - startDate: '2025-01-10', - nextRenewalDate: '2027-01-10', - notifyDaysBefore: 3, - webhookEnabled: true, - notes: '', - description: '', - websiteUrl: 'https://netflix.com/', - tagNames: ['Video'], - logoRef: null, - logoImportStatus: 'none', - warnings: ['价格 "$10" 的币种符号存在歧义，已默认按 USD 导入'] - }, - { - sourceId: 2, - name: 'Spotify', - amount: 15, - currency: 'USD', - status: 'active', - autoRenew: true, - billingIntervalCount: 1, - billingIntervalUnit: 'month', - startDate: '2026-04-02', - nextRenewalDate: '2026-05-02', - notifyDaysBefore: 3, - webhookEnabled: true, - notes: '', - description: '', - websiteUrl: 'https://spotify.com', - tagNames: ['Music'], - logoRef: null, - logoImportStatus: 'none', - warnings: [] - } - ], - warnings: [] - }) }) afterEach(() => { vi.useRealTimers() }) - it('batches imported tags, subscription tags and subscription order writes', async () => { + it('batches imported tags, subscriptions, joins, and order writes for normal imports', async () => { + previewState.getImportPreview.mockResolvedValue({ + preview: { + importToken: 'token-1', + isWallos: true, + summary: { + fileType: 'db', + subscriptionsTotal: 2, + tagsTotal: 2, + usedTagsTotal: 2, + supportedSubscriptions: 2, + skippedSubscriptions: 0, + globalNotifyDays: 3, + zipLogoMatched: 0, + zipLogoMissing: 0 + }, + usedTags: [ + { sourceId: 1, name: 'Video', sortOrder: 1 }, + { sourceId: 2, name: 'Music', sortOrder: 2 } + ], + tags: [], + subscriptionsPreview: [ + { + sourceId: 1, + name: 'Netflix', + amount: 10, + currency: 'USD', + status: 'active', + autoRenew: true, + billingIntervalCount: 1, + billingIntervalUnit: 'year', + startDate: '2025-01-10', + nextRenewalDate: '2027-01-10', + notifyDaysBefore: 3, + webhookEnabled: true, + notes: '', + description: '', + websiteUrl: 'https://netflix.com/', + tagNames: ['Video'], + logoRef: null, + logoImportStatus: 'none', + warnings: [] + }, + { + sourceId: 2, + name: 'Spotify', + amount: 15, + currency: 'USD', + status: 'active', + autoRenew: true, + billingIntervalCount: 1, + billingIntervalUnit: 'month', + startDate: '2026-04-02', + nextRenewalDate: '2026-05-02', + notifyDaysBefore: 3, + webhookEnabled: true, + notes: '', + description: '', + websiteUrl: 'https://spotify.com/', + tagNames: ['Music'], + logoRef: null, + logoImportStatus: 'none', + warnings: [] + } + ], + warnings: [] + }, + logoManifest: {} + }) + prismaMock.tag.findMany .mockResolvedValueOnce([]) .mockResolvedValueOnce([ @@ -142,14 +152,14 @@ describe('commitWallosImport', () => { ] }) expect(prismaMock.subscription.createMany).toHaveBeenCalledTimes(1) - expect(prismaMock.subscription.createMany.mock.calls[0][0].data).toHaveLength(2) const createdRows = prismaMock.subscription.createMany.mock.calls[0][0].data const createdIds = createdRows.map((item: { id: string }) => item.id) expect(createdRows[0]).toMatchObject({ currency: 'USD', websiteUrl: 'https://netflix.com/', nextRenewalDate: new Date('2027-01-10T00:00:00.000Z'), - status: 'active' + status: 'active', + logoUrl: null }) expect(prismaMock.subscriptionTag.createMany).toHaveBeenCalledWith({ data: [ @@ -157,13 +167,82 @@ describe('commitWallosImport', () => { { subscriptionId: createdIds[1], tagId: 'tag_music' } ] }) - expect(appendSubscriptionOrders).toHaveBeenCalledTimes(1) expect(appendSubscriptionOrders).toHaveBeenCalledWith(createdIds) + expect(previewState.deleteImportPreview).toHaveBeenCalledWith('token-1') expect(result).toMatchObject({ importedTags: 2, importedSubscriptions: 2, - skippedSubscriptions: 0 + skippedSubscriptions: 0, + importedLogos: 0 }) - expect(previewState.deleteImportPreview).toHaveBeenCalledWith('token-1') + }) + + it('reuses preview-stage zip logo manifest instead of re-uploading logos during commit', async () => { + previewState.getImportPreview.mockResolvedValue({ + preview: { + importToken: 'token-zip', + isWallos: true, + summary: { + fileType: 'zip', + subscriptionsTotal: 1, + tagsTotal: 1, + usedTagsTotal: 1, + supportedSubscriptions: 1, + skippedSubscriptions: 0, + globalNotifyDays: 3, + zipLogoMatched: 1, + zipLogoMissing: 0 + }, + usedTags: [{ sourceId: 1, name: 'Video', sortOrder: 1 }], + tags: [], + subscriptionsPreview: [ + { + sourceId: 1, + name: 'Netflix', + amount: 10, + currency: 'USD', + status: 'active', + autoRenew: true, + billingIntervalCount: 1, + billingIntervalUnit: 'year', + startDate: '2025-01-10', + nextRenewalDate: '2027-01-10', + notifyDaysBefore: 3, + webhookEnabled: true, + notes: '', + description: '', + websiteUrl: 'https://netflix.com/', + tagNames: ['Video'], + logoRef: 'abc.png', + logoImportStatus: 'ready-from-zip', + warnings: [] + } + ], + warnings: [] + }, + logoManifest: { + 'abc.png': { + logoRef: 'abc.png', + r2Key: 'logos/imports/wallos/token-zip/abc.png', + logoUrl: '/static/logos/logos%2Fimports%2Fwallos%2Ftoken-zip%2Fabc.png', + contentType: 'image/png', + uploaded: true + } + } + }) + + prismaMock.tag.findMany.mockResolvedValue([{ id: 'tag_video', name: 'Video' }]) + prismaMock.subscription.createMany.mockResolvedValue({ count: 1 }) + prismaMock.subscriptionTag.createMany.mockResolvedValue({ count: 1 }) + + const result = await commitWallosImport({ importToken: 'token-zip' }) + + expect(prismaMock.subscription.createMany).toHaveBeenCalledTimes(1) + expect(prismaMock.subscription.createMany.mock.calls[0][0].data[0]).toMatchObject({ + logoUrl: '/static/logos/logos%2Fimports%2Fwallos%2Ftoken-zip%2Fabc.png', + logoSource: 'wallos-zip' + }) + expect(result.importedLogos).toBe(1) + expect(previewState.deleteImportPreview).toHaveBeenCalledWith('token-zip') }) }) diff --git a/apps/api/tests/unit/wallos-import.test.ts b/apps/api/tests/unit/wallos-import.test.ts index dea1bf8..3020bb4 100644 --- a/apps/api/tests/unit/wallos-import.test.ts +++ b/apps/api/tests/unit/wallos-import.test.ts @@ -1,14 +1,96 @@ +import { createRequire } from 'node:module' +import path from 'node:path' +import AdmZip from 'adm-zip' +import initSqlJs from 'sql.js' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mapWallosBillingInterval, - resolveWallosEffectiveNextPayment, mapWallosSubscriptionStatus, previewWallosImportFromBase64ForTest, + resolveWallosEffectiveNextPayment, resolveWallosNotifyDays } from '../../src/services/wallos-import.service' -function encodeJson(rows: unknown[]) { - return Buffer.from(JSON.stringify(rows), 'utf8').toString('base64') +const require = createRequire(import.meta.url) + +async function createWallosFixtureBuffer() { + const SQL = await initSqlJs({ + locateFile: (file: string) => path.resolve(path.dirname(require.resolve('sql.js/dist/sql-wasm.wasm')), file) + }) + + const db = new SQL.Database() + db.run(` + CREATE TABLE categories (id INTEGER PRIMARY KEY, name TEXT NOT NULL, "order" INTEGER DEFAULT 0, user_id INTEGER DEFAULT 1); + CREATE TABLE currencies (id INTEGER PRIMARY KEY, name TEXT NOT NULL, symbol TEXT NOT NULL, code TEXT NOT NULL, rate TEXT NOT NULL, user_id INTEGER DEFAULT 1); + CREATE TABLE cycles (id INTEGER PRIMARY KEY, days INTEGER NOT NULL, name TEXT NOT NULL); + CREATE TABLE frequencies (id INTEGER PRIMARY KEY, name INTEGER NOT NULL); + CREATE TABLE notification_settings (days INTEGER DEFAULT 0, user_id INTEGER DEFAULT 1); + CREATE TABLE subscriptions ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + logo TEXT, + price REAL NOT NULL, + currency_id INTEGER, + next_payment DATE, + cycle INTEGER, + frequency INTEGER, + notes TEXT, + payment_method_id INTEGER, + payer_user_id INTEGER, + category_id INTEGER, + notify BOOLEAN DEFAULT false, + url VARCHAR(255), + inactive BOOLEAN DEFAULT false, + notify_days_before INTEGER DEFAULT 0, + user_id INTEGER DEFAULT 1, + cancellation_date DATE, + replacement_subscription_id INTEGER DEFAULT NULL, + start_date INTEGER DEFAULT NULL, + auto_renew INTEGER DEFAULT 1 + ); + `) + + db.run(` + INSERT INTO categories (id, name, "order") VALUES + (1, 'No category', 1), + (2, 'VPS', 2), + (3, 'UnusedTag', 3); + + INSERT INTO currencies (id, name, symbol, code, rate) VALUES + (1, '人民币', '¥', 'CNY', '1'), + (2, 'US Dollar', '$', 'USD', '7'); + + INSERT INTO cycles (id, days, name) VALUES + (1, 30, 'Monthly'), + (2, 365, 'Yearly'); + + INSERT INTO frequencies (id, name) VALUES + (1, 1), + (2, 2); + + INSERT INTO notification_settings (days) VALUES (3); + + INSERT INTO subscriptions + (id, name, logo, price, currency_id, next_payment, cycle, frequency, notes, category_id, notify, url, inactive, notify_days_before, start_date, auto_renew) + VALUES + (10, 'Test VPS', 'abc.png', 10, 2, '2025-01-10', 2, 1, 'note', 2, 1, 'example.com', 0, -1, 1736467200, 1), + (11, 'No category sub', NULL, 5, 1, '2026-07-01', 1, 1, '', 1, 0, NULL, 0, NULL, NULL, 0); + `) + + const buffer = Buffer.from(db.export()) + db.close() + return buffer +} + +function encodeBase64(buffer: Buffer) { + return buffer.toString('base64') +} + +async function createWallosZipBase64() { + const zip = new AdmZip() + zip.addFile('backup/db/wallos.db', await createWallosFixtureBuffer()) + zip.addFile('backup/images/abc.png', Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) + return zip.toBuffer().toString('base64') } describe('wallos import helpers', () => { @@ -21,7 +103,7 @@ describe('wallos import helpers', () => { vi.useRealTimers() }) - it('maps standard billing intervals', () => { + it('maps standard billing intervals and wallos status helpers', () => { expect(mapWallosBillingInterval(30, 2)).toMatchObject({ billingIntervalCount: 2, billingIntervalUnit: 'month', @@ -32,9 +114,7 @@ describe('wallos import helpers', () => { billingIntervalCount: 45, billingIntervalUnit: 'day' }) - }) - it('maps status and notify config', () => { expect(mapWallosSubscriptionStatus({ inactive: 1, cancellationDate: null, nextPayment: '2026-06-01' })).toBe('paused') expect(mapWallosSubscriptionStatus({ inactive: 0, cancellationDate: '2026-04-01', nextPayment: '2026-06-01' })).toBe( 'cancelled' @@ -63,43 +143,14 @@ describe('wallos import helpers', () => { webhookEnabled: true, notifyDaysBefore: 3 }) - expect(resolveWallosNotifyDays({ notify: 1, notifyDaysBefore: 0, globalNotifyDays: 3 })).toEqual({ - webhookEnabled: true, - notifyDaysBefore: 3 - }) }) - it('only accepts wallos json in worker lite and keeps used tags only', async () => { + it('supports sqlite db imports with wallos semantics and used-tag filtering', async () => { const preview = await previewWallosImportFromBase64ForTest( { - filename: 'wallos.json', - contentType: 'application/json', - base64: encodeJson([ - { - Name: 'Test VPS', - Price: '$10.00', - Category: 'VPS', - 'Next Payment': '2026-06-01', - Notifications: 'enabled', - Renewal: 'automatic', - Notes: 'note', - URL: 'https://example.com/a' - }, - { - Name: 'No category sub', - Price: '¥5', - Category: 'No category', - 'Next Payment': '2026-07-01', - Notifications: 'disabled', - Renewal: 'manual' - }, - { - Name: '', - Price: '$1', - Category: 'UnusedTag', - 'Next Payment': '' - } - ]) + filename: 'wallos.db', + contentType: 'application/octet-stream', + base64: encodeBase64(await createWallosFixtureBuffer()) }, { defaultNotifyDays: 3, @@ -107,20 +158,22 @@ describe('wallos import helpers', () => { } ) - expect(preview.isWallos).toBe(true) - expect(preview.summary.fileType).toBe('json') + expect(preview.summary.fileType).toBe('db') expect(preview.usedTags.map((item) => item.name)).toEqual(['VPS']) expect(preview.summary.usedTagsTotal).toBe(1) - expect(preview.summary.skippedSubscriptions).toBe(1) - expect(preview.summary.zipLogoMatched).toBe(0) + + const categorized = preview.subscriptionsPreview.find((item) => item.name === 'Test VPS') + expect(categorized).toMatchObject({ + autoRenew: true, + websiteUrl: 'https://example.com/', + nextRenewalDate: '2027-01-10', + status: 'active', + notifyDaysBefore: 3, + tagNames: ['VPS'] + }) const uncategorized = preview.subscriptionsPreview.find((item) => item.name === 'No category sub') expect(uncategorized?.tagNames).toEqual([]) - - const categorized = preview.subscriptionsPreview.find((item) => item.name === 'Test VPS') - expect(categorized?.tagNames).toEqual(['VPS']) - expect(categorized?.autoRenew).toBe(true) - expect(categorized?.logoImportStatus).toBe('none') }) it('normalizes wallos json preview with wallos semantics and warnings', async () => { @@ -128,18 +181,20 @@ describe('wallos import helpers', () => { { filename: 'wallos.json', contentType: 'application/json', - base64: encodeJson([ - { - Name: 'Legacy Auto Renew', - Price: '$10', - Category: 'Video', - 'Payment Cycle': 'Yearly', - 'Next Payment': '2025-01-10', - Renewal: 'Automatic', - URL: 'netflix.com', - Notes: '' - } - ]) + base64: Buffer.from( + JSON.stringify([ + { + Name: 'Legacy Auto Renew', + Price: '$10', + Category: 'Video', + 'Payment Cycle': 'Yearly', + 'Next Payment': '2025-01-10', + Renewal: 'Automatic', + URL: 'netflix.com', + Notes: '' + } + ]) + ).toString('base64') }, { defaultNotifyDays: 3, @@ -148,7 +203,6 @@ describe('wallos import helpers', () => { ) const item = preview.subscriptionsPreview[0] - expect(item).toBeDefined() expect(item?.nextRenewalDate).toBe('2027-01-10') expect(item?.status).toBe('active') expect(item?.websiteUrl).toBe('https://netflix.com/') @@ -158,10 +212,27 @@ describe('wallos import helpers', () => { 'Wallos JSON 不包含 start_date，已使用 Next Payment 代填开始日期' ]) ) - expect(preview.warnings).toEqual( - expect.arrayContaining([ - expect.stringContaining('网址 "netflix.com" 缺少协议') - ]) + }) + + it('allows zip imports without R2 and warns that zip logos are ignored', async () => { + const preview = await previewWallosImportFromBase64ForTest( + { + filename: 'wallos-backup.zip', + contentType: 'application/zip', + base64: await createWallosZipBase64() + }, + { + defaultNotifyDays: 3, + baseCurrency: 'CNY' + } ) + + expect(preview.summary.fileType).toBe('zip') + expect(preview.summary.zipLogoMatched).toBe(1) + expect(preview.warnings).toEqual(expect.arrayContaining([expect.stringContaining('当前 Worker 未启用 R2')])) + expect(preview.subscriptionsPreview.find((item) => item.name === 'Test VPS')).toMatchObject({ + logoImportStatus: 'none', + logoRef: null + }) }) }) diff --git a/apps/web/src/components/WallosImportModal.vue b/apps/web/src/components/WallosImportModal.vue index 23bfb45..be94eb4 100644 --- a/apps/web/src/components/WallosImportModal.vue +++ b/apps/web/src/components/WallosImportModal.vue @@ -2,11 +2,11 @@ - Cloudflare Worker 版本当前仅支持上传 Wallos 的 JSON 导出文件，仍会只导入实际被订阅使用到的标签。 + 支持上传 Wallos 的 JSON、SQLite 数据库或 ZIP 包。当前只导入实际被订阅使用到的标签。 - + 选择文件 {{ selectedFileName || '未选择文件' }} 生成预览 @@ -38,8 +38,8 @@ -
Logo 导入
-
Cloudflare Worker 不支持
+
ZIP Logo 匹配
+
{{ preview.summary.zipLogoMatched }}/{{ preview.summary.zipLogoMatched + preview.summary.zipLogoMissing }}
 @@ -160,8 +160,8 @@ const subscriptionColumns = [ render: (row: WallosImportSubscriptionPreview) => ({ none: '无', - 'pending-file-match': '不支持', - 'ready-from-zip': '不支持' + 'pending-file-match': '待匹配', + 'ready-from-zip': 'ZIP 可导入' })[row.logoImportStatus] } ] @@ -250,8 +250,8 @@ function unitText(unit: WallosImportSubscriptionPreview['billingIntervalUnit']) function fileTypeText(type: WallosImportInspectResult['summary']['fileType']) { return { json: 'JSON', - db: '已停用', - zip: '已停用' + db: 'SQLite', + zip: 'ZIP' }[type] } @@ -278,6 +278,15 @@ function fileTypeText(type: WallosImportInspectResult['summary']['fileType']) { color: var(--app-text-strong); } +.warning-list { + margin-top: 12px; + margin-bottom: 0; + padding-left: 18px; + color: var(--app-text-secondary); + display: grid; + gap: 8px; +} + .warning-header { display: flex; align-items: center; @@ -287,12 +296,4 @@ function fileTypeText(type: WallosImportInspectResult['summary']['fileType']) { font-size: 13px; } -.warning-list { - margin-top: 12px; - margin-bottom: 0; - padding-left: 18px; - color: var(--app-text-secondary); - display: grid; - gap: 8px; -} diff --git a/apps/web/src/pages/SettingsPage.vue b/apps/web/src/pages/SettingsPage.vue index dec77ad..185eba3 100644 --- a/apps/web/src/pages/SettingsPage.vue +++ b/apps/web/src/pages/SettingsPage.vue @@ -11,7 +11,7 @@ 当前运行时：Cloudflare Worker。 KV 不使用； R2 {{ settingsForm.storageCapabilities.r2Enabled ? '已启用' : '未启用，仅支持远程 Logo 引用' }}； - Wallos 导入模式：仅 JSON。 + Wallos 导入模式：JSON / SQLite / ZIP。 @@ -521,9 +521,9 @@ 可导出全部订阅为 CSV / JSON，也可在这里导入 Wallos 数据。 - 当前 Cloudflare Worker 版本仅支持 JSON 导入。 + 当前 Cloudflare Worker 版本支持 JSON、SQLite 与 ZIP 导入。 @@ -689,7 +689,7 @@ const settingsForm = reactive({ kvEnabled: false, r2Enabled: false, logoStorageEnabled: false, - wallosImportMode: 'json-only' + wallosImportMode: 'json-db-zip' }, aiConfig: { ...DEFAULT_AI_CONFIG, diff --git a/apps/web/src/stores/app.ts b/apps/web/src/stores/app.ts index 485f766..da46f63 100644 --- a/apps/web/src/stores/app.ts +++ b/apps/web/src/stores/app.ts @@ -60,7 +60,7 @@ export const useAppStore = defineStore('app', () => { kvEnabled: false, r2Enabled: false, logoStorageEnabled: false, - wallosImportMode: 'json-only' + wallosImportMode: 'json-db-zip' }, aiConfig: { ...DEFAULT_AI_CONFIG, diff --git a/apps/web/src/types/api.ts b/apps/web/src/types/api.ts index e44942e..59798f8 100644 --- a/apps/web/src/types/api.ts +++ b/apps/web/src/types/api.ts @@ -227,7 +227,7 @@ export interface StorageCapabilities { kvEnabled: boolean r2Enabled: boolean logoStorageEnabled: boolean - wallosImportMode: 'json-only' + wallosImportMode: 'json-only' | 'json-db-zip' } export interface Settings { diff --git a/apps/web/tests/unit/utils/worker-capabilities.test.ts b/apps/web/tests/unit/utils/worker-capabilities.test.ts index 0ecb7fc..3867bf4 100644 --- a/apps/web/tests/unit/utils/worker-capabilities.test.ts +++ b/apps/web/tests/unit/utils/worker-capabilities.test.ts @@ -10,7 +10,7 @@ describe('worker capabilities helpers', () => { kvEnabled: true, r2Enabled: true, logoStorageEnabled: true, - wallosImportMode: 'json-only' + wallosImportMode: 'json-db-zip' } }) ).toBe(true) @@ -25,7 +25,7 @@ describe('worker capabilities helpers', () => { kvEnabled: false, r2Enabled: false, logoStorageEnabled: false, - wallosImportMode: 'json-only' + wallosImportMode: 'json-db-zip' } }) ).toBe(false) diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 5d62d48..56e2f61 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -192,7 +192,7 @@ export const StorageCapabilitiesSchema = z.object({ kvEnabled: z.boolean().default(false), r2Enabled: z.boolean().default(false), logoStorageEnabled: z.boolean().default(false), - wallosImportMode: z.literal('json-only').default('json-only') + wallosImportMode: z.enum(['json-only', 'json-db-zip']).default('json-db-zip') }) export const SettingsSchema = z.object({