From 07422d7c1b475f626db783e3cc0589c58c00ca17 Mon Sep 17 00:00:00 2001 From: Cyrus Auyeung Date: Thu, 23 Jul 2026 10:34:59 +0800 Subject: [PATCH] =?UTF-8?q?fix(canvas):=20=E5=B0=8F=E8=AF=B4=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=20-=20=E5=A2=9E=E5=BC=BA=20TXT=20=E5=88=86=E7=AB=A0?= =?UTF-8?q?=E4=B8=8E=E7=BC=96=E7=A0=81=E8=AF=86=E5=88=AB=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canvas/canvas-document-editor-modal.tsx | 4 +- web/src/lib/canvas/canvas-document.ts | 51 +++++++++++++++++-- web/test/canvas-document.test.ts | 48 +++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 web/test/canvas-document.test.ts diff --git a/web/src/components/canvas/canvas-document-editor-modal.tsx b/web/src/components/canvas/canvas-document-editor-modal.tsx index de108d655..011d47708 100644 --- a/web/src/components/canvas/canvas-document-editor-modal.tsx +++ b/web/src/components/canvas/canvas-document-editor-modal.tsx @@ -45,7 +45,7 @@ import { TextStyle } from "@tiptap/extension-text-style"; import Color from "@tiptap/extension-color"; import type { Editor } from "@tiptap/core"; import type { CanvasDocumentChapter, CanvasNodeData, CanvasRichDocument } from "@/types/canvas"; -import { buildRichDocument, createDocumentChapter, emptyDocument, normalizeDocumentChapters, splitTextIntoChapters } from "@/lib/canvas/canvas-document"; +import { buildRichDocument, createDocumentChapter, decodeNovelText, emptyDocument, normalizeDocumentChapters, splitTextIntoChapters } from "@/lib/canvas/canvas-document"; import { canvasThemes } from "@/lib/canvas-theme"; import { useThemeStore } from "@/stores/use-theme-store"; @@ -281,7 +281,7 @@ export function CanvasDocumentEditorModal({ node, open, saving = false, analyzin if (file.name.toLowerCase().endsWith(".docx")) { const mammoth = await import("mammoth"); text = (await mammoth.extractRawText({ arrayBuffer: await file.arrayBuffer() })).value; - } else text = await file.text(); + } else text = decodeNovelText(await file.arrayBuffer()); if (!text.trim()) throw new Error("文件中没有可编辑的文字"); const imported = splitTextIntoChapters(text); setChapters(imported); diff --git a/web/src/lib/canvas/canvas-document.ts b/web/src/lib/canvas/canvas-document.ts index ccbdfffe7..913095aa2 100644 --- a/web/src/lib/canvas/canvas-document.ts +++ b/web/src/lib/canvas/canvas-document.ts @@ -1,6 +1,19 @@ import type { JSONContent } from "@tiptap/core"; import type { CanvasDocumentChapter, CanvasRichDocument } from "@/types/canvas"; +const maxChapterHeadingLength = 120; +const chineseChapterNumber = "[零〇○一二两三四五六七八九十百千万亿壹贰叁肆伍陆柒捌玖拾佰仟\\d]+"; +const englishChapterNumber = "(?:\\d+|[ivxlcdm]+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)"; +const chapterTitleSuffix = "(?:\\s*[-—–:：·.]\\s*.*|\\s+.*)?"; +const chapterHeadingPatterns = [ + new RegExp(`^(?:正文\\s*)?第\\s*${chineseChapterNumber}\\s*[章节卷回部篇集季幕]${chapterTitleSuffix}$`, "i"), + new RegExp(`^(?:卷|篇|部|集|季|幕)\\s*${chineseChapterNumber}${chapterTitleSuffix}$`, "i"), + new RegExp(`^(?:序|序幕|序章|序言|前言|前记|楔子|引子|引言|开篇|终章|尾声|后记)${chapterTitleSuffix}$`, "i"), + new RegExp(`^(?:番外(?:篇)?|附录)(?:\\s*(?:第\\s*)?${chineseChapterNumber}\\s*[章节回篇]?)?${chapterTitleSuffix}$`, "i"), + new RegExp(`^(?:chapter|book|part|volume)\\s+${englishChapterNumber}\\b${chapterTitleSuffix}$`, "i"), + new RegExp(`^(?:prologue|preface|introduction|epilogue|afterword|appendix)${chapterTitleSuffix}$`, "i"), +]; + export function emptyDocument(): JSONContent { return { type: "doc", content: [{ type: "paragraph" }] }; } @@ -40,19 +53,51 @@ export function normalizeDocumentChapters(document: CanvasRichDocument | undefin return [chapter]; } +function chapterTitleFromLine(line: string) { + const trimmed = line.trim(); + if (!trimmed || Array.from(trimmed).length > maxChapterHeadingLength) return null; + + const title = trimmed + .replace(/^[#＃]{1,6}\s*/, "") + .replace(/^[\s=_*~\-—–]+|[\s=_*~\-—–]+$/g, "") + .trim(); + const candidate = title + .normalize("NFKC") + .replace(/[【】\[\]「」『』〈〉《》〖〗]/g, " ") + .replace(/\s+/g, " ") + .trim(); + + // 只把较短且整行符合章节结构的文本当作标题，避免正文中提到“第一章”时误切分。 + return chapterHeadingPatterns.some((pattern) => pattern.test(candidate)) ? title : null; +} + +export function decodeNovelText(buffer: ArrayBuffer) { + const bytes = new Uint8Array(buffer); + if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return new TextDecoder("utf-8").decode(bytes.subarray(3)); + if (bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder("utf-16le").decode(bytes.subarray(2)); + if (bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder("utf-16be").decode(bytes.subarray(2)); + + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + // 中文网文 TXT 常由旧版编辑器导出为 GBK；GB18030 兼容 GBK，并能覆盖更多汉字。 + return new TextDecoder("gb18030").decode(bytes); + } +} + export function splitTextIntoChapters(text: string) { const normalized = text.replace(/\r\n?/g, "\n").trim(); if (!normalized) return [createDocumentChapter("第 1 章")]; const lines = normalized.split("\n"); - const headingPattern = /^\s*(第[零〇一二两三四五六七八九十百千万\d]+[章节卷回部篇]|chapter\s+\d+\b|序章|楔子|引子|尾声|后记)(?:[\s：:.-]+.*)?\s*$/i; const sections: Array<{ title: string; lines: string[] }> = []; let current: { title: string; lines: string[] } | null = null; const preface: string[] = []; for (const line of lines) { - if (headingPattern.test(line.trim())) { + const chapterTitle = chapterTitleFromLine(line); + if (chapterTitle) { if (current) sections.push(current); - current = { title: line.trim(), lines: [] }; + current = { title: chapterTitle, lines: [] }; continue; } if (current) current.lines.push(line); diff --git a/web/test/canvas-document.test.ts b/web/test/canvas-document.test.ts new file mode 100644 index 000000000..7fa552e75 --- /dev/null +++ b/web/test/canvas-document.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; +import { decodeNovelText, splitTextIntoChapters } from "../src/lib/canvas/canvas-document"; + +describe("splitTextIntoChapters", () => { + test("识别常见中文、Markdown、装饰符与英文标题", () => { + const chapters = splitTextIntoChapters(`书名与作者 + +第 1 章 初入江湖 +第一章正文 + +＃ 第２章 风雨欲来 +第二章正文 + +【第三章】再会故人 +第三章正文 + +卷四 山河故人 +第四章正文 + +Epilogue: 重逢 +尾声正文`); + + expect(chapters.map((chapter) => chapter.title)).toEqual(["序章", "第 1 章 初入江湖", "第２章 风雨欲来", "【第三章】再会故人", "卷四 山河故人", "Epilogue: 重逢"]); + expect(chapters[0].plainText).toBe("书名与作者"); + expect(chapters[2].plainText).toBe("第二章正文"); + }); + + test("普通正文和未识别文本保持为单个章节", () => { + const text = "这是第一章中发生的故事。\n角色继续前行。"; + const chapters = splitTextIntoChapters(text); + + expect(chapters).toHaveLength(1); + expect(chapters[0].title).toBe("第 1 章"); + expect(chapters[0].plainText).toBe(text); + }); +}); + +describe("decodeNovelText", () => { + test("解码带 BOM 的 UTF-16LE", () => { + const bytes = new Uint8Array([0xff, 0xfe, 0x2c, 0x7b, 0x00, 0x4e, 0xe0, 0x7a]); + expect(decodeNovelText(bytes.buffer)).toBe("第一章"); + }); + + test("UTF-8 严格解码失败后回退到 GB18030", () => { + const bytes = new Uint8Array([0xb5, 0xda, 0xd2, 0xbb, 0xd5, 0xc2, 0x0a, 0xd5, 0xfd, 0xce, 0xc4]); + expect(decodeNovelText(bytes.buffer)).toBe("第一章\n正文"); + }); +});