feat(data-process): 接入三种文档切分引擎

backend/app/modules/data_process/algorithms.py

@@ -1,8 +1,7 @@
 """数据处理模块使用的无副作用算法。
 
 本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
-所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
-``source[chunk.start:chunk.end]``。
+所有偏移量均为 Python 字符串偏移量。
 """
 
 from __future__ import annotations
@@ -16,7 +15,6 @@ import re
 import unicodedata
 import xml.etree.ElementTree as ET
 import zipfile
-from bisect import bisect_left
 from collections import Counter
 from collections.abc import Iterable, Mapping, Sequence
 from copy import deepcopy
@@ -31,7 +29,6 @@ from docx.oxml.table import CT_Tbl
 from docx.oxml.text.paragraph import CT_P
 from docx.table import Table
 from docx.text.paragraph import Paragraph
-from llama_index.core.node_parser import SentenceSplitter, TokenTextSplitter
 from openpyxl import load_workbook
 from openpyxl.utils.cell import range_boundaries
 from pptx import Presentation
@@ -48,7 +45,6 @@ TextFormat = Literal[
     "xlsx",
     "pptx",
 ]
-ChunkMethod = Literal["structure", "fixed", "custom"]
 DatasetSplit = Literal["train", "validation", "test"]
 StructuredPreprocessOption = Literal[
     "clean_invalid",
@@ -153,7 +149,6 @@ _ENGLISH_NAME_CONTEXT_PATTERN = re.compile(
     r"(?P<name>[A-Za-z][A-Za-z'’-]*(?:[ \t]+[A-Za-z][A-Za-z'’-]*){0,3})"
 )
 _TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
-_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
 
 
 @dataclass(frozen=True, slots=True)
@@ -184,18 +179,6 @@ class DocumentNoiseSpan:
     kind: Literal["page_number", "repeated_margin", "table_of_contents"]
 
 
-@dataclass(frozen=True, slots=True)
-class TextChunk:
-    """带有可追溯来源位置的非结构化文本切片。"""
-
-    content: str
-    start: int
-    end: int
-    start_line: int
-    end_line: int
-    token_count: int
-
-
 @dataclass(frozen=True, slots=True)
 class QualityScore:
     """标准 instruction/input/output 记录的可解释质量分。"""
@@ -1975,41 +1958,6 @@ def estimate_token_count(text: str) -> int:
     return len(_TOKEN_PATTERN.findall(text))
 
 
-def _token_spans(text: str) -> list[tuple[int, int]]:
-    return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
-
-
-def _deterministic_tokenizer(text: str) -> list[str]:
-    """LlamaIndex splitter 使用的稳定 tokenizer，与预览 token 计数完全一致。"""
-
-    return _TOKEN_PATTERN.findall(text)
-
-
-def _sentence_chunks(text: str) -> list[str]:
-    """按项目既有中英文句界切句，避免 SentenceSplitter 触发 NLTK 下载。"""
-
-    chunks: list[str] = []
-    cursor = 0
-    for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(text):
-        end = match.end()
-        if end > cursor:
-            chunks.append(text[cursor:end])
-        cursor = end
-    if cursor < len(text):
-        chunks.append(text[cursor:])
-    return chunks or [text]
-
-
-def _line_number(newline_offsets: list[int], offset: int) -> int:
-    # 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
-    return bisect_left(newline_offsets, offset) + 1
-
-
-def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
-    starts = [span[0] for span in spans]
-    return bisect_left(starts, offset)
-
-
 def _protected_markdown_ranges(
     text: str,
     *,
@@ -2110,12 +2058,6 @@ def _protected_markdown_ranges(
     return merged
 
 
-def _range_containing(
-    ranges: Sequence[tuple[int, int]], offset: int
-) -> tuple[int, int] | None:
-    return next((item for item in ranges if item[0] < offset < item[1]), None)
-
-
 def protected_context_ranges(
     text: str,
     *,
@@ -2444,269 +2386,6 @@ def is_near_duplicate(
     )
 
 
-def _boundary_for_method(
-    text: str,
-    spans: list[tuple[int, int]],
-    start_index: int,
-    ideal_end_index: int,
-    minimum_end_index: int,
-    method: ChunkMethod,
-    custom_delimiter: str,
-    splitter: SentenceSplitter | TokenTextSplitter | None,
-) -> tuple[int, int | None]:
-    if method == "fixed" and not isinstance(splitter, TokenTextSplitter):
-        raise RuntimeError("fixed chunking requires TokenTextSplitter")
-
-    start_offset = spans[start_index][0]
-    ideal_end_offset = spans[ideal_end_index - 1][1]
-    minimum_end_offset = spans[minimum_end_index - 1][1]
-    search_text = text[start_offset:ideal_end_offset]
-
-    if method == "fixed":
-        # 直接让 TokenTextSplitter 处理真实文本；开启空白保留后将首块边界
-        # 投影回稳定 token span，offset 和实际 overlap 仍由外层统一维护。
-        lookahead_end = min(len(spans), ideal_end_index + 1)
-        window_end = spans[lookahead_end - 1][1]
-        window = text[start_offset:window_end]
-        chunks = splitter.split_text(window)
-        first_chunk = chunks[0] if chunks else ""
-        if first_chunk and window.startswith(first_chunk):
-            boundary_offset = start_offset + len(first_chunk)
-            boundary_index = _token_index_at_or_after(spans, boundary_offset)
-            if boundary_index > start_index:
-                return min(boundary_index, ideal_end_index), boundary_offset
-        return ideal_end_index, None
-
-    if method == "custom":
-        delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
-        if not delimiter:
-            raise ValueError("custom_delimiter is required for custom chunking")
-        relative_minimum = max(0, minimum_end_offset - start_offset)
-        delimiter_start = search_text.rfind(delimiter, relative_minimum)
-        if delimiter_start >= 0:
-            boundary_offset = start_offset + delimiter_start + len(delimiter)
-            boundary_index = _token_index_at_or_after(spans, boundary_offset)
-            if boundary_index > start_index:
-                return min(boundary_index, ideal_end_index), boundary_offset
-        return ideal_end_index, None
-
-    if method == "structure":
-        if not isinstance(splitter, SentenceSplitter):
-            raise RuntimeError("structure chunking requires SentenceSplitter")
-        # 多给一个 token 使 splitter 确实执行限长；只取首块并投影回原文。
-        lookahead_end = min(len(spans), ideal_end_index + 1)
-        window_end = spans[lookahead_end - 1][1]
-        window = text[start_offset:window_end]
-        chunks = splitter.split_text(window)
-        first_chunk = chunks[0] if chunks else ""
-        if first_chunk and window.startswith(first_chunk):
-            boundary_offset = start_offset + len(first_chunk)
-        else:
-            boundary_offset = ideal_end_offset
-        boundary_index = _token_index_at_or_after(spans, boundary_offset)
-        if boundary_index >= minimum_end_index:
-            return min(boundary_index, ideal_end_index), boundary_offset
-    return ideal_end_index, None
-
-
-def _chunk_normalized_text(
-    normalized: str,
-    *,
-    method: ChunkMethod,
-    chunk_size: int = 800,
-    chunk_overlap: int = 100,
-    min_chunk_size: int = 100,
-    custom_delimiter: str = "",
-    preserve_code_blocks: bool = False,
-    preserve_tables: bool = False,
-    preserve_lists: bool = False,
-) -> list[TextChunk]:
-    if not normalized:
-        return []
-    spans = _token_spans(normalized)
-    if not spans:
-        return []
-
-    newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
-    protected_ranges = _protected_markdown_ranges(
-        normalized,
-        preserve_code_blocks=preserve_code_blocks,
-        preserve_tables=preserve_tables,
-        preserve_lists=preserve_lists,
-    )
-    splitter: SentenceSplitter | TokenTextSplitter | None
-    if method == "fixed":
-        splitter = TokenTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            tokenizer=_deterministic_tokenizer,
-            separator=" ",
-            backup_separators=["\n"],
-            keep_whitespaces=True,
-            include_metadata=False,
-            include_prev_next_rel=False,
-        )
-    elif method == "structure":
-        splitter = SentenceSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            tokenizer=_deterministic_tokenizer,
-            chunking_tokenizer_fn=_sentence_chunks,
-            include_metadata=False,
-            include_prev_next_rel=False,
-        )
-    else:
-        splitter = None
-    chunks: list[TextChunk] = []
-    start_index = 0
-
-    while start_index < len(spans):
-        ideal_end_index = min(len(spans), start_index + chunk_size)
-        if ideal_end_index == len(spans):
-            end_index, end_override = ideal_end_index, len(normalized)
-        else:
-            minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
-            end_index, end_override = _boundary_for_method(
-                normalized,
-                spans,
-                start_index,
-                ideal_end_index,
-                minimum_end_index,
-                method,
-                custom_delimiter,
-                splitter,
-            )
-        if end_index <= start_index:
-            end_index = min(len(spans), start_index + chunk_size)
-            end_override = None
-
-        start_offset = spans[start_index][0]
-        end_offset = end_override if end_override is not None else spans[end_index - 1][1]
-        end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
-        split_range = _range_containing(protected_ranges, end_offset)
-        if split_range:
-            before_index = _token_index_at_or_after(spans, split_range[0])
-            if before_index - start_index >= min_chunk_size:
-                end_index = before_index
-                end_offset = split_range[0]
-            else:
-                end_index = min(
-                    len(spans),
-                    max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
-                )
-                end_offset = split_range[1]
-        content = normalized[start_offset:end_offset]
-        chunks.append(
-            TextChunk(
-                content=content,
-                start=start_offset,
-                end=end_offset,
-                start_line=_line_number(newline_offsets, start_offset),
-                end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
-                token_count=end_index - start_index,
-            )
-        )
-
-        if end_index >= len(spans):
-            break
-        next_start = max(start_index + 1, end_index - chunk_overlap)
-        overlap_range = _range_containing(protected_ranges, spans[next_start][0])
-        if overlap_range:
-            candidate = _token_index_at_or_after(spans, overlap_range[0])
-            if candidate <= start_index:
-                candidate = _token_index_at_or_after(spans, overlap_range[1])
-            next_start = min(len(spans), max(start_index + 1, candidate))
-        start_index = next_start
-
-    return chunks
-
-
-def _structure_sections(text: str) -> list[tuple[int, int]]:
-    structure = detect_document_structure(text)
-    if not structure.headings:
-        return [(0, len(text))]
-
-    sections: list[tuple[int, int]] = []
-    first_start = structure.headings[0].start
-    if first_start > 0:
-        sections.append((0, first_start))
-    sections.extend(
-        (
-            heading.start,
-            structure.headings[index + 1].start
-            if index + 1 < len(structure.headings)
-            else len(text),
-        )
-        for index, heading in enumerate(structure.headings)
-    )
-    return sections
-
-
-def chunk_unstructured(
-    text: str,
-    *,
-    method: ChunkMethod = "structure",
-    chunk_size: int = 800,
-    chunk_overlap: int = 100,
-    min_chunk_size: int = 100,
-    custom_delimiter: str = "",
-    preserve_code_blocks: bool = False,
-    preserve_tables: bool = False,
-    preserve_lists: bool = False,
-) -> list[TextChunk]:
-    """按确定性 token 切分，并保留规范化原文的 offset、行号和实际 overlap。"""
-
-    if method not in {"structure", "fixed", "custom"}:
-        raise ValueError(f"unsupported chunk method: {method}")
-    if chunk_size <= 0:
-        raise ValueError("chunk_size must be greater than 0")
-    if chunk_overlap < 0 or chunk_overlap >= chunk_size:
-        raise ValueError("chunk_overlap must be in [0, chunk_size)")
-    if min_chunk_size <= 0 or min_chunk_size > chunk_size:
-        raise ValueError("min_chunk_size must be in [1, chunk_size]")
-    if chunk_overlap + min_chunk_size > chunk_size:
-        raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
-    if method == "custom" and not custom_delimiter:
-        raise ValueError("custom_delimiter is required for custom chunking")
-
-    normalized = normalize_text(text)
-    if not normalized:
-        return []
-    section_ranges = (
-        _structure_sections(normalized)
-        if method == "structure"
-        else [(0, len(normalized))]
-    )
-    newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
-    chunks: list[TextChunk] = []
-    for section_start, section_end in section_ranges:
-        section = normalized[section_start:section_end]
-        for chunk in _chunk_normalized_text(
-            section,
-            method=method,
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            min_chunk_size=min_chunk_size,
-            custom_delimiter=custom_delimiter,
-            preserve_code_blocks=preserve_code_blocks,
-            preserve_tables=preserve_tables,
-            preserve_lists=preserve_lists,
-        ):
-            start = section_start + chunk.start
-            end = section_start + chunk.end
-            chunks.append(
-                TextChunk(
-                    content=normalized[start:end],
-                    start=start,
-                    end=end,
-                    start_line=_line_number(newline_offsets, start),
-                    end_line=_line_number(newline_offsets, max(start, end - 1)),
-                    token_count=chunk.token_count,
-                )
-            )
-    return chunks
-
-
 def record_fingerprint(record: Mapping[str, Any]) -> str:
     """计算与字典键顺序无关的稳定记录指纹。"""
 
@@ -3029,7 +2708,6 @@ def generate_standard_records(
 
 
 __all__ = [
-    "ChunkMethod",
     "DatasetSplit",
     "DocumentHeading",
     "DocumentNoiseSpan",
@@ -3039,10 +2717,8 @@ __all__ = [
     "QualityScore",
     "SUPPORTED_TEXT_FORMATS",
     "StructuredPreprocessOption",
-    "TextChunk",
     "TextFormat",
     "canonical_record_json",
-    "chunk_unstructured",
     "content_quality_flags",
     "decode_utf8",
     "desensitize_pii",