feat(data-process): 接入三种文档切分引擎

backend/app/api/v1/endpoints/data_process.py

@@ -30,13 +30,10 @@ from psycopg.rows import dict_row
 
 from app.modules.data_process.algorithms import (
     ParsedText,
-    TextChunk,
     canonical_record_json,
-    chunk_unstructured,
     content_quality_flags,
     desensitize_pii,
     desensitize_structured_record,
-    detect_document_structure,
     detect_pdf_document_noise,
     estimate_token_count,
     extract_pdf_page_texts,
@@ -48,6 +45,13 @@ from app.modules.data_process.algorithms import (
     remove_document_noise,
     score_quality,
 )
+from app.modules.data_process.document_chunking import (
+    DocumentChunk,
+    chunk_fixed_text,
+    chunk_layout_document,
+    chunk_semantic_text,
+    merge_short_chunks,
+)
 from app.modules.data_process.generation import generate_model_records
 from app.modules.data_process.storage import (
     LocalDataProcessStorage,
@@ -238,152 +242,58 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
     )
 
 
-def _shift_chunk(chunk: TextChunk, offset: int, source_text: str) -> TextChunk:
-    start = offset + chunk.start
-    end = offset + chunk.end
-    return TextChunk(
-        content=chunk.content,
-        start=start,
-        end=end,
-        start_line=source_text.count("\n", 0, start) + 1,
-        end_line=source_text.count("\n", 0, max(start, end - 1)) + 1,
-        token_count=chunk.token_count,
-    )
-
-
-def _merge_short_chunks(
-    chunks: list[TextChunk],
-    source_text: str,
-    *,
-    min_token_count: int,
-    chunk_size: int,
-) -> list[TextChunk]:
-    """按原文顺序合并相邻短块，同时严格遵守切片大小上限。"""
-
-    merged: list[TextChunk] = []
-    index = 0
-    while index < len(chunks):
-        current = chunks[index]
-        if current.token_count >= min_token_count:
-            merged.append(current)
-            index += 1
-            continue
-
-        candidates: list[tuple[int, int, int]] = []
-        if merged:
-            candidates.append((merged[-1].start, current.end, -1))
-        if index + 1 < len(chunks):
-            candidates.append((current.start, chunks[index + 1].end, 1))
-        selected = next(
-            (
-                (start, end, direction)
-                for start, end, direction in candidates
-                if estimate_token_count(source_text[start:end]) <= chunk_size
-            ),
-            None,
-        )
-        if selected is None:
-            merged.append(current)
-            index += 1
-            continue
-
-        start, end, direction = selected
-        combined = TextChunk(
-            content=source_text[start:end],
-            start=start,
-            end=end,
-            start_line=source_text.count("\n", 0, start) + 1,
-            end_line=source_text.count("\n", 0, max(start, end - 1)) + 1,
-            token_count=estimate_token_count(source_text[start:end]),
-        )
-        if direction < 0:
-            merged[-1] = combined
-            index += 1
-        else:
-            merged.append(combined)
-            index += 2
-    return merged
-
-
 def _chunk_source_text(
-    text: str,
+    source: dict[str, Any],
     config: dict[str, Any],
     preprocess_options: set[str],
-) -> list[tuple[TextChunk, tuple[str, ...]]]:
-    """按可选文档结构分段后切片，结构边界之间不共享 overlap。"""
+) -> list[DocumentChunk]:
+    """根据任务配置调用真实的 Docling/LlamaIndex 切分器。"""
 
-    method = str(_value(config, "chunk_method", "chunkMethod", "structure"))
-    detect_structure = (
-        method == "structure" or "detect_document_structure" in preprocess_options
-    )
+    method = str(_value(config, "chunk_method", "chunkMethod", "layout_hybrid"))
     preserve_context = "preserve_context" in preprocess_options
-    merge_short = "merge_short_content" in preprocess_options
     chunk_size = int(_value(config, "chunk_size", "chunkSize", 800))
-    configured_minimum = int(_value(config, "min_chunk_size", "minChunkSize", 100))
-    minimum = configured_minimum if merge_short else 1
     overlap = (
         int(_value(config, "chunk_overlap", "chunkOverlap", 100))
         if preserve_context
         else 0
     )
-    common = {
-        "method": method,
-        "chunk_size": chunk_size,
-        "chunk_overlap": overlap,
-        "min_chunk_size": minimum,
-        "custom_delimiter": str(
-            _value(config, "custom_delimiter", "customDelimiter", "") or ""
-        ),
-        "preserve_code_blocks": bool(
-            _value(config, "preserve_code_blocks", "preserveCodeBlocks", False)
-        ),
-        "preserve_tables": bool(
-            _value(config, "preserve_tables", "preserveTables", False)
-        ),
-        "preserve_lists": bool(
-            _value(config, "preserve_lists", "preserveLists", False)
-        ),
-    }
-
-    sections: list[tuple[int, int, tuple[str, ...]]] = [(0, len(text), ())]
-    if detect_structure:
-        structure = detect_document_structure(text)
-        if structure.headings:
-            sections = []
-            first_start = structure.headings[0].start
-            if first_start > 0 and text[:first_start].strip():
-                sections.append((0, first_start, ()))
-            stack: list[tuple[int, str]] = []
-            for index, heading in enumerate(structure.headings):
-                while stack and stack[-1][0] >= heading.level:
-                    stack.pop()
-                stack.append((heading.level, heading.title))
-                end = (
-                    structure.headings[index + 1].start
-                    if index + 1 < len(structure.headings)
-                    else len(text)
-                )
-                sections.append((heading.start, end, tuple(title for _, title in stack)))
-
-    result: list[tuple[TextChunk, tuple[str, ...]]] = []
-    for start, end, heading_path in sections:
-        section_text = text[start:end]
-        local_chunks = chunk_unstructured(section_text, **common)
-        shifted = [_shift_chunk(chunk, start, text) for chunk in local_chunks]
-        result.extend((chunk, heading_path) for chunk in shifted)
-
-    if merge_short and result:
-        # 结构分段只负责提供标题路径和隔离 overlap，不应让目录项或短小节
-        # 突破 min_chunk_size 约束。合并后保留首个原始块的标题路径。
-        heading_paths = {chunk.start: heading_path for chunk, heading_path in result}
-        merged = _merge_short_chunks(
-            [chunk for chunk, _ in result],
-            text,
-            min_token_count=configured_minimum,
+    text = str(source.get("content") or "")
+    if method == "layout_hybrid":
+        raw = source.get("raw_content")
+        if not isinstance(raw, bytes):
+            raise InvalidStateError("版面结构混合切分需要原始文件，请重新上传后再处理")
+        chunks = chunk_layout_document(
+            raw,
+            filename=str(source.get("name") or "document.pdf"),
+            source_text=text,
             chunk_size=chunk_size,
         )
-        result = [(chunk, heading_paths.get(chunk.start, ())) for chunk in merged]
-    return result
+    elif method == "semantic":
+        chunks = chunk_semantic_text(
+            text,
+            chunk_size=chunk_size,
+            chunk_overlap=overlap,
+            breakpoint_percentile_threshold=int(
+                _value(
+                    config,
+                    "semantic_breakpoint_percentile",
+                    "semanticBreakpointPercentile",
+                    95,
+                )
+            ),
+        )
+    elif method == "fixed":
+        chunks = chunk_fixed_text(text, chunk_size=chunk_size, chunk_overlap=overlap)
+    else:
+        raise ValueError(f"unsupported chunk method: {method}")
+    if "merge_short_content" in preprocess_options:
+        chunks = merge_short_chunks(
+            chunks,
+            source_text=text,
+            min_token_count=int(_value(config, "min_chunk_size", "minChunkSize", 100)),
+            max_token_count=chunk_size,
+        )
+    return chunks
 
 
 _NEGATION_MARKERS = frozenset({"不", "无", "未", "否", "没有", "并非", "not", "no", "never"})
@@ -456,23 +366,27 @@ def _build_preview_items(
         parsed = _parse_stored_source(source)
         if process_type == "unstructured":
             document_noise_spans = tuple(source.get("document_noise_spans") or ())
-            chunks = _chunk_source_text(parsed.text, config, preprocess_options)
-            for chunk, heading_path in chunks:
+            chunks = _chunk_source_text(source, config, preprocess_options)
+            for chunk in chunks:
                 content = (
                     remove_document_noise(
-                        chunk.content,
+                        chunk.contextualized_content,
                         document_noise_spans,
-                        source_offset=chunk.start,
+                        source_offset=chunk.source_start or 0,
                     )
-                    if should_clean_invalid and document_noise_spans
-                    else chunk.content
+                    if (
+                        should_clean_invalid
+                        and document_noise_spans
+                        and chunk.source_start is not None
+                    )
+                    else chunk.contextualized_content
                 )
                 preprocess_flags = content_quality_flags(
                     content,
                     min_chars=0,
                     min_tokens=0,
                 )
-                if content != chunk.content:
+                if content != chunk.contextualized_content:
                     preprocess_flags = (*preprocess_flags, "document_noise_removed")
                 flag_set = set(preprocess_flags)
                 if "clean_invalid_content" in preprocess_options and flag_set & {
@@ -508,25 +422,28 @@ def _build_preview_items(
                 quality = _preview_quality(content, config)
                 quality["pii_replacements"] = pii_counts
                 quality["preprocess_flags"] = list(preprocess_flags)
-                chunk_method = str(
-                    _value(config, "chunk_method", "chunkMethod", "structure")
+                quality["chunk_method"] = str(
+                    _value(config, "chunk_method", "chunkMethod", "layout_hybrid")
                 )
-                if (
-                    chunk_method == "structure"
-                    or "detect_document_structure" in preprocess_options
-                ):
-                    quality["heading_path"] = list(heading_path)
+                quality["heading_path"] = list(chunk.heading_path)
+                quality["source_pages"] = list(chunk.source_pages)
+                quality["doc_item_refs"] = list(chunk.doc_item_refs)
+                quality["source_bboxes"] = list(chunk.source_bboxes)
                 append_item(
                     {
                         "source_file_id": source["id"],
-                        "original_content": chunk.content,
+                        "original_content": chunk.original_content,
                         "edited_content": content,
-                        "source_start": chunk.start,
-                        "source_end": chunk.end,
-                        "source_start_line": chunk.start_line,
-                        "source_end_line": chunk.end_line,
+                        "source_start": chunk.source_start,
+                        "source_end": chunk.source_end,
+                        "source_start_line": chunk.source_start_line,
+                        "source_end_line": chunk.source_end_line,
                         "token_count": estimate_token_count(content),
-                        "status": "modified" if content != chunk.content else "original",
+                        "status": (
+                            "modified"
+                            if content != chunk.original_content
+                            else "original"
+                        ),
                         "quality_score": quality,
                     }
                 )
@@ -1265,13 +1182,21 @@ def _prepare_preview_items(
     ]
     if not sources:
         raise InvalidStateError("at least one source file is required")
-    preprocess_options = _preprocess_options(task.get("config") or {})
-    if (
-        task.get("process_type") == "unstructured"
-        and preprocess_options & {"clean_invalid", "clean_invalid_content"}
+    config = task.get("config") or {}
+    preprocess_options = _preprocess_options(config)
+    chunk_method = str(
+        _value(config, "chunk_method", "chunkMethod", "layout_hybrid")
+    )
+    is_unstructured = task.get("process_type") == "unstructured"
+    if is_unstructured and (
+        chunk_method == "layout_hybrid"
+        or preprocess_options & {"clean_invalid", "clean_invalid_content"}
     ):
         for index, source in enumerate(sources):
-            if str(source.get("file_format") or "").lower() != "pdf":
+            if (
+                chunk_method != "layout_hybrid"
+                and str(source.get("file_format") or "").lower() != "pdf"
+            ):
                 continue
             storage_object_id = str(source.get("storage_object_id") or "")
             actual_size = storage.file_size(
@@ -1280,10 +1205,10 @@ def _prepare_preview_items(
                 expected_source_file_id=str(source["id"]),
             )
             if actual_size is None:
-                logger.info(
-                    "skip PDF document noise detection for unavailable legacy source %s",
-                    source["id"],
-                )
+                if chunk_method == "layout_hybrid":
+                    raise InvalidStateError(
+                        "版面结构混合切分无法读取原始文件，请重新上传后再处理"
+                    )
                 continue
             expected_size = int(source.get("size_bytes") or 0)
             if expected_size and actual_size != expected_size:
@@ -1296,6 +1221,13 @@ def _prepare_preview_items(
                     expected_size=actual_size,
                 )
             )
+            enriched = dict(source)
+            if chunk_method == "layout_hybrid":
+                enriched["raw_content"] = raw
+                sources[index] = enriched
+                continue
+            if str(source.get("file_format") or "").lower() != "pdf":
+                continue
             pages = extract_pdf_page_texts(raw)
             extracted_text = "\n\n".join(page.text for page in pages if page.text)
             if extracted_text != str(source.get("content") or ""):
@@ -1304,7 +1236,6 @@ def _prepare_preview_items(
                     source["id"],
                 )
                 continue
-            enriched = dict(source)
             enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
             sources[index] = enriched
     items = _build_preview_items(task, sources)

backend/app/modules/data_process/algorithms.py

@@ -1,8 +1,7 @@
 """数据处理模块使用的无副作用算法。
 
 本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
-所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
-``source[chunk.start:chunk.end]``。
+所有偏移量均为 Python 字符串偏移量。
 """
 
 from __future__ import annotations
@@ -16,7 +15,6 @@ import re
 import unicodedata
 import xml.etree.ElementTree as ET
 import zipfile
-from bisect import bisect_left
 from collections import Counter
 from collections.abc import Iterable, Mapping, Sequence
 from copy import deepcopy
@@ -31,7 +29,6 @@ from docx.oxml.table import CT_Tbl
 from docx.oxml.text.paragraph import CT_P
 from docx.table import Table
 from docx.text.paragraph import Paragraph
-from llama_index.core.node_parser import SentenceSplitter, TokenTextSplitter
 from openpyxl import load_workbook
 from openpyxl.utils.cell import range_boundaries
 from pptx import Presentation
@@ -48,7 +45,6 @@ TextFormat = Literal[
     "xlsx",
     "pptx",
 ]
-ChunkMethod = Literal["structure", "fixed", "custom"]
 DatasetSplit = Literal["train", "validation", "test"]
 StructuredPreprocessOption = Literal[
     "clean_invalid",
@@ -153,7 +149,6 @@ _ENGLISH_NAME_CONTEXT_PATTERN = re.compile(
     r"(?P<name>[A-Za-z][A-Za-z'’-]*(?:[ \t]+[A-Za-z][A-Za-z'’-]*){0,3})"
 )
 _TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
-_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
 
 
 @dataclass(frozen=True, slots=True)
@@ -184,18 +179,6 @@ class DocumentNoiseSpan:
     kind: Literal["page_number", "repeated_margin", "table_of_contents"]
 
 
-@dataclass(frozen=True, slots=True)
-class TextChunk:
-    """带有可追溯来源位置的非结构化文本切片。"""
-
-    content: str
-    start: int
-    end: int
-    start_line: int
-    end_line: int
-    token_count: int
-
-
 @dataclass(frozen=True, slots=True)
 class QualityScore:
     """标准 instruction/input/output 记录的可解释质量分。"""
@@ -1975,41 +1958,6 @@ def estimate_token_count(text: str) -> int:
     return len(_TOKEN_PATTERN.findall(text))
 
 
-def _token_spans(text: str) -> list[tuple[int, int]]:
-    return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
-
-
-def _deterministic_tokenizer(text: str) -> list[str]:
-    """LlamaIndex splitter 使用的稳定 tokenizer，与预览 token 计数完全一致。"""
-
-    return _TOKEN_PATTERN.findall(text)
-
-
-def _sentence_chunks(text: str) -> list[str]:
-    """按项目既有中英文句界切句，避免 SentenceSplitter 触发 NLTK 下载。"""
-
-    chunks: list[str] = []
-    cursor = 0
-    for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(text):
-        end = match.end()
-        if end > cursor:
-            chunks.append(text[cursor:end])
-        cursor = end
-    if cursor < len(text):
-        chunks.append(text[cursor:])
-    return chunks or [text]
-
-
-def _line_number(newline_offsets: list[int], offset: int) -> int:
-    # 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
-    return bisect_left(newline_offsets, offset) + 1
-
-
-def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
-    starts = [span[0] for span in spans]
-    return bisect_left(starts, offset)
-
-
 def _protected_markdown_ranges(
     text: str,
     *,
@@ -2110,12 +2058,6 @@ def _protected_markdown_ranges(
     return merged
 
 
-def _range_containing(
-    ranges: Sequence[tuple[int, int]], offset: int
-) -> tuple[int, int] | None:
-    return next((item for item in ranges if item[0] < offset < item[1]), None)
-
-
 def protected_context_ranges(
     text: str,
     *,
@@ -2444,269 +2386,6 @@ def is_near_duplicate(
     )
 
 
-def _boundary_for_method(
-    text: str,
-    spans: list[tuple[int, int]],
-    start_index: int,
-    ideal_end_index: int,
-    minimum_end_index: int,
-    method: ChunkMethod,
-    custom_delimiter: str,
-    splitter: SentenceSplitter | TokenTextSplitter | None,
-) -> tuple[int, int | None]:
-    if method == "fixed" and not isinstance(splitter, TokenTextSplitter):
-        raise RuntimeError("fixed chunking requires TokenTextSplitter")
-
-    start_offset = spans[start_index][0]
-    ideal_end_offset = spans[ideal_end_index - 1][1]
-    minimum_end_offset = spans[minimum_end_index - 1][1]
-    search_text = text[start_offset:ideal_end_offset]
-
-    if method == "fixed":
-        # 直接让 TokenTextSplitter 处理真实文本；开启空白保留后将首块边界
-        # 投影回稳定 token span，offset 和实际 overlap 仍由外层统一维护。
-        lookahead_end = min(len(spans), ideal_end_index + 1)
-        window_end = spans[lookahead_end - 1][1]
-        window = text[start_offset:window_end]
-        chunks = splitter.split_text(window)
-        first_chunk = chunks[0] if chunks else ""
-        if first_chunk and window.startswith(first_chunk):
-            boundary_offset = start_offset + len(first_chunk)
-            boundary_index = _token_index_at_or_after(spans, boundary_offset)
-            if boundary_index > start_index:
-                return min(boundary_index, ideal_end_index), boundary_offset
-        return ideal_end_index, None
-
-    if method == "custom":
-        delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
-        if not delimiter:
-            raise ValueError("custom_delimiter is required for custom chunking")
-        relative_minimum = max(0, minimum_end_offset - start_offset)
-        delimiter_start = search_text.rfind(delimiter, relative_minimum)
-        if delimiter_start >= 0:
-            boundary_offset = start_offset + delimiter_start + len(delimiter)
-            boundary_index = _token_index_at_or_after(spans, boundary_offset)
-            if boundary_index > start_index:
-                return min(boundary_index, ideal_end_index), boundary_offset
-        return ideal_end_index, None
-
-    if method == "structure":
-        if not isinstance(splitter, SentenceSplitter):
-            raise RuntimeError("structure chunking requires SentenceSplitter")
-        # 多给一个 token 使 splitter 确实执行限长；只取首块并投影回原文。
-        lookahead_end = min(len(spans), ideal_end_index + 1)
-        window_end = spans[lookahead_end - 1][1]
-        window = text[start_offset:window_end]
-        chunks = splitter.split_text(window)
-        first_chunk = chunks[0] if chunks else ""
-        if first_chunk and window.startswith(first_chunk):
-            boundary_offset = start_offset + len(first_chunk)
-        else:
-            boundary_offset = ideal_end_offset
-        boundary_index = _token_index_at_or_after(spans, boundary_offset)
-        if boundary_index >= minimum_end_index:
-            return min(boundary_index, ideal_end_index), boundary_offset
-    return ideal_end_index, None
-
-
-def _chunk_normalized_text(
-    normalized: str,
-    *,
-    method: ChunkMethod,
-    chunk_size: int = 800,
-    chunk_overlap: int = 100,
-    min_chunk_size: int = 100,
-    custom_delimiter: str = "",
-    preserve_code_blocks: bool = False,
-    preserve_tables: bool = False,
-    preserve_lists: bool = False,
-) -> list[TextChunk]:
-    if not normalized:
-        return []
-    spans = _token_spans(normalized)
-    if not spans:
-        return []
-
-    newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
-    protected_ranges = _protected_markdown_ranges(
-        normalized,
-        preserve_code_blocks=preserve_code_blocks,
-        preserve_tables=preserve_tables,
-        preserve_lists=preserve_lists,
-    )
-    splitter: SentenceSplitter | TokenTextSplitter | None
-    if method == "fixed":
-        splitter = TokenTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            tokenizer=_deterministic_tokenizer,
-            separator=" ",
-            backup_separators=["\n"],
-            keep_whitespaces=True,
-            include_metadata=False,
-            include_prev_next_rel=False,
-        )
-    elif method == "structure":
-        splitter = SentenceSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            tokenizer=_deterministic_tokenizer,
-            chunking_tokenizer_fn=_sentence_chunks,
-            include_metadata=False,
-            include_prev_next_rel=False,
-        )
-    else:
-        splitter = None
-    chunks: list[TextChunk] = []
-    start_index = 0
-
-    while start_index < len(spans):
-        ideal_end_index = min(len(spans), start_index + chunk_size)
-        if ideal_end_index == len(spans):
-            end_index, end_override = ideal_end_index, len(normalized)
-        else:
-            minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
-            end_index, end_override = _boundary_for_method(
-                normalized,
-                spans,
-                start_index,
-                ideal_end_index,
-                minimum_end_index,
-                method,
-                custom_delimiter,
-                splitter,
-            )
-        if end_index <= start_index:
-            end_index = min(len(spans), start_index + chunk_size)
-            end_override = None
-
-        start_offset = spans[start_index][0]
-        end_offset = end_override if end_override is not None else spans[end_index - 1][1]
-        end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
-        split_range = _range_containing(protected_ranges, end_offset)
-        if split_range:
-            before_index = _token_index_at_or_after(spans, split_range[0])
-            if before_index - start_index >= min_chunk_size:
-                end_index = before_index
-                end_offset = split_range[0]
-            else:
-                end_index = min(
-                    len(spans),
-                    max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
-                )
-                end_offset = split_range[1]
-        content = normalized[start_offset:end_offset]
-        chunks.append(
-            TextChunk(
-                content=content,
-                start=start_offset,
-                end=end_offset,
-                start_line=_line_number(newline_offsets, start_offset),
-                end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
-                token_count=end_index - start_index,
-            )
-        )
-
-        if end_index >= len(spans):
-            break
-        next_start = max(start_index + 1, end_index - chunk_overlap)
-        overlap_range = _range_containing(protected_ranges, spans[next_start][0])
-        if overlap_range:
-            candidate = _token_index_at_or_after(spans, overlap_range[0])
-            if candidate <= start_index:
-                candidate = _token_index_at_or_after(spans, overlap_range[1])
-            next_start = min(len(spans), max(start_index + 1, candidate))
-        start_index = next_start
-
-    return chunks
-
-
-def _structure_sections(text: str) -> list[tuple[int, int]]:
-    structure = detect_document_structure(text)
-    if not structure.headings:
-        return [(0, len(text))]
-
-    sections: list[tuple[int, int]] = []
-    first_start = structure.headings[0].start
-    if first_start > 0:
-        sections.append((0, first_start))
-    sections.extend(
-        (
-            heading.start,
-            structure.headings[index + 1].start
-            if index + 1 < len(structure.headings)
-            else len(text),
-        )
-        for index, heading in enumerate(structure.headings)
-    )
-    return sections
-
-
-def chunk_unstructured(
-    text: str,
-    *,
-    method: ChunkMethod = "structure",
-    chunk_size: int = 800,
-    chunk_overlap: int = 100,
-    min_chunk_size: int = 100,
-    custom_delimiter: str = "",
-    preserve_code_blocks: bool = False,
-    preserve_tables: bool = False,
-    preserve_lists: bool = False,
-) -> list[TextChunk]:
-    """按确定性 token 切分，并保留规范化原文的 offset、行号和实际 overlap。"""
-
-    if method not in {"structure", "fixed", "custom"}:
-        raise ValueError(f"unsupported chunk method: {method}")
-    if chunk_size <= 0:
-        raise ValueError("chunk_size must be greater than 0")
-    if chunk_overlap < 0 or chunk_overlap >= chunk_size:
-        raise ValueError("chunk_overlap must be in [0, chunk_size)")
-    if min_chunk_size <= 0 or min_chunk_size > chunk_size:
-        raise ValueError("min_chunk_size must be in [1, chunk_size]")
-    if chunk_overlap + min_chunk_size > chunk_size:
-        raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
-    if method == "custom" and not custom_delimiter:
-        raise ValueError("custom_delimiter is required for custom chunking")
-
-    normalized = normalize_text(text)
-    if not normalized:
-        return []
-    section_ranges = (
-        _structure_sections(normalized)
-        if method == "structure"
-        else [(0, len(normalized))]
-    )
-    newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
-    chunks: list[TextChunk] = []
-    for section_start, section_end in section_ranges:
-        section = normalized[section_start:section_end]
-        for chunk in _chunk_normalized_text(
-            section,
-            method=method,
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-            min_chunk_size=min_chunk_size,
-            custom_delimiter=custom_delimiter,
-            preserve_code_blocks=preserve_code_blocks,
-            preserve_tables=preserve_tables,
-            preserve_lists=preserve_lists,
-        ):
-            start = section_start + chunk.start
-            end = section_start + chunk.end
-            chunks.append(
-                TextChunk(
-                    content=normalized[start:end],
-                    start=start,
-                    end=end,
-                    start_line=_line_number(newline_offsets, start),
-                    end_line=_line_number(newline_offsets, max(start, end - 1)),
-                    token_count=chunk.token_count,
-                )
-            )
-    return chunks
-
-
 def record_fingerprint(record: Mapping[str, Any]) -> str:
     """计算与字典键顺序无关的稳定记录指纹。"""
 
@@ -3029,7 +2708,6 @@ def generate_standard_records(
 
 
 __all__ = [
-    "ChunkMethod",
     "DatasetSplit",
     "DocumentHeading",
     "DocumentNoiseSpan",
@@ -3039,10 +2717,8 @@ __all__ = [
     "QualityScore",
     "SUPPORTED_TEXT_FORMATS",
     "StructuredPreprocessOption",
-    "TextChunk",
     "TextFormat",
     "canonical_record_json",
-    "chunk_unstructured",
     "content_quality_flags",
     "decode_utf8",
     "desensitize_pii",

backend/app/modules/data_process/document_chunking.py

@@ -0,0 +1,444 @@
+"""基于 Docling 与 LlamaIndex 的文档切分实现。"""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+import unicodedata
+from dataclasses import dataclass
+from functools import lru_cache
+from io import BytesIO
+from typing import Any, Literal
+
+import tiktoken
+from docling_core.transforms.chunker.hierarchical_chunker import ChunkingSerializerProvider
+from llama_index.core import Document
+from llama_index.core.base.embeddings.base import BaseEmbedding
+from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
+
+from app.modules.data_process.algorithms import normalize_text
+
+ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
+
+_PAGE_FURNITURE = re.compile(
+    r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
+)
+_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
+_CONVERTER_LOCK = threading.Lock()
+
+
+@dataclass(frozen=True, slots=True)
+class DocumentChunk:
+    """切片正文及其在原文件中的可追溯信息。"""
+
+    original_content: str
+    contextualized_content: str
+    source_start: int | None
+    source_end: int | None
+    source_start_line: int | None
+    source_end_line: int | None
+    token_count: int
+    heading_path: tuple[str, ...] = ()
+    source_pages: tuple[int, ...] = ()
+    doc_item_refs: tuple[str, ...] = ()
+    source_bboxes: tuple[dict[str, Any], ...] = ()
+
+
+def _sentence_chunks(text: str) -> list[str]:
+    """提供稳定的中英文句界，避免 LlamaIndex 默认分词器下载额外资源。"""
+
+    boundary = re.compile(
+        r".*?(?:\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)|$)",
+        re.DOTALL,
+    )
+    return [part for part in boundary.findall(text) if part]
+
+
+@lru_cache(maxsize=1)
+def _tokenizer() -> tiktoken.Encoding:
+    return tiktoken.get_encoding("cl100k_base")
+
+
+def _text_chunks(
+    text: str,
+    *,
+    chunk_size: int,
+    chunk_overlap: int,
+) -> list[DocumentChunk]:
+    normalized = normalize_text(text)
+    if not normalized:
+        return []
+    splitter = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        tokenizer=_tokenizer().encode,
+        chunking_tokenizer_fn=_sentence_chunks,
+        include_metadata=False,
+        include_prev_next_rel=False,
+    )
+    nodes = splitter.get_nodes_from_documents([Document(text=normalized)])
+    return _nodes_to_chunks(nodes, normalized)
+
+
+def chunk_fixed_text(
+    text: str,
+    *,
+    chunk_size: int,
+    chunk_overlap: int,
+) -> list[DocumentChunk]:
+    """使用 LlamaIndex SentenceSplitter 按句界控制固定 Token 长度。"""
+
+    return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+
+@lru_cache(maxsize=1)
+def _semantic_embedding_model() -> BaseEmbedding:
+    # 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
+    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+
+    return HuggingFaceEmbedding(
+        model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
+        device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
+        trust_remote_code=False,
+    )
+
+
+def chunk_semantic_text(
+    text: str,
+    *,
+    chunk_size: int,
+    chunk_overlap: int,
+    breakpoint_percentile_threshold: int,
+    embed_model: BaseEmbedding | None = None,
+) -> list[DocumentChunk]:
+    """使用 LlamaIndex SemanticSplitter 识别主题跳变，再限制最大长度。"""
+
+    normalized = normalize_text(text)
+    if not normalized:
+        return []
+    splitter = SemanticSplitterNodeParser.from_defaults(
+        embed_model=embed_model or _semantic_embedding_model(),
+        breakpoint_percentile_threshold=breakpoint_percentile_threshold,
+        buffer_size=1,
+        sentence_splitter=_sentence_chunks,
+        include_metadata=False,
+        include_prev_next_rel=False,
+    )
+    semantic_nodes = splitter.get_nodes_from_documents([Document(text=normalized)])
+    result: list[DocumentChunk] = []
+    search_from = 0
+    for node in semantic_nodes:
+        content = node.get_content().strip()
+        if not content:
+            continue
+        start = _locate_text(normalized, content, search_from)
+        if start is None:
+            start = _locate_text(normalized, content, 0)
+        if start is None:
+            continue
+        if len(_tokenizer().encode(content)) <= chunk_size:
+            result.append(_make_text_chunk(normalized, start, start + len(content)))
+        else:
+            for child in _text_chunks(
+                content,
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+            ):
+                if child.source_start is None or child.source_end is None:
+                    continue
+                result.append(
+                    _make_text_chunk(
+                        normalized,
+                        start + child.source_start,
+                        start + child.source_end,
+                    )
+                )
+        search_from = start + len(content)
+    return result
+
+
+def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
+    chunks: list[DocumentChunk] = []
+    search_from = 0
+    for node in nodes:
+        content = node.get_content().strip()
+        if not content:
+            continue
+        raw_start = getattr(node, "start_char_idx", None)
+        raw_end = getattr(node, "end_char_idx", None)
+        if (
+            isinstance(raw_start, int)
+            and isinstance(raw_end, int)
+            and source_text[raw_start:raw_end].strip() == content
+        ):
+            start = raw_start + len(source_text[raw_start:raw_end]) - len(source_text[raw_start:raw_end].lstrip())
+        else:
+            start = _locate_text(source_text, content, search_from)
+            if start is None:
+                start = _locate_text(source_text, content, 0)
+        if start is None:
+            continue
+        end = start + len(content)
+        chunks.append(_make_text_chunk(source_text, start, end))
+        search_from = max(search_from, end)
+    return chunks
+
+
+def _locate_text(source: str, content: str, start: int) -> int | None:
+    position = source.find(content, start)
+    return position if position >= 0 else None
+
+
+def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
+    content = source[start:end]
+    return DocumentChunk(
+        original_content=content,
+        contextualized_content=content,
+        source_start=start,
+        source_end=end,
+        source_start_line=source.count("\n", 0, start) + 1,
+        source_end_line=source.count("\n", 0, max(start, end - 1)) + 1,
+        token_count=len(_tokenizer().encode(content)),
+    )
+
+
+@lru_cache(maxsize=1)
+def _document_converter():
+    from docling.document_converter import DocumentConverter
+
+    return DocumentConverter()
+
+
+class _MarkdownSerializerProvider(ChunkingSerializerProvider):
+    def get_serializer(self, doc: Any):
+        from docling_core.transforms.chunker.hierarchical_chunker import ChunkingDocSerializer
+        from docling_core.transforms.serializer.markdown import (
+            MarkdownParams,
+            MarkdownTableSerializer,
+        )
+        from docling_core.types.doc import DocItemLabel
+
+        excluded = {
+            DocItemLabel.DOCUMENT_INDEX,
+            DocItemLabel.PAGE_HEADER,
+            DocItemLabel.PAGE_FOOTER,
+        }
+        return ChunkingDocSerializer(
+            doc=doc,
+            table_serializer=MarkdownTableSerializer(),
+            params=MarkdownParams(
+                labels=set(DocItemLabel) - excluded,
+                compact_tables=True,
+                image_placeholder="",
+                escape_html=False,
+                escape_underscores=False,
+            ),
+        )
+
+
+def _clean_layout_text(value: str) -> str:
+    return normalize_text(_PAGE_FURNITURE.sub("", value)).strip()
+
+
+def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
+    compact: list[str] = []
+    offsets: list[int] = []
+    for index, character in enumerate(unicodedata.normalize("NFKC", value)):
+        if _COMPACT_CHARACTER.fullmatch(character):
+            compact.append(character.casefold())
+            offsets.append(index)
+    return "".join(compact), offsets
+
+
+def _project_layout_span(
+    source_text: str,
+    content: str,
+    *,
+    compact_source: str,
+    source_offsets: list[int],
+    compact_start: int,
+) -> tuple[int | None, int | None, int]:
+    compact_content, _ = _compact_with_offsets(content)
+    if len(compact_content) < 4:
+        return None, None, compact_start
+    position = compact_source.find(compact_content, compact_start)
+    if position < 0:
+        position = compact_source.find(compact_content)
+    if position < 0:
+        return None, None, compact_start
+    start = source_offsets[position]
+    end = source_offsets[position + len(compact_content) - 1] + 1
+    while start > 0 and source_text[start - 1] not in "\r\n":
+        start -= 1
+    while end < len(source_text) and source_text[end] not in "\r\n":
+        end += 1
+    return start, end, position + len(compact_content)
+
+
+def chunk_layout_document(
+    raw: bytes,
+    *,
+    filename: str,
+    source_text: str,
+    chunk_size: int,
+) -> list[DocumentChunk]:
+    """使用 Docling HybridChunker 按版面层级、列表与表格边界切分。"""
+
+    from docling.chunking import HybridChunker
+    from docling.datamodel.base_models import DocumentStream
+    from docling.exceptions import BaseError as DoclingError
+    from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
+    from docling_core.types.doc import DocItemLabel
+
+    try:
+        with _CONVERTER_LOCK:
+            conversion = _document_converter().convert(
+                DocumentStream(name=filename, stream=BytesIO(raw))
+            )
+    except DoclingError as exc:
+        raise ValueError(f"文档版面解析失败: {exc}") from exc
+    chunker = HybridChunker(
+        tokenizer=OpenAITokenizer(tokenizer=_tokenizer(), max_tokens=chunk_size),
+        serializer_provider=_MarkdownSerializerProvider(),
+        merge_peers=True,
+        repeat_table_header=True,
+    )
+    compact_source, source_offsets = _compact_with_offsets(source_text)
+    compact_start = 0
+    result: list[DocumentChunk] = []
+    excluded = {
+        DocItemLabel.DOCUMENT_INDEX,
+        DocItemLabel.PAGE_HEADER,
+        DocItemLabel.PAGE_FOOTER,
+    }
+    for raw_chunk in chunker.chunk(conversion.document):
+        doc_items = tuple(raw_chunk.meta.doc_items or ())
+        if doc_items and all(item.label in excluded for item in doc_items):
+            continue
+        content = _clean_layout_text(raw_chunk.text)
+        if not content:
+            continue
+        contextualized = _clean_layout_text(chunker.contextualize(raw_chunk)) or content
+        start, end, compact_start = _project_layout_span(
+            source_text,
+            content,
+            compact_source=compact_source,
+            source_offsets=source_offsets,
+            compact_start=compact_start,
+        )
+        original = source_text[start:end] if start is not None and end is not None else content
+        pages: set[int] = set()
+        refs: list[str] = []
+        bboxes: list[dict[str, Any]] = []
+        for item in doc_items:
+            refs.append(str(item.self_ref))
+            for provenance in item.prov or ():
+                pages.add(int(provenance.page_no))
+                bbox = provenance.bbox
+                bboxes.append(
+                    {
+                        "page": int(provenance.page_no),
+                        "left": float(bbox.l),
+                        "top": float(bbox.t),
+                        "right": float(bbox.r),
+                        "bottom": float(bbox.b),
+                        "origin": str(bbox.coord_origin.value),
+                    }
+                )
+        result.append(
+            DocumentChunk(
+                original_content=original,
+                contextualized_content=contextualized,
+                source_start=start,
+                source_end=end,
+                source_start_line=(source_text.count("\n", 0, start) + 1 if start is not None else None),
+                source_end_line=(
+                    source_text.count("\n", 0, max(start or 0, (end or 1) - 1)) + 1
+                    if end is not None
+                    else None
+                ),
+                token_count=len(_tokenizer().encode(contextualized)),
+                heading_path=tuple(str(item) for item in (raw_chunk.meta.headings or ())),
+                source_pages=tuple(sorted(pages)),
+                doc_item_refs=tuple(refs),
+                source_bboxes=tuple(bboxes),
+            )
+        )
+    return result
+
+
+def merge_short_chunks(
+    chunks: list[DocumentChunk],
+    *,
+    source_text: str,
+    min_token_count: int,
+    max_token_count: int,
+) -> list[DocumentChunk]:
+    """在不突破长度上限的前提下，把过短块并入相邻内容。"""
+
+    result: list[DocumentChunk] = []
+    index = 0
+    while index < len(chunks):
+        current = chunks[index]
+        if current.token_count >= min_token_count:
+            result.append(current)
+            index += 1
+            continue
+        if index + 1 < len(chunks):
+            combined = _combine_chunks(current, chunks[index + 1], source_text)
+            if combined.token_count <= max_token_count:
+                result.append(combined)
+                index += 2
+                continue
+        if result:
+            combined = _combine_chunks(result[-1], current, source_text)
+            if combined.token_count <= max_token_count:
+                result[-1] = combined
+                index += 1
+                continue
+        result.append(current)
+        index += 1
+    return result
+
+
+def _combine_chunks(
+    left: DocumentChunk,
+    right: DocumentChunk,
+    source_text: str,
+) -> DocumentChunk:
+    contextualized = "\n\n".join(
+        part for part in (left.contextualized_content, right.contextualized_content) if part
+    )
+    start = left.source_start
+    end = right.source_end
+    has_contiguous_source = (
+        start is not None
+        and left.source_end is not None
+        and right.source_start is not None
+        and end is not None
+        and left.source_end <= right.source_start
+    )
+    original = (
+        source_text[start:end]
+        if has_contiguous_source and start is not None and end is not None
+        else "\n\n".join(
+            part for part in (left.original_content, right.original_content) if part
+        )
+    )
+    if not has_contiguous_source:
+        start = None
+        end = None
+    return DocumentChunk(
+        original_content=original,
+        contextualized_content=contextualized,
+        source_start=start,
+        source_end=end,
+        source_start_line=left.source_start_line if start is not None else None,
+        source_end_line=right.source_end_line if end is not None else None,
+        token_count=len(_tokenizer().encode(contextualized)),
+        heading_path=left.heading_path or right.heading_path,
+        source_pages=tuple(sorted(set(left.source_pages) | set(right.source_pages))),
+        doc_item_refs=left.doc_item_refs + right.doc_item_refs,
+        source_bboxes=left.source_bboxes + right.source_bboxes,
+    )

backend/app/schemas/data_process.py

@@ -13,23 +13,26 @@ def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, defa
 
 
 def _validate_process_config(config: dict[str, Any]) -> None:
-    chunk_method = _config_value(config, "chunk_method", "chunkMethod", "structure")
+    chunk_method = _config_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
     if not isinstance(chunk_method, str) or chunk_method not in {
-        "structure",
+        "layout_hybrid",
+        "semantic",
         "fixed",
-        "custom",
     }:
-        raise ValueError("chunk_method must be one of: structure, fixed, custom")
-    custom_delimiter = _config_value(
+        raise ValueError("chunk_method must be one of: layout_hybrid, semantic, fixed")
+
+    semantic_percentile = _config_value(
         config,
-        "custom_delimiter",
-        "customDelimiter",
-        "",
+        "semantic_breakpoint_percentile",
+        "semanticBreakpointPercentile",
+        95,
     )
-    if chunk_method == "custom" and (
-        not isinstance(custom_delimiter, str) or not custom_delimiter
+    if (
+        isinstance(semantic_percentile, bool)
+        or not isinstance(semantic_percentile, int)
+        or not 1 <= semantic_percentile <= 99
     ):
-        raise ValueError("custom_delimiter is required for custom chunking")
+        raise ValueError("semantic_breakpoint_percentile must be an integer in [1, 99]")
 
     split = _config_value(config, "dataset_split", "datasetSplit", None)
     if split is not None: