feat(data-process): 接入三种文档切分引擎
This commit is contained in:
caoxiaozhu
@@ -1,8 +1,7 @@
|
||||
"""数据处理模块使用的无副作用算法。
|
||||
|
||||
本模块不访问数据库、文件系统或网络，便于 API、后台任务和测试共同复用。
|
||||
所有偏移量均为 Python 字符串偏移量，``TextChunk.content`` 始终等于
|
||||
``source[chunk.start:chunk.end]``。
|
||||
所有偏移量均为 Python 字符串偏移量。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,7 +15,6 @@ import re
|
||||
import unicodedata
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from bisect import bisect_left
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
@@ -31,7 +29,6 @@ from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
from llama_index.core.node_parser import SentenceSplitter, TokenTextSplitter
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils.cell import range_boundaries
|
||||
from pptx import Presentation
|
||||
@@ -48,7 +45,6 @@ TextFormat = Literal[
|
||||
"xlsx",
|
||||
"pptx",
|
||||
]
|
||||
ChunkMethod = Literal["structure", "fixed", "custom"]
|
||||
DatasetSplit = Literal["train", "validation", "test"]
|
||||
StructuredPreprocessOption = Literal[
|
||||
"clean_invalid",
|
||||
@@ -153,7 +149,6 @@ _ENGLISH_NAME_CONTEXT_PATTERN = re.compile(
|
||||
r"(?P<name>[A-Za-z][A-Za-z'’-]*(?:[ \t]+[A-Za-z][A-Za-z'’-]*){0,3})"
|
||||
)
|
||||
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
||||
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -184,18 +179,6 @@ class DocumentNoiseSpan:
|
||||
kind: Literal["page_number", "repeated_margin", "table_of_contents"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
|
||||
content: str
|
||||
start: int
|
||||
end: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
token_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QualityScore:
|
||||
"""标准 instruction/input/output 记录的可解释质量分。"""
|
||||
@@ -1975,41 +1958,6 @@ def estimate_token_count(text: str) -> int:
|
||||
return len(_TOKEN_PATTERN.findall(text))
|
||||
|
||||
|
||||
def _token_spans(text: str) -> list[tuple[int, int]]:
|
||||
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
||||
|
||||
|
||||
def _deterministic_tokenizer(text: str) -> list[str]:
|
||||
"""LlamaIndex splitter 使用的稳定 tokenizer，与预览 token 计数完全一致。"""
|
||||
|
||||
return _TOKEN_PATTERN.findall(text)
|
||||
|
||||
|
||||
def _sentence_chunks(text: str) -> list[str]:
|
||||
"""按项目既有中英文句界切句，避免 SentenceSplitter 触发 NLTK 下载。"""
|
||||
|
||||
chunks: list[str] = []
|
||||
cursor = 0
|
||||
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(text):
|
||||
end = match.end()
|
||||
if end > cursor:
|
||||
chunks.append(text[cursor:end])
|
||||
cursor = end
|
||||
if cursor < len(text):
|
||||
chunks.append(text[cursor:])
|
||||
return chunks or [text]
|
||||
|
||||
|
||||
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
||||
# 换行符本身仍属于上一行；只有严格位于 offset 之前的换行才推进行号。
|
||||
return bisect_left(newline_offsets, offset) + 1
|
||||
|
||||
|
||||
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
||||
starts = [span[0] for span in spans]
|
||||
return bisect_left(starts, offset)
|
||||
|
||||
|
||||
def _protected_markdown_ranges(
|
||||
text: str,
|
||||
*,
|
||||
@@ -2110,12 +2058,6 @@ def _protected_markdown_ranges(
|
||||
return merged
|
||||
|
||||
|
||||
def _range_containing(
|
||||
ranges: Sequence[tuple[int, int]], offset: int
|
||||
) -> tuple[int, int] | None:
|
||||
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
||||
|
||||
|
||||
def protected_context_ranges(
|
||||
text: str,
|
||||
*,
|
||||
@@ -2444,269 +2386,6 @@ def is_near_duplicate(
|
||||
)
|
||||
|
||||
|
||||
def _boundary_for_method(
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
start_index: int,
|
||||
ideal_end_index: int,
|
||||
minimum_end_index: int,
|
||||
method: ChunkMethod,
|
||||
custom_delimiter: str,
|
||||
splitter: SentenceSplitter | TokenTextSplitter | None,
|
||||
) -> tuple[int, int | None]:
|
||||
if method == "fixed" and not isinstance(splitter, TokenTextSplitter):
|
||||
raise RuntimeError("fixed chunking requires TokenTextSplitter")
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
ideal_end_offset = spans[ideal_end_index - 1][1]
|
||||
minimum_end_offset = spans[minimum_end_index - 1][1]
|
||||
search_text = text[start_offset:ideal_end_offset]
|
||||
|
||||
if method == "fixed":
|
||||
# 直接让 TokenTextSplitter 处理真实文本；开启空白保留后将首块边界
|
||||
# 投影回稳定 token span，offset 和实际 overlap 仍由外层统一维护。
|
||||
lookahead_end = min(len(spans), ideal_end_index + 1)
|
||||
window_end = spans[lookahead_end - 1][1]
|
||||
window = text[start_offset:window_end]
|
||||
chunks = splitter.split_text(window)
|
||||
first_chunk = chunks[0] if chunks else ""
|
||||
if first_chunk and window.startswith(first_chunk):
|
||||
boundary_offset = start_offset + len(first_chunk)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "custom":
|
||||
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
||||
if not delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
relative_minimum = max(0, minimum_end_offset - start_offset)
|
||||
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
||||
if delimiter_start >= 0:
|
||||
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "structure":
|
||||
if not isinstance(splitter, SentenceSplitter):
|
||||
raise RuntimeError("structure chunking requires SentenceSplitter")
|
||||
# 多给一个 token 使 splitter 确实执行限长；只取首块并投影回原文。
|
||||
lookahead_end = min(len(spans), ideal_end_index + 1)
|
||||
window_end = spans[lookahead_end - 1][1]
|
||||
window = text[start_offset:window_end]
|
||||
chunks = splitter.split_text(window)
|
||||
first_chunk = chunks[0] if chunks else ""
|
||||
if first_chunk and window.startswith(first_chunk):
|
||||
boundary_offset = start_offset + len(first_chunk)
|
||||
else:
|
||||
boundary_offset = ideal_end_offset
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index >= minimum_end_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
|
||||
def _chunk_normalized_text(
|
||||
normalized: str,
|
||||
*,
|
||||
method: ChunkMethod,
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
if not normalized:
|
||||
return []
|
||||
spans = _token_spans(normalized)
|
||||
if not spans:
|
||||
return []
|
||||
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
protected_ranges = _protected_markdown_ranges(
|
||||
normalized,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
)
|
||||
splitter: SentenceSplitter | TokenTextSplitter | None
|
||||
if method == "fixed":
|
||||
splitter = TokenTextSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
tokenizer=_deterministic_tokenizer,
|
||||
separator=" ",
|
||||
backup_separators=["\n"],
|
||||
keep_whitespaces=True,
|
||||
include_metadata=False,
|
||||
include_prev_next_rel=False,
|
||||
)
|
||||
elif method == "structure":
|
||||
splitter = SentenceSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
tokenizer=_deterministic_tokenizer,
|
||||
chunking_tokenizer_fn=_sentence_chunks,
|
||||
include_metadata=False,
|
||||
include_prev_next_rel=False,
|
||||
)
|
||||
else:
|
||||
splitter = None
|
||||
chunks: list[TextChunk] = []
|
||||
start_index = 0
|
||||
|
||||
while start_index < len(spans):
|
||||
ideal_end_index = min(len(spans), start_index + chunk_size)
|
||||
if ideal_end_index == len(spans):
|
||||
end_index, end_override = ideal_end_index, len(normalized)
|
||||
else:
|
||||
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
||||
end_index, end_override = _boundary_for_method(
|
||||
normalized,
|
||||
spans,
|
||||
start_index,
|
||||
ideal_end_index,
|
||||
minimum_end_index,
|
||||
method,
|
||||
custom_delimiter,
|
||||
splitter,
|
||||
)
|
||||
if end_index <= start_index:
|
||||
end_index = min(len(spans), start_index + chunk_size)
|
||||
end_override = None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
||||
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
||||
split_range = _range_containing(protected_ranges, end_offset)
|
||||
if split_range:
|
||||
before_index = _token_index_at_or_after(spans, split_range[0])
|
||||
if before_index - start_index >= min_chunk_size:
|
||||
end_index = before_index
|
||||
end_offset = split_range[0]
|
||||
else:
|
||||
end_index = min(
|
||||
len(spans),
|
||||
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
||||
)
|
||||
end_offset = split_range[1]
|
||||
content = normalized[start_offset:end_offset]
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=content,
|
||||
start=start_offset,
|
||||
end=end_offset,
|
||||
start_line=_line_number(newline_offsets, start_offset),
|
||||
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
||||
token_count=end_index - start_index,
|
||||
)
|
||||
)
|
||||
|
||||
if end_index >= len(spans):
|
||||
break
|
||||
next_start = max(start_index + 1, end_index - chunk_overlap)
|
||||
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
||||
if overlap_range:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
||||
if candidate <= start_index:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
||||
next_start = min(len(spans), max(start_index + 1, candidate))
|
||||
start_index = next_start
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def _structure_sections(text: str) -> list[tuple[int, int]]:
|
||||
structure = detect_document_structure(text)
|
||||
if not structure.headings:
|
||||
return [(0, len(text))]
|
||||
|
||||
sections: list[tuple[int, int]] = []
|
||||
first_start = structure.headings[0].start
|
||||
if first_start > 0:
|
||||
sections.append((0, first_start))
|
||||
sections.extend(
|
||||
(
|
||||
heading.start,
|
||||
structure.headings[index + 1].start
|
||||
if index + 1 < len(structure.headings)
|
||||
else len(text),
|
||||
)
|
||||
for index, heading in enumerate(structure.headings)
|
||||
)
|
||||
return sections
|
||||
|
||||
|
||||
def chunk_unstructured(
|
||||
text: str,
|
||||
*,
|
||||
method: ChunkMethod = "structure",
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
"""按确定性 token 切分，并保留规范化原文的 offset、行号和实际 overlap。"""
|
||||
|
||||
if method not in {"structure", "fixed", "custom"}:
|
||||
raise ValueError(f"unsupported chunk method: {method}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
||||
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
||||
if chunk_overlap + min_chunk_size > chunk_size:
|
||||
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
||||
if method == "custom" and not custom_delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
section_ranges = (
|
||||
_structure_sections(normalized)
|
||||
if method == "structure"
|
||||
else [(0, len(normalized))]
|
||||
)
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
chunks: list[TextChunk] = []
|
||||
for section_start, section_end in section_ranges:
|
||||
section = normalized[section_start:section_end]
|
||||
for chunk in _chunk_normalized_text(
|
||||
section,
|
||||
method=method,
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
min_chunk_size=min_chunk_size,
|
||||
custom_delimiter=custom_delimiter,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
):
|
||||
start = section_start + chunk.start
|
||||
end = section_start + chunk.end
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=normalized[start:end],
|
||||
start=start,
|
||||
end=end,
|
||||
start_line=_line_number(newline_offsets, start),
|
||||
end_line=_line_number(newline_offsets, max(start, end - 1)),
|
||||
token_count=chunk.token_count,
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||||
|
||||
@@ -3029,7 +2708,6 @@ def generate_standard_records(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"DocumentHeading",
|
||||
"DocumentNoiseSpan",
|
||||
@@ -3039,10 +2717,8 @@ __all__ = [
|
||||
"QualityScore",
|
||||
"SUPPORTED_TEXT_FORMATS",
|
||||
"StructuredPreprocessOption",
|
||||
"TextChunk",
|
||||
"TextFormat",
|
||||
"canonical_record_json",
|
||||
"chunk_unstructured",
|
||||
"content_quality_flags",
|
||||
"decode_utf8",
|
||||
"desensitize_pii",
|
||||
|
||||
444
backend/app/modules/data_process/document_chunking.py
Normal file
444
backend/app/modules/data_process/document_chunking.py
Normal file
@@ -0,0 +1,444 @@
|
||||
"""基于 Docling 与 LlamaIndex 的文档切分实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from io import BytesIO
|
||||
from typing import Any, Literal
|
||||
|
||||
import tiktoken
|
||||
from docling_core.transforms.chunker.hierarchical_chunker import ChunkingSerializerProvider
|
||||
from llama_index.core import Document
|
||||
from llama_index.core.base.embeddings.base import BaseEmbedding
|
||||
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
|
||||
|
||||
from app.modules.data_process.algorithms import normalize_text
|
||||
|
||||
ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
|
||||
|
||||
_PAGE_FURNITURE = re.compile(
|
||||
r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
|
||||
)
|
||||
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
||||
_CONVERTER_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentChunk:
|
||||
"""切片正文及其在原文件中的可追溯信息。"""
|
||||
|
||||
original_content: str
|
||||
contextualized_content: str
|
||||
source_start: int | None
|
||||
source_end: int | None
|
||||
source_start_line: int | None
|
||||
source_end_line: int | None
|
||||
token_count: int
|
||||
heading_path: tuple[str, ...] = ()
|
||||
source_pages: tuple[int, ...] = ()
|
||||
doc_item_refs: tuple[str, ...] = ()
|
||||
source_bboxes: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
|
||||
def _sentence_chunks(text: str) -> list[str]:
|
||||
"""提供稳定的中英文句界，避免 LlamaIndex 默认分词器下载额外资源。"""
|
||||
|
||||
boundary = re.compile(
|
||||
r".*?(?:\n\s*\n|[。！？!?；;](?:[\"'”’）】》]*)|\.(?:\s+|$)|$)",
|
||||
re.DOTALL,
|
||||
)
|
||||
return [part for part in boundary.findall(text) if part]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _tokenizer() -> tiktoken.Encoding:
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
|
||||
def _text_chunks(
|
||||
text: str,
|
||||
*,
|
||||
chunk_size: int,
|
||||
chunk_overlap: int,
|
||||
) -> list[DocumentChunk]:
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
splitter = SentenceSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
tokenizer=_tokenizer().encode,
|
||||
chunking_tokenizer_fn=_sentence_chunks,
|
||||
include_metadata=False,
|
||||
include_prev_next_rel=False,
|
||||
)
|
||||
nodes = splitter.get_nodes_from_documents([Document(text=normalized)])
|
||||
return _nodes_to_chunks(nodes, normalized)
|
||||
|
||||
|
||||
def chunk_fixed_text(
|
||||
text: str,
|
||||
*,
|
||||
chunk_size: int,
|
||||
chunk_overlap: int,
|
||||
) -> list[DocumentChunk]:
|
||||
"""使用 LlamaIndex SentenceSplitter 按句界控制固定 Token 长度。"""
|
||||
|
||||
return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _semantic_embedding_model() -> BaseEmbedding:
|
||||
# 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
|
||||
return HuggingFaceEmbedding(
|
||||
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
|
||||
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
|
||||
trust_remote_code=False,
|
||||
)
|
||||
|
||||
|
||||
def chunk_semantic_text(
|
||||
text: str,
|
||||
*,
|
||||
chunk_size: int,
|
||||
chunk_overlap: int,
|
||||
breakpoint_percentile_threshold: int,
|
||||
embed_model: BaseEmbedding | None = None,
|
||||
) -> list[DocumentChunk]:
|
||||
"""使用 LlamaIndex SemanticSplitter 识别主题跳变，再限制最大长度。"""
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
splitter = SemanticSplitterNodeParser.from_defaults(
|
||||
embed_model=embed_model or _semantic_embedding_model(),
|
||||
breakpoint_percentile_threshold=breakpoint_percentile_threshold,
|
||||
buffer_size=1,
|
||||
sentence_splitter=_sentence_chunks,
|
||||
include_metadata=False,
|
||||
include_prev_next_rel=False,
|
||||
)
|
||||
semantic_nodes = splitter.get_nodes_from_documents([Document(text=normalized)])
|
||||
result: list[DocumentChunk] = []
|
||||
search_from = 0
|
||||
for node in semantic_nodes:
|
||||
content = node.get_content().strip()
|
||||
if not content:
|
||||
continue
|
||||
start = _locate_text(normalized, content, search_from)
|
||||
if start is None:
|
||||
start = _locate_text(normalized, content, 0)
|
||||
if start is None:
|
||||
continue
|
||||
if len(_tokenizer().encode(content)) <= chunk_size:
|
||||
result.append(_make_text_chunk(normalized, start, start + len(content)))
|
||||
else:
|
||||
for child in _text_chunks(
|
||||
content,
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
):
|
||||
if child.source_start is None or child.source_end is None:
|
||||
continue
|
||||
result.append(
|
||||
_make_text_chunk(
|
||||
normalized,
|
||||
start + child.source_start,
|
||||
start + child.source_end,
|
||||
)
|
||||
)
|
||||
search_from = start + len(content)
|
||||
return result
|
||||
|
||||
|
||||
def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
|
||||
chunks: list[DocumentChunk] = []
|
||||
search_from = 0
|
||||
for node in nodes:
|
||||
content = node.get_content().strip()
|
||||
if not content:
|
||||
continue
|
||||
raw_start = getattr(node, "start_char_idx", None)
|
||||
raw_end = getattr(node, "end_char_idx", None)
|
||||
if (
|
||||
isinstance(raw_start, int)
|
||||
and isinstance(raw_end, int)
|
||||
and source_text[raw_start:raw_end].strip() == content
|
||||
):
|
||||
start = raw_start + len(source_text[raw_start:raw_end]) - len(source_text[raw_start:raw_end].lstrip())
|
||||
else:
|
||||
start = _locate_text(source_text, content, search_from)
|
||||
if start is None:
|
||||
start = _locate_text(source_text, content, 0)
|
||||
if start is None:
|
||||
continue
|
||||
end = start + len(content)
|
||||
chunks.append(_make_text_chunk(source_text, start, end))
|
||||
search_from = max(search_from, end)
|
||||
return chunks
|
||||
|
||||
|
||||
def _locate_text(source: str, content: str, start: int) -> int | None:
|
||||
position = source.find(content, start)
|
||||
return position if position >= 0 else None
|
||||
|
||||
|
||||
def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
|
||||
content = source[start:end]
|
||||
return DocumentChunk(
|
||||
original_content=content,
|
||||
contextualized_content=content,
|
||||
source_start=start,
|
||||
source_end=end,
|
||||
source_start_line=source.count("\n", 0, start) + 1,
|
||||
source_end_line=source.count("\n", 0, max(start, end - 1)) + 1,
|
||||
token_count=len(_tokenizer().encode(content)),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _document_converter():
|
||||
from docling.document_converter import DocumentConverter
|
||||
|
||||
return DocumentConverter()
|
||||
|
||||
|
||||
class _MarkdownSerializerProvider(ChunkingSerializerProvider):
|
||||
def get_serializer(self, doc: Any):
|
||||
from docling_core.transforms.chunker.hierarchical_chunker import ChunkingDocSerializer
|
||||
from docling_core.transforms.serializer.markdown import (
|
||||
MarkdownParams,
|
||||
MarkdownTableSerializer,
|
||||
)
|
||||
from docling_core.types.doc import DocItemLabel
|
||||
|
||||
excluded = {
|
||||
DocItemLabel.DOCUMENT_INDEX,
|
||||
DocItemLabel.PAGE_HEADER,
|
||||
DocItemLabel.PAGE_FOOTER,
|
||||
}
|
||||
return ChunkingDocSerializer(
|
||||
doc=doc,
|
||||
table_serializer=MarkdownTableSerializer(),
|
||||
params=MarkdownParams(
|
||||
labels=set(DocItemLabel) - excluded,
|
||||
compact_tables=True,
|
||||
image_placeholder="",
|
||||
escape_html=False,
|
||||
escape_underscores=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _clean_layout_text(value: str) -> str:
|
||||
return normalize_text(_PAGE_FURNITURE.sub("", value)).strip()
|
||||
|
||||
|
||||
def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
|
||||
compact: list[str] = []
|
||||
offsets: list[int] = []
|
||||
for index, character in enumerate(unicodedata.normalize("NFKC", value)):
|
||||
if _COMPACT_CHARACTER.fullmatch(character):
|
||||
compact.append(character.casefold())
|
||||
offsets.append(index)
|
||||
return "".join(compact), offsets
|
||||
|
||||
|
||||
def _project_layout_span(
|
||||
source_text: str,
|
||||
content: str,
|
||||
*,
|
||||
compact_source: str,
|
||||
source_offsets: list[int],
|
||||
compact_start: int,
|
||||
) -> tuple[int | None, int | None, int]:
|
||||
compact_content, _ = _compact_with_offsets(content)
|
||||
if len(compact_content) < 4:
|
||||
return None, None, compact_start
|
||||
position = compact_source.find(compact_content, compact_start)
|
||||
if position < 0:
|
||||
position = compact_source.find(compact_content)
|
||||
if position < 0:
|
||||
return None, None, compact_start
|
||||
start = source_offsets[position]
|
||||
end = source_offsets[position + len(compact_content) - 1] + 1
|
||||
while start > 0 and source_text[start - 1] not in "\r\n":
|
||||
start -= 1
|
||||
while end < len(source_text) and source_text[end] not in "\r\n":
|
||||
end += 1
|
||||
return start, end, position + len(compact_content)
|
||||
|
||||
|
||||
def chunk_layout_document(
|
||||
raw: bytes,
|
||||
*,
|
||||
filename: str,
|
||||
source_text: str,
|
||||
chunk_size: int,
|
||||
) -> list[DocumentChunk]:
|
||||
"""使用 Docling HybridChunker 按版面层级、列表与表格边界切分。"""
|
||||
|
||||
from docling.chunking import HybridChunker
|
||||
from docling.datamodel.base_models import DocumentStream
|
||||
from docling.exceptions import BaseError as DoclingError
|
||||
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
|
||||
from docling_core.types.doc import DocItemLabel
|
||||
|
||||
try:
|
||||
with _CONVERTER_LOCK:
|
||||
conversion = _document_converter().convert(
|
||||
DocumentStream(name=filename, stream=BytesIO(raw))
|
||||
)
|
||||
except DoclingError as exc:
|
||||
raise ValueError(f"文档版面解析失败: {exc}") from exc
|
||||
chunker = HybridChunker(
|
||||
tokenizer=OpenAITokenizer(tokenizer=_tokenizer(), max_tokens=chunk_size),
|
||||
serializer_provider=_MarkdownSerializerProvider(),
|
||||
merge_peers=True,
|
||||
repeat_table_header=True,
|
||||
)
|
||||
compact_source, source_offsets = _compact_with_offsets(source_text)
|
||||
compact_start = 0
|
||||
result: list[DocumentChunk] = []
|
||||
excluded = {
|
||||
DocItemLabel.DOCUMENT_INDEX,
|
||||
DocItemLabel.PAGE_HEADER,
|
||||
DocItemLabel.PAGE_FOOTER,
|
||||
}
|
||||
for raw_chunk in chunker.chunk(conversion.document):
|
||||
doc_items = tuple(raw_chunk.meta.doc_items or ())
|
||||
if doc_items and all(item.label in excluded for item in doc_items):
|
||||
continue
|
||||
content = _clean_layout_text(raw_chunk.text)
|
||||
if not content:
|
||||
continue
|
||||
contextualized = _clean_layout_text(chunker.contextualize(raw_chunk)) or content
|
||||
start, end, compact_start = _project_layout_span(
|
||||
source_text,
|
||||
content,
|
||||
compact_source=compact_source,
|
||||
source_offsets=source_offsets,
|
||||
compact_start=compact_start,
|
||||
)
|
||||
original = source_text[start:end] if start is not None and end is not None else content
|
||||
pages: set[int] = set()
|
||||
refs: list[str] = []
|
||||
bboxes: list[dict[str, Any]] = []
|
||||
for item in doc_items:
|
||||
refs.append(str(item.self_ref))
|
||||
for provenance in item.prov or ():
|
||||
pages.add(int(provenance.page_no))
|
||||
bbox = provenance.bbox
|
||||
bboxes.append(
|
||||
{
|
||||
"page": int(provenance.page_no),
|
||||
"left": float(bbox.l),
|
||||
"top": float(bbox.t),
|
||||
"right": float(bbox.r),
|
||||
"bottom": float(bbox.b),
|
||||
"origin": str(bbox.coord_origin.value),
|
||||
}
|
||||
)
|
||||
result.append(
|
||||
DocumentChunk(
|
||||
original_content=original,
|
||||
contextualized_content=contextualized,
|
||||
source_start=start,
|
||||
source_end=end,
|
||||
source_start_line=(source_text.count("\n", 0, start) + 1 if start is not None else None),
|
||||
source_end_line=(
|
||||
source_text.count("\n", 0, max(start or 0, (end or 1) - 1)) + 1
|
||||
if end is not None
|
||||
else None
|
||||
),
|
||||
token_count=len(_tokenizer().encode(contextualized)),
|
||||
heading_path=tuple(str(item) for item in (raw_chunk.meta.headings or ())),
|
||||
source_pages=tuple(sorted(pages)),
|
||||
doc_item_refs=tuple(refs),
|
||||
source_bboxes=tuple(bboxes),
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def merge_short_chunks(
|
||||
chunks: list[DocumentChunk],
|
||||
*,
|
||||
source_text: str,
|
||||
min_token_count: int,
|
||||
max_token_count: int,
|
||||
) -> list[DocumentChunk]:
|
||||
"""在不突破长度上限的前提下，把过短块并入相邻内容。"""
|
||||
|
||||
result: list[DocumentChunk] = []
|
||||
index = 0
|
||||
while index < len(chunks):
|
||||
current = chunks[index]
|
||||
if current.token_count >= min_token_count:
|
||||
result.append(current)
|
||||
index += 1
|
||||
continue
|
||||
if index + 1 < len(chunks):
|
||||
combined = _combine_chunks(current, chunks[index + 1], source_text)
|
||||
if combined.token_count <= max_token_count:
|
||||
result.append(combined)
|
||||
index += 2
|
||||
continue
|
||||
if result:
|
||||
combined = _combine_chunks(result[-1], current, source_text)
|
||||
if combined.token_count <= max_token_count:
|
||||
result[-1] = combined
|
||||
index += 1
|
||||
continue
|
||||
result.append(current)
|
||||
index += 1
|
||||
return result
|
||||
|
||||
|
||||
def _combine_chunks(
|
||||
left: DocumentChunk,
|
||||
right: DocumentChunk,
|
||||
source_text: str,
|
||||
) -> DocumentChunk:
|
||||
contextualized = "\n\n".join(
|
||||
part for part in (left.contextualized_content, right.contextualized_content) if part
|
||||
)
|
||||
start = left.source_start
|
||||
end = right.source_end
|
||||
has_contiguous_source = (
|
||||
start is not None
|
||||
and left.source_end is not None
|
||||
and right.source_start is not None
|
||||
and end is not None
|
||||
and left.source_end <= right.source_start
|
||||
)
|
||||
original = (
|
||||
source_text[start:end]
|
||||
if has_contiguous_source and start is not None and end is not None
|
||||
else "\n\n".join(
|
||||
part for part in (left.original_content, right.original_content) if part
|
||||
)
|
||||
)
|
||||
if not has_contiguous_source:
|
||||
start = None
|
||||
end = None
|
||||
return DocumentChunk(
|
||||
original_content=original,
|
||||
contextualized_content=contextualized,
|
||||
source_start=start,
|
||||
source_end=end,
|
||||
source_start_line=left.source_start_line if start is not None else None,
|
||||
source_end_line=right.source_end_line if end is not None else None,
|
||||
token_count=len(_tokenizer().encode(contextualized)),
|
||||
heading_path=left.heading_path or right.heading_path,
|
||||
source_pages=tuple(sorted(set(left.source_pages) | set(right.source_pages))),
|
||||
doc_item_refs=left.doc_item_refs + right.doc_item_refs,
|
||||
source_bboxes=left.source_bboxes + right.source_bboxes,
|
||||
)
|
||||
Reference in New Issue
Block a user