From 93373bc61f47a72e78b3d1ab097950b0d80a55d3 Mon Sep 17 00:00:00 2001
From: caoxiaozhu <leocaoxiaozhu@gmail.com>
Date: Fri, 24 Jul 2026 14:23:10 +0800
Subject: [PATCH] =?UTF-8?q?fix(data-process):=20=E5=90=88=E5=B9=B6?=
 =?UTF-8?q?=E6=96=87=E6=A1=A3=E7=BB=93=E6=9E=84=E7=9F=AD=E5=88=87=E7=89=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/app/api/v1/endpoints/data_process.py | 23 ++++++++++++--------
 backend/tests/test_data_process_api.py       | 20 ++++++++++++++++-
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py
index 96cd93f..14228f6 100644
--- a/backend/app/api/v1/endpoints/data_process.py
+++ b/backend/app/api/v1/endpoints/data_process.py
@@ -256,7 +256,7 @@ def _merge_short_chunks(
     min_token_count: int,
     chunk_size: int,
 ) -> list[TextChunk]:
-    """在同一章节内合并短块，同时严格遵守切片大小上限。"""
+    """按原文顺序合并相邻短块，同时严格遵守切片大小上限。"""
 
     merged: list[TextChunk] = []
     index = 0
@@ -308,7 +308,7 @@ def _chunk_source_text(
     config: dict[str, Any],
     preprocess_options: set[str],
 ) -> list[tuple[TextChunk, tuple[str, ...]]]:
-    """按可选文档结构分段后切片，章节之间绝不共享 overlap。"""
+    """按可选文档结构分段后切片，结构边界之间不共享 overlap。"""
 
     method = str(_value(config, "chunk_method", "chunkMethod", "structure"))
     detect_structure = (
@@ -368,14 +368,19 @@ def _chunk_source_text(
         section_text = text[start:end]
         local_chunks = chunk_unstructured(section_text, **common)
         shifted = [_shift_chunk(chunk, start, text) for chunk in local_chunks]
-        if merge_short:
-            shifted = _merge_short_chunks(
-                shifted,
-                text,
-                min_token_count=configured_minimum,
-                chunk_size=chunk_size,
-            )
         result.extend((chunk, heading_path) for chunk in shifted)
+
+    if merge_short and result:
+        # 结构分段只负责提供标题路径和隔离 overlap，不应让目录项或短小节
+        # 突破 min_chunk_size 约束。合并后保留首个原始块的标题路径。
+        heading_paths = {chunk.start: heading_path for chunk, heading_path in result}
+        merged = _merge_short_chunks(
+            [chunk for chunk, _ in result],
+            text,
+            min_token_count=configured_minimum,
+            chunk_size=chunk_size,
+        )
+        result = [(chunk, heading_paths.get(chunk.start, ())) for chunk in merged]
     return result
 
 
diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py
index 54e390a..c28aa1f 100644
--- a/backend/tests/test_data_process_api.py
+++ b/backend/tests/test_data_process_api.py
@@ -1402,7 +1402,6 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
     )
     assert without_merge[0]["token_count"] < 5
     assert with_merge[0]["token_count"] >= 5
-
     mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。"
     assert len(_preview_task(mojibake, options=[])) == 1
     assert _preview_task(mojibake, options=["filter_low_quality"]) == []
@@ -1451,6 +1450,25 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
     assert "[EMAIL]" in masked["edited_content"]
 
 
+def test_merge_short_content_applies_across_adjacent_structure_sections() -> None:
+    content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
+    items = _preview_task(
+        content,
+        options=["merge_short_content"],
+        config={
+            "chunk_method": "structure",
+            "chunk_size": 40,
+            "chunk_overlap": 0,
+            "min_chunk_size": 20,
+        },
+    )
+
+    assert len(items) == 2
+    assert all(20 <= item["token_count"] <= 40 for item in items)
+    assert items[0]["source_start_line"] == 1
+    assert items[0]["source_end_line"] == 8
+
+
 def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None:
     for file_format in ("pdf", "docx", "pptx"):
         items = _preview_task(