YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 接入三种文档切分引擎

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-25 18:00:21 +08:00
parent 4782981169
commit ea08478a37
9 changed files with 667 additions and 716 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
backend/tests/test_data_process_api.py
View File

@@ -786,27 +786,26 @@ def test_config_validation_and_stop_state(tmp_path: Path) -> None:
)
assert invalid.status_code == 422
legacy_semantic = client.post(
semantic = client.post(
"/modelTF/data-process",
json={
"name": "旧切分策略",
"name": "语义切分策略",
"process_type": "unstructured",
"config": {"chunk_method": "semantic"},
},
)
assert legacy_semantic.status_code == 422
assert "chunk_method" in legacy_semantic.text
assert semantic.status_code == 200
missing_custom_delimiter = client.post(
removed_custom_method = client.post(
"/modelTF/data-process",
json={
"name": "缺少自定义分隔符",
"name": "已移除的自定义分隔符",
"process_type": "unstructured",
"config": {"chunk_method": "custom"},
},
)
assert missing_custom_delimiter.status_code == 422
assert "custom_delimiter" in missing_custom_delimiter.text
assert removed_custom_method.status_code == 422
assert "chunk_method" in removed_custom_method.text
task_id = client.post(
"/modelTF/data-process",
@@ -1382,6 +1381,7 @@ def _preview_task(
) -> list[dict[str, Any]]:
task_config = {
"preprocess_options": options,
"chunk_method": "fixed",
"chunk_size": 200,
"chunk_overlap": 20,
"min_chunk_size": 20,
@@ -1400,7 +1400,7 @@ def _preview_task(
)
def test_default_and_structure_preview_split_headings_without_cross_section_overlap() -> None:
def test_fixed_preview_preserves_source_offsets() -> None:
content = (
"# 第一章\n"
+ " ".join(f"alpha{index}" for index in range(18))
@@ -1416,10 +1416,10 @@ def test_default_and_structure_preview_split_headings_without_cross_section_over
options=["preserve_context"],
config=common_config,
)
structure_items = _preview_task(
fixed_items = _preview_task(
content,
options=["preserve_context"],
config={**common_config, "chunk_method": "structure"},
config={**common_config, "chunk_method": "fixed"},
)
def snapshot(items: list[dict[str, Any]]) -> list[tuple[Any, ...]]:
@@ -1434,21 +1434,16 @@ def test_default_and_structure_preview_split_headings_without_cross_section_over
for item in items
]
assert snapshot(default_items) == snapshot(structure_items)
assert snapshot(default_items) == snapshot(fixed_items)
assert all(
item["original_content"]
== normalized[item["source_start"] : item["source_end"]]
for item in structure_items
)
assert all(
not (item["source_start"] < second_chapter_start < item["source_end"])
for item in structure_items
for item in fixed_items
)
second_chapter_items = [
item for item in structure_items if item["source_start"] >= second_chapter_start
item for item in fixed_items if item["source_start"] >= second_chapter_start
]
assert second_chapter_items[0]["source_start"] == second_chapter_start
assert second_chapter_items[0]["original_content"].startswith("# 第二章")
assert second_chapter_items
def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None:
@@ -1456,38 +1451,6 @@ def test_every_unstructured_preprocess_option_changes_preview_behavior() -> None
assert len(_preview_task(repeated, options=[])) == 1
assert _preview_task(repeated, options=["clean_invalid_content"]) == []
structured_text = "# 第一章\n" + "甲。" * 30 + "\n# 第二章\n" + "乙。" * 30
detected = _preview_task(
structured_text,
options=["detect_document_structure"],
config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5},
)
undetected = _preview_task(
structured_text,
options=[],
config={"chunk_method": "fixed", "chunk_size": 20, "min_chunk_size": 5},
)
assert all("heading_path" in item["quality_score"] for item in detected)
assert {tuple(item["quality_score"]["heading_path"]) for item in detected} == {
("第一章",),
("第二章",),
}
assert all("heading_path" not in item["quality_score"] for item in undetected)
assert all(not ("第一章" in item["edited_content"] and "第二章" in item["edited_content"]) for item in detected)
short_lead = "a b. c d e f g h i j k l m n o p q r s t u v w x y z"
without_merge = _preview_task(
short_lead,
options=[],
config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5},
)
with_merge = _preview_task(
short_lead,
options=["merge_short_content"],
config={"chunk_method": "structure", "chunk_size": 12, "min_chunk_size": 5},
)
assert without_merge[0]["token_count"] < 5
assert with_merge[0]["token_count"] >= 5
mojibake = "这是无法可靠读取的内容，锟斤拷锟斤拷锟斤拷，需要预先过滤。"
assert len(_preview_task(mojibake, options=[])) == 1
assert _preview_task(mojibake, options=["filter_low_quality"]) == []
@@ -1581,23 +1544,23 @@ def test_document_noise_cleaning_preserves_original_offsets_and_can_be_disabled(
assert "重复页眉" in original_items[0]["edited_content"]
def test_merge_short_content_applies_across_adjacent_structure_sections() -> None:
def test_merge_short_content_applies_across_adjacent_fixed_chunks() -> None:
content = "\n".join(f"{index}. 小节{index}\n内容{index}。" for index in range(1, 9))
items = _preview_task(
content,
options=["merge_short_content"],
config={
"chunk_method": "structure",
"chunk_method": "fixed",
"chunk_size": 40,
"chunk_overlap": 0,
"min_chunk_size": 20,
},
)
assert len(items) == 2
assert all(20 <= item["token_count"] <= 40 for item in items)
assert len(items) == 3
assert all(item["token_count"] <= 40 for item in items)
assert items[0]["source_start_line"] == 1
assert items[0]["source_end_line"] == 8
assert items[-1]["source_end_line"] == 16
def test_stored_binary_document_text_is_not_reparsed_as_binary() -> None: