feat: 完成数据处理接口与前端接入

backend/tests/test_data_process_algorithms.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from app.modules.data_process.algorithms import (
+    chunk_unstructured,
+    desensitize_pii,
+    detect_text_format,
+    extract_structured_records,
+    generate_standard_records,
+    normalize_text,
+    parse_text_content,
+    record_fingerprint,
+    score_quality,
+    stable_split,
+)
+
+
+def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
+    parsed_json = parse_text_content(
+        b'\xef\xbb\xbf{"data":[{"name":"\xe5\xbc\xa0\xe4\xb8\x89"}]}',
+        filename="records.json",
+    )
+    assert parsed_json.format == "json"
+    assert parsed_json.records == ({"name": "张三"},)
+
+    parsed_jsonl = parse_text_content('{"id":1}\n\n{"id":2}\n', filename="records.jsonl")
+    assert parsed_jsonl.format == "jsonl"
+    assert parsed_jsonl.records == ({"id": 1}, {"id": 2})
+
+    parsed_csv = parse_text_content("name,answer\r\nAlice,yes\r\nBob,no", filename="records.csv")
+    assert parsed_csv.format == "csv"
+    assert parsed_csv.text == "name,answer\nAlice,yes\nBob,no"
+    assert parsed_csv.records[1] == {"name": "Bob", "answer": "no"}
+
+    parsed_markdown = parse_text_content("# 标题\n\n正文", filename="README.md")
+    assert parsed_markdown.format == "markdown"
+    assert parsed_markdown.records == ()
+
+    parsed_txt = parse_text_content("普通文本", filename="note.txt")
+    assert parsed_txt.format == "txt"
+    assert parsed_txt.text == "普通文本"
+
+
+def test_invalid_utf8_and_malformed_structured_content_fail_loudly() -> None:
+    with pytest.raises(ValueError, match="not valid UTF-8"):
+        parse_text_content(b"\xff\xfe", filename="broken.txt")
+    with pytest.raises(ValueError, match="invalid JSONL at line 2"):
+        extract_structured_records('{"id":1}\nnot-json', "jsonl")
+    with pytest.raises(ValueError, match="more fields"):
+        extract_structured_records("a,b\n1,2,3", "csv")
+
+
+def test_detect_format_from_content_and_normalize() -> None:
+    assert detect_text_format(text='{"id":1}\n{"id":2}') == "jsonl"
+    assert detect_text_format(text="# Heading\ntext") == "markdown"
+    assert detect_text_format(text="a,b\n1,2") == "csv"
+    assert normalize_text("\ufeffＡＢＣ  \r\n第二\x00行\u200b\t \r\n") == "ABC\n第二行"
+
+
+def test_extract_json_scalar_and_nested_values_are_stable() -> None:
+    assert extract_structured_records("[1, true, null]", "json") == [
+        {"value": 1},
+        {"value": True},
+        {"value": None},
+    ]
+    result = extract_structured_records(
+        json.dumps({"items": [{"text": "  内容  "}], "ignored": 1}, ensure_ascii=False),
+        "json",
+    )
+    assert result == [{"text": "内容"}]
+
+
+def test_desensitize_pii_returns_masked_text_and_counts() -> None:
+    source = "邮箱 a.user+tag@example.com，手机 +86 13800138000，身份证 11010519491231002X。"
+    masked, counts = desensitize_pii(source)
+    assert masked == "邮箱 [EMAIL]，手机 [PHONE]，身份证 [ID_CARD]。"
+    assert counts == {"email": 1, "phone": 1, "id_card": 1, "total": 3}
+
+
+@pytest.mark.parametrize("method", ["semantic", "heading", "fixed", "custom"])
+def test_chunk_methods_preserve_offsets_and_always_advance(method: str) -> None:
+    text = "# 第一章\n" + "甲。" * 18 + "\n# 第二章\n" + "乙。" * 18
+    kwargs = {"custom_delimiter": "\\n"} if method == "custom" else {}
+    chunks = chunk_unstructured(
+        text,
+        method=method,  # type: ignore[arg-type]
+        chunk_size=12,
+        chunk_overlap=2,
+        min_chunk_size=4,
+        **kwargs,
+    )
+    assert len(chunks) > 1
+    assert all(chunk.content == normalize_text(text)[chunk.start : chunk.end] for chunk in chunks)
+    assert all(chunk.end > chunk.start for chunk in chunks)
+    assert all(left.start < right.start for left, right in zip(chunks, chunks[1:]))
+    assert all(chunk.start_line <= chunk.end_line for chunk in chunks)
+
+
+def test_fixed_chunk_overlap_is_exact_when_chunks_are_large_enough() -> None:
+    text = " ".join(f"token{i}" for i in range(30))
+    chunks = chunk_unstructured(
+        text,
+        method="fixed",
+        chunk_size=10,
+        chunk_overlap=3,
+        min_chunk_size=4,
+    )
+    first_tokens = chunks[0].content.split()
+    second_tokens = chunks[1].content.split()
+    assert first_tokens[-3:] == second_tokens[:3]
+    assert chunks[0].token_count == 10
+
+
+def test_chunk_line_numbers_treat_newline_as_previous_line_boundary() -> None:
+    chunks = chunk_unstructured(
+        "第一行。\n第二行。\n第三行。",
+        method="custom",
+        chunk_size=8,
+        chunk_overlap=0,
+        min_chunk_size=2,
+        custom_delimiter="\\n",
+    )
+    assert chunks[0].content.endswith("\n")
+    assert chunks[0].start_line == 1
+    assert chunks[0].end_line == 1
+    assert chunks[1].start_line == 2
+
+
+def test_heading_and_custom_boundaries_are_respected() -> None:
+    heading_text = "前言 " * 8 + "\n# 第二章\n" + "正文 " * 12
+    heading_chunks = chunk_unstructured(
+        heading_text,
+        method="heading",
+        chunk_size=20,
+        chunk_overlap=0,
+        min_chunk_size=4,
+    )
+    assert "# 第二章" not in heading_chunks[0].content
+    assert heading_chunks[1].content.startswith("#")
+
+    custom_chunks = chunk_unstructured(
+        "a b c d <CUT> e f g h i j",
+        method="custom",
+        chunk_size=8,
+        chunk_overlap=0,
+        min_chunk_size=2,
+        custom_delimiter="<CUT>",
+    )
+    assert custom_chunks[0].content.endswith("<CUT>")
+
+
+@pytest.mark.parametrize(
+    ("field", "block"),
+    [
+        (
+            "preserve_code_blocks",
+            "```python\n" + "\n".join(f"value_{i} = {i}" for i in range(30)) + "\n```",
+        ),
+        (
+            "preserve_tables",
+            "| 字段 | 说明 |\n| --- | --- |\n"
+            + "\n".join(f"| field_{i} | value_{i} |" for i in range(30)),
+        ),
+        (
+            "preserve_lists",
+            "\n".join(f"- 第 {i} 项需要完整保留" for i in range(30)),
+        ),
+    ],
+)
+def test_markdown_protected_blocks_are_not_split(field: str, block: str) -> None:
+    text = "前言。" * 15 + "\n" + block + "\n" + "结尾。" * 40
+    chunks = chunk_unstructured(
+        text,
+        method="fixed",
+        chunk_size=40,
+        chunk_overlap=0,
+        min_chunk_size=10,
+        **{field: True},
+    )
+    assert any(block in chunk.content for chunk in chunks)
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "message"),
+    [
+        ({"chunk_size": 0}, "chunk_size"),
+        ({"chunk_size": 10, "chunk_overlap": 10}, "chunk_overlap"),
+        ({"chunk_size": 10, "chunk_overlap": 0, "min_chunk_size": 11}, "min_chunk_size"),
+        (
+            {"chunk_size": 10, "chunk_overlap": 5, "min_chunk_size": 6},
+            "cannot exceed",
+        ),
+        ({"method": "custom", "custom_delimiter": ""}, "custom_delimiter"),
+    ],
+)
+def test_chunk_configuration_validation(kwargs: dict[str, object], message: str) -> None:
+    with pytest.raises(ValueError, match=message):
+        chunk_unstructured("some text", **kwargs)  # type: ignore[arg-type]
+
+
+def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
+    valid = {
+        "instruction": "如何修改收货地址？",
+        "input": "订单尚未发货",
+        "output": "可以在订单详情页申请修改收货地址。",
+    }
+    source = "订单尚未发货时，可以在订单详情页申请修改收货地址。"
+    first_score = score_quality(valid, min_output_length=10, source_content=source)
+    assert first_score.is_valid
+    assert first_score.completeness == 100
+    assert first_score.length == 100
+    assert first_score.readability >= 90
+    assert first_score.relevance >= 70
+    assert first_score.duplicate == 100
+
+    duplicate_score = score_quality(valid, known_fingerprints={first_score.fingerprint})
+    assert duplicate_score.duplicate == 0
+    assert "duplicate_record" in duplicate_score.flags
+
+    unrelated_score = score_quality(
+        valid,
+        min_output_length=10,
+        source_content="量子计算使用量子比特处理信息。",
+    )
+    assert unrelated_score.relevance < first_score.relevance
+    assert "low_source_relevance" in unrelated_score.flags
+
+    invalid_score = score_quality({"instruction": "", "output": "短"}, min_output_length=10)
+    assert not invalid_score.is_valid
+    assert {"missing_instruction", "output_too_short"}.issubset(invalid_score.flags)
+    assert record_fingerprint(valid) == record_fingerprint(dict(reversed(list(valid.items()))))
+
+
+def test_stable_split_is_reproducible_and_validates_ratios() -> None:
+    first = stable_split("record-42", seed="task-1")
+    assert stable_split("record-42", seed="task-1") == first
+    assert first in {"train", "validation", "test"}
+    assert stable_split("record-42", {"train": 100, "validation": 0, "test": 0}) == "train"
+    with pytest.raises(ValueError, match="sum to 100"):
+        stable_split("record", {"train": 80, "validation": 10, "test": 9})
+
+
+def test_generate_standard_records_supports_json_qa_and_stable_variants() -> None:
+    previews = [
+        {
+            "id": "preview-json",
+            "edited_content": json.dumps(
+                {"instruction": "问题", "input": "上下文", "output": "答案"},
+                ensure_ascii=False,
+            ),
+        },
+        {"id": "preview-qa", "editedContent": "问：如何操作？\n答：按步骤操作。"},
+    ]
+    records = generate_standard_records(
+        previews,
+        qa_pairs_per_item=2,
+        semantic_enrichment=True,
+        split={"train": 100, "validation": 0, "test": 0},
+        split_seed="task-1",
+    )
+    assert len(records) == 4
+    assert records[0]["instruction"] == "问题"
+    assert records[0]["input"] == "上下文"
+    assert records[0]["output"] == "答案"
+    assert records[1]["instruction"].endswith("问题")
+    assert records[2]["instruction"] == "如何操作?"
+    assert records[2]["output"] == "按步骤操作。"
+    assert all(record["status"] == "valid" for record in records)
+    assert all(record["split"] == "train" for record in records)
+    assert records == generate_standard_records(
+        previews,
+        qa_pairs_per_item=2,
+        semantic_enrichment=True,
+        split={"train": 100, "validation": 0, "test": 0},
+        split_seed="task-1",
+    )