fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
caoxiaozhu
@@ -5,6 +5,7 @@ import json
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
record_fingerprint,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
structured_json_dumps,
|
||||
)
|
||||
|
||||
|
||||
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
|
||||
root_json = parse_text_content('{"id":1}', filename="root.json")
|
||||
assert root_json.record_locators == (
|
||||
{
|
||||
"kind": "json",
|
||||
"record_index": 1,
|
||||
"json_pointer": "",
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
},
|
||||
)
|
||||
|
||||
wrapped_json = parse_text_content(
|
||||
'{"records":[{"id":1},{"id":1}]}',
|
||||
filename="wrapped.json",
|
||||
)
|
||||
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
|
||||
"/records/0",
|
||||
"/records/1",
|
||||
]
|
||||
|
||||
parsed_jsonl = parse_text_content(
|
||||
'{"id":1}\r\n\r\n{"id":1}',
|
||||
filename="records.jsonl",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == [(1, 1, 1), (2, 3, 3)]
|
||||
assert [
|
||||
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == ['{"id":1}', '{"id":1}']
|
||||
|
||||
parsed_csv = parse_text_content(
|
||||
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
|
||||
filename="records.csv",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_csv.record_locators
|
||||
] == [(1, 2, 3), (2, 5, 5)]
|
||||
assert [
|
||||
parsed_csv.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_csv.record_locators
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
|
||||
processed = preprocess_structured_records_with_lineage(
|
||||
[
|
||||
{"id": "A", "value": "first", "empty": ""},
|
||||
{"id": "", "value": "invalid", "empty": ""},
|
||||
{"id": "A", "value": "duplicate identity", "empty": ""},
|
||||
{"id": "B", "value": "second", "empty": ""},
|
||||
],
|
||||
["clean_invalid", "deduplicate"],
|
||||
)
|
||||
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
|
||||
assert [entry.record for entry in processed] == [
|
||||
{"id": "A", "value": "first"},
|
||||
{"id": "", "value": "invalid"},
|
||||
{"id": "A", "value": "duplicate identity"},
|
||||
{"id": "B", "value": "second"},
|
||||
]
|
||||
|
||||
|
||||
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
|
||||
assert parsed_pdf.format == "pdf"
|
||||
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
|
||||
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
|
||||
)
|
||||
assert parsed_xlsx.record_locators == (
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
)
|
||||
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
|
||||
|
||||
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
|
||||
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
assert parsed_pptx.records == ()
|
||||
|
||||
|
||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||
workbook = Workbook()
|
||||
first = workbook.active
|
||||
first.title = "甲表"
|
||||
first.append(["说明"])
|
||||
first.append([])
|
||||
first.append(["id", "value"])
|
||||
first.append([1, "same"])
|
||||
first.append([1, "same"])
|
||||
second = workbook.create_sheet("乙表")
|
||||
second.append(["id", "value"])
|
||||
second.append([1, "same"])
|
||||
output = io.BytesIO()
|
||||
workbook.save(output)
|
||||
workbook.close()
|
||||
|
||||
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
|
||||
assert parsed.records == (
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
)
|
||||
assert [
|
||||
(
|
||||
locator["record_index"],
|
||||
locator["sheet_index"],
|
||||
locator["sheet_name"],
|
||||
locator["row_number"],
|
||||
locator["sheet_record_index"],
|
||||
)
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(1, 0, "甲表", 4, 0),
|
||||
(2, 0, "甲表", 5, 1),
|
||||
(3, 1, "乙表", 2, 0),
|
||||
]
|
||||
|
||||
|
||||
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
|
||||
pages = _pdf_page_texts(
|
||||
"""
|
||||
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
|
||||
|
||||
assert extract_structured_records(
|
||||
'{"items":[{"text":" 内容 "}],"total":1}',
|
||||
"json",
|
||||
) == [{"text": " 内容 "}]
|
||||
|
||||
|
||||
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
|
||||
source = '{"code":"００１","text":" 内容 ","quote":"＂"}'
|
||||
parsed = parse_text_content(source, filename="records.json")
|
||||
assert parsed.text == source
|
||||
assert parsed.records == (
|
||||
{"code": "００１", "text": " 内容 ", "quote": "＂"},
|
||||
)
|
||||
|
||||
invalid_values = (
|
||||
'{"id":1,"id":2}',
|
||||
'{"nested":{"id":1,"id":2}}',
|
||||
'{"value":NaN}',
|
||||
'{"value":Infinity}',
|
||||
'{"value":-Infinity}',
|
||||
'{"value":"bad\x00control"}',
|
||||
)
|
||||
for invalid in invalid_values:
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content(invalid, filename="invalid.json")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content("｛\"id\":1｝", filename="invalid.json")
|
||||
with pytest.raises(ValueError, match="nesting exceeds"):
|
||||
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
|
||||
|
||||
|
||||
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
|
||||
source = (
|
||||
' {"code":"００１","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}\r\n\r\n'
|
||||
'{"id":2}\r\n'
|
||||
)
|
||||
parsed = parse_text_content(source, filename="records.jsonl")
|
||||
assert parsed.text == source
|
||||
assert parsed.records[0] == {
|
||||
"code": "００１",
|
||||
"text": " 内容 ",
|
||||
"value": Decimal("0.123456789012345678901234567890"),
|
||||
}
|
||||
assert [
|
||||
source[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(
|
||||
'{"code":"００１","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}'
|
||||
),
|
||||
'{"id":2}',
|
||||
]
|
||||
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
|
||||
|
||||
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
|
||||
parse_text_content(invalid, filename="invalid.jsonl")
|
||||
|
||||
|
||||
def test_json_record_contract_avoids_business_field_collisions() -> None:
|
||||
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
|
||||
{"id": 1},
|
||||
{"id": 2},
|
||||
]
|
||||
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
|
||||
{"id": 1, "data": [{"id": 2}]}
|
||||
]
|
||||
assert extract_structured_records(
|
||||
'{"records":[{"id":1}],"data":[{"id":2}]}',
|
||||
"json",
|
||||
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
|
||||
assert extract_structured_records(
|
||||
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
|
||||
"json",
|
||||
) == [{"id": 1}]
|
||||
assert extract_structured_records(
|
||||
'{"payload":{"data":[{"id":2}],"total":1}}',
|
||||
"json",
|
||||
) == [{"id": 2}]
|
||||
assert extract_structured_records('{"records":[],"total":0}', "json") == []
|
||||
# 包装数组中的非对象不是记录集合，整体按一条业务对象保留。
|
||||
assert extract_structured_records('{"data":[1,2]}', "json") == [
|
||||
{"data": [1, 2]}
|
||||
]
|
||||
|
||||
|
||||
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
|
||||
pretty = (
|
||||
'{\n "records": [\n {"id": 1},\n'
|
||||
' {\n "id": 2\n }\n ],\n "total": 2\n}'
|
||||
)
|
||||
parsed = parse_text_content(pretty, filename="pretty.json")
|
||||
assert [
|
||||
pretty[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id": 1}', '{\n "id": 2\n }']
|
||||
assert [
|
||||
(locator["start_line"], locator["end_line"])
|
||||
for locator in parsed.record_locators
|
||||
] == [(3, 3), (4, 6)]
|
||||
|
||||
minified = '[{"id":1},{"id":2}]'
|
||||
parsed = parse_text_content(minified, filename="minified.json")
|
||||
assert [
|
||||
minified[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
|
||||
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
|
||||
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
|
||||
parsed = parse_text_content(source, filename="precise.json")
|
||||
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
|
||||
assert parsed.records[1]["value"] == Decimal("1e400")
|
||||
assert structured_json_dumps(parsed.records[0]) == (
|
||||
'{"value":0.123456789012345678901234567890}'
|
||||
)
|
||||
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
|
||||
assert isinstance(parsed.records[0]["value"], Decimal)
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
assert preprocess_structured_records(clean_source, []) == clean_source
|
||||
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
|
||||
{"id": "1", "name": "有效"},
|
||||
{"id": "", "name": "缺少关键字段"},
|
||||
{"id": "2", "name": "有效"},
|
||||
]
|
||||
|
||||
hierarchy = [
|
||||
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
|
||||
{"id": "", "parent_id": "", "name": "", "empty": ""},
|
||||
]
|
||||
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
|
||||
{"id": "1", "parent_id": None, "name": "根节点"},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点"},
|
||||
]
|
||||
|
||||
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
|
||||
assert "profile" in preprocess_structured_records(nested, [])[0]
|
||||
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
|
||||
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
duplicates = [
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "C-1", "value": "updated"},
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "", "value": "blank-one"},
|
||||
{"customer_id": "", "value": "blank-two"},
|
||||
]
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 4
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 5
|
||||
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
|
||||
assert [record["value"] for record in deduplicated] == [
|
||||
"first",
|
||||
"updated",
|
||||
"blank-one",
|
||||
"blank-two",
|
||||
]
|
||||
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
|
||||
masked, counts = desensitize_structured_record(
|
||||
{
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "张三",
|
||||
"contact_name": "李四",
|
||||
"姓名": "王五",
|
||||
"profile.name": "赵六",
|
||||
}
|
||||
)
|
||||
|
||||
assert masked == {
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "[NAME]",
|
||||
"contact_name": "[NAME]",
|
||||
"姓名": "[NAME]",
|
||||
"profile.name": "[NAME]",
|
||||
}
|
||||
assert counts == {
|
||||
"email": 0,
|
||||
"phone": 0,
|
||||
"id_card": 0,
|
||||
"name": 4,
|
||||
"total": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址？",
|
||||
|
||||
Reference in New Issue
Block a user