feat: 新增外部数据源拉取与 DPO 输出格式支持

- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源
- 新增 DPO (Direct Preference Optimization) 输出类型
- 支持 chosen/rejected 字段的编辑、校验和发布
- 完善数据预处理切分逻辑和元数据管理
- 移除 OCR 扫描 PDF 功能,保持基础文本解析能力

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
caoxiaozhu
2026-08-11 14:17:45 +08:00
parent f809825a7d
commit 5f6e7523cf
26 changed files with 927 additions and 144 deletions

View File

@@ -1335,6 +1335,39 @@ def test_publish_rejects_invalid_reasoning_output_format() -> None:
)
def test_publish_dpo_writes_chosen_and_rejected_jsonl() -> None:
conn = _PublishConnection(
[
{
"id": "result-dpo",
"status": "valid",
"instruction": "如何处理扫描 PDF",
"input": "",
"output": "仅在无文本层时执行 OCR。",
"chosen": "仅在无文本层时执行 OCR。",
"rejected": "所有 PDF 都执行 OCR。",
"preview_item_id": "preview-dpo",
}
]
)
published = _PublishStore(conn, {"output_type": "dpo"}).publish(
"task-dpo",
{
"dataset_name": "偏好数据",
"storage_type": "local",
"format": "dpo",
"split": {"train": 100, "validation": 0, "test": 0},
},
)
record = conn.records[0]["raw"]
assert record["chosen"] == "仅在无文本层时执行 OCR。"
assert record["rejected"] == "所有 PDF 都执行 OCR。"
assert "output" not in record
assert published["datasets"][0]["metadata"]["format"] == "dpo"
def test_source_storage_descriptor_accepts_owned_local_and_legacy_db_references() -> None:
task_id = "dpt_task"
source_file_id = "dpsf_source"