feat: 新增外部数据源拉取与 DPO 输出格式支持
- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源 - 新增 DPO (Direct Preference Optimization) 输出类型 - 支持 chosen/rejected 字段的编辑、校验和发布 - 完善数据预处理切分逻辑和元数据管理 - 移除 OCR 扫描 PDF 功能,保持基础文本解析能力 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1335,6 +1335,39 @@ def test_publish_rejects_invalid_reasoning_output_format() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_publish_dpo_writes_chosen_and_rejected_jsonl() -> None:
|
||||
conn = _PublishConnection(
|
||||
[
|
||||
{
|
||||
"id": "result-dpo",
|
||||
"status": "valid",
|
||||
"instruction": "如何处理扫描 PDF?",
|
||||
"input": "",
|
||||
"output": "仅在无文本层时执行 OCR。",
|
||||
"chosen": "仅在无文本层时执行 OCR。",
|
||||
"rejected": "所有 PDF 都执行 OCR。",
|
||||
"preview_item_id": "preview-dpo",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
published = _PublishStore(conn, {"output_type": "dpo"}).publish(
|
||||
"task-dpo",
|
||||
{
|
||||
"dataset_name": "偏好数据",
|
||||
"storage_type": "local",
|
||||
"format": "dpo",
|
||||
"split": {"train": 100, "validation": 0, "test": 0},
|
||||
},
|
||||
)
|
||||
|
||||
record = conn.records[0]["raw"]
|
||||
assert record["chosen"] == "仅在无文本层时执行 OCR。"
|
||||
assert record["rejected"] == "所有 PDF 都执行 OCR。"
|
||||
assert "output" not in record
|
||||
assert published["datasets"][0]["metadata"]["format"] == "dpo"
|
||||
|
||||
|
||||
def test_source_storage_descriptor_accepts_owned_local_and_legacy_db_references() -> None:
|
||||
task_id = "dpt_task"
|
||||
source_file_id = "dpsf_source"
|
||||
|
||||
Reference in New Issue
Block a user