feat: 新增外部数据源拉取与 DPO 输出格式支持

- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源
- 新增 DPO (Direct Preference Optimization) 输出类型
- 支持 chosen/rejected 字段的编辑、校验和发布
- 完善数据预处理切分逻辑和元数据管理
- 移除 OCR 扫描 PDF 功能,保持基础文本解析能力

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
caoxiaozhu
2026-08-11 14:17:45 +08:00
parent f809825a7d
commit 5f6e7523cf
26 changed files with 927 additions and 144 deletions

View File

@@ -1840,6 +1840,43 @@ def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
assert response.json()["detail"]["code"] == 501
def test_external_source_mode_belongs_to_step_three_structured_task(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
local_task_id = client.post(
"/modelTF/data-process",
json={
"name": "本地结构化任务",
"process_type": "structured",
"config": {"source_mode": "local"},
},
).json()["data"]["id"]
rejected = client.post(
f"/modelTF/data-process/{local_task_id}/external/test",
json={"type": "mysql", "url": "mysql://db.example/test"},
)
assert rejected.status_code == 409
external_task_id = client.post(
"/modelTF/data-process",
json={
"name": "外部结构化任务",
"process_type": "structured",
"config": {"source_mode": "external"},
},
).json()["data"]["id"]
accepted_as_external = client.post(
f"/modelTF/data-process/{external_task_id}/external/test",
json={"type": "mysql", "url": "mysql://db.example/test"},
)
assert accepted_as_external.status_code == 501
local_upload = client.post(
f"/modelTF/data-process/{external_task_id}/source-files",
files={"files": ("records.jsonl", b'{"id":1}\n', "application/jsonl")},
)
assert local_upload.status_code == 409
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(