feat: 新增外部数据源拉取与 DPO 输出格式支持
- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源 - 新增 DPO (Direct Preference Optimization) 输出类型 - 支持 chosen/rejected 字段的编辑、校验和发布 - 完善数据预处理切分逻辑和元数据管理 - 移除 OCR 扫描 PDF 功能,保持基础文本解析能力 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1840,6 +1840,43 @@ def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
|
||||
assert response.json()["detail"]["code"] == 501
|
||||
|
||||
|
||||
def test_external_source_mode_belongs_to_step_three_structured_task(tmp_path: Path) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
local_task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "本地结构化任务",
|
||||
"process_type": "structured",
|
||||
"config": {"source_mode": "local"},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
rejected = client.post(
|
||||
f"/modelTF/data-process/{local_task_id}/external/test",
|
||||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||||
)
|
||||
assert rejected.status_code == 409
|
||||
|
||||
external_task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "外部结构化任务",
|
||||
"process_type": "structured",
|
||||
"config": {"source_mode": "external"},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
accepted_as_external = client.post(
|
||||
f"/modelTF/data-process/{external_task_id}/external/test",
|
||||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||||
)
|
||||
assert accepted_as_external.status_code == 501
|
||||
|
||||
local_upload = client.post(
|
||||
f"/modelTF/data-process/{external_task_id}/source-files",
|
||||
files={"files": ("records.jsonl", b'{"id":1}\n', "application/jsonl")},
|
||||
)
|
||||
assert local_upload.status_code == 409
|
||||
|
||||
|
||||
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
|
||||
Reference in New Issue
Block a user