feat: 新增外部数据源拉取与 DPO 输出格式支持
- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源 - 新增 DPO (Direct Preference Optimization) 输出类型 - 支持 chosen/rejected 字段的编辑、校验和发布 - 完善数据预处理切分逻辑和元数据管理 - 移除 OCR 扫描 PDF 功能，保持基础文本解析能力 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
caoxiaozhu
@@ -16,7 +16,7 @@ from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from threading import BoundedSemaphore, Lock
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlsplit
|
||||
from urllib.parse import parse_qs, quote, urlsplit
|
||||
|
||||
import httpx
|
||||
import psycopg
|
||||
@@ -676,8 +676,10 @@ def _run_generation(
|
||||
qa_pairs_per_item=int(pairs or 1),
|
||||
on_progress=report_progress,
|
||||
)
|
||||
elif output_type == "reasoning":
|
||||
raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
|
||||
elif output_type in {"reasoning", "dpo"}:
|
||||
if output_type == "reasoning":
|
||||
raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
|
||||
raise InvalidStateError("DPO 输出必须配置可用的数据生成模型")
|
||||
else:
|
||||
generated = generate_standard_records(
|
||||
preview_items,
|
||||
@@ -1077,7 +1079,10 @@ async def upload_source_files(
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
process_type = str(task["process_type"])
|
||||
if process_type == "external":
|
||||
source_mode = str(
|
||||
_value(task.get("config") or {}, "source_mode", "sourceMode", "local")
|
||||
)
|
||||
if process_type == "external" or source_mode == "external":
|
||||
raise InvalidStateError(
|
||||
"external tasks must import data through the external source endpoint"
|
||||
)
|
||||
@@ -1375,6 +1380,10 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
|
||||
raise fail(501, f"external data source type is not supported: {payload.type}")
|
||||
if parsed_url.username or parsed_url.password:
|
||||
raise fail(400, "database credentials must use the account and password fields")
|
||||
if set(parse_qs(parsed_url.query)) & {
|
||||
"password", "secret", "token", "api_key", "user", "username"
|
||||
}:
|
||||
raise fail(400, "database URL query must not contain credentials")
|
||||
if payload.auth_mode not in {"none", "basic"}:
|
||||
raise fail(400, "PostgreSQL supports only none or basic authentication")
|
||||
if payload.auth_mode == "basic" and not payload.username:
|
||||
@@ -1413,10 +1422,14 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
|
||||
"set DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB=true only in a trusted deployment",
|
||||
)
|
||||
kwargs: dict[str, Any] = {
|
||||
"connect_timeout": 5,
|
||||
"connect_timeout": payload.connect_timeout_seconds,
|
||||
"row_factory": dict_row,
|
||||
"application_name": "yg-ft-data-process-readonly",
|
||||
"options": "-c default_transaction_read_only=on -c statement_timeout=30000",
|
||||
"options": (
|
||||
"-c default_transaction_read_only=on "
|
||||
f"-c statement_timeout={payload.statement_timeout_seconds * 1000}"
|
||||
),
|
||||
"sslmode": payload.ssl_mode,
|
||||
}
|
||||
if payload.auth_mode == "basic" and payload.username:
|
||||
kwargs["user"] = payload.username
|
||||
@@ -1425,6 +1438,17 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
|
||||
return psycopg.connect(payload.url, **kwargs)
|
||||
|
||||
|
||||
def _assert_external_source_task(task: dict[str, Any]) -> None:
|
||||
if str(task.get("process_type")) == "external":
|
||||
return
|
||||
config = task.get("config") or {}
|
||||
source_mode = str(_value(config, "source_mode", "sourceMode", "local"))
|
||||
if str(task.get("process_type")) != "structured" or source_mode != "external":
|
||||
raise InvalidStateError(
|
||||
"external source access requires a structured task with source_mode=external"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/external/test")
|
||||
def test_external_source(
|
||||
task_id: str,
|
||||
@@ -1433,10 +1457,7 @@ def test_external_source(
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
if str(task.get("process_type")) != "external":
|
||||
raise InvalidStateError(
|
||||
"external source access requires an external data processing task"
|
||||
)
|
||||
_assert_external_source_task(task)
|
||||
try:
|
||||
with _external_postgres_connection(payload) as conn:
|
||||
conn.execute("SELECT 1 AS ok").fetchone()
|
||||
@@ -1462,12 +1483,14 @@ def pull_external_source(
|
||||
raise fail(400, "a read-only SELECT or WITH query is required for external pull")
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
if str(task.get("process_type")) != "external":
|
||||
raise InvalidStateError("external pull requires an external data processing task")
|
||||
_assert_external_source_task(task)
|
||||
try:
|
||||
with _external_postgres_connection(payload) as conn:
|
||||
conn.execute("SET TRANSACTION READ ONLY")
|
||||
conn.execute("SET LOCAL statement_timeout = '30s'")
|
||||
conn.execute(
|
||||
"SELECT set_config('statement_timeout', %s, true)",
|
||||
(f"{payload.statement_timeout_seconds}s",),
|
||||
)
|
||||
cursor = conn.execute(query)
|
||||
rows: list[dict[str, Any]] = []
|
||||
content_parts: list[str] = []
|
||||
@@ -1518,6 +1541,9 @@ def pull_external_source(
|
||||
"external_type": payload.type,
|
||||
"external_host": urlsplit(payload.url).hostname,
|
||||
"external_limit": payload.limit,
|
||||
"external_ssl_mode": payload.ssl_mode,
|
||||
"external_connect_timeout_seconds": payload.connect_timeout_seconds,
|
||||
"external_statement_timeout_seconds": payload.statement_timeout_seconds,
|
||||
},
|
||||
}
|
||||
],
|
||||
@@ -1566,7 +1592,10 @@ def _prepare_preview_items(
|
||||
for index, source in enumerate(sources):
|
||||
source_format = str(source.get("file_format") or "").lower()
|
||||
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
|
||||
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
|
||||
needs_layout_raw = (
|
||||
is_unstructured
|
||||
and chunk_method == "layout_hybrid"
|
||||
)
|
||||
needs_pdf_noise = (
|
||||
is_unstructured
|
||||
and not needs_layout_raw
|
||||
@@ -2038,6 +2067,8 @@ def restore_result(
|
||||
"instruction": current.get("original_instruction") or current.get("instruction") or "",
|
||||
"input": current.get("original_input") or current.get("input") or "",
|
||||
"output": current.get("original_output") or current.get("output") or "",
|
||||
"chosen": current.get("original_chosen") or current.get("chosen") or "",
|
||||
"rejected": current.get("original_rejected") or current.get("rejected") or "",
|
||||
}
|
||||
preview_id = current.get("preview_item_id")
|
||||
source_content = ""
|
||||
@@ -2070,6 +2101,8 @@ def restore_result(
|
||||
"instruction": restored["instruction"],
|
||||
"input": restored["input"],
|
||||
"output": restored["output"],
|
||||
"chosen": restored["chosen"],
|
||||
"rejected": restored["rejected"],
|
||||
"quality_score": asdict(quality),
|
||||
"expected_updated_at": current.get("updated_at"),
|
||||
},
|
||||
@@ -2141,13 +2174,15 @@ def _generate_result_replacement(
|
||||
).strip().lower()
|
||||
previous_instruction = str(current.get("instruction") or "")[:1000]
|
||||
previous_output = str(current.get("output") or "")[:1000]
|
||||
previous_rejected = str(current.get("rejected") or "")[:1000]
|
||||
base_prompt = str(
|
||||
_value(config, "generation_prompt", "generationPrompt", "") or ""
|
||||
)
|
||||
regeneration_instruction = (
|
||||
"这是一次失败结果的重新生成。请使用新的提问角度和表达，"
|
||||
"不要复述旧结果。旧问题："
|
||||
f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
|
||||
f"{previous_instruction or '无'}；旧优选答案：{previous_output or '无'}；"
|
||||
f"旧拒选答案：{previous_rejected or '无'}。"
|
||||
)
|
||||
runtime_config = {
|
||||
**config,
|
||||
|
||||
Reference in New Issue
Block a user