Merge branch 'ft_wyt' of http://www.caoxiaozhu.com:13001/YG-Soft/YG_FT into ft_wyt

backend/app/api/v1/endpoints/data_process.py

@@ -16,7 +16,7 @@ from dataclasses import asdict
 from pathlib import Path
 from threading import BoundedSemaphore, Lock
 from typing import Any, Literal
-from urllib.parse import quote, urlsplit
+from urllib.parse import parse_qs, quote, urlsplit
 
 import httpx
 import psycopg
@@ -676,8 +676,10 @@ def _run_generation(
                 qa_pairs_per_item=int(pairs or 1),
                 on_progress=report_progress,
             )
-        elif output_type == "reasoning":
-            raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
+        elif output_type in {"reasoning", "dpo"}:
+            if output_type == "reasoning":
+                raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
+            raise InvalidStateError("DPO 输出必须配置可用的数据生成模型")
         else:
             generated = generate_standard_records(
                 preview_items,
@@ -1077,7 +1079,10 @@ async def upload_source_files(
         with api_errors():
             task = store.get_task(task_id)
             process_type = str(task["process_type"])
-            if process_type == "external":
+            source_mode = str(
+                _value(task.get("config") or {}, "source_mode", "sourceMode", "local")
+            )
+            if process_type == "external" or source_mode == "external":
                 raise InvalidStateError(
                     "external tasks must import data through the external source endpoint"
                 )
@@ -1375,6 +1380,10 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
         raise fail(501, f"external data source type is not supported: {payload.type}")
     if parsed_url.username or parsed_url.password:
         raise fail(400, "database credentials must use the account and password fields")
+    if set(parse_qs(parsed_url.query)) & {
+        "password", "secret", "token", "api_key", "user", "username"
+    }:
+        raise fail(400, "database URL query must not contain credentials")
     if payload.auth_mode not in {"none", "basic"}:
         raise fail(400, "PostgreSQL supports only none or basic authentication")
     if payload.auth_mode == "basic" and not payload.username:
@@ -1413,10 +1422,14 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
                 "set DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB=true only in a trusted deployment",
             )
     kwargs: dict[str, Any] = {
-        "connect_timeout": 5,
+        "connect_timeout": payload.connect_timeout_seconds,
         "row_factory": dict_row,
         "application_name": "yg-ft-data-process-readonly",
-        "options": "-c default_transaction_read_only=on -c statement_timeout=30000",
+        "options": (
+            "-c default_transaction_read_only=on "
+            f"-c statement_timeout={payload.statement_timeout_seconds * 1000}"
+        ),
+        "sslmode": payload.ssl_mode,
     }
     if payload.auth_mode == "basic" and payload.username:
         kwargs["user"] = payload.username
@@ -1425,6 +1438,17 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
     return psycopg.connect(payload.url, **kwargs)
 
 
+def _assert_external_source_task(task: dict[str, Any]) -> None:
+    if str(task.get("process_type")) == "external":
+        return
+    config = task.get("config") or {}
+    source_mode = str(_value(config, "source_mode", "sourceMode", "local"))
+    if str(task.get("process_type")) != "structured" or source_mode != "external":
+        raise InvalidStateError(
+            "external source access requires a structured task with source_mode=external"
+        )
+
+
 @router.post("/{task_id}/external/test")
 def test_external_source(
     task_id: str,
@@ -1433,10 +1457,7 @@ def test_external_source(
 ) -> dict[str, Any]:
     with api_errors():
         task = store.get_task(task_id)
-        if str(task.get("process_type")) != "external":
-            raise InvalidStateError(
-                "external source access requires an external data processing task"
-            )
+        _assert_external_source_task(task)
         try:
             with _external_postgres_connection(payload) as conn:
                 conn.execute("SELECT 1 AS ok").fetchone()
@@ -1462,12 +1483,14 @@ def pull_external_source(
         raise fail(400, "a read-only SELECT or WITH query is required for external pull")
     with api_errors():
         task = store.get_task(task_id)
-        if str(task.get("process_type")) != "external":
-            raise InvalidStateError("external pull requires an external data processing task")
+        _assert_external_source_task(task)
         try:
             with _external_postgres_connection(payload) as conn:
                 conn.execute("SET TRANSACTION READ ONLY")
-                conn.execute("SET LOCAL statement_timeout = '30s'")
+                conn.execute(
+                    "SELECT set_config('statement_timeout', %s, true)",
+                    (f"{payload.statement_timeout_seconds}s",),
+                )
                 cursor = conn.execute(query)
                 rows: list[dict[str, Any]] = []
                 content_parts: list[str] = []
@@ -1518,6 +1541,9 @@ def pull_external_source(
                         "external_type": payload.type,
                         "external_host": urlsplit(payload.url).hostname,
                         "external_limit": payload.limit,
+                        "external_ssl_mode": payload.ssl_mode,
+                        "external_connect_timeout_seconds": payload.connect_timeout_seconds,
+                        "external_statement_timeout_seconds": payload.statement_timeout_seconds,
                     },
                 }
             ],
@@ -1566,7 +1592,10 @@ def _prepare_preview_items(
         for index, source in enumerate(sources):
             source_format = str(source.get("file_format") or "").lower()
             needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
-            needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
+            needs_layout_raw = (
+                is_unstructured
+                and chunk_method == "layout_hybrid"
+            )
             needs_pdf_noise = (
                 is_unstructured
                 and not needs_layout_raw
@@ -2038,6 +2067,8 @@ def restore_result(
             "instruction": current.get("original_instruction") or current.get("instruction") or "",
             "input": current.get("original_input") or current.get("input") or "",
             "output": current.get("original_output") or current.get("output") or "",
+            "chosen": current.get("original_chosen") or current.get("chosen") or "",
+            "rejected": current.get("original_rejected") or current.get("rejected") or "",
         }
         preview_id = current.get("preview_item_id")
         source_content = ""
@@ -2070,6 +2101,8 @@ def restore_result(
                 "instruction": restored["instruction"],
                 "input": restored["input"],
                 "output": restored["output"],
+                "chosen": restored["chosen"],
+                "rejected": restored["rejected"],
                 "quality_score": asdict(quality),
                 "expected_updated_at": current.get("updated_at"),
             },
@@ -2141,13 +2174,15 @@ def _generate_result_replacement(
     ).strip().lower()
     previous_instruction = str(current.get("instruction") or "")[:1000]
     previous_output = str(current.get("output") or "")[:1000]
+    previous_rejected = str(current.get("rejected") or "")[:1000]
     base_prompt = str(
         _value(config, "generation_prompt", "generationPrompt", "") or ""
     )
     regeneration_instruction = (
         "这是一次失败结果的重新生成。请使用新的提问角度和表达，"
         "不要复述旧结果。旧问题："
-        f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
+        f"{previous_instruction or '无'}；旧优选答案：{previous_output or '无'}；"
+        f"旧拒选答案：{previous_rejected or '无'}。"
     )
     runtime_config = {
         **config,