feat(data-process): 完善后台生成与失败重试

backend/app/modules/data_process/generation.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 import re
 from collections.abc import Callable, Iterable, Mapping
 from typing import Any
@@ -22,6 +23,10 @@ class ModelGenerationError(ValueError):
     """模型配置、响应或调用失败。"""
 
 
+class _TerminalModelGenerationError(ModelGenerationError):
+    """使用相同参数重试也无法恢复的模型响应错误。"""
+
+
 OUTPUT_TYPE_STANDARD = "standard"
 OUTPUT_TYPE_REASONING = "reasoning"
 SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
@@ -31,6 +36,25 @@ SUPPORTED_REASONING_DETAILS = {
     REASONING_DETAIL_NORMAL,
     REASONING_DETAIL_DETAILED,
 }
+MINIMAX_M3_API_HOSTS = {"api.minimax.io", "api.minimaxi.com"}
+MINIMAX_M3_MIN_COMPLETION_TOKENS = 4096
+logger = logging.getLogger(__name__)
+
+
+def _is_retryable_generation_error(exc: Exception) -> bool:
+    if isinstance(exc, _TerminalModelGenerationError):
+        return False
+    if isinstance(exc, httpx.HTTPStatusError):
+        status_code = exc.response.status_code
+        return status_code in {408, 425, 429} or status_code >= 500
+    if isinstance(exc, httpx.RequestError):
+        return True
+    return isinstance(exc, (json.JSONDecodeError, ModelGenerationError))
+
+
+def _is_official_minimax_m3(endpoint: str, model_name: str) -> bool:
+    host = (urlsplit(endpoint).hostname or "").casefold()
+    return host in MINIMAX_M3_API_HOSTS and model_name.casefold() == "minimax-m3"
 
 
 def chat_completions_url(value: str) -> str:
@@ -59,32 +83,139 @@ def chat_completions_url(value: str) -> str:
     return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
 
 
-def _message_content(payload: Mapping[str, Any]) -> str:
+def _response_choice(payload: Mapping[str, Any]) -> Mapping[str, Any]:
     try:
-        content = payload["choices"][0]["message"]["content"]
+        choice = payload["choices"][0]
     except (KeyError, IndexError, TypeError) as exc:
-        raise ModelGenerationError(
-            "model response does not contain choices[0].message.content"
-        ) from exc
+        raise ModelGenerationError("模型响应缺少 choices[0]") from exc
+    if not isinstance(choice, Mapping):
+        raise ModelGenerationError("模型响应 choices[0] 不是对象")
+    return choice
+
+
+def _response_finish_reason(payload: Mapping[str, Any]) -> str:
+    try:
+        return str(_response_choice(payload).get("finish_reason") or "").strip().lower()
+    except ModelGenerationError:
+        return ""
+
+
+def _response_content_length(payload: Mapping[str, Any]) -> int:
+    try:
+        message = _response_choice(payload).get("message")
+        if not isinstance(message, Mapping):
+            return 0
+        content = message.get("content")
+        if isinstance(content, str):
+            return len(content)
+        if isinstance(content, list):
+            return sum(
+                len(str(item.get("text") or ""))
+                for item in content
+                if isinstance(item, Mapping)
+            )
+    except ModelGenerationError:
+        pass
+    return 0
+
+
+def _raise_for_terminal_response(payload: Mapping[str, Any]) -> Mapping[str, Any]:
+    choice = _response_choice(payload)
+    base_response = payload.get("base_resp")
+    status_code: Any = None
+    status_message = ""
+    if isinstance(base_response, Mapping):
+        status_code = base_response.get("status_code")
+        status_message = re.sub(
+            r"\s+", " ", str(base_response.get("status_msg") or "")
+        ).strip()[:200]
+
+    if bool(payload.get("input_sensitive")) or status_code in {1026, "1026"}:
+        raise _TerminalModelGenerationError(
+            f"模型输入触发内容安全拦截（code={status_code or 1026}）"
+        )
+    if bool(payload.get("output_sensitive")) or status_code in {1027, "1027"}:
+        raise _TerminalModelGenerationError(
+            f"模型输出触发内容安全拦截（code={status_code or 1027}）"
+        )
+
+    finish_reason = str(choice.get("finish_reason") or "").strip().lower()
+    if finish_reason == "length":
+        raise _TerminalModelGenerationError(
+            "模型输出因达到 Token 上限被截断（finish_reason=length），"
+            "请提高最大输出长度后重试"
+        )
+    if finish_reason == "content_filter":
+        raise _TerminalModelGenerationError(
+            "模型输出被内容安全策略拦截（finish_reason=content_filter）"
+        )
+    if finish_reason in {"tool_calls", "function_call"}:
+        raise _TerminalModelGenerationError(
+            f"模型返回了当前生成任务不支持的工具调用（finish_reason={finish_reason}）"
+        )
+    if status_code not in {None, "", 0, "0"}:
+        detail = f"：{status_message}" if status_message else ""
+        raise _TerminalModelGenerationError(
+            f"模型服务返回业务错误（code={status_code}）{detail}"
+        )
+    return choice
+
+
+def _message_content(payload: Mapping[str, Any]) -> str:
+    choice = _raise_for_terminal_response(payload)
+    message = choice.get("message")
+    if not isinstance(message, Mapping):
+        raise ModelGenerationError("模型响应缺少 choices[0].message")
+    content = message.get("content")
     if isinstance(content, str):
-        return content
-    if isinstance(content, list):
+        result = content
+    elif isinstance(content, list):
         parts = [
             str(item.get("text") or "")
             for item in content
             if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
         ]
-        if parts:
-            return "".join(parts)
-    raise ModelGenerationError("model response content must be text")
+        result = "".join(parts)
+    elif content is None:
+        result = ""
+    else:
+        raise ModelGenerationError("模型响应 content 必须是文本")
+    if not result.strip():
+        raise ModelGenerationError("模型返回的最终内容为空，未生成可解析的 JSON")
+    return result
+
+
+def _json_documents(content: str) -> list[Any]:
+    decoder = json.JSONDecoder()
+    documents: list[Any] = []
+    cursor = 0
+    while cursor < len(content):
+        match = re.search(r"[\[{]", content[cursor:])
+        if not match:
+            break
+        start = cursor + match.start()
+        try:
+            value, end = decoder.raw_decode(content[start:])
+        except json.JSONDecodeError:
+            cursor = start + 1
+            continue
+        if isinstance(value, (Mapping, list)):
+            documents.append(value)
+        cursor = start + max(end, 1)
+    return documents
 
 
 def _json_payload(content: str) -> Any:
     # 只移除模型在 JSON 之前自行输出的思考过程，不能破坏 JSON 字段中的训练内容。
+    cleaned = content.strip()
+    if re.match(r"^\s*<think>", cleaned, flags=re.IGNORECASE) and not re.match(
+        r"^\s*<think>[\s\S]*?</think>", cleaned, flags=re.IGNORECASE
+    ):
+        raise ModelGenerationError("模型思考内容未闭合，响应可能已被截断")
     cleaned = re.sub(
-        r"^\s*<think>[\s\S]*?</think>\s*",
+        r"^\s*(?:<think>[\s\S]*?</think>\s*)+",
         "",
-        content,
+        cleaned,
         count=1,
         flags=re.IGNORECASE,
     ).strip()
@@ -93,10 +224,16 @@ def _json_payload(content: str) -> Any:
         cleaned = fenced.group(1).strip()
     try:
         return json.loads(cleaned)
-    except json.JSONDecodeError as exc:
+    except json.JSONDecodeError as direct_error:
+        documents = _json_documents(cleaned)
+        if len(documents) == 1:
+            return documents[0]
+        if len(documents) > 1:
+            raise ModelGenerationError("模型响应包含多个 JSON 对象，无法确定应使用哪一个")
         raise ModelGenerationError(
-            f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
-        ) from exc
+            "模型响应中没有找到唯一且完整的 JSON 对象"
+            f"（第 {direct_error.lineno} 行，第 {direct_error.colno} 列）"
+        ) from direct_error
 
 
 def _result_items(payload: Any) -> list[Mapping[str, Any]]:
@@ -206,6 +343,7 @@ def generate_model_records(
     model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
     if not model_name:
         raise ModelGenerationError("generation model name is required")
+    is_minimax_m3 = _is_official_minimax_m3(endpoint, model_name)
 
     temperature = float(config.get("temperature", 0.7))
     max_tokens = int(config.get("max_tokens", 1024))
@@ -246,9 +384,18 @@ def generate_model_records(
                         reasoning_detail=reasoning_detail,
                     ),
                     "temperature": temperature,
-                    "max_tokens": max_tokens,
                 }
-                if bool(config.get("json_mode", False)):
+                if is_minimax_m3:
+                    request_payload.update(
+                        reasoning_split=True,
+                        max_completion_tokens=max(
+                            max_tokens,
+                            MINIMAX_M3_MIN_COMPLETION_TOKENS,
+                        ),
+                    )
+                else:
+                    request_payload["max_tokens"] = max_tokens
+                if bool(config.get("json_mode", False)) and not is_minimax_m3:
                     request_payload["response_format"] = {"type": "json_object"}
 
                 last_error: Exception | None = None
@@ -264,7 +411,24 @@ def generate_model_records(
                         body = response.json()
                         if not isinstance(body, Mapping):
                             raise ModelGenerationError("model response body must be a JSON object")
-                        candidate_items = _result_items(_json_payload(_message_content(body)))
+                        try:
+                            candidate_items = _result_items(
+                                _json_payload(_message_content(body))
+                            )
+                        except ModelGenerationError as exc:
+                            logger.warning(
+                                "data process model response rejected task_id=%s model=%s "
+                                "finish_reason=%s response_chars=%s input_sensitive=%s "
+                                "output_sensitive=%s reason=%s",
+                                task_id,
+                                model_name,
+                                _response_finish_reason(body) or "missing",
+                                _response_content_length(body),
+                                bool(body.get("input_sensitive")),
+                                bool(body.get("output_sensitive")),
+                                str(exc),
+                            )
+                            raise
                         if len(candidate_items) < batch_count:
                             raise ModelGenerationError(
                                 "model response contains fewer result objects than requested: "
@@ -278,6 +442,8 @@ def generate_model_records(
                         ModelGenerationError,
                     ) as exc:
                         last_error = exc
+                        if not _is_retryable_generation_error(exc):
+                            break
 
                 if generated_items is None:
                     error_message = str(last_error or "model generation failed")[:2000]