Merge branch 'ft_wyt' of http://www.caoxiaozhu.com:13001/YG-Soft/YG_FT into ft_wyt

backend/app/modules/data_process/algorithms/parsers/pdf.py

@@ -104,7 +104,9 @@ def extract_pdf_page_texts(raw: bytes) -> tuple[PdfPageText, ...]:
         )
         has_text = True
     if not has_text:
-        raise ValueError("PDF contains no extractable text; scanned PDF requires OCR")
+        raise ValueError(
+            "PDF contains no extractable text; scanned or image-only PDF files are not supported"
+        )
     return tuple(pages)
 
 def _pdf_page_lines(page: PdfPageText) -> tuple[_PdfLine, ...]:

backend/app/modules/data_process/document_chunking.py

@@ -90,8 +90,10 @@ def _tokenizer() -> tiktoken.Encoding:
                         token, rank = line.split()
                         mergeable_ranks[base64.b64decode(token)] = int(rank)
 
-                # 构造 Encoding 对象
-                import tiktoken.core
+                # 构造 Encoding 对象（模块顶部已 import tiktoken，
+                # 此处不能再 import tiktoken.core，否则会把 tiktoken
+                # 变成局部变量，使函数开头的 tiktoken.get_encoding 抛
+                # UnboundLocalError）
                 return tiktoken.core.Encoding(
                     name="cl100k_base",
                     pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
@@ -260,9 +262,17 @@ def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
 
 @lru_cache(maxsize=1)
 def _document_converter():
-    from docling.document_converter import DocumentConverter
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+    from docling.document_converter import DocumentConverter, PdfFormatOption
 
-    return DocumentConverter()
+    pipeline_options = PdfPipelineOptions()
+    pipeline_options.do_ocr = False
+    return DocumentConverter(
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
+        }
+    )
 
 
 class _MarkdownSerializerProvider(ChunkingSerializerProvider):

backend/app/modules/data_process/generation.py

@@ -45,6 +45,25 @@ MINIMAX_M3_API_HOSTS = {"api.minimax.io", "api.minimaxi.com"}
 MINIMAX_M3_MIN_COMPLETION_TOKENS = 4096
 logger = logging.getLogger(__name__)
 
+# 问题表述风格规则：防止模型产出“请描述/请说明”式模板化问句。
+_QUESTION_STYLE_RULE = (
+    "各条问题必须覆盖不同的信息点并使用不同的句式，只替换关键词套用同一句式视为重复。"
+    "问题表述要像真实用户自然提出的问题：具体、口语化、直奔信息点，"
+    "避免“请描述”“请说明”“根据文档”等模板化开头，"
+    "也不要把原文句子直接改成问句；多条问题时交替使用直接疑问、场景式提问、追问式等句式。"
+    "表述示例（仅示意风格，不要照搬内容）："
+    "避免——“请描述系统的权限控制机制”；"
+    "推荐——“不同角色能看到的菜单不一样，平台是怎么控制的？”"
+)
+
+# 任务配置未提供提示语时的兜底，与前端内置默认提示语保持同等信息量。
+_DEFAULT_GENERATION_PROMPT = (
+    "你是一名专业的数据生成专家。请基于来源内容生成高质量、"
+    "可直接用于监督微调的问答数据：问题聚焦核心信息点、"
+    "表述像真实用户自然提出的问题，具体、口语化，多条问题使用不同句式；"
+    "答案严格依据来源内容，准确、完整、语言自然，不引入来源之外的信息。"
+)
+
 
 def _is_retryable_generation_error(exc: Exception) -> bool:
     if isinstance(exc, _TerminalModelGenerationError):
@@ -310,11 +329,11 @@ def _prompt_messages(
         )
     schema_instruction = (
         f"必须只返回 JSON 对象，格式为 {schema}；items 必须包含 {count} 条。"
-        f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条，"
-        "各条必须使用不同的提问角度和表述，避免重复。"
-        f"{output_rule}不要输出 Markdown 代码围栏或 JSON 之外的说明。"
+        f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条。"
+        f"{_QUESTION_STYLE_RULE}{output_rule}"
+        "不要输出 Markdown 代码围栏或 JSON 之外的说明。"
     )
-    base_prompt = normalize_text(prompt) or "请根据来源内容生成可用于监督微调的问答数据。"
+    base_prompt = normalize_text(prompt) or _DEFAULT_GENERATION_PROMPT
     if "{{ content }}" in base_prompt:
         user_prompt = base_prompt.replace("{{ content }}", content)
         return [

backend/tests/test_data_process_algorithms.py

@@ -654,7 +654,10 @@ def test_office_zip_bomb_and_invalid_pdf_are_rejected_before_parsing() -> None:
     blank_writer = PdfWriter()
     blank_writer.add_blank_page(width=612, height=792)
     blank_writer.write(blank_pdf)
-    with pytest.raises(ValueError, match="scanned PDF requires OCR"):
+    with pytest.raises(
+        ValueError,
+        match="scanned or image-only PDF files are not supported",
+    ):
         parse_text_content(blank_pdf.getvalue(), filename="scanned.pdf")
 
     aes_pdf_without_open_password = parse_text_content(

backend/tests/test_data_process_generation.py

@@ -35,6 +35,8 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
         assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
         assert "你正在生成标准监督微调问答数据" in payload["messages"][0]["content"]
         assert "禁止输出分析、推理过程" in payload["messages"][0]["content"]
+        assert "真实用户自然提出的问题" in payload["messages"][0]["content"]
+        assert "模板化开头" in payload["messages"][0]["content"]
         return httpx.Response(
             200,
             json={
@@ -88,6 +90,59 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
     assert progress_updates == [(1, 1)]
 
 
+def test_generate_model_records_falls_back_to_rich_default_prompt() -> None:
+    captured: dict[str, str] = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        payload = json.loads(request.content)
+        captured["system"] = payload["messages"][0]["content"]
+        captured["user"] = payload["messages"][1]["content"]
+        return httpx.Response(
+            200,
+            json={
+                "choices": [
+                    {
+                        "message": {
+                            "content": json.dumps(
+                                {
+                                    "items": [
+                                        {
+                                            "instruction": "平台如何控制不同角色的菜单可见性？",
+                                            "input": "",
+                                            "output": "按角色分配权限。",
+                                        }
+                                    ]
+                                },
+                                ensure_ascii=False,
+                            )
+                        }
+                    }
+                ]
+            },
+        )
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    records = generate_model_records(
+        [{"id": "preview-1", "edited_content": "平台按角色分配菜单权限。"}],
+        model={
+            "name": "Qwen",
+            "online_model_name": "qwen-plus",
+            "api_url": "model.example",
+        },
+        config={},
+        task_id="task-fallback",
+        split={"train": 100, "validation": 0, "test": 0},
+        qa_pairs_per_item=1,
+        client=client,
+    )
+
+    assert len(records) == 1
+    assert records[0]["status"] == "valid"
+    assert "数据生成专家" in captured["system"]
+    assert "真实用户自然提出的问题" in captured["system"]
+    assert "平台按角色分配菜单权限。" in captured["user"]
+
+
 def test_generate_model_records_builds_native_dpo_pair() -> None:
     def handler(request: httpx.Request) -> httpx.Response:
         payload = json.loads(request.content)

backend/tests/test_document_chunking.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
 
+import sys
+from types import ModuleType, SimpleNamespace
+
 from llama_index.core.embeddings import MockEmbedding
 
 from app.modules.data_process.document_chunking import (
     DocumentChunk,
     _compact_with_offsets,
+    _document_converter,
     _project_layout_span,
     chunk_fixed_text,
     chunk_semantic_text,
@@ -42,6 +46,48 @@ def test_semantic_splitter_uses_llamaindex_and_reapplies_maximum_size() -> None:
     assert "".join(chunk.original_content for chunk in chunks) == text
 
 
+def test_layout_converter_disables_ocr(monkeypatch) -> None:
+    class FakePipelineOptions:
+        def __init__(self) -> None:
+            self.do_ocr = True
+
+    class FakePdfFormatOption:
+        def __init__(self, *, pipeline_options) -> None:
+            self.pipeline_options = pipeline_options
+
+    class FakeDocumentConverter:
+        def __init__(self, *, format_options) -> None:
+            self.format_options = format_options
+
+    docling_module = ModuleType("docling")
+    docling_module.__path__ = []
+    document_converter_module = ModuleType("docling.document_converter")
+    document_converter_module.DocumentConverter = FakeDocumentConverter
+    document_converter_module.PdfFormatOption = FakePdfFormatOption
+    datamodel_module = ModuleType("docling.datamodel")
+    datamodel_module.__path__ = []
+    base_models_module = ModuleType("docling.datamodel.base_models")
+    base_models_module.InputFormat = SimpleNamespace(PDF="pdf")
+    pipeline_options_module = ModuleType("docling.datamodel.pipeline_options")
+    pipeline_options_module.PdfPipelineOptions = FakePipelineOptions
+    for name, module in {
+        "docling": docling_module,
+        "docling.document_converter": document_converter_module,
+        "docling.datamodel": datamodel_module,
+        "docling.datamodel.base_models": base_models_module,
+        "docling.datamodel.pipeline_options": pipeline_options_module,
+    }.items():
+        monkeypatch.setitem(sys.modules, name, module)
+
+    _document_converter.cache_clear()
+    try:
+        converter = _document_converter()
+        options = converter.format_options["pdf"].pipeline_options
+        assert options.do_ocr is False
+    finally:
+        _document_converter.cache_clear()
+
+
 def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() -> None:
     source = "标题\n第一条 这是正文。\n第二条 后续正文。"
     compact_source, offsets = _compact_with_offsets(source)