feat(data-process): 支持单项生成五十条数据
This commit is contained in:
@@ -34,6 +34,8 @@ from openpyxl.utils.cell import range_boundaries
|
||||
from pptx import Presentation
|
||||
from pypdf import PdfReader
|
||||
|
||||
from app.modules.data_process.constants import MAX_QA_PAIRS_PER_ITEM
|
||||
|
||||
TextFormat = Literal[
|
||||
"json",
|
||||
"jsonl",
|
||||
@@ -2659,8 +2661,10 @@ def generate_standard_records(
|
||||
LLM 响应解析后的统一落库步骤。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
||||
if not 1 <= qa_pairs_per_item <= MAX_QA_PAIRS_PER_ITEM:
|
||||
raise ValueError(
|
||||
f"qa_pairs_per_item must be in [1, {MAX_QA_PAIRS_PER_ITEM}]"
|
||||
)
|
||||
prefixes = (
|
||||
"请结合实际情况说明:",
|
||||
"请用通俗易懂的方式说明:",
|
||||
@@ -2677,7 +2681,13 @@ def generate_standard_records(
|
||||
variant_instruction = instruction
|
||||
if variant_index:
|
||||
if semantic_enrichment:
|
||||
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
||||
prefix = prefixes[variant_index % len(prefixes)]
|
||||
if variant_index >= len(prefixes):
|
||||
prefix = (
|
||||
f"{prefix.removesuffix(':')}"
|
||||
f"(问法 {variant_index + 1}):"
|
||||
)
|
||||
variant_instruction = f"{prefix}{instruction}"
|
||||
else:
|
||||
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
||||
raw_id = f"{preview_id}:{variant_index + 1}"
|
||||
|
||||
Reference in New Issue
Block a user