YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 支持单项生成五十条数据

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-27 09:11:51 +08:00
parent 25d75f40c7
commit 9428c6b785
7 changed files with 400 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
backend/app/modules/data_process/generation.py
View File

@@ -12,6 +12,10 @@ from urllib.parse import urlsplit, urlunsplit
import httpx
from app.modules.data_process.algorithms import normalize_text, stable_split_assignments
from app.modules.data_process.constants import (
MAX_QA_PAIRS_PER_ITEM,
MODEL_GENERATION_BATCH_SIZE,
)
class ModelGenerationError(ValueError):
@@ -96,10 +100,20 @@ def _result_items(payload: Any) -> list[Mapping[str, Any]]:
return items
def _prompt_messages(prompt: str, content: str, count: int) -> list[dict[str, str]]:
def _prompt_messages(
prompt: str,
content: str,
count: int,
*,
start_index: int,
total_count: int,
) -> list[dict[str, str]]:
end_index = start_index + count - 1
schema_instruction = (
f"必须只返回 JSON 对象，格式为 {{\"items\":[{{\"instruction\":\"...\","
f"\"input\":\"...\",\"output\":\"...\"}}]}}；items 必须包含 {count} 条。"
f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条，"
"各条必须使用不同的提问角度和表述，避免重复。"
"instruction 和 output 不得为空，不要输出 Markdown 代码围栏或分析过程。"
)
base_prompt = (
@@ -131,11 +145,14 @@ def generate_model_records(
) -> list[dict[str, Any]]:
"""调用 OpenAI 兼容接口，将预览切片生成标准训练记录。
单条调用失败会产生可人工修复的 invalid 结果，不会丢弃整批任务。
每个切片按安全批次调用模型；失败批次会产生一条可人工修复的
invalid 结果，已经成功的批次不会丢失。
"""
if not 1 <= qa_pairs_per_item <= 5:
raise ModelGenerationError("qa_pairs_per_item must be in [1, 5]")
if not 1 <= qa_pairs_per_item <= MAX_QA_PAIRS_PER_ITEM:
raise ModelGenerationError(
f"qa_pairs_per_item must be in [1, {MAX_QA_PAIRS_PER_ITEM}]"
)
endpoint = chat_completions_url(str(model.get("api_url") or ""))
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
if not model_name:
@@ -161,84 +178,126 @@ def generate_model_records(
content = normalize_text(
str(item.get("edited_content") or item.get("original_content") or "")
)
request_payload: dict[str, Any] = {
"model": model_name,
"messages": _prompt_messages(
str(config.get("generation_prompt") or ""),
content,
qa_pairs_per_item,
),
"temperature": temperature,
"max_tokens": max_tokens,
}
if bool(config.get("json_mode", False)):
request_payload["response_format"] = {"type": "json_object"}
last_error: Exception | None = None
generated_items: list[Mapping[str, Any]] | None = None
for _ in range(retries + 1):
try:
response = http_client.post(endpoint, headers=headers, json=request_payload)
response.raise_for_status()
body = response.json()
if not isinstance(body, Mapping):
raise ModelGenerationError("model response body must be a JSON object")
generated_items = _result_items(_json_payload(_message_content(body)))
break
except (httpx.HTTPError, json.JSONDecodeError, ModelGenerationError) as exc:
last_error = exc
if generated_items is None:
error_message = str(last_error or "model generation failed")[:2000]
result_id = f"result_{hashlib.sha256(f'{preview_id}:error'.encode()).hexdigest()[:16]}"
results.append(
{
"id": result_id,
"preview_item_id": preview_id,
"instruction": "模型生成失败，请人工补充",
"input": content,
"output": "",
"original_instruction": "模型生成失败，请人工补充",
"original_input": content,
"original_output": "",
"status": "invalid",
"error": error_message,
"split": "train",
}
for batch_offset in range(0, qa_pairs_per_item, MODEL_GENERATION_BATCH_SIZE):
batch_count = min(
MODEL_GENERATION_BATCH_SIZE,
qa_pairs_per_item - batch_offset,
)
if on_progress:
on_progress(item_index + 1, total_items)
continue
batch_start = batch_offset + 1
batch_end = batch_offset + batch_count
request_payload: dict[str, Any] = {
"model": model_name,
"messages": _prompt_messages(
str(config.get("generation_prompt") or ""),
content,
batch_count,
start_index=batch_start,
total_count=qa_pairs_per_item,
),
"temperature": temperature,
"max_tokens": max_tokens,
}
if bool(config.get("json_mode", False)):
request_payload["response_format"] = {"type": "json_object"}
for variant_index, value in enumerate(generated_items[:qa_pairs_per_item]):
instruction = normalize_text(str(value.get("instruction") or value.get("question") or ""))
input_text = normalize_text(str(value.get("input") or value.get("context") or ""))
output = normalize_text(
str(
value.get("output")
or value.get("answer")
or value.get("response")
or ""
last_error: Exception | None = None
generated_items: list[Mapping[str, Any]] | None = None
for _ in range(retries + 1):
try:
response = http_client.post(
endpoint,
headers=headers,
json=request_payload,
)
response.raise_for_status()
body = response.json()
if not isinstance(body, Mapping):
raise ModelGenerationError(
"model response body must be a JSON object"
)
candidate_items = _result_items(
_json_payload(_message_content(body))
)
if len(candidate_items) < batch_count:
raise ModelGenerationError(
"model response contains fewer result objects than requested: "
f"expected {batch_count}, got {len(candidate_items)}"
)
generated_items = candidate_items
break
except (
httpx.HTTPError,
json.JSONDecodeError,
ModelGenerationError,
) as exc:
last_error = exc
if generated_items is None:
error_message = str(last_error or "model generation failed")[:2000]
failure_instruction = (
f"模型生成失败，请人工补充（第 {batch_start}-{batch_end} 条）"
)
result_id = (
"result_"
f"{hashlib.sha256(f'{preview_id}:error:{batch_start}'.encode()).hexdigest()[:16]}"
)
results.append(
{
"id": result_id,
"preview_item_id": preview_id,
"instruction": failure_instruction,
"input": content,
"output": "",
"original_instruction": failure_instruction,
"original_input": content,
"original_output": "",
"status": "invalid",
"error": error_message,
"split": "train",
}
)
continue
for batch_index, value in enumerate(generated_items[:batch_count]):
variant_index = batch_offset + batch_index
instruction = normalize_text(
str(value.get("instruction") or value.get("question") or "")
)
input_text = normalize_text(
str(value.get("input") or value.get("context") or "")
)
output = normalize_text(
str(
value.get("output")
or value.get("answer")
or value.get("response")
or ""
)
)
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
result_id = (
f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
)
valid = bool(instruction and output)
results.append(
{
"id": result_id,
"preview_item_id": preview_id,
"instruction": instruction,
"input": input_text,
"output": output,
"original_instruction": instruction,
"original_input": input_text,
"original_output": output,
"status": "valid" if valid else "invalid",
"error": (
None
if valid
else "model result is missing instruction or output"
),
"split": "train",
}
)
)
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
valid = bool(instruction and output)
results.append(
{
"id": result_id,
"preview_item_id": preview_id,
"instruction": instruction,
"input": input_text,
"output": output,
"original_instruction": instruction,
"original_input": input_text,
"original_output": output,
"status": "valid" if valid else "invalid",
"error": None if valid else "model result is missing instruction or output",
"split": "train",
}
)
if on_progress:
on_progress(item_index + 1, total_items)
finally: