YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增外部数据源拉取与 DPO 输出格式支持

Browse Source 
- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源
- 新增 DPO (Direct Preference Optimization) 输出类型
- 支持 chosen/rejected 字段的编辑、校验和发布
- 完善数据预处理切分逻辑和元数据管理
- 移除 OCR 扫描 PDF 功能，保持基础文本解析能力

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
caoxiaozhu
2026-08-11 14:17:45 +08:00
parent f809825a7d
commit 5f6e7523cf
26 changed files with 927 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -50,16 +50,15 @@ YG_FT/
## 前后端一键启动
首次使用前，请先按下方“后端启动”和“前端启动”说明安装依赖，并确保
PostgreSQL 已可用。之后在项目根目录执行：
首次使用前请确保前端依赖已安装、PostgreSQL 已可用。之后在项目根目录执行：
```bash
bash ./start.sh
```
脚本会同时启动前端 `http://localhost:16801` 和后端
`http://127.0.0.1:17861`，按 `Ctrl+C` 会同时停止两个服务。脚本只负责
启动前后端，不会自动安装依赖，也不会启动 PostgreSQL、Redis 或算力服务。
脚本会自动补装后端 `requirements.txt`，然后同时启动前端 `http://localhost:16801` 和后端
`http://127.0.0.1:17861`，按 `Ctrl+C` 会同时停止两个服务。脚本不会自动安装
前端依赖，也不会启动 PostgreSQL、Redis 或算力服务。
仅检查依赖和端口而不启动服务：

65
backend/app/api/v1/endpoints/data_process.py
View File

@@ -16,7 +16,7 @@ from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Literal
from urllib.parse import quote, urlsplit
from urllib.parse import parse_qs, quote, urlsplit
import httpx
import psycopg
@@ -676,8 +676,10 @@ def _run_generation(
qa_pairs_per_item=int(pairs or 1),
on_progress=report_progress,
)
elif output_type == "reasoning":
raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
elif output_type in {"reasoning", "dpo"}:
if output_type == "reasoning":
raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
raise InvalidStateError("DPO 输出必须配置可用的数据生成模型")
else:
generated = generate_standard_records(
preview_items,
@@ -1077,7 +1079,10 @@ async def upload_source_files(
with api_errors():
task = store.get_task(task_id)
process_type = str(task["process_type"])
if process_type == "external":
source_mode = str(
_value(task.get("config") or {}, "source_mode", "sourceMode", "local")
)
if process_type == "external" or source_mode == "external":
raise InvalidStateError(
"external tasks must import data through the external source endpoint"
)
@@ -1375,6 +1380,10 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
raise fail(501, f"external data source type is not supported: {payload.type}")
if parsed_url.username or parsed_url.password:
raise fail(400, "database credentials must use the account and password fields")
if set(parse_qs(parsed_url.query)) & {
"password", "secret", "token", "api_key", "user", "username"
}:
raise fail(400, "database URL query must not contain credentials")
if payload.auth_mode not in {"none", "basic"}:
raise fail(400, "PostgreSQL supports only none or basic authentication")
if payload.auth_mode == "basic" and not payload.username:
@@ -1413,10 +1422,14 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
"set DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB=true only in a trusted deployment",
)
kwargs: dict[str, Any] = {
"connect_timeout": 5,
"connect_timeout": payload.connect_timeout_seconds,
"row_factory": dict_row,
"application_name": "yg-ft-data-process-readonly",
"options": "-c default_transaction_read_only=on -c statement_timeout=30000",
"options": (
"-c default_transaction_read_only=on "
f"-c statement_timeout={payload.statement_timeout_seconds * 1000}"
),
"sslmode": payload.ssl_mode,
}
if payload.auth_mode == "basic" and payload.username:
kwargs["user"] = payload.username
@@ -1425,6 +1438,17 @@ def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Con
return psycopg.connect(payload.url, **kwargs)
def _assert_external_source_task(task: dict[str, Any]) -> None:
if str(task.get("process_type")) == "external":
return
config = task.get("config") or {}
source_mode = str(_value(config, "source_mode", "sourceMode", "local"))
if str(task.get("process_type")) != "structured" or source_mode != "external":
raise InvalidStateError(
"external source access requires a structured task with source_mode=external"
)
@router.post("/{task_id}/external/test")
def test_external_source(
task_id: str,
@@ -1433,10 +1457,7 @@ def test_external_source(
) -> dict[str, Any]:
with api_errors():
task = store.get_task(task_id)
if str(task.get("process_type")) != "external":
raise InvalidStateError(
"external source access requires an external data processing task"
)
_assert_external_source_task(task)
try:
with _external_postgres_connection(payload) as conn:
conn.execute("SELECT 1 AS ok").fetchone()
@@ -1462,12 +1483,14 @@ def pull_external_source(
raise fail(400, "a read-only SELECT or WITH query is required for external pull")
with api_errors():
task = store.get_task(task_id)
if str(task.get("process_type")) != "external":
raise InvalidStateError("external pull requires an external data processing task")
_assert_external_source_task(task)
try:
with _external_postgres_connection(payload) as conn:
conn.execute("SET TRANSACTION READ ONLY")
conn.execute("SET LOCAL statement_timeout = '30s'")
conn.execute(
"SELECT set_config('statement_timeout', %s, true)",
(f"{payload.statement_timeout_seconds}s",),
)
cursor = conn.execute(query)
rows: list[dict[str, Any]] = []
content_parts: list[str] = []
@@ -1518,6 +1541,9 @@ def pull_external_source(
"external_type": payload.type,
"external_host": urlsplit(payload.url).hostname,
"external_limit": payload.limit,
"external_ssl_mode": payload.ssl_mode,
"external_connect_timeout_seconds": payload.connect_timeout_seconds,
"external_statement_timeout_seconds": payload.statement_timeout_seconds,
},
}
],
@@ -1566,7 +1592,10 @@ def _prepare_preview_items(
for index, source in enumerate(sources):
source_format = str(source.get("file_format") or "").lower()
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
needs_layout_raw = (
is_unstructured
and chunk_method == "layout_hybrid"
)
needs_pdf_noise = (
is_unstructured
and not needs_layout_raw
@@ -2038,6 +2067,8 @@ def restore_result(
"instruction": current.get("original_instruction") or current.get("instruction") or "",
"input": current.get("original_input") or current.get("input") or "",
"output": current.get("original_output") or current.get("output") or "",
"chosen": current.get("original_chosen") or current.get("chosen") or "",
"rejected": current.get("original_rejected") or current.get("rejected") or "",
}
preview_id = current.get("preview_item_id")
source_content = ""
@@ -2070,6 +2101,8 @@ def restore_result(
"instruction": restored["instruction"],
"input": restored["input"],
"output": restored["output"],
"chosen": restored["chosen"],
"rejected": restored["rejected"],
"quality_score": asdict(quality),
"expected_updated_at": current.get("updated_at"),
},
@@ -2141,13 +2174,15 @@ def _generate_result_replacement(
).strip().lower()
previous_instruction = str(current.get("instruction") or "")[:1000]
previous_output = str(current.get("output") or "")[:1000]
previous_rejected = str(current.get("rejected") or "")[:1000]
base_prompt = str(
_value(config, "generation_prompt", "generationPrompt", "") or ""
)
regeneration_instruction = (
"这是一次失败结果的重新生成。请使用新的提问角度和表达，"
"不要复述旧结果。旧问题："
f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
f"{previous_instruction or '无'}；旧优选答案：{previous_output or '无'}；"
f"旧拒选答案：{previous_rejected or '无'}。"
)
runtime_config = {
**config,

9
backend/app/db/sql/002_data_process.sql
View File

@@ -268,9 +268,13 @@ CREATE TABLE IF NOT EXISTS data_process_results (
instruction TEXT NOT NULL,
input TEXT NOT NULL DEFAULT '',
output TEXT NOT NULL,
chosen TEXT NOT NULL DEFAULT '',
rejected TEXT NOT NULL DEFAULT '',
original_instruction TEXT,
original_input TEXT,
original_output TEXT,
original_chosen TEXT,
original_rejected TEXT,
status VARCHAR(20) NOT NULL DEFAULT 'valid'
CHECK (status IN ('valid', 'modified', 'invalid')),
error TEXT,
@@ -280,6 +284,11 @@ CREATE TABLE IF NOT EXISTS data_process_results (
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS chosen TEXT NOT NULL DEFAULT '';
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS rejected TEXT NOT NULL DEFAULT '';
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_chosen TEXT;
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_rejected TEXT;
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
ON data_process_results(task_id, status, id);
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split

57
backend/app/modules/data_process/document_chunking.py
View File

@@ -57,7 +57,62 @@ def _sentence_chunks(text: str) -> list[str]:
@lru_cache(maxsize=1)
def _tokenizer() -> tiktoken.Encoding:
return tiktoken.get_encoding("cl100k_base")
"""加载 cl100k_base 编码器，优先在线下载，失败时使用本地缓存以支持离线环境。"""
import os
import base64
# 先设置缓存目录环境变量
offline_cache = os.path.expanduser("~/.cache/tiktoken")
os.environ.setdefault("TIKTOKEN_CACHE_DIR", offline_cache)
try:
# 尝试标准方式加载
return tiktoken.get_encoding("cl100k_base")
except Exception:
# 如果失败，尝试手动从本地文件构造
try:
from pathlib import Path
local_file = Path(offline_cache) / "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
if not local_file.exists():
# 尝试另一个可能的文件名
local_file = Path(offline_cache) / "cl100k_base.tiktoken"
if local_file.exists():
# 读取 BPE 文件内容
with open(local_file, "rb") as f:
contents = f.read()
# 解析 BPE 文件
mergeable_ranks = {}
for line in contents.splitlines():
if line:
token, rank = line.split()
mergeable_ranks[base64.b64decode(token)] = int(rank)
# 构造 Encoding 对象
import tiktoken.core
return tiktoken.core.Encoding(
name="cl100k_base",
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
mergeable_ranks=mergeable_ranks,
special_tokens={
"<|endoftext|>": 100257,
"<|fim_prefix|>": 100258,
"<|fim_middle|>": 100259,
"<|fim_suffix|>": 100260,
"<|endofprompt|>": 100276,
},
)
except Exception:
pass
raise RuntimeError(
f"无法加载 cl100k_base 编码器\n"
f"请确保以下任一条件满足:\n"
f"1. 服务器可以访问网络\n"
f"2. 本地存在缓存文件: {offline_cache}/9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
)
def _text_chunks(

BIN
backend/app/modules/data_process/document_chunking.txt Normal file
View File

Binary file not shown.

64
backend/app/modules/data_process/generation.py
View File

@@ -29,7 +29,12 @@ class _TerminalModelGenerationError(ModelGenerationError):
OUTPUT_TYPE_STANDARD = "standard"
OUTPUT_TYPE_REASONING = "reasoning"
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
OUTPUT_TYPE_DPO = "dpo"
SUPPORTED_OUTPUT_TYPES = {
OUTPUT_TYPE_STANDARD,
OUTPUT_TYPE_REASONING,
OUTPUT_TYPE_DPO,
}
REASONING_DETAIL_NORMAL = "normal"
REASONING_DETAIL_DETAILED = "detailed"
SUPPORTED_REASONING_DETAILS = {
@@ -285,6 +290,18 @@ def _prompt_messages(
"这是思维链输出模式，即使其他提示语要求省略分析，也不得省略 reasoning。"
"不要自行添加 <think> 标签，系统会在保存时统一组装。"
)
elif output_type == OUTPUT_TYPE_DPO:
schema = (
'{"items":[{"instruction":"...","input":"...",'
'"chosen":"...","rejected":"..."}]}'
)
output_rule = (
"你正在生成用于直接偏好优化（DPO）的成对偏好数据。"
"instruction、chosen 和 rejected 均不得为空；chosen 必须是忠于来源、"
"准确完整的优选回答，rejected 必须是表面合理但存在明确质量缺陷的拒选回答。"
"两者不得相同；rejected 不得包含违法危险内容，也不得用空白、乱码或无关文本凑数。"
"不要输出分析过程或 <think> 标签。"
)
else:
schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}'
output_rule = (
@@ -461,9 +478,13 @@ def generate_model_records(
"instruction": failure_instruction,
"input": content,
"output": "",
"chosen": "",
"rejected": "",
"original_instruction": failure_instruction,
"original_input": content,
"original_output": "",
"original_chosen": "",
"original_rejected": "",
"status": "invalid",
"error": error_message,
"split": "train",
@@ -479,6 +500,8 @@ def generate_model_records(
input_text = normalize_text(
str(value.get("input") or value.get("context") or "")
)
chosen = ""
rejected = ""
if output_type == OUTPUT_TYPE_REASONING:
reasoning = normalize_text(
re.sub(
@@ -508,6 +531,36 @@ def generate_model_records(
)
valid = bool(instruction and reasoning and answer)
missing_error = "model result is missing instruction, reasoning or answer"
elif output_type == OUTPUT_TYPE_DPO:
chosen = normalize_text(str(value.get("chosen") or ""))
rejected = normalize_text(str(value.get("rejected") or ""))
chosen = normalize_text(
re.sub(
r"<think>[\s\S]*?(?:</think>|$)",
"",
chosen,
flags=re.IGNORECASE,
)
)
rejected = normalize_text(
re.sub(
r"<think>[\s\S]*?(?:</think>|$)",
"",
rejected,
flags=re.IGNORECASE,
)
)
output = chosen
valid = bool(
instruction
and chosen
and rejected
and chosen.strip() != rejected.strip()
)
missing_error = (
"model result is missing instruction, chosen or rejected, "
"or chosen equals rejected"
)
else:
output = normalize_text(
str(
@@ -527,7 +580,10 @@ def generate_model_records(
)
valid = bool(instruction and output)
missing_error = "model result is missing instruction or output"
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
raw_id = (
f"{preview_id}:{variant_index + 1}:{instruction}:"
f"{output}:{rejected}"
)
result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
results.append(
{
@@ -536,9 +592,13 @@ def generate_model_records(
"instruction": instruction,
"input": input_text,
"output": output,
"chosen": chosen,
"rejected": rejected,
"original_instruction": instruction,
"original_input": input_text,
"original_output": output,
"original_chosen": chosen,
"original_rejected": rejected,
"status": "valid" if valid else "invalid",
"error": (None if valid else missing_error),
"split": "train",

112
backend/app/modules/data_process/store.py
View File

@@ -140,6 +140,12 @@ def _reasoning_output_is_valid(value: Any) -> bool:
)
def _dpo_fields_are_valid(row: dict[str, Any]) -> bool:
chosen = str(row.get("chosen") or "").strip()
rejected = str(row.get("rejected") or "").strip()
return bool(chosen and rejected and chosen != rejected)
def _preview_config_value(config: dict[str, Any], key: str, default: Any) -> Any:
if key in config:
return config[key]
@@ -846,7 +852,11 @@ class DataProcessStore:
)
instruction = str(record.get("instruction") or raw.get("instruction") or "")
input_text = str(record.get("input") or raw.get("input") or "")
output = str(record.get("output") or raw.get("output") or "")
chosen = str(raw.get("chosen") or "")
rejected = str(raw.get("rejected") or "")
output = str(
record.get("output") or raw.get("output") or chosen or ""
)
split = str(record.get("split") or raw.get("split") or "") or None
status = str(record.get("status") or "valid")
if status not in {"valid", "modified", "invalid"}:
@@ -856,10 +866,11 @@ class DataProcessStore:
"""
INSERT INTO data_process_results
(id, task_id, preview_item_id, instruction, input, output,
original_instruction, original_input, original_output, status,
chosen, rejected, original_instruction, original_input,
original_output, original_chosen, original_rejected, status,
error, split, quality_score, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
NULL, %s, '{}', %s, %s)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, NULL, %s, '{}', %s, %s)
""",
(
result_id,
@@ -868,9 +879,13 @@ class DataProcessStore:
instruction,
input_text,
output,
chosen,
rejected,
instruction,
input_text,
output,
chosen,
rejected,
status,
split,
created_at,
@@ -2024,9 +2039,11 @@ class DataProcessStore:
"""
INSERT INTO data_process_results
(id, task_id, preview_item_id, instruction, input, output,
original_instruction, original_input, original_output, status, error,
chosen, rejected, original_instruction, original_input,
original_output, original_chosen, original_rejected, status, error,
split, quality_score, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s)
""",
(
result.get("id") or new_id("dpr"),
@@ -2035,9 +2052,13 @@ class DataProcessStore:
result.get("instruction") or "",
result.get("input") or "",
result.get("output") or "",
result.get("chosen") or "",
result.get("rejected") or "",
result.get("original_instruction", result.get("instruction") or ""),
result.get("original_input", result.get("input") or ""),
result.get("original_output", result.get("output") or ""),
result.get("original_chosen", result.get("chosen") or ""),
result.get("original_rejected", result.get("rejected") or ""),
result.get("status") or "valid",
result.get("error"),
result.get("split"),
@@ -2121,7 +2142,7 @@ class DataProcessStore:
rows = conn.execute(
"""
SELECT status, instruction, output
SELECT status, instruction, output, chosen, rejected
FROM data_process_results
WHERE task_id=%s
""",
@@ -2139,6 +2160,10 @@ class DataProcessStore:
_task_output_type(task) == "reasoning"
and not _reasoning_output_is_valid(row.get("output"))
)
or (
_task_output_type(task) == "dpo"
and not _dpo_fields_are_valid(row)
)
)
if invalid_count:
raise InvalidStateError(
@@ -2210,7 +2235,9 @@ class DataProcessStore:
def update_result(
self, task_id: str, result_id: str, payload: dict[str, Any]
) -> dict[str, Any]:
allowed = {"instruction", "input", "output", "quality_score"}
allowed = {
"instruction", "input", "output", "chosen", "rejected", "quality_score"
}
values = {key: value for key, value in payload.items() if key in allowed}
if "quality_score" in values:
values["quality_score"] = json_dumps(values["quality_score"])
@@ -2232,20 +2259,28 @@ class DataProcessStore:
current_updated_at = _serialize_value(current.get("updated_at"))
if expected_updated_at and expected_updated_at != current_updated_at:
raise ConflictError("data process result was modified by another request")
output_type = _task_output_type(task)
if output_type == "dpo" and "chosen" in values:
values["output"] = values["chosen"]
merged = {**current, **values}
quality = payload.get("quality_score") or {}
instruction_valid = bool(str(merged.get("instruction") or "").strip())
output_valid = bool(str(merged.get("output") or "").strip())
reasoning_valid = (
_task_output_type(task) != "reasoning"
output_type != "reasoning"
or _reasoning_output_is_valid(merged.get("output"))
)
hard_valid = instruction_valid and output_valid and reasoning_valid
dpo_valid = output_type != "dpo" or _dpo_fields_are_valid(merged)
hard_valid = instruction_valid and output_valid and reasoning_valid and dpo_valid
quality_valid = bool(quality.get("is_valid", hard_valid))
changed = any(
str(merged.get(field) or "")
!= str(merged.get(f"original_{field}") or "")
for field in ("instruction", "input", "output")
for field in (
("instruction", "input", "chosen", "rejected")
if output_type == "dpo"
else ("instruction", "input", "output")
)
)
status = "invalid" if not hard_valid or not quality_valid else (
"modified" if changed else "valid"
@@ -2255,6 +2290,8 @@ class DataProcessStore:
format_error = (
"思维链输出必须包含非空的 <think>...</think> 推理过程和最终答案"
if instruction_valid and output_valid and not reasoning_valid
else "DPO 输出必须包含不同的非空 Chosen 和 Rejected 回答"
if instruction_valid and not dpo_valid
else "Instruction 和 Output 不能为空"
if not instruction_valid or not output_valid
else None
@@ -2318,18 +2355,23 @@ class DataProcessStore:
instruction = str(replacement.get("instruction") or "").strip()
input_text = str(replacement.get("input") or "").strip()
output = str(replacement.get("output") or "").strip()
chosen = str(replacement.get("chosen") or "").strip()
rejected = str(replacement.get("rejected") or "").strip()
quality_score = replacement.get("quality_score") or {}
if not instruction or not output or not bool(quality_score.get("is_valid")):
raise InvalidStateError("regenerated result did not pass quality validation")
if _task_output_type(task) == "reasoning" and not _reasoning_output_is_valid(output):
raise InvalidStateError("regenerated reasoning result has an invalid output format")
if _task_output_type(task) == "dpo" and not _dpo_fields_are_valid(replacement):
raise InvalidStateError("regenerated DPO result has invalid preference fields")
now = utcnow()
row = conn.execute(
"""
UPDATE data_process_results
SET instruction=%s, input=%s, output=%s,
SET instruction=%s, input=%s, output=%s, chosen=%s, rejected=%s,
original_instruction=%s, original_input=%s, original_output=%s,
original_chosen=%s, original_rejected=%s,
status='valid', error=NULL, quality_score=%s, updated_at=%s
WHERE id=%s AND task_id=%s
RETURNING *
@@ -2338,9 +2380,13 @@ class DataProcessStore:
instruction,
input_text,
output,
chosen,
rejected,
instruction,
input_text,
output,
chosen,
rejected,
json_dumps(quality_score),
now,
result_id,
@@ -2434,6 +2480,10 @@ class DataProcessStore:
_task_output_type(task) == "reasoning"
and not _reasoning_output_is_valid(row.get("output"))
)
or (
_task_output_type(task) == "dpo"
and not _dpo_fields_are_valid(row)
)
)
if invalid_count:
raise InvalidStateError(f"task contains {invalid_count} invalid results")
@@ -2449,15 +2499,27 @@ class DataProcessStore:
requested_split,
seed=task_id,
)
records = [
{
"instruction": row["instruction"],
"input": row["input"],
"output": row["output"],
"split": assignment,
}
for row, assignment in zip(rows, assignments, strict=True)
]
if _task_output_type(task) == "dpo":
records = [
{
"instruction": row["instruction"],
"input": row["input"],
"chosen": row["chosen"],
"rejected": row["rejected"],
"split": assignment,
}
for row, assignment in zip(rows, assignments, strict=True)
]
else:
records = [
{
"instruction": row["instruction"],
"input": row["input"],
"output": row["output"],
"split": assignment,
}
for row, assignment in zip(rows, assignments, strict=True)
]
split_order = ("train", "validation", "test")
split_counts = {
split_name: assignments.count(split_name) for split_name in split_order
@@ -2498,7 +2560,11 @@ class DataProcessStore:
"reasoning_detail": _task_reasoning_detail(task),
"source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)],
"source_result_ids": source_result_ids,
"format": payload.get("format") or "alpaca_jsonl",
"format": (
"dpo"
if _task_output_type(task) == "dpo"
else payload.get("format") or "alpaca_jsonl"
),
"split": requested_split,
}
@@ -2741,7 +2807,7 @@ class DataProcessStore:
record["split"],
record["instruction"],
record["input"],
record["output"],
record.get("output") or record.get("chosen") or "",
json_dumps(
{
**record,

33
backend/app/schemas/data_process.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
from enum import StrEnum
from typing import Any, Literal
from urllib.parse import parse_qs, urlsplit
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
@@ -15,6 +16,31 @@ def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, defa
def _validate_process_config(config: dict[str, Any]) -> None:
output_type = _config_value(config, "output_type", "outputType", "standard")
if output_type not in {"standard", "reasoning", "dpo"}:
raise ValueError("output_type must be one of: standard, reasoning, dpo")
source_mode = _config_value(config, "source_mode", "sourceMode", "local")
if source_mode not in {"local", "external"}:
raise ValueError("source_mode must be one of: local, external")
external_source = _config_value(config, "external_source", "externalSource", None)
if external_source is not None:
if not isinstance(external_source, dict):
raise ValueError("external_source must be an object")
if any(
key.lower() in {"password", "secret", "token", "api_key"}
for key in external_source
):
raise ValueError("external_source must not persist credentials")
external_url = str(external_source.get("url") or "").strip()
if external_url:
parsed_external_url = urlsplit(external_url)
sensitive_query_keys = {"password", "secret", "token", "api_key", "user", "username"}
if parsed_external_url.username or parsed_external_url.password or (
set(parse_qs(parsed_external_url.query)) & sensitive_query_keys
):
raise ValueError("external_source URL must not contain credentials")
chunk_method = _config_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
if not isinstance(chunk_method, str) or chunk_method not in {
"layout_hybrid",
@@ -303,6 +329,9 @@ class ExternalSourceRequest(BaseModel):
username: str | None = Field(default=None, max_length=150)
password: str | None = Field(default=None, max_length=500)
limit: int = Field(default=1000, ge=1, le=100_000)
connect_timeout_seconds: int = Field(default=5, ge=1, le=30)
statement_timeout_seconds: int = Field(default=30, ge=1, le=300)
ssl_mode: Literal["disable", "prefer", "require", "verify-ca", "verify-full"] = "prefer"
class ExternalPullRequest(ExternalSourceRequest):
@@ -324,6 +353,8 @@ class ResultUpdate(BaseModel):
instruction: str | None = None
input: str | None = None
output: str | None = None
chosen: str | None = None
rejected: str | None = None
expected_updated_at: str | None = None
@@ -374,7 +405,7 @@ class PublishRequest(BaseModel):
dataset_type: Literal["train", "test", "eval", "val", "other"] = "train"
storage_type: Literal["local"] = "local"
split: DatasetSplit = Field(default_factory=DatasetSplit)
format: Literal["alpaca_jsonl", "jsonl"] = "alpaca_jsonl"
format: Literal["alpaca_jsonl", "jsonl", "dpo"] = "alpaca_jsonl"
description: str = ""
@field_validator("dataset_name")

37
backend/tests/test_data_process_api.py
View File

@@ -1840,6 +1840,43 @@ def test_external_source_never_returns_fake_success(tmp_path: Path) -> None:
assert response.json()["detail"]["code"] == 501
def test_external_source_mode_belongs_to_step_three_structured_task(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
local_task_id = client.post(
"/modelTF/data-process",
json={
"name": "本地结构化任务",
"process_type": "structured",
"config": {"source_mode": "local"},
},
).json()["data"]["id"]
rejected = client.post(
f"/modelTF/data-process/{local_task_id}/external/test",
json={"type": "mysql", "url": "mysql://db.example/test"},
)
assert rejected.status_code == 409
external_task_id = client.post(
"/modelTF/data-process",
json={
"name": "外部结构化任务",
"process_type": "structured",
"config": {"source_mode": "external"},
},
).json()["data"]["id"]
accepted_as_external = client.post(
f"/modelTF/data-process/{external_task_id}/external/test",
json={"type": "mysql", "url": "mysql://db.example/test"},
)
assert accepted_as_external.status_code == 501
local_upload = client.post(
f"/modelTF/data-process/{external_task_id}/source-files",
files={"files": ("records.jsonl", b'{"id":1}\n', "application/jsonl")},
)
assert local_upload.status_code == 409
def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(

68
backend/tests/test_data_process_generation.py
View File

@@ -88,6 +88,74 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
assert progress_updates == [(1, 1)]
def test_generate_model_records_builds_native_dpo_pair() -> None:
def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
system_prompt = payload["messages"][0]["content"]
assert '"chosen"' in system_prompt
assert '"rejected"' in system_prompt
assert "直接偏好优化" in system_prompt
return httpx.Response(
200,
json={
"choices": [{
"message": {
"content": json.dumps({
"items": [{
"instruction": "系统如何处理扫描 PDF？",
"input": "",
"chosen": "仅在没有文本层时调用 OCR，并保留页码。",
"rejected": "所有 PDF 都重复执行 OCR。",
}],
}, ensure_ascii=False),
},
}],
},
)
records = generate_model_records(
[{"id": "preview-dpo", "edited_content": "扫描 PDF 缺少文本层时执行 OCR。"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"output_type": "dpo", "generation_retries": 0},
task_id="task-dpo",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert records[0]["status"] == "valid"
assert records[0]["chosen"] == "仅在没有文本层时调用 OCR,并保留页码。"
assert records[0]["rejected"] == "所有 PDF 都重复执行 OCR。"
assert records[0]["output"] == records[0]["chosen"]
def test_generate_model_records_rejects_equal_dpo_pair() -> None:
def handler(_request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={"choices": [{"message": {"content": json.dumps({
"items": [{
"instruction": "问题",
"chosen": "相同回答",
"rejected": "相同回答",
}],
}, ensure_ascii=False)}}]},
)
records = generate_model_records(
[{"id": "preview-dpo-invalid", "edited_content": "来源"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"output_type": "dpo", "generation_retries": 0},
task_id="task-dpo-invalid",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert records[0]["status"] == "invalid"
assert "chosen equals rejected" in records[0]["error"]
def test_minimax_m3_uses_split_reasoning_and_completion_token_budget() -> None:
requests: list[dict[str, object]] = []

4
backend/tests/test_data_process_migration.py
View File

@@ -37,6 +37,10 @@ def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
for value in ("idle", "queued", "running", "completed", "failed", "cancelled"):
assert f"'{value}'" in sql
assert "CREATE TABLE IF NOT EXISTS data_process_results" in sql
assert "ADD COLUMN IF NOT EXISTS chosen TEXT NOT NULL DEFAULT ''" in sql
assert "ADD COLUMN IF NOT EXISTS rejected TEXT NOT NULL DEFAULT ''" in sql
assert "ADD COLUMN IF NOT EXISTS original_chosen TEXT" in sql
assert "ADD COLUMN IF NOT EXISTS original_rejected TEXT" in sql
assert sql.count("BEGIN;") == 1
assert sql.rstrip().endswith("COMMIT;")

33
backend/tests/test_data_process_store.py
View File

@@ -1335,6 +1335,39 @@ def test_publish_rejects_invalid_reasoning_output_format() -> None:
)
def test_publish_dpo_writes_chosen_and_rejected_jsonl() -> None:
conn = _PublishConnection(
[
{
"id": "result-dpo",
"status": "valid",
"instruction": "如何处理扫描 PDF？",
"input": "",
"output": "仅在无文本层时执行 OCR。",
"chosen": "仅在无文本层时执行 OCR。",
"rejected": "所有 PDF 都执行 OCR。",
"preview_item_id": "preview-dpo",
}
]
)
published = _PublishStore(conn, {"output_type": "dpo"}).publish(
"task-dpo",
{
"dataset_name": "偏好数据",
"storage_type": "local",
"format": "dpo",
"split": {"train": 100, "validation": 0, "test": 0},
},
)
record = conn.records[0]["raw"]
assert record["chosen"] == "仅在无文本层时执行 OCR。"
assert record["rejected"] == "所有 PDF 都执行 OCR。"
assert "output" not in record
assert published["datasets"][0]["metadata"]["format"] == "dpo"
def test_source_storage_descriptor_accepts_owned_local_and_legacy_db_references() -> None:
task_id = "dpt_task"
source_file_id = "dpsf_source"

6
docker/app/Dockerfile.backend
View File

@@ -2,7 +2,8 @@ FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1
PIP_NO_CACHE_DIR=1 \
TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache
WORKDIR /app
@@ -16,6 +17,9 @@ RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \
&& chmod -R 0775 /opt/yg-ft /data/yg-ft
# 离线打包 tiktoken cl100k_base 词表，避免无网环境下运行时联网下载
COPY docker/app/tiktoken /opt/tiktoken_cache
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

4
frontend/scripts/regression-data-process-detail.mjs
View File

@@ -101,7 +101,9 @@ assert.match(detailSource, /!\/\(\?:password\|secret\|token\|api_key\)\/i\.test\
assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置仍直接展示内部模型快照')
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : value === 'dpo' \? 'DPO 偏好对' : '标准回答'/, '处理配置没有转换输出类型枚举')
assert.match(detailSource, /prop="chosen"[\s\S]*?prop="rejected"/, 'DPO 结果没有展示 Chosen 与 Rejected')
assert.match(detailSource, /publishForm\.format = isDpoOutput\.value \? 'dpo'/, 'DPO 发布没有锁定原生格式')
assert.match(detailSource, /reasoning_detail:\s*'推理详细程度'/, '处理配置没有显示推理详细程度名称')
assert.match(detailSource, /value === 'detailed' \? '详细推理' : '普通推理'/, '处理配置没有转换推理详细程度枚举')
assert.match(detailSource, /key !== 'reasoning_detail' \|\| config\.output_type === 'reasoning'/, '标准回答任务不应展示无关的推理详细程度')

21
frontend/scripts/regression-data-process-wizard.mjs
View File

@@ -122,13 +122,13 @@ assert.match(regenerationSource, /route\.name === 'data-process-regenerate'[\s\S
assert.match(viewSource, /const currentStep = ref\(0\)/, '重新生成必须从向导第一步开始')
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) {
for (const title of ['创建任务', '大模型选择', '数据来源', '数据预览', '开始生成', '结果编辑与保存']) {
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤：${title}`)
}
assert.match(
viewSource,
/\{ id: 'create',[\s\S]*?\{ id: 'model',[\s\S]*?\{ id: 'upload',[\s\S]*?\{ id: 'preview',[\s\S]*?\{ id: 'generate',[\s\S]*?\{ id: 'results'/,
'六步向导顺序必须为创建任务、大模型选择、上传文件、数据预览、开始生成、结果编辑与保存',
'六步向导顺序必须为创建任务、大模型选择、数据来源、数据预览、开始生成、结果编辑与保存',
)
assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减')
assert.match(
@@ -380,7 +380,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
assert.match(viewSource, /<ModelSelectionStep\s+[\s\S]*?v-else-if="currentStepId === 'model'"/, '第二步没有挂载独立大模型选择组件')
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：选择大模型'/, '第一步主按钮没有指向大模型选择')
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：上传文件'/, '第二步主按钮没有指向上传文件')
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：选择数据来源'/, '第二步主按钮没有指向数据来源选择')
assert.match(
viewSource,
/if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续：数据预览'/,
@@ -1032,7 +1032,7 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
assert.match(stateSource, /configuredOutputType === 'reasoning' \|\| configuredOutputType === 'dpo'[\s\S]*?\? configuredOutputType[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
@@ -1412,7 +1412,7 @@ assert.match(
)
assert.match(
sourceUploadSource,
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
'无文件时未保留原有大拖拽上传区或上传配置',
)
assert.match(
@@ -1432,7 +1432,7 @@ assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构
assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换')
assert.match(
sourceUploadSource,
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding \|\| externalPulling">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
'有文件时缺少标题右侧的继续上传触发器或上传配置',
)
assert.match(
@@ -1476,4 +1476,13 @@ assert.match(
)
assert.match(previewSource, /height:\s*clamp\(560px,\s*calc\(100vh - 370px\),\s*720px\)/, '对照预览高度不足以展示切片正文')
assert.doesNotMatch(taskSetupSource, /外来数据源拉取/, '外部数据源仍错误地放在第一步处理类型中')
assert.match(sourceUploadSource, /<h3>数据来源<\/h3>[\s\S]*?本地上传[\s\S]*?外部数据源/, '第三步缺少本地与外部数据来源选择')
for (const field of ['地址 / URL', '鉴权方式', 'SSL 模式', '连接超时', '查询超时', '只读查询语句', '拉取条数', '落地文件名']) {
assert.ok(sourceUploadSource.includes(field), `外部数据源标准配置缺少：${field}`)
}
assert.match(generationControlSource, /label="DPO 偏好对" value="dpo"/, '输出类型缺少 DPO 偏好对')
assert.match(resultEditorSource, /Chosen[\s\S]*?Rejected/, '结果编辑器缺少 DPO 成对字段')
assert.match(viewSource, /sourceConfigForBackend\(sourceMode\.value, externalSource\)/, '任务配置没有保存第三步数据来源模式')
console.log('数据处理六步向导回归检查通过')

13
frontend/src/types/dataProcess.ts
View File

@@ -4,7 +4,7 @@ export type DataProcessStatus = 'pending' | 'running' | 'completed' | 'failed' |
export type DataProcessType = 'structured' | 'unstructured' | 'external'
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
export type DataProcessSplit = 'train' | 'validation' | 'test'
export type DataProcessOutputType = 'standard' | 'reasoning'
export type DataProcessOutputType = 'standard' | 'reasoning' | 'dpo'
export type DataProcessReasoningDetail = 'normal' | 'detailed'
export type DataProcessWorkflowStep = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
export type DataProcessPreviewStatus = 'idle' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'
@@ -221,6 +221,9 @@ export interface DataProcessExternalSourcePayload {
username?: string
password?: string
limit: number
connect_timeout_seconds: number
statement_timeout_seconds: number
ssl_mode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
query?: string
file_name?: string
}
@@ -342,9 +345,13 @@ export interface DataProcessResult {
instruction: string
input: string
output: string
chosen?: string
rejected?: string
original_instruction?: string | null
original_input?: string | null
original_output?: string | null
original_chosen?: string | null
original_rejected?: string | null
status: DataProcessResultStatus
error?: string | null
split?: DataProcessSplit | null
@@ -356,6 +363,8 @@ export interface DataProcessResultUpdatePayload {
instruction: string
input: string
output: string
chosen?: string
rejected?: string
expected_updated_at?: string
}
@@ -410,7 +419,7 @@ export interface DataProcessPublishPayload {
dataset_type: 'train' | 'test' | 'eval' | 'val' | 'other'
storage_type: 'local'
split: DataProcessDatasetSplit
format: 'alpaca_jsonl' | 'jsonl'
format: 'alpaca_jsonl' | 'jsonl' | 'dpo'
}
export interface DataProcessPublishResult {

91
frontend/src/views/data-process/DataProcessCreateView.vue
View File

@@ -20,6 +20,7 @@ import {
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
@@ -39,26 +40,20 @@ import {
updateDataProcessPreview,
updateDataProcessTask,
updateDataProcessWorkflowStep,
type DataProcessExternalSourcePayload,
type DataProcessPreviewItem,
} from '@/api/modules/dataProcess'
import type {
DataProcessConfig,
DataProcessPreviewProgress,
DataProcessTask,
DataProcessWorkflowStep,
} from '@/types/dataProcess'
import type { DataProcessConfig, DataProcessPreviewProgress, DataProcessTask, DataProcessWorkflowStep } from '@/types/dataProcess'
import type {
ExternalDataSource,
GenerationControlOptions,
PreviewItem,
ProcessType,
SourceMode,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
UploadedDataFile,
} from './create/types'
const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
@@ -71,7 +66,7 @@ const PREVIEW_MODEL_VERSION = 'backend-pipeline-v4'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
{ id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
{ id: 'upload', title: '数据来源', desc: '选择本地上传或外部数据源拉取' },
{ id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
@@ -81,11 +76,13 @@ const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id
const task = reactive({ name: '', description: '' })
const taskId = ref<string | null>(null)
const processType = ref<ProcessType>('structured')
const sourceMode = ref<SourceMode>('local')
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
const modelSelectionOptions = computed<GenerationControlOptions>(() => (
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
))
const activeOutputType = computed(() => modelSelectionOptions.value.outputType)
const uploadedFiles = ref<UploadedDataFile[]>([])
const previewBuilding = ref(false)
const {
@@ -93,16 +90,7 @@ const {
startPreviewBuild,
stopPreviewPolling,
} = useDataProcessPreviewBuild()
const externalSource = reactive<ExternalDataSource>({
type: 'postgresql',
url: '',
authMode: 'none',
username: '',
password: '',
limit: 1000,
query: '',
fileName: 'external-data.jsonl',
})
const externalSource = reactive<ExternalDataSource>(createDefaultExternalSource())
const externalPulling = ref(false)
const externalConnected = ref(false)
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
@@ -135,6 +123,7 @@ const {
} = useDataProcessGeneration({
taskId,
dirty,
outputType: activeOutputType,
beforeGenerate: beforeStartGeneration,
})
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
@@ -174,7 +163,7 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
}))
const primaryActionLabel = computed(() => {
if (currentStepId.value === 'create') return '继续：选择大模型'
if (currentStepId.value === 'model') return '继续：上传文件'
if (currentStepId.value === 'model') return '继续：选择数据来源'
if (currentStepId.value === 'upload') {
if (sourceUploading.value) return '正在上传'
return previewBuilding.value ? '正在切分' : '继续：数据预览'
@@ -198,7 +187,6 @@ function goToStep(stepId: StepId) {
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
if (nextStepIndex >= 0) currentStep.value = nextStepIndex
}
function updateModelSelectionOptions(value: GenerationControlOptions) {
if (processType.value === 'unstructured') {
unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
@@ -206,12 +194,12 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
}
structuredOptions.value = { ...structuredOptions.value, ...value }
}
function toBackendConfig(): DataProcessConfig {
const options = processType.value === 'unstructured'
? unstructuredOptions.value
: structuredOptions.value
const common = {
...sourceConfigForBackend(sourceMode.value, externalSource),
preprocess_options: [...options.preprocessOptions],
semantic_enrichment: options.semanticEnrichment,
dataset_split: { ...options.datasetSplit },
@@ -247,7 +235,6 @@ function toBackendConfig(): DataProcessConfig {
qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
}
}
function taskPayload() {
return {
name: task.name.trim(),
@@ -256,12 +243,10 @@ function taskPayload() {
config: toBackendConfig(),
}
}
async function persistWorkflowStep(step: StepId) {
if (!taskId.value) return
await updateDataProcessWorkflowStep(taskId.value, step as DataProcessWorkflowStep)
}
async function saveTaskConfiguration() {
if (isRegeneration.value) {
const regenerated = await prepareRegeneration(taskPayload())
@@ -277,19 +262,6 @@ async function saveTaskConfiguration() {
return saved
}
function externalPayload(): DataProcessExternalSourcePayload {
return {
type: 'postgresql',
url: externalSource.url.trim(),
auth_mode: externalSource.authMode,
username: externalSource.username || undefined,
password: externalSource.password || undefined,
limit: externalSource.limit,
query: externalSource.query?.trim() || undefined,
file_name: externalSource.fileName || 'external-data.jsonl',
}
}
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
const sourceLocator = item.quality_score?.source_locator
return {
@@ -378,7 +350,7 @@ async function beforeStartGeneration() {
)
}
watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
[() => task.name, () => task.description, processType, sourceMode, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!hydrating.value) dirty.value = true
},
@@ -396,6 +368,13 @@ watch(processType, (nextType, previousType) => {
ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
})
watch(processType, (nextType) => {
if (nextType === 'unstructured' && sourceMode.value === 'external') {
sourceMode.value = 'local'
externalConnected.value = false
}
})
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (hydrating.value) return
if (currentSignature === previousSignature) return
@@ -453,6 +432,16 @@ function updateExternalSource(value: ExternalDataSource) {
externalConnected.value = false
}
async function updateSourceMode(value: SourceMode) {
const previous = sourceMode.value
sourceMode.value = value
externalConnected.value = false
dirty.value = true
externalPulling.value = true
try { await saveTaskConfiguration() } catch { sourceMode.value = previous }
finally { externalPulling.value = false }
}
async function handleTestConnection() {
if (!taskId.value) {
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
@@ -464,7 +453,8 @@ async function handleTestConnection() {
}
externalPulling.value = true
try {
const result = await testDataProcessExternalSource(taskId.value, externalPayload())
await saveTaskConfiguration()
const result = await testDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
externalConnected.value = result.connected
if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
else ElMessage.warning(result.message || '数据源连接失败')
@@ -490,7 +480,8 @@ async function handlePullData() {
}
externalPulling.value = true
try {
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
await saveTaskConfiguration()
const response = await pullDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
const newFiles: UploadedDataFile[] = []
for (const file of response.files) {
const content = await loadCanonicalSourceContent(taskId.value, file.id)
@@ -620,6 +611,12 @@ function applyPreviewProgress(progress: DataProcessPreviewProgress) {
async function completePreviewWorkspace() {
previewItems.value = await loadAllPreviewItems()
for (const file of uploadedFiles.value) {
if (!file.content && file.sourceFileId && file.fileFormat?.replace('.', '') === 'pdf') {
file.content = await loadCanonicalSourceContent(taskId.value as string, file.sourceFileId)
.catch(() => file.content)
}
}
const configSignature = buildPreviewConfigSignature()
const previewCounts = new Map<string, number>()
for (const item of previewItems.value) {
@@ -678,7 +675,7 @@ async function nextFromUpload() {
return
}
if (uploadedFiles.value.length === 0) {
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
ElMessage.warning(sourceMode.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
@@ -992,6 +989,9 @@ async function initializeExistingWorkflow() {
const sourceTask = await loadRegenerationSource()
if (!sourceTask) return
taskId.value = String(sourceTask.id)
const restoredSource = restoreExternalSourceConfig(sourceTask.config || {})
sourceMode.value = restoredSource.mode
Object.assign(externalSource, restoredSource.source)
if (!isWorkflowResume.value) return
if (sourceTask.status === 'completed' && sourceTask.results_confirmed !== false) {
allowLeave = true
@@ -1102,6 +1102,7 @@ onMounted(() => {
<SourceUploadStep
v-else-if="currentStepId === 'upload'"
:process-type="processType"
:source-mode="sourceMode"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
@@ -1109,6 +1110,7 @@ onMounted(() => {
:preview-building="previewBuilding"
:source-uploading="sourceUploading"
@update:external-source="updateExternalSource"
@update:source-mode="updateSourceMode"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@use-sample="useSampleFile"
@@ -1154,6 +1156,7 @@ onMounted(() => {
:preview-items="previewItems"
:regenerating-result-id="regeneratingResultId"
:bulk-regeneration="bulkRegeneration"
:output-type="activeOutputType"
@update:field="updateResultField"
@regenerate:all="regenerateAllResults"
@regenerate:item="regenerateResult"
@@ -1179,8 +1182,8 @@ onMounted(() => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
:loading="externalPulling || modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="externalPulling || hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

57
frontend/src/views/data-process/DataProcessDetailView.vue
View File

@@ -53,7 +53,7 @@ const resultCellTooltipOptions = {
} as const
let resultFilterTimer: ReturnType<typeof setTimeout> | null = null
const editForm = reactive({ instruction: '', input: '', output: '' })
const editForm = reactive({ instruction: '', input: '', output: '', chosen: '', rejected: '' })
const publishForm = reactive<DataProcessPublishPayload>({
dataset_name: '',
dataset_type: 'train',
@@ -69,6 +69,8 @@ const processTypeMap: Record<DataProcessType, string> = {
}
const configLabelMap: Record<string, string> = {
source_mode: '数据来源',
external_source: '外部数据源配置',
preprocess_options: '预处理规则',
dataset_split: '数据集划分',
generation_model_id: '数据生成模型',
@@ -212,6 +214,7 @@ const outputDatasetName = computed(() => (
))
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
const hasPublishedOutputs = computed(() => outputDatasets.value.length > 0)
const isDpoOutput = computed(() => detail.value?.config?.output_type === 'dpo')
const hasCurrentPublishedDataset = computed(() => (
Boolean(outputDatasetId.value)
&& outputDatasets.value.some((dataset) => String(dataset.id) === String(outputDatasetId.value))
@@ -294,6 +297,18 @@ const configRows = computed(() => {
})
function formatConfigValue(key: string, value: unknown) {
if (key === 'source_mode') return value === 'external' ? '外部数据源' : '本地上传'
if (key === 'external_source' && value && typeof value === 'object') {
const source = value as Record<string, unknown>
return [
source.type,
source.url,
source.ssl_mode ? `SSL ${source.ssl_mode}` : '',
source.limit ? `上限 ${source.limit} 条` : '',
source.connect_timeout_seconds ? `连接 ${source.connect_timeout_seconds}s` : '',
source.statement_timeout_seconds ? `查询 ${source.statement_timeout_seconds}s` : '',
].filter(Boolean).join(' · ')
}
if (key === 'generation_model_id') {
const snapshot = detail.value?.config?.generation_model_snapshot
if (snapshot && typeof snapshot === 'object' && !Array.isArray(snapshot)) {
@@ -305,7 +320,7 @@ function formatConfigValue(key: string, value: unknown) {
return chunkMethodLabelMap[value] || value
}
if (key === 'output_type') {
return value === 'reasoning' ? '思维链回答' : '标准回答'
return value === 'reasoning' ? '思维链回答' : value === 'dpo' ? 'DPO 偏好对' : '标准回答'
}
if (key === 'reasoning_detail') {
return value === 'detailed' ? '详细推理' : '普通推理'
@@ -461,14 +476,26 @@ function openResultEditor(result: DataProcessResult) {
instruction: result.instruction,
input: result.input,
output: result.output,
chosen: result.chosen || result.output || '',
rejected: result.rejected || '',
})
editDialogVisible.value = true
}
async function saveResult() {
if (!editingResult.value) return
if (!editForm.instruction.trim() || !editForm.output.trim()) {
ElMessage.warning('Instruction 和 Output 不能为空')
const invalid = isDpoOutput.value
? !editForm.instruction.trim()
|| !editForm.chosen.trim()
|| !editForm.rejected.trim()
|| editForm.chosen.trim() === editForm.rejected.trim()
: !editForm.instruction.trim() || !editForm.output.trim()
if (invalid) {
ElMessage.warning(
isDpoOutput.value
? 'Instruction、Chosen、Rejected 均不能为空，且两个回答不能相同'
: 'Instruction 和 Output 不能为空',
)
return
}
@@ -478,6 +505,8 @@ async function saveResult() {
instruction: editForm.instruction,
input: editForm.input,
output: editForm.output,
chosen: editForm.chosen,
rejected: editForm.rejected,
expected_updated_at: editingResult.value.updated_at,
})
replaceResult(updated)
@@ -530,6 +559,7 @@ function openPublishDialog() {
if (!detail.value) return
publishForm.dataset_name = `${detail.value.name}-数据集`
publishForm.split = configuredSplit()
publishForm.format = isDpoOutput.value ? 'dpo' : 'alpaca_jsonl'
publishDialogVisible.value = true
}
@@ -808,7 +838,11 @@ onBeforeUnmount(() => {
<el-table-column type="index" label="#" width="56" align="center" />
<el-table-column label="指令" min-width="190" show-overflow-tooltip prop="instruction" />
<el-table-column label="输入" min-width="160" show-overflow-tooltip prop="input" />
<el-table-column label="输出" min-width="230" show-overflow-tooltip prop="output" />
<el-table-column v-if="!isDpoOutput" label="输出" min-width="230" show-overflow-tooltip prop="output" />
<template v-else>
<el-table-column label="Chosen" min-width="210" show-overflow-tooltip prop="chosen" />
<el-table-column label="Rejected" min-width="210" show-overflow-tooltip prop="rejected" />
</template>
<el-table-column label="质量分" width="88" align="center">
<template #default="{ row }">
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
@@ -862,9 +896,17 @@ onBeforeUnmount(() => {
<el-form-item label="Input">
<el-input v-model="editForm.input" type="textarea" :rows="3" maxlength="10000" show-word-limit />
</el-form-item>
<el-form-item label="Output" required>
<el-form-item v-if="!isDpoOutput" label="Output" required>
<el-input v-model="editForm.output" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
<template v-else>
<el-form-item label="Chosen（优选回答）" required>
<el-input v-model="editForm.chosen" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
<el-form-item label="Rejected（拒选回答）" required>
<el-input v-model="editForm.rejected" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
</template>
</el-form>
<template #footer>
<el-button @click="editDialogVisible = false">取消</el-button>
@@ -884,7 +926,8 @@ onBeforeUnmount(() => {
</el-select>
</el-form-item>
<el-form-item label="输出格式">
<el-select v-model="publishForm.format">
<el-select v-model="publishForm.format" :disabled="isDpoOutput">
<el-option v-if="isDpoOutput" label="DPO JSONL" value="dpo" />
<el-option label="Alpaca JSONL" value="alpaca_jsonl" />
<el-option label="JSONL" value="jsonl" />
</el-select>

11
frontend/src/views/data-process/create/GenerationOptionsPanel.vue
View File

@@ -76,7 +76,7 @@ function updateQualityRules(value: Array<string | number>) {
}
function updateOutputType(value: string | number | boolean | undefined) {
const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
const outputType = value === 'reasoning' || value === 'dpo' ? value : 'standard'
emit('update:options', {
...props.options,
outputType,
@@ -153,6 +153,9 @@ function modelMeta(model: ModelItem) {
<small v-if="options.outputType === 'reasoning'">
当前使用思维链专用提示语；系统还会按所选详细程度约束推理结构
</small>
<small v-else-if="options.outputType === 'dpo'">
当前使用 DPO 专用提示语，模型会同时生成 Chosen 与 Rejected 偏好回答
</small>
<small v-else>当前使用标准回答提示语，只要求问题和最终答案</small>
</div>
<div class="prompt-input-wrapper">
@@ -237,7 +240,7 @@ function modelMeta(model: ModelItem) {
<div class="output-type-row">
<div class="field-copy">
<strong>输出类型</strong>
<small>控制答案是否包含可用于推理模型训练的思维链内容</small>
<small>选择标准监督、思维链或 DPO 成对偏好数据</small>
</div>
<el-select
class="output-type-select"
@@ -247,6 +250,7 @@ function modelMeta(model: ModelItem) {
>
<el-option label="标准回答" value="standard" />
<el-option label="思维链回答" value="reasoning" />
<el-option label="DPO 偏好对" value="dpo" />
</el-select>
</div>
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
@@ -272,6 +276,9 @@ function modelMeta(model: ModelItem) {
<template v-else>保留关键依据与必要步骤，</template>
最终按 <code>&lt;think&gt;推理过程&lt;/think&gt;</code> 加最终答案保存。
</template>
<template v-else-if="options.outputType === 'dpo'">
保存同一问题的 <code>chosen</code> 优选回答和 <code>rejected</code> 拒选回答，用于直接偏好优化训练。
</template>
<template v-else>仅保存最终答案，不包含推理过程。</template>
</p>
</div>

37
frontend/src/views/data-process/create/ResultEditorStep.vue
View File

@@ -1,6 +1,7 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
const props = defineProps<{
items: ResultItem[]
@@ -8,11 +9,12 @@ const props = defineProps<{
selectedId: string | null
regeneratingResultId: string | null
bulkRegeneration: BulkResultRegenerationState
outputType: DataProcessOutputType
}>()
const emit = defineEmits<{
'update:selectedId': [value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output', value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
'regenerate:item': [id: string]
'regenerate:all': []
}>()
@@ -59,6 +61,11 @@ const selectedSourceMeta = computed(() => {
const source = selectedSource.value
if (!source) return ''
const parts: string[] = []
if (source.sourcePages?.length) {
const first = source.sourcePages[0]
const last = source.sourcePages[source.sourcePages.length - 1]
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
}
if (source.sourceStartLine != null) {
parts.push(
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
@@ -75,6 +82,8 @@ const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !keyword
|| item.instruction.toLowerCase().includes(keyword)
|| item.output.toLowerCase().includes(keyword)
|| item.chosen.toLowerCase().includes(keyword)
|| item.rejected.toLowerCase().includes(keyword)
|| String(index + 1).includes(keyword)
return matchesSearch && (!invalidOnly.value || item.status === 'invalid')
}))
@@ -135,7 +144,7 @@ function selectRelative(offset: number) {
<span class="result-index">#{{ String(items.findIndex((entry) => entry.id === item.id) + 1).padStart(3, '0') }}</span>
<span class="result-copy">
<strong>{{ item.instruction || '未填写指令' }}</strong>
<small>{{ item.output || '未填写输出' }}</small>
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
</span>
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
<i
@@ -207,7 +216,7 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'input', $event)"
/>
</div>
<div class="field-editor">
<div v-if="outputType !== 'dpo'" class="field-editor">
<label>Output <em>必填</em></label>
<el-input
:model-value="selectedItem.output"
@@ -217,6 +226,28 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'output', $event)"
/>
</div>
<template v-else>
<div class="field-editor dpo-field is-chosen">
<label>Chosen <em>优选回答，必填</em></label>
<el-input
:model-value="selectedItem.chosen"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'chosen', $event)"
/>
</div>
<div class="field-editor dpo-field is-rejected">
<label>Rejected <em>拒选回答，必填</em></label>
<el-input
:model-value="selectedItem.rejected"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'rejected', $event)"
/>
</div>
</template>
<div v-if="selectedItem.error" class="validation-error">
<i class="fa fa-exclamation-circle" /> {{ selectedItem.error }}
</div>

165
frontend/src/views/data-process/create/SourceUploadStep.vue
View File

@@ -1,10 +1,11 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import type { UploadFile } from 'element-plus'
import type { ExternalDataSource, ProcessType, UploadedDataFile } from './types'
import type { ExternalDataSource, ProcessType, SourceMode, UploadedDataFile } from './types'
const props = defineProps<{
processType: ProcessType
sourceMode: SourceMode
uploadedFiles: UploadedDataFile[]
externalSource: ExternalDataSource
externalPulling: boolean
@@ -14,6 +15,7 @@ const props = defineProps<{
}>()
const emit = defineEmits<{
'update:sourceMode': [value: SourceMode]
'update:externalSource': [value: ExternalDataSource]
'file-change': [file: UploadFile]
'remove-file': [uid: string | number]
@@ -31,6 +33,14 @@ const AUTH_MODES = [
{ value: 'basic', label: '账号密码' },
]
const SSL_MODES = [
{ value: 'disable', label: '关闭' },
{ value: 'prefer', label: '优先使用（默认）' },
{ value: 'require', label: '必须加密' },
{ value: 'verify-ca', label: '验证 CA' },
{ value: 'verify-full', label: '完整验证' },
]
const FILE_PAGE_SIZE = 10
const currentFilePage = ref(1)
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
@@ -45,7 +55,10 @@ const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
}
const isExternal = computed(() => props.processType === 'external')
const isExternal = computed(() => props.sourceMode === 'external')
const sourceModeLocked = computed(() => (
props.uploadedFiles.length > 0 || props.sourceUploading || props.previewBuilding || props.externalPulling
))
const uploadAccept = computed(() => props.processType === 'unstructured'
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
@@ -124,6 +137,44 @@ function getFileError(file: UploadedDataFile) {
<template>
<section class="source-upload-step" aria-labelledby="source-upload-title">
<div class="source-mode-section">
<div class="section-title-row">
<div>
<h3>数据来源</h3>
<p>在当前步骤选择本地文件上传或从外部系统拉取，处理类型保持不变</p>
</div>
</div>
<div class="source-mode-options">
<button
type="button"
class="source-mode-option"
:class="{ 'is-active': sourceMode === 'local' }"
:disabled="sourceModeLocked"
@click="emit('update:sourceMode', 'local')"
>
<i class="fa fa-upload" aria-hidden="true" />
<span><strong>本地上传</strong><small>从当前设备选择文件</small></span>
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
</button>
<el-tooltip
:content="processType === 'structured' ? '从标准化外部数据源配置拉取' : '当前仅支持把外部数据库拉取为结构化数据'"
>
<button
type="button"
class="source-mode-option"
:class="{ 'is-active': sourceMode === 'external' }"
:disabled="sourceModeLocked || processType !== 'structured'"
@click="emit('update:sourceMode', 'external')"
>
<i class="fa fa-cloud-download" aria-hidden="true" />
<span><strong>外部数据源</strong><small>连接数据库并执行只读拉取</small></span>
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
</button>
</el-tooltip>
</div>
<small v-if="sourceModeLocked" class="source-mode-lock-hint">已有来源文件时不可切换；删除现有文件后可重新选择。</small>
</div>
<div v-if="isExternal" class="form-section external-section">
<div class="section-title-row">
<div>
@@ -202,6 +253,45 @@ function getFileError(file: UploadedDataFile) {
@update:model-value="updateExternalField('limit', Number($event) || 0)"
/>
</el-form-item>
<el-form-item label="SSL 模式">
<el-select
:model-value="externalSource.sslMode"
aria-label="SSL 模式"
@update:model-value="updateExternalField('sslMode', $event)"
>
<el-option v-for="item in SSL_MODES" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
</el-form-item>
<el-form-item label="连接超时">
<el-input-number
:model-value="externalSource.connectTimeoutSeconds"
:min="1"
:max="30"
controls-position="right"
aria-label="连接超时秒数"
@update:model-value="updateExternalField('connectTimeoutSeconds', Number($event) || 5)"
/>
<small>单位：秒</small>
</el-form-item>
<el-form-item label="查询超时">
<el-input-number
:model-value="externalSource.statementTimeoutSeconds"
:min="1"
:max="300"
controls-position="right"
aria-label="查询超时秒数"
@update:model-value="updateExternalField('statementTimeoutSeconds', Number($event) || 30)"
/>
<small>单位：秒</small>
</el-form-item>
<el-form-item label="落地文件名">
<el-input
:model-value="externalSource.fileName"
placeholder="external-data.jsonl"
aria-label="外部数据落地文件名"
@update:model-value="updateExternalField('fileName', $event)"
/>
</el-form-item>
<el-form-item label="只读查询语句" class="external-query-field">
<el-input
:model-value="externalSource.query"
@@ -314,7 +404,7 @@ function getFileError(file: UploadedDataFile) {
v-if="uploadedFiles.length === 0"
link
type="primary"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
@click="emit('use-sample')"
>
使用示例数据
@@ -326,7 +416,7 @@ function getFileError(file: UploadedDataFile) {
drag
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
@@ -353,13 +443,13 @@ function getFileError(file: UploadedDataFile) {
<el-upload
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
aria-label="继续添加源数据文件"
>
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
<el-button size="small" type="primary" :disabled="previewBuilding || externalPulling">继续上传</el-button>
</el-upload>
</div>
</div>
@@ -430,6 +520,68 @@ function getFileError(file: UploadedDataFile) {
width: 100%;
}
.source-mode-section {
padding-bottom: 22px;
margin-bottom: 22px;
border-bottom: 1px solid #edf0f5;
}
.source-mode-options {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 14px;
margin-top: 14px;
}
.source-mode-option {
position: relative;
display: flex;
align-items: center;
gap: 12px;
min-height: 76px;
padding: 14px 16px;
color: #667085;
text-align: left;
background: #fff;
border: 1px solid #dfe3ea;
border-radius: 8px;
cursor: pointer;
> .fa:first-child {
width: 28px;
color: #5b50f2;
font-size: 20px;
text-align: center;
}
span {
display: flex;
flex: 1;
flex-direction: column;
gap: 5px;
}
strong { color: #344054; font-size: 14px; }
small { color: #8a93a3; font-size: 12px; }
.selection-mark { color: transparent; }
&.is-active {
background: #fafaff;
border-color: #5b50f2;
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
}
&.is-active .selection-mark { color: #5b50f2; }
&:disabled { cursor: not-allowed; opacity: 0.55; }
}
.source-mode-lock-hint {
display: block;
margin-top: 9px;
color: #8a93a3;
font-size: 11px;
}
.form-section {
padding: 0;
@@ -672,6 +824,7 @@ function getFileError(file: UploadedDataFile) {
}
@media (max-width: 900px) {
.source-mode-options,
.external-section .external-grid {
grid-template-columns: minmax(0, 1fr);
}

16
frontend/src/views/data-process/create/TaskSetupStep.vue
View File

@@ -188,20 +188,6 @@ defineExpose({ validate })
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
<button
type="button"
class="type-option"
:class="{ 'is-active': processType === 'external' }"
:disabled="processTypeLocked"
@click="emit('update:processType', 'external')"
>
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
<span>
<strong>外来数据源拉取</strong>
<small>适用于数据库、API 接口等需远程拉取的外部数据</small>
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
</div>
</el-form-item>
</div>
@@ -274,7 +260,7 @@ defineExpose({ validate })
.type-options {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 16px;
width: 100%;
}

25
frontend/src/views/data-process/create/dataProcessCreateState.ts
View File

@@ -73,10 +73,21 @@ export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名具有深厚专
3. 信息不足时如实处理：遇到来源信息缺失或相互矛盾时，需在推理中明确指出，并基于来源做出合理假设，但不得编造与来源无关的内容。
4. 最终答案须能从前面的推理过程中自然得出，语言简洁、结构清晰。`
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
return outputType === 'reasoning'
? DEFAULT_REASONING_GENERATION_PROMPT
: DEFAULT_STANDARD_GENERATION_PROMPT
export const DEFAULT_DPO_GENERATION_PROMPT = `你是一名偏好数据构造专家。请基于下方来源内容，生成可用于直接偏好优化（DPO）的成对问答数据。
来源内容：
{{ content }}
任务要求：
1. 问题应聚焦来源内容的核心信息，并具有实际训练价值。
2. Chosen 必须准确、完整、清晰且严格忠于来源内容。
3. Rejected 应当表面合理但包含可辨认的缺陷，例如遗漏关键条件、事实偏差、逻辑不完整或表达含混；不得用乱码、空话或危险内容凑数。
4. Chosen 与 Rejected 必须明显不同，且都直接回答同一个问题。`
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning' | 'dpo') {
if (outputType === 'reasoning') return DEFAULT_REASONING_GENERATION_PROMPT
if (outputType === 'dpo') return DEFAULT_DPO_GENERATION_PROMPT
return DEFAULT_STANDARD_GENERATION_PROMPT
}
export function isBuiltInGenerationPrompt(value: string) {
@@ -92,6 +103,7 @@ export function isBuiltInGenerationPrompt(value: string) {
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_3,
DEFAULT_STANDARD_GENERATION_PROMPT,
DEFAULT_REASONING_GENERATION_PROMPT,
DEFAULT_DPO_GENERATION_PROMPT,
].map(normalize).includes(normalizedValue)
}
@@ -171,8 +183,9 @@ function generationOptionsFromConfig(
config: DataProcessConfig,
defaults: GenerationControlOptions,
): GenerationControlOptions {
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
? 'reasoning'
const configuredOutputType = configValue(config, 'output_type', defaults.outputType)
const outputType = configuredOutputType === 'reasoning' || configuredOutputType === 'dpo'
? configuredOutputType
: 'standard'
const configuredPrompt = String(
configValue(config, 'generation_prompt', defaults.generationPrompt),

77
frontend/src/views/data-process/create/externalSourceConfig.ts Normal file
View File

@@ -0,0 +1,77 @@
import type { DataProcessExternalSourcePayload } from '@/api/modules/dataProcess'
import type { DataProcessConfig } from '@/types/dataProcess'
import type { ExternalDataSource, SourceMode } from './types'
export function createDefaultExternalSource(): ExternalDataSource {
return {
type: 'postgresql',
url: '',
authMode: 'none',
username: '',
password: '',
limit: 1000,
connectTimeoutSeconds: 5,
statementTimeoutSeconds: 30,
sslMode: 'prefer',
query: '',
fileName: 'external-data.jsonl',
}
}
export function externalSourcePayload(source: ExternalDataSource): DataProcessExternalSourcePayload {
return {
type: source.type,
url: source.url.trim(),
auth_mode: source.authMode,
username: source.username?.trim() || undefined,
password: source.password || undefined,
limit: source.limit,
connect_timeout_seconds: source.connectTimeoutSeconds,
statement_timeout_seconds: source.statementTimeoutSeconds,
ssl_mode: source.sslMode,
query: source.query?.trim() || undefined,
file_name: source.fileName || 'external-data.jsonl',
}
}
export function sourceConfigForBackend(mode: SourceMode, source: ExternalDataSource) {
const payload = externalSourcePayload(source)
const { password: _password, ...safeSource } = payload
return {
source_mode: mode,
external_source: mode === 'external' ? safeSource : undefined,
}
}
export function restoreExternalSourceConfig(config: DataProcessConfig): {
mode: SourceMode
source: ExternalDataSource
} {
const defaults = createDefaultExternalSource()
const value = config.external_source
if (!value || typeof value !== 'object') {
return {
mode: config.source_mode === 'external' ? 'external' : 'local',
source: defaults,
}
}
const source = value as Record<string, unknown>
const sslMode = String(source.ssl_mode || '')
return {
mode: config.source_mode === 'external' ? 'external' : 'local',
source: {
...defaults,
url: String(source.url || ''),
authMode: source.auth_mode === 'basic' ? 'basic' : 'none',
username: String(source.username || ''),
limit: Number(source.limit) || defaults.limit,
connectTimeoutSeconds: Number(source.connect_timeout_seconds) || defaults.connectTimeoutSeconds,
statementTimeoutSeconds: Number(source.statement_timeout_seconds) || defaults.statementTimeoutSeconds,
sslMode: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'].includes(sslMode)
? sslMode as ExternalDataSource['sslMode']
: defaults.sslMode,
query: String(source.query || ''),
fileName: String(source.file_name || defaults.fileName),
},
}
}

10
frontend/src/views/data-process/create/types.ts
View File

@@ -5,6 +5,7 @@ import type {
} from '@/types/dataProcess'
export type ProcessType = 'structured' | 'unstructured' | 'external'
export type SourceMode = 'local' | 'external'
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
@@ -88,6 +89,9 @@ export interface ExternalDataSource {
username?: string
password?: string
limit: number
connectTimeoutSeconds: number
statementTimeoutSeconds: number
sslMode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
query?: string
fileName?: string
}
@@ -167,12 +171,18 @@ export interface ResultItem {
instruction: string
input: string
output: string
chosen: string
rejected: string
originalInstruction: string
originalInput: string
originalOutput: string
originalChosen: string
originalRejected: string
savedInstruction: string
savedInput: string
savedOutput: string
savedChosen: string
savedRejected: string
savedStatus: 'valid' | 'modified' | 'invalid'
status: 'valid' | 'modified' | 'invalid'
error?: string

47
frontend/src/views/data-process/create/useDataProcessGeneration.ts
View File

@@ -12,10 +12,12 @@ import {
type DataProcessResult,
} from '@/api/modules/dataProcess'
import type { BulkResultRegenerationState, GenerationState, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
interface GenerationBindings {
taskId: Ref<string | null>
dirty: Ref<boolean>
outputType: Ref<DataProcessOutputType>
beforeGenerate?: () => Promise<boolean | void>
}
@@ -32,12 +34,18 @@ function mapResult(item: DataProcessResult): ResultItem {
instruction: item.instruction,
input: item.input || '',
output: item.output,
chosen: item.chosen || item.output || '',
rejected: item.rejected || '',
originalInstruction: item.original_instruction ?? item.instruction,
originalInput: item.original_input ?? item.input ?? '',
originalOutput: item.original_output ?? item.output,
originalChosen: item.original_chosen ?? item.chosen ?? item.output ?? '',
originalRejected: item.original_rejected ?? item.rejected ?? '',
savedInstruction: item.instruction,
savedInput: item.input || '',
savedOutput: item.output,
savedChosen: item.chosen || item.output || '',
savedRejected: item.rejected || '',
savedStatus: item.status,
status: item.status,
error: item.error || undefined,
@@ -246,15 +254,34 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
}
}
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
function updateResultField(
id: string,
field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected',
value: string,
) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item[field] = value
const valid = item.instruction.trim() && item.output.trim()
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
if (field === 'chosen') item.output = value
const isDpo = bindings.outputType.value === 'dpo'
const valid = isDpo
? Boolean(
item.instruction.trim()
&& item.chosen.trim()
&& item.rejected.trim()
&& item.chosen.trim() !== item.rejected.trim()
)
: Boolean(item.instruction.trim() && item.output.trim())
item.error = valid
? undefined
: isDpo
? 'Instruction、Chosen、Rejected 均不能为空，且两个回答不能相同'
: 'Instruction 和 Output 不能为空'
const changed = item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
|| item.chosen !== item.originalChosen
|| item.rejected !== item.originalRejected
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
bindings.dirty.value = true
}
@@ -311,6 +338,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
item.instruction !== item.savedInstruction
|| item.input !== item.savedInput
|| item.output !== item.savedOutput
|| item.chosen !== item.savedChosen
|| item.rejected !== item.savedRejected
))
if (unsaved) {
selectedResultId.value = unsaved.id
@@ -409,12 +438,16 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
item.instruction !== item.savedInstruction
|| item.input !== item.savedInput
|| item.output !== item.savedOutput
|| item.chosen !== item.savedChosen
|| item.rejected !== item.savedRejected
))
for (const item of changed) {
const saved = await updateDataProcessResult(taskId, item.id, {
instruction: item.instruction,
input: item.input,
output: item.output,
chosen: item.chosen,
rejected: item.rejected,
expected_updated_at: item.updatedAt,
})
const index = results.value.findIndex((entry) => entry.id === item.id)
@@ -425,7 +458,13 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
const requiredFieldsInvalid = bindings.outputType.value === 'dpo'
? !item.instruction.trim()
|| !item.chosen.trim()
|| !item.rejected.trim()
|| item.chosen.trim() === item.rejected.trim()
: !item.instruction.trim() || !item.output.trim()
if (requiredFieldsInvalid || item.status === 'invalid') {
item.error ||= '结果未通过后端质量校验，请修改后重新保存'
item.status = 'invalid'
firstInvalidId ??= item.id