diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 9dd3b1f..032619b 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -556,12 +556,16 @@ def _run_generation( if task["process_type"] == "unstructured" else _value(config, "qa_pairs_per_row", "qaPairsPerRow", 1) ) + output_type = str( + _value(config, "output_type", "outputType", "standard") + ).strip().lower() if generation_model: runtime_config = { **config, "generation_prompt": _value( config, "generation_prompt", "generationPrompt", "" ), + "output_type": output_type, "max_tokens": _value(config, "max_tokens", "maxTokens", 1024), "json_mode": _value(config, "json_mode", "jsonMode", False), } @@ -583,6 +587,8 @@ def _run_generation( qa_pairs_per_item=int(pairs or 1), on_progress=report_progress, ) + elif output_type == "reasoning": + raise InvalidStateError("思维链输出必须配置可用的数据生成模型") else: generated = generate_standard_records( preview_items, diff --git a/backend/app/modules/data_process/generation.py b/backend/app/modules/data_process/generation.py index 1c1e14f..cc6ce7e 100644 --- a/backend/app/modules/data_process/generation.py +++ b/backend/app/modules/data_process/generation.py @@ -22,6 +22,11 @@ class ModelGenerationError(ValueError): """模型配置、响应或调用失败。""" +OUTPUT_TYPE_STANDARD = "standard" +OUTPUT_TYPE_REASONING = "reasoning" +SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING} + + def chat_completions_url(value: str) -> str: """把域名、基础 URL 或完整地址统一为 chat completions 地址。""" @@ -52,7 +57,9 @@ def _message_content(payload: Mapping[str, Any]) -> str: try: content = payload["choices"][0]["message"]["content"] except (KeyError, IndexError, TypeError) as exc: - raise ModelGenerationError("model response does not contain choices[0].message.content") from exc + raise ModelGenerationError( + "model response does not contain choices[0].message.content" + ) from exc if isinstance(content, str): return content if isinstance(content, list): @@ -67,7 +74,14 @@ def _message_content(payload: Mapping[str, Any]) -> str: def _json_payload(content: str) -> Any: - cleaned = re.sub(r"[\s\S]*?", "", content, flags=re.IGNORECASE).strip() + # 只移除模型在 JSON 之前自行输出的思考过程，不能破坏 JSON 字段中的训练内容。 + cleaned = re.sub( + r"^\s*[\s\S]*?\s*", + "", + content, + count=1, + flags=re.IGNORECASE, + ).strip() fenced = re.fullmatch(r"```(?:json)?\s*([\s\S]*?)\s*```", cleaned, flags=re.IGNORECASE) if fenced: cleaned = fenced.group(1).strip() @@ -107,19 +121,27 @@ def _prompt_messages( *, start_index: int, total_count: int, + output_type: str, ) -> list[dict[str, str]]: end_index = start_index + count - 1 + if output_type == OUTPUT_TYPE_REASONING: + schema = '{"items":[{"instruction":"...","input":"...","reasoning":"...","answer":"..."}]}' + output_rule = ( + "instruction、reasoning 和 answer 均不得为空；reasoning 必须是基于来源内容、" + "可核对且简洁的推理步骤，answer 只写最终答案。" + "这是思维链输出模式，即使其他提示语要求省略分析，也不得省略 reasoning。" + "不要自行添加 标签，系统会在保存时统一组装。" + ) + else: + schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}' + output_rule = "instruction 和 output 不得为空，不要输出分析过程。" schema_instruction = ( - f"必须只返回 JSON 对象，格式为 {{\"items\":[{{\"instruction\":\"...\"," - f"\"input\":\"...\",\"output\":\"...\"}}]}}；items 必须包含 {count} 条。" + f"必须只返回 JSON 对象，格式为 {schema}；items 必须包含 {count} 条。" f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条，" "各条必须使用不同的提问角度和表述，避免重复。" - "instruction 和 output 不得为空，不要输出 Markdown 代码围栏或分析过程。" - ) - base_prompt = ( - normalize_text(prompt) - or "请根据来源内容生成可用于监督微调的问答数据。" + f"{output_rule}不要输出 Markdown 代码围栏或 JSON 之外的说明。" ) + base_prompt = normalize_text(prompt) or "请根据来源内容生成可用于监督微调的问答数据。" if "{{ content }}" in base_prompt: user_prompt = base_prompt.replace("{{ content }}", content) return [ @@ -150,9 +172,10 @@ def generate_model_records( """ if not 1 <= qa_pairs_per_item <= MAX_QA_PAIRS_PER_ITEM: - raise ModelGenerationError( - f"qa_pairs_per_item must be in [1, {MAX_QA_PAIRS_PER_ITEM}]" - ) + raise ModelGenerationError(f"qa_pairs_per_item must be in [1, {MAX_QA_PAIRS_PER_ITEM}]") + output_type = str(config.get("output_type") or OUTPUT_TYPE_STANDARD).strip().lower() + if output_type not in SUPPORTED_OUTPUT_TYPES: + raise ModelGenerationError(f"output_type must be one of {sorted(SUPPORTED_OUTPUT_TYPES)}") endpoint = chat_completions_url(str(model.get("api_url") or "")) model_name = str(model.get("online_model_name") or model.get("name") or "").strip() if not model_name: @@ -193,6 +216,7 @@ def generate_model_records( batch_count, start_index=batch_start, total_count=qa_pairs_per_item, + output_type=output_type, ), "temperature": temperature, "max_tokens": max_tokens, @@ -212,12 +236,8 @@ def generate_model_records( response.raise_for_status() body = response.json() if not isinstance(body, Mapping): - raise ModelGenerationError( - "model response body must be a JSON object" - ) - candidate_items = _result_items( - _json_payload(_message_content(body)) - ) + raise ModelGenerationError("model response body must be a JSON object") + candidate_items = _result_items(_json_payload(_message_content(body))) if len(candidate_items) < batch_count: raise ModelGenerationError( "model response contains fewer result objects than requested: " @@ -266,19 +286,56 @@ def generate_model_records( input_text = normalize_text( str(value.get("input") or value.get("context") or "") ) - output = normalize_text( - str( - value.get("output") - or value.get("answer") - or value.get("response") - or "" + if output_type == OUTPUT_TYPE_REASONING: + reasoning = normalize_text( + re.sub( + r"", + "", + str(value.get("reasoning") or value.get("analysis") or ""), + flags=re.IGNORECASE, + ) ) - ) + answer = normalize_text( + re.sub( + r"", + "", + str( + value.get("answer") + or value.get("final_answer") + or value.get("output") + or "" + ), + flags=re.IGNORECASE, + ) + ) + output = ( + f"\n{reasoning}\n\n{answer}" + if reasoning and answer + else answer or (f"\n{reasoning}\n" if reasoning else "") + ) + valid = bool(instruction and reasoning and answer) + missing_error = "model result is missing instruction, reasoning or answer" + else: + output = normalize_text( + str( + value.get("output") + or value.get("answer") + or value.get("response") + or "" + ) + ) + output = normalize_text( + re.sub( + r"[\s\S]*?(?:|$)", + "", + output, + flags=re.IGNORECASE, + ) + ) + valid = bool(instruction and output) + missing_error = "model result is missing instruction or output" raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}" - result_id = ( - f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}" - ) - valid = bool(instruction and output) + result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}" results.append( { "id": result_id, @@ -290,11 +347,7 @@ def generate_model_records( "original_input": input_text, "original_output": output, "status": "valid" if valid else "invalid", - "error": ( - None - if valid - else "model result is missing instruction or output" - ), + "error": (None if valid else missing_error), "split": "train", } ) diff --git a/backend/app/modules/data_process/store.py b/backend/app/modules/data_process/store.py index df6ed3e..b6d584a 100644 --- a/backend/app/modules/data_process/store.py +++ b/backend/app/modules/data_process/store.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib import json +import re import uuid from collections.abc import Iterator, Sequence from contextlib import contextmanager @@ -87,6 +88,31 @@ def _json_value(value: Any, default: Any) -> Any: return default +def _task_output_type(task: dict[str, Any]) -> str: + config = _json_value(task.get("config"), {}) + if not isinstance(config, dict): + return "standard" + return str(config.get("output_type") or config.get("outputType") or "standard") + + +def _reasoning_output_is_valid(value: Any) -> bool: + match = re.fullmatch( + r"\s*\s*(?P[\s\S]*?)\s*\s*(?P[\s\S]+?)\s*", + str(value or ""), + flags=re.IGNORECASE, + ) + return bool( + match + and match.group("reasoning").strip() + and match.group("answer").strip() + and all( + tag not in part.lower() + for tag in (" Any: if key in config: return config[key] @@ -1565,10 +1591,13 @@ class DataProcessStore: raise ConflictError("data process result was modified by another request") merged = {**current, **values} quality = payload.get("quality_score") or {} - hard_valid = bool( - str(merged.get("instruction") or "").strip() - and str(merged.get("output") or "").strip() + instruction_valid = bool(str(merged.get("instruction") or "").strip()) + output_valid = bool(str(merged.get("output") or "").strip()) + reasoning_valid = ( + _task_output_type(task) != "reasoning" + or _reasoning_output_is_valid(merged.get("output")) ) + hard_valid = instruction_valid and output_valid and reasoning_valid quality_valid = bool(quality.get("is_valid", hard_valid)) changed = any( str(merged.get(field) or "") @@ -1580,8 +1609,16 @@ class DataProcessStore: ) values["status"] = status flags = quality.get("flags") if isinstance(quality, dict) else None + format_error = ( + "思维链输出必须包含非空的 ... 推理过程和最终答案" + if instruction_valid and output_valid and not reasoning_valid + else "Instruction 和 Output 不能为空" + if not instruction_valid or not output_valid + else None + ) values["error"] = ", ".join(str(flag) for flag in flags or []) or ( - "quality validation failed" if status == "invalid" else None + format_error + or ("quality validation failed" if status == "invalid" else None) ) values["updated_at"] = utcnow() assignments = ", ".join(f"{key}=%s" for key in values) @@ -1671,6 +1708,10 @@ class DataProcessStore: if row["status"] == "invalid" or not str(row.get("instruction") or "").strip() or not str(row.get("output") or "").strip() + or ( + _task_output_type(task) == "reasoning" + and not _reasoning_output_is_valid(row.get("output")) + ) ) if invalid_count: raise InvalidStateError(f"task contains {invalid_count} invalid results") @@ -1731,6 +1772,7 @@ class DataProcessStore: "source": "data_process", "storage_backend": "database", "source_task_id": task_id, + "output_type": _task_output_type(task), "source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)], "source_result_ids": source_result_ids, "format": payload.get("format") or "alpaca_jsonl", diff --git a/backend/tests/test_data_process_api.py b/backend/tests/test_data_process_api.py index 6dc36ef..6f5db51 100644 --- a/backend/tests/test_data_process_api.py +++ b/backend/tests/test_data_process_api.py @@ -1405,6 +1405,40 @@ def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> N assert store.tasks[task_id]["status"] == "running" +def test_reasoning_output_requires_generation_model() -> None: + store = FakeDataProcessStore() + task = store.create_task( + { + "name": "思维链模型校验", + "process_type": "structured", + "config": {"output_type": "reasoning"}, + } + ) + task_id = task["id"] + store.replace_preview_items( + task_id, + [ + { + "source_file_id": None, + "original_content": "需要推理的来源内容", + "edited_content": "需要推理的来源内容", + "status": "manual", + } + ], + ) + started = store.start_generation(task_id, replace_existing=True) + + data_process_endpoint._run_generation( + store, + task_id, + started["generation_run_id"], + ) + + failed = store.get_task(task_id) + assert failed["status"] == "failed" + assert failed["failure_reason"] == "思维链输出必须配置可用的数据生成模型" + + def test_result_status_cannot_be_forged_by_client(tmp_path: Path) -> None: client, _, _ = make_client(tmp_path) task_id = client.post( diff --git a/backend/tests/test_data_process_generation.py b/backend/tests/test_data_process_generation.py index 2961bcf..303fdca 100644 --- a/backend/tests/test_data_process_generation.py +++ b/backend/tests/test_data_process_generation.py @@ -86,6 +86,119 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None: assert progress_updates == [(1, 1)] +def test_generate_model_records_builds_reasoning_output_with_think_tags() -> None: + def handler(request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content) + system_prompt = payload["messages"][0]["content"] + assert '"reasoning":"...","answer":"..."' in system_prompt + assert "系统会在保存时统一组装" in system_prompt + content = "模型接口自己的分析" + json.dumps( + { + "items": [ + { + "instruction": "计算两项费用合计", + "input": "交通费 30 元，餐费 20 元", + "reasoning": "先识别两项费用，再计算 30 + 20。", + "answer": "合计 50 元。", + } + ] + }, + ensure_ascii=False, + ) + return httpx.Response( + 200, + json={"choices": [{"message": {"content": content}}]}, + ) + + records = generate_model_records( + [{"id": "preview-reasoning", "edited_content": "交通费 30 元，餐费 20 元"}], + model={"name": "model", "api_url": "https://model.example/v1"}, + config={"output_type": "reasoning"}, + task_id="task-reasoning", + split={"train": 100, "validation": 0, "test": 0}, + qa_pairs_per_item=1, + client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + assert records[0]["status"] == "valid" + assert records[0]["output"] == ( + "\n先识别两项费用,再计算 30 + 20。\n\n合计 50 元。" + ) + + +def test_generate_model_records_marks_reasoning_without_reasoning_field_invalid() -> None: + response = { + "choices": [ + { + "message": { + "content": json.dumps( + { + "items": [ + { + "instruction": "问题", + "answer": "只有最终答案", + } + ] + }, + ensure_ascii=False, + ) + } + } + ] + } + client = httpx.Client( + transport=httpx.MockTransport(lambda _: httpx.Response(200, json=response)) + ) + + records = generate_model_records( + [{"id": "preview-missing-reasoning", "edited_content": "来源正文"}], + model={"name": "model", "api_url": "https://model.example/v1"}, + config={"output_type": "reasoning"}, + task_id="task-missing-reasoning", + split={"train": 100, "validation": 0, "test": 0}, + qa_pairs_per_item=1, + client=client, + ) + + assert records[0]["status"] == "invalid" + assert records[0]["output"] == "只有最终答案" + assert "reasoning" in records[0]["error"] + + +def test_standard_output_removes_model_think_block() -> None: + content = json.dumps( + { + "items": [ + { + "instruction": "问题", + "output": "不应保存的分析最终答案", + } + ] + }, + ensure_ascii=False, + ) + client = httpx.Client( + transport=httpx.MockTransport( + lambda _: httpx.Response( + 200, + json={"choices": [{"message": {"content": content}}]}, + ) + ) + ) + + records = generate_model_records( + [{"id": "preview-standard", "edited_content": "来源正文"}], + model={"name": "model", "api_url": "https://model.example/v1"}, + config={"output_type": "standard"}, + task_id="task-standard", + split={"train": 100, "validation": 0, "test": 0}, + qa_pairs_per_item=1, + client=client, + ) + + assert records[0]["output"] == "最终答案" + + def test_generate_model_records_keeps_partial_failure_for_manual_repair() -> None: client = httpx.Client( transport=httpx.MockTransport( @@ -267,3 +380,15 @@ def test_generate_model_records_rejects_out_of_range_count( split={"train": 100, "validation": 0, "test": 0}, qa_pairs_per_item=qa_pairs_per_item, ) + + +def test_generate_model_records_rejects_unknown_output_type() -> None: + with pytest.raises(ModelGenerationError, match="output_type"): + generate_model_records( + [], + model={"name": "model", "api_url": "https://model.example/v1"}, + config={"output_type": "unknown"}, + task_id="task-invalid-output", + split={"train": 100, "validation": 0, "test": 0}, + qa_pairs_per_item=1, + ) diff --git a/backend/tests/test_data_process_store.py b/backend/tests/test_data_process_store.py index 220a220..285a3f9 100644 --- a/backend/tests/test_data_process_store.py +++ b/backend/tests/test_data_process_store.py @@ -15,6 +15,7 @@ from app.modules.data_process.store import ( InvalidStateError, _decode_row, _preview_config_changed, + _reasoning_output_is_valid, _source_storage_descriptor, ) @@ -114,14 +115,25 @@ class _PublishConnection: ) if normalized.startswith("INSERT INTO dataset_records"): self.records.append( - {"dataset_id": params[1], "line_no": params[4], "split": params[5]} + { + "dataset_id": params[1], + "line_no": params[4], + "split": params[5], + "output": params[8], + "raw": json.loads(params[9]), + } ) return _Result() class _PublishStore(DataProcessStore): - def __init__(self, conn: _PublishConnection): + def __init__( + self, + conn: _PublishConnection, + task_config: dict[str, Any] | None = None, + ): self._conn = conn + self._task_config = task_config or {} @contextmanager def connect(self) -> Iterator[_PublishConnection]: @@ -135,7 +147,7 @@ class _PublishStore(DataProcessStore): "id": task_id, "status": "completed", "description": "", - "config": {}, + "config": self._task_config, "output_dataset_id": train_dataset and train_dataset["id"], } @@ -967,7 +979,11 @@ def test_publish_creates_three_independent_datasets_with_exact_counts() -> None: "status": "valid", "instruction": f"问题 {index}", "input": "", - "output": f"答案 {index}", + "output": ( + "\n先读取制度条款。\n\n答案 0" + if index == 0 + else f"答案 {index}" + ), "preview_item_id": f"preview-{index}", } for index in range(28) @@ -993,6 +1009,10 @@ def test_publish_creates_three_independent_datasets_with_exact_counts() -> None: item["id"] for item in conn.datasets } assert len(conn.records) == 28 + reasoning_record = next( + item for item in conn.records if item["output"].startswith("") + ) + assert reasoning_record["raw"]["output"] == reasoning_record["output"] assert published["dataset"]["type"] == "train" assert len(published["datasets"]) == 3 assert published["split_counts"] == {"train": 22, "validation": 3, "test": 3} @@ -1078,6 +1098,49 @@ def test_publish_keeps_all_three_datasets_when_a_small_split_is_empty() -> None: assert len(conn.files) == 3 +@pytest.mark.parametrize( + ("output", "expected"), + [ + ("\n推理步骤\n\n最终答案", True), + ("\n最终答案", False), + ("只有推理", False), + ("没有标签的最终答案", False), + ("外层嵌套答案", False), + ], +) +def test_reasoning_output_validator_requires_one_complete_pair( + output: str, + expected: bool, +) -> None: + assert _reasoning_output_is_valid(output) is expected + + +def test_publish_rejects_invalid_reasoning_output_format() -> None: + conn = _PublishConnection( + [ + { + "id": "result-reasoning-invalid", + "status": "valid", + "instruction": "需要推理的问题", + "input": "", + "output": "只有最终答案", + "preview_item_id": "preview-reasoning-invalid", + } + ] + ) + + with pytest.raises(InvalidStateError, match="1 invalid results"): + _PublishStore(conn, {"output_type": "reasoning"}).publish( + "task-reasoning-invalid", + { + "dataset_name": "无效思维链", + "storage_type": "local", + "format": "alpaca_jsonl", + "split": {"train": 80, "validation": 10, "test": 10}, + }, + ) + + def test_source_storage_descriptor_accepts_owned_local_and_legacy_db_references() -> None: task_id = "dpt_task" source_file_id = "dpsf_source" diff --git a/frontend/scripts/regression-data-process-detail.mjs b/frontend/scripts/regression-data-process-detail.mjs index 25efb08..be872d4 100644 --- a/frontend/scripts/regression-data-process-detail.mjs +++ b/frontend/scripts/regression-data-process-detail.mjs @@ -86,6 +86,8 @@ assert.doesNotMatch(detailSource, /^\s*max-width:\s*\d+px/m, '详情页不应使 assert.match(detailSource, /!\/\(\?:password\|secret\|token\|api_key\)\/i\.test\(key\)/, '处理配置没有过滤敏感凭据字段') assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置仍直接展示内部模型快照') assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文') +assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称') +assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举') assert.match(detailSource, /const parsed = typeof value === 'number' \? value : Number\(value\)/, '详情页不兼容 PostgreSQL 数字字符串') assert.match(detailSource, /new Date\(startTime\.value\)\.getTime\(\)/, '详情页没有在后端耗时缺失时按开始、完成时间回算') assert.match(detailSource, /outputCount \+ numeric\(detail\.value\?\.filtered_count\)/, '结果保留率没有使用输出和过滤结果的同口径分母') diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index 9d0a078..f73ad83 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -452,6 +452,7 @@ assert.match(apiSource, /regenerateDataProcessTask[\s\S]*?\/regenerate`/, '重 for (const field of [ 'generationModelId', 'generationPrompt', + 'outputType', 'qualityFilterEnabled', 'filterLowQuality', 'filterShortContent', @@ -472,11 +473,14 @@ assert.match(modelSelectionSource, /class="form-section"/, '大模型选择步 assert.doesNotMatch(modelSelectionSource, /max-width:\s*980px/, '大模型选择步骤不应使用比第一步更窄的固定内容宽度') assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') assert.doesNotMatch(generationControlSource, /

大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类') -for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { +for (const label of ['大模型', '数据生成模型', '默认提示语', '输出类型', '标准回答', '思维链回答', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { assert.ok(generationControlSource.includes(label), `生成控制界面缺少：${label}`) } assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索') assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制') +assert.match(generationControlSource, /aria-label="输出类型"/, '输出类型选项缺少可访问名称') +assert.match(generationControlSource, /<think>推理过程<\/think>/, '思维链选项没有说明最终保存格式') +assert.match(generationControlSource, /\.output-type-options[\s\S]*?min-height:\s*44px/, '输出类型选项的点击区域不足 44px') assert.match(generationControlSource, /\.model-field\s*\{[\s\S]*?display:\s*flex[\s\S]*?flex-direction:\s*column/, '大模型字段没有使用稳定的纵向表单布局') assert.match(generationControlSource, /\.generation-config-group\s*\{[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有保留统一配置面板边框') assert.match(generationControlSource, /\.model-config-group\s*\{[\s\S]*?padding:\s*0[\s\S]*?border:\s*0/, '独立大模型步骤仍存在嵌套卡片挤压') @@ -487,6 +491,7 @@ assert.match( ) assert.match(stateSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') +assert.equal((stateSource.match(/outputType:\s*'standard'/g) || []).length, 2, '结构化与非结构化任务应默认生成标准回答') assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示') assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示') assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') @@ -659,6 +664,7 @@ for (const [backendField, frontendField] of [ ['semantic_enrichment', 'semanticEnrichment'], ['generation_model_id', 'generationModelId'], ['generation_prompt', 'generationPrompt'], + ['output_type', 'outputType'], ['temperature', 'temperature'], ['max_tokens', 'maxTokens'], ['json_mode', 'jsonMode'], @@ -702,6 +708,7 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\ assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0') assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化') assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化') +assert.match(stateSource, /outputType:\s*configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填') assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射') assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射') assert.match(stateSource, /configValue\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填') @@ -778,6 +785,7 @@ for (const field of [ 'datasetSplit', 'generationModelId', 'generationPrompt', + 'outputType', 'qualityFilterEnabled', 'filterLowQuality', 'filterShortContent', diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 573e5c8..b99fe5c 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -4,6 +4,7 @@ export type DataProcessStatus = 'pending' | 'running' | 'completed' | 'failed' | export type DataProcessType = 'structured' | 'unstructured' | 'external' export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid' export type DataProcessSplit = 'train' | 'validation' | 'test' +export type DataProcessOutputType = 'standard' | 'reasoning' export interface DataProcessPage { items: T[] @@ -20,6 +21,7 @@ export interface DataProcessDatasetSplit { export type DataProcessConfig = Record & { dataset_split?: DataProcessDatasetSplit + output_type?: DataProcessOutputType } export interface DataProcessTask { diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index f869587..999db3e 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -195,13 +195,13 @@ function toBackendConfig(): DataProcessConfig { const options = processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value - const common = { preprocess_options: [...options.preprocessOptions], semantic_enrichment: options.semanticEnrichment, dataset_split: { ...options.datasetSplit }, generation_model_id: options.generationModelId, generation_prompt: options.generationPrompt, + output_type: options.outputType, temperature: options.temperature, max_tokens: options.maxTokens, json_mode: options.jsonMode, diff --git a/frontend/src/views/data-process/DataProcessDetailView.vue b/frontend/src/views/data-process/DataProcessDetailView.vue index 2c873b0..0c7a029 100644 --- a/frontend/src/views/data-process/DataProcessDetailView.vue +++ b/frontend/src/views/data-process/DataProcessDetailView.vue @@ -70,6 +70,7 @@ const configLabelMap: Record = { dataset_split: '数据集划分', generation_model_id: '数据生成模型', generation_prompt: '生成提示语', + output_type: '输出类型', temperature: '生成温度', max_tokens: '最大输出长度', json_mode: 'JSON 输出', @@ -245,6 +246,9 @@ function formatConfigValue(key: string, value: unknown) { if (key === 'chunk_method' && typeof value === 'string') { return chunkMethodLabelMap[value] || value } + if (key === 'output_type') { + return value === 'reasoning' ? '思维链回答' : '标准回答' + } if (key === 'dataset_split' && value && typeof value === 'object') { const split = value as Partial return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%` diff --git a/frontend/src/views/data-process/create/GenerationOptionsPanel.vue b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue index cccfa32..438de61 100644 --- a/frontend/src/views/data-process/create/GenerationOptionsPanel.vue +++ b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue @@ -30,6 +30,10 @@ function updateQualityRules(value: Array) { }) } +function updateOutputType(value: string | number | boolean | undefined) { + updateField('outputType', value === 'reasoning' ? 'reasoning' : 'standard') +} + const selectedQualityRules = () => [ props.options.filterLowQuality ? 'low_quality' : '', props.options.filterShortContent ? 'short_content' : '', @@ -106,6 +110,28 @@ function modelMeta(model: ModelItem) { +
+
+ 输出类型 + 控制答案是否包含可用于推理模型训练的思维链内容 +
+ + 标准回答 + 思维链回答 + +

+ + +

+
+