2 Commits

Author SHA1 Message Date
caoxiaozhu
f97245b814 feat(data-process): 增加可配置思维链生成 2026-07-27 14:41:38 +08:00
caoxiaozhu
97cdb5cc68 fix(data-process): 收准结构化预处理能力文案 2026-07-27 14:36:16 +08:00
13 changed files with 247 additions and 36 deletions

View File

@@ -566,6 +566,9 @@ def _run_generation(
config, "generation_prompt", "generationPrompt", "" config, "generation_prompt", "generationPrompt", ""
), ),
"output_type": output_type, "output_type": output_type,
"reasoning_detail": _value(
config, "reasoning_detail", "reasoningDetail", "normal"
),
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024), "max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
"json_mode": _value(config, "json_mode", "jsonMode", False), "json_mode": _value(config, "json_mode", "jsonMode", False),
} }

View File

@@ -25,6 +25,12 @@ class ModelGenerationError(ValueError):
OUTPUT_TYPE_STANDARD = "standard" OUTPUT_TYPE_STANDARD = "standard"
OUTPUT_TYPE_REASONING = "reasoning" OUTPUT_TYPE_REASONING = "reasoning"
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING} SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
REASONING_DETAIL_NORMAL = "normal"
REASONING_DETAIL_DETAILED = "detailed"
SUPPORTED_REASONING_DETAILS = {
REASONING_DETAIL_NORMAL,
REASONING_DETAIL_DETAILED,
}
def chat_completions_url(value: str) -> str: def chat_completions_url(value: str) -> str:
@@ -122,19 +128,32 @@ def _prompt_messages(
start_index: int, start_index: int,
total_count: int, total_count: int,
output_type: str, output_type: str,
reasoning_detail: str,
) -> list[dict[str, str]]: ) -> list[dict[str, str]]:
end_index = start_index + count - 1 end_index = start_index + count - 1
if output_type == OUTPUT_TYPE_REASONING: if output_type == OUTPUT_TYPE_REASONING:
schema = '{"items":[{"instruction":"...","input":"...","reasoning":"...","answer":"..."}]}' schema = '{"items":[{"instruction":"...","input":"...","reasoning":"...","answer":"..."}]}'
detail_rule = (
"推理详细程度为“详细”:完整展开问题条件、来源依据、中间计算或推导,"
"并在得出答案前核对结论;每一步都必须能从来源内容中验证。"
if reasoning_detail == REASONING_DETAIL_DETAILED
else
"推理详细程度为“普通”:只保留得出答案所需的关键依据和必要步骤,"
"避免冗长复述、套话和无依据扩展。"
)
output_rule = ( output_rule = (
"你正在生成用于训练推理模型的思维链数据,而不是普通问答数据。"
"instruction、reasoning 和 answer 均不得为空reasoning 必须是基于来源内容、" "instruction、reasoning 和 answer 均不得为空reasoning 必须是基于来源内容、"
"可核对且简洁的推理步骤answer 只写最终答案。" f"可核对的推理过程answer 只写最终答案。{detail_rule}"
"这是思维链输出模式,即使其他提示语要求省略分析,也不得省略 reasoning。" "这是思维链输出模式,即使其他提示语要求省略分析,也不得省略 reasoning。"
"不要自行添加 <think> 标签,系统会在保存时统一组装。" "不要自行添加 <think> 标签,系统会在保存时统一组装。"
) )
else: else:
schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}' schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}'
output_rule = "instruction 和 output 不得为空,不要输出分析过程。" output_rule = (
"你正在生成标准监督微调问答数据。instruction 和 output 不得为空;"
"output 只写最终答案,禁止输出分析、推理过程或 <think> 标签。"
)
schema_instruction = ( schema_instruction = (
f"必须只返回 JSON 对象,格式为 {schema}items 必须包含 {count} 条。" f"必须只返回 JSON 对象,格式为 {schema}items 必须包含 {count} 条。"
f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条," f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条,"
@@ -176,6 +195,13 @@ def generate_model_records(
output_type = str(config.get("output_type") or OUTPUT_TYPE_STANDARD).strip().lower() output_type = str(config.get("output_type") or OUTPUT_TYPE_STANDARD).strip().lower()
if output_type not in SUPPORTED_OUTPUT_TYPES: if output_type not in SUPPORTED_OUTPUT_TYPES:
raise ModelGenerationError(f"output_type must be one of {sorted(SUPPORTED_OUTPUT_TYPES)}") raise ModelGenerationError(f"output_type must be one of {sorted(SUPPORTED_OUTPUT_TYPES)}")
reasoning_detail = str(
config.get("reasoning_detail") or REASONING_DETAIL_NORMAL
).strip().lower()
if reasoning_detail not in SUPPORTED_REASONING_DETAILS:
raise ModelGenerationError(
f"reasoning_detail must be one of {sorted(SUPPORTED_REASONING_DETAILS)}"
)
endpoint = chat_completions_url(str(model.get("api_url") or "")) endpoint = chat_completions_url(str(model.get("api_url") or ""))
model_name = str(model.get("online_model_name") or model.get("name") or "").strip() model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
if not model_name: if not model_name:
@@ -217,6 +243,7 @@ def generate_model_records(
start_index=batch_start, start_index=batch_start,
total_count=qa_pairs_per_item, total_count=qa_pairs_per_item,
output_type=output_type, output_type=output_type,
reasoning_detail=reasoning_detail,
), ),
"temperature": temperature, "temperature": temperature,
"max_tokens": max_tokens, "max_tokens": max_tokens,

View File

@@ -95,6 +95,17 @@ def _task_output_type(task: dict[str, Any]) -> str:
return str(config.get("output_type") or config.get("outputType") or "standard") return str(config.get("output_type") or config.get("outputType") or "standard")
def _task_reasoning_detail(task: dict[str, Any]) -> str:
config = _json_value(task.get("config"), {})
if not isinstance(config, dict):
return "normal"
return str(
config.get("reasoning_detail")
or config.get("reasoningDetail")
or "normal"
)
def _reasoning_output_is_valid(value: Any) -> bool: def _reasoning_output_is_valid(value: Any) -> bool:
match = re.fullmatch( match = re.fullmatch(
r"\s*<think>\s*(?P<reasoning>[\s\S]*?)\s*</think>\s*(?P<answer>[\s\S]+?)\s*", r"\s*<think>\s*(?P<reasoning>[\s\S]*?)\s*</think>\s*(?P<answer>[\s\S]+?)\s*",
@@ -1773,6 +1784,7 @@ class DataProcessStore:
"storage_backend": "database", "storage_backend": "database",
"source_task_id": task_id, "source_task_id": task_id,
"output_type": _task_output_type(task), "output_type": _task_output_type(task),
"reasoning_detail": _task_reasoning_detail(task),
"source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)], "source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)],
"source_result_ids": source_result_ids, "source_result_ids": source_result_ids,
"format": payload.get("format") or "alpaca_jsonl", "format": payload.get("format") or "alpaca_jsonl",

View File

@@ -33,6 +33,8 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
assert payload["model"] == "qwen-plus" assert payload["model"] == "qwen-plus"
assert payload["response_format"] == {"type": "json_object"} assert payload["response_format"] == {"type": "json_object"}
assert "客户反馈页面加载慢" in payload["messages"][1]["content"] assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
assert "你正在生成标准监督微调问答数据" in payload["messages"][0]["content"]
assert "禁止输出分析、推理过程" in payload["messages"][0]["content"]
return httpx.Response( return httpx.Response(
200, 200,
json={ json={
@@ -91,6 +93,8 @@ def test_generate_model_records_builds_reasoning_output_with_think_tags() -> Non
payload = json.loads(request.content) payload = json.loads(request.content)
system_prompt = payload["messages"][0]["content"] system_prompt = payload["messages"][0]["content"]
assert '"reasoning":"...","answer":"..."' in system_prompt assert '"reasoning":"...","answer":"..."' in system_prompt
assert "你正在生成用于训练推理模型的思维链数据" in system_prompt
assert "推理详细程度为“普通”" in system_prompt
assert "系统会在保存时统一组装" in system_prompt assert "系统会在保存时统一组装" in system_prompt
content = "<think>模型接口自己的分析</think>" + json.dumps( content = "<think>模型接口自己的分析</think>" + json.dumps(
{ {
@@ -126,6 +130,49 @@ def test_generate_model_records_builds_reasoning_output_with_think_tags() -> Non
) )
def test_generate_model_records_uses_detailed_reasoning_instruction() -> None:
def handler(request: httpx.Request) -> httpx.Response:
system_prompt = json.loads(request.content)["messages"][0]["content"]
assert "推理详细程度为“详细”" in system_prompt
assert "完整展开问题条件、来源依据、中间计算或推导" in system_prompt
return httpx.Response(
200,
json={
"choices": [
{
"message": {
"content": json.dumps(
{
"items": [
{
"instruction": "计算报销总额",
"reasoning": "条件为交通费 30 元和餐费 20 元。分别核对后相加30 + 20 = 50。",
"answer": "报销总额为 50 元。",
}
]
},
ensure_ascii=False,
)
}
}
]
},
)
records = generate_model_records(
[{"id": "preview-detailed", "edited_content": "交通费 30 元,餐费 20 元"}],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"output_type": "reasoning", "reasoning_detail": "detailed"},
task_id="task-detailed",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
client=httpx.Client(transport=httpx.MockTransport(handler)),
)
assert records[0]["status"] == "valid"
assert "分别核对后相加" in records[0]["output"]
def test_generate_model_records_marks_reasoning_without_reasoning_field_invalid() -> None: def test_generate_model_records_marks_reasoning_without_reasoning_field_invalid() -> None:
response = { response = {
"choices": [ "choices": [
@@ -392,3 +439,15 @@ def test_generate_model_records_rejects_unknown_output_type() -> None:
split={"train": 100, "validation": 0, "test": 0}, split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1, qa_pairs_per_item=1,
) )
def test_generate_model_records_rejects_unknown_reasoning_detail() -> None:
with pytest.raises(ModelGenerationError, match="reasoning_detail"):
generate_model_records(
[],
model={"name": "model", "api_url": "https://model.example/v1"},
config={"output_type": "reasoning", "reasoning_detail": "verbose"},
task_id="task-invalid-reasoning-detail",
split={"train": 100, "validation": 0, "test": 0},
qa_pairs_per_item=1,
)

View File

@@ -88,6 +88,9 @@ assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文') assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称') assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举') assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
assert.match(detailSource, /reasoning_detail:\s*'推理详细程度'/, '处理配置没有显示推理详细程度名称')
assert.match(detailSource, /value === 'detailed' \? '详细推理' : '普通推理'/, '处理配置没有转换推理详细程度枚举')
assert.match(detailSource, /key !== 'reasoning_detail' \|\| config\.output_type === 'reasoning'/, '标准回答任务不应展示无关的推理详细程度')
assert.match(detailSource, /const parsed = typeof value === 'number' \? value : Number\(value\)/, '详情页不兼容 PostgreSQL 数字字符串') assert.match(detailSource, /const parsed = typeof value === 'number' \? value : Number\(value\)/, '详情页不兼容 PostgreSQL 数字字符串')
assert.match(detailSource, /new Date\(startTime\.value\)\.getTime\(\)/, '详情页没有在后端耗时缺失时按开始、完成时间回算') assert.match(detailSource, /new Date\(startTime\.value\)\.getTime\(\)/, '详情页没有在后端耗时缺失时按开始、完成时间回算')
assert.match(detailSource, /outputCount \+ numeric\(detail\.value\?\.filtered_count\)/, '结果保留率没有使用输出和过滤结果的同口径分母') assert.match(detailSource, /outputCount \+ numeric\(detail\.value\?\.filtered_count\)/, '结果保留率没有使用输出和过滤结果的同口径分母')

View File

@@ -352,8 +352,16 @@ assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示') assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
const expectedStructuredOptions = [ const expectedStructuredOptions = [
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'], ['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
['detect_structure', '识别表格结构', '识别多级表头与合并单元格,并将嵌套字段展平'], [
['deduplicate', '重复数据去重', '基于整行精确匹配和关键字段组合删除重复记录'], 'detect_structure',
'嵌套结构展平',
'展平嵌套对象和可解析的 JSON 字段Excel 表头与合并单元格在上传时自动解析',
],
[
'deduplicate',
'重复记录去重',
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
],
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'], ['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'], ['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'], ['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
@@ -363,7 +371,7 @@ for (const [value, label, description] of expectedStructuredOptions) {
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`) assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`) assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
} }
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{ value: '([^']+)', label:/g)] const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
.map((match) => match[1]) .map((match) => match[1])
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确') assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一') assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
@@ -453,6 +461,7 @@ for (const field of [
'generationModelId', 'generationModelId',
'generationPrompt', 'generationPrompt',
'outputType', 'outputType',
'reasoningDetail',
'qualityFilterEnabled', 'qualityFilterEnabled',
'filterLowQuality', 'filterLowQuality',
'filterShortContent', 'filterShortContent',
@@ -473,12 +482,14 @@ assert.match(modelSelectionSource, /class="form-section"/, '大模型选择步
assert.doesNotMatch(modelSelectionSource, /max-width:\s*980px/, '大模型选择步骤不应使用比第一步更窄的固定内容宽度') assert.doesNotMatch(modelSelectionSource, /max-width:\s*980px/, '大模型选择步骤不应使用比第一步更窄的固定内容宽度')
assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类') assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
for (const label of ['大模型', '数据生成模型', '默认提示语', '输出类型', '标准回答', '思维链回答', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { for (const label of ['大模型', '数据生成模型', '默认提示语', '输出类型', '标准回答', '思维链回答', '推理详细程度', '普通推理(推荐)', '详细推理', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`) assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`)
} }
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索') assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制') assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
assert.match(generationControlSource, /aria-label="输出类型"/, '输出类型选项缺少可访问名称') assert.match(generationControlSource, /aria-label="输出类型"/, '输出类型选项缺少可访问名称')
assert.match(generationControlSource, /v-if="options\.outputType === 'reasoning'" class="output-type-row"/, '推理详细程度没有按思维链模式渐进显示')
assert.match(generationControlSource, /aria-label="推理详细程度"/, '推理详细程度缺少可访问名称')
assert.match(generationControlSource, /&lt;think&gt;推理过程&lt;\/think&gt;/, '思维链选项没有说明最终保存格式') assert.match(generationControlSource, /&lt;think&gt;推理过程&lt;\/think&gt;/, '思维链选项没有说明最终保存格式')
assert.match(generationControlSource, /<el-select[\s\S]*?class="output-type-select"[\s\S]*?aria-label="输出类型"/, '输出类型必须使用右侧下拉选择') assert.match(generationControlSource, /<el-select[\s\S]*?class="output-type-select"[\s\S]*?aria-label="输出类型"/, '输出类型必须使用右侧下拉选择')
assert.doesNotMatch(generationControlSource, /class="output-type-options"/, '输出类型不应继续使用横向按钮组') assert.doesNotMatch(generationControlSource, /class="output-type-options"/, '输出类型不应继续使用横向按钮组')
@@ -503,9 +514,12 @@ assert.match(
/\.model-config-group \.advanced-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*1fr/, /\.model-config-group \.advanced-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*1fr/,
'大模型高级参数没有改为与第一步一致的纵向布局', '大模型高级参数没有改为与第一步一致的纵向布局',
) )
assert.match(stateSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') assert.match(stateSource, /const DEFAULT_STANDARD_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '标准回答缺少独立默认提示语')
assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') assert.match(stateSource, /const DEFAULT_REASONING_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '思维链回答缺少独立默认提示语')
assert.equal((stateSource.match(/outputType:\s*'standard'/g) || []).length, 2, '结构化与非结构化任务应默认生成标准回答') assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_STANDARD_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务应默认使用标准回答提示语')
assert.match(generationControlSource, /isBuiltInGenerationPrompt\(props\.options\.generationPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)[\s\S]*?: props\.options\.generationPrompt/, '切换输出类型时没有在保留自定义提示语的前提下切换内置提示语')
assert.equal((stateSource.match(/^\s{4}outputType:\s*'standard',/gm) || []).length, 2, '结构化与非结构化任务应默认生成标准回答')
assert.equal((stateSource.match(/reasoningDetail:\s*'normal'/g) || []).length, 2, '结构化与非结构化任务应默认使用普通推理')
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示') assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示') assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
@@ -679,6 +693,7 @@ for (const [backendField, frontendField] of [
['generation_model_id', 'generationModelId'], ['generation_model_id', 'generationModelId'],
['generation_prompt', 'generationPrompt'], ['generation_prompt', 'generationPrompt'],
['output_type', 'outputType'], ['output_type', 'outputType'],
['reasoning_detail', 'reasoningDetail'],
['temperature', 'temperature'], ['temperature', 'temperature'],
['max_tokens', 'maxTokens'], ['max_tokens', 'maxTokens'],
['json_mode', 'jsonMode'], ['json_mode', 'jsonMode'],
@@ -722,7 +737,9 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0') assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化') assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化') assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /outputType:\s*configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填') assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射') assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射') assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填') assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填')
@@ -800,6 +817,7 @@ for (const field of [
'generationModelId', 'generationModelId',
'generationPrompt', 'generationPrompt',
'outputType', 'outputType',
'reasoningDetail',
'qualityFilterEnabled', 'qualityFilterEnabled',
'filterLowQuality', 'filterLowQuality',
'filterShortContent', 'filterShortContent',

View File

@@ -5,6 +5,7 @@ export type DataProcessType = 'structured' | 'unstructured' | 'external'
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid' export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
export type DataProcessSplit = 'train' | 'validation' | 'test' export type DataProcessSplit = 'train' | 'validation' | 'test'
export type DataProcessOutputType = 'standard' | 'reasoning' export type DataProcessOutputType = 'standard' | 'reasoning'
export type DataProcessReasoningDetail = 'normal' | 'detailed'
export interface DataProcessPage<T> { export interface DataProcessPage<T> {
items: T[] items: T[]
@@ -22,6 +23,7 @@ export interface DataProcessDatasetSplit {
export type DataProcessConfig = Record<string, unknown> & { export type DataProcessConfig = Record<string, unknown> & {
dataset_split?: DataProcessDatasetSplit dataset_split?: DataProcessDatasetSplit
output_type?: DataProcessOutputType output_type?: DataProcessOutputType
reasoning_detail?: DataProcessReasoningDetail
} }
export interface DataProcessTask { export interface DataProcessTask {

View File

@@ -202,6 +202,7 @@ function toBackendConfig(): DataProcessConfig {
generation_model_id: options.generationModelId, generation_model_id: options.generationModelId,
generation_prompt: options.generationPrompt, generation_prompt: options.generationPrompt,
output_type: options.outputType, output_type: options.outputType,
reasoning_detail: options.reasoningDetail,
temperature: options.temperature, temperature: options.temperature,
max_tokens: options.maxTokens, max_tokens: options.maxTokens,
json_mode: options.jsonMode, json_mode: options.jsonMode,
@@ -210,7 +211,6 @@ function toBackendConfig(): DataProcessConfig {
filter_short_content: options.filterShortContent, filter_short_content: options.filterShortContent,
min_output_length: options.minOutputLength, min_output_length: options.minOutputLength,
} }
if (processType.value === 'unstructured') { if (processType.value === 'unstructured') {
return { return {
...common, ...common,

View File

@@ -71,6 +71,7 @@ const configLabelMap: Record<string, string> = {
generation_model_id: '数据生成模型', generation_model_id: '数据生成模型',
generation_prompt: '生成提示语', generation_prompt: '生成提示语',
output_type: '输出类型', output_type: '输出类型',
reasoning_detail: '推理详细程度',
temperature: '生成温度', temperature: '生成温度',
max_tokens: '最大输出长度', max_tokens: '最大输出长度',
json_mode: 'JSON 输出', json_mode: 'JSON 输出',
@@ -99,8 +100,8 @@ const chunkMethodLabelMap: Record<string, string> = {
const preprocessOptionLabelMap: Record<string, string> = { const preprocessOptionLabelMap: Record<string, string> = {
clean_invalid: '清理无效数据', clean_invalid: '清理无效数据',
detect_structure: '识别表格结构', detect_structure: '嵌套结构展平',
deduplicate: '重复数据去重', deduplicate: '重复记录去重',
normalize_format: '数据格式标准化', normalize_format: '数据格式标准化',
filter_anomaly: '异常数据过滤', filter_anomaly: '异常数据过滤',
desensitize: '敏感信息脱敏', desensitize: '敏感信息脱敏',
@@ -225,15 +226,19 @@ const durationText = computed(() => {
return minutes ? `${minutes}${restSeconds}` : `${restSeconds}` return minutes ? `${minutes}${restSeconds}` : `${restSeconds}`
}) })
const configRows = computed(() => Object.entries(detail.value?.config || {}) const configRows = computed(() => {
const config = detail.value?.config || {}
return Object.entries(config)
.filter(([key]) => ( .filter(([key]) => (
key !== 'generation_model_snapshot' key !== 'generation_model_snapshot'
&& (key !== 'reasoning_detail' || config.output_type === 'reasoning')
&& !/(?:password|secret|token|api_key)/i.test(key) && !/(?:password|secret|token|api_key)/i.test(key)
)) ))
.map(([key, value]) => ({ .map(([key, value]) => ({
label: configLabelMap[key] || key.split('_').join(' '), label: configLabelMap[key] || key.split('_').join(' '),
value: formatConfigValue(key, value), value: formatConfigValue(key, value),
}))) }))
})
function formatConfigValue(key: string, value: unknown) { function formatConfigValue(key: string, value: unknown) {
if (key === 'generation_model_id') { if (key === 'generation_model_id') {
@@ -249,6 +254,9 @@ function formatConfigValue(key: string, value: unknown) {
if (key === 'output_type') { if (key === 'output_type') {
return value === 'reasoning' ? '思维链回答' : '标准回答' return value === 'reasoning' ? '思维链回答' : '标准回答'
} }
if (key === 'reasoning_detail') {
return value === 'detailed' ? '详细推理' : '普通推理'
}
if (key === 'dataset_split' && value && typeof value === 'object') { if (key === 'dataset_split' && value && typeof value === 'object') {
const split = value as Partial<DataProcessDatasetSplit> const split = value as Partial<DataProcessDatasetSplit>
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%` return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`

View File

@@ -1,7 +1,10 @@
<script setup lang="ts"> <script setup lang="ts">
import { ref } from 'vue'
import type { ModelItem } from '@/types' import type { ModelItem } from '@/types'
import type { GenerationControlOptions } from './types' import type { GenerationControlOptions } from './types'
import {
defaultGenerationPrompt,
isBuiltInGenerationPrompt,
} from './dataProcessCreateState'
const props = defineProps<{ const props = defineProps<{
options: GenerationControlOptions options: GenerationControlOptions
@@ -31,7 +34,18 @@ function updateQualityRules(value: Array<string | number>) {
} }
function updateOutputType(value: string | number | boolean | undefined) { function updateOutputType(value: string | number | boolean | undefined) {
updateField('outputType', value === 'reasoning' ? 'reasoning' : 'standard') const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
emit('update:options', {
...props.options,
outputType,
generationPrompt: isBuiltInGenerationPrompt(props.options.generationPrompt)
? defaultGenerationPrompt(outputType)
: props.options.generationPrompt,
})
}
function updateReasoningDetail(value: string | number | boolean | undefined) {
updateField('reasoningDetail', value === 'detailed' ? 'detailed' : 'normal')
} }
const selectedQualityRules = () => [ const selectedQualityRules = () => [
@@ -91,7 +105,10 @@ function modelMeta(model: ModelItem) {
<div class="model-field"> <div class="model-field">
<div class="field-copy"> <div class="field-copy">
<strong>默认提示语</strong> <strong>默认提示语</strong>
<small>用于约束生成内容的格式语气和完整性可按任务需要修改</small> <small v-if="options.outputType === 'reasoning'">
当前使用思维链专用提示语系统还会按所选详细程度约束推理结构
</small>
<small v-else>当前使用标准回答提示语只要求问题和最终答案</small>
</div> </div>
<el-input <el-input
class="prompt-input" class="prompt-input"
@@ -177,9 +194,28 @@ function modelMeta(model: ModelItem) {
<el-option label="思维链回答" value="reasoning" /> <el-option label="思维链回答" value="reasoning" />
</el-select> </el-select>
</div> </div>
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
<div class="field-copy">
<strong>推理详细程度</strong>
<small>控制推理过程保留关键步骤或完整展开依据与中间推导</small>
</div>
<el-select
class="output-type-select"
:model-value="options.reasoningDetail"
aria-label="推理详细程度"
@update:model-value="updateReasoningDetail"
>
<el-option label="普通推理(推荐)" value="normal" />
<el-option label="详细推理" value="detailed" />
</el-select>
</div>
<p class="output-type-hint"> <p class="output-type-hint">
<template v-if="options.outputType === 'reasoning'"> <template v-if="options.outputType === 'reasoning'">
生成结果将按 <code>&lt;think&gt;推理过程&lt;/think&gt;</code> <template v-if="options.reasoningDetail === 'detailed'">
完整展开条件来源依据中间推导和结论核对
</template>
<template v-else>保留关键依据与必要步骤</template>
最终按 <code>&lt;think&gt;推理过程&lt;/think&gt;</code>
</template> </template>
<template v-else>仅保存最终答案不包含推理过程</template> <template v-else>仅保存最终答案不包含推理过程</template>
</p> </p>

View File

@@ -27,8 +27,16 @@ const PREPROCESS_OPTIONS: Array<{
description: string description: string
}> = [ }> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' }, { value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别多级表头与合并单元格,并将嵌套字段展平' }, {
{ value: 'deduplicate', label: '重复数据去重', description: '基于整行精确匹配和关键字段组合删除重复记录' }, value: 'detect_structure',
label: '嵌套结构展平',
description: '展平嵌套对象和可解析的 JSON 字段Excel 表头与合并单元格在上传时自动解析',
},
{
value: 'deduplicate',
label: '重复记录去重',
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
},
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' }, { value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' }, { value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' }, { value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },

View File

@@ -9,7 +9,25 @@ import type {
} from './types' } from './types'
import { normalizeQaPairsGenerationCount } from './types' import { normalizeQaPairsGenerationCount } from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容,答案应事实清晰、语言自然,不要添加无关说明。' const LEGACY_DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容,答案应事实清晰、语言自然,不要添加无关说明。'
export const DEFAULT_STANDARD_GENERATION_PROMPT = '你是一名专业的数据生成助手。请严格依据输入内容生成准确、完整、可直接用于监督微调的问答数据。只生成问题和最终答案,不输出分析、推理过程或来源中不存在的信息;答案应事实清晰、语言自然。'
export const DEFAULT_REASONING_GENERATION_PROMPT = '你是一名专业的推理数据生成助手。请严格依据输入内容生成问题、可核验的推理过程和最终答案。推理需要说明关键依据与必要步骤,不得引入来源中不存在的事实;最终答案应准确、完整且语言自然。'
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
return outputType === 'reasoning'
? DEFAULT_REASONING_GENERATION_PROMPT
: DEFAULT_STANDARD_GENERATION_PROMPT
}
export function isBuiltInGenerationPrompt(value: string) {
return [
LEGACY_DEFAULT_GENERATION_PROMPT,
DEFAULT_STANDARD_GENERATION_PROMPT,
DEFAULT_REASONING_GENERATION_PROMPT,
].includes(value)
}
export function createDefaultStructuredOptions(): StructuredProcessOptions { export function createDefaultStructuredOptions(): StructuredProcessOptions {
return { return {
@@ -18,8 +36,9 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
qaPairsPerRow: 1, qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 }, datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '', generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT, generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
outputType: 'standard', outputType: 'standard',
reasoningDetail: 'normal',
temperature: 0.7, temperature: 0.7,
maxTokens: 1024, maxTokens: 1024,
jsonMode: false, jsonMode: false,
@@ -52,8 +71,9 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
qaPairsPerChunk: 1, qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 }, datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '', generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT, generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
outputType: 'standard', outputType: 'standard',
reasoningDetail: 'normal',
temperature: 0.7, temperature: 0.7,
maxTokens: 1024, maxTokens: 1024,
jsonMode: false, jsonMode: false,
@@ -92,12 +112,21 @@ function generationOptionsFromConfig(
config: DataProcessConfig, config: DataProcessConfig,
defaults: GenerationControlOptions, defaults: GenerationControlOptions,
): GenerationControlOptions { ): GenerationControlOptions {
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
? 'reasoning'
: 'standard'
const configuredPrompt = String(
configValue(config, 'generation_prompt', defaults.generationPrompt),
)
return { return {
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId), generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
generationPrompt: String(configValue(config, 'generation_prompt', defaults.generationPrompt)), generationPrompt: isBuiltInGenerationPrompt(configuredPrompt)
outputType: configValue(config, 'output_type', defaults.outputType) === 'reasoning' ? defaultGenerationPrompt(outputType)
? 'reasoning' : configuredPrompt,
: 'standard', outputType,
reasoningDetail: configValue(config, 'reasoning_detail', defaults.reasoningDetail) === 'detailed'
? 'detailed'
: 'normal',
temperature: numberValue(config, 'temperature', defaults.temperature), temperature: numberValue(config, 'temperature', defaults.temperature),
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens), maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)), jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
@@ -224,6 +253,7 @@ export function generationAffectingOptionsFor(
generationModelId: options.generationModelId, generationModelId: options.generationModelId,
generationPrompt: options.generationPrompt, generationPrompt: options.generationPrompt,
outputType: options.outputType, outputType: options.outputType,
reasoningDetail: options.reasoningDetail,
temperature: options.temperature, temperature: options.temperature,
maxTokens: options.maxTokens, maxTokens: options.maxTokens,
jsonMode: options.jsonMode, jsonMode: options.jsonMode,

View File

@@ -1,4 +1,8 @@
import type { DataProcessOutputType, DataProcessPreviewFileStatus } from '@/types/dataProcess' import type {
DataProcessOutputType,
DataProcessPreviewFileStatus,
DataProcessReasoningDetail,
} from '@/types/dataProcess'
export type ProcessType = 'structured' | 'unstructured' | 'external' export type ProcessType = 'structured' | 'unstructured' | 'external'
@@ -33,6 +37,7 @@ export interface GenerationControlOptions {
generationModelId: string | number | '' generationModelId: string | number | ''
generationPrompt: string generationPrompt: string
outputType: DataProcessOutputType outputType: DataProcessOutputType
reasoningDetail: DataProcessReasoningDetail
temperature: number temperature: number
maxTokens: number maxTokens: number
jsonMode: boolean jsonMode: boolean