Compare commits
2 Commits
d8a11e4949
...
f97245b814
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f97245b814 | ||
|
|
97cdb5cc68 |
@@ -566,6 +566,9 @@ def _run_generation(
|
|||||||
config, "generation_prompt", "generationPrompt", ""
|
config, "generation_prompt", "generationPrompt", ""
|
||||||
),
|
),
|
||||||
"output_type": output_type,
|
"output_type": output_type,
|
||||||
|
"reasoning_detail": _value(
|
||||||
|
config, "reasoning_detail", "reasoningDetail", "normal"
|
||||||
|
),
|
||||||
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
|
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
|
||||||
"json_mode": _value(config, "json_mode", "jsonMode", False),
|
"json_mode": _value(config, "json_mode", "jsonMode", False),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ class ModelGenerationError(ValueError):
|
|||||||
OUTPUT_TYPE_STANDARD = "standard"
|
OUTPUT_TYPE_STANDARD = "standard"
|
||||||
OUTPUT_TYPE_REASONING = "reasoning"
|
OUTPUT_TYPE_REASONING = "reasoning"
|
||||||
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
|
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
|
||||||
|
REASONING_DETAIL_NORMAL = "normal"
|
||||||
|
REASONING_DETAIL_DETAILED = "detailed"
|
||||||
|
SUPPORTED_REASONING_DETAILS = {
|
||||||
|
REASONING_DETAIL_NORMAL,
|
||||||
|
REASONING_DETAIL_DETAILED,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def chat_completions_url(value: str) -> str:
|
def chat_completions_url(value: str) -> str:
|
||||||
@@ -122,19 +128,32 @@ def _prompt_messages(
|
|||||||
start_index: int,
|
start_index: int,
|
||||||
total_count: int,
|
total_count: int,
|
||||||
output_type: str,
|
output_type: str,
|
||||||
|
reasoning_detail: str,
|
||||||
) -> list[dict[str, str]]:
|
) -> list[dict[str, str]]:
|
||||||
end_index = start_index + count - 1
|
end_index = start_index + count - 1
|
||||||
if output_type == OUTPUT_TYPE_REASONING:
|
if output_type == OUTPUT_TYPE_REASONING:
|
||||||
schema = '{"items":[{"instruction":"...","input":"...","reasoning":"...","answer":"..."}]}'
|
schema = '{"items":[{"instruction":"...","input":"...","reasoning":"...","answer":"..."}]}'
|
||||||
|
detail_rule = (
|
||||||
|
"推理详细程度为“详细”:完整展开问题条件、来源依据、中间计算或推导,"
|
||||||
|
"并在得出答案前核对结论;每一步都必须能从来源内容中验证。"
|
||||||
|
if reasoning_detail == REASONING_DETAIL_DETAILED
|
||||||
|
else
|
||||||
|
"推理详细程度为“普通”:只保留得出答案所需的关键依据和必要步骤,"
|
||||||
|
"避免冗长复述、套话和无依据扩展。"
|
||||||
|
)
|
||||||
output_rule = (
|
output_rule = (
|
||||||
|
"你正在生成用于训练推理模型的思维链数据,而不是普通问答数据。"
|
||||||
"instruction、reasoning 和 answer 均不得为空;reasoning 必须是基于来源内容、"
|
"instruction、reasoning 和 answer 均不得为空;reasoning 必须是基于来源内容、"
|
||||||
"可核对且简洁的推理步骤,answer 只写最终答案。"
|
f"可核对的推理过程,answer 只写最终答案。{detail_rule}"
|
||||||
"这是思维链输出模式,即使其他提示语要求省略分析,也不得省略 reasoning。"
|
"这是思维链输出模式,即使其他提示语要求省略分析,也不得省略 reasoning。"
|
||||||
"不要自行添加 <think> 标签,系统会在保存时统一组装。"
|
"不要自行添加 <think> 标签,系统会在保存时统一组装。"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}'
|
schema = '{"items":[{"instruction":"...","input":"...","output":"..."}]}'
|
||||||
output_rule = "instruction 和 output 不得为空,不要输出分析过程。"
|
output_rule = (
|
||||||
|
"你正在生成标准监督微调问答数据。instruction 和 output 不得为空;"
|
||||||
|
"output 只写最终答案,禁止输出分析、推理过程或 <think> 标签。"
|
||||||
|
)
|
||||||
schema_instruction = (
|
schema_instruction = (
|
||||||
f"必须只返回 JSON 对象,格式为 {schema};items 必须包含 {count} 条。"
|
f"必须只返回 JSON 对象,格式为 {schema};items 必须包含 {count} 条。"
|
||||||
f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条,"
|
f"这是总计 {total_count} 条中的第 {start_index}-{end_index} 条,"
|
||||||
@@ -176,6 +195,13 @@ def generate_model_records(
|
|||||||
output_type = str(config.get("output_type") or OUTPUT_TYPE_STANDARD).strip().lower()
|
output_type = str(config.get("output_type") or OUTPUT_TYPE_STANDARD).strip().lower()
|
||||||
if output_type not in SUPPORTED_OUTPUT_TYPES:
|
if output_type not in SUPPORTED_OUTPUT_TYPES:
|
||||||
raise ModelGenerationError(f"output_type must be one of {sorted(SUPPORTED_OUTPUT_TYPES)}")
|
raise ModelGenerationError(f"output_type must be one of {sorted(SUPPORTED_OUTPUT_TYPES)}")
|
||||||
|
reasoning_detail = str(
|
||||||
|
config.get("reasoning_detail") or REASONING_DETAIL_NORMAL
|
||||||
|
).strip().lower()
|
||||||
|
if reasoning_detail not in SUPPORTED_REASONING_DETAILS:
|
||||||
|
raise ModelGenerationError(
|
||||||
|
f"reasoning_detail must be one of {sorted(SUPPORTED_REASONING_DETAILS)}"
|
||||||
|
)
|
||||||
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||||
if not model_name:
|
if not model_name:
|
||||||
@@ -217,6 +243,7 @@ def generate_model_records(
|
|||||||
start_index=batch_start,
|
start_index=batch_start,
|
||||||
total_count=qa_pairs_per_item,
|
total_count=qa_pairs_per_item,
|
||||||
output_type=output_type,
|
output_type=output_type,
|
||||||
|
reasoning_detail=reasoning_detail,
|
||||||
),
|
),
|
||||||
"temperature": temperature,
|
"temperature": temperature,
|
||||||
"max_tokens": max_tokens,
|
"max_tokens": max_tokens,
|
||||||
|
|||||||
@@ -95,6 +95,17 @@ def _task_output_type(task: dict[str, Any]) -> str:
|
|||||||
return str(config.get("output_type") or config.get("outputType") or "standard")
|
return str(config.get("output_type") or config.get("outputType") or "standard")
|
||||||
|
|
||||||
|
|
||||||
|
def _task_reasoning_detail(task: dict[str, Any]) -> str:
|
||||||
|
config = _json_value(task.get("config"), {})
|
||||||
|
if not isinstance(config, dict):
|
||||||
|
return "normal"
|
||||||
|
return str(
|
||||||
|
config.get("reasoning_detail")
|
||||||
|
or config.get("reasoningDetail")
|
||||||
|
or "normal"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _reasoning_output_is_valid(value: Any) -> bool:
|
def _reasoning_output_is_valid(value: Any) -> bool:
|
||||||
match = re.fullmatch(
|
match = re.fullmatch(
|
||||||
r"\s*<think>\s*(?P<reasoning>[\s\S]*?)\s*</think>\s*(?P<answer>[\s\S]+?)\s*",
|
r"\s*<think>\s*(?P<reasoning>[\s\S]*?)\s*</think>\s*(?P<answer>[\s\S]+?)\s*",
|
||||||
@@ -1773,6 +1784,7 @@ class DataProcessStore:
|
|||||||
"storage_backend": "database",
|
"storage_backend": "database",
|
||||||
"source_task_id": task_id,
|
"source_task_id": task_id,
|
||||||
"output_type": _task_output_type(task),
|
"output_type": _task_output_type(task),
|
||||||
|
"reasoning_detail": _task_reasoning_detail(task),
|
||||||
"source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)],
|
"source_file_ids": [item["id"] for item in self._source_ids(conn, task_id)],
|
||||||
"source_result_ids": source_result_ids,
|
"source_result_ids": source_result_ids,
|
||||||
"format": payload.get("format") or "alpaca_jsonl",
|
"format": payload.get("format") or "alpaca_jsonl",
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
|
|||||||
assert payload["model"] == "qwen-plus"
|
assert payload["model"] == "qwen-plus"
|
||||||
assert payload["response_format"] == {"type": "json_object"}
|
assert payload["response_format"] == {"type": "json_object"}
|
||||||
assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
|
assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
|
||||||
|
assert "你正在生成标准监督微调问答数据" in payload["messages"][0]["content"]
|
||||||
|
assert "禁止输出分析、推理过程" in payload["messages"][0]["content"]
|
||||||
return httpx.Response(
|
return httpx.Response(
|
||||||
200,
|
200,
|
||||||
json={
|
json={
|
||||||
@@ -91,6 +93,8 @@ def test_generate_model_records_builds_reasoning_output_with_think_tags() -> Non
|
|||||||
payload = json.loads(request.content)
|
payload = json.loads(request.content)
|
||||||
system_prompt = payload["messages"][0]["content"]
|
system_prompt = payload["messages"][0]["content"]
|
||||||
assert '"reasoning":"...","answer":"..."' in system_prompt
|
assert '"reasoning":"...","answer":"..."' in system_prompt
|
||||||
|
assert "你正在生成用于训练推理模型的思维链数据" in system_prompt
|
||||||
|
assert "推理详细程度为“普通”" in system_prompt
|
||||||
assert "系统会在保存时统一组装" in system_prompt
|
assert "系统会在保存时统一组装" in system_prompt
|
||||||
content = "<think>模型接口自己的分析</think>" + json.dumps(
|
content = "<think>模型接口自己的分析</think>" + json.dumps(
|
||||||
{
|
{
|
||||||
@@ -126,6 +130,49 @@ def test_generate_model_records_builds_reasoning_output_with_think_tags() -> Non
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_model_records_uses_detailed_reasoning_instruction() -> None:
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
system_prompt = json.loads(request.content)["messages"][0]["content"]
|
||||||
|
assert "推理详细程度为“详细”" in system_prompt
|
||||||
|
assert "完整展开问题条件、来源依据、中间计算或推导" in system_prompt
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
json={
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"message": {
|
||||||
|
"content": json.dumps(
|
||||||
|
{
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"instruction": "计算报销总额",
|
||||||
|
"reasoning": "条件为交通费 30 元和餐费 20 元。分别核对后相加,30 + 20 = 50。",
|
||||||
|
"answer": "报销总额为 50 元。",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
records = generate_model_records(
|
||||||
|
[{"id": "preview-detailed", "edited_content": "交通费 30 元,餐费 20 元"}],
|
||||||
|
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||||
|
config={"output_type": "reasoning", "reasoning_detail": "detailed"},
|
||||||
|
task_id="task-detailed",
|
||||||
|
split={"train": 100, "validation": 0, "test": 0},
|
||||||
|
qa_pairs_per_item=1,
|
||||||
|
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert records[0]["status"] == "valid"
|
||||||
|
assert "分别核对后相加" in records[0]["output"]
|
||||||
|
|
||||||
|
|
||||||
def test_generate_model_records_marks_reasoning_without_reasoning_field_invalid() -> None:
|
def test_generate_model_records_marks_reasoning_without_reasoning_field_invalid() -> None:
|
||||||
response = {
|
response = {
|
||||||
"choices": [
|
"choices": [
|
||||||
@@ -392,3 +439,15 @@ def test_generate_model_records_rejects_unknown_output_type() -> None:
|
|||||||
split={"train": 100, "validation": 0, "test": 0},
|
split={"train": 100, "validation": 0, "test": 0},
|
||||||
qa_pairs_per_item=1,
|
qa_pairs_per_item=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_model_records_rejects_unknown_reasoning_detail() -> None:
|
||||||
|
with pytest.raises(ModelGenerationError, match="reasoning_detail"):
|
||||||
|
generate_model_records(
|
||||||
|
[],
|
||||||
|
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||||
|
config={"output_type": "reasoning", "reasoning_detail": "verbose"},
|
||||||
|
task_id="task-invalid-reasoning-detail",
|
||||||
|
split={"train": 100, "validation": 0, "test": 0},
|
||||||
|
qa_pairs_per_item=1,
|
||||||
|
)
|
||||||
|
|||||||
@@ -88,6 +88,9 @@ assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置
|
|||||||
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
|
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
|
||||||
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
|
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
|
||||||
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
|
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
|
||||||
|
assert.match(detailSource, /reasoning_detail:\s*'推理详细程度'/, '处理配置没有显示推理详细程度名称')
|
||||||
|
assert.match(detailSource, /value === 'detailed' \? '详细推理' : '普通推理'/, '处理配置没有转换推理详细程度枚举')
|
||||||
|
assert.match(detailSource, /key !== 'reasoning_detail' \|\| config\.output_type === 'reasoning'/, '标准回答任务不应展示无关的推理详细程度')
|
||||||
assert.match(detailSource, /const parsed = typeof value === 'number' \? value : Number\(value\)/, '详情页不兼容 PostgreSQL 数字字符串')
|
assert.match(detailSource, /const parsed = typeof value === 'number' \? value : Number\(value\)/, '详情页不兼容 PostgreSQL 数字字符串')
|
||||||
assert.match(detailSource, /new Date\(startTime\.value\)\.getTime\(\)/, '详情页没有在后端耗时缺失时按开始、完成时间回算')
|
assert.match(detailSource, /new Date\(startTime\.value\)\.getTime\(\)/, '详情页没有在后端耗时缺失时按开始、完成时间回算')
|
||||||
assert.match(detailSource, /outputCount \+ numeric\(detail\.value\?\.filtered_count\)/, '结果保留率没有使用输出和过滤结果的同口径分母')
|
assert.match(detailSource, /outputCount \+ numeric\(detail\.value\?\.filtered_count\)/, '结果保留率没有使用输出和过滤结果的同口径分母')
|
||||||
|
|||||||
@@ -352,8 +352,16 @@ assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]
|
|||||||
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
||||||
const expectedStructuredOptions = [
|
const expectedStructuredOptions = [
|
||||||
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
|
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
|
||||||
['detect_structure', '识别表格结构', '识别多级表头与合并单元格,并将嵌套字段展平'],
|
[
|
||||||
['deduplicate', '重复数据去重', '基于整行精确匹配和关键字段组合删除重复记录'],
|
'detect_structure',
|
||||||
|
'嵌套结构展平',
|
||||||
|
'展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'deduplicate',
|
||||||
|
'重复记录去重',
|
||||||
|
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||||
|
],
|
||||||
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
|
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
|
||||||
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
|
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
|
||||||
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||||
@@ -363,7 +371,7 @@ for (const [value, label, description] of expectedStructuredOptions) {
|
|||||||
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
|
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
|
||||||
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
|
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
|
||||||
}
|
}
|
||||||
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{ value: '([^']+)', label:/g)]
|
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
|
||||||
.map((match) => match[1])
|
.map((match) => match[1])
|
||||||
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
|
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
|
||||||
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
|
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
|
||||||
@@ -453,6 +461,7 @@ for (const field of [
|
|||||||
'generationModelId',
|
'generationModelId',
|
||||||
'generationPrompt',
|
'generationPrompt',
|
||||||
'outputType',
|
'outputType',
|
||||||
|
'reasoningDetail',
|
||||||
'qualityFilterEnabled',
|
'qualityFilterEnabled',
|
||||||
'filterLowQuality',
|
'filterLowQuality',
|
||||||
'filterShortContent',
|
'filterShortContent',
|
||||||
@@ -473,12 +482,14 @@ assert.match(modelSelectionSource, /class="form-section"/, '大模型选择步
|
|||||||
assert.doesNotMatch(modelSelectionSource, /max-width:\s*980px/, '大模型选择步骤不应使用比第一步更窄的固定内容宽度')
|
assert.doesNotMatch(modelSelectionSource, /max-width:\s*980px/, '大模型选择步骤不应使用比第一步更窄的固定内容宽度')
|
||||||
assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
|
assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
|
||||||
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
|
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
|
||||||
for (const label of ['大模型', '数据生成模型', '默认提示语', '输出类型', '标准回答', '思维链回答', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
|
for (const label of ['大模型', '数据生成模型', '默认提示语', '输出类型', '标准回答', '思维链回答', '推理详细程度', '普通推理(推荐)', '详细推理', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
|
||||||
assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`)
|
assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`)
|
||||||
}
|
}
|
||||||
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
|
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
|
||||||
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
|
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
|
||||||
assert.match(generationControlSource, /aria-label="输出类型"/, '输出类型选项缺少可访问名称')
|
assert.match(generationControlSource, /aria-label="输出类型"/, '输出类型选项缺少可访问名称')
|
||||||
|
assert.match(generationControlSource, /v-if="options\.outputType === 'reasoning'" class="output-type-row"/, '推理详细程度没有按思维链模式渐进显示')
|
||||||
|
assert.match(generationControlSource, /aria-label="推理详细程度"/, '推理详细程度缺少可访问名称')
|
||||||
assert.match(generationControlSource, /<think>推理过程<\/think>/, '思维链选项没有说明最终保存格式')
|
assert.match(generationControlSource, /<think>推理过程<\/think>/, '思维链选项没有说明最终保存格式')
|
||||||
assert.match(generationControlSource, /<el-select[\s\S]*?class="output-type-select"[\s\S]*?aria-label="输出类型"/, '输出类型必须使用右侧下拉选择')
|
assert.match(generationControlSource, /<el-select[\s\S]*?class="output-type-select"[\s\S]*?aria-label="输出类型"/, '输出类型必须使用右侧下拉选择')
|
||||||
assert.doesNotMatch(generationControlSource, /class="output-type-options"/, '输出类型不应继续使用横向按钮组')
|
assert.doesNotMatch(generationControlSource, /class="output-type-options"/, '输出类型不应继续使用横向按钮组')
|
||||||
@@ -503,9 +514,12 @@ assert.match(
|
|||||||
/\.model-config-group \.advanced-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*1fr/,
|
/\.model-config-group \.advanced-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*1fr/,
|
||||||
'大模型高级参数没有改为与第一步一致的纵向布局',
|
'大模型高级参数没有改为与第一步一致的纵向布局',
|
||||||
)
|
)
|
||||||
assert.match(stateSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语')
|
assert.match(stateSource, /const DEFAULT_STANDARD_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '标准回答缺少独立默认提示语')
|
||||||
assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语')
|
assert.match(stateSource, /const DEFAULT_REASONING_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '思维链回答缺少独立默认提示语')
|
||||||
assert.equal((stateSource.match(/outputType:\s*'standard'/g) || []).length, 2, '结构化与非结构化任务应默认生成标准回答')
|
assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_STANDARD_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务应默认使用标准回答提示语')
|
||||||
|
assert.match(generationControlSource, /isBuiltInGenerationPrompt\(props\.options\.generationPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)[\s\S]*?: props\.options\.generationPrompt/, '切换输出类型时没有在保留自定义提示语的前提下切换内置提示语')
|
||||||
|
assert.equal((stateSource.match(/^\s{4}outputType:\s*'standard',/gm) || []).length, 2, '结构化与非结构化任务应默认生成标准回答')
|
||||||
|
assert.equal((stateSource.match(/reasoningDetail:\s*'normal'/g) || []).length, 2, '结构化与非结构化任务应默认使用普通推理')
|
||||||
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
|
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
|
||||||
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
|
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
|
||||||
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
|
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
|
||||||
@@ -679,6 +693,7 @@ for (const [backendField, frontendField] of [
|
|||||||
['generation_model_id', 'generationModelId'],
|
['generation_model_id', 'generationModelId'],
|
||||||
['generation_prompt', 'generationPrompt'],
|
['generation_prompt', 'generationPrompt'],
|
||||||
['output_type', 'outputType'],
|
['output_type', 'outputType'],
|
||||||
|
['reasoning_detail', 'reasoningDetail'],
|
||||||
['temperature', 'temperature'],
|
['temperature', 'temperature'],
|
||||||
['max_tokens', 'maxTokens'],
|
['max_tokens', 'maxTokens'],
|
||||||
['json_mode', 'jsonMode'],
|
['json_mode', 'jsonMode'],
|
||||||
@@ -722,7 +737,9 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
|
|||||||
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
|
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
|
||||||
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
|
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
|
||||||
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
|
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
|
||||||
assert.match(stateSource, /outputType:\s*configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
||||||
|
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
|
||||||
|
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
|
||||||
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
|
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
|
||||||
assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射')
|
assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射')
|
||||||
assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填')
|
assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填')
|
||||||
@@ -800,6 +817,7 @@ for (const field of [
|
|||||||
'generationModelId',
|
'generationModelId',
|
||||||
'generationPrompt',
|
'generationPrompt',
|
||||||
'outputType',
|
'outputType',
|
||||||
|
'reasoningDetail',
|
||||||
'qualityFilterEnabled',
|
'qualityFilterEnabled',
|
||||||
'filterLowQuality',
|
'filterLowQuality',
|
||||||
'filterShortContent',
|
'filterShortContent',
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export type DataProcessType = 'structured' | 'unstructured' | 'external'
|
|||||||
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
|
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
|
||||||
export type DataProcessSplit = 'train' | 'validation' | 'test'
|
export type DataProcessSplit = 'train' | 'validation' | 'test'
|
||||||
export type DataProcessOutputType = 'standard' | 'reasoning'
|
export type DataProcessOutputType = 'standard' | 'reasoning'
|
||||||
|
export type DataProcessReasoningDetail = 'normal' | 'detailed'
|
||||||
|
|
||||||
export interface DataProcessPage<T> {
|
export interface DataProcessPage<T> {
|
||||||
items: T[]
|
items: T[]
|
||||||
@@ -22,6 +23,7 @@ export interface DataProcessDatasetSplit {
|
|||||||
export type DataProcessConfig = Record<string, unknown> & {
|
export type DataProcessConfig = Record<string, unknown> & {
|
||||||
dataset_split?: DataProcessDatasetSplit
|
dataset_split?: DataProcessDatasetSplit
|
||||||
output_type?: DataProcessOutputType
|
output_type?: DataProcessOutputType
|
||||||
|
reasoning_detail?: DataProcessReasoningDetail
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DataProcessTask {
|
export interface DataProcessTask {
|
||||||
|
|||||||
@@ -202,6 +202,7 @@ function toBackendConfig(): DataProcessConfig {
|
|||||||
generation_model_id: options.generationModelId,
|
generation_model_id: options.generationModelId,
|
||||||
generation_prompt: options.generationPrompt,
|
generation_prompt: options.generationPrompt,
|
||||||
output_type: options.outputType,
|
output_type: options.outputType,
|
||||||
|
reasoning_detail: options.reasoningDetail,
|
||||||
temperature: options.temperature,
|
temperature: options.temperature,
|
||||||
max_tokens: options.maxTokens,
|
max_tokens: options.maxTokens,
|
||||||
json_mode: options.jsonMode,
|
json_mode: options.jsonMode,
|
||||||
@@ -210,7 +211,6 @@ function toBackendConfig(): DataProcessConfig {
|
|||||||
filter_short_content: options.filterShortContent,
|
filter_short_content: options.filterShortContent,
|
||||||
min_output_length: options.minOutputLength,
|
min_output_length: options.minOutputLength,
|
||||||
}
|
}
|
||||||
|
|
||||||
if (processType.value === 'unstructured') {
|
if (processType.value === 'unstructured') {
|
||||||
return {
|
return {
|
||||||
...common,
|
...common,
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ const configLabelMap: Record<string, string> = {
|
|||||||
generation_model_id: '数据生成模型',
|
generation_model_id: '数据生成模型',
|
||||||
generation_prompt: '生成提示语',
|
generation_prompt: '生成提示语',
|
||||||
output_type: '输出类型',
|
output_type: '输出类型',
|
||||||
|
reasoning_detail: '推理详细程度',
|
||||||
temperature: '生成温度',
|
temperature: '生成温度',
|
||||||
max_tokens: '最大输出长度',
|
max_tokens: '最大输出长度',
|
||||||
json_mode: 'JSON 输出',
|
json_mode: 'JSON 输出',
|
||||||
@@ -99,8 +100,8 @@ const chunkMethodLabelMap: Record<string, string> = {
|
|||||||
|
|
||||||
const preprocessOptionLabelMap: Record<string, string> = {
|
const preprocessOptionLabelMap: Record<string, string> = {
|
||||||
clean_invalid: '清理无效数据',
|
clean_invalid: '清理无效数据',
|
||||||
detect_structure: '识别表格结构',
|
detect_structure: '嵌套结构展平',
|
||||||
deduplicate: '重复数据去重',
|
deduplicate: '重复记录去重',
|
||||||
normalize_format: '数据格式标准化',
|
normalize_format: '数据格式标准化',
|
||||||
filter_anomaly: '异常数据过滤',
|
filter_anomaly: '异常数据过滤',
|
||||||
desensitize: '敏感信息脱敏',
|
desensitize: '敏感信息脱敏',
|
||||||
@@ -225,15 +226,19 @@ const durationText = computed(() => {
|
|||||||
return minutes ? `${minutes} 分 ${restSeconds} 秒` : `${restSeconds} 秒`
|
return minutes ? `${minutes} 分 ${restSeconds} 秒` : `${restSeconds} 秒`
|
||||||
})
|
})
|
||||||
|
|
||||||
const configRows = computed(() => Object.entries(detail.value?.config || {})
|
const configRows = computed(() => {
|
||||||
|
const config = detail.value?.config || {}
|
||||||
|
return Object.entries(config)
|
||||||
.filter(([key]) => (
|
.filter(([key]) => (
|
||||||
key !== 'generation_model_snapshot'
|
key !== 'generation_model_snapshot'
|
||||||
|
&& (key !== 'reasoning_detail' || config.output_type === 'reasoning')
|
||||||
&& !/(?:password|secret|token|api_key)/i.test(key)
|
&& !/(?:password|secret|token|api_key)/i.test(key)
|
||||||
))
|
))
|
||||||
.map(([key, value]) => ({
|
.map(([key, value]) => ({
|
||||||
label: configLabelMap[key] || key.split('_').join(' '),
|
label: configLabelMap[key] || key.split('_').join(' '),
|
||||||
value: formatConfigValue(key, value),
|
value: formatConfigValue(key, value),
|
||||||
})))
|
}))
|
||||||
|
})
|
||||||
|
|
||||||
function formatConfigValue(key: string, value: unknown) {
|
function formatConfigValue(key: string, value: unknown) {
|
||||||
if (key === 'generation_model_id') {
|
if (key === 'generation_model_id') {
|
||||||
@@ -249,6 +254,9 @@ function formatConfigValue(key: string, value: unknown) {
|
|||||||
if (key === 'output_type') {
|
if (key === 'output_type') {
|
||||||
return value === 'reasoning' ? '思维链回答' : '标准回答'
|
return value === 'reasoning' ? '思维链回答' : '标准回答'
|
||||||
}
|
}
|
||||||
|
if (key === 'reasoning_detail') {
|
||||||
|
return value === 'detailed' ? '详细推理' : '普通推理'
|
||||||
|
}
|
||||||
if (key === 'dataset_split' && value && typeof value === 'object') {
|
if (key === 'dataset_split' && value && typeof value === 'object') {
|
||||||
const split = value as Partial<DataProcessDatasetSplit>
|
const split = value as Partial<DataProcessDatasetSplit>
|
||||||
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref } from 'vue'
|
|
||||||
import type { ModelItem } from '@/types'
|
import type { ModelItem } from '@/types'
|
||||||
import type { GenerationControlOptions } from './types'
|
import type { GenerationControlOptions } from './types'
|
||||||
|
import {
|
||||||
|
defaultGenerationPrompt,
|
||||||
|
isBuiltInGenerationPrompt,
|
||||||
|
} from './dataProcessCreateState'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
options: GenerationControlOptions
|
options: GenerationControlOptions
|
||||||
@@ -31,7 +34,18 @@ function updateQualityRules(value: Array<string | number>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function updateOutputType(value: string | number | boolean | undefined) {
|
function updateOutputType(value: string | number | boolean | undefined) {
|
||||||
updateField('outputType', value === 'reasoning' ? 'reasoning' : 'standard')
|
const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
|
||||||
|
emit('update:options', {
|
||||||
|
...props.options,
|
||||||
|
outputType,
|
||||||
|
generationPrompt: isBuiltInGenerationPrompt(props.options.generationPrompt)
|
||||||
|
? defaultGenerationPrompt(outputType)
|
||||||
|
: props.options.generationPrompt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateReasoningDetail(value: string | number | boolean | undefined) {
|
||||||
|
updateField('reasoningDetail', value === 'detailed' ? 'detailed' : 'normal')
|
||||||
}
|
}
|
||||||
|
|
||||||
const selectedQualityRules = () => [
|
const selectedQualityRules = () => [
|
||||||
@@ -91,7 +105,10 @@ function modelMeta(model: ModelItem) {
|
|||||||
<div class="model-field">
|
<div class="model-field">
|
||||||
<div class="field-copy">
|
<div class="field-copy">
|
||||||
<strong>默认提示语</strong>
|
<strong>默认提示语</strong>
|
||||||
<small>用于约束生成内容的格式、语气和完整性,可按任务需要修改</small>
|
<small v-if="options.outputType === 'reasoning'">
|
||||||
|
当前使用思维链专用提示语;系统还会按所选详细程度约束推理结构
|
||||||
|
</small>
|
||||||
|
<small v-else>当前使用标准回答提示语,只要求问题和最终答案</small>
|
||||||
</div>
|
</div>
|
||||||
<el-input
|
<el-input
|
||||||
class="prompt-input"
|
class="prompt-input"
|
||||||
@@ -177,9 +194,28 @@ function modelMeta(model: ModelItem) {
|
|||||||
<el-option label="思维链回答" value="reasoning" />
|
<el-option label="思维链回答" value="reasoning" />
|
||||||
</el-select>
|
</el-select>
|
||||||
</div>
|
</div>
|
||||||
|
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
|
||||||
|
<div class="field-copy">
|
||||||
|
<strong>推理详细程度</strong>
|
||||||
|
<small>控制推理过程保留关键步骤,或完整展开依据与中间推导</small>
|
||||||
|
</div>
|
||||||
|
<el-select
|
||||||
|
class="output-type-select"
|
||||||
|
:model-value="options.reasoningDetail"
|
||||||
|
aria-label="推理详细程度"
|
||||||
|
@update:model-value="updateReasoningDetail"
|
||||||
|
>
|
||||||
|
<el-option label="普通推理(推荐)" value="normal" />
|
||||||
|
<el-option label="详细推理" value="detailed" />
|
||||||
|
</el-select>
|
||||||
|
</div>
|
||||||
<p class="output-type-hint">
|
<p class="output-type-hint">
|
||||||
<template v-if="options.outputType === 'reasoning'">
|
<template v-if="options.outputType === 'reasoning'">
|
||||||
生成结果将按 <code><think>推理过程</think></code> 加最终答案的格式保存。
|
<template v-if="options.reasoningDetail === 'detailed'">
|
||||||
|
完整展开条件、来源依据、中间推导和结论核对,
|
||||||
|
</template>
|
||||||
|
<template v-else>保留关键依据与必要步骤,</template>
|
||||||
|
最终按 <code><think>推理过程</think></code> 加最终答案保存。
|
||||||
</template>
|
</template>
|
||||||
<template v-else>仅保存最终答案,不包含推理过程。</template>
|
<template v-else>仅保存最终答案,不包含推理过程。</template>
|
||||||
</p>
|
</p>
|
||||||
|
|||||||
@@ -27,8 +27,16 @@ const PREPROCESS_OPTIONS: Array<{
|
|||||||
description: string
|
description: string
|
||||||
}> = [
|
}> = [
|
||||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
||||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别多级表头与合并单元格,并将嵌套字段展平' },
|
{
|
||||||
{ value: 'deduplicate', label: '重复数据去重', description: '基于整行精确匹配和关键字段组合删除重复记录' },
|
value: 'detect_structure',
|
||||||
|
label: '嵌套结构展平',
|
||||||
|
description: '展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: 'deduplicate',
|
||||||
|
label: '重复记录去重',
|
||||||
|
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||||
|
},
|
||||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
||||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||||
|
|||||||
@@ -9,7 +9,25 @@ import type {
|
|||||||
} from './types'
|
} from './types'
|
||||||
import { normalizeQaPairsGenerationCount } from './types'
|
import { normalizeQaPairsGenerationCount } from './types'
|
||||||
|
|
||||||
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容,答案应事实清晰、语言自然,不要添加无关说明。'
|
const LEGACY_DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容,答案应事实清晰、语言自然,不要添加无关说明。'
|
||||||
|
|
||||||
|
export const DEFAULT_STANDARD_GENERATION_PROMPT = '你是一名专业的数据生成助手。请严格依据输入内容生成准确、完整、可直接用于监督微调的问答数据。只生成问题和最终答案,不输出分析、推理过程或来源中不存在的信息;答案应事实清晰、语言自然。'
|
||||||
|
|
||||||
|
export const DEFAULT_REASONING_GENERATION_PROMPT = '你是一名专业的推理数据生成助手。请严格依据输入内容生成问题、可核验的推理过程和最终答案。推理需要说明关键依据与必要步骤,不得引入来源中不存在的事实;最终答案应准确、完整且语言自然。'
|
||||||
|
|
||||||
|
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
|
||||||
|
return outputType === 'reasoning'
|
||||||
|
? DEFAULT_REASONING_GENERATION_PROMPT
|
||||||
|
: DEFAULT_STANDARD_GENERATION_PROMPT
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isBuiltInGenerationPrompt(value: string) {
|
||||||
|
return [
|
||||||
|
LEGACY_DEFAULT_GENERATION_PROMPT,
|
||||||
|
DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||||
|
DEFAULT_REASONING_GENERATION_PROMPT,
|
||||||
|
].includes(value)
|
||||||
|
}
|
||||||
|
|
||||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||||
return {
|
return {
|
||||||
@@ -18,8 +36,9 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
|||||||
qaPairsPerRow: 1,
|
qaPairsPerRow: 1,
|
||||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||||
generationModelId: '',
|
generationModelId: '',
|
||||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||||
outputType: 'standard',
|
outputType: 'standard',
|
||||||
|
reasoningDetail: 'normal',
|
||||||
temperature: 0.7,
|
temperature: 0.7,
|
||||||
maxTokens: 1024,
|
maxTokens: 1024,
|
||||||
jsonMode: false,
|
jsonMode: false,
|
||||||
@@ -52,8 +71,9 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
|||||||
qaPairsPerChunk: 1,
|
qaPairsPerChunk: 1,
|
||||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||||
generationModelId: '',
|
generationModelId: '',
|
||||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||||
outputType: 'standard',
|
outputType: 'standard',
|
||||||
|
reasoningDetail: 'normal',
|
||||||
temperature: 0.7,
|
temperature: 0.7,
|
||||||
maxTokens: 1024,
|
maxTokens: 1024,
|
||||||
jsonMode: false,
|
jsonMode: false,
|
||||||
@@ -92,12 +112,21 @@ function generationOptionsFromConfig(
|
|||||||
config: DataProcessConfig,
|
config: DataProcessConfig,
|
||||||
defaults: GenerationControlOptions,
|
defaults: GenerationControlOptions,
|
||||||
): GenerationControlOptions {
|
): GenerationControlOptions {
|
||||||
|
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
|
||||||
|
? 'reasoning'
|
||||||
|
: 'standard'
|
||||||
|
const configuredPrompt = String(
|
||||||
|
configValue(config, 'generation_prompt', defaults.generationPrompt),
|
||||||
|
)
|
||||||
return {
|
return {
|
||||||
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
|
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
|
||||||
generationPrompt: String(configValue(config, 'generation_prompt', defaults.generationPrompt)),
|
generationPrompt: isBuiltInGenerationPrompt(configuredPrompt)
|
||||||
outputType: configValue(config, 'output_type', defaults.outputType) === 'reasoning'
|
? defaultGenerationPrompt(outputType)
|
||||||
? 'reasoning'
|
: configuredPrompt,
|
||||||
: 'standard',
|
outputType,
|
||||||
|
reasoningDetail: configValue(config, 'reasoning_detail', defaults.reasoningDetail) === 'detailed'
|
||||||
|
? 'detailed'
|
||||||
|
: 'normal',
|
||||||
temperature: numberValue(config, 'temperature', defaults.temperature),
|
temperature: numberValue(config, 'temperature', defaults.temperature),
|
||||||
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
|
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
|
||||||
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
|
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
|
||||||
@@ -224,6 +253,7 @@ export function generationAffectingOptionsFor(
|
|||||||
generationModelId: options.generationModelId,
|
generationModelId: options.generationModelId,
|
||||||
generationPrompt: options.generationPrompt,
|
generationPrompt: options.generationPrompt,
|
||||||
outputType: options.outputType,
|
outputType: options.outputType,
|
||||||
|
reasoningDetail: options.reasoningDetail,
|
||||||
temperature: options.temperature,
|
temperature: options.temperature,
|
||||||
maxTokens: options.maxTokens,
|
maxTokens: options.maxTokens,
|
||||||
jsonMode: options.jsonMode,
|
jsonMode: options.jsonMode,
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
import type { DataProcessOutputType, DataProcessPreviewFileStatus } from '@/types/dataProcess'
|
import type {
|
||||||
|
DataProcessOutputType,
|
||||||
|
DataProcessPreviewFileStatus,
|
||||||
|
DataProcessReasoningDetail,
|
||||||
|
} from '@/types/dataProcess'
|
||||||
|
|
||||||
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||||
|
|
||||||
@@ -33,6 +37,7 @@ export interface GenerationControlOptions {
|
|||||||
generationModelId: string | number | ''
|
generationModelId: string | number | ''
|
||||||
generationPrompt: string
|
generationPrompt: string
|
||||||
outputType: DataProcessOutputType
|
outputType: DataProcessOutputType
|
||||||
|
reasoningDetail: DataProcessReasoningDetail
|
||||||
temperature: number
|
temperature: number
|
||||||
maxTokens: number
|
maxTokens: number
|
||||||
jsonMode: boolean
|
jsonMode: boolean
|
||||||
|
|||||||
Reference in New Issue
Block a user