YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data-process): 放宽问答生成数量上限

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-27 09:11:58 +08:00
parent 9428c6b785
commit 762f866175
5 changed files with 48 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
frontend/scripts/regression-data-process-wizard.mjs
View File

@@ -382,7 +382,10 @@ for (const splitField of ['train', 'validation', 'test']) {
`数据集划分字段 ${splitField} 缺少 0～100 的整数限制`,
)
}
assert.match(structuredOptionsSource, /<el-input-number[\s\S]*options\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
assert.match(typesSource, /QA_PAIRS_GENERATION_LIMITS\s*=\s*\{ min: 1, max: 50 \}/, '问答生成数量统一范围必须为 1 到 50')
assert.match(typesSource, /function normalizeQaPairsGenerationCount[\s\S]*?Math\.trunc\(parsed\)[\s\S]*?QA_PAIRS_GENERATION_LIMITS\.max[\s\S]*?QA_PAIRS_GENERATION_LIMITS\.min/, '问答生成数量缺少统一整数归一化和边界裁剪')
assert.match(structuredOptionsSource, /options\.qaPairsPerRow[\s\S]*?:min="QA_PAIRS_GENERATION_LIMITS\.min"[\s\S]*?:max="QA_PAIRS_GENERATION_LIMITS\.max"/, '每行生成数量必须使用统一的 1 到 50 限制')
assert.match(structuredOptionsSource, /支持 1～50 条；数量越大，处理耗时和 Token 消耗越高/, '结构化生成数量缺少耗时与 Token 消耗说明')
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
@@ -604,7 +607,9 @@ for (const label of ['每个切片生成数量', '数据集划分']) {
for (const removedLabel of ['上下文范围', '问题类型', '跳过无法回答的内容']) {
assert.ok(!taskSetupFeatureSource.includes(removedLabel), `简化后仍显示低频选项：${removedLabel}`)
}
assert.match(unstructuredOptionsSource, /options\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3')
assert.match(unstructuredOptionsSource, /qaPairsPerChunk:\s*QA_PAIRS_GENERATION_LIMITS/, '每切片生成数量裁剪必须使用统一的 1 到 50 限制')
assert.match(unstructuredOptionsSource, /options\.qaPairsPerChunk[\s\S]*?:min="QA_PAIRS_GENERATION_LIMITS\.min"[\s\S]*?:max="QA_PAIRS_GENERATION_LIMITS\.max"/, '每个切片生成数量必须使用统一的 1 到 50 限制')
assert.match(unstructuredOptionsSource, /支持 1～50 条；数量越大，处理耗时和 Token 消耗越高/, '非结构化生成数量缺少耗时与 Token 消耗说明')
assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验')
assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界')
assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明')
@@ -617,6 +622,7 @@ assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 10
assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
assert.match(stateSource, /semanticBreakpointPercentile:\s*95/, '默认语义断点百分位必须为 95')
assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
assert.match(stateSource, /qaPairsPerRow:\s*1/, '默认每行必须生成 1 个问答对')
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
@@ -694,6 +700,8 @@ for (const marker of ['name: task.name.trim()', 'description: task.description.t
}
assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\)/, '配置反向映射没有区分缺失值与 false/0')
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填')

17
frontend/src/views/data-process/create/StructuredOptionsPanel.vue
View File

@@ -4,6 +4,10 @@ import type {
PreprocessOption,
StructuredProcessOptions,
} from './types'
import {
normalizeQaPairsGenerationCount,
QA_PAIRS_GENERATION_LIMITS,
} from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
@@ -41,6 +45,10 @@ function updateGenerationOptions(value: GenerationControlOptions) {
emit('update:options', { ...props.options, ...value })
}
function updateQaPairsPerRow(value: number | undefined) {
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = Array.from(new Set(value.filter(
@@ -97,15 +105,16 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每行生成数量</strong>
<small>每行结构化数据生成的问答对数量</small>
<small>支持 1～50 条；数量越大，处理耗时和 Token 消耗越高</small>
</div>
<el-input-number
:model-value="options.qaPairsPerRow"
:min="1"
:max="5"
:min="QA_PAIRS_GENERATION_LIMITS.min"
:max="QA_PAIRS_GENERATION_LIMITS.max"
:step="1"
:precision="0"
controls-position="right"
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
@update:model-value="updateQaPairsPerRow"
/>
</div>
<DatasetSplitEditor

9
frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue
View File

@@ -6,6 +6,7 @@ import type {
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
} from './types'
import { QA_PAIRS_GENERATION_LIMITS } from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
@@ -40,7 +41,7 @@ const UNSTRUCTURED_NUMBER_LIMITS = {
chunkOverlap: { min: 0, max: 500 },
minChunkSize: { min: 20, max: 500 },
semanticBreakpointPercentile: { min: 1, max: 99 },
qaPairsPerChunk: { min: 1, max: 3 },
qaPairsPerChunk: QA_PAIRS_GENERATION_LIMITS,
} as const
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
@@ -285,12 +286,12 @@ defineExpose({ revealValidation })
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每个切片生成数量</strong>
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
<small>支持 1～50 条；数量越大，处理耗时和 Token 消耗越高</small>
</div>
<el-input-number
:model-value="options.qaPairsPerChunk"
:min="1"
:max="3"
:min="QA_PAIRS_GENERATION_LIMITS.min"
:max="QA_PAIRS_GENERATION_LIMITS.max"
:step="1"
:precision="0"
controls-position="right"

11
frontend/src/views/data-process/create/dataProcessCreateState.ts
View File

@@ -7,6 +7,7 @@ import type {
UnstructuredProcessOptions,
ProcessType,
} from './types'
import { normalizeQaPairsGenerationCount } from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
@@ -128,7 +129,10 @@ export function createStructuredOptionsFromConfig(config: DataProcessConfig): St
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
qaPairsPerRow: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
defaults.qaPairsPerRow,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}
@@ -163,7 +167,10 @@ export function createUnstructuredOptionsFromConfig(config: DataProcessConfig):
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
qaPairsPerChunk: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
defaults.qaPairsPerChunk,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}

11
frontend/src/views/data-process/create/types.ts
View File

@@ -4,6 +4,17 @@ export type ProcessType = 'structured' | 'unstructured' | 'external'
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
export const QA_PAIRS_GENERATION_LIMITS = { min: 1, max: 50 } as const
export function normalizeQaPairsGenerationCount(value: unknown, fallback = 1): number {
const parsed = Number(value)
const normalized = Number.isFinite(parsed) ? Math.trunc(parsed) : fallback
return Math.min(
QA_PAIRS_GENERATION_LIMITS.max,
Math.max(QA_PAIRS_GENERATION_LIMITS.min, normalized),
)
}
export type PreprocessOption =
| 'clean_invalid'
| 'detect_structure'