feat(data-process): 放宽问答生成数量上限

This commit is contained in:
caoxiaozhu
2026-07-27 09:11:58 +08:00
parent 9428c6b785
commit 762f866175
5 changed files with 48 additions and 12 deletions

View File

@@ -4,6 +4,10 @@ import type {
PreprocessOption,
StructuredProcessOptions,
} from './types'
import {
normalizeQaPairsGenerationCount,
QA_PAIRS_GENERATION_LIMITS,
} from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
@@ -41,6 +45,10 @@ function updateGenerationOptions(value: GenerationControlOptions) {
emit('update:options', { ...props.options, ...value })
}
function updateQaPairsPerRow(value: number | undefined) {
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = Array.from(new Set(value.filter(
@@ -97,15 +105,16 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每行生成数量</strong>
<small>每行结构化数据生成的问答对数量</small>
<small>支持 150 数量越大处理耗时和 Token 消耗越高</small>
</div>
<el-input-number
:model-value="options.qaPairsPerRow"
:min="1"
:max="5"
:min="QA_PAIRS_GENERATION_LIMITS.min"
:max="QA_PAIRS_GENERATION_LIMITS.max"
:step="1"
:precision="0"
controls-position="right"
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
@update:model-value="updateQaPairsPerRow"
/>
</div>
<DatasetSplitEditor

View File

@@ -6,6 +6,7 @@ import type {
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
} from './types'
import { QA_PAIRS_GENERATION_LIMITS } from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
@@ -40,7 +41,7 @@ const UNSTRUCTURED_NUMBER_LIMITS = {
chunkOverlap: { min: 0, max: 500 },
minChunkSize: { min: 20, max: 500 },
semanticBreakpointPercentile: { min: 1, max: 99 },
qaPairsPerChunk: { min: 1, max: 3 },
qaPairsPerChunk: QA_PAIRS_GENERATION_LIMITS,
} as const
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
@@ -285,12 +286,12 @@ defineExpose({ revealValidation })
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每个切片生成数量</strong>
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
<small>支持 150 数量越大处理耗时和 Token 消耗越高</small>
</div>
<el-input-number
:model-value="options.qaPairsPerChunk"
:min="1"
:max="3"
:min="QA_PAIRS_GENERATION_LIMITS.min"
:max="QA_PAIRS_GENERATION_LIMITS.max"
:step="1"
:precision="0"
controls-position="right"

View File

@@ -7,6 +7,7 @@ import type {
UnstructuredProcessOptions,
ProcessType,
} from './types'
import { normalizeQaPairsGenerationCount } from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
@@ -128,7 +129,10 @@ export function createStructuredOptionsFromConfig(config: DataProcessConfig): St
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
qaPairsPerRow: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
defaults.qaPairsPerRow,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}
@@ -163,7 +167,10 @@ export function createUnstructuredOptionsFromConfig(config: DataProcessConfig):
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
qaPairsPerChunk: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
defaults.qaPairsPerChunk,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}

View File

@@ -4,6 +4,17 @@ export type ProcessType = 'structured' | 'unstructured' | 'external'
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
export const QA_PAIRS_GENERATION_LIMITS = { min: 1, max: 50 } as const
export function normalizeQaPairsGenerationCount(value: unknown, fallback = 1): number {
const parsed = Number(value)
const normalized = Number.isFinite(parsed) ? Math.trunc(parsed) : fallback
return Math.min(
QA_PAIRS_GENERATION_LIMITS.max,
Math.max(QA_PAIRS_GENERATION_LIMITS.min, normalized),
)
}
export type PreprocessOption =
| 'clean_invalid'
| 'detect_structure'