feat(data-process): 放宽问答生成数量上限
This commit is contained in:
@@ -4,6 +4,10 @@ import type {
|
||||
PreprocessOption,
|
||||
StructuredProcessOptions,
|
||||
} from './types'
|
||||
import {
|
||||
normalizeQaPairsGenerationCount,
|
||||
QA_PAIRS_GENERATION_LIMITS,
|
||||
} from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
@@ -41,6 +45,10 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:options', { ...props.options, ...value })
|
||||
}
|
||||
|
||||
function updateQaPairsPerRow(value: number | undefined) {
|
||||
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
@@ -97,15 +105,16 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每行生成数量</strong>
|
||||
<small>每行结构化数据生成的问答对数量</small>
|
||||
<small>支持 1~50 条;数量越大,处理耗时和 Token 消耗越高</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerRow"
|
||||
:min="1"
|
||||
:max="5"
|
||||
:min="QA_PAIRS_GENERATION_LIMITS.min"
|
||||
:max="QA_PAIRS_GENERATION_LIMITS.max"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
|
||||
@update:model-value="updateQaPairsPerRow"
|
||||
/>
|
||||
</div>
|
||||
<DatasetSplitEditor
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import { QA_PAIRS_GENERATION_LIMITS } from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
@@ -40,7 +41,7 @@ const UNSTRUCTURED_NUMBER_LIMITS = {
|
||||
chunkOverlap: { min: 0, max: 500 },
|
||||
minChunkSize: { min: 20, max: 500 },
|
||||
semanticBreakpointPercentile: { min: 1, max: 99 },
|
||||
qaPairsPerChunk: { min: 1, max: 3 },
|
||||
qaPairsPerChunk: QA_PAIRS_GENERATION_LIMITS,
|
||||
} as const
|
||||
|
||||
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
|
||||
@@ -285,12 +286,12 @@ defineExpose({ revealValidation })
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每个切片生成数量</strong>
|
||||
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
|
||||
<small>支持 1~50 条;数量越大,处理耗时和 Token 消耗越高</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerChunk"
|
||||
:min="1"
|
||||
:max="3"
|
||||
:min="QA_PAIRS_GENERATION_LIMITS.min"
|
||||
:max="QA_PAIRS_GENERATION_LIMITS.max"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
UnstructuredProcessOptions,
|
||||
ProcessType,
|
||||
} from './types'
|
||||
import { normalizeQaPairsGenerationCount } from './types'
|
||||
|
||||
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
|
||||
|
||||
@@ -128,7 +129,10 @@ export function createStructuredOptionsFromConfig(config: DataProcessConfig): St
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
|
||||
qaPairsPerRow: normalizeQaPairsGenerationCount(
|
||||
numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
|
||||
defaults.qaPairsPerRow,
|
||||
),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
@@ -163,7 +167,10 @@ export function createUnstructuredOptionsFromConfig(config: DataProcessConfig):
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
|
||||
qaPairsPerChunk: normalizeQaPairsGenerationCount(
|
||||
numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
|
||||
defaults.qaPairsPerChunk,
|
||||
),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,17 @@ export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||
|
||||
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
|
||||
export const QA_PAIRS_GENERATION_LIMITS = { min: 1, max: 50 } as const
|
||||
|
||||
export function normalizeQaPairsGenerationCount(value: unknown, fallback = 1): number {
|
||||
const parsed = Number(value)
|
||||
const normalized = Number.isFinite(parsed) ? Math.trunc(parsed) : fallback
|
||||
return Math.min(
|
||||
QA_PAIRS_GENERATION_LIMITS.max,
|
||||
Math.max(QA_PAIRS_GENERATION_LIMITS.min, normalized),
|
||||
)
|
||||
}
|
||||
|
||||
export type PreprocessOption =
|
||||
| 'clean_invalid'
|
||||
| 'detect_structure'
|
||||
|
||||
Reference in New Issue
Block a user