feat(data-process): 放宽问答生成数量上限

This commit is contained in:
caoxiaozhu
2026-07-27 09:11:58 +08:00
parent 9428c6b785
commit 762f866175
5 changed files with 48 additions and 12 deletions

View File

@@ -7,6 +7,7 @@ import type {
UnstructuredProcessOptions,
ProcessType,
} from './types'
import { normalizeQaPairsGenerationCount } from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
@@ -128,7 +129,10 @@ export function createStructuredOptionsFromConfig(config: DataProcessConfig): St
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
qaPairsPerRow: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
defaults.qaPairsPerRow,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}
@@ -163,7 +167,10 @@ export function createUnstructuredOptionsFromConfig(config: DataProcessConfig):
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
qaPairsPerChunk: normalizeQaPairsGenerationCount(
numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
defaults.qaPairsPerChunk,
),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}