import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess' import type { GenerationControlOptions, PreprocessOption, StructuredProcessOptions, UnstructuredPreprocessOption, UnstructuredProcessOptions, ProcessType, } from './types' import { normalizeQaPairsGenerationCount } from './types' const LEGACY_DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容,答案应事实清晰、语言自然,不要添加无关说明。' const PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT = '你是一名专业的数据生成助手。请严格依据输入内容生成准确、完整、可直接用于监督微调的问答数据。只生成问题和最终答案,不输出分析、推理过程或来源中不存在的信息;答案应事实清晰、语言自然。' const PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT = '你是一名专业的推理数据生成助手。请严格依据输入内容生成问题、可核验的推理过程和最终答案。推理需要说明关键依据与必要步骤,不得引入来源中不存在的事实;最终答案应准确、完整且语言自然。' export const DEFAULT_STANDARD_GENERATION_PROMPT = `你是一名专业的数据生成助手。请根据下方来源内容生成可用于监督微调的问答数据。 来源内容: {{ content }} 要求:严格依据来源生成问题和最终答案,不输出分析、中间思考过程或来源中不存在的信息;答案应准确、完整、语言自然。` export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名专业的推理数据生成助手。请根据下方来源内容生成可用于推理模型训练的问答数据。 来源内容: {{ content }} 要求:每条数据必须包含问题、可核验的中间思考过程和最终答案。中间思考过程需说明关键依据、必要步骤和中间计算,不得跳过推理只给结论,也不得引入来源中不存在的事实。` export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') { return outputType === 'reasoning' ? DEFAULT_REASONING_GENERATION_PROMPT : DEFAULT_STANDARD_GENERATION_PROMPT } export function isBuiltInGenerationPrompt(value: string) { return [ LEGACY_DEFAULT_GENERATION_PROMPT, PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT, PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT, DEFAULT_STANDARD_GENERATION_PROMPT, DEFAULT_REASONING_GENERATION_PROMPT, ].includes(value) } export function createDefaultStructuredOptions(): StructuredProcessOptions { return { preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], semanticEnrichment: false, qaPairsPerRow: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, generationModelId: '', generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT, outputType: 'standard', reasoningDetail: 'normal', temperature: 0.7, maxTokens: 1024, jsonMode: false, qualityFilterEnabled: false, filterLowQuality: true, filterShortContent: true, minOutputLength: 20, } } export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions { return { preprocessOptions: [ 'clean_invalid_content', 'detect_document_structure', 'merge_short_content', 'filter_low_quality', 'deduplicate_content', 'preserve_context', ], chunkMethod: 'layout_hybrid', chunkSize: 800, chunkOverlap: 100, minChunkSize: 100, semanticBreakpointPercentile: 95, preserveTables: true, preserveCodeBlocks: true, preserveLists: true, semanticEnrichment: false, qaPairsPerChunk: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, generationModelId: '', generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT, outputType: 'standard', reasoningDetail: 'normal', temperature: 0.7, maxTokens: 1024, jsonMode: false, qualityFilterEnabled: false, filterLowQuality: true, filterShortContent: true, minOutputLength: 20, } } function configValue(config: DataProcessConfig, key: string, fallback: T): T { return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback } function numberValue(config: DataProcessConfig, key: string, fallback: number): number { const value = Number(configValue(config, key, fallback)) return Number.isFinite(value) ? value : fallback } function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) { const value = config.dataset_split if (!value || typeof value !== 'object') return { ...fallback } const split = value as unknown as Record const splitNumber = (key: keyof DataProcessDatasetSplit) => { const parsed = Number(split[key]) return Number.isFinite(parsed) ? parsed : fallback[key] } return { train: splitNumber('train'), validation: splitNumber('validation'), test: splitNumber('test'), } } function generationOptionsFromConfig( config: DataProcessConfig, defaults: GenerationControlOptions, ): GenerationControlOptions { const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning' ? 'reasoning' : 'standard' const configuredPrompt = String( configValue(config, 'generation_prompt', defaults.generationPrompt), ) return { generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId), generationPrompt: isBuiltInGenerationPrompt(configuredPrompt) ? defaultGenerationPrompt(outputType) : configuredPrompt, outputType, reasoningDetail: configValue(config, 'reasoning_detail', defaults.reasoningDetail) === 'detailed' ? 'detailed' : 'normal', temperature: numberValue(config, 'temperature', defaults.temperature), maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens), jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)), qualityFilterEnabled: Boolean(configValue( config, 'quality_filter_enabled', defaults.qualityFilterEnabled, )), filterLowQuality: Boolean(configValue( config, 'filter_low_quality', defaults.filterLowQuality, )), filterShortContent: Boolean(configValue( config, 'filter_short_content', defaults.filterShortContent, )), minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength), } } export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions { const defaults = createDefaultStructuredOptions() const preprocessOptions = configValue(config, 'preprocess_options', []) return { ...defaults, ...generationOptionsFromConfig(config, defaults), preprocessOptions: Array.isArray(preprocessOptions) ? preprocessOptions.map(String) as PreprocessOption[] : defaults.preprocessOptions, semanticEnrichment: Boolean(configValue( config, 'semantic_enrichment', defaults.semanticEnrichment, )), qaPairsPerRow: normalizeQaPairsGenerationCount( numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow), defaults.qaPairsPerRow, ), datasetSplit: datasetSplitValue(config, defaults.datasetSplit), } } export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions { const defaults = createDefaultUnstructuredOptions() const preprocessOptions = configValue(config, 'preprocess_options', []) return { ...defaults, ...generationOptionsFromConfig(config, defaults), preprocessOptions: Array.isArray(preprocessOptions) ? preprocessOptions.map(String) as UnstructuredPreprocessOption[] : defaults.preprocessOptions, chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod), chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize), chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap), minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize), semanticBreakpointPercentile: numberValue( config, 'semantic_breakpoint_percentile', defaults.semanticBreakpointPercentile, ), preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)), preserveCodeBlocks: Boolean(configValue( config, 'preserve_code_blocks', defaults.preserveCodeBlocks, )), preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)), semanticEnrichment: Boolean(configValue( config, 'semantic_enrichment', defaults.semanticEnrichment, )), qaPairsPerChunk: normalizeQaPairsGenerationCount( numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk), defaults.qaPairsPerChunk, ), datasetSplit: datasetSplitValue(config, defaults.datasetSplit), } } export function previewAffectingOptionsFor( processType: ProcessType, structured: StructuredProcessOptions, unstructured: UnstructuredProcessOptions, ) { if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions } if (processType !== 'unstructured') return null const { preprocessOptions, chunkMethod, chunkSize, chunkOverlap, minChunkSize, semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, } = unstructured return { preprocessOptions, chunkMethod, chunkSize, chunkOverlap, minChunkSize, semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, } } export function generationAffectingOptionsFor( processType: ProcessType, structured: StructuredProcessOptions, unstructured: UnstructuredProcessOptions, ) { const options = processType === 'unstructured' ? unstructured : structured if (processType === 'external') return null const common = { semanticEnrichment: options.semanticEnrichment, datasetSplit: options.datasetSplit, generationModelId: options.generationModelId, generationPrompt: options.generationPrompt, outputType: options.outputType, reasoningDetail: options.reasoningDetail, temperature: options.temperature, maxTokens: options.maxTokens, jsonMode: options.jsonMode, qualityFilterEnabled: options.qualityFilterEnabled, filterLowQuality: options.filterLowQuality, filterShortContent: options.filterShortContent, minOutputLength: options.minOutputLength, } return processType === 'unstructured' ? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk } : { ...common, qaPairsPerRow: structured.qaPairsPerRow } }