import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess' import type { GenerationControlOptions, PreprocessOption, StructuredProcessOptions, UnstructuredPreprocessOption, UnstructuredProcessOptions, ProcessType, } from './types' import { normalizeQaPairsGenerationCount } from './types' export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。' export function createDefaultStructuredOptions(): StructuredProcessOptions { return { preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], semanticEnrichment: false, qaPairsPerRow: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, generationModelId: '', generationPrompt: DEFAULT_GENERATION_PROMPT, temperature: 0.7, maxTokens: 1024, jsonMode: false, qualityFilterEnabled: false, filterLowQuality: true, filterShortContent: true, minOutputLength: 20, } } export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions { return { preprocessOptions: [ 'clean_invalid_content', 'detect_document_structure', 'merge_short_content', 'filter_low_quality', 'deduplicate_content', 'preserve_context', ], chunkMethod: 'layout_hybrid', chunkSize: 800, chunkOverlap: 100, minChunkSize: 100, semanticBreakpointPercentile: 95, preserveTables: true, preserveCodeBlocks: true, preserveLists: true, semanticEnrichment: false, qaPairsPerChunk: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, generationModelId: '', generationPrompt: DEFAULT_GENERATION_PROMPT, temperature: 0.7, maxTokens: 1024, jsonMode: false, qualityFilterEnabled: false, filterLowQuality: true, filterShortContent: true, minOutputLength: 20, } } function configValue(config: DataProcessConfig, key: string, fallback: T): T { return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback } function numberValue(config: DataProcessConfig, key: string, fallback: number): number { const value = Number(configValue(config, key, fallback)) return Number.isFinite(value) ? value : fallback } function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) { const value = config.dataset_split if (!value || typeof value !== 'object') return { ...fallback } const split = value as unknown as Record const splitNumber = (key: keyof DataProcessDatasetSplit) => { const parsed = Number(split[key]) return Number.isFinite(parsed) ? parsed : fallback[key] } return { train: splitNumber('train'), validation: splitNumber('validation'), test: splitNumber('test'), } } function generationOptionsFromConfig( config: DataProcessConfig, defaults: GenerationControlOptions, ): GenerationControlOptions { return { generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId), generationPrompt: String(configValue(config, 'generation_prompt', defaults.generationPrompt)), temperature: numberValue(config, 'temperature', defaults.temperature), maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens), jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)), qualityFilterEnabled: Boolean(configValue( config, 'quality_filter_enabled', defaults.qualityFilterEnabled, )), filterLowQuality: Boolean(configValue( config, 'filter_low_quality', defaults.filterLowQuality, )), filterShortContent: Boolean(configValue( config, 'filter_short_content', defaults.filterShortContent, )), minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength), } } export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions { const defaults = createDefaultStructuredOptions() const preprocessOptions = configValue(config, 'preprocess_options', []) return { ...defaults, ...generationOptionsFromConfig(config, defaults), preprocessOptions: Array.isArray(preprocessOptions) ? preprocessOptions.map(String) as PreprocessOption[] : defaults.preprocessOptions, semanticEnrichment: Boolean(configValue( config, 'semantic_enrichment', defaults.semanticEnrichment, )), qaPairsPerRow: normalizeQaPairsGenerationCount( numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow), defaults.qaPairsPerRow, ), datasetSplit: datasetSplitValue(config, defaults.datasetSplit), } } export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions { const defaults = createDefaultUnstructuredOptions() const preprocessOptions = configValue(config, 'preprocess_options', []) return { ...defaults, ...generationOptionsFromConfig(config, defaults), preprocessOptions: Array.isArray(preprocessOptions) ? preprocessOptions.map(String) as UnstructuredPreprocessOption[] : defaults.preprocessOptions, chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod), chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize), chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap), minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize), semanticBreakpointPercentile: numberValue( config, 'semantic_breakpoint_percentile', defaults.semanticBreakpointPercentile, ), preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)), preserveCodeBlocks: Boolean(configValue( config, 'preserve_code_blocks', defaults.preserveCodeBlocks, )), preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)), semanticEnrichment: Boolean(configValue( config, 'semantic_enrichment', defaults.semanticEnrichment, )), qaPairsPerChunk: normalizeQaPairsGenerationCount( numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk), defaults.qaPairsPerChunk, ), datasetSplit: datasetSplitValue(config, defaults.datasetSplit), } } export function previewAffectingOptionsFor( processType: ProcessType, structured: StructuredProcessOptions, unstructured: UnstructuredProcessOptions, ) { if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions } if (processType !== 'unstructured') return null const { preprocessOptions, chunkMethod, chunkSize, chunkOverlap, minChunkSize, semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, } = unstructured return { preprocessOptions, chunkMethod, chunkSize, chunkOverlap, minChunkSize, semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, } } export function generationAffectingOptionsFor( processType: ProcessType, structured: StructuredProcessOptions, unstructured: UnstructuredProcessOptions, ) { const options = processType === 'unstructured' ? unstructured : structured if (processType === 'external') return null const common = { semanticEnrichment: options.semanticEnrichment, datasetSplit: options.datasetSplit, generationModelId: options.generationModelId, generationPrompt: options.generationPrompt, temperature: options.temperature, maxTokens: options.maxTokens, jsonMode: options.jsonMode, qualityFilterEnabled: options.qualityFilterEnabled, filterLowQuality: options.filterLowQuality, filterShortContent: options.filterShortContent, minOutputLength: options.minOutputLength, } return processType === 'unstructured' ? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk } : { ...common, qaPairsPerRow: structured.qaPairsPerRow } }