YG_FT/frontend/src/views/data-process/create/dataProcessCreateState.ts

import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess'
import type {
  GenerationControlOptions,
  PreprocessOption,
  StructuredProcessOptions,
  UnstructuredPreprocessOption,
  UnstructuredProcessOptions,
  ProcessType,
} from './types'
import { normalizeQaPairsGenerationCount } from './types'

const LEGACY_DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容，答案应事实清晰、语言自然，不要添加无关说明。'

const PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT = '你是一名专业的数据生成助手。请严格依据输入内容生成准确、完整、可直接用于监督微调的问答数据。只生成问题和最终答案，不输出分析、推理过程或来源中不存在的信息；答案应事实清晰、语言自然。'

const PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT = '你是一名专业的推理数据生成助手。请严格依据输入内容生成问题、可核验的推理过程和最终答案。推理需要说明关键依据与必要步骤，不得引入来源中不存在的事实；最终答案应准确、完整且语言自然。'

export const DEFAULT_STANDARD_GENERATION_PROMPT = `你是一名专业的数据生成助手。请根据下方来源内容生成可用于监督微调的问答数据。
来源内容：
{{ content }}
要求：严格依据来源生成问题和最终答案，不输出分析、中间思考过程或来源中不存在的信息；答案应准确、完整、语言自然。`

export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名专业的推理数据生成助手。请根据下方来源内容生成可用于推理模型训练的问答数据。
来源内容：
{{ content }}
要求：每条数据必须包含问题、可核验的中间思考过程和最终答案。中间思考过程需说明关键依据、必要步骤和中间计算，不得跳过推理只给结论，也不得引入来源中不存在的事实。`

export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
  return outputType === 'reasoning'
    ? DEFAULT_REASONING_GENERATION_PROMPT
    : DEFAULT_STANDARD_GENERATION_PROMPT
}

export function isBuiltInGenerationPrompt(value: string) {
  return [
    LEGACY_DEFAULT_GENERATION_PROMPT,
    PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT,
    PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT,
    DEFAULT_STANDARD_GENERATION_PROMPT,
    DEFAULT_REASONING_GENERATION_PROMPT,
  ].includes(value)
}

export function createDefaultStructuredOptions(): StructuredProcessOptions {
  return {
    preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
    semanticEnrichment: false,
    qaPairsPerRow: 1,
    datasetSplit: { train: 80, validation: 10, test: 10 },
    generationModelId: '',
    generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
    outputType: 'standard',
    reasoningDetail: 'normal',
    temperature: 0.7,
    maxTokens: 1024,
    jsonMode: false,
    qualityFilterEnabled: false,
    filterLowQuality: true,
    filterShortContent: true,
    minOutputLength: 20,
  }
}

export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
  return {
    preprocessOptions: [
      'clean_invalid_content',
      'detect_document_structure',
      'merge_short_content',
      'filter_low_quality',
      'deduplicate_content',
      'preserve_context',
    ],
    chunkMethod: 'layout_hybrid',
    chunkSize: 800,
    chunkOverlap: 100,
    minChunkSize: 100,
    semanticBreakpointPercentile: 95,
    preserveTables: true,
    preserveCodeBlocks: true,
    preserveLists: true,
    semanticEnrichment: false,
    qaPairsPerChunk: 1,
    datasetSplit: { train: 80, validation: 10, test: 10 },
    generationModelId: '',
    generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
    outputType: 'standard',
    reasoningDetail: 'normal',
    temperature: 0.7,
    maxTokens: 1024,
    jsonMode: false,
    qualityFilterEnabled: false,
    filterLowQuality: true,
    filterShortContent: true,
    minOutputLength: 20,
  }
}

function configValue<T>(config: DataProcessConfig, key: string, fallback: T): T {
  return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback
}

function numberValue(config: DataProcessConfig, key: string, fallback: number): number {
  const value = Number(configValue(config, key, fallback))
  return Number.isFinite(value) ? value : fallback
}

function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) {
  const value = config.dataset_split
  if (!value || typeof value !== 'object') return { ...fallback }
  const split = value as unknown as Record<string, unknown>
  const splitNumber = (key: keyof DataProcessDatasetSplit) => {
    const parsed = Number(split[key])
    return Number.isFinite(parsed) ? parsed : fallback[key]
  }
  return {
    train: splitNumber('train'),
    validation: splitNumber('validation'),
    test: splitNumber('test'),
  }
}

function generationOptionsFromConfig(
  config: DataProcessConfig,
  defaults: GenerationControlOptions,
): GenerationControlOptions {
  const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
    ? 'reasoning'
    : 'standard'
  const configuredPrompt = String(
    configValue(config, 'generation_prompt', defaults.generationPrompt),
  )
  return {
    generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
    generationPrompt: isBuiltInGenerationPrompt(configuredPrompt)
      ? defaultGenerationPrompt(outputType)
      : configuredPrompt,
    outputType,
    reasoningDetail: configValue(config, 'reasoning_detail', defaults.reasoningDetail) === 'detailed'
      ? 'detailed'
      : 'normal',
    temperature: numberValue(config, 'temperature', defaults.temperature),
    maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
    jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
    qualityFilterEnabled: Boolean(configValue(
      config,
      'quality_filter_enabled',
      defaults.qualityFilterEnabled,
    )),
    filterLowQuality: Boolean(configValue(
      config,
      'filter_low_quality',
      defaults.filterLowQuality,
    )),
    filterShortContent: Boolean(configValue(
      config,
      'filter_short_content',
      defaults.filterShortContent,
    )),
    minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength),
  }
}

export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
  const defaults = createDefaultStructuredOptions()
  const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
  return {
    ...defaults,
    ...generationOptionsFromConfig(config, defaults),
    preprocessOptions: Array.isArray(preprocessOptions)
      ? preprocessOptions.map(String) as PreprocessOption[]
      : defaults.preprocessOptions,
    semanticEnrichment: Boolean(configValue(
      config,
      'semantic_enrichment',
      defaults.semanticEnrichment,
    )),
    qaPairsPerRow: normalizeQaPairsGenerationCount(
      numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
      defaults.qaPairsPerRow,
    ),
    datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
  }
}

export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions {
  const defaults = createDefaultUnstructuredOptions()
  const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
  return {
    ...defaults,
    ...generationOptionsFromConfig(config, defaults),
    preprocessOptions: Array.isArray(preprocessOptions)
      ? preprocessOptions.map(String) as UnstructuredPreprocessOption[]
      : defaults.preprocessOptions,
    chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod),
    chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize),
    chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap),
    minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize),
    semanticBreakpointPercentile: numberValue(
      config,
      'semantic_breakpoint_percentile',
      defaults.semanticBreakpointPercentile,
    ),
    preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)),
    preserveCodeBlocks: Boolean(configValue(
      config,
      'preserve_code_blocks',
      defaults.preserveCodeBlocks,
    )),
    preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)),
    semanticEnrichment: Boolean(configValue(
      config,
      'semantic_enrichment',
      defaults.semanticEnrichment,
    )),
    qaPairsPerChunk: normalizeQaPairsGenerationCount(
      numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
      defaults.qaPairsPerChunk,
    ),
    datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
  }
}

export function previewAffectingOptionsFor(
  processType: ProcessType,
  structured: StructuredProcessOptions,
  unstructured: UnstructuredProcessOptions,
) {
  if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions }
  if (processType !== 'unstructured') return null
  const {
    preprocessOptions,
    chunkMethod,
    chunkSize,
    chunkOverlap,
    minChunkSize,
    semanticBreakpointPercentile,
    preserveTables,
    preserveCodeBlocks,
    preserveLists,
  } = unstructured
  return {
    preprocessOptions,
    chunkMethod,
    chunkSize,
    chunkOverlap,
    minChunkSize,
    semanticBreakpointPercentile,
    preserveTables,
    preserveCodeBlocks,
    preserveLists,
  }
}

export function generationAffectingOptionsFor(
  processType: ProcessType,
  structured: StructuredProcessOptions,
  unstructured: UnstructuredProcessOptions,
) {
  const options = processType === 'unstructured' ? unstructured : structured
  if (processType === 'external') return null
  const common = {
    semanticEnrichment: options.semanticEnrichment,
    datasetSplit: options.datasetSplit,
    generationModelId: options.generationModelId,
    generationPrompt: options.generationPrompt,
    outputType: options.outputType,
    reasoningDetail: options.reasoningDetail,
    temperature: options.temperature,
    maxTokens: options.maxTokens,
    jsonMode: options.jsonMode,
    qualityFilterEnabled: options.qualityFilterEnabled,
    filterLowQuality: options.filterLowQuality,
    filterShortContent: options.filterShortContent,
    minOutputLength: options.minOutputLength,
  }
  return processType === 'unstructured'
    ? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk }
    : { ...common, qaPairsPerRow: structured.qaPairsPerRow }
}