YG_FT/frontend/src/views/data-process/DataProcessCreateView.vue

<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, type UploadFile } from 'element-plus'
import { storeToRefs } from 'pinia'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue'
import ModelSelectionStep from './create/ModelSelectionStep.vue'
import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import {
  createDefaultStructuredOptions,
  createDefaultUnstructuredOptions,
} from './create/dataProcessCreateState'
import {
  DATA_PROCESS_DRAFT_STORAGE_KEY,
  useDataProcessDraft,
} from './create/useDataProcessDraft'
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useModelsStore } from '@/stores/models'
import type {
  ExternalDataSource,
  GenerationControlOptions,
  PreviewItem,
  ProcessType,
  StepId,
  StructuredProcessOptions,
  UnstructuredProcessOptions,
  UploadedDataFile,
} from './create/types'

const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'

const WIZARD_STEPS = [
  { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
  { id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
  { id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
  { id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
  { id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
  { id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }>
const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('structured')
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
const modelSelectionOptions = computed<GenerationControlOptions>(() => (
  processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
))
const uploadedFiles = ref<UploadedDataFile[]>([])

const externalSource = reactive<ExternalDataSource>({
  type: 'mysql',
  url: '',
  authMode: 'none',
  username: '',
  password: '',
  token: '',
  limit: 1000,
})
const externalPulling = ref(false)
const externalConnected = ref(false)
let connectionTimer: ReturnType<typeof setTimeout> | null = null
let pullTimer: ReturnType<typeof setTimeout> | null = null

const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const previewSignature = ref('')
const previewItems = ref<PreviewItem[]>([])
const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const dirty = ref(false)
const restoringDraft = ref(false)
let allowLeave = false

const {
  generation,
  results,
  selectedResultId,
  resetDownstream,
  restoreResult,
  startGeneration,
  stopGeneration,
  stopGenerationTimer,
  updateResultField,
  validateResults,
} = useDataProcessGeneration({
  previewItems,
  processType,
  structuredOptions,
  unstructuredOptions,
  dirty,
})

const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
const previewItemsByFile = computed(() => {
  const grouped = new Map<string, PreviewItem[]>()
  for (const item of previewItems.value) {
    const items = grouped.get(item.sourceFileId) || []
    items.push(item)
    grouped.set(item.sourceFileId, items)
  }
  return grouped
})
const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value))
const activePreviewItems = computed(() => previewItemsByFile.value.get(selectedPreviewFileId.value || '') || [])
const activeSourceText = computed(() => activePreviewFile.value?.content ?? '')
const previewFiles = computed(() => uploadedFiles.value.map((file) => {
  const items = previewItemsByFile.value.get(String(file.uid)) || []
  return {
    id: String(file.uid),
    name: file.name,
    count: items.length,
    modifiedCount: items.filter((item) => item.status !== 'original').length,
  }
}))
const primaryActionLabel = computed(() => {
  if (currentStepId.value === 'create') return '继续：选择大模型'
  if (currentStepId.value === 'model') return '继续：上传文件'
  if (currentStepId.value === 'upload') return '继续：数据预览'
  if (currentStepId.value === 'preview') return '确认预览并继续'
  if (currentStepId.value === 'results') return '保存任务'
  if (generation.status === 'running') return '正在生成'
  if (generation.status === 'success') return '查看生成结果'
  if (generation.status === 'failed') return '重新生成'
  return '开始生成'
})

const primaryActionIcon = computed(() => {
  if (currentStepId.value === 'results') return 'fa-check'
  if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
  return 'fa-arrow-right'
})

const previousStepLabel = computed(() => currentStep.value > 0
  ? WIZARD_STEPS[currentStep.value - 1].title
  : '')

function goToStep(stepId: StepId) {
  const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
  if (nextStepIndex >= 0) currentStep.value = nextStepIndex
}

function updateModelSelectionOptions(value: GenerationControlOptions) {
  if (processType.value === 'unstructured') {
    unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
    return
  }
  structuredOptions.value = { ...structuredOptions.value, ...value }
}

const { persistDraft, restoreDraft } = useDataProcessDraft({
  currentStepId,
  task,
  processType,
  structuredOptions,
  unstructuredOptions,
  externalSource,
  restoringDraft,
  dirty,
  goToStep,
})

function previewAffectingOptions() {
  if (processType.value === 'structured') {
    return {
      preprocessOptions: structuredOptions.value.preprocessOptions,
    }
  }
  if (processType.value === 'unstructured') {
    const {
      preprocessOptions,
      chunkMethod,
      chunkSize,
      chunkOverlap,
      minChunkSize,
      customDelimiter,
      preserveTables,
      preserveCodeBlocks,
      preserveLists,
    } = unstructuredOptions.value
    return {
      preprocessOptions,
      chunkMethod,
      chunkSize,
      chunkOverlap,
      minChunkSize,
      customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
      preserveTables,
      preserveCodeBlocks,
      preserveLists,
    }
  }
  return null
}

function generationAffectingOptions() {
  if (processType.value === 'structured') {
    const {
      semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
      temperature, maxTokens, jsonMode,
      qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
    } = structuredOptions.value
    return {
      semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
      temperature, maxTokens, jsonMode,
      qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
    }
  }
  if (processType.value === 'unstructured') {
    const {
      semanticEnrichment,
      qaPairsPerChunk,
      datasetSplit,
      generationModelId,
      generationPrompt,
      temperature,
      maxTokens,
      jsonMode,
      qualityFilterEnabled,
      filterLowQuality,
      filterShortContent,
      minOutputLength,
    } = unstructuredOptions.value
    return {
      semanticEnrichment,
      qaPairsPerChunk,
      datasetSplit,
      generationModelId,
      generationPrompt,
      temperature,
      maxTokens,
      jsonMode,
      qualityFilterEnabled,
      filterLowQuality,
      filterShortContent,
      minOutputLength,
    }
  }
  return null
}

const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))

function buildPreviewSignature() {
  const filesSignature = uploadedFiles.value
    .map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
    .join('|')
  return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
}

watch(
  [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
  () => {
    if (!restoringDraft.value) dirty.value = true
  },
  { deep: true },
)

watch(processType, (nextType, previousType) => {
  if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
  resetSourceDataForProcessTypeChange()
  ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
})

watch(generationOptionsSignature, (currentSignature, previousSignature) => {
  if (restoringDraft.value || currentSignature === previousSignature) return
  resetDownstream()
})

watch(
  [task, processType, structuredOptions, unstructuredOptions, externalSource],
  persistDraft,
  { deep: true },
)

function scrollToStepTop() {
  document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
}

watch(currentStep, () => nextTick(scrollToStepTop))

async function handleFileChange(uploadFile: UploadFile) {
  const raw = uploadFile.raw
  if (!raw) return
  if (raw.size > 200 * 1024 * 1024) {
    ElMessage.warning('单文件不能超过 200MB')
    return
  }

  const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
  const textExtensions = ['txt', 'md', 'json', 'jsonl', 'csv']
  let content = ''
  if (textExtensions.includes(extension)) {
    try {
      content = await raw.text()
    } catch {
      content = ''
    }
  }

  const fileContent = content.trim() ? content : DEFAULT_SOURCE_TEXT
  const linesCount = fileContent.split('\n').filter((line) => line.trim()).length
  
  // Prevent duplicate upload of the same file
  if (!uploadedFiles.value.some(f => f.name === raw.name && f.size === raw.size)) {
    uploadedFiles.value.push({
      uid: uploadFile.uid || Date.now() + Math.random(),
      name: raw.name,
      size: raw.size,
      count: linesCount,
      content: fileContent
    })
  }

  dirty.value = true
}

function useSampleFile() {
  uploadedFiles.value = [{
    uid: 'sample-1',
    name: 'finance_qa.jsonl',
    size: 128 * 1024 * 1024,
    count: DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length,
    content: DEFAULT_SOURCE_TEXT
  }]
  dirty.value = true
}

function updateExternalSource(value: ExternalDataSource) {
  Object.assign(externalSource, value)
  externalConnected.value = false
}

function handleTestConnection() {
  if (!externalSource.url.trim()) {
    ElMessage.warning('请先填写数据源地址')
    return
  }
  if (connectionTimer) clearTimeout(connectionTimer)
  externalPulling.value = true
  connectionTimer = setTimeout(() => {
    connectionTimer = null
    externalPulling.value = false
    externalConnected.value = true
    ElMessage.success('数据源连接测试成功')
  }, 1500)
}

function handlePullData() {
  if (!externalSource.url.trim()) {
    ElMessage.warning('请先填写数据源地址')
    return
  }
  if (pullTimer) clearTimeout(pullTimer)
  externalPulling.value = true
  pullTimer = setTimeout(() => {
    pullTimer = null
    externalPulling.value = false
    externalConnected.value = true
    const typeName = externalSource.type.toUpperCase()
    const id = `external-${Date.now()}`
    uploadedFiles.value.push({
      uid: id,
      name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
      size: Math.min(externalSource.limit, 5000) * 64,
      count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
      content: DEFAULT_SOURCE_TEXT,
    })
    dirty.value = true
    ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
  }, 2000)
}

function handleRemoveFile(uid: string | number) {
  const index = uploadedFiles.value.findIndex(f => f.uid === uid)
  if (index > -1) {
    uploadedFiles.value.splice(index, 1)
    previewSignature.value = ''
    previewItems.value = []
    selectedPreviewId.value = null
    resetDownstream()
    dirty.value = true
  }
}

function resetSourceDataForProcessTypeChange() {
  uploadedFiles.value = []
  previewSignature.value = ''
  previewItems.value = []
  selectedPreviewFileId.value = null
  selectedPreviewId.value = null
  selectedPreviewIdsByFile.value = {}
  externalConnected.value = false
  resetDownstream()
}

async function nextFromCreate() {
  const valid = await taskSetupRef.value?.validate()
  if (!valid) return
  goToStep('model')
}

async function nextFromModel() {
  const valid = await modelSelectionRef.value?.validate()
  if (!valid) return
  goToStep('upload')
}

function nextFromUpload() {
  if (uploadedFiles.value.length === 0) {
    ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
    return
  }

  const signature = buildPreviewSignature()
  if (signature !== previewSignature.value) {
    previewItems.value = uploadedFiles.value.flatMap((file) =>
      buildPreviewItems(
        file.content,
        processType.value,
        String(file.uid),
        processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
      ),
    )
    selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
    selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
    selectedPreviewIdsByFile.value = selectedPreviewId.value && selectedPreviewFileId.value
      ? { [selectedPreviewFileId.value]: selectedPreviewId.value }
      : {}
    previewSignature.value = signature
    resetDownstream()
  }
  goToStep('preview')
}

function selectPreviewFile(fileId: string) {
  if (selectedPreviewFileId.value && selectedPreviewId.value) {
    selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
  }
  selectedPreviewFileId.value = fileId
  selectedPreviewId.value = selectedPreviewIdsByFile.value[fileId]
    || activePreviewItems.value[0]?.id
    || null
}

function selectPreviewItem(id: string) {
  selectedPreviewId.value = id
  if (selectedPreviewFileId.value) selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = id
}

function updatePreviewContent(id: string, value: string) {
  const item = previewItems.value.find((entry) => entry.id === id)
  if (!item) return
  item.editedContent = value
  item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
  item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
  resetDownstream()
  dirty.value = true
}

function restorePreviewItem(id: string) {
  const item = previewItems.value.find((entry) => entry.id === id)
  if (!item || item.sourceStart == null) return
  item.editedContent = item.originalContent
  item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
  item.status = 'original'
  resetDownstream()
  dirty.value = true
}

function addPreviewItem() {
  if (!selectedPreviewFileId.value) return
  const id = `manual-${Date.now()}`
  previewItems.value.push({
    id,
    sourceFileId: selectedPreviewFileId.value,
    originalContent: '',
    editedContent: '',
    sourceStart: null,
    sourceEnd: null,
    sourceStartLine: null,
    sourceEndLine: null,
    tokenCount: 1,
    status: 'manual',
  })
  selectPreviewItem(id)
  resetDownstream()
  dirty.value = true
}

async function removePreviewItem(id: string) {
  const confirmed = await confirmDialogRef.value?.open({
    title: '删除预览内容？',
    message: '删除只影响本次处理，不会修改源文件。删除后可重新从源文件生成预览。',
    confirmText: '删除',
    cancelText: '取消',
    tone: 'danger',
  })
  if (!confirmed) return

  const index = previewItems.value.findIndex((item) => item.id === id)
  if (index < 0) return
  previewItems.value.splice(index, 1)
  selectedPreviewId.value = activePreviewItems.value[Math.min(index, activePreviewItems.value.length - 1)]?.id ?? null
  if (selectedPreviewFileId.value && selectedPreviewId.value) {
    selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
  }
  resetDownstream()
  dirty.value = true
}

async function handlePrimaryAction() {
  if (currentStepId.value === 'create') {
    await nextFromCreate()
    return
  }
  if (currentStepId.value === 'model') {
    await nextFromModel()
    return
  }
  if (currentStepId.value === 'upload') {
    nextFromUpload()
    return
  }
  if (currentStepId.value === 'preview') {
    if (!previewItems.value.length) {
      ElMessage.warning('当前没有可生成的预览内容')
      return
    }
    goToStep('generate')
    return
  }
  if (currentStepId.value === 'generate') {
    if (generation.status === 'success') {
      goToStep('results')
    } else if (generation.status !== 'running') {
      startGeneration()
    }
    return
  }
  await saveTask()
}

function handleBack() {
  if (generation.status === 'running') {
    ElMessage.warning('请先停止当前生成任务')
    return
  }
  if (currentStep.value > 0) currentStep.value -= 1
}


async function saveTask() {
  if (!validateResults()) {
    ElMessage.warning('请先修正校验失败的结果')
    return
  }
  dirty.value = false
  localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
  allowLeave = true
  ElMessage.success('数据处理任务已保存')
  await router.push('/data-process')
}

async function handleCancel() {
  if (!dirty.value) {
    allowLeave = true
    router.back()
    return
  }
  const confirmed = await confirmDialogRef.value?.open({
    title: '确认离开当前页面？',
    message: '当前存在未保存修改，离开后这些修改将不会保留。',
    confirmText: '放弃修改',
    cancelText: '继续编辑',
    tone: 'danger',
  })
  if (!confirmed) return
  allowLeave = true
  router.back()
}

onBeforeRouteLeave(async () => {
  if (allowLeave || !dirty.value) return true
  const confirmed = await confirmDialogRef.value?.open({
    title: '确认离开当前页面？',
    message: '当前存在未保存修改，离开后这些修改将不会保留。',
    confirmText: '放弃修改',
    cancelText: '继续编辑',
    tone: 'danger',
  })
  if (confirmed) allowLeave = true
  return Boolean(confirmed)
})

onBeforeUnmount(() => {
  stopGenerationTimer()
  if (connectionTimer) clearTimeout(connectionTimer)
  if (pullTimer) clearTimeout(pullTimer)
})
onMounted(() => {
  restoreDraft()
  modelsStore.load()
})
</script>

<template>
  <div class="create-wizard-layout">
    <main class="wizard-main">
      <div class="wizard-main-inner">
        <div class="wizard-steps-container">
          <div class="custom-wizard-steps">
            <template v-for="(step, index) in WIZARD_STEPS" :key="step.id">
              <div 
                v-if="index !== 0" 
                class="step-connector"
                :class="{ 'is-active': currentStep >= index }"
              ></div>
              <div 
                class="step-item"
                :class="{ 
                  'is-active': currentStep === index, 
                  'is-completed': currentStep > index 
                }"
                :aria-current="currentStep === index ? 'step' : undefined"
              >
                <div class="step-node">
                  <div class="step-icon">
                    <i v-if="currentStep > index" class="fa fa-check" />
                    <span v-else>{{ index + 1 }}</span>
                  </div>
                  <div class="step-text">
                    <div class="step-title">{{ step.title }}</div>
                  </div>
                </div>
              </div>
            </template>
          </div>
        </div>

        <div class="wizard-content">
          <TaskSetupStep
            v-if="currentStepId === 'create'"
            ref="taskSetupRef"
            v-model:name="task.name"
            v-model:description="task.description"
            v-model:process-type="processType"
            v-model:structured-options="structuredOptions"
            v-model:unstructured-options="unstructuredOptions"
          />

          <ModelSelectionStep
            v-else-if="currentStepId === 'model'"
            ref="modelSelectionRef"
            :options="modelSelectionOptions"
            :models="generationModels"
            @update:options="updateModelSelectionOptions"
          />

          <SourceUploadStep
            v-else-if="currentStepId === 'upload'"
            :process-type="processType"
            :uploaded-files="uploadedFiles"
            :external-source="externalSource"
            :external-pulling="externalPulling"
            :external-connected="externalConnected"
            @update:external-source="updateExternalSource"
            @file-change="handleFileChange"
            @remove-file="handleRemoveFile"
            @use-sample="useSampleFile"
            @test-connection="handleTestConnection"
            @pull-data="handlePullData"
          />

          <PreviewCompareStep
            v-else-if="currentStepId === 'preview'"
            :selected-id="selectedPreviewId"
            :selected-file-id="selectedPreviewFileId"
            :source-text="activeSourceText"
            :items="activePreviewItems"
            :process-type="processType"
            :file-name="activePreviewFile?.name ?? ''"
            :files="previewFiles"
            @update:selected-id="selectPreviewItem"
            @update:selected-file-id="selectPreviewFile"
            @update:item-content="updatePreviewContent"
            @restore:item="restorePreviewItem"
            @add:item="addPreviewItem"
            @remove:item="removePreviewItem"
          />

          <GenerationStep
            v-else-if="currentStepId === 'generate'"
            :task-name="task.name"
            :process-type="processType"
            :file-name="fileName"
            :preview-count="previewItems.length"
            :modified-count="modifiedPreviewCount"
            :generation="generation"
            @stop="stopGeneration"
            @retry="startGeneration"
          />

          <ResultEditorStep
            v-else-if="currentStepId === 'results'"
            v-model:selected-id="selectedResultId"
            :items="results"
            @update:field="updateResultField"
            @restore:item="restoreResult"
          />
        </div>
      </div>
    </main>

    <footer class="wizard-footer">
      <div class="footer-left">
        <el-button v-if="currentStep > 0" @click="handleBack">
          <i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回：{{ previousStepLabel }}
        </el-button>
        <el-button v-else @click="handleCancel">取消</el-button>
      </div>
      <div class="footer-center">
      </div>
      <div class="footer-right">
        <el-button
          class="wizard-primary-action"
          type="primary"
          :loading="generation.status === 'running'"
          :disabled="currentStepId === 'generate' && generation.status === 'running'"
          @click="handlePrimaryAction"
        >
          {{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
        </el-button>
      </div>
    </footer>
  </div>

  <AppConfirmDialog ref="confirmDialogRef" />
</template>

<style scoped lang="scss" src="./create/data-process-create.scss"></style>