feat(data-process): 优化上传切分流程与PDF高亮预览

2026-07-24 11:28:02 +08:00
parent 33d0ed2e01
commit ad64e44860
18 changed files with 1940 additions and 395 deletions
--- a/frontend/src/views/data-process/DataProcessCreateView.vue
+++ b/frontend/src/views/data-process/DataProcessCreateView.vue
@@ -15,29 +15,27 @@ import {
  createDefaultStructuredOptions,
  createDefaultUnstructuredOptions,
 } from './create/dataProcessCreateState'
-import {
-  DATA_PROCESS_DRAFT_STORAGE_KEY,
-  useDataProcessDraft,
-} from './create/useDataProcessDraft'
 import { useDataProcessGeneration } from './create/useDataProcessGeneration'
+import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
+import {
+  mapDataProcessSourceFile,
+  useDataProcessSourceUpload,
+  validateSourceFileSelection,
+} from './create/useDataProcessSourceUpload'
 import { useModelsStore } from '@/stores/models'
 import {
-  buildDataProcessPreview,
  createDataProcessPreview,
  createDataProcessTask,
  deleteDataProcessPreview,
  deleteDataProcessSourceFile,
  getDataProcessPreview,
  getDataProcessSourceContent,
-  getDataProcessTask,
  pullDataProcessExternalSource,
  testDataProcessExternalSource,
  updateDataProcessPreview,
  updateDataProcessTask,
-  uploadDataProcessSourceFiles,
  type DataProcessExternalSourcePayload,
  type DataProcessPreviewItem,
-  type DataProcessSourceFile,
 } from '@/api/modules/dataProcess'
 import type { DataProcessConfig } from '@/types/dataProcess'
 import type {
@@ -54,15 +52,12 @@ import type {
 const router = useRouter()
 const modelsStore = useModelsStore()
 const { list: modelList } = storeToRefs(modelsStore)
-const generationModels = computed(() => modelList.value.filter((model) => (
-  model.type === 'LLM'
-  && (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
-)))
+// 候选范围与模型管理保持一致，不在数据处理页面重复定义模型过滤规则。
+const generationModels = computed(() => modelList.value)
 const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
 const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
 const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
-const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'
-
+const PREVIEW_MODEL_VERSION = 'backend-pipeline-v2'
 const WIZARD_STEPS = [
  { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
  { id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
@@ -82,7 +77,8 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
  processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
 ))
 const uploadedFiles = ref<UploadedDataFile[]>([])
-
+const previewBuilding = ref(false)
+const { buildPreviewsByFile } = useDataProcessPreviewBuild()
 const externalSource = reactive<ExternalDataSource>({
  type: 'postgresql',
  url: '',
@@ -95,7 +91,6 @@ const externalSource = reactive<ExternalDataSource>({
 })
 const externalPulling = ref(false)
 const externalConnected = ref(false)
-
 const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
 const previewSignature = ref('')
 const previewItems = ref<PreviewItem[]>([])
@@ -103,7 +98,6 @@ const selectedPreviewFileId = ref<string | null>(null)
 const selectedPreviewId = ref<string | null>(null)
 const selectedPreviewIdsByFile = ref<Record<string, string>>({})
 const dirty = ref(false)
-const restoringDraft = ref(false)
 let allowLeave = false

 const {
@@ -123,6 +117,19 @@ const {
  dirty,
  beforeGenerate: syncPreviewChanges,
 })
+const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
+  taskId,
+  uploadedFiles,
+  onUploaded(file) {
+    previewSignature.value = ''
+    resetDownstream()
+    dirty.value = true
+    ElMessage.success(`文件 ${file.name} 上传成功，等待切分`)
+  },
+})
+const hasUnfinishedUploads = computed(() => uploadedFiles.value.some((file) => (
+  file.status !== 'ready' || !file.sourceFileId
+)))

 const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
 const previewItemsByFile = computed(() => {
@@ -149,7 +156,10 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
 const primaryActionLabel = computed(() => {
  if (currentStepId.value === 'create') return '继续：选择大模型'
  if (currentStepId.value === 'model') return '继续：上传文件'
-  if (currentStepId.value === 'upload') return '继续：数据预览'
+  if (currentStepId.value === 'upload') {
+    if (sourceUploading.value) return '正在上传'
+    return previewBuilding.value ? '正在切分' : '继续：数据预览'
+  }
  if (currentStepId.value === 'preview') return '确认预览并继续'
  if (currentStepId.value === 'results') return '保存任务'
  if (generation.status === 'running') return '正在生成'
@@ -260,33 +270,6 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
  }
 }

-function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
-  return {
-    uid: String(file.id),
-    sourceFileId: String(file.id),
-    name: file.name,
-    size: file.size_bytes,
-    count: file.record_count,
-    content,
-    fileFormat: file.file_format,
-    checksumSha256: file.checksum_sha256,
-    status: 'ready',
-  }
-}
-
-const { persistDraft, restoreDraft } = useDataProcessDraft({
-  taskId,
-  currentStepId,
-  task,
-  processType,
-  structuredOptions,
-  unstructuredOptions,
-  externalSource,
-  restoringDraft,
-  dirty,
-  goToStep,
-})
-
 function previewAffectingOptions() {
  if (processType.value === 'structured') {
    return {
@@ -368,92 +351,70 @@ function generationAffectingOptions() {

 const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))

+function buildPreviewConfigSignature() {
+  return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}`
+}
+
 function buildPreviewSignature() {
  const filesSignature = uploadedFiles.value
    .map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
    .join('|')
-  return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
+  return `${buildPreviewConfigSignature()}:${filesSignature}`
 }

 watch(
  [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
-  () => {
-    if (!restoringDraft.value) dirty.value = true
-  },
+  () => { dirty.value = true },
  { deep: true },
 )

 watch(processType, (nextType, previousType) => {
-  if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
+  if (nextType === previousType || uploadedFiles.value.length === 0) return
  resetSourceDataForProcessTypeChange()
  ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
 })

 watch(generationOptionsSignature, (currentSignature, previousSignature) => {
-  if (restoringDraft.value || currentSignature === previousSignature) return
+  if (currentSignature === previousSignature) return
  resetDownstream()
 })

-watch(
-  [taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
-  persistDraft,
-  { deep: true },
-)
-
 function scrollToStepTop() {
  document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
 }

 watch(currentStep, () => nextTick(scrollToStepTop))

-async function handleFileChange(uploadFile: UploadFile) {
+function handleFileChange(uploadFile: UploadFile) {
  const raw = uploadFile.raw
  if (!raw) return
  if (!taskId.value) {
    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
    return
  }
-  if (raw.size > 200 * 1024 * 1024) {
-    ElMessage.warning('单文件不能超过 200MB')
+  const validation = validateSourceFileSelection(raw, processType.value, uploadedFiles.value)
+  if (!validation.valid) {
+    ElMessage[validation.severity](validation.message)
    return
  }

-  const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
-  const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
-  if (!textExtensions.has(extension)) {
-    ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV；不会用示例内容替代无法解析的文件')
-    return
-  }
-
-  if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
-    ElMessage.warning('同名且同大小的文件已经上传')
-    return
-  }
-
-  let content = ''
-  try {
-    content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
-  } catch {
-    ElMessage.error('文件不是有效的 UTF-8 文本，请转换编码后重试')
-    return
-  }
-  if (!content.trim()) {
-    ElMessage.warning('不能上传空文件')
-    return
-  }
-
-  try {
-    const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
-    const source = uploaded.files[0]
-    if (!source) throw new Error('后端未返回源文件记录')
-    uploadedFiles.value.push(mapSourceFile(source, content))
-    previewSignature.value = ''
-    resetDownstream()
-    dirty.value = true
-    ElMessage.success(`文件 ${source.name} 上传成功`)
-  } catch {
-    // 请求层已展示后端的解析或格式错误。
-  }
+  // 必须先同步插入文件行，再交给队列；这样选择完成后页面会立即展示全部文件。
+  const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
+  uploadedFiles.value.push({
+    uid: localUid,
+    rawFile: raw,
+    name: raw.name,
+    size: raw.size,
+    count: 0,
+    content: '',
+    fileFormat: validation.extension,
+    status: 'queued',
+    uploadProgress: 0,
+    previewStatus: 'waiting',
+    previewProgress: 0,
+  })
+  dirty.value = true
+  enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
 }

 async function useSampleFile() {
@@ -462,32 +423,7 @@ async function useSampleFile() {
    return
  }
  const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
-  await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
-}
-
-async function restoreRegisteredSources() {
-  if (!taskId.value) return
-  try {
-    const savedTask = await getDataProcessTask(taskId.value)
-    const sources = savedTask.source_files || []
-    const restoredFiles = await Promise.all(sources.map(async (file) => {
-      try {
-        const source = await getDataProcessSourceContent(taskId.value!, file.id, {
-          start_line: 1,
-          line_count: 5000,
-        })
-        return mapSourceFile(file, source.content)
-      } catch {
-        return mapSourceFile(file)
-      }
-    }))
-    uploadedFiles.value = restoredFiles
-    if (restoredFiles.length) {
-      ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
-    }
-  } catch {
-    ElMessage.warning('草稿任务暂时无法从后端同步，请检查服务后重试')
-  }
+  handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
 }

 function updateExternalSource(value: ExternalDataSource) {
@@ -539,7 +475,7 @@ async function handlePullData() {
        start_line: 1,
        line_count: 5000,
      })
-      newFiles.push(mapSourceFile(file, source.content))
+      newFiles.push(mapDataProcessSourceFile(file, source.content))
    }
    uploadedFiles.value.push(...newFiles)
    externalConnected.value = true
@@ -557,11 +493,19 @@ async function handlePullData() {
 async function handleRemoveFile(uid: string | number) {
  const index = uploadedFiles.value.findIndex(f => f.uid === uid)
  if (index < 0 || !taskId.value) return
-  try {
-    await deleteDataProcessSourceFile(taskId.value, uid)
-  } catch {
+  const file = uploadedFiles.value[index]
+  if (!file) return
+  if (file.status === 'uploading') {
+    ElMessage.warning('当前文件正在上传，请等待上传结束后再删除')
    return
  }
+  if (file.sourceFileId) {
+    try {
+      await deleteDataProcessSourceFile(taskId.value, file.sourceFileId)
+    } catch {
+      return
+    }
+  }
  uploadedFiles.value.splice(index, 1)
  previewSignature.value = ''
  previewItems.value = []
@@ -596,7 +540,6 @@ async function nextFromModel() {
      : await createDataProcessTask(taskPayload())
    taskId.value = String(saved.id)
    dirty.value = true
-    persistDraft()
    goToStep('upload')
  } catch {
    // 请求层已展示名称冲突或配置非法等具体原因。
@@ -604,6 +547,7 @@ async function nextFromModel() {
 }

 async function nextFromUpload() {
+  if (previewBuilding.value || sourceUploading.value) return
  if (!taskId.value) {
    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
    return
@@ -612,13 +556,61 @@ async function nextFromUpload() {
    ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
    return
  }
+  const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
+  if (failedUploads.length) {
+    ElMessage.warning(`${failedUploads.length} 个文件上传失败，请删除后重新选择`)
+    return
+  }
+  if (hasUnfinishedUploads.value) {
+    ElMessage.warning('请等待所有文件上传完成后再开始切分')
+    return
+  }

  const signature = buildPreviewSignature()
-  if (signature !== previewSignature.value) {
+  const configSignature = buildPreviewConfigSignature()
+  const allFilesSucceeded = () => uploadedFiles.value.every((file) => (
+    file.previewStatus === 'success' && file.previewConfigSignature === configSignature
+  ))
+
+  if (signature === previewSignature.value && previewItems.value.length && allFilesSucceeded()) {
+    goToStep('preview')
+    return
+  }
+
+  for (const file of uploadedFiles.value) {
+    if (file.previewConfigSignature === configSignature) continue
+    file.previewStatus = 'waiting'
+    file.previewProgress = 0
+    file.previewError = undefined
+    file.previewCount = undefined
+  }
+
+  previewBuilding.value = true
+  try {
+    const pendingFileIds = uploadedFiles.value
+      .filter((file) => file.previewStatus !== 'success' || file.previewConfigSignature !== configSignature)
+      .map((file) => file.sourceFileId)
+      .filter((fileId): fileId is string => Boolean(fileId))
+    // 构建接口是同步响应：请求中只展示真实的“处理中”，响应成功后才记为 100%。
+    await buildPreviewsByFile(taskId.value, pendingFileIds, (progress) => {
+      const file = uploadedFiles.value.find((item) => (
+        String(item.sourceFileId) === String(progress.source_file_id)
+      ))
+      if (!file) return
+      file.previewStatus = progress.status
+      file.previewProgress = progress.progress
+      file.previewCount = progress.preview_count
+      file.previewError = progress.error
+      if (progress.status === 'success') file.previewConfigSignature = configSignature
+    })
+
+    const failedCount = uploadedFiles.value.filter((file) => file.previewStatus === 'failed').length
+    if (failedCount || !allFilesSucceeded()) {
+      ElMessage.warning(`${failedCount || 1} 个文件切分失败；修正问题后点击“继续：数据预览”即可重试`)
+      return
+    }
+
    try {
-      await buildDataProcessPreview(taskId.value, {
-        source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
-      })
      const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
      const items = [...first.items]
      const pages = Math.ceil(first.total / first.page_size)
@@ -628,6 +620,7 @@ async function nextFromUpload() {
      }
      previewItems.value = items.map(mapPreviewItem)
    } catch {
+      ElMessage.warning('文件切分已经完成，但预览加载失败；请再次点击继续重试加载')
      return
    }
    if (!previewItems.value.length) {
@@ -641,8 +634,10 @@ async function nextFromUpload() {
      : {}
    previewSignature.value = signature
    resetDownstream()
+    goToStep('preview')
+  } finally {
+    previewBuilding.value = false
  }
-  goToStep('preview')
 }

 function selectPreviewFile(fileId: string) {
@@ -770,6 +765,14 @@ async function handlePrimaryAction() {
 }

 function handleBack() {
+  if (sourceUploading.value) {
+    ElMessage.warning('请等待当前文件上传完成')
+    return
+  }
+  if (previewBuilding.value) {
+    ElMessage.warning('请等待当前文件切分完成')
+    return
+  }
  if (generation.status === 'running') {
    ElMessage.warning('请先停止当前生成任务')
    return
@@ -793,7 +796,6 @@ async function saveTask() {
    return
  }
  dirty.value = false
-  localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
  allowLeave = true
  ElMessage.success('数据处理任务已保存')
  await router.push('/data-process')
@@ -818,6 +820,10 @@ async function handleCancel() {
 }

 onBeforeRouteLeave(async () => {
+  if (sourceUploading.value) {
+    ElMessage.warning('请等待当前文件上传完成后再离开页面')
+    return false
+  }
  if (allowLeave || !dirty.value) return true
  const confirmed = await confirmDialogRef.value?.open({
    title: '确认离开当前页面？',
@@ -833,9 +839,9 @@ onBeforeRouteLeave(async () => {
 onBeforeUnmount(() => {
  stopGenerationTimer()
 })
-onMounted(async () => {
-  restoreDraft()
-  await Promise.all([modelsStore.load(), restoreRegisteredSources()])
+onMounted(() => {
+  localStorage.removeItem('yg-data-process-create-draft')
+  void modelsStore.load(true)
 })
 </script>

@@ -899,6 +905,8 @@ onMounted(async () => {
            :external-source="externalSource"
            :external-pulling="externalPulling"
            :external-connected="externalConnected"
+            :preview-building="previewBuilding"
+            :source-uploading="sourceUploading"
            @update:external-source="updateExternalSource"
            @file-change="handleFileChange"
            @remove-file="handleRemoveFile"
@@ -915,6 +923,8 @@ onMounted(async () => {
            :items="activePreviewItems"
            :process-type="processType"
            :file-name="activePreviewFile?.name ?? ''"
+            :task-id="taskId"
+            :source-file-id="activePreviewFile?.sourceFileId ?? activePreviewFile?.uid ?? null"
            :files="previewFiles"
            @update:selected-id="selectPreviewItem"
            @update:selected-file-id="selectPreviewFile"
@@ -949,7 +959,7 @@ onMounted(async () => {

    <footer class="wizard-footer">
      <div class="footer-left">
-        <el-button v-if="currentStep > 0" @click="handleBack">
+        <el-button v-if="currentStep > 0" :disabled="previewBuilding || sourceUploading" @click="handleBack">
          <i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回：{{ previousStepLabel }}
        </el-button>
        <el-button v-else @click="handleCancel">取消</el-button>
@@ -960,8 +970,8 @@ onMounted(async () => {
        <el-button
          class="wizard-primary-action"
          type="primary"
-          :loading="generation.status === 'running'"
-          :disabled="currentStepId === 'generate' && generation.status === 'running'"
+          :loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
+          :disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
          @click="handlePrimaryAction"
        >
          {{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />