feat: 完成数据处理接口与前端接入

2026-07-23 15:10:13 +08:00
parent f453234057
commit f04dc479bb
29 changed files with 7126 additions and 1144 deletions
--- a/frontend/src/views/data-process/DataProcessCreateView.vue
+++ b/frontend/src/views/data-process/DataProcessCreateView.vue
@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
 import PreviewCompareStep from './create/PreviewCompareStep.vue'
 import GenerationStep from './create/GenerationStep.vue'
 import ResultEditorStep from './create/ResultEditorStep.vue'
-import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
+import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
 import {
  createDefaultStructuredOptions,
  createDefaultUnstructuredOptions,
@@ -21,6 +21,25 @@ import {
 } from './create/useDataProcessDraft'
 import { useDataProcessGeneration } from './create/useDataProcessGeneration'
 import { useModelsStore } from '@/stores/models'
+import {
+  buildDataProcessPreview,
+  createDataProcessPreview,
+  createDataProcessTask,
+  deleteDataProcessPreview,
+  deleteDataProcessSourceFile,
+  getDataProcessPreview,
+  getDataProcessSourceContent,
+  getDataProcessTask,
+  pullDataProcessExternalSource,
+  testDataProcessExternalSource,
+  updateDataProcessPreview,
+  updateDataProcessTask,
+  uploadDataProcessSourceFiles,
+  type DataProcessExternalSourcePayload,
+  type DataProcessPreviewItem,
+  type DataProcessSourceFile,
+} from '@/api/modules/dataProcess'
+import type { DataProcessConfig } from '@/types/dataProcess'
 import type {
  ExternalDataSource,
  GenerationControlOptions,
@@ -35,11 +54,14 @@ import type {
 const router = useRouter()
 const modelsStore = useModelsStore()
 const { list: modelList } = storeToRefs(modelsStore)
-const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
+const generationModels = computed(() => modelList.value.filter((model) => (
+  model.type === 'LLM'
+  && (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
+)))
 const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
 const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
 const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
-const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
+const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'

 const WIZARD_STEPS = [
  { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
@@ -52,6 +74,7 @@ const WIZARD_STEPS = [
 const currentStep = ref(0)
 const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
 const task = reactive({ name: '', description: '' })
+const taskId = ref<string | null>(null)
 const processType = ref<ProcessType>('structured')
 const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
 const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
@@ -61,18 +84,17 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
 const uploadedFiles = ref<UploadedDataFile[]>([])

 const externalSource = reactive<ExternalDataSource>({
-  type: 'mysql',
+  type: 'postgresql',
  url: '',
  authMode: 'none',
  username: '',
  password: '',
-  token: '',
  limit: 1000,
+  query: '',
+  fileName: 'external-data.jsonl',
 })
 const externalPulling = ref(false)
 const externalConnected = ref(false)
-let connectionTimer: ReturnType<typeof setTimeout> | null = null
-let pullTimer: ReturnType<typeof setTimeout> | null = null

 const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
 const previewSignature = ref('')
@@ -88,6 +110,7 @@ const {
  generation,
  results,
  selectedResultId,
+  persistResultChanges,
  resetDownstream,
  restoreResult,
  startGeneration,
@@ -96,11 +119,9 @@ const {
  updateResultField,
  validateResults,
 } = useDataProcessGeneration({
-  previewItems,
-  processType,
-  structuredOptions,
-  unstructuredOptions,
+  taskId,
  dirty,
+  beforeGenerate: syncPreviewChanges,
 })

 const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
@@ -160,7 +181,101 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
  structuredOptions.value = { ...structuredOptions.value, ...value }
 }

+function toBackendConfig(): DataProcessConfig {
+  const options = processType.value === 'unstructured'
+    ? unstructuredOptions.value
+    : structuredOptions.value
+
+  const common = {
+    preprocess_options: [...options.preprocessOptions],
+    semantic_enrichment: options.semanticEnrichment,
+    dataset_split: { ...options.datasetSplit },
+    generation_model_id: options.generationModelId,
+    generation_prompt: options.generationPrompt,
+    temperature: options.temperature,
+    max_tokens: options.maxTokens,
+    json_mode: options.jsonMode,
+    quality_filter_enabled: options.qualityFilterEnabled,
+    filter_low_quality: options.filterLowQuality,
+    filter_short_content: options.filterShortContent,
+    min_output_length: options.minOutputLength,
+  }
+
+  if (processType.value === 'unstructured') {
+    return {
+      ...common,
+      chunk_method: unstructuredOptions.value.chunkMethod,
+      chunk_size: unstructuredOptions.value.chunkSize,
+      chunk_overlap: unstructuredOptions.value.chunkOverlap,
+      min_chunk_size: unstructuredOptions.value.minChunkSize,
+      custom_delimiter: unstructuredOptions.value.customDelimiter,
+      preserve_tables: unstructuredOptions.value.preserveTables,
+      preserve_code_blocks: unstructuredOptions.value.preserveCodeBlocks,
+      preserve_lists: unstructuredOptions.value.preserveLists,
+      qa_pairs_per_chunk: unstructuredOptions.value.qaPairsPerChunk,
+    }
+  }
+
+  return {
+    ...common,
+    qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
+  }
+}
+
+function taskPayload() {
+  return {
+    name: task.name.trim(),
+    description: task.description.trim(),
+    process_type: processType.value,
+    config: toBackendConfig(),
+  }
+}
+
+function externalPayload(): DataProcessExternalSourcePayload {
+  return {
+    type: externalSource.type,
+    url: externalSource.url.trim(),
+    auth_mode: externalSource.authMode,
+    username: externalSource.username || undefined,
+    password: externalSource.password || undefined,
+    limit: externalSource.limit,
+    query: externalSource.query?.trim() || undefined,
+    file_name: externalSource.fileName || 'external-data.jsonl',
+  }
+}
+
+function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
+  return {
+    id: String(item.id),
+    sourceFileId: String(item.source_file_id),
+    originalContent: item.original_content,
+    editedContent: item.edited_content,
+    sourceStart: item.source_start,
+    sourceEnd: item.source_end,
+    sourceStartLine: item.source_start_line,
+    sourceEndLine: item.source_end_line,
+    tokenCount: item.token_count,
+    status: item.status,
+    updatedAt: item.updated_at,
+  }
+}
+
+function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
+  return {
+    uid: String(file.id),
+    sourceFileId: String(file.id),
+    name: file.name,
+    size: file.size_bytes,
+    count: file.record_count,
+    content,
+    fileFormat: file.file_format,
+    checksumSha256: file.checksum_sha256,
+    status: 'ready',
+  }
+}
+
 const { persistDraft, restoreDraft } = useDataProcessDraft({
+  taskId,
  currentStepId,
  task,
  processType,
@@ -255,7 +370,7 @@ const generationOptionsSignature = computed(() => JSON.stringify(generationAffec

 function buildPreviewSignature() {
  const filesSignature = uploadedFiles.value
-    .map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
+    .map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
    .join('|')
  return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
 }
@@ -280,7 +395,7 @@ watch(generationOptionsSignature, (currentSignature, previousSignature) => {
 })

 watch(
-  [task, processType, structuredOptions, unstructuredOptions, externalSource],
+  [taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
  persistDraft,
  { deep: true },
 )
@@ -294,48 +409,85 @@ watch(currentStep, () => nextTick(scrollToStepTop))
 async function handleFileChange(uploadFile: UploadFile) {
  const raw = uploadFile.raw
  if (!raw) return
+  if (!taskId.value) {
+    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
+    return
+  }
  if (raw.size > 200 * 1024 * 1024) {
    ElMessage.warning('单文件不能超过 200MB')
    return
  }

  const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
-  const textExtensions = ['txt', 'md', 'json', 'jsonl', 'csv']
+  const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
+  if (!textExtensions.has(extension)) {
+    ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV；不会用示例内容替代无法解析的文件')
+    return
+  }
+
+  if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
+    ElMessage.warning('同名且同大小的文件已经上传')
+    return
+  }
+
  let content = ''
-  if (textExtensions.includes(extension)) {
-    try {
-      content = await raw.text()
-    } catch {
-      content = ''
-    }
+  try {
+    content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
+  } catch {
+    ElMessage.error('文件不是有效的 UTF-8 文本，请转换编码后重试')
+    return
+  }
+  if (!content.trim()) {
+    ElMessage.warning('不能上传空文件')
+    return
  }

-  const fileContent = content.trim() ? content : DEFAULT_SOURCE_TEXT
-  const linesCount = fileContent.split('\n').filter((line) => line.trim()).length
-  
-  // Prevent duplicate upload of the same file
-  if (!uploadedFiles.value.some(f => f.name === raw.name && f.size === raw.size)) {
-    uploadedFiles.value.push({
-      uid: uploadFile.uid || Date.now() + Math.random(),
-      name: raw.name,
-      size: raw.size,
-      count: linesCount,
-      content: fileContent
-    })
+  try {
+    const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
+    const source = uploaded.files[0]
+    if (!source) throw new Error('后端未返回源文件记录')
+    uploadedFiles.value.push(mapSourceFile(source, content))
+    previewSignature.value = ''
+    resetDownstream()
+    dirty.value = true
+    ElMessage.success(`文件 ${source.name} 上传成功`)
+  } catch {
+    // 请求层已展示后端的解析或格式错误。
  }
-
-  dirty.value = true
 }

-function useSampleFile() {
-  uploadedFiles.value = [{
-    uid: 'sample-1',
-    name: 'finance_qa.jsonl',
-    size: 128 * 1024 * 1024,
-    count: DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length,
-    content: DEFAULT_SOURCE_TEXT
-  }]
-  dirty.value = true
+async function useSampleFile() {
+  if (!taskId.value) {
+    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
+    return
+  }
+  const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
+  await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
+}
+
+async function restoreRegisteredSources() {
+  if (!taskId.value) return
+  try {
+    const savedTask = await getDataProcessTask(taskId.value)
+    const sources = savedTask.source_files || []
+    const restoredFiles = await Promise.all(sources.map(async (file) => {
+      try {
+        const source = await getDataProcessSourceContent(taskId.value!, file.id, {
+          start_line: 1,
+          line_count: 5000,
+        })
+        return mapSourceFile(file, source.content)
+      } catch {
+        return mapSourceFile(file)
+      }
+    }))
+    uploadedFiles.value = restoredFiles
+    if (restoredFiles.length) {
+      ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
+    }
+  } catch {
+    ElMessage.warning('草稿任务暂时无法从后端同步，请检查服务后重试')
+  }
 }

 function updateExternalSource(value: ExternalDataSource) {
@@ -343,58 +495,81 @@ function updateExternalSource(value: ExternalDataSource) {
  externalConnected.value = false
 }

-function handleTestConnection() {
+async function handleTestConnection() {
+  if (!taskId.value) {
+    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
+    return
+  }
  if (!externalSource.url.trim()) {
    ElMessage.warning('请先填写数据源地址')
    return
  }
-  if (connectionTimer) clearTimeout(connectionTimer)
  externalPulling.value = true
-  connectionTimer = setTimeout(() => {
-    connectionTimer = null
+  try {
+    const result = await testDataProcessExternalSource(taskId.value, externalPayload())
+    externalConnected.value = result.connected
+    if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
+    else ElMessage.warning(result.message || '数据源连接失败')
+  } catch {
+    externalConnected.value = false
+  } finally {
    externalPulling.value = false
-    externalConnected.value = true
-    ElMessage.success('数据源连接测试成功')
-  }, 1500)
+  }
 }

-function handlePullData() {
+async function handlePullData() {
+  if (!taskId.value) {
+    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
+    return
+  }
  if (!externalSource.url.trim()) {
    ElMessage.warning('请先填写数据源地址')
    return
  }
-  if (pullTimer) clearTimeout(pullTimer)
+  if (!externalSource.query?.trim()) {
+    ElMessage.warning('请先填写只读 SELECT 查询语句')
+    return
+  }
  externalPulling.value = true
-  pullTimer = setTimeout(() => {
-    pullTimer = null
-    externalPulling.value = false
+  try {
+    const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
+    const newFiles: UploadedDataFile[] = []
+    for (const file of response.files) {
+      const source = await getDataProcessSourceContent(taskId.value, file.id, {
+        start_line: 1,
+        line_count: 5000,
+      })
+      newFiles.push(mapSourceFile(file, source.content))
+    }
+    uploadedFiles.value.push(...newFiles)
    externalConnected.value = true
-    const typeName = externalSource.type.toUpperCase()
-    const id = `external-${Date.now()}`
-    uploadedFiles.value.push({
-      uid: id,
-      name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
-      size: Math.min(externalSource.limit, 5000) * 64,
-      count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
-      content: DEFAULT_SOURCE_TEXT,
-    })
-    dirty.value = true
-    ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
-  }, 2000)
-}
-
-function handleRemoveFile(uid: string | number) {
-  const index = uploadedFiles.value.findIndex(f => f.uid === uid)
-  if (index > -1) {
-    uploadedFiles.value.splice(index, 1)
    previewSignature.value = ''
-    previewItems.value = []
-    selectedPreviewId.value = null
    resetDownstream()
    dirty.value = true
+    ElMessage.success(`已成功登记 ${newFiles.length} 个外部源文件`)
+  } catch {
+    externalConnected.value = false
+  } finally {
+    externalPulling.value = false
  }
 }

+async function handleRemoveFile(uid: string | number) {
+  const index = uploadedFiles.value.findIndex(f => f.uid === uid)
+  if (index < 0 || !taskId.value) return
+  try {
+    await deleteDataProcessSourceFile(taskId.value, uid)
+  } catch {
+    return
+  }
+  uploadedFiles.value.splice(index, 1)
+  previewSignature.value = ''
+  previewItems.value = []
+  selectedPreviewId.value = null
+  resetDownstream()
+  dirty.value = true
+}
+
 function resetSourceDataForProcessTypeChange() {
  uploadedFiles.value = []
  previewSignature.value = ''
@@ -415,10 +590,24 @@ async function nextFromCreate() {
 async function nextFromModel() {
  const valid = await modelSelectionRef.value?.validate()
  if (!valid) return
-  goToStep('upload')
+  try {
+    const saved = taskId.value
+      ? await updateDataProcessTask(taskId.value, taskPayload())
+      : await createDataProcessTask(taskPayload())
+    taskId.value = String(saved.id)
+    dirty.value = true
+    persistDraft()
+    goToStep('upload')
+  } catch {
+    // 请求层已展示名称冲突或配置非法等具体原因。
+  }
 }

-function nextFromUpload() {
+async function nextFromUpload() {
+  if (!taskId.value) {
+    ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
+    return
+  }
  if (uploadedFiles.value.length === 0) {
    ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
    return
@@ -426,14 +615,25 @@ function nextFromUpload() {

  const signature = buildPreviewSignature()
  if (signature !== previewSignature.value) {
-    previewItems.value = uploadedFiles.value.flatMap((file) =>
-      buildPreviewItems(
-        file.content,
-        processType.value,
-        String(file.uid),
-        processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
-      ),
-    )
+    try {
+      await buildDataProcessPreview(taskId.value, {
+        source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
+      })
+      const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
+      const items = [...first.items]
+      const pages = Math.ceil(first.total / first.page_size)
+      for (let page = 2; page <= pages; page += 1) {
+        const next = await getDataProcessPreview(taskId.value, { page, page_size: 500 })
+        items.push(...next.items)
+      }
+      previewItems.value = items.map(mapPreviewItem)
+    } catch {
+      return
+    }
+    if (!previewItems.value.length) {
+      ElMessage.warning('源文件没有生成可用的预览条目，请检查文件内容和预处理配置')
+      return
+    }
    selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
    selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
    selectedPreviewIdsByFile.value = selectedPreviewId.value && selectedPreviewFileId.value
@@ -464,40 +664,50 @@ function updatePreviewContent(id: string, value: string) {
  const item = previewItems.value.find((entry) => entry.id === id)
  if (!item) return
  item.editedContent = value
-  item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
+  item.tokenCount = estimateTokenCount(value)
  item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
  resetDownstream()
  dirty.value = true
 }

+async function syncPreviewChanges() {
+  if (!taskId.value) throw new Error('任务尚未创建')
+  const changedItems = previewItems.value.filter((item) => item.status === 'modified' || item.status === 'manual')
+  for (const item of changedItems) {
+    const saved = await updateDataProcessPreview(taskId.value, item.id, {
+      edited_content: item.editedContent,
+      expected_updated_at: item.updatedAt,
+    })
+    const index = previewItems.value.findIndex((entry) => entry.id === item.id)
+    if (index >= 0) previewItems.value[index] = mapPreviewItem(saved)
+  }
+}
+
 function restorePreviewItem(id: string) {
  const item = previewItems.value.find((entry) => entry.id === id)
  if (!item || item.sourceStart == null) return
  item.editedContent = item.originalContent
-  item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
+  item.tokenCount = estimateTokenCount(item.originalContent)
  item.status = 'original'
  resetDownstream()
  dirty.value = true
 }

-function addPreviewItem() {
-  if (!selectedPreviewFileId.value) return
-  const id = `manual-${Date.now()}`
-  previewItems.value.push({
-    id,
-    sourceFileId: selectedPreviewFileId.value,
-    originalContent: '',
-    editedContent: '',
-    sourceStart: null,
-    sourceEnd: null,
-    sourceStartLine: null,
-    sourceEndLine: null,
-    tokenCount: 1,
-    status: 'manual',
-  })
-  selectPreviewItem(id)
-  resetDownstream()
-  dirty.value = true
+async function addPreviewItem() {
+  if (!selectedPreviewFileId.value || !taskId.value) return
+  try {
+    const created = await createDataProcessPreview(taskId.value, {
+      source_file_id: selectedPreviewFileId.value,
+      edited_content: '',
+    })
+    const item = mapPreviewItem(created)
+    previewItems.value.push(item)
+    selectPreviewItem(item.id)
+    resetDownstream()
+    dirty.value = true
+  } catch {
+    // 请求层已展示错误。
+  }
 }

 async function removePreviewItem(id: string) {
@@ -512,6 +722,12 @@ async function removePreviewItem(id: string) {

  const index = previewItems.value.findIndex((item) => item.id === id)
  if (index < 0) return
+  if (!taskId.value) return
+  try {
+    await deleteDataProcessPreview(taskId.value, id)
+  } catch {
+    return
+  }
  previewItems.value.splice(index, 1)
  selectedPreviewId.value = activePreviewItems.value[Math.min(index, activePreviewItems.value.length - 1)]?.id ?? null
  if (selectedPreviewFileId.value && selectedPreviewId.value) {
@@ -531,7 +747,7 @@ async function handlePrimaryAction() {
    return
  }
  if (currentStepId.value === 'upload') {
-    nextFromUpload()
+    await nextFromUpload()
    return
  }
  if (currentStepId.value === 'preview') {
@@ -546,7 +762,7 @@ async function handlePrimaryAction() {
    if (generation.status === 'success') {
      goToStep('results')
    } else if (generation.status !== 'running') {
-      startGeneration()
+      await startGeneration()
    }
    return
  }
@@ -567,6 +783,15 @@ async function saveTask() {
    ElMessage.warning('请先修正校验失败的结果')
    return
  }
+  try {
+    await persistResultChanges()
+  } catch {
+    return
+  }
+  if (!validateResults()) {
+    ElMessage.warning('仍有结果未通过后端质量校验，请继续修正')
+    return
+  }
  dirty.value = false
  localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
  allowLeave = true
@@ -607,12 +832,10 @@ onBeforeRouteLeave(async () => {

 onBeforeUnmount(() => {
  stopGenerationTimer()
-  if (connectionTimer) clearTimeout(connectionTimer)
-  if (pullTimer) clearTimeout(pullTimer)
 })
-onMounted(() => {
+onMounted(async () => {
  restoreDraft()
-  modelsStore.load()
+  await Promise.all([modelsStore.load(), restoreRegisteredSources()])
 })
 </script>