fix: 完善数据预处理与 JSON 上传链路

frontend/src/views/data-process/create/useDataProcessSourceUpload.ts

@@ -6,7 +6,6 @@ import {
 } from '@/api/modules/dataProcess'
 import type { ProcessType, UploadedDataFile } from './types'
 
-const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
 const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
 const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
   'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
 const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
 const MAX_SOURCE_FILE_COUNT = 20
 const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
+const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
 
 interface SourceUploadJob {
   uid: string
   file: File
-  extension: string
 }
 
 interface SourceUploadOptions {
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
         : '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
     }
   }
-  if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
-    return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
-  }
   if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
     return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
   }
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
   return { valid: true, extension }
 }
 
+function unicodeCodePointLength(value: string) {
+  let length = 0
+  for (const _character of value) length += 1
+  return length
+}
+
+/** 分页读取服务端保存的规范化正文，避免重新使用浏览器本地解码结果。 */
+export async function loadCanonicalSourceContent(
+  taskId: string | number,
+  fileId: string | number,
+) {
+  const chunks: string[] = []
+  let offset = 0
+  while (true) {
+    const source = await getDataProcessSourceContent(taskId, fileId, {
+      offset,
+      limit: SOURCE_CONTENT_PAGE_CHARS,
+    })
+    const content = source.content || ''
+    chunks.push(content)
+    if (!source.has_more) break
+    const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
+    if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常，请删除文件后重试')
+    offset = nextOffset
+  }
+  return chunks.join('')
+}
+
 export function mapDataProcessSourceFile(
   file: DataProcessSourceFile,
   content = '',
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
     pending.uploadError = undefined
 
     try {
-      let content = ''
-      if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
-        try {
-          content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
-        } catch {
-          throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
-        }
-        if (!content.trim()) throw new Error('不能上传空文件')
-      }
-
       const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
         pending.uploadProgress = progress
       })
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
 
       // 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
       Object.assign(pending, mapDataProcessSourceFile(source), {
-        rawFile: job.file,
         status: 'uploading',
         uploadProgress: 99,
       })
-      if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
-        try {
-          const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
-            start_line: 1,
-            line_count: 10_000,
-          })
-          pending.content = parsed.content
-        } catch {
-          // 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
-        }
-      } else {
-        pending.content = content
+      try {
+        pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
+      } catch {
+        throw new Error('文件已上传，但服务端规范化内容读取失败，请删除文件后重试')
       }
 
       pending.status = 'ready'