YG_FT/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts

import { ref, type Ref } from 'vue'
import {
  getDataProcessSourceContent,
  uploadDataProcessSourceFiles,
  type DataProcessSourceFile,
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'

const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
  'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
])
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024

interface SourceUploadJob {
  uid: string
  file: File
  extension: string
}

interface SourceUploadOptions {
  taskId: Ref<string | null>
  uploadedFiles: Ref<UploadedDataFile[]>
  onUploaded: (file: UploadedDataFile) => void
}

type SourceFileValidation =
  | { valid: true; extension: string }
  | { valid: false; severity: 'error' | 'warning'; message: string }

export function validateSourceFileSelection(
  raw: File,
  processType: ProcessType,
  selectedFiles: UploadedDataFile[],
): SourceFileValidation {
  if (raw.size > MAX_SOURCE_FILE_BYTES) {
    return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
  }
  const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
  if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
    return {
      valid: false,
      severity: 'error',
      message: '旧版 DOC、XLS、PPT 文件暂不支持，请分别转换为 DOCX、XLSX、PPTX 后上传',
    }
  }
  const supportedExtensions = processType === 'unstructured'
    ? UNSTRUCTURED_FILE_EXTENSIONS
    : STRUCTURED_FILE_EXTENSIONS
  if (!supportedExtensions.has(extension)) {
    return {
      valid: false,
      severity: 'error',
      message: processType === 'unstructured'
        ? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
        : '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
    }
  }
  if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
    return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
  }
  if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
    return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
  }
  const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
  if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
    return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
  }
  return { valid: true, extension }
}

export function mapDataProcessSourceFile(
  file: DataProcessSourceFile,
  content = '',
): UploadedDataFile {
  return {
    uid: String(file.id),
    sourceFileId: String(file.id),
    name: file.name,
    size: file.size_bytes,
    count: file.record_count,
    content,
    fileFormat: file.file_format,
    checksumSha256: file.checksum_sha256,
    status: 'ready',
    uploadProgress: 100,
    previewStatus: 'waiting',
    previewProgress: 0,
  }
}

function getUploadErrorMessage(error: unknown) {
  const responseData = (error as {
    response?: { data?: { detail?: string | { message?: string }; message?: string } }
  })?.response?.data
  const detail = responseData?.detail
  if (typeof detail === 'string') return detail
  if (detail?.message) return detail.message
  if (responseData?.message) return responseData.message
  if (error instanceof Error && error.message) return error.message
  return '上传失败，请删除该文件后重新选择'
}

export function useDataProcessSourceUpload(options: SourceUploadOptions) {
  const sourceUploading = ref(false)
  const queue: SourceUploadJob[] = []
  let worker: Promise<void> | null = null

  async function uploadOne(job: SourceUploadJob) {
    const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
    if (!pending) return

    const currentTaskId = options.taskId.value
    if (!currentTaskId) {
      pending.status = 'failed'
      pending.uploadError = '任务尚未创建，请返回模型选择步骤后重试'
      return
    }

    pending.status = 'uploading'
    pending.uploadProgress = 0
    pending.uploadError = undefined

    try {
      let content = ''
      if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
        try {
          content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
        } catch {
          throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
        }
        if (!content.trim()) throw new Error('不能上传空文件')
      }

      const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
        pending.uploadProgress = progress
      })
      const source = uploaded.files[0]
      if (!source) throw new Error('后端未返回源文件记录')

      // 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
      Object.assign(pending, mapDataProcessSourceFile(source), {
        rawFile: job.file,
        status: 'uploading',
        uploadProgress: 99,
      })
      if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
        try {
          const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
            start_line: 1,
            line_count: 10_000,
          })
          pending.content = parsed.content
        } catch {
          // 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
        }
      } else {
        pending.content = content
      }

      pending.status = 'ready'
      pending.uploadProgress = 100
      options.onUploaded(pending)
    } catch (error) {
      pending.status = 'failed'
      pending.uploadError = getUploadErrorMessage(error)
    }
  }

  async function drainQueue() {
    sourceUploading.value = true
    try {
      while (queue.length) {
        const job = queue.shift()
        if (job) await uploadOne(job)
      }
    } finally {
      sourceUploading.value = false
    }
  }

  function enqueueSourceUpload(job: SourceUploadJob) {
    queue.push(job)
    if (worker) return
    worker = drainQueue().finally(() => {
      worker = null
    })
  }

  return {
    enqueueSourceUpload,
    sourceUploading,
  }
}