YG_FT/frontend/src/views/data-process/create/useDataProcessSourceUpload.ts

import { ref, type Ref } from 'vue'
import {
  getDataProcessSourceContent,
  uploadDataProcessSourceFiles,
  type DataProcessSourceFile,
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'

const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
  'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
])
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000

interface SourceUploadJob {
  uid: string
  file: File
}

interface SourceUploadOptions {
  taskId: Ref<string | null>
  uploadedFiles: Ref<UploadedDataFile[]>
  onUploaded: (file: UploadedDataFile) => void
}

type SourceFileValidation =
  | { valid: true; extension: string }
  | { valid: false; severity: 'error' | 'warning'; message: string }

export function validateSourceFileSelection(
  raw: File,
  processType: ProcessType,
  selectedFiles: UploadedDataFile[],
): SourceFileValidation {
  if (raw.size > MAX_SOURCE_FILE_BYTES) {
    return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
  }
  const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
  if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
    return {
      valid: false,
      severity: 'error',
      message: '旧版 DOC、XLS、PPT 文件暂不支持，请分别转换为 DOCX、XLSX、PPTX 后上传',
    }
  }
  const supportedExtensions = processType === 'unstructured'
    ? UNSTRUCTURED_FILE_EXTENSIONS
    : STRUCTURED_FILE_EXTENSIONS
  if (!supportedExtensions.has(extension)) {
    return {
      valid: false,
      severity: 'error',
      message: processType === 'unstructured'
        ? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
        : '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
    }
  }
  if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
    return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
  }
  const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
  if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
    return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
  }
  return { valid: true, extension }
}

function unicodeCodePointLength(value: string) {
  let length = 0
  for (const _character of value) length += 1
  return length
}

/** 分页读取服务端保存的规范化正文，避免重新使用浏览器本地解码结果。 */
export async function loadCanonicalSourceContent(
  taskId: string | number,
  fileId: string | number,
) {
  const chunks: string[] = []
  let offset = 0
  while (true) {
    const source = await getDataProcessSourceContent(taskId, fileId, {
      offset,
      limit: SOURCE_CONTENT_PAGE_CHARS,
    })
    const content = source.content || ''
    chunks.push(content)
    if (!source.has_more) break
    const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
    if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常，请删除文件后重试')
    offset = nextOffset
  }
  return chunks.join('')
}

export function mapDataProcessSourceFile(
  file: DataProcessSourceFile,
  content = '',
): UploadedDataFile {
  return {
    uid: String(file.id),
    sourceFileId: String(file.id),
    name: file.name,
    size: file.size_bytes,
    count: file.record_count,
    content,
    fileFormat: file.file_format,
    checksumSha256: file.checksum_sha256,
    status: 'ready',
    uploadProgress: 100,
    previewStatus: 'waiting',
    previewProgress: 0,
  }
}

function getUploadErrorMessage(error: unknown) {
  const responseData = (error as {
    response?: { data?: { detail?: string | { message?: string }; message?: string } }
  })?.response?.data
  const detail = responseData?.detail
  if (typeof detail === 'string') return detail
  if (detail?.message) return detail.message
  if (responseData?.message) return responseData.message
  if (error instanceof Error && error.message) return error.message
  return '上传失败，请删除该文件后重新选择'
}

export function useDataProcessSourceUpload(options: SourceUploadOptions) {
  const sourceUploading = ref(false)
  const queue: SourceUploadJob[] = []
  let worker: Promise<void> | null = null

  async function uploadOne(job: SourceUploadJob) {
    const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
    if (!pending) return

    const currentTaskId = options.taskId.value
    if (!currentTaskId) {
      pending.status = 'failed'
      pending.uploadError = '任务尚未创建，请返回模型选择步骤后重试'
      return
    }

    pending.status = 'uploading'
    pending.uploadProgress = 0
    pending.uploadError = undefined

    try {
      const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
        pending.uploadProgress = progress
      })
      const source = uploaded.files[0]
      if (!source) throw new Error('后端未返回源文件记录')

      // 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
      Object.assign(pending, mapDataProcessSourceFile(source), {
        status: 'uploading',
        uploadProgress: 99,
      })
      try {
        pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
      } catch {
        throw new Error('文件已上传，但服务端规范化内容读取失败，请删除文件后重试')
      }

      pending.status = 'ready'
      pending.uploadProgress = 100
      options.onUploaded(pending)
    } catch (error) {
      pending.status = 'failed'
      pending.uploadError = getUploadErrorMessage(error)
    }
  }

  async function drainQueue() {
    sourceUploading.value = true
    try {
      while (queue.length) {
        const job = queue.shift()
        if (job) await uploadOne(job)
      }
    } finally {
      sourceUploading.value = false
    }
  }

  function enqueueSourceUpload(job: SourceUploadJob) {
    queue.push(job)
    if (worker) return
    worker = drainQueue().finally(() => {
      worker = null
    })
  }

  return {
    enqueueSourceUpload,
    sourceUploading,
  }
}