import { ref, type Ref } from 'vue' import { getDataProcessSourceContent, uploadDataProcessSourceFiles, type DataProcessSourceFile, } from '@/api/modules/dataProcess' import type { ProcessType, UploadedDataFile } from './types' const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx']) const UNSTRUCTURED_FILE_EXTENSIONS = new Set([ 'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson', ]) const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt']) const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024 const MAX_SOURCE_FILE_COUNT = 20 const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024 const SOURCE_CONTENT_PAGE_CHARS = 1_000_000 interface SourceUploadJob { uid: string file: File } interface SourceUploadOptions { taskId: Ref uploadedFiles: Ref onUploaded: (file: UploadedDataFile) => void } type SourceFileValidation = | { valid: true; extension: string } | { valid: false; severity: 'error' | 'warning'; message: string } export function validateSourceFileSelection( raw: File, processType: ProcessType, selectedFiles: UploadedDataFile[], ): SourceFileValidation { if (raw.size > MAX_SOURCE_FILE_BYTES) { return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' } } const extension = raw.name.split('.').pop()?.toLowerCase() ?? '' if (LEGACY_OFFICE_EXTENSIONS.has(extension)) { return { valid: false, severity: 'error', message: '旧版 DOC、XLS、PPT 文件暂不支持,请分别转换为 DOCX、XLSX、PPTX 后上传', } } const supportedExtensions = processType === 'unstructured' ? UNSTRUCTURED_FILE_EXTENSIONS : STRUCTURED_FILE_EXTENSIONS if (!supportedExtensions.has(extension)) { return { valid: false, severity: 'error', message: processType === 'unstructured' ? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON' : '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX', } } if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) { return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` } } const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0) if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) { return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' } } return { valid: true, extension } } function unicodeCodePointLength(value: string) { let length = 0 for (const _character of value) length += 1 return length } /** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */ export async function loadCanonicalSourceContent( taskId: string | number, fileId: string | number, ) { const chunks: string[] = [] let offset = 0 while (true) { const source = await getDataProcessSourceContent(taskId, fileId, { offset, limit: SOURCE_CONTENT_PAGE_CHARS, }) const content = source.content || '' chunks.push(content) if (!source.has_more) break const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content) if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试') offset = nextOffset } return chunks.join('') } export function mapDataProcessSourceFile( file: DataProcessSourceFile, content = '', ): UploadedDataFile { return { uid: String(file.id), sourceFileId: String(file.id), name: file.name, size: file.size_bytes, count: file.record_count, content, fileFormat: file.file_format, checksumSha256: file.checksum_sha256, status: 'ready', uploadProgress: 100, previewStatus: 'waiting', previewProgress: 0, } } function getUploadErrorMessage(error: unknown) { const responseData = (error as { response?: { data?: { detail?: string | { message?: string }; message?: string } } })?.response?.data const detail = responseData?.detail if (typeof detail === 'string') return detail if (detail?.message) return detail.message if (responseData?.message) return responseData.message if (error instanceof Error && error.message) return error.message return '上传失败,请删除该文件后重新选择' } export function useDataProcessSourceUpload(options: SourceUploadOptions) { const sourceUploading = ref(false) const queue: SourceUploadJob[] = [] let worker: Promise | null = null async function uploadOne(job: SourceUploadJob) { const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid) if (!pending) return const currentTaskId = options.taskId.value if (!currentTaskId) { pending.status = 'failed' pending.uploadError = '任务尚未创建,请返回模型选择步骤后重试' return } pending.status = 'uploading' pending.uploadProgress = 0 pending.uploadError = undefined try { const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => { pending.uploadProgress = progress }) const source = uploaded.files[0] if (!source) throw new Error('后端未返回源文件记录') // 先登记后端 ID,确保正文读取失败时仍可正确删除已落库的文件。 Object.assign(pending, mapDataProcessSourceFile(source), { status: 'uploading', uploadProgress: 99, }) try { pending.content = await loadCanonicalSourceContent(currentTaskId, source.id) } catch { throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试') } pending.status = 'ready' pending.uploadProgress = 100 options.onUploaded(pending) } catch (error) { pending.status = 'failed' pending.uploadError = getUploadErrorMessage(error) } } async function drainQueue() { sourceUploading.value = true try { while (queue.length) { const job = queue.shift() if (job) await uploadOne(job) } } finally { sourceUploading.value = false } } function enqueueSourceUpload(job: SourceUploadJob) { queue.push(job) if (worker) return worker = drainQueue().finally(() => { worker = null }) } return { enqueueSourceUpload, sourceUploading, } }