feat(data-process): 优化上传切分流程与PDF高亮预览
This commit is contained in:
caoxiaozhu
@@ -0,0 +1,198 @@
|
||||
import { ref, type Ref } from 'vue'
|
||||
import {
|
||||
getDataProcessSourceContent,
|
||||
uploadDataProcessSourceFiles,
|
||||
type DataProcessSourceFile,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
])
|
||||
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
taskId: Ref<string | null>
|
||||
uploadedFiles: Ref<UploadedDataFile[]>
|
||||
onUploaded: (file: UploadedDataFile) => void
|
||||
}
|
||||
|
||||
type SourceFileValidation =
|
||||
| { valid: true; extension: string }
|
||||
| { valid: false; severity: 'error' | 'warning'; message: string }
|
||||
|
||||
export function validateSourceFileSelection(
|
||||
raw: File,
|
||||
processType: ProcessType,
|
||||
selectedFiles: UploadedDataFile[],
|
||||
): SourceFileValidation {
|
||||
if (raw.size > MAX_SOURCE_FILE_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
|
||||
}
|
||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||
if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: '旧版 DOC、XLS、PPT 文件暂不支持，请分别转换为 DOCX、XLSX、PPTX 后上传',
|
||||
}
|
||||
}
|
||||
const supportedExtensions = processType === 'unstructured'
|
||||
? UNSTRUCTURED_FILE_EXTENSIONS
|
||||
: STRUCTURED_FILE_EXTENSIONS
|
||||
if (!supportedExtensions.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: processType === 'unstructured'
|
||||
? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
|
||||
if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
|
||||
}
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
): UploadedDataFile {
|
||||
return {
|
||||
uid: String(file.id),
|
||||
sourceFileId: String(file.id),
|
||||
name: file.name,
|
||||
size: file.size_bytes,
|
||||
count: file.record_count,
|
||||
content,
|
||||
fileFormat: file.file_format,
|
||||
checksumSha256: file.checksum_sha256,
|
||||
status: 'ready',
|
||||
uploadProgress: 100,
|
||||
previewStatus: 'waiting',
|
||||
previewProgress: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function getUploadErrorMessage(error: unknown) {
|
||||
const responseData = (error as {
|
||||
response?: { data?: { detail?: string | { message?: string }; message?: string } }
|
||||
})?.response?.data
|
||||
const detail = responseData?.detail
|
||||
if (typeof detail === 'string') return detail
|
||||
if (detail?.message) return detail.message
|
||||
if (responseData?.message) return responseData.message
|
||||
if (error instanceof Error && error.message) return error.message
|
||||
return '上传失败，请删除该文件后重新选择'
|
||||
}
|
||||
|
||||
export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
const sourceUploading = ref(false)
|
||||
const queue: SourceUploadJob[] = []
|
||||
let worker: Promise<void> | null = null
|
||||
|
||||
async function uploadOne(job: SourceUploadJob) {
|
||||
const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
|
||||
if (!pending) return
|
||||
|
||||
const currentTaskId = options.taskId.value
|
||||
if (!currentTaskId) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = '任务尚未创建，请返回模型选择步骤后重试'
|
||||
return
|
||||
}
|
||||
|
||||
pending.status = 'uploading'
|
||||
pending.uploadProgress = 0
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
const source = uploaded.files[0]
|
||||
if (!source) throw new Error('后端未返回源文件记录')
|
||||
|
||||
// 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
pending.uploadProgress = 100
|
||||
options.onUploaded(pending)
|
||||
} catch (error) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = getUploadErrorMessage(error)
|
||||
}
|
||||
}
|
||||
|
||||
async function drainQueue() {
|
||||
sourceUploading.value = true
|
||||
try {
|
||||
while (queue.length) {
|
||||
const job = queue.shift()
|
||||
if (job) await uploadOne(job)
|
||||
}
|
||||
} finally {
|
||||
sourceUploading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueSourceUpload(job: SourceUploadJob) {
|
||||
queue.push(job)
|
||||
if (worker) return
|
||||
worker = drainQueue().finally(() => {
|
||||
worker = null
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
enqueueSourceUpload,
|
||||
sourceUploading,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user