feat(data-process): 优化上传切分流程与PDF高亮预览
This commit is contained in:
@@ -3,6 +3,7 @@ import type {
|
||||
DataProcessExternalSourcePayload,
|
||||
DataProcessExternalTestResult,
|
||||
DataProcessPage,
|
||||
DataProcessPdfPages,
|
||||
DataProcessPreviewBuildPayload,
|
||||
DataProcessPreviewBuildResult,
|
||||
DataProcessPreviewCreatePayload,
|
||||
@@ -27,8 +28,13 @@ export type {
|
||||
DataProcessExternalSourcePayload,
|
||||
DataProcessExternalTestResult,
|
||||
DataProcessPage,
|
||||
DataProcessPdfPageRange,
|
||||
DataProcessPdfPages,
|
||||
DataProcessPreviewBuildPayload,
|
||||
DataProcessPreviewBuildFileResult,
|
||||
DataProcessPreviewBuildResult,
|
||||
DataProcessPreviewFileBuildProgress,
|
||||
DataProcessPreviewFileStatus,
|
||||
DataProcessPreviewCreatePayload,
|
||||
DataProcessPreviewItem,
|
||||
DataProcessPreviewUpdatePayload,
|
||||
@@ -71,13 +77,25 @@ export const updateDataProcessTask = (taskId: string | number, payload: DataProc
|
||||
export const deleteDataProcessTask = (taskId: string | number) =>
|
||||
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||
|
||||
export function uploadDataProcessSourceFiles(taskId: string | number, files: File[]) {
|
||||
export function uploadDataProcessSourceFiles(
|
||||
taskId: string | number,
|
||||
files: File[],
|
||||
onProgress?: (progress: number) => void,
|
||||
) {
|
||||
const formData = new FormData()
|
||||
files.forEach((file) => formData.append('files', file))
|
||||
return post<{ files: DataProcessSourceFile[] }>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files`,
|
||||
formData,
|
||||
{ headers: { 'Content-Type': 'multipart/form-data' } },
|
||||
{
|
||||
timeout: 5 * 60 * 1000,
|
||||
onUploadProgress: (event) => {
|
||||
if (!event.total) return
|
||||
// 发送完成不等于服务端解析完成;收到成功响应前最多展示 99%。
|
||||
const progress = Math.round((event.loaded / event.total) * 100)
|
||||
onProgress?.(Math.min(99, Math.max(0, progress)))
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -95,6 +113,18 @@ export const getDataProcessSourceContent = (
|
||||
params,
|
||||
)
|
||||
|
||||
export const getDataProcessSourceRawUrl = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) => `/modelTF/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/raw`
|
||||
|
||||
export const getDataProcessPdfPages = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) => get<DataProcessPdfPages>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/pdf-pages`,
|
||||
)
|
||||
|
||||
export const testDataProcessExternalSource = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessExternalSourcePayload,
|
||||
@@ -120,6 +150,7 @@ export const buildDataProcessPreview = (
|
||||
) => post<DataProcessPreviewBuildResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/build`,
|
||||
payload,
|
||||
{ timeout: 5 * 60 * 1000 },
|
||||
)
|
||||
|
||||
export function getDataProcessPreview(
|
||||
|
||||
@@ -93,6 +93,17 @@ export interface DataProcessSourceContent {
|
||||
total_chars?: number
|
||||
}
|
||||
|
||||
export interface DataProcessPdfPageRange {
|
||||
page_number: number
|
||||
source_start: number
|
||||
source_end: number
|
||||
}
|
||||
|
||||
export interface DataProcessPdfPages {
|
||||
page_count: number
|
||||
pages: DataProcessPdfPageRange[]
|
||||
}
|
||||
|
||||
export interface DataProcessExternalSourcePayload {
|
||||
type: 'postgresql'
|
||||
url: string
|
||||
@@ -129,7 +140,26 @@ export interface DataProcessPreviewBuildPayload {
|
||||
source_file_ids?: Array<string | number>
|
||||
}
|
||||
|
||||
export type DataProcessPreviewBuildResult = DataProcessPage<DataProcessPreviewItem>
|
||||
export type DataProcessPreviewFileStatus = 'waiting' | 'processing' | 'success' | 'failed'
|
||||
|
||||
export interface DataProcessPreviewBuildFileResult {
|
||||
source_file_id: string | number
|
||||
preview_count: number
|
||||
status: 'completed' | 'empty'
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewFileBuildProgress {
|
||||
source_file_id: string | number
|
||||
status: Exclude<DataProcessPreviewFileStatus, 'waiting'>
|
||||
progress: number
|
||||
preview_count?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewBuildResult extends DataProcessPage<DataProcessPreviewItem> {
|
||||
file_counts?: Record<string, number>
|
||||
files?: DataProcessPreviewBuildFileResult[]
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewCreatePayload {
|
||||
source_file_id?: string | number | null
|
||||
|
||||
@@ -15,29 +15,27 @@ import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
} from './create/dataProcessCreateState'
|
||||
import {
|
||||
DATA_PROCESS_DRAFT_STORAGE_KEY,
|
||||
useDataProcessDraft,
|
||||
} from './create/useDataProcessDraft'
|
||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import {
|
||||
mapDataProcessSourceFile,
|
||||
useDataProcessSourceUpload,
|
||||
validateSourceFileSelection,
|
||||
} from './create/useDataProcessSourceUpload'
|
||||
import { useModelsStore } from '@/stores/models'
|
||||
import {
|
||||
buildDataProcessPreview,
|
||||
createDataProcessPreview,
|
||||
createDataProcessTask,
|
||||
deleteDataProcessPreview,
|
||||
deleteDataProcessSourceFile,
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
pullDataProcessExternalSource,
|
||||
testDataProcessExternalSource,
|
||||
updateDataProcessPreview,
|
||||
updateDataProcessTask,
|
||||
uploadDataProcessSourceFiles,
|
||||
type DataProcessExternalSourcePayload,
|
||||
type DataProcessPreviewItem,
|
||||
type DataProcessSourceFile,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { DataProcessConfig } from '@/types/dataProcess'
|
||||
import type {
|
||||
@@ -54,15 +52,12 @@ import type {
|
||||
const router = useRouter()
|
||||
const modelsStore = useModelsStore()
|
||||
const { list: modelList } = storeToRefs(modelsStore)
|
||||
const generationModels = computed(() => modelList.value.filter((model) => (
|
||||
model.type === 'LLM'
|
||||
&& (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
|
||||
)))
|
||||
// 候选范围与模型管理保持一致,不在数据处理页面重复定义模型过滤规则。
|
||||
const generationModels = computed(() => modelList.value)
|
||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
|
||||
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'
|
||||
|
||||
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v2'
|
||||
const WIZARD_STEPS = [
|
||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
|
||||
@@ -82,7 +77,8 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
|
||||
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
|
||||
))
|
||||
const uploadedFiles = ref<UploadedDataFile[]>([])
|
||||
|
||||
const previewBuilding = ref(false)
|
||||
const { buildPreviewsByFile } = useDataProcessPreviewBuild()
|
||||
const externalSource = reactive<ExternalDataSource>({
|
||||
type: 'postgresql',
|
||||
url: '',
|
||||
@@ -95,7 +91,6 @@ const externalSource = reactive<ExternalDataSource>({
|
||||
})
|
||||
const externalPulling = ref(false)
|
||||
const externalConnected = ref(false)
|
||||
|
||||
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
||||
const previewSignature = ref('')
|
||||
const previewItems = ref<PreviewItem[]>([])
|
||||
@@ -103,7 +98,6 @@ const selectedPreviewFileId = ref<string | null>(null)
|
||||
const selectedPreviewId = ref<string | null>(null)
|
||||
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
|
||||
const dirty = ref(false)
|
||||
const restoringDraft = ref(false)
|
||||
let allowLeave = false
|
||||
|
||||
const {
|
||||
@@ -123,6 +117,19 @@ const {
|
||||
dirty,
|
||||
beforeGenerate: syncPreviewChanges,
|
||||
})
|
||||
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
|
||||
taskId,
|
||||
uploadedFiles,
|
||||
onUploaded(file) {
|
||||
previewSignature.value = ''
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
ElMessage.success(`文件 ${file.name} 上传成功,等待切分`)
|
||||
},
|
||||
})
|
||||
const hasUnfinishedUploads = computed(() => uploadedFiles.value.some((file) => (
|
||||
file.status !== 'ready' || !file.sourceFileId
|
||||
)))
|
||||
|
||||
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
|
||||
const previewItemsByFile = computed(() => {
|
||||
@@ -149,7 +156,10 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
|
||||
const primaryActionLabel = computed(() => {
|
||||
if (currentStepId.value === 'create') return '继续:选择大模型'
|
||||
if (currentStepId.value === 'model') return '继续:上传文件'
|
||||
if (currentStepId.value === 'upload') return '继续:数据预览'
|
||||
if (currentStepId.value === 'upload') {
|
||||
if (sourceUploading.value) return '正在上传'
|
||||
return previewBuilding.value ? '正在切分' : '继续:数据预览'
|
||||
}
|
||||
if (currentStepId.value === 'preview') return '确认预览并继续'
|
||||
if (currentStepId.value === 'results') return '保存任务'
|
||||
if (generation.status === 'running') return '正在生成'
|
||||
@@ -260,33 +270,6 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
}
|
||||
}
|
||||
|
||||
function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
|
||||
return {
|
||||
uid: String(file.id),
|
||||
sourceFileId: String(file.id),
|
||||
name: file.name,
|
||||
size: file.size_bytes,
|
||||
count: file.record_count,
|
||||
content,
|
||||
fileFormat: file.file_format,
|
||||
checksumSha256: file.checksum_sha256,
|
||||
status: 'ready',
|
||||
}
|
||||
}
|
||||
|
||||
const { persistDraft, restoreDraft } = useDataProcessDraft({
|
||||
taskId,
|
||||
currentStepId,
|
||||
task,
|
||||
processType,
|
||||
structuredOptions,
|
||||
unstructuredOptions,
|
||||
externalSource,
|
||||
restoringDraft,
|
||||
dirty,
|
||||
goToStep,
|
||||
})
|
||||
|
||||
function previewAffectingOptions() {
|
||||
if (processType.value === 'structured') {
|
||||
return {
|
||||
@@ -368,92 +351,70 @@ function generationAffectingOptions() {
|
||||
|
||||
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
|
||||
|
||||
function buildPreviewConfigSignature() {
|
||||
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}`
|
||||
}
|
||||
|
||||
function buildPreviewSignature() {
|
||||
const filesSignature = uploadedFiles.value
|
||||
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
|
||||
.join('|')
|
||||
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
|
||||
return `${buildPreviewConfigSignature()}:${filesSignature}`
|
||||
}
|
||||
|
||||
watch(
|
||||
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
() => {
|
||||
if (!restoringDraft.value) dirty.value = true
|
||||
},
|
||||
() => { dirty.value = true },
|
||||
{ deep: true },
|
||||
)
|
||||
|
||||
watch(processType, (nextType, previousType) => {
|
||||
if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
|
||||
if (nextType === previousType || uploadedFiles.value.length === 0) return
|
||||
resetSourceDataForProcessTypeChange()
|
||||
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
|
||||
})
|
||||
|
||||
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||
if (restoringDraft.value || currentSignature === previousSignature) return
|
||||
if (currentSignature === previousSignature) return
|
||||
resetDownstream()
|
||||
})
|
||||
|
||||
watch(
|
||||
[taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
persistDraft,
|
||||
{ deep: true },
|
||||
)
|
||||
|
||||
function scrollToStepTop() {
|
||||
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
|
||||
}
|
||||
|
||||
watch(currentStep, () => nextTick(scrollToStepTop))
|
||||
|
||||
async function handleFileChange(uploadFile: UploadFile) {
|
||||
function handleFileChange(uploadFile: UploadFile) {
|
||||
const raw = uploadFile.raw
|
||||
if (!raw) return
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
if (raw.size > 200 * 1024 * 1024) {
|
||||
ElMessage.warning('单文件不能超过 200MB')
|
||||
const validation = validateSourceFileSelection(raw, processType.value, uploadedFiles.value)
|
||||
if (!validation.valid) {
|
||||
ElMessage[validation.severity](validation.message)
|
||||
return
|
||||
}
|
||||
|
||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||
const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
|
||||
if (!textExtensions.has(extension)) {
|
||||
ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV;不会用示例内容替代无法解析的文件')
|
||||
return
|
||||
}
|
||||
|
||||
if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
ElMessage.warning('同名且同大小的文件已经上传')
|
||||
return
|
||||
}
|
||||
|
||||
let content = ''
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
|
||||
} catch {
|
||||
ElMessage.error('文件不是有效的 UTF-8 文本,请转换编码后重试')
|
||||
return
|
||||
}
|
||||
if (!content.trim()) {
|
||||
ElMessage.warning('不能上传空文件')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
|
||||
const source = uploaded.files[0]
|
||||
if (!source) throw new Error('后端未返回源文件记录')
|
||||
uploadedFiles.value.push(mapSourceFile(source, content))
|
||||
previewSignature.value = ''
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
ElMessage.success(`文件 ${source.name} 上传成功`)
|
||||
} catch {
|
||||
// 请求层已展示后端的解析或格式错误。
|
||||
}
|
||||
// 必须先同步插入文件行,再交给队列;这样选择完成后页面会立即展示全部文件。
|
||||
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
|
||||
uploadedFiles.value.push({
|
||||
uid: localUid,
|
||||
rawFile: raw,
|
||||
name: raw.name,
|
||||
size: raw.size,
|
||||
count: 0,
|
||||
content: '',
|
||||
fileFormat: validation.extension,
|
||||
status: 'queued',
|
||||
uploadProgress: 0,
|
||||
previewStatus: 'waiting',
|
||||
previewProgress: 0,
|
||||
})
|
||||
dirty.value = true
|
||||
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
|
||||
}
|
||||
|
||||
async function useSampleFile() {
|
||||
@@ -462,32 +423,7 @@ async function useSampleFile() {
|
||||
return
|
||||
}
|
||||
const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
|
||||
await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
|
||||
}
|
||||
|
||||
async function restoreRegisteredSources() {
|
||||
if (!taskId.value) return
|
||||
try {
|
||||
const savedTask = await getDataProcessTask(taskId.value)
|
||||
const sources = savedTask.source_files || []
|
||||
const restoredFiles = await Promise.all(sources.map(async (file) => {
|
||||
try {
|
||||
const source = await getDataProcessSourceContent(taskId.value!, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
return mapSourceFile(file, source.content)
|
||||
} catch {
|
||||
return mapSourceFile(file)
|
||||
}
|
||||
}))
|
||||
uploadedFiles.value = restoredFiles
|
||||
if (restoredFiles.length) {
|
||||
ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
|
||||
}
|
||||
} catch {
|
||||
ElMessage.warning('草稿任务暂时无法从后端同步,请检查服务后重试')
|
||||
}
|
||||
handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
|
||||
}
|
||||
|
||||
function updateExternalSource(value: ExternalDataSource) {
|
||||
@@ -539,7 +475,7 @@ async function handlePullData() {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
newFiles.push(mapSourceFile(file, source.content))
|
||||
newFiles.push(mapDataProcessSourceFile(file, source.content))
|
||||
}
|
||||
uploadedFiles.value.push(...newFiles)
|
||||
externalConnected.value = true
|
||||
@@ -557,11 +493,19 @@ async function handlePullData() {
|
||||
async function handleRemoveFile(uid: string | number) {
|
||||
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
|
||||
if (index < 0 || !taskId.value) return
|
||||
try {
|
||||
await deleteDataProcessSourceFile(taskId.value, uid)
|
||||
} catch {
|
||||
const file = uploadedFiles.value[index]
|
||||
if (!file) return
|
||||
if (file.status === 'uploading') {
|
||||
ElMessage.warning('当前文件正在上传,请等待上传结束后再删除')
|
||||
return
|
||||
}
|
||||
if (file.sourceFileId) {
|
||||
try {
|
||||
await deleteDataProcessSourceFile(taskId.value, file.sourceFileId)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
}
|
||||
uploadedFiles.value.splice(index, 1)
|
||||
previewSignature.value = ''
|
||||
previewItems.value = []
|
||||
@@ -596,7 +540,6 @@ async function nextFromModel() {
|
||||
: await createDataProcessTask(taskPayload())
|
||||
taskId.value = String(saved.id)
|
||||
dirty.value = true
|
||||
persistDraft()
|
||||
goToStep('upload')
|
||||
} catch {
|
||||
// 请求层已展示名称冲突或配置非法等具体原因。
|
||||
@@ -604,6 +547,7 @@ async function nextFromModel() {
|
||||
}
|
||||
|
||||
async function nextFromUpload() {
|
||||
if (previewBuilding.value || sourceUploading.value) return
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
|
||||
return
|
||||
@@ -612,13 +556,61 @@ async function nextFromUpload() {
|
||||
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
||||
return
|
||||
}
|
||||
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
|
||||
if (failedUploads.length) {
|
||||
ElMessage.warning(`${failedUploads.length} 个文件上传失败,请删除后重新选择`)
|
||||
return
|
||||
}
|
||||
if (hasUnfinishedUploads.value) {
|
||||
ElMessage.warning('请等待所有文件上传完成后再开始切分')
|
||||
return
|
||||
}
|
||||
|
||||
const signature = buildPreviewSignature()
|
||||
if (signature !== previewSignature.value) {
|
||||
const configSignature = buildPreviewConfigSignature()
|
||||
const allFilesSucceeded = () => uploadedFiles.value.every((file) => (
|
||||
file.previewStatus === 'success' && file.previewConfigSignature === configSignature
|
||||
))
|
||||
|
||||
if (signature === previewSignature.value && previewItems.value.length && allFilesSucceeded()) {
|
||||
goToStep('preview')
|
||||
return
|
||||
}
|
||||
|
||||
for (const file of uploadedFiles.value) {
|
||||
if (file.previewConfigSignature === configSignature) continue
|
||||
file.previewStatus = 'waiting'
|
||||
file.previewProgress = 0
|
||||
file.previewError = undefined
|
||||
file.previewCount = undefined
|
||||
}
|
||||
|
||||
previewBuilding.value = true
|
||||
try {
|
||||
const pendingFileIds = uploadedFiles.value
|
||||
.filter((file) => file.previewStatus !== 'success' || file.previewConfigSignature !== configSignature)
|
||||
.map((file) => file.sourceFileId)
|
||||
.filter((fileId): fileId is string => Boolean(fileId))
|
||||
// 构建接口是同步响应:请求中只展示真实的“处理中”,响应成功后才记为 100%。
|
||||
await buildPreviewsByFile(taskId.value, pendingFileIds, (progress) => {
|
||||
const file = uploadedFiles.value.find((item) => (
|
||||
String(item.sourceFileId) === String(progress.source_file_id)
|
||||
))
|
||||
if (!file) return
|
||||
file.previewStatus = progress.status
|
||||
file.previewProgress = progress.progress
|
||||
file.previewCount = progress.preview_count
|
||||
file.previewError = progress.error
|
||||
if (progress.status === 'success') file.previewConfigSignature = configSignature
|
||||
})
|
||||
|
||||
const failedCount = uploadedFiles.value.filter((file) => file.previewStatus === 'failed').length
|
||||
if (failedCount || !allFilesSucceeded()) {
|
||||
ElMessage.warning(`${failedCount || 1} 个文件切分失败;修正问题后点击“继续:数据预览”即可重试`)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
await buildDataProcessPreview(taskId.value, {
|
||||
source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
|
||||
})
|
||||
const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
@@ -628,6 +620,7 @@ async function nextFromUpload() {
|
||||
}
|
||||
previewItems.value = items.map(mapPreviewItem)
|
||||
} catch {
|
||||
ElMessage.warning('文件切分已经完成,但预览加载失败;请再次点击继续重试加载')
|
||||
return
|
||||
}
|
||||
if (!previewItems.value.length) {
|
||||
@@ -641,8 +634,10 @@ async function nextFromUpload() {
|
||||
: {}
|
||||
previewSignature.value = signature
|
||||
resetDownstream()
|
||||
goToStep('preview')
|
||||
} finally {
|
||||
previewBuilding.value = false
|
||||
}
|
||||
goToStep('preview')
|
||||
}
|
||||
|
||||
function selectPreviewFile(fileId: string) {
|
||||
@@ -770,6 +765,14 @@ async function handlePrimaryAction() {
|
||||
}
|
||||
|
||||
function handleBack() {
|
||||
if (sourceUploading.value) {
|
||||
ElMessage.warning('请等待当前文件上传完成')
|
||||
return
|
||||
}
|
||||
if (previewBuilding.value) {
|
||||
ElMessage.warning('请等待当前文件切分完成')
|
||||
return
|
||||
}
|
||||
if (generation.status === 'running') {
|
||||
ElMessage.warning('请先停止当前生成任务')
|
||||
return
|
||||
@@ -793,7 +796,6 @@ async function saveTask() {
|
||||
return
|
||||
}
|
||||
dirty.value = false
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
allowLeave = true
|
||||
ElMessage.success('数据处理任务已保存')
|
||||
await router.push('/data-process')
|
||||
@@ -818,6 +820,10 @@ async function handleCancel() {
|
||||
}
|
||||
|
||||
onBeforeRouteLeave(async () => {
|
||||
if (sourceUploading.value) {
|
||||
ElMessage.warning('请等待当前文件上传完成后再离开页面')
|
||||
return false
|
||||
}
|
||||
if (allowLeave || !dirty.value) return true
|
||||
const confirmed = await confirmDialogRef.value?.open({
|
||||
title: '确认离开当前页面?',
|
||||
@@ -833,9 +839,9 @@ onBeforeRouteLeave(async () => {
|
||||
onBeforeUnmount(() => {
|
||||
stopGenerationTimer()
|
||||
})
|
||||
onMounted(async () => {
|
||||
restoreDraft()
|
||||
await Promise.all([modelsStore.load(), restoreRegisteredSources()])
|
||||
onMounted(() => {
|
||||
localStorage.removeItem('yg-data-process-create-draft')
|
||||
void modelsStore.load(true)
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -899,6 +905,8 @@ onMounted(async () => {
|
||||
:external-source="externalSource"
|
||||
:external-pulling="externalPulling"
|
||||
:external-connected="externalConnected"
|
||||
:preview-building="previewBuilding"
|
||||
:source-uploading="sourceUploading"
|
||||
@update:external-source="updateExternalSource"
|
||||
@file-change="handleFileChange"
|
||||
@remove-file="handleRemoveFile"
|
||||
@@ -915,6 +923,8 @@ onMounted(async () => {
|
||||
:items="activePreviewItems"
|
||||
:process-type="processType"
|
||||
:file-name="activePreviewFile?.name ?? ''"
|
||||
:task-id="taskId"
|
||||
:source-file-id="activePreviewFile?.sourceFileId ?? activePreviewFile?.uid ?? null"
|
||||
:files="previewFiles"
|
||||
@update:selected-id="selectPreviewItem"
|
||||
@update:selected-file-id="selectPreviewFile"
|
||||
@@ -949,7 +959,7 @@ onMounted(async () => {
|
||||
|
||||
<footer class="wizard-footer">
|
||||
<div class="footer-left">
|
||||
<el-button v-if="currentStep > 0" @click="handleBack">
|
||||
<el-button v-if="currentStep > 0" :disabled="previewBuilding || sourceUploading" @click="handleBack">
|
||||
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回:{{ previousStepLabel }}
|
||||
</el-button>
|
||||
<el-button v-else @click="handleCancel">取消</el-button>
|
||||
@@ -960,8 +970,8 @@ onMounted(async () => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="generation.status === 'running'"
|
||||
:disabled="currentStepId === 'generate' && generation.status === 'running'"
|
||||
:loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -84,6 +84,12 @@ const configLabelMap: Record<string, string> = {
|
||||
preserve_lists: '保留列表',
|
||||
}
|
||||
|
||||
const chunkMethodLabelMap: Record<string, string> = {
|
||||
structure: '文档结构',
|
||||
fixed: '固定 Token',
|
||||
custom: '自定义分隔符',
|
||||
}
|
||||
|
||||
function numeric(value: number | undefined) {
|
||||
return Number.isFinite(value) ? Number(value) : 0
|
||||
}
|
||||
@@ -135,6 +141,9 @@ const configRows = computed(() => Object.entries(detail.value?.config || {})
|
||||
})))
|
||||
|
||||
function formatConfigValue(key: string, value: unknown) {
|
||||
if (key === 'chunk_method' && typeof value === 'string') {
|
||||
return chunkMethodLabelMap[value] || value
|
||||
}
|
||||
if (key === 'dataset_split' && value && typeof value === 'object') {
|
||||
const split = value as Partial<DataProcessDatasetSplit>
|
||||
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
||||
|
||||
@@ -41,6 +41,15 @@ function sectionValidationMessage() {
|
||||
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
|
||||
return isModelMessage ? '' : props.validationMessage
|
||||
}
|
||||
|
||||
function modelMeta(model: ModelItem) {
|
||||
const source = model.model_source === 'api' || model.model_source === 'online'
|
||||
? '在线模型'
|
||||
: model.model_source === 'local'
|
||||
? '本地模型'
|
||||
: model.model_source || '模型管理'
|
||||
return model.type ? `${source} · ${model.type}` : source
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -66,11 +75,11 @@ function sectionValidationMessage() {
|
||||
>
|
||||
<div class="model-option">
|
||||
<span>{{ model.name }}</span>
|
||||
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
|
||||
<small>{{ modelMeta(model) }}</small>
|
||||
</div>
|
||||
</el-option>
|
||||
<template #empty>
|
||||
<div class="model-empty">暂无可用的大模型,请先在模型管理中添加</div>
|
||||
<div class="model-empty">暂无模型,请先在模型管理中添加</div>
|
||||
</template>
|
||||
</el-select>
|
||||
</div>
|
||||
|
||||
650
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
650
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
@@ -0,0 +1,650 @@
|
||||
<script setup lang="ts">
|
||||
import {
|
||||
GlobalWorkerOptions,
|
||||
TextLayer,
|
||||
getDocument,
|
||||
type PDFDocumentLoadingTask,
|
||||
type PDFDocumentProxy,
|
||||
type RenderTask,
|
||||
} from 'pdfjs-dist'
|
||||
import pdfWorkerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url'
|
||||
import { computed, nextTick, onBeforeUnmount, ref, shallowRef, watch } from 'vue'
|
||||
import {
|
||||
getDataProcessPdfPages,
|
||||
getDataProcessSourceRawUrl,
|
||||
type DataProcessPdfPageRange,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { PreviewItem } from './types'
|
||||
|
||||
GlobalWorkerOptions.workerSrc = pdfWorkerUrl
|
||||
|
||||
const props = defineProps<{
|
||||
taskId: string | number | null
|
||||
sourceFileId: string | number | null
|
||||
fileName: string
|
||||
selectedItem: PreviewItem | null
|
||||
}>()
|
||||
|
||||
const scrollRef = ref<HTMLElement | null>(null)
|
||||
const pageRef = ref<HTMLElement | null>(null)
|
||||
const canvasRef = ref<HTMLCanvasElement | null>(null)
|
||||
const textLayerRef = ref<HTMLElement | null>(null)
|
||||
const documentRef = shallowRef<PDFDocumentProxy | null>(null)
|
||||
const pageRanges = ref<DataProcessPdfPageRange[]>([])
|
||||
const currentPage = ref(1)
|
||||
const pageCount = ref(0)
|
||||
const zoom = ref(1)
|
||||
const renderedScale = ref(1)
|
||||
const loading = ref(false)
|
||||
const rendering = ref(false)
|
||||
const errorMessage = ref('')
|
||||
const highlightState = ref<'idle' | 'highlighted' | 'unmatched' | 'manual'>('idle')
|
||||
|
||||
let loadingTask: PDFDocumentLoadingTask | null = null
|
||||
let renderTask: RenderTask | null = null
|
||||
let textLayer: TextLayer | null = null
|
||||
let loadSequence = 0
|
||||
let renderSequence = 0
|
||||
let renderedPageNumber = 0
|
||||
let resizeFrame = 0
|
||||
|
||||
const sourceUrl = computed(() => (
|
||||
props.taskId != null && props.sourceFileId != null
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
|
||||
const pageStyle = computed(() => ({
|
||||
'--total-scale-factor': String(renderedScale.value),
|
||||
}))
|
||||
|
||||
const locationText = computed(() => {
|
||||
if (highlightState.value === 'highlighted') return `切片原文已在第 ${currentPage.value} 页高亮`
|
||||
if (highlightState.value === 'unmatched') return `已定位第 ${currentPage.value} 页,未匹配到可高亮文字`
|
||||
if (highlightState.value === 'manual') return '手动新增切片没有原文位置'
|
||||
return pageCount.value ? `第 ${currentPage.value} / ${pageCount.value} 页` : '正在读取 PDF'
|
||||
})
|
||||
|
||||
function pageForItem(item: PreviewItem | null) {
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return null
|
||||
return pageRanges.value.find((page) => (
|
||||
item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end
|
||||
)) ?? pageRanges.value.find((page) => (
|
||||
item.sourceStart! < page.source_end && item.sourceEnd! > page.source_start
|
||||
)) ?? null
|
||||
}
|
||||
|
||||
function selectedTextForPage(page: DataProcessPdfPageRange) {
|
||||
const item = props.selectedItem
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return ''
|
||||
const intersectionStart = Math.max(item.sourceStart, page.source_start)
|
||||
const intersectionEnd = Math.min(item.sourceEnd, page.source_end)
|
||||
if (intersectionEnd <= intersectionStart) return ''
|
||||
const relativeStart = Math.max(0, intersectionStart - item.sourceStart)
|
||||
const relativeEnd = Math.max(relativeStart, intersectionEnd - item.sourceStart)
|
||||
return item.originalContent.slice(relativeStart, relativeEnd)
|
||||
}
|
||||
|
||||
function normalizeLocatorText(value: string) {
|
||||
return value.normalize('NFKC').replace(/[^\p{L}\p{N}]+/gu, '').toLocaleLowerCase()
|
||||
}
|
||||
|
||||
function nearestOccurrence(haystack: string, needle: string, expectedIndex: number) {
|
||||
let nearest = -1
|
||||
let nearestDistance = Number.POSITIVE_INFINITY
|
||||
let cursor = haystack.indexOf(needle)
|
||||
while (cursor >= 0) {
|
||||
const distance = Math.abs(cursor - expectedIndex)
|
||||
if (distance < nearestDistance) {
|
||||
nearest = cursor
|
||||
nearestDistance = distance
|
||||
}
|
||||
cursor = haystack.indexOf(needle, cursor + 1)
|
||||
}
|
||||
return nearest
|
||||
}
|
||||
|
||||
function findHighlight(
|
||||
itemStrings: string[],
|
||||
selectedText: string,
|
||||
page: DataProcessPdfPageRange,
|
||||
) {
|
||||
const itemRanges: Array<{ start: number; end: number }> = []
|
||||
let pageText = ''
|
||||
for (const item of itemStrings) {
|
||||
const start = pageText.length
|
||||
pageText += normalizeLocatorText(item)
|
||||
itemRanges.push({ start, end: pageText.length })
|
||||
}
|
||||
|
||||
const target = normalizeLocatorText(selectedText)
|
||||
if (!pageText || !target) return []
|
||||
const itemStart = props.selectedItem?.sourceStart ?? page.source_start
|
||||
const sourceLength = Math.max(1, page.source_end - page.source_start)
|
||||
const expectedRatio = Math.min(1, Math.max(0, (itemStart - page.source_start) / sourceLength))
|
||||
const expectedIndex = Math.round(pageText.length * expectedRatio)
|
||||
const anchorLengths = [target.length, 120, 80, 48, 24, 12, 8]
|
||||
.map((length) => Math.min(length, target.length))
|
||||
.filter((length, index, values) => length >= 4 && values.indexOf(length) === index)
|
||||
|
||||
for (const anchorLength of anchorLengths) {
|
||||
const anchor = target.slice(0, anchorLength)
|
||||
const matchStart = nearestOccurrence(pageText, anchor, expectedIndex)
|
||||
if (matchStart < 0) continue
|
||||
const matchEnd = matchStart + anchor.length
|
||||
return itemRanges.reduce<number[]>((matches, range, index) => {
|
||||
if (range.end > matchStart && range.start < matchEnd) matches.push(index)
|
||||
return matches
|
||||
}, [])
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
function applySelectedHighlight(page: DataProcessPdfPageRange | null) {
|
||||
const item = props.selectedItem
|
||||
for (const element of textLayer?.textDivs ?? []) {
|
||||
element.classList.remove('is-slice-highlighted')
|
||||
}
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) {
|
||||
highlightState.value = item ? 'manual' : 'idle'
|
||||
return
|
||||
}
|
||||
if (!page || page.page_number !== currentPage.value || !textLayer) {
|
||||
highlightState.value = 'idle'
|
||||
return
|
||||
}
|
||||
const matches = findHighlight(
|
||||
textLayer.textContentItemsStr,
|
||||
selectedTextForPage(page),
|
||||
page,
|
||||
)
|
||||
for (const index of matches) {
|
||||
textLayer.textDivs[index]?.classList.add('is-slice-highlighted')
|
||||
}
|
||||
highlightState.value = matches.length ? 'highlighted' : 'unmatched'
|
||||
}
|
||||
|
||||
async function renderCurrentPage(force = false) {
|
||||
const document = documentRef.value
|
||||
const pageContainer = pageRef.value
|
||||
const canvas = canvasRef.value
|
||||
const layerContainer = textLayerRef.value
|
||||
const scroller = scrollRef.value
|
||||
if (!document || !pageContainer || !canvas || !layerContainer || !scroller) return
|
||||
|
||||
const selectedPage = pageForItem(props.selectedItem)
|
||||
if (!force && renderedPageNumber === currentPage.value && textLayer) {
|
||||
applySelectedHighlight(selectedPage)
|
||||
return
|
||||
}
|
||||
|
||||
const sequence = ++renderSequence
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
renderTask = null
|
||||
textLayer = null
|
||||
rendering.value = true
|
||||
errorMessage.value = ''
|
||||
|
||||
try {
|
||||
const page = await document.getPage(currentPage.value)
|
||||
if (sequence !== renderSequence) return
|
||||
const baseViewport = page.getViewport({ scale: 1 })
|
||||
const availableWidth = Math.max(280, scroller.clientWidth - 36)
|
||||
const scale = (availableWidth / baseViewport.width) * zoom.value
|
||||
const viewport = page.getViewport({ scale })
|
||||
const outputScale = Math.max(1, window.devicePixelRatio || 1)
|
||||
const context = canvas.getContext('2d')
|
||||
if (!context) throw new Error('浏览器无法创建 PDF 画布')
|
||||
|
||||
renderedScale.value = scale
|
||||
pageContainer.style.width = `${viewport.width}px`
|
||||
pageContainer.style.height = `${viewport.height}px`
|
||||
canvas.width = Math.floor(viewport.width * outputScale)
|
||||
canvas.height = Math.floor(viewport.height * outputScale)
|
||||
canvas.style.width = `${viewport.width}px`
|
||||
canvas.style.height = `${viewport.height}px`
|
||||
layerContainer.replaceChildren()
|
||||
|
||||
const activeRenderTask = page.render({
|
||||
canvas,
|
||||
viewport,
|
||||
transform: outputScale === 1 ? undefined : [outputScale, 0, 0, outputScale, 0, 0],
|
||||
})
|
||||
renderTask = activeRenderTask
|
||||
// 立即挂接取消处理,避免 ResizeObserver 触发重绘时产生未处理的取消异常。
|
||||
const canvasRenderPromise = activeRenderTask.promise.catch((error: unknown) => {
|
||||
if (
|
||||
sequence !== renderSequence
|
||||
|| (error instanceof Error && error.name === 'RenderingCancelledException')
|
||||
) return
|
||||
throw error
|
||||
})
|
||||
const textContent = await page.getTextContent()
|
||||
if (sequence !== renderSequence) return
|
||||
const activeTextLayer = new TextLayer({
|
||||
textContentSource: textContent,
|
||||
container: layerContainer,
|
||||
viewport,
|
||||
})
|
||||
textLayer = activeTextLayer
|
||||
await Promise.all([canvasRenderPromise, activeTextLayer.render()])
|
||||
if (sequence !== renderSequence) return
|
||||
renderedPageNumber = currentPage.value
|
||||
applySelectedHighlight(selectedPage)
|
||||
await nextTick()
|
||||
pageContainer.querySelector<HTMLElement>('.is-slice-highlighted')
|
||||
?.scrollIntoView({ block: 'center', inline: 'center', behavior: 'smooth' })
|
||||
} catch (error) {
|
||||
if (sequence !== renderSequence) return
|
||||
errorMessage.value = error instanceof Error ? error.message : 'PDF 页面渲染失败'
|
||||
} finally {
|
||||
if (sequence === renderSequence) rendering.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function locateSelectedItem() {
|
||||
if (!documentRef.value) return
|
||||
const item = props.selectedItem
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) {
|
||||
applySelectedHighlight(null)
|
||||
return
|
||||
}
|
||||
const page = pageForItem(item)
|
||||
if (!page) {
|
||||
highlightState.value = 'unmatched'
|
||||
return
|
||||
}
|
||||
if (currentPage.value !== page.page_number) {
|
||||
currentPage.value = page.page_number
|
||||
await renderCurrentPage(true)
|
||||
return
|
||||
}
|
||||
await renderCurrentPage(false)
|
||||
}
|
||||
|
||||
async function loadPdf() {
|
||||
const sequence = ++loadSequence
|
||||
++renderSequence
|
||||
renderedPageNumber = 0
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
await loadingTask?.destroy()
|
||||
loadingTask = null
|
||||
documentRef.value = null
|
||||
pageRanges.value = []
|
||||
pageCount.value = 0
|
||||
currentPage.value = 1
|
||||
errorMessage.value = ''
|
||||
highlightState.value = 'idle'
|
||||
if (!sourceUrl.value || props.taskId == null || props.sourceFileId == null) return
|
||||
|
||||
loading.value = true
|
||||
try {
|
||||
const task = getDocument({ url: sourceUrl.value })
|
||||
loadingTask = task
|
||||
const [mapping, document] = await Promise.all([
|
||||
getDataProcessPdfPages(props.taskId, props.sourceFileId),
|
||||
task.promise,
|
||||
])
|
||||
if (sequence !== loadSequence) {
|
||||
await task.destroy()
|
||||
return
|
||||
}
|
||||
documentRef.value = document
|
||||
pageRanges.value = mapping.pages
|
||||
pageCount.value = document.numPages
|
||||
await nextTick()
|
||||
await locateSelectedItem()
|
||||
if (!props.selectedItem) await renderCurrentPage(true)
|
||||
} catch (error) {
|
||||
if (sequence !== loadSequence) return
|
||||
errorMessage.value = error instanceof Error ? error.message : 'PDF 原文件加载失败'
|
||||
} finally {
|
||||
if (sequence === loadSequence) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function changePage(delta: number) {
|
||||
const nextPage = Math.min(pageCount.value, Math.max(1, currentPage.value + delta))
|
||||
if (nextPage === currentPage.value) return
|
||||
currentPage.value = nextPage
|
||||
await renderCurrentPage(true)
|
||||
}
|
||||
|
||||
async function changeZoom(delta: number) {
|
||||
const nextZoom = Math.min(2, Math.max(0.6, Number((zoom.value + delta).toFixed(1))))
|
||||
if (nextZoom === zoom.value) return
|
||||
zoom.value = nextZoom
|
||||
await renderCurrentPage(true)
|
||||
}
|
||||
|
||||
function scheduleResizeRender() {
|
||||
window.cancelAnimationFrame(resizeFrame)
|
||||
resizeFrame = window.requestAnimationFrame(() => {
|
||||
if (documentRef.value) void renderCurrentPage(true)
|
||||
})
|
||||
}
|
||||
|
||||
const resizeObserver = new ResizeObserver(scheduleResizeRender)
|
||||
watch(scrollRef, (element, previous) => {
|
||||
if (previous) resizeObserver.unobserve(previous)
|
||||
if (element) resizeObserver.observe(element)
|
||||
})
|
||||
watch(sourceUrl, () => void loadPdf(), { immediate: true })
|
||||
watch(() => props.selectedItem?.id, () => void locateSelectedItem())
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
++loadSequence
|
||||
++renderSequence
|
||||
window.cancelAnimationFrame(resizeFrame)
|
||||
resizeObserver.disconnect()
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
void loadingTask?.destroy()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="pdf-source-viewer" aria-label="PDF 原文件预览">
|
||||
<div v-if="sourceUrl" class="pdf-toolbar">
|
||||
<div class="toolbar-group">
|
||||
<button type="button" aria-label="上一页" :disabled="currentPage <= 1 || loading" @click="changePage(-1)">
|
||||
<i class="fa fa-chevron-left" aria-hidden="true" />
|
||||
</button>
|
||||
<span class="page-indicator">{{ currentPage }} / {{ pageCount || '—' }}</span>
|
||||
<button type="button" aria-label="下一页" :disabled="currentPage >= pageCount || loading" @click="changePage(1)">
|
||||
<i class="fa fa-chevron-right" aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
<span class="location-state" :class="`is-${highlightState}`" role="status">
|
||||
<i v-if="highlightState === 'highlighted'" class="fa fa-map-marker" aria-hidden="true" />
|
||||
{{ locationText }}
|
||||
</span>
|
||||
<div class="toolbar-group">
|
||||
<button type="button" aria-label="缩小" :disabled="zoom <= 0.6 || loading" @click="changeZoom(-0.1)">−</button>
|
||||
<span class="zoom-indicator">{{ Math.round(zoom * 100) }}%</span>
|
||||
<button type="button" aria-label="放大" :disabled="zoom >= 2 || loading" @click="changeZoom(0.1)">+</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="sourceUrl" ref="scrollRef" class="pdf-scroll" tabindex="0" :aria-label="`PDF 预览:${fileName}`">
|
||||
<div
|
||||
ref="pageRef"
|
||||
class="pdf-page"
|
||||
:style="pageStyle"
|
||||
:data-page-number="currentPage"
|
||||
>
|
||||
<canvas ref="canvasRef" class="pdf-canvas" />
|
||||
<div ref="textLayerRef" class="pdf-text-layer" />
|
||||
</div>
|
||||
<div v-if="loading || rendering" class="pdf-loading" role="status">
|
||||
<i class="fa fa-spinner fa-spin" aria-hidden="true" />
|
||||
{{ loading ? '正在加载 PDF…' : '正在渲染页面…' }}
|
||||
</div>
|
||||
<div v-if="errorMessage" class="pdf-error" role="alert">
|
||||
<i class="fa fa-exclamation-circle" aria-hidden="true" />
|
||||
<strong>PDF 预览失败</strong>
|
||||
<span>{{ errorMessage }}</span>
|
||||
<a :href="sourceUrl" target="_blank" rel="noopener noreferrer">在新窗口打开原件</a>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="pdf-unavailable" role="status">
|
||||
<i class="fa fa-file-pdf-o" aria-hidden="true" />
|
||||
<strong>PDF 原文件暂不可预览</strong>
|
||||
<span>请重新上传该文件后再试。</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.pdf-source-viewer {
|
||||
display: flex;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
background: #525659;
|
||||
}
|
||||
|
||||
.pdf-toolbar {
|
||||
display: grid;
|
||||
min-height: 44px;
|
||||
flex: 0 0 auto;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 0 12px;
|
||||
color: #f2f4f7;
|
||||
background: #323639;
|
||||
border-bottom: 1px solid #1f2427;
|
||||
}
|
||||
|
||||
.toolbar-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
|
||||
button {
|
||||
display: inline-flex;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #f2f4f7;
|
||||
font: inherit;
|
||||
font-size: 16px;
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
|
||||
&:hover:not(:disabled) {
|
||||
background: rgb(255 255 255 / 12%);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
color: #7d8387;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.page-indicator,
|
||||
.zoom-indicator {
|
||||
min-width: 54px;
|
||||
color: #e4e7ec;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.location-state {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
color: #d0d5dd;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
|
||||
&.is-highlighted {
|
||||
color: #ffd666;
|
||||
}
|
||||
|
||||
&.is-unmatched {
|
||||
color: #fdb022;
|
||||
}
|
||||
}
|
||||
|
||||
.pdf-scroll {
|
||||
position: relative;
|
||||
display: flex;
|
||||
min-height: 538px;
|
||||
flex: 1;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
overflow: auto;
|
||||
padding: 18px;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.pdf-page {
|
||||
position: relative;
|
||||
flex: 0 0 auto;
|
||||
overflow: hidden;
|
||||
background: #fff;
|
||||
box-shadow: 0 2px 12px rgb(0 0 0 / 34%);
|
||||
}
|
||||
|
||||
.pdf-canvas {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.pdf-text-layer {
|
||||
--min-font-size: 1;
|
||||
--text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
|
||||
--min-font-size-inv: calc(1 / var(--min-font-size));
|
||||
|
||||
position: absolute;
|
||||
z-index: 1;
|
||||
inset: 0;
|
||||
overflow: clip;
|
||||
color-scheme: only light;
|
||||
line-height: 1;
|
||||
letter-spacing: normal;
|
||||
word-spacing: normal;
|
||||
text-align: initial;
|
||||
text-size-adjust: none;
|
||||
forced-color-adjust: none;
|
||||
transform-origin: 0 0;
|
||||
caret-color: CanvasText;
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(span),
|
||||
.pdf-text-layer :deep(br) {
|
||||
position: absolute;
|
||||
color: transparent;
|
||||
white-space: pre;
|
||||
cursor: text;
|
||||
user-select: text;
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
.pdf-text-layer > :deep(:not(.markedContent)),
|
||||
.pdf-text-layer :deep(.markedContent span:not(.markedContent)) {
|
||||
--font-height: 0;
|
||||
--scale-x: 1;
|
||||
--rotate: 0deg;
|
||||
|
||||
z-index: 1;
|
||||
font-size: calc(var(--text-scale-factor) * var(--font-height));
|
||||
transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(.markedContent) {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(.is-slice-highlighted) {
|
||||
margin: -2px;
|
||||
padding: 2px;
|
||||
background: rgb(255 202 40 / 48%);
|
||||
border-radius: 3px;
|
||||
box-shadow: 0 0 0 1px rgb(245 158 11 / 38%);
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(::selection) {
|
||||
color: transparent;
|
||||
background: rgb(37 99 235 / 30%);
|
||||
}
|
||||
|
||||
.pdf-loading,
|
||||
.pdf-error {
|
||||
position: absolute;
|
||||
z-index: 3;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transform: translate(-50%, -50%);
|
||||
}
|
||||
|
||||
.pdf-loading {
|
||||
gap: 8px;
|
||||
padding: 10px 14px;
|
||||
color: #f2f4f7;
|
||||
font-size: 13px;
|
||||
background: rgb(31 36 39 / 86%);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.pdf-error {
|
||||
width: min(360px, calc(100% - 32px));
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 20px;
|
||||
color: #667085;
|
||||
text-align: center;
|
||||
background: #fff;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 20px rgb(0 0 0 / 22%);
|
||||
|
||||
i {
|
||||
color: #d92d20;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
}
|
||||
|
||||
span,
|
||||
a {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.pdf-unavailable {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
color: #667085;
|
||||
background: #f8f9fb;
|
||||
|
||||
i {
|
||||
color: #d92d20;
|
||||
font-size: 36px;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
span {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.pdf-toolbar {
|
||||
gap: 5px;
|
||||
padding: 0 6px;
|
||||
}
|
||||
|
||||
.location-state {
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.pdf-scroll {
|
||||
min-height: 420px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -1,5 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||
import { sourceLines } from './previewModel'
|
||||
import type { PreviewItem, ProcessType } from './types'
|
||||
|
||||
@@ -9,6 +10,8 @@ const props = defineProps<{
|
||||
selectedId: string | null
|
||||
processType: ProcessType
|
||||
fileName: string
|
||||
taskId: string | number | null
|
||||
sourceFileId: string | number | null
|
||||
files: { id: string; name: string; count: number; modifiedCount: number }[]
|
||||
selectedFileId: string | null
|
||||
}>()
|
||||
@@ -31,6 +34,7 @@ const editorDraft = ref('')
|
||||
const lines = computed(() => sourceLines(props.sourceText))
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||
const isPdfSource = computed(() => /\.pdf$/i.test(props.fileName))
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !search.value.trim()
|
||||
@@ -102,7 +106,7 @@ watch(selectedItem, async (item) => {
|
||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||
}
|
||||
|
||||
if (item.sourceStart == null) return
|
||||
if (isPdfSource.value || item.sourceStart == null) return
|
||||
await nextTick()
|
||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||
@@ -160,7 +164,14 @@ function lineRange(item: PreviewItem) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
||||
<PdfSourceViewer
|
||||
v-if="isPdfSource"
|
||||
:task-id="taskId"
|
||||
:source-file-id="sourceFileId"
|
||||
:file-name="fileName"
|
||||
:selected-item="selectedItem ?? null"
|
||||
/>
|
||||
<div v-else ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
||||
<div
|
||||
v-for="line in lines"
|
||||
:key="line.number"
|
||||
@@ -198,6 +209,8 @@ function lineRange(item: PreviewItem) {
|
||||
:key="item.id"
|
||||
class="preview-item"
|
||||
:class="{ 'is-active': item.id === selectedItem?.id }"
|
||||
:data-preview-id="item.id"
|
||||
:aria-selected="item.id === selectedItem?.id"
|
||||
role="button"
|
||||
tabindex="0"
|
||||
@click="selectItem(item.id)"
|
||||
|
||||
@@ -9,6 +9,8 @@ const props = defineProps<{
|
||||
externalSource: ExternalDataSource
|
||||
externalPulling: boolean
|
||||
externalConnected: boolean
|
||||
previewBuilding: boolean
|
||||
sourceUploading: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -31,14 +33,23 @@ const AUTH_MODES = [
|
||||
|
||||
const FILE_PAGE_SIZE = 10
|
||||
const currentFilePage = ref(1)
|
||||
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
|
||||
|
||||
const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
|
||||
queued: { label: '等待上传', icon: 'fa-clock-o' },
|
||||
uploading: { label: '正在上传', icon: 'fa-cloud-upload' },
|
||||
waiting: { label: '等待切分', icon: 'fa-clock-o' },
|
||||
processing: { label: '正在切分', icon: 'fa-spinner fa-spin' },
|
||||
success: { label: '切分完成', icon: 'fa-check-circle' },
|
||||
'upload-failed': { label: '上传失败', icon: 'fa-exclamation-circle' },
|
||||
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
|
||||
}
|
||||
|
||||
const isExternal = computed(() => props.processType === 'external')
|
||||
|
||||
// 后端首版严格支持这些可验证的文本格式;不要把无法解析的二进制文档
|
||||
// 静默替换成示例正文。
|
||||
const uploadAccept = computed(() => props.processType === 'unstructured'
|
||||
? '.txt,.md,.json,.jsonl'
|
||||
: '.json,.jsonl,.csv,.txt,.md')
|
||||
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
|
||||
: '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx')
|
||||
|
||||
const pagedUploadedFiles = computed(() => {
|
||||
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
|
||||
@@ -67,6 +78,48 @@ function formatSize(size: number) {
|
||||
if (size >= 1024 * 1024) return `${(size / 1024 / 1024).toFixed(1)} MB`
|
||||
return `${(size / 1024).toFixed(1)} KB`
|
||||
}
|
||||
|
||||
function getFileStage(file: UploadedDataFile): FileStage {
|
||||
if (file.status === 'queued' || file.status === 'uploading') return file.status
|
||||
if (file.status === 'failed') return 'upload-failed'
|
||||
if (file.previewStatus === 'processing' || file.previewStatus === 'success') return file.previewStatus
|
||||
if (file.previewStatus === 'failed') return 'preview-failed'
|
||||
return 'waiting'
|
||||
}
|
||||
|
||||
function getFileProgress(file: UploadedDataFile) {
|
||||
const stage = getFileStage(file)
|
||||
const progress = stage === 'queued' || stage === 'uploading' || stage === 'upload-failed'
|
||||
? file.uploadProgress
|
||||
: stage === 'waiting' ? 100 : file.previewProgress ?? 0
|
||||
return Math.min(100, Math.max(0, progress))
|
||||
}
|
||||
|
||||
function isFileProcessing(file: UploadedDataFile) {
|
||||
return getFileStage(file) === 'processing'
|
||||
}
|
||||
|
||||
function getFileBarPercentage(file: UploadedDataFile) {
|
||||
// Element Plus 的不定进度动画需要非零宽度;这里不作为完成百分比展示。
|
||||
return isFileProcessing(file) ? 100 : getFileProgress(file)
|
||||
}
|
||||
|
||||
function getFileProgressStatus(file: UploadedDataFile): 'success' | 'exception' | undefined {
|
||||
const stage = getFileStage(file)
|
||||
if (stage === 'waiting' || stage === 'success') return 'success'
|
||||
if (stage === 'upload-failed' || stage === 'preview-failed') return 'exception'
|
||||
return undefined
|
||||
}
|
||||
|
||||
function getFileProgressText(file: UploadedDataFile) {
|
||||
if (isFileProcessing(file)) return '处理中'
|
||||
if (getFileStage(file) === 'waiting') return '已上传'
|
||||
return `${getFileProgress(file)}%`
|
||||
}
|
||||
|
||||
function getFileError(file: UploadedDataFile) {
|
||||
return file.uploadError || file.previewError
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -167,7 +220,7 @@ function formatSize(size: number) {
|
||||
<div class="external-actions">
|
||||
<el-button
|
||||
:loading="externalPulling && !externalConnected"
|
||||
:disabled="externalPulling"
|
||||
:disabled="externalPulling || previewBuilding"
|
||||
plain
|
||||
@click="emit('test-connection')"
|
||||
>
|
||||
@@ -176,7 +229,7 @@ function formatSize(size: number) {
|
||||
<el-button
|
||||
type="primary"
|
||||
:loading="externalPulling"
|
||||
:disabled="externalPulling"
|
||||
:disabled="externalPulling || previewBuilding"
|
||||
@click="emit('pull-data')"
|
||||
>
|
||||
拉取数据
|
||||
@@ -200,10 +253,34 @@ function formatSize(size: number) {
|
||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||
</span>
|
||||
</div>
|
||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 拉取成功</span>
|
||||
<div
|
||||
class="file-preview-progress"
|
||||
:class="`is-${getFileStage(file)}`"
|
||||
:title="getFileError(file)"
|
||||
>
|
||||
<div class="file-status" role="status" aria-live="polite">
|
||||
<span>
|
||||
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||
</span>
|
||||
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="getFileBarPercentage(file)"
|
||||
:indeterminate="isFileProcessing(file)"
|
||||
:duration="1.5"
|
||||
:stroke-width="5"
|
||||
:show-text="false"
|
||||
:status="getFileProgressStatus(file)"
|
||||
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||
:aria-valuetext="isFileProcessing(file) ? '正在切分,进度未知' : getFileProgressText(file)"
|
||||
/>
|
||||
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||
</div>
|
||||
<el-button
|
||||
link
|
||||
type="danger"
|
||||
:disabled="previewBuilding || file.status === 'uploading'"
|
||||
:aria-label="`删除数据集 ${file.name}`"
|
||||
@click="emit('remove-file', file.uid)"
|
||||
>
|
||||
@@ -233,7 +310,13 @@ function formatSize(size: number) {
|
||||
<h3 id="source-upload-title">源数据上传</h3>
|
||||
<p>上传后可在下一步检查内容和切分效果,支持同时添加多个文件</p>
|
||||
</div>
|
||||
<el-button v-if="uploadedFiles.length === 0" link type="primary" @click="emit('use-sample')">
|
||||
<el-button
|
||||
v-if="uploadedFiles.length === 0"
|
||||
link
|
||||
type="primary"
|
||||
:disabled="previewBuilding"
|
||||
@click="emit('use-sample')"
|
||||
>
|
||||
使用示例数据
|
||||
</el-button>
|
||||
</div>
|
||||
@@ -243,6 +326,7 @@ function formatSize(size: number) {
|
||||
drag
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
@@ -253,25 +337,29 @@ function formatSize(size: number) {
|
||||
<template #tip>
|
||||
<div class="el-upload__tip">
|
||||
{{ processType === 'unstructured'
|
||||
? '支持 TXT、Markdown、PDF、Word、JSON、JSONL,单文件不超过 200MB'
|
||||
: '支持 JSON、JSONL、CSV、Excel,单文件不超过 200MB' }}
|
||||
? '支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON;旧版 DOC/PPT 请先转换,单文件不超过 200MB'
|
||||
: '支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX;旧版 XLS 请先转换,单文件不超过 200MB' }}
|
||||
</div>
|
||||
</template>
|
||||
</el-upload>
|
||||
|
||||
<section v-else class="uploaded-file-list" aria-label="已上传文件列表">
|
||||
<div class="uploaded-file-list-header">
|
||||
<span>已添加 {{ uploadedFiles.length }} 个文件</span>
|
||||
<span>
|
||||
已选择 {{ uploadedFiles.length }} 个文件
|
||||
<small v-if="sourceUploading" class="upload-queue-status"> · 正在逐个上传</small>
|
||||
</span>
|
||||
<div class="continue-upload">
|
||||
<el-upload
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
aria-label="继续添加源数据文件"
|
||||
>
|
||||
<el-button size="small" type="primary">继续上传</el-button>
|
||||
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
|
||||
</el-upload>
|
||||
</div>
|
||||
</div>
|
||||
@@ -285,10 +373,34 @@ function formatSize(size: number) {
|
||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||
</span>
|
||||
</div>
|
||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 校验通过</span>
|
||||
<div
|
||||
class="file-preview-progress"
|
||||
:class="`is-${getFileStage(file)}`"
|
||||
:title="getFileError(file)"
|
||||
>
|
||||
<div class="file-status" role="status" aria-live="polite">
|
||||
<span>
|
||||
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||
</span>
|
||||
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="getFileBarPercentage(file)"
|
||||
:indeterminate="isFileProcessing(file)"
|
||||
:duration="1.5"
|
||||
:stroke-width="5"
|
||||
:show-text="false"
|
||||
:status="getFileProgressStatus(file)"
|
||||
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||
:aria-valuetext="isFileProcessing(file) ? '正在切分,进度未知' : getFileProgressText(file)"
|
||||
/>
|
||||
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||
</div>
|
||||
<el-button
|
||||
link
|
||||
type="danger"
|
||||
:disabled="previewBuilding || file.status === 'uploading'"
|
||||
:aria-label="`删除文件 ${file.name}`"
|
||||
@click="emit('remove-file', file.uid)"
|
||||
>
|
||||
@@ -441,6 +553,11 @@ function formatSize(size: number) {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.upload-queue-status {
|
||||
color: #5b50f2;
|
||||
font-size: inherit;
|
||||
}
|
||||
|
||||
.uploaded-file-pagination {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
@@ -495,9 +612,59 @@ function formatSize(size: number) {
|
||||
}
|
||||
}
|
||||
|
||||
.file-preview-progress {
|
||||
display: flex;
|
||||
flex: 0 1 220px;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
min-width: 150px;
|
||||
|
||||
&.is-queued .file-status,
|
||||
&.is-waiting .file-status {
|
||||
color: #8a93a3;
|
||||
}
|
||||
|
||||
&.is-uploading .file-status,
|
||||
&.is-processing .file-status {
|
||||
color: #5b50f2;
|
||||
}
|
||||
|
||||
&.is-success .file-status {
|
||||
color: #2ca66a;
|
||||
}
|
||||
|
||||
&.is-upload-failed .file-status,
|
||||
&.is-preview-failed .file-status,
|
||||
.file-preview-error {
|
||||
color: #d94b4b;
|
||||
}
|
||||
}
|
||||
|
||||
.file-status {
|
||||
color: #2ca66a;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
font-size: 12px;
|
||||
|
||||
> span:first-child {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.file-progress-value {
|
||||
flex: 0 0 auto;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.file-preview-error {
|
||||
overflow: hidden;
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.uploaded-file :deep(.el-button) {
|
||||
@@ -514,8 +681,12 @@ function formatSize(size: number) {
|
||||
padding: 8px 10px;
|
||||
}
|
||||
|
||||
.file-status {
|
||||
.file-preview-progress {
|
||||
flex: 0 1 auto;
|
||||
min-width: 130px;
|
||||
}
|
||||
|
||||
.file-status {
|
||||
line-height: 1.4;
|
||||
white-space: normal;
|
||||
}
|
||||
@@ -545,7 +716,8 @@ function formatSize(size: number) {
|
||||
min-width: calc(100% - 40px);
|
||||
}
|
||||
|
||||
.file-status {
|
||||
.file-preview-progress {
|
||||
flex: 1 0 calc(100% - 38px);
|
||||
margin-left: 38px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,12 +22,12 @@ const PREPROCESS_OPTIONS: Array<{
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
||||
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一编码、空白、字段名和 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别多级表头与合并单元格,并将嵌套字段展平' },
|
||||
{ value: 'deduplicate', label: '重复数据去重', description: '基于整行精确匹配和关键字段组合删除重复记录' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||
]
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
@@ -43,9 +43,11 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = value.filter(
|
||||
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
|
||||
)
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
(option): option is PreprocessOption => (
|
||||
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||
),
|
||||
)))
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -30,9 +30,8 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
|
||||
]
|
||||
|
||||
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||||
{ value: 'semantic', label: '自动语义切分' },
|
||||
{ value: 'heading', label: '按标题和段落' },
|
||||
{ value: 'fixed', label: '按固定长度' },
|
||||
{ value: 'structure', label: '文档结构' },
|
||||
{ value: 'fixed', label: '固定 Token' },
|
||||
{ value: 'custom', label: '自定义分隔符' },
|
||||
]
|
||||
|
||||
@@ -70,9 +69,9 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
|
||||
function updateSmartPreprocess(value: string | number | boolean) {
|
||||
const enabled = Boolean(value)
|
||||
const remainingOptions = props.options.preprocessOptions.filter(
|
||||
const remainingOptions = Array.from(new Set(props.options.preprocessOptions.filter(
|
||||
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
||||
)
|
||||
)))
|
||||
updateField(
|
||||
'preprocessOptions',
|
||||
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
||||
@@ -80,9 +79,11 @@ function updateSmartPreprocess(value: string | number | boolean) {
|
||||
}
|
||||
|
||||
function updateDesensitize(value: string | number | boolean) {
|
||||
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
|
||||
(option) => option !== 'desensitize',
|
||||
)
|
||||
const preprocessOptions: UnstructuredPreprocessOption[] = Array.from(new Set(
|
||||
props.options.preprocessOptions.filter(
|
||||
(option) => option !== 'desensitize',
|
||||
),
|
||||
))
|
||||
if (Boolean(value)) preprocessOptions.push('desensitize')
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
@@ -116,7 +117,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认使用推荐策略,只需决定是否需要脱敏</p>
|
||||
<p>默认启用结构感知的推荐策略,只需决定是否需要脱敏</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="preprocess-option-grid">
|
||||
@@ -127,7 +128,7 @@ defineExpose({ revealValidation })
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>智能预处理</strong>
|
||||
<small>自动清理、解析、去重及保留上下文</small>
|
||||
<small>清理无效内容、感知文档结构、合并短块、预过滤低质量内容、近重复去重,并通过重叠保护上下文</small>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
@@ -138,7 +139,7 @@ defineExpose({ revealValidation })
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>敏感信息脱敏</strong>
|
||||
<small>处理姓名、手机号等隐私信息</small>
|
||||
<small>识别并脱敏姓名、手机号、邮箱和身份证号</small>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
@@ -159,7 +160,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>切分选项</h3>
|
||||
<p>以语义完整为优先,将长文档拆成可独立生成问答的内容块</p>
|
||||
<p>按文档结构、固定 Token 或自定义分隔符拆分长文档</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -178,7 +179,7 @@ defineExpose({ revealValidation })
|
||||
:value="method.value"
|
||||
/>
|
||||
</el-select>
|
||||
<small>推荐使用自动语义切分,在长度限制内优先保留完整句段</small>
|
||||
<small>推荐使用文档结构,按标题、段落和句子边界保留完整内容块</small>
|
||||
</label>
|
||||
|
||||
<label class="config-field">
|
||||
|
||||
@@ -30,7 +30,7 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
chunkMethod: 'semantic',
|
||||
chunkMethod: 'structure',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { DataProcessPreviewFileStatus } from '@/types/dataProcess'
|
||||
|
||||
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||
|
||||
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
@@ -44,7 +46,7 @@ export type UnstructuredPreprocessOption =
|
||||
| 'desensitize'
|
||||
| 'preserve_context'
|
||||
|
||||
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
|
||||
export type ChunkMethod = 'structure' | 'fixed' | 'custom'
|
||||
|
||||
export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
preprocessOptions: UnstructuredPreprocessOption[]
|
||||
@@ -62,7 +64,7 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
}
|
||||
|
||||
export interface ExternalDataSource {
|
||||
type: string
|
||||
type: 'postgresql'
|
||||
url: string
|
||||
authMode: 'none' | 'basic'
|
||||
username?: string
|
||||
@@ -75,14 +77,21 @@ export interface ExternalDataSource {
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
rawFile?: File
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
content: string
|
||||
fileFormat?: string
|
||||
checksumSha256?: string
|
||||
status?: 'uploading' | 'ready' | 'failed'
|
||||
error?: string
|
||||
status: 'queued' | 'uploading' | 'ready' | 'failed'
|
||||
uploadProgress: number
|
||||
uploadError?: string
|
||||
previewStatus?: DataProcessPreviewFileStatus
|
||||
previewProgress?: number
|
||||
previewError?: string
|
||||
previewCount?: number
|
||||
previewConfigSignature?: string
|
||||
}
|
||||
|
||||
export interface SourceLine {
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import { nextTick, type Reactive, type Ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
ProcessType,
|
||||
StepId,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
|
||||
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 7
|
||||
|
||||
interface DraftSnapshot {
|
||||
schemaVersion?: number
|
||||
taskId?: string
|
||||
currentStepId?: StepId
|
||||
task?: { name?: string; description?: string }
|
||||
processType?: ProcessType
|
||||
structuredOptions?: Partial<StructuredProcessOptions>
|
||||
unstructuredOptions?: Partial<UnstructuredProcessOptions>
|
||||
externalSource?: Partial<ExternalDataSource>
|
||||
}
|
||||
|
||||
interface DraftBindings {
|
||||
taskId: Ref<string | null>
|
||||
currentStepId: Readonly<Ref<StepId>>
|
||||
task: Reactive<{ name: string; description: string }>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
externalSource: Reactive<ExternalDataSource>
|
||||
restoringDraft: Ref<boolean>
|
||||
dirty: Ref<boolean>
|
||||
goToStep: (stepId: StepId) => void
|
||||
}
|
||||
|
||||
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
|
||||
return {
|
||||
type: typeof source.type === 'string' ? source.type : 'postgresql',
|
||||
url: typeof source.url === 'string' ? source.url : '',
|
||||
authMode: source.authMode === 'basic' ? 'basic' as const : 'none' as const,
|
||||
username: typeof source.username === 'string' ? source.username : '',
|
||||
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
|
||||
query: typeof source.query === 'string' ? source.query : '',
|
||||
fileName: typeof source.fileName === 'string' ? source.fileName : 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
function isDraftSnapshot(value: unknown): value is DraftSnapshot {
|
||||
return Boolean(value && typeof value === 'object')
|
||||
}
|
||||
|
||||
/**
|
||||
* 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。
|
||||
*/
|
||||
export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
function draftSnapshot(): DraftSnapshot {
|
||||
return {
|
||||
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
|
||||
taskId: bindings.taskId.value || undefined,
|
||||
currentStepId: bindings.currentStepId.value,
|
||||
task: { ...bindings.task },
|
||||
processType: bindings.processType.value,
|
||||
structuredOptions: {
|
||||
...bindings.structuredOptions.value,
|
||||
preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...bindings.structuredOptions.value.datasetSplit },
|
||||
},
|
||||
unstructuredOptions: {
|
||||
...bindings.unstructuredOptions.value,
|
||||
preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit },
|
||||
},
|
||||
externalSource: sanitizeExternalSource(bindings.externalSource),
|
||||
}
|
||||
}
|
||||
|
||||
function writeDraft(showWarning: boolean) {
|
||||
try {
|
||||
localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
|
||||
} catch {
|
||||
if (showWarning) ElMessage.warning('草稿保存失败,请检查浏览器存储空间')
|
||||
}
|
||||
}
|
||||
|
||||
function persistDraft() {
|
||||
if (!bindings.restoringDraft.value) writeDraft(true)
|
||||
}
|
||||
|
||||
function restoreDraft() {
|
||||
try {
|
||||
const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
if (!raw) return
|
||||
const snapshot: unknown = JSON.parse(raw)
|
||||
if (!isDraftSnapshot(snapshot)) return
|
||||
|
||||
bindings.restoringDraft.value = true
|
||||
bindings.goToStep('create')
|
||||
bindings.taskId.value = typeof snapshot.taskId === 'string' ? snapshot.taskId : null
|
||||
bindings.task.name = snapshot.task?.name || ''
|
||||
bindings.task.description = snapshot.task?.description || ''
|
||||
bindings.processType.value = snapshot.processType === 'unstructured' || snapshot.processType === 'external'
|
||||
? snapshot.processType
|
||||
: 'structured'
|
||||
|
||||
if (snapshot.structuredOptions) {
|
||||
bindings.structuredOptions.value = {
|
||||
...bindings.structuredOptions.value,
|
||||
...snapshot.structuredOptions,
|
||||
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
|
||||
? snapshot.structuredOptions.preprocessOptions
|
||||
: bindings.structuredOptions.value.preprocessOptions,
|
||||
datasetSplit: {
|
||||
...bindings.structuredOptions.value.datasetSplit,
|
||||
...snapshot.structuredOptions.datasetSplit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot.unstructuredOptions) {
|
||||
bindings.unstructuredOptions.value = {
|
||||
...bindings.unstructuredOptions.value,
|
||||
...snapshot.unstructuredOptions,
|
||||
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
|
||||
? snapshot.unstructuredOptions.preprocessOptions
|
||||
: bindings.unstructuredOptions.value.preprocessOptions,
|
||||
datasetSplit: {
|
||||
...bindings.unstructuredOptions.value.datasetSplit,
|
||||
...snapshot.unstructuredOptions.datasetSplit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
|
||||
password: '',
|
||||
})
|
||||
bindings.dirty.value = false
|
||||
|
||||
nextTick(() => {
|
||||
bindings.restoringDraft.value = false
|
||||
// 立即覆盖 v5 及更早草稿,清除其中可能存在的敏感值和大段正文。
|
||||
writeDraft(false)
|
||||
})
|
||||
ElMessage.info('已恢复上次的任务配置')
|
||||
} catch {
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
|
||||
return { draftSnapshot, persistDraft, restoreDraft }
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { buildDataProcessPreview } from '@/api/modules/dataProcess'
|
||||
import type { DataProcessPreviewFileBuildProgress } from '@/types/dataProcess'
|
||||
|
||||
export function useDataProcessPreviewBuild() {
|
||||
async function buildPreviewsByFile(
|
||||
taskId: string | number,
|
||||
sourceFileIds: Array<string | number>,
|
||||
onProgress: (progress: DataProcessPreviewFileBuildProgress) => void,
|
||||
) {
|
||||
const results: DataProcessPreviewFileBuildProgress[] = []
|
||||
for (const sourceFileId of sourceFileIds) {
|
||||
onProgress({ source_file_id: sourceFileId, status: 'processing', progress: 0 })
|
||||
let progress: DataProcessPreviewFileBuildProgress
|
||||
try {
|
||||
const built = await buildDataProcessPreview(taskId, {
|
||||
replace_existing: true,
|
||||
source_file_ids: [sourceFileId],
|
||||
})
|
||||
const sourceId = String(sourceFileId)
|
||||
const fileResult = built.files?.find((item) => String(item.source_file_id) === sourceId)
|
||||
const previewCount = fileResult?.preview_count
|
||||
?? built.file_counts?.[sourceId]
|
||||
?? built.items.filter((item) => String(item.source_file_id) === sourceId).length
|
||||
progress = previewCount > 0
|
||||
? { source_file_id: sourceFileId, status: 'success', progress: 100, preview_count: previewCount }
|
||||
: {
|
||||
source_file_id: sourceFileId,
|
||||
status: 'failed',
|
||||
progress: 0,
|
||||
preview_count: 0,
|
||||
error: '该文件没有生成可用的预览条目,请检查内容或切分配置',
|
||||
}
|
||||
} catch (error) {
|
||||
progress = {
|
||||
source_file_id: sourceFileId,
|
||||
status: 'failed',
|
||||
progress: 0,
|
||||
error: error instanceof Error && error.message ? error.message : '切分失败,请重试',
|
||||
}
|
||||
}
|
||||
results.push(progress)
|
||||
onProgress(progress)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
return { buildPreviewsByFile }
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import { ref, type Ref } from 'vue'
|
||||
import {
|
||||
getDataProcessSourceContent,
|
||||
uploadDataProcessSourceFiles,
|
||||
type DataProcessSourceFile,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
])
|
||||
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
taskId: Ref<string | null>
|
||||
uploadedFiles: Ref<UploadedDataFile[]>
|
||||
onUploaded: (file: UploadedDataFile) => void
|
||||
}
|
||||
|
||||
type SourceFileValidation =
|
||||
| { valid: true; extension: string }
|
||||
| { valid: false; severity: 'error' | 'warning'; message: string }
|
||||
|
||||
export function validateSourceFileSelection(
|
||||
raw: File,
|
||||
processType: ProcessType,
|
||||
selectedFiles: UploadedDataFile[],
|
||||
): SourceFileValidation {
|
||||
if (raw.size > MAX_SOURCE_FILE_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
|
||||
}
|
||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||
if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: '旧版 DOC、XLS、PPT 文件暂不支持,请分别转换为 DOCX、XLSX、PPTX 后上传',
|
||||
}
|
||||
}
|
||||
const supportedExtensions = processType === 'unstructured'
|
||||
? UNSTRUCTURED_FILE_EXTENSIONS
|
||||
: STRUCTURED_FILE_EXTENSIONS
|
||||
if (!supportedExtensions.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: processType === 'unstructured'
|
||||
? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
|
||||
if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
|
||||
}
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
): UploadedDataFile {
|
||||
return {
|
||||
uid: String(file.id),
|
||||
sourceFileId: String(file.id),
|
||||
name: file.name,
|
||||
size: file.size_bytes,
|
||||
count: file.record_count,
|
||||
content,
|
||||
fileFormat: file.file_format,
|
||||
checksumSha256: file.checksum_sha256,
|
||||
status: 'ready',
|
||||
uploadProgress: 100,
|
||||
previewStatus: 'waiting',
|
||||
previewProgress: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function getUploadErrorMessage(error: unknown) {
|
||||
const responseData = (error as {
|
||||
response?: { data?: { detail?: string | { message?: string }; message?: string } }
|
||||
})?.response?.data
|
||||
const detail = responseData?.detail
|
||||
if (typeof detail === 'string') return detail
|
||||
if (detail?.message) return detail.message
|
||||
if (responseData?.message) return responseData.message
|
||||
if (error instanceof Error && error.message) return error.message
|
||||
return '上传失败,请删除该文件后重新选择'
|
||||
}
|
||||
|
||||
export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
const sourceUploading = ref(false)
|
||||
const queue: SourceUploadJob[] = []
|
||||
let worker: Promise<void> | null = null
|
||||
|
||||
async function uploadOne(job: SourceUploadJob) {
|
||||
const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
|
||||
if (!pending) return
|
||||
|
||||
const currentTaskId = options.taskId.value
|
||||
if (!currentTaskId) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = '任务尚未创建,请返回模型选择步骤后重试'
|
||||
return
|
||||
}
|
||||
|
||||
pending.status = 'uploading'
|
||||
pending.uploadProgress = 0
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
const source = uploaded.files[0]
|
||||
if (!source) throw new Error('后端未返回源文件记录')
|
||||
|
||||
// 先登记后端 ID,确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
pending.uploadProgress = 100
|
||||
options.onUploaded(pending)
|
||||
} catch (error) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = getUploadErrorMessage(error)
|
||||
}
|
||||
}
|
||||
|
||||
async function drainQueue() {
|
||||
sourceUploading.value = true
|
||||
try {
|
||||
while (queue.length) {
|
||||
const job = queue.shift()
|
||||
if (job) await uploadOne(job)
|
||||
}
|
||||
} finally {
|
||||
sourceUploading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueSourceUpload(job: SourceUploadJob) {
|
||||
queue.push(job)
|
||||
if (worker) return
|
||||
worker = drainQueue().finally(() => {
|
||||
worker = null
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
enqueueSourceUpload,
|
||||
sourceUploading,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user