feat(data-process): 优化上传切分流程与PDF高亮预览

This commit is contained in:
caoxiaozhu
2026-07-24 11:28:02 +08:00
parent 33d0ed2e01
commit ad64e44860
18 changed files with 1940 additions and 395 deletions

View File

@@ -15,29 +15,27 @@ import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
} from './create/dataProcessCreateState'
import {
DATA_PROCESS_DRAFT_STORAGE_KEY,
useDataProcessDraft,
} from './create/useDataProcessDraft'
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import {
mapDataProcessSourceFile,
useDataProcessSourceUpload,
validateSourceFileSelection,
} from './create/useDataProcessSourceUpload'
import { useModelsStore } from '@/stores/models'
import {
buildDataProcessPreview,
createDataProcessPreview,
createDataProcessTask,
deleteDataProcessPreview,
deleteDataProcessSourceFile,
getDataProcessPreview,
getDataProcessSourceContent,
getDataProcessTask,
pullDataProcessExternalSource,
testDataProcessExternalSource,
updateDataProcessPreview,
updateDataProcessTask,
uploadDataProcessSourceFiles,
type DataProcessExternalSourcePayload,
type DataProcessPreviewItem,
type DataProcessSourceFile,
} from '@/api/modules/dataProcess'
import type { DataProcessConfig } from '@/types/dataProcess'
import type {
@@ -54,15 +52,12 @@ import type {
const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => (
model.type === 'LLM'
&& (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
)))
// 候选范围与模型管理保持一致,不在数据处理页面重复定义模型过滤规则。
const generationModels = computed(() => modelList.value)
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v2'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
@@ -82,7 +77,8 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
))
const uploadedFiles = ref<UploadedDataFile[]>([])
const previewBuilding = ref(false)
const { buildPreviewsByFile } = useDataProcessPreviewBuild()
const externalSource = reactive<ExternalDataSource>({
type: 'postgresql',
url: '',
@@ -95,7 +91,6 @@ const externalSource = reactive<ExternalDataSource>({
})
const externalPulling = ref(false)
const externalConnected = ref(false)
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const previewSignature = ref('')
const previewItems = ref<PreviewItem[]>([])
@@ -103,7 +98,6 @@ const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const dirty = ref(false)
const restoringDraft = ref(false)
let allowLeave = false
const {
@@ -123,6 +117,19 @@ const {
dirty,
beforeGenerate: syncPreviewChanges,
})
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
taskId,
uploadedFiles,
onUploaded(file) {
previewSignature.value = ''
resetDownstream()
dirty.value = true
ElMessage.success(`文件 ${file.name} 上传成功,等待切分`)
},
})
const hasUnfinishedUploads = computed(() => uploadedFiles.value.some((file) => (
file.status !== 'ready' || !file.sourceFileId
)))
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
const previewItemsByFile = computed(() => {
@@ -149,7 +156,10 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
const primaryActionLabel = computed(() => {
if (currentStepId.value === 'create') return '继续:选择大模型'
if (currentStepId.value === 'model') return '继续:上传文件'
if (currentStepId.value === 'upload') return '继续:数据预览'
if (currentStepId.value === 'upload') {
if (sourceUploading.value) return '正在上传'
return previewBuilding.value ? '正在切分' : '继续:数据预览'
}
if (currentStepId.value === 'preview') return '确认预览并继续'
if (currentStepId.value === 'results') return '保存任务'
if (generation.status === 'running') return '正在生成'
@@ -260,33 +270,6 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
}
}
function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
return {
uid: String(file.id),
sourceFileId: String(file.id),
name: file.name,
size: file.size_bytes,
count: file.record_count,
content,
fileFormat: file.file_format,
checksumSha256: file.checksum_sha256,
status: 'ready',
}
}
const { persistDraft, restoreDraft } = useDataProcessDraft({
taskId,
currentStepId,
task,
processType,
structuredOptions,
unstructuredOptions,
externalSource,
restoringDraft,
dirty,
goToStep,
})
function previewAffectingOptions() {
if (processType.value === 'structured') {
return {
@@ -368,92 +351,70 @@ function generationAffectingOptions() {
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
function buildPreviewConfigSignature() {
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}`
}
function buildPreviewSignature() {
const filesSignature = uploadedFiles.value
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
.join('|')
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
return `${buildPreviewConfigSignature()}:${filesSignature}`
}
watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
() => { dirty.value = true },
{ deep: true },
)
watch(processType, (nextType, previousType) => {
if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
if (nextType === previousType || uploadedFiles.value.length === 0) return
resetSourceDataForProcessTypeChange()
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
})
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (restoringDraft.value || currentSignature === previousSignature) return
if (currentSignature === previousSignature) return
resetDownstream()
})
watch(
[taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
persistDraft,
{ deep: true },
)
function scrollToStepTop() {
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
}
watch(currentStep, () => nextTick(scrollToStepTop))
async function handleFileChange(uploadFile: UploadFile) {
function handleFileChange(uploadFile: UploadFile) {
const raw = uploadFile.raw
if (!raw) return
if (!taskId.value) {
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
return
}
if (raw.size > 200 * 1024 * 1024) {
ElMessage.warning('单文件不能超过 200MB')
const validation = validateSourceFileSelection(raw, processType.value, uploadedFiles.value)
if (!validation.valid) {
ElMessage[validation.severity](validation.message)
return
}
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
if (!textExtensions.has(extension)) {
ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV不会用示例内容替代无法解析的文件')
return
}
if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
ElMessage.warning('同名且同大小的文件已经上传')
return
}
let content = ''
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
} catch {
ElMessage.error('文件不是有效的 UTF-8 文本,请转换编码后重试')
return
}
if (!content.trim()) {
ElMessage.warning('不能上传空文件')
return
}
try {
const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
const source = uploaded.files[0]
if (!source) throw new Error('后端未返回源文件记录')
uploadedFiles.value.push(mapSourceFile(source, content))
previewSignature.value = ''
resetDownstream()
dirty.value = true
ElMessage.success(`文件 ${source.name} 上传成功`)
} catch {
// 请求层已展示后端的解析或格式错误。
}
// 必须先同步插入文件行,再交给队列;这样选择完成后页面会立即展示全部文件。
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
uploadedFiles.value.push({
uid: localUid,
rawFile: raw,
name: raw.name,
size: raw.size,
count: 0,
content: '',
fileFormat: validation.extension,
status: 'queued',
uploadProgress: 0,
previewStatus: 'waiting',
previewProgress: 0,
})
dirty.value = true
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
}
async function useSampleFile() {
@@ -462,32 +423,7 @@ async function useSampleFile() {
return
}
const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
}
async function restoreRegisteredSources() {
if (!taskId.value) return
try {
const savedTask = await getDataProcessTask(taskId.value)
const sources = savedTask.source_files || []
const restoredFiles = await Promise.all(sources.map(async (file) => {
try {
const source = await getDataProcessSourceContent(taskId.value!, file.id, {
start_line: 1,
line_count: 5000,
})
return mapSourceFile(file, source.content)
} catch {
return mapSourceFile(file)
}
}))
uploadedFiles.value = restoredFiles
if (restoredFiles.length) {
ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
}
} catch {
ElMessage.warning('草稿任务暂时无法从后端同步,请检查服务后重试')
}
handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
}
function updateExternalSource(value: ExternalDataSource) {
@@ -539,7 +475,7 @@ async function handlePullData() {
start_line: 1,
line_count: 5000,
})
newFiles.push(mapSourceFile(file, source.content))
newFiles.push(mapDataProcessSourceFile(file, source.content))
}
uploadedFiles.value.push(...newFiles)
externalConnected.value = true
@@ -557,11 +493,19 @@ async function handlePullData() {
async function handleRemoveFile(uid: string | number) {
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
if (index < 0 || !taskId.value) return
try {
await deleteDataProcessSourceFile(taskId.value, uid)
} catch {
const file = uploadedFiles.value[index]
if (!file) return
if (file.status === 'uploading') {
ElMessage.warning('当前文件正在上传,请等待上传结束后再删除')
return
}
if (file.sourceFileId) {
try {
await deleteDataProcessSourceFile(taskId.value, file.sourceFileId)
} catch {
return
}
}
uploadedFiles.value.splice(index, 1)
previewSignature.value = ''
previewItems.value = []
@@ -596,7 +540,6 @@ async function nextFromModel() {
: await createDataProcessTask(taskPayload())
taskId.value = String(saved.id)
dirty.value = true
persistDraft()
goToStep('upload')
} catch {
// 请求层已展示名称冲突或配置非法等具体原因。
@@ -604,6 +547,7 @@ async function nextFromModel() {
}
async function nextFromUpload() {
if (previewBuilding.value || sourceUploading.value) return
if (!taskId.value) {
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
return
@@ -612,13 +556,61 @@ async function nextFromUpload() {
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
if (failedUploads.length) {
ElMessage.warning(`${failedUploads.length} 个文件上传失败,请删除后重新选择`)
return
}
if (hasUnfinishedUploads.value) {
ElMessage.warning('请等待所有文件上传完成后再开始切分')
return
}
const signature = buildPreviewSignature()
if (signature !== previewSignature.value) {
const configSignature = buildPreviewConfigSignature()
const allFilesSucceeded = () => uploadedFiles.value.every((file) => (
file.previewStatus === 'success' && file.previewConfigSignature === configSignature
))
if (signature === previewSignature.value && previewItems.value.length && allFilesSucceeded()) {
goToStep('preview')
return
}
for (const file of uploadedFiles.value) {
if (file.previewConfigSignature === configSignature) continue
file.previewStatus = 'waiting'
file.previewProgress = 0
file.previewError = undefined
file.previewCount = undefined
}
previewBuilding.value = true
try {
const pendingFileIds = uploadedFiles.value
.filter((file) => file.previewStatus !== 'success' || file.previewConfigSignature !== configSignature)
.map((file) => file.sourceFileId)
.filter((fileId): fileId is string => Boolean(fileId))
// 构建接口是同步响应:请求中只展示真实的“处理中”,响应成功后才记为 100%。
await buildPreviewsByFile(taskId.value, pendingFileIds, (progress) => {
const file = uploadedFiles.value.find((item) => (
String(item.sourceFileId) === String(progress.source_file_id)
))
if (!file) return
file.previewStatus = progress.status
file.previewProgress = progress.progress
file.previewCount = progress.preview_count
file.previewError = progress.error
if (progress.status === 'success') file.previewConfigSignature = configSignature
})
const failedCount = uploadedFiles.value.filter((file) => file.previewStatus === 'failed').length
if (failedCount || !allFilesSucceeded()) {
ElMessage.warning(`${failedCount || 1} 个文件切分失败;修正问题后点击“继续:数据预览”即可重试`)
return
}
try {
await buildDataProcessPreview(taskId.value, {
source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
})
const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
const items = [...first.items]
const pages = Math.ceil(first.total / first.page_size)
@@ -628,6 +620,7 @@ async function nextFromUpload() {
}
previewItems.value = items.map(mapPreviewItem)
} catch {
ElMessage.warning('文件切分已经完成,但预览加载失败;请再次点击继续重试加载')
return
}
if (!previewItems.value.length) {
@@ -641,8 +634,10 @@ async function nextFromUpload() {
: {}
previewSignature.value = signature
resetDownstream()
goToStep('preview')
} finally {
previewBuilding.value = false
}
goToStep('preview')
}
function selectPreviewFile(fileId: string) {
@@ -770,6 +765,14 @@ async function handlePrimaryAction() {
}
function handleBack() {
if (sourceUploading.value) {
ElMessage.warning('请等待当前文件上传完成')
return
}
if (previewBuilding.value) {
ElMessage.warning('请等待当前文件切分完成')
return
}
if (generation.status === 'running') {
ElMessage.warning('请先停止当前生成任务')
return
@@ -793,7 +796,6 @@ async function saveTask() {
return
}
dirty.value = false
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
allowLeave = true
ElMessage.success('数据处理任务已保存')
await router.push('/data-process')
@@ -818,6 +820,10 @@ async function handleCancel() {
}
onBeforeRouteLeave(async () => {
if (sourceUploading.value) {
ElMessage.warning('请等待当前文件上传完成后再离开页面')
return false
}
if (allowLeave || !dirty.value) return true
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?',
@@ -833,9 +839,9 @@ onBeforeRouteLeave(async () => {
onBeforeUnmount(() => {
stopGenerationTimer()
})
onMounted(async () => {
restoreDraft()
await Promise.all([modelsStore.load(), restoreRegisteredSources()])
onMounted(() => {
localStorage.removeItem('yg-data-process-create-draft')
void modelsStore.load(true)
})
</script>
@@ -899,6 +905,8 @@ onMounted(async () => {
:external-source="externalSource"
:external-pulling="externalPulling"
:external-connected="externalConnected"
:preview-building="previewBuilding"
:source-uploading="sourceUploading"
@update:external-source="updateExternalSource"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@@ -915,6 +923,8 @@ onMounted(async () => {
:items="activePreviewItems"
:process-type="processType"
:file-name="activePreviewFile?.name ?? ''"
:task-id="taskId"
:source-file-id="activePreviewFile?.sourceFileId ?? activePreviewFile?.uid ?? null"
:files="previewFiles"
@update:selected-id="selectPreviewItem"
@update:selected-file-id="selectPreviewFile"
@@ -949,7 +959,7 @@ onMounted(async () => {
<footer class="wizard-footer">
<div class="footer-left">
<el-button v-if="currentStep > 0" @click="handleBack">
<el-button v-if="currentStep > 0" :disabled="previewBuilding || sourceUploading" @click="handleBack">
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回{{ previousStepLabel }}
</el-button>
<el-button v-else @click="handleCancel">取消</el-button>
@@ -960,8 +970,8 @@ onMounted(async () => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="generation.status === 'running'"
:disabled="currentStepId === 'generate' && generation.status === 'running'"
:loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

View File

@@ -84,6 +84,12 @@ const configLabelMap: Record<string, string> = {
preserve_lists: '保留列表',
}
const chunkMethodLabelMap: Record<string, string> = {
structure: '文档结构',
fixed: '固定 Token',
custom: '自定义分隔符',
}
function numeric(value: number | undefined) {
return Number.isFinite(value) ? Number(value) : 0
}
@@ -135,6 +141,9 @@ const configRows = computed(() => Object.entries(detail.value?.config || {})
})))
function formatConfigValue(key: string, value: unknown) {
if (key === 'chunk_method' && typeof value === 'string') {
return chunkMethodLabelMap[value] || value
}
if (key === 'dataset_split' && value && typeof value === 'object') {
const split = value as Partial<DataProcessDatasetSplit>
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`

View File

@@ -41,6 +41,15 @@ function sectionValidationMessage() {
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
return isModelMessage ? '' : props.validationMessage
}
function modelMeta(model: ModelItem) {
const source = model.model_source === 'api' || model.model_source === 'online'
? '在线模型'
: model.model_source === 'local'
? '本地模型'
: model.model_source || '模型管理'
return model.type ? `${source} · ${model.type}` : source
}
</script>
<template>
@@ -66,11 +75,11 @@ function sectionValidationMessage() {
>
<div class="model-option">
<span>{{ model.name }}</span>
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
<small>{{ modelMeta(model) }}</small>
</div>
</el-option>
<template #empty>
<div class="model-empty">暂无可用的大模型请先在模型管理中添加</div>
<div class="model-empty">暂无模型请先在模型管理中添加</div>
</template>
</el-select>
</div>

View File

@@ -0,0 +1,650 @@
<script setup lang="ts">
import {
GlobalWorkerOptions,
TextLayer,
getDocument,
type PDFDocumentLoadingTask,
type PDFDocumentProxy,
type RenderTask,
} from 'pdfjs-dist'
import pdfWorkerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url'
import { computed, nextTick, onBeforeUnmount, ref, shallowRef, watch } from 'vue'
import {
getDataProcessPdfPages,
getDataProcessSourceRawUrl,
type DataProcessPdfPageRange,
} from '@/api/modules/dataProcess'
import type { PreviewItem } from './types'
GlobalWorkerOptions.workerSrc = pdfWorkerUrl
const props = defineProps<{
taskId: string | number | null
sourceFileId: string | number | null
fileName: string
selectedItem: PreviewItem | null
}>()
const scrollRef = ref<HTMLElement | null>(null)
const pageRef = ref<HTMLElement | null>(null)
const canvasRef = ref<HTMLCanvasElement | null>(null)
const textLayerRef = ref<HTMLElement | null>(null)
const documentRef = shallowRef<PDFDocumentProxy | null>(null)
const pageRanges = ref<DataProcessPdfPageRange[]>([])
const currentPage = ref(1)
const pageCount = ref(0)
const zoom = ref(1)
const renderedScale = ref(1)
const loading = ref(false)
const rendering = ref(false)
const errorMessage = ref('')
const highlightState = ref<'idle' | 'highlighted' | 'unmatched' | 'manual'>('idle')
let loadingTask: PDFDocumentLoadingTask | null = null
let renderTask: RenderTask | null = null
let textLayer: TextLayer | null = null
let loadSequence = 0
let renderSequence = 0
let renderedPageNumber = 0
let resizeFrame = 0
const sourceUrl = computed(() => (
props.taskId != null && props.sourceFileId != null
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
: ''
))
const pageStyle = computed(() => ({
'--total-scale-factor': String(renderedScale.value),
}))
const locationText = computed(() => {
if (highlightState.value === 'highlighted') return `切片原文已在第 ${currentPage.value} 页高亮`
if (highlightState.value === 'unmatched') return `已定位第 ${currentPage.value} 页,未匹配到可高亮文字`
if (highlightState.value === 'manual') return '手动新增切片没有原文位置'
return pageCount.value ? `${currentPage.value} / ${pageCount.value}` : '正在读取 PDF'
})
function pageForItem(item: PreviewItem | null) {
if (!item || item.sourceStart == null || item.sourceEnd == null) return null
return pageRanges.value.find((page) => (
item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end
)) ?? pageRanges.value.find((page) => (
item.sourceStart! < page.source_end && item.sourceEnd! > page.source_start
)) ?? null
}
function selectedTextForPage(page: DataProcessPdfPageRange) {
const item = props.selectedItem
if (!item || item.sourceStart == null || item.sourceEnd == null) return ''
const intersectionStart = Math.max(item.sourceStart, page.source_start)
const intersectionEnd = Math.min(item.sourceEnd, page.source_end)
if (intersectionEnd <= intersectionStart) return ''
const relativeStart = Math.max(0, intersectionStart - item.sourceStart)
const relativeEnd = Math.max(relativeStart, intersectionEnd - item.sourceStart)
return item.originalContent.slice(relativeStart, relativeEnd)
}
function normalizeLocatorText(value: string) {
return value.normalize('NFKC').replace(/[^\p{L}\p{N}]+/gu, '').toLocaleLowerCase()
}
function nearestOccurrence(haystack: string, needle: string, expectedIndex: number) {
let nearest = -1
let nearestDistance = Number.POSITIVE_INFINITY
let cursor = haystack.indexOf(needle)
while (cursor >= 0) {
const distance = Math.abs(cursor - expectedIndex)
if (distance < nearestDistance) {
nearest = cursor
nearestDistance = distance
}
cursor = haystack.indexOf(needle, cursor + 1)
}
return nearest
}
function findHighlight(
itemStrings: string[],
selectedText: string,
page: DataProcessPdfPageRange,
) {
const itemRanges: Array<{ start: number; end: number }> = []
let pageText = ''
for (const item of itemStrings) {
const start = pageText.length
pageText += normalizeLocatorText(item)
itemRanges.push({ start, end: pageText.length })
}
const target = normalizeLocatorText(selectedText)
if (!pageText || !target) return []
const itemStart = props.selectedItem?.sourceStart ?? page.source_start
const sourceLength = Math.max(1, page.source_end - page.source_start)
const expectedRatio = Math.min(1, Math.max(0, (itemStart - page.source_start) / sourceLength))
const expectedIndex = Math.round(pageText.length * expectedRatio)
const anchorLengths = [target.length, 120, 80, 48, 24, 12, 8]
.map((length) => Math.min(length, target.length))
.filter((length, index, values) => length >= 4 && values.indexOf(length) === index)
for (const anchorLength of anchorLengths) {
const anchor = target.slice(0, anchorLength)
const matchStart = nearestOccurrence(pageText, anchor, expectedIndex)
if (matchStart < 0) continue
const matchEnd = matchStart + anchor.length
return itemRanges.reduce<number[]>((matches, range, index) => {
if (range.end > matchStart && range.start < matchEnd) matches.push(index)
return matches
}, [])
}
return []
}
function applySelectedHighlight(page: DataProcessPdfPageRange | null) {
const item = props.selectedItem
for (const element of textLayer?.textDivs ?? []) {
element.classList.remove('is-slice-highlighted')
}
if (!item || item.sourceStart == null || item.sourceEnd == null) {
highlightState.value = item ? 'manual' : 'idle'
return
}
if (!page || page.page_number !== currentPage.value || !textLayer) {
highlightState.value = 'idle'
return
}
const matches = findHighlight(
textLayer.textContentItemsStr,
selectedTextForPage(page),
page,
)
for (const index of matches) {
textLayer.textDivs[index]?.classList.add('is-slice-highlighted')
}
highlightState.value = matches.length ? 'highlighted' : 'unmatched'
}
async function renderCurrentPage(force = false) {
const document = documentRef.value
const pageContainer = pageRef.value
const canvas = canvasRef.value
const layerContainer = textLayerRef.value
const scroller = scrollRef.value
if (!document || !pageContainer || !canvas || !layerContainer || !scroller) return
const selectedPage = pageForItem(props.selectedItem)
if (!force && renderedPageNumber === currentPage.value && textLayer) {
applySelectedHighlight(selectedPage)
return
}
const sequence = ++renderSequence
renderTask?.cancel()
textLayer?.cancel()
renderTask = null
textLayer = null
rendering.value = true
errorMessage.value = ''
try {
const page = await document.getPage(currentPage.value)
if (sequence !== renderSequence) return
const baseViewport = page.getViewport({ scale: 1 })
const availableWidth = Math.max(280, scroller.clientWidth - 36)
const scale = (availableWidth / baseViewport.width) * zoom.value
const viewport = page.getViewport({ scale })
const outputScale = Math.max(1, window.devicePixelRatio || 1)
const context = canvas.getContext('2d')
if (!context) throw new Error('浏览器无法创建 PDF 画布')
renderedScale.value = scale
pageContainer.style.width = `${viewport.width}px`
pageContainer.style.height = `${viewport.height}px`
canvas.width = Math.floor(viewport.width * outputScale)
canvas.height = Math.floor(viewport.height * outputScale)
canvas.style.width = `${viewport.width}px`
canvas.style.height = `${viewport.height}px`
layerContainer.replaceChildren()
const activeRenderTask = page.render({
canvas,
viewport,
transform: outputScale === 1 ? undefined : [outputScale, 0, 0, outputScale, 0, 0],
})
renderTask = activeRenderTask
// 立即挂接取消处理,避免 ResizeObserver 触发重绘时产生未处理的取消异常。
const canvasRenderPromise = activeRenderTask.promise.catch((error: unknown) => {
if (
sequence !== renderSequence
|| (error instanceof Error && error.name === 'RenderingCancelledException')
) return
throw error
})
const textContent = await page.getTextContent()
if (sequence !== renderSequence) return
const activeTextLayer = new TextLayer({
textContentSource: textContent,
container: layerContainer,
viewport,
})
textLayer = activeTextLayer
await Promise.all([canvasRenderPromise, activeTextLayer.render()])
if (sequence !== renderSequence) return
renderedPageNumber = currentPage.value
applySelectedHighlight(selectedPage)
await nextTick()
pageContainer.querySelector<HTMLElement>('.is-slice-highlighted')
?.scrollIntoView({ block: 'center', inline: 'center', behavior: 'smooth' })
} catch (error) {
if (sequence !== renderSequence) return
errorMessage.value = error instanceof Error ? error.message : 'PDF 页面渲染失败'
} finally {
if (sequence === renderSequence) rendering.value = false
}
}
async function locateSelectedItem() {
if (!documentRef.value) return
const item = props.selectedItem
if (!item || item.sourceStart == null || item.sourceEnd == null) {
applySelectedHighlight(null)
return
}
const page = pageForItem(item)
if (!page) {
highlightState.value = 'unmatched'
return
}
if (currentPage.value !== page.page_number) {
currentPage.value = page.page_number
await renderCurrentPage(true)
return
}
await renderCurrentPage(false)
}
async function loadPdf() {
const sequence = ++loadSequence
++renderSequence
renderedPageNumber = 0
renderTask?.cancel()
textLayer?.cancel()
await loadingTask?.destroy()
loadingTask = null
documentRef.value = null
pageRanges.value = []
pageCount.value = 0
currentPage.value = 1
errorMessage.value = ''
highlightState.value = 'idle'
if (!sourceUrl.value || props.taskId == null || props.sourceFileId == null) return
loading.value = true
try {
const task = getDocument({ url: sourceUrl.value })
loadingTask = task
const [mapping, document] = await Promise.all([
getDataProcessPdfPages(props.taskId, props.sourceFileId),
task.promise,
])
if (sequence !== loadSequence) {
await task.destroy()
return
}
documentRef.value = document
pageRanges.value = mapping.pages
pageCount.value = document.numPages
await nextTick()
await locateSelectedItem()
if (!props.selectedItem) await renderCurrentPage(true)
} catch (error) {
if (sequence !== loadSequence) return
errorMessage.value = error instanceof Error ? error.message : 'PDF 原文件加载失败'
} finally {
if (sequence === loadSequence) loading.value = false
}
}
async function changePage(delta: number) {
const nextPage = Math.min(pageCount.value, Math.max(1, currentPage.value + delta))
if (nextPage === currentPage.value) return
currentPage.value = nextPage
await renderCurrentPage(true)
}
async function changeZoom(delta: number) {
const nextZoom = Math.min(2, Math.max(0.6, Number((zoom.value + delta).toFixed(1))))
if (nextZoom === zoom.value) return
zoom.value = nextZoom
await renderCurrentPage(true)
}
function scheduleResizeRender() {
window.cancelAnimationFrame(resizeFrame)
resizeFrame = window.requestAnimationFrame(() => {
if (documentRef.value) void renderCurrentPage(true)
})
}
const resizeObserver = new ResizeObserver(scheduleResizeRender)
watch(scrollRef, (element, previous) => {
if (previous) resizeObserver.unobserve(previous)
if (element) resizeObserver.observe(element)
})
watch(sourceUrl, () => void loadPdf(), { immediate: true })
watch(() => props.selectedItem?.id, () => void locateSelectedItem())
onBeforeUnmount(() => {
++loadSequence
++renderSequence
window.cancelAnimationFrame(resizeFrame)
resizeObserver.disconnect()
renderTask?.cancel()
textLayer?.cancel()
void loadingTask?.destroy()
})
</script>
<template>
<div class="pdf-source-viewer" aria-label="PDF 原文件预览">
<div v-if="sourceUrl" class="pdf-toolbar">
<div class="toolbar-group">
<button type="button" aria-label="上一页" :disabled="currentPage <= 1 || loading" @click="changePage(-1)">
<i class="fa fa-chevron-left" aria-hidden="true" />
</button>
<span class="page-indicator">{{ currentPage }} / {{ pageCount || '—' }}</span>
<button type="button" aria-label="下一页" :disabled="currentPage >= pageCount || loading" @click="changePage(1)">
<i class="fa fa-chevron-right" aria-hidden="true" />
</button>
</div>
<span class="location-state" :class="`is-${highlightState}`" role="status">
<i v-if="highlightState === 'highlighted'" class="fa fa-map-marker" aria-hidden="true" />
{{ locationText }}
</span>
<div class="toolbar-group">
<button type="button" aria-label="缩小" :disabled="zoom <= 0.6 || loading" @click="changeZoom(-0.1)"></button>
<span class="zoom-indicator">{{ Math.round(zoom * 100) }}%</span>
<button type="button" aria-label="放大" :disabled="zoom >= 2 || loading" @click="changeZoom(0.1)">+</button>
</div>
</div>
<div v-if="sourceUrl" ref="scrollRef" class="pdf-scroll" tabindex="0" :aria-label="`PDF 预览:${fileName}`">
<div
ref="pageRef"
class="pdf-page"
:style="pageStyle"
:data-page-number="currentPage"
>
<canvas ref="canvasRef" class="pdf-canvas" />
<div ref="textLayerRef" class="pdf-text-layer" />
</div>
<div v-if="loading || rendering" class="pdf-loading" role="status">
<i class="fa fa-spinner fa-spin" aria-hidden="true" />
{{ loading ? '正在加载 PDF…' : '正在渲染页面…' }}
</div>
<div v-if="errorMessage" class="pdf-error" role="alert">
<i class="fa fa-exclamation-circle" aria-hidden="true" />
<strong>PDF 预览失败</strong>
<span>{{ errorMessage }}</span>
<a :href="sourceUrl" target="_blank" rel="noopener noreferrer">在新窗口打开原件</a>
</div>
</div>
<div v-else class="pdf-unavailable" role="status">
<i class="fa fa-file-pdf-o" aria-hidden="true" />
<strong>PDF 原文件暂不可预览</strong>
<span>请重新上传该文件后再试</span>
</div>
</div>
</template>
<style scoped lang="scss">
.pdf-source-viewer {
display: flex;
min-height: 0;
flex: 1;
flex-direction: column;
overflow: hidden;
background: #525659;
}
.pdf-toolbar {
display: grid;
min-height: 44px;
flex: 0 0 auto;
grid-template-columns: auto minmax(0, 1fr) auto;
align-items: center;
gap: 12px;
padding: 0 12px;
color: #f2f4f7;
background: #323639;
border-bottom: 1px solid #1f2427;
}
.toolbar-group {
display: flex;
align-items: center;
gap: 5px;
button {
display: inline-flex;
width: 30px;
height: 30px;
align-items: center;
justify-content: center;
color: #f2f4f7;
font: inherit;
font-size: 16px;
background: transparent;
border: 0;
border-radius: 4px;
cursor: pointer;
&:hover:not(:disabled) {
background: rgb(255 255 255 / 12%);
}
&:disabled {
color: #7d8387;
cursor: not-allowed;
}
}
}
.page-indicator,
.zoom-indicator {
min-width: 54px;
color: #e4e7ec;
font-size: 12px;
text-align: center;
}
.location-state {
min-width: 0;
overflow: hidden;
color: #d0d5dd;
font-size: 12px;
text-align: center;
text-overflow: ellipsis;
white-space: nowrap;
&.is-highlighted {
color: #ffd666;
}
&.is-unmatched {
color: #fdb022;
}
}
.pdf-scroll {
position: relative;
display: flex;
min-height: 538px;
flex: 1;
align-items: flex-start;
justify-content: center;
overflow: auto;
padding: 18px;
outline: none;
}
.pdf-page {
position: relative;
flex: 0 0 auto;
overflow: hidden;
background: #fff;
box-shadow: 0 2px 12px rgb(0 0 0 / 34%);
}
.pdf-canvas {
position: absolute;
inset: 0;
display: block;
}
.pdf-text-layer {
--min-font-size: 1;
--text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
--min-font-size-inv: calc(1 / var(--min-font-size));
position: absolute;
z-index: 1;
inset: 0;
overflow: clip;
color-scheme: only light;
line-height: 1;
letter-spacing: normal;
word-spacing: normal;
text-align: initial;
text-size-adjust: none;
forced-color-adjust: none;
transform-origin: 0 0;
caret-color: CanvasText;
}
.pdf-text-layer :deep(span),
.pdf-text-layer :deep(br) {
position: absolute;
color: transparent;
white-space: pre;
cursor: text;
user-select: text;
transform-origin: 0 0;
}
.pdf-text-layer > :deep(:not(.markedContent)),
.pdf-text-layer :deep(.markedContent span:not(.markedContent)) {
--font-height: 0;
--scale-x: 1;
--rotate: 0deg;
z-index: 1;
font-size: calc(var(--text-scale-factor) * var(--font-height));
transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
}
.pdf-text-layer :deep(.markedContent) {
display: contents;
}
.pdf-text-layer :deep(.is-slice-highlighted) {
margin: -2px;
padding: 2px;
background: rgb(255 202 40 / 48%);
border-radius: 3px;
box-shadow: 0 0 0 1px rgb(245 158 11 / 38%);
}
.pdf-text-layer :deep(::selection) {
color: transparent;
background: rgb(37 99 235 / 30%);
}
.pdf-loading,
.pdf-error {
position: absolute;
z-index: 3;
top: 50%;
left: 50%;
display: flex;
align-items: center;
justify-content: center;
transform: translate(-50%, -50%);
}
.pdf-loading {
gap: 8px;
padding: 10px 14px;
color: #f2f4f7;
font-size: 13px;
background: rgb(31 36 39 / 86%);
border-radius: 6px;
}
.pdf-error {
width: min(360px, calc(100% - 32px));
flex-direction: column;
gap: 8px;
padding: 20px;
color: #667085;
text-align: center;
background: #fff;
border-radius: 8px;
box-shadow: 0 4px 20px rgb(0 0 0 / 22%);
i {
color: #d92d20;
font-size: 28px;
}
strong {
color: #344054;
}
span,
a {
font-size: 12px;
}
}
.pdf-unavailable {
display: flex;
width: 100%;
flex: 1;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 8px;
color: #667085;
background: #f8f9fb;
i {
color: #d92d20;
font-size: 36px;
}
strong {
color: #344054;
font-size: 14px;
}
span {
font-size: 12px;
}
}
@media (max-width: 900px) {
.pdf-toolbar {
gap: 5px;
padding: 0 6px;
}
.location-state {
font-size: 11px;
}
.pdf-scroll {
min-height: 420px;
}
}
</style>

View File

@@ -1,5 +1,6 @@
<script setup lang="ts">
import { computed, nextTick, ref, watch } from 'vue'
import PdfSourceViewer from './PdfSourceViewer.vue'
import { sourceLines } from './previewModel'
import type { PreviewItem, ProcessType } from './types'
@@ -9,6 +10,8 @@ const props = defineProps<{
selectedId: string | null
processType: ProcessType
fileName: string
taskId: string | number | null
sourceFileId: string | number | null
files: { id: string; name: string; count: number; modifiedCount: number }[]
selectedFileId: string | null
}>()
@@ -31,6 +34,7 @@ const editorDraft = ref('')
const lines = computed(() => sourceLines(props.sourceText))
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
const isPdfSource = computed(() => /\.pdf$/i.test(props.fileName))
const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !search.value.trim()
@@ -102,7 +106,7 @@ watch(selectedItem, async (item) => {
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
}
if (item.sourceStart == null) return
if (isPdfSource.value || item.sourceStart == null) return
await nextTick()
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
@@ -160,7 +164,14 @@ function lineRange(item: PreviewItem) {
</div>
</div>
<div ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
<PdfSourceViewer
v-if="isPdfSource"
:task-id="taskId"
:source-file-id="sourceFileId"
:file-name="fileName"
:selected-item="selectedItem ?? null"
/>
<div v-else ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
<div
v-for="line in lines"
:key="line.number"
@@ -198,6 +209,8 @@ function lineRange(item: PreviewItem) {
:key="item.id"
class="preview-item"
:class="{ 'is-active': item.id === selectedItem?.id }"
:data-preview-id="item.id"
:aria-selected="item.id === selectedItem?.id"
role="button"
tabindex="0"
@click="selectItem(item.id)"

View File

@@ -9,6 +9,8 @@ const props = defineProps<{
externalSource: ExternalDataSource
externalPulling: boolean
externalConnected: boolean
previewBuilding: boolean
sourceUploading: boolean
}>()
const emit = defineEmits<{
@@ -31,14 +33,23 @@ const AUTH_MODES = [
const FILE_PAGE_SIZE = 10
const currentFilePage = ref(1)
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
queued: { label: '等待上传', icon: 'fa-clock-o' },
uploading: { label: '正在上传', icon: 'fa-cloud-upload' },
waiting: { label: '等待切分', icon: 'fa-clock-o' },
processing: { label: '正在切分', icon: 'fa-spinner fa-spin' },
success: { label: '切分完成', icon: 'fa-check-circle' },
'upload-failed': { label: '上传失败', icon: 'fa-exclamation-circle' },
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
}
const isExternal = computed(() => props.processType === 'external')
// 后端首版严格支持这些可验证的文本格式;不要把无法解析的二进制文档
// 静默替换成示例正文。
const uploadAccept = computed(() => props.processType === 'unstructured'
? '.txt,.md,.json,.jsonl'
: '.json,.jsonl,.csv,.txt,.md')
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
: '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx')
const pagedUploadedFiles = computed(() => {
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
@@ -67,6 +78,48 @@ function formatSize(size: number) {
if (size >= 1024 * 1024) return `${(size / 1024 / 1024).toFixed(1)} MB`
return `${(size / 1024).toFixed(1)} KB`
}
function getFileStage(file: UploadedDataFile): FileStage {
if (file.status === 'queued' || file.status === 'uploading') return file.status
if (file.status === 'failed') return 'upload-failed'
if (file.previewStatus === 'processing' || file.previewStatus === 'success') return file.previewStatus
if (file.previewStatus === 'failed') return 'preview-failed'
return 'waiting'
}
function getFileProgress(file: UploadedDataFile) {
const stage = getFileStage(file)
const progress = stage === 'queued' || stage === 'uploading' || stage === 'upload-failed'
? file.uploadProgress
: stage === 'waiting' ? 100 : file.previewProgress ?? 0
return Math.min(100, Math.max(0, progress))
}
function isFileProcessing(file: UploadedDataFile) {
return getFileStage(file) === 'processing'
}
function getFileBarPercentage(file: UploadedDataFile) {
// Element Plus 的不定进度动画需要非零宽度;这里不作为完成百分比展示。
return isFileProcessing(file) ? 100 : getFileProgress(file)
}
function getFileProgressStatus(file: UploadedDataFile): 'success' | 'exception' | undefined {
const stage = getFileStage(file)
if (stage === 'waiting' || stage === 'success') return 'success'
if (stage === 'upload-failed' || stage === 'preview-failed') return 'exception'
return undefined
}
function getFileProgressText(file: UploadedDataFile) {
if (isFileProcessing(file)) return '处理中'
if (getFileStage(file) === 'waiting') return '已上传'
return `${getFileProgress(file)}%`
}
function getFileError(file: UploadedDataFile) {
return file.uploadError || file.previewError
}
</script>
<template>
@@ -167,7 +220,7 @@ function formatSize(size: number) {
<div class="external-actions">
<el-button
:loading="externalPulling && !externalConnected"
:disabled="externalPulling"
:disabled="externalPulling || previewBuilding"
plain
@click="emit('test-connection')"
>
@@ -176,7 +229,7 @@ function formatSize(size: number) {
<el-button
type="primary"
:loading="externalPulling"
:disabled="externalPulling"
:disabled="externalPulling || previewBuilding"
@click="emit('pull-data')"
>
拉取数据
@@ -200,10 +253,34 @@ function formatSize(size: number) {
<template v-if="file.count"> · {{ file.count.toLocaleString() }} </template>
</span>
</div>
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 拉取成功</span>
<div
class="file-preview-progress"
:class="`is-${getFileStage(file)}`"
:title="getFileError(file)"
>
<div class="file-status" role="status" aria-live="polite">
<span>
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
{{ FILE_STAGE_META[getFileStage(file)].label }}
</span>
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
</div>
<el-progress
:percentage="getFileBarPercentage(file)"
:indeterminate="isFileProcessing(file)"
:duration="1.5"
:stroke-width="5"
:show-text="false"
:status="getFileProgressStatus(file)"
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
:aria-valuetext="isFileProcessing(file) ? '正在切分进度未知' : getFileProgressText(file)"
/>
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
</div>
<el-button
link
type="danger"
:disabled="previewBuilding || file.status === 'uploading'"
:aria-label="`删除数据集 ${file.name}`"
@click="emit('remove-file', file.uid)"
>
@@ -233,7 +310,13 @@ function formatSize(size: number) {
<h3 id="source-upload-title">源数据上传</h3>
<p>上传后可在下一步检查内容和切分效果支持同时添加多个文件</p>
</div>
<el-button v-if="uploadedFiles.length === 0" link type="primary" @click="emit('use-sample')">
<el-button
v-if="uploadedFiles.length === 0"
link
type="primary"
:disabled="previewBuilding"
@click="emit('use-sample')"
>
使用示例数据
</el-button>
</div>
@@ -243,6 +326,7 @@ function formatSize(size: number) {
drag
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
@@ -253,25 +337,29 @@ function formatSize(size: number) {
<template #tip>
<div class="el-upload__tip">
{{ processType === 'unstructured'
? '支持 TXT、Markdown、PDF、Word、JSON、JSONL,单文件不超过 200MB'
: '支持 JSON、JSONL、CSV、Excel,单文件不超过 200MB' }}
? '支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON旧版 DOC/PPT 请先转换,单文件不超过 200MB'
: '支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX旧版 XLS 请先转换,单文件不超过 200MB' }}
</div>
</template>
</el-upload>
<section v-else class="uploaded-file-list" aria-label="已上传文件列表">
<div class="uploaded-file-list-header">
<span>已添加 {{ uploadedFiles.length }} 个文件</span>
<span>
已选择 {{ uploadedFiles.length }} 个文件
<small v-if="sourceUploading" class="upload-queue-status"> · 正在逐个上传</small>
</span>
<div class="continue-upload">
<el-upload
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
aria-label="继续添加源数据文件"
>
<el-button size="small" type="primary">继续上传</el-button>
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
</el-upload>
</div>
</div>
@@ -285,10 +373,34 @@ function formatSize(size: number) {
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
</span>
</div>
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 校验通过</span>
<div
class="file-preview-progress"
:class="`is-${getFileStage(file)}`"
:title="getFileError(file)"
>
<div class="file-status" role="status" aria-live="polite">
<span>
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
{{ FILE_STAGE_META[getFileStage(file)].label }}
</span>
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
</div>
<el-progress
:percentage="getFileBarPercentage(file)"
:indeterminate="isFileProcessing(file)"
:duration="1.5"
:stroke-width="5"
:show-text="false"
:status="getFileProgressStatus(file)"
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
:aria-valuetext="isFileProcessing(file) ? '正在切分,进度未知' : getFileProgressText(file)"
/>
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
</div>
<el-button
link
type="danger"
:disabled="previewBuilding || file.status === 'uploading'"
:aria-label="`删除文件 ${file.name}`"
@click="emit('remove-file', file.uid)"
>
@@ -441,6 +553,11 @@ function formatSize(size: number) {
margin-top: 0;
}
.upload-queue-status {
color: #5b50f2;
font-size: inherit;
}
.uploaded-file-pagination {
display: flex;
justify-content: flex-end;
@@ -495,9 +612,59 @@ function formatSize(size: number) {
}
}
.file-preview-progress {
display: flex;
flex: 0 1 220px;
flex-direction: column;
gap: 5px;
min-width: 150px;
&.is-queued .file-status,
&.is-waiting .file-status {
color: #8a93a3;
}
&.is-uploading .file-status,
&.is-processing .file-status {
color: #5b50f2;
}
&.is-success .file-status {
color: #2ca66a;
}
&.is-upload-failed .file-status,
&.is-preview-failed .file-status,
.file-preview-error {
color: #d94b4b;
}
}
.file-status {
color: #2ca66a;
display: flex;
align-items: center;
justify-content: space-between;
gap: 8px;
font-size: 12px;
> span:first-child {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
.file-progress-value {
flex: 0 0 auto;
font-variant-numeric: tabular-nums;
}
.file-preview-error {
overflow: hidden;
font-size: 11px;
line-height: 1.4;
text-overflow: ellipsis;
white-space: nowrap;
}
.uploaded-file :deep(.el-button) {
@@ -514,8 +681,12 @@ function formatSize(size: number) {
padding: 8px 10px;
}
.file-status {
.file-preview-progress {
flex: 0 1 auto;
min-width: 130px;
}
.file-status {
line-height: 1.4;
white-space: normal;
}
@@ -545,7 +716,8 @@ function formatSize(size: number) {
min-width: calc(100% - 40px);
}
.file-status {
.file-preview-progress {
flex: 1 0 calc(100% - 38px);
margin-left: 38px;
}
}

View File

@@ -22,12 +22,12 @@ const PREPROCESS_OPTIONS: Array<{
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头合并单元格' },
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
{ value: 'normalize_format', label: '数据格式标准化', description: '统一编码、空白、字段名 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别多级表头合并单元格,并将嵌套字段展平' },
{ value: 'deduplicate', label: '重复数据去重', description: '基于整行精确匹配和关键字段组合删除重复记录' },
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
]
function updateField<K extends keyof StructuredProcessOptions>(
@@ -43,9 +43,11 @@ function updateGenerationOptions(value: GenerationControlOptions) {
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter(
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
)
const preprocessOptions = Array.from(new Set(value.filter(
(option): option is PreprocessOption => (
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
),
)))
updateField('preprocessOptions', preprocessOptions)
}
</script>

View File

@@ -30,9 +30,8 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
]
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
{ value: 'semantic', label: '自动语义切分' },
{ value: 'heading', label: '按标题和段落' },
{ value: 'fixed', label: '按固定长度' },
{ value: 'structure', label: '文档结构' },
{ value: 'fixed', label: '固定 Token' },
{ value: 'custom', label: '自定义分隔符' },
]
@@ -70,9 +69,9 @@ function updateGenerationOptions(value: GenerationControlOptions) {
function updateSmartPreprocess(value: string | number | boolean) {
const enabled = Boolean(value)
const remainingOptions = props.options.preprocessOptions.filter(
const remainingOptions = Array.from(new Set(props.options.preprocessOptions.filter(
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
)
)))
updateField(
'preprocessOptions',
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
@@ -80,9 +79,11 @@ function updateSmartPreprocess(value: string | number | boolean) {
}
function updateDesensitize(value: string | number | boolean) {
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
(option) => option !== 'desensitize',
)
const preprocessOptions: UnstructuredPreprocessOption[] = Array.from(new Set(
props.options.preprocessOptions.filter(
(option) => option !== 'desensitize',
),
))
if (Boolean(value)) preprocessOptions.push('desensitize')
updateField('preprocessOptions', preprocessOptions)
}
@@ -116,7 +117,7 @@ defineExpose({ revealValidation })
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>默认使用推荐策略只需决定是否需要脱敏</p>
<p>默认启用结构感知的推荐策略只需决定是否需要脱敏</p>
</div>
</div>
<div class="preprocess-option-grid">
@@ -127,7 +128,7 @@ defineExpose({ revealValidation })
/>
<span class="preprocess-option-copy">
<strong>智能预处理</strong>
<small>自动清理解析去重及保留上下文</small>
<small>清理无效内容感知文档结构合并短块预过滤低质量内容近重复去重并通过重叠保护上下文</small>
</span>
</label>
@@ -138,7 +139,7 @@ defineExpose({ revealValidation })
/>
<span class="preprocess-option-copy">
<strong>敏感信息脱敏</strong>
<small>处理姓名手机号等隐私信息</small>
<small>识别并脱敏姓名手机号邮箱和身份证号</small>
</span>
</label>
@@ -159,7 +160,7 @@ defineExpose({ revealValidation })
<div class="section-title-row">
<div>
<h3>切分选项</h3>
<p>以语义完整为优先将长文档拆成可独立生成问答的内容块</p>
<p>按文档结构固定 Token 或自定义分隔符拆分长文档</p>
</div>
</div>
@@ -178,7 +179,7 @@ defineExpose({ revealValidation })
:value="method.value"
/>
</el-select>
<small>推荐使用自动语义切分在长度限制内优先保留完整句段</small>
<small>推荐使用文档结构按标题段落和句子边界保留完整内容块</small>
</label>
<label class="config-field">

View File

@@ -30,7 +30,7 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
'deduplicate_content',
'preserve_context',
],
chunkMethod: 'semantic',
chunkMethod: 'structure',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,

View File

@@ -1,3 +1,5 @@
import type { DataProcessPreviewFileStatus } from '@/types/dataProcess'
export type ProcessType = 'structured' | 'unstructured' | 'external'
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
@@ -44,7 +46,7 @@ export type UnstructuredPreprocessOption =
| 'desensitize'
| 'preserve_context'
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
export type ChunkMethod = 'structure' | 'fixed' | 'custom'
export interface UnstructuredProcessOptions extends GenerationControlOptions {
preprocessOptions: UnstructuredPreprocessOption[]
@@ -62,7 +64,7 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
}
export interface ExternalDataSource {
type: string
type: 'postgresql'
url: string
authMode: 'none' | 'basic'
username?: string
@@ -75,14 +77,21 @@ export interface ExternalDataSource {
export interface UploadedDataFile {
uid: string | number
sourceFileId?: string
rawFile?: File
name: string
size: number
count: number
content: string
fileFormat?: string
checksumSha256?: string
status?: 'uploading' | 'ready' | 'failed'
error?: string
status: 'queued' | 'uploading' | 'ready' | 'failed'
uploadProgress: number
uploadError?: string
previewStatus?: DataProcessPreviewFileStatus
previewProgress?: number
previewError?: string
previewCount?: number
previewConfigSignature?: string
}
export interface SourceLine {

View File

@@ -1,152 +0,0 @@
import { nextTick, type Reactive, type Ref } from 'vue'
import { ElMessage } from 'element-plus'
import type {
ExternalDataSource,
ProcessType,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 7
interface DraftSnapshot {
schemaVersion?: number
taskId?: string
currentStepId?: StepId
task?: { name?: string; description?: string }
processType?: ProcessType
structuredOptions?: Partial<StructuredProcessOptions>
unstructuredOptions?: Partial<UnstructuredProcessOptions>
externalSource?: Partial<ExternalDataSource>
}
interface DraftBindings {
taskId: Ref<string | null>
currentStepId: Readonly<Ref<StepId>>
task: Reactive<{ name: string; description: string }>
processType: Ref<ProcessType>
structuredOptions: Ref<StructuredProcessOptions>
unstructuredOptions: Ref<UnstructuredProcessOptions>
externalSource: Reactive<ExternalDataSource>
restoringDraft: Ref<boolean>
dirty: Ref<boolean>
goToStep: (stepId: StepId) => void
}
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
return {
type: typeof source.type === 'string' ? source.type : 'postgresql',
url: typeof source.url === 'string' ? source.url : '',
authMode: source.authMode === 'basic' ? 'basic' as const : 'none' as const,
username: typeof source.username === 'string' ? source.username : '',
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
query: typeof source.query === 'string' ? source.query : '',
fileName: typeof source.fileName === 'string' ? source.fileName : 'external-data.jsonl',
}
}
function isDraftSnapshot(value: unknown): value is DraftSnapshot {
return Boolean(value && typeof value === 'object')
}
/**
* 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。
*/
export function useDataProcessDraft(bindings: DraftBindings) {
function draftSnapshot(): DraftSnapshot {
return {
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
taskId: bindings.taskId.value || undefined,
currentStepId: bindings.currentStepId.value,
task: { ...bindings.task },
processType: bindings.processType.value,
structuredOptions: {
...bindings.structuredOptions.value,
preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions],
datasetSplit: { ...bindings.structuredOptions.value.datasetSplit },
},
unstructuredOptions: {
...bindings.unstructuredOptions.value,
preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions],
datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit },
},
externalSource: sanitizeExternalSource(bindings.externalSource),
}
}
function writeDraft(showWarning: boolean) {
try {
localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
} catch {
if (showWarning) ElMessage.warning('草稿保存失败,请检查浏览器存储空间')
}
}
function persistDraft() {
if (!bindings.restoringDraft.value) writeDraft(true)
}
function restoreDraft() {
try {
const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
if (!raw) return
const snapshot: unknown = JSON.parse(raw)
if (!isDraftSnapshot(snapshot)) return
bindings.restoringDraft.value = true
bindings.goToStep('create')
bindings.taskId.value = typeof snapshot.taskId === 'string' ? snapshot.taskId : null
bindings.task.name = snapshot.task?.name || ''
bindings.task.description = snapshot.task?.description || ''
bindings.processType.value = snapshot.processType === 'unstructured' || snapshot.processType === 'external'
? snapshot.processType
: 'structured'
if (snapshot.structuredOptions) {
bindings.structuredOptions.value = {
...bindings.structuredOptions.value,
...snapshot.structuredOptions,
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
? snapshot.structuredOptions.preprocessOptions
: bindings.structuredOptions.value.preprocessOptions,
datasetSplit: {
...bindings.structuredOptions.value.datasetSplit,
...snapshot.structuredOptions.datasetSplit,
},
}
}
if (snapshot.unstructuredOptions) {
bindings.unstructuredOptions.value = {
...bindings.unstructuredOptions.value,
...snapshot.unstructuredOptions,
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
? snapshot.unstructuredOptions.preprocessOptions
: bindings.unstructuredOptions.value.preprocessOptions,
datasetSplit: {
...bindings.unstructuredOptions.value.datasetSplit,
...snapshot.unstructuredOptions.datasetSplit,
},
}
}
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
password: '',
})
bindings.dirty.value = false
nextTick(() => {
bindings.restoringDraft.value = false
// 立即覆盖 v5 及更早草稿,清除其中可能存在的敏感值和大段正文。
writeDraft(false)
})
ElMessage.info('已恢复上次的任务配置')
} catch {
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
}
}
return { draftSnapshot, persistDraft, restoreDraft }
}

View File

@@ -0,0 +1,48 @@
import { buildDataProcessPreview } from '@/api/modules/dataProcess'
import type { DataProcessPreviewFileBuildProgress } from '@/types/dataProcess'
export function useDataProcessPreviewBuild() {
async function buildPreviewsByFile(
taskId: string | number,
sourceFileIds: Array<string | number>,
onProgress: (progress: DataProcessPreviewFileBuildProgress) => void,
) {
const results: DataProcessPreviewFileBuildProgress[] = []
for (const sourceFileId of sourceFileIds) {
onProgress({ source_file_id: sourceFileId, status: 'processing', progress: 0 })
let progress: DataProcessPreviewFileBuildProgress
try {
const built = await buildDataProcessPreview(taskId, {
replace_existing: true,
source_file_ids: [sourceFileId],
})
const sourceId = String(sourceFileId)
const fileResult = built.files?.find((item) => String(item.source_file_id) === sourceId)
const previewCount = fileResult?.preview_count
?? built.file_counts?.[sourceId]
?? built.items.filter((item) => String(item.source_file_id) === sourceId).length
progress = previewCount > 0
? { source_file_id: sourceFileId, status: 'success', progress: 100, preview_count: previewCount }
: {
source_file_id: sourceFileId,
status: 'failed',
progress: 0,
preview_count: 0,
error: '该文件没有生成可用的预览条目,请检查内容或切分配置',
}
} catch (error) {
progress = {
source_file_id: sourceFileId,
status: 'failed',
progress: 0,
error: error instanceof Error && error.message ? error.message : '切分失败,请重试',
}
}
results.push(progress)
onProgress(progress)
}
return results
}
return { buildPreviewsByFile }
}

View File

@@ -0,0 +1,198 @@
import { ref, type Ref } from 'vue'
import {
getDataProcessSourceContent,
uploadDataProcessSourceFiles,
type DataProcessSourceFile,
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
])
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
interface SourceUploadJob {
uid: string
file: File
extension: string
}
interface SourceUploadOptions {
taskId: Ref<string | null>
uploadedFiles: Ref<UploadedDataFile[]>
onUploaded: (file: UploadedDataFile) => void
}
type SourceFileValidation =
| { valid: true; extension: string }
| { valid: false; severity: 'error' | 'warning'; message: string }
export function validateSourceFileSelection(
raw: File,
processType: ProcessType,
selectedFiles: UploadedDataFile[],
): SourceFileValidation {
if (raw.size > MAX_SOURCE_FILE_BYTES) {
return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
}
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
return {
valid: false,
severity: 'error',
message: '旧版 DOC、XLS、PPT 文件暂不支持,请分别转换为 DOCX、XLSX、PPTX 后上传',
}
}
const supportedExtensions = processType === 'unstructured'
? UNSTRUCTURED_FILE_EXTENSIONS
: STRUCTURED_FILE_EXTENSIONS
if (!supportedExtensions.has(extension)) {
return {
valid: false,
severity: 'error',
message: processType === 'unstructured'
? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
}
}
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
}
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
}
const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
}
return { valid: true, extension }
}
export function mapDataProcessSourceFile(
file: DataProcessSourceFile,
content = '',
): UploadedDataFile {
return {
uid: String(file.id),
sourceFileId: String(file.id),
name: file.name,
size: file.size_bytes,
count: file.record_count,
content,
fileFormat: file.file_format,
checksumSha256: file.checksum_sha256,
status: 'ready',
uploadProgress: 100,
previewStatus: 'waiting',
previewProgress: 0,
}
}
function getUploadErrorMessage(error: unknown) {
const responseData = (error as {
response?: { data?: { detail?: string | { message?: string }; message?: string } }
})?.response?.data
const detail = responseData?.detail
if (typeof detail === 'string') return detail
if (detail?.message) return detail.message
if (responseData?.message) return responseData.message
if (error instanceof Error && error.message) return error.message
return '上传失败,请删除该文件后重新选择'
}
export function useDataProcessSourceUpload(options: SourceUploadOptions) {
const sourceUploading = ref(false)
const queue: SourceUploadJob[] = []
let worker: Promise<void> | null = null
async function uploadOne(job: SourceUploadJob) {
const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
if (!pending) return
const currentTaskId = options.taskId.value
if (!currentTaskId) {
pending.status = 'failed'
pending.uploadError = '任务尚未创建,请返回模型选择步骤后重试'
return
}
pending.status = 'uploading'
pending.uploadProgress = 0
pending.uploadError = undefined
try {
let content = ''
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
} catch {
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
}
if (!content.trim()) throw new Error('不能上传空文件')
}
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
pending.uploadProgress = progress
})
const source = uploaded.files[0]
if (!source) throw new Error('后端未返回源文件记录')
// 先登记后端 ID确保正文读取失败时仍可正确删除已落库的文件。
Object.assign(pending, mapDataProcessSourceFile(source), {
rawFile: job.file,
status: 'uploading',
uploadProgress: 99,
})
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
start_line: 1,
line_count: 10_000,
})
pending.content = parsed.content
} catch {
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
}
} else {
pending.content = content
}
pending.status = 'ready'
pending.uploadProgress = 100
options.onUploaded(pending)
} catch (error) {
pending.status = 'failed'
pending.uploadError = getUploadErrorMessage(error)
}
}
async function drainQueue() {
sourceUploading.value = true
try {
while (queue.length) {
const job = queue.shift()
if (job) await uploadOne(job)
}
} finally {
sourceUploading.value = false
}
}
function enqueueSourceUpload(job: SourceUploadJob) {
queue.push(job)
if (worker) return
worker = drainQueue().finally(() => {
worker = null
})
}
return {
enqueueSourceUpload,
sourceUploading,
}
}