diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 437047d..6df2d34 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -16,6 +16,7 @@ "element-plus": "^2.9.1", "marked": "^15.0.5", "md-editor-v3": "^5.1.4", + "pdfjs-dist": "^6.1.200", "pinia": "^2.3.0", "vue": "^3.5.13", "vue-echarts": "^8.0.1", @@ -1213,6 +1214,256 @@ "integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==", "license": "MIT" }, + "node_modules/@napi-rs/canvas": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.2.tgz", + "integrity": "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "1.0.2", + "@napi-rs/canvas-darwin-arm64": "1.0.2", + "@napi-rs/canvas-darwin-x64": "1.0.2", + "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", + "@napi-rs/canvas-linux-arm64-musl": "1.0.2", + "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", + "@napi-rs/canvas-linux-x64-gnu": "1.0.2", + "@napi-rs/canvas-linux-x64-musl": "1.0.2", + "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", + "@napi-rs/canvas-win32-x64-msvc": "1.0.2" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.2.tgz", + "integrity": "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.2.tgz", + "integrity": "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.2.tgz", + "integrity": "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.2.tgz", + "integrity": "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.2.tgz", + "integrity": "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.2.tgz", + "integrity": "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.2.tgz", + "integrity": "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.2.tgz", + "integrity": "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.2.tgz", + "integrity": "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.2.tgz", + "integrity": "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.2.tgz", + "integrity": "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3443,6 +3694,18 @@ "dev": true, "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "6.1.200", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.1.200.tgz", + "integrity": "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==", + "license": "Apache-2.0", + "engines": { + "node": ">=22.13.0 || >=24" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^1.0.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index dc2037e..a5a46ae 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -36,6 +36,7 @@ "element-plus": "^2.9.1", "marked": "^15.0.5", "md-editor-v3": "^5.1.4", + "pdfjs-dist": "^6.1.200", "pinia": "^2.3.0", "vue": "^3.5.13", "vue-echarts": "^8.0.1", diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index 84dbd60..73f43e6 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -12,17 +12,19 @@ const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmD const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue') const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts') const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts') +const sourceUploadWorkerPath = path.join(createDir, 'useDataProcessSourceUpload.ts') const viewSource = await readFile(viewPath, 'utf8') const layoutSource = await readFile(layoutPath, 'utf8') -const [draftSource, stateSource, generationSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([ - readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'), +const [stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([ readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'), readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'), + readFile(path.join(createDir, 'useDataProcessPreviewBuild.ts'), 'utf8'), + readFile(sourceUploadWorkerPath, 'utf8'), readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'), readFile(apiPath, 'utf8'), readFile(contractTypesPath, 'utf8'), ]) -const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n') +const implementationSource = [viewSource, stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource].join('\n') assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog') const confirmDialogSource = await readFile(confirmDialogPath, 'utf8') @@ -58,9 +60,13 @@ assert.match( /@media \(max-width: 1100px\)[\s\S]*?\.step-title\s*\{[\s\S]*?display:\s*none[\s\S]*?\.step-item\.is-active \.step-title\s*\{[\s\S]*?display:\s*block/, '六步向导在中等宽度下没有收起非当前步骤标题', ) -assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化') -assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取') -assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿') +assert.equal(existsSync(path.join(createDir, 'useDataProcessDraft.ts')), false, '不应保留新建任务草稿模块') +assert.doesNotMatch( + viewSource, + /useDataProcessDraft|persistDraft|restoreDraft|restoringDraft|localStorage\.(?:setItem|getItem)/, + '新建任务不应保存或恢复草稿', +) +assert.match(viewSource, /localStorage\.removeItem\('yg-data-process-create-draft'\)/, '进入新建页时应清理遗留草稿') assert.ok(viewSource.split('\n').length < 1000, 'DataProcessCreateView 拆分后仍超过 1000 行') assert.match(viewSource, /useDataProcessGeneration\(\{/, '生成流程没有拆分到独立 composable') @@ -79,13 +85,16 @@ for (const component of expectedComponents) { const typesPath = path.join(createDir, 'types.ts') const modelPath = path.join(createDir, 'previewModel.ts') +const pdfViewerPath = path.join(createDir, 'PdfSourceViewer.vue') assert.ok(existsSync(typesPath), '缺少向导类型定义') assert.ok(existsSync(modelPath), '缺少来源映射模型') +assert.ok(existsSync(pdfViewerPath), '缺少 PDF 原文件预览组件') -const [typesSource, modelSource, previewSource] = await Promise.all([ +const [typesSource, modelSource, previewSource, pdfViewerSource] = await Promise.all([ readFile(typesPath, 'utf8'), readFile(modelPath, 'utf8'), readFile(path.join(createDir, 'PreviewCompareStep.vue'), 'utf8'), + readFile(pdfViewerPath, 'utf8'), ]) for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedContent']) { @@ -98,8 +107,8 @@ assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后 assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态') assert.match( viewSource, - /buildDataProcessPreview\(taskId\.value,\s*\{[\s\S]*?source_file_ids:\s*uploadedFiles\.value\.map/, - '预览没有通过后端按已上传源文件构建', + /const \{ buildPreviewsByFile \} = useDataProcessPreviewBuild\(\)/, + '父页面没有通过独立 composable 构建逐文件预览', ) assert.match(viewSource, /getDataProcessPreview\(taskId\.value,\s*\{ page:\s*1, page_size:\s*500 \}\)/, '预览构建后没有分页读取后端数据') assert.doesNotMatch(viewSource, /buildPreviewItems\(/, '创建向导仍在本地构建集成预览数据') @@ -142,6 +151,21 @@ assert.match(previewSource, /\.editor-actions\s*\{[\s\S]*?justify-content:\s*fle assert.doesNotMatch(previewSource, /item-token|item-status|modifiedOnly|仅看已修改/, '切片列表不应再显示 Token 或修改状态') assert.match(previewSource, /\.preview-editor\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?overflow-y:\s*auto/, '编辑模式必须占据右侧剩余区域并可滚动') assert.match(previewSource, /@media \(max-width: 900px\)/, '第四步缺少窄屏上下布局') +assert.match(previewSource, /= 7, '安全草稿格式版本不得低于 v7') - +assert.match( + viewSource, + /if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续：数据预览'/, + '第三步主按钮没有依次反映上传、切分状态并指向数据预览', +) const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()') const nextFromModelStart = viewSource.indexOf('async function nextFromModel()', nextFromCreateStart) const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromModelStart) @@ -213,36 +237,95 @@ assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, ' assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务') assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件') assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据') -assert.match(nextFromUploadSource, /buildDataProcessPreview\(/, '上传步骤没有调用后端构建预览') +assert.match( + nextFromUploadSource, + /const pendingFileIds = uploadedFiles\.value[\s\S]*?\.filter\(\(file\) => file\.previewStatus !== 'success' \|\| file\.previewConfigSignature !== configSignature\)[\s\S]*?\.map\(\(file\) => file\.sourceFileId\)[\s\S]*?\.filter\(\(fileId\): fileId is string => Boolean\(fileId\)\)/, + '上传步骤没有仅选择待处理或失败文件，无法跳过成功文件并重试失败文件', +) +assert.match( + nextFromUploadSource, + /await buildPreviewsByFile\(taskId\.value, pendingFileIds, \(progress\) => \{/, + '上传步骤没有通过独立 composable 串行构建待处理文件', +) +assert.match( + nextFromUploadSource, + /file\.previewStatus = progress\.status[\s\S]*?file\.previewProgress = progress\.progress[\s\S]*?file\.previewError = progress\.error/, + '上传步骤没有把单文件构建进度与错误回写到对应文件', +) +assert.doesNotMatch( + nextFromUploadSource, + /source_file_ids:\s*uploadedFiles\.value\.map/, + '上传步骤仍一次性批量构建全部文件预览', +) +assert.match( + previewBuildSource, + /for \(const sourceFileId of sourceFileIds\)[\s\S]*?onProgress\(\{ source_file_id: sourceFileId, status: 'processing', progress: 0 \}\)/, + '逐文件构建 composable 没有串行处理并先上报 processing 进度', +) +assert.match( + previewBuildSource, + /buildDataProcessPreview\(taskId,\s*\{[\s\S]*?replace_existing:\s*true[\s\S]*?source_file_ids:\s*\[sourceFileId\]/, + '逐文件构建 composable 没有按单个源文件 ID 替换预览', +) +assert.match( + previewBuildSource, + /status: 'success', progress: 100, preview_count: previewCount/, + '单文件构建成功后没有上报 100% 和预览数量', +) +assert.match( + previewBuildSource, + /catch \(error\)[\s\S]*?status: 'failed'[\s\S]*?progress: 0[\s\S]*?error:/, + '单文件构建失败后没有保留可重试错误', +) assert.match(nextFromUploadSource, /getDataProcessPreview\(/, '上传步骤没有读取后端预览结果') -assert.match(nextFromUploadSource, /goToStep\('preview'\)/, '上传步骤完成后没有进入数据预览') +assert.match( + nextFromUploadSource, + /const allFilesSucceeded = \(\) => uploadedFiles\.value\.every\(\(file\) => \([\s\S]*?file\.previewStatus === 'success'[\s\S]*?file\.previewConfigSignature === configSignature/, + '上传步骤缺少全部文件 success 且配置签名一致的完成判定', +) +const previewSuccessGateIndex = nextFromUploadSource.indexOf('if (failedCount || !allFilesSucceeded())') +const previewStepIndex = nextFromUploadSource.lastIndexOf("goToStep('preview')") +assert.ok(previewSuccessGateIndex >= 0, '上传步骤没有在任一文件未成功时停留当前步骤') +assert.match( + nextFromUploadSource, + /if \(signature === previewSignature\.value && previewItems\.value\.length && allFilesSucceeded\(\)\) \{\s*goToStep\('preview'\)/, + '缓存预览快速路径没有要求全部文件 success', +) +assert.equal( + (nextFromUploadSource.match(/goToStep\('preview'\)/g) || []).length, + 2, + '上传步骤只能通过全成功缓存路径或本轮全成功路径进入预览', +) +assert.ok( + previewStepIndex > previewSuccessGateIndex, + '上传步骤必须在全部文件 success 后才能进入数据预览', +) assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识') assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成') assert.match(viewSource, /generation\.status === 'success'[\s\S]*?goToStep\('results'\)/, '生成成功后没有进入结果编辑与保存') -assert.match(draftSource, /currentStepId:\s*bindings\.currentStepId\.value/, '草稿没有保存稳定步骤标识') -const draftSnapshotSource = draftSource.slice( - draftSource.indexOf('function draftSnapshot()'), - draftSource.indexOf('function writeDraft'), -) -for (const forbiddenField of ['uploadedFiles', 'previewItems', 'results', 'password', 'token']) { - assert.ok(!draftSnapshotSource.includes(forbiddenField), `安全草稿不应持久化：${forbiddenField}`) -} -assert.match(draftSource, /writeDraft\(false\)/, '恢复旧草稿后没有立即覆盖潜在敏感数据') assert.match(viewSource, / match[1]) +assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确') +assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一') +assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重') assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类') for (const splitName of ['训练集', '验证集', '测试集']) { assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`) @@ -260,8 +343,6 @@ for (const splitField of ['train', 'validation', 'test']) { assert.match(structuredOptionsSource, //, '父页面缺少结构化配置状态') assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10') -assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿') -assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复') assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置') assert.match(generationSource, /generateDataProcess\(taskId\)/, '开始生成没有调用真实 API') assert.match(generationSource, /getDataProcessProgress\(taskId\)/, '生成状态没有通过真实 API 轮询') @@ -284,13 +365,39 @@ for (const apiName of [ ]) { assert.match(apiSource, new RegExp(`export (?:const|async function|function) ${apiName}\\b`), `API 模块缺少 ${apiName}`) } -assert.match(viewSource, /uploadDataProcessSourceFiles\(taskId\.value,\s*\[raw\]\)/, '文件上传没有调用真实 API') +assert.match(sourceUploadWorkerSource, /uploadDataProcessSourceFiles\(currentTaskId,\s*\[job\.file\]/, '文件上传没有逐文件调用真实 API') +assert.match(sourceUploadWorkerSource, /STRUCTURED_FILE_EXTENSIONS = new Set\(\['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'\]\)/, '结构化文件扩展名白名单不完整') +for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson']) { + assert.ok(sourceUploadWorkerSource.includes(`'${extension}'`), `非结构化文件扩展名白名单缺少 ${extension}`) +} +assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别') +assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示') +assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验') +assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本') assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段') +assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%') +assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时') assert.match(apiSource, /\/preview\/build/, 'API 模块缺少后端预览构建路径') +assert.match(apiSource, /\/preview\/build`[\s\S]*?\{ timeout: 5 \* 60 \* 1000 \}/, '单文件切分请求缺少 5 分钟超时') assert.match(apiSource, /\/progress`/, 'API 模块缺少生成进度路径') assert.match(apiSource, /\/results`/, 'API 模块缺少结果分页路径') assert.match(apiSource, /\/publish`/, 'API 模块缺少数据集发布路径') assert.match(contractTypesSource, /source_file_ids\?: Array/, '预览构建契约缺少源文件 ID 列表') +assert.match( + contractTypesSource, + /export type DataProcessPreviewFileStatus = 'waiting' \| 'processing' \| 'success' \| 'failed'/, + '文件预览状态契约不完整', +) +for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) { + assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段：${field}`) +} +assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整') +assert.match(viewSource, /uploadedFiles\.value\.push\([\s\S]*?status: 'queued'[\s\S]*?enqueueSourceUpload/, '文件选择后没有先进入列表再加入上传队列') +assert.match(sourceUploadWorkerSource, /while \(queue\.length\) \{[\s\S]*?await uploadOne\(job\)/, '多文件上传没有由单一队列逐个等待') +assert.doesNotMatch(sourceUploadWorkerSource, /Promise\.(?:all|allSettled)/, '上传队列不得并发消费文件') +assert.match(sourceUploadWorkerSource, /pending\.status = 'ready'[\s\S]*?pending\.uploadProgress = 100/, '服务端响应成功后没有将文件置为上传完成') +assert.match(viewSource, /failedUploads[\s\S]*?hasUnfinishedUploads[\s\S]*?buildPreviewsByFile/, '上传失败或未完成时没有阻断切分') +assert.doesNotMatch(viewSource, /file\.sourceFileId \|\| file\.uid/, '切分或删除仍可能把本地临时 UID 当成后端文件 ID') assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段') for (const field of [ @@ -302,7 +409,7 @@ for (const field of [ 'minOutputLength', ]) { assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`) - assert.ok(implementationSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`) + assert.ok(implementationSource.includes(field), `父页面默认值缺少字段：${field}`) } assert.match(structuredOptionsSource, /GenerationOptionsPanel/, '结构化生成选项没有复用统一的质量筛选组件') assert.match(unstructuredOptionsSource, /GenerationOptionsPanel/, '非结构化生成选项没有复用统一的质量筛选组件') @@ -336,10 +443,23 @@ assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最 assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') assert.match(taskSetupSource, /qualityValidationMessage/, '质量规则缺少继续前校验') assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表') -assert.match(viewSource, /model\.type === 'LLM'/, '数据生成模型列表没有排除非大模型') +const generationModelsStart = viewSource.indexOf('const generationModels') +const generationModelsEnd = viewSource.indexOf('const taskSetupRef', generationModelsStart) +const generationModelsSource = viewSource.slice(generationModelsStart, generationModelsEnd) +assert.match(generationModelsSource, /computed\(\(\) => modelList\.value\)/, '数据生成模型候选没有直接使用模型管理完整列表') +assert.doesNotMatch(generationModelsSource, /\.filter\(|model\.type|model\.model_source|model\.status|model\.purpose/, '数据生成模型候选仍按类型、来源、状态或用途静默过滤') +assert.match(viewSource, /modelsStore\.load\(true\)/, '进入创建向导时没有强制刷新模型管理列表') assert.match(viewSource, / match[1]) +assert.deepEqual(smartOptionValues, expectedSmartPreprocessOptions, '智能预处理内部值与后端语义不一致') +assert.equal(new Set(smartOptionValues).size, smartOptionValues.length, '非结构化智能预处理 value 必须唯一') +for (const descriptionPart of [ + '清理无效内容', + '感知文档结构', + '合并短块', + '预过滤低质量内容', + '近重复去重', + '重叠保护上下文', +]) { + assert.ok(unstructuredOptionsSource.includes(descriptionPart), `智能预处理说明缺少语义：${descriptionPart}`) +} +assert.ok(unstructuredOptionsSource.includes('姓名、手机号、邮箱和身份证号'), '非结构化脱敏说明缺少完整字段范围') +assert.match(unstructuredOptionsSource, /Array\.from\(new Set\(props\.options\.preprocessOptions\.filter\(/, '非结构化预处理选中值没有去重') assert.match(unstructuredOptionsSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项') assert.match(unstructuredOptionsSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑') assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑') assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类') -for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) { - assert.ok(unstructuredOptionsSource.includes(method), `切分方式缺少选项：${method}`) +const expectedChunkMethods = [ + ['structure', '文档结构'], + ['fixed', '固定 Token'], + ['custom', '自定义分隔符'], +] +const chunkMethodsStart = unstructuredOptionsSource.indexOf('const CHUNK_METHODS') +const chunkMethodsEnd = unstructuredOptionsSource.indexOf('const UNSTRUCTURED_NUMBER_LIMITS', chunkMethodsStart) +const chunkMethodsSource = unstructuredOptionsSource.slice(chunkMethodsStart, chunkMethodsEnd) +const chunkMethodValues = [...chunkMethodsSource.matchAll(/value: '([^']+)'/g)].map((match) => match[1]) +assert.deepEqual(chunkMethodValues, expectedChunkMethods.map(([value]) => value), '切分方式值集合不准确') +for (const [value, label] of expectedChunkMethods) { + assert.ok(chunkMethodsSource.includes(`value: '${value}', label: '${label}'`), `切分方式缺少选项：${label}`) } +for (const removedMethod of ['semantic', 'heading']) { + assert.ok(!chunkMethodsSource.includes(`value: '${removedMethod}'`), `切分方式仍保留已移除值：${removedMethod}`) +} +assert.ok(unstructuredOptionsSource.includes('推荐使用文档结构'), '切分方式缺少文档结构默认推荐说明') for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) { assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`) } @@ -395,19 +555,87 @@ assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), ' assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局') assert.match(viewSource, /const unstructuredOptions = ref/, '父页面缺少非结构化配置状态') -assert.match(stateSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分') +assert.match(stateSource, /chunkMethod:\s*'structure'/, '非结构化默认切分方式必须为文档结构') assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') -assert.match(draftSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.bindings\.unstructuredOptions\.value/, '非结构化配置没有写入草稿') -assert.match(draftSource, /bindings\.unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复') assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置') -assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识') -assert.match(draftSource, /schemaVersion:\s*DATA_PROCESS_DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识') -assert.match(draftSource, /bindings\.goToStep\('create'\)/, '恢复配置后应回到安全的创建步骤') assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断') +function defaultPreprocessValues(functionName, nextFunctionName) { + const start = stateSource.indexOf(`export function ${functionName}`) + const end = nextFunctionName ? stateSource.indexOf(`export function ${nextFunctionName}`, start) : stateSource.length + const functionSource = stateSource.slice(start, end) + const match = functionSource.match(/preprocessOptions:\s*\[([\s\S]*?)\]/) + assert.ok(match, `${functionName} 缺少 preprocessOptions 默认值`) + return [...match[1].matchAll(/'([^']+)'/g)].map((item) => item[1]) +} + +const defaultStructuredPreprocess = defaultPreprocessValues( + 'createDefaultStructuredOptions', + 'createDefaultUnstructuredOptions', +) +assert.deepEqual( + defaultStructuredPreprocess, + ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], + '结构化默认预处理配置不准确', +) +assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复') +const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions') +assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整') +assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复') + +const backendConfigStart = viewSource.indexOf('function toBackendConfig()') +const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart) +assert.ok(backendConfigStart >= 0 && backendConfigEnd > backendConfigStart, '缺少任务后端配置映射') +const backendConfigSource = viewSource.slice(backendConfigStart, backendConfigEnd) +assert.ok(backendConfigSource.includes('preprocess_options: [...options.preprocessOptions]'), '预处理选项没有完整传入任务配置') +assert.ok(backendConfigSource.includes('dataset_split: { ...options.datasetSplit }'), '数据集划分没有完整传入任务配置') +for (const [backendField, frontendField] of [ + ['semantic_enrichment', 'semanticEnrichment'], + ['generation_model_id', 'generationModelId'], + ['generation_prompt', 'generationPrompt'], + ['temperature', 'temperature'], + ['max_tokens', 'maxTokens'], + ['json_mode', 'jsonMode'], + ['quality_filter_enabled', 'qualityFilterEnabled'], + ['filter_low_quality', 'filterLowQuality'], + ['filter_short_content', 'filterShortContent'], + ['min_output_length', 'minOutputLength'], +]) { + assert.ok( + backendConfigSource.includes(`${backendField}: options.${frontendField}`), + `公共配置 ${frontendField} 没有传入 task payload`, + ) +} +for (const [backendField, frontendField] of [ + ['chunk_method', 'chunkMethod'], + ['chunk_size', 'chunkSize'], + ['chunk_overlap', 'chunkOverlap'], + ['min_chunk_size', 'minChunkSize'], + ['custom_delimiter', 'customDelimiter'], + ['preserve_tables', 'preserveTables'], + ['preserve_code_blocks', 'preserveCodeBlocks'], + ['preserve_lists', 'preserveLists'], + ['qa_pairs_per_chunk', 'qaPairsPerChunk'], +]) { + assert.ok( + backendConfigSource.includes(`${backendField}: unstructuredOptions.value.${frontendField}`), + `非结构化配置 ${frontendField} 没有传入 task payload`, + ) +} +assert.ok( + backendConfigSource.includes('qa_pairs_per_row: structuredOptions.value.qaPairsPerRow'), + '结构化每行生成数量没有传入 task payload', +) +const taskPayloadStart = viewSource.indexOf('function taskPayload()') +const taskPayloadEnd = viewSource.indexOf('function externalPayload()', taskPayloadStart) +const taskPayloadSource = viewSource.slice(taskPayloadStart, taskPayloadEnd) +for (const marker of ['name: task.name.trim()', 'description: task.description.trim()', 'process_type: processType.value', 'config: toBackendConfig()']) { + assert.ok(taskPayloadSource.includes(marker), `任务创建 payload 缺少：${marker}`) +} + const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()') const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart) assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数') @@ -667,7 +895,7 @@ for (const property of ['max-height', 'overflow', 'overflow-x', 'overflow-y']) { for (const marker of [ 'uploaded-file-list-header', 'uploaded-file-items', - '已添加 {{ uploadedFiles.length }} 个文件', + '已选择 {{ uploadedFiles.length }} 个文件', ':title="file.name"', ]) { assert.ok(sourceUploadSource.includes(marker), `源数据文件列表缺少：${marker}`) @@ -720,7 +948,7 @@ assert.match( ) assert.match( sourceUploadSource, - /\s*已添加 \{\{ uploadedFiles\.length \}\} 个文件<\/span>/, + /[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?正在逐个上传/, '文件列表标题结构或文件数量文案缺失', ) assert.match( @@ -730,17 +958,27 @@ assert.match( ) assert.match( sourceUploadSource, - /]*>/, + /]*>/, '无文件时未保留原有大拖拽上传区或上传配置', ) assert.match( sourceUploadSource, - /]*>[\s\S]*?\s*\s*\{\{\s*processType === 'unstructured'\s*\? '支持 TXT、Markdown、PDF、Word、JSON、JSONL，单文件不超过 200MB'\s*:\s*'支持 JSON、JSONL、CSV、Excel，单文件不超过 200MB'\s*\}\}/, - '无文件时大拖拽上传区缺少格式提示槽、处理类型分支或完整格式提示', + /[\s\S]*?processType === 'unstructured'/, + '无文件时大拖拽上传区缺少按处理类型展示的格式提示', ) +assert.ok( + sourceUploadSource.includes("? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'"), + '非结构化上传 accept 不完整', +) +assert.ok( + sourceUploadSource.includes(": '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx'"), + '结构化上传 accept 不完整', +) +assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构化格式提示没有说明旧版 DOC/PPT 需转换') +assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换') assert.match( sourceUploadSource, - /\s*已添加 \{\{ uploadedFiles\.length \}\} 个文件<\/span>\s*\s* emit\('file-change', file\)"[^>]*>\s*继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/, + /[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?\s* emit\('file-change', file\)"[^>]*>\s*继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/, '有文件时缺少标题右侧的继续上传触发器或上传配置', ) assert.match( @@ -754,7 +992,20 @@ assert.match( '继续上传未覆盖内层上传节点的宽度和顶部间距', ) assert.match(sourceUploadSource, /\.uploaded-file\s*\{[^}]*min-height:\s*48px/, '文件行没有保持 48px 最小高度') -assert.match(sourceUploadSource, /]*\/> 校验通过<\/span>/, '文件行缺少校验成功状态') +assert.match(sourceUploadSource, /success:\s*\{ label: '切分完成', icon: 'fa-check-circle' \}/, '文件行缺少切分成功状态') +assert.match(sourceUploadSource, /previewBuilding/, '上传组件没有接收逐文件预览构建状态') +assert.match(sourceUploadSource, /sourceUploading/, '上传组件没有接收串行上传状态') +assert.match( + sourceUploadSource, + / del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`) -export function uploadDataProcessSourceFiles(taskId: string | number, files: File[]) { +export function uploadDataProcessSourceFiles( + taskId: string | number, + files: File[], + onProgress?: (progress: number) => void, +) { const formData = new FormData() files.forEach((file) => formData.append('files', file)) return post<{ files: DataProcessSourceFile[] }>( `/data-process/${encodeURIComponent(taskId)}/source-files`, formData, - { headers: { 'Content-Type': 'multipart/form-data' } }, + { + timeout: 5 * 60 * 1000, + onUploadProgress: (event) => { + if (!event.total) return + // 发送完成不等于服务端解析完成；收到成功响应前最多展示 99%。 + const progress = Math.round((event.loaded / event.total) * 100) + onProgress?.(Math.min(99, Math.max(0, progress))) + }, + }, ) } @@ -95,6 +113,18 @@ export const getDataProcessSourceContent = ( params, ) +export const getDataProcessSourceRawUrl = ( + taskId: string | number, + fileId: string | number, +) => `/modelTF/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/raw` + +export const getDataProcessPdfPages = ( + taskId: string | number, + fileId: string | number, +) => get( + `/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/pdf-pages`, +) + export const testDataProcessExternalSource = ( taskId: string | number, payload: DataProcessExternalSourcePayload, @@ -120,6 +150,7 @@ export const buildDataProcessPreview = ( ) => post( `/data-process/${encodeURIComponent(taskId)}/preview/build`, payload, + { timeout: 5 * 60 * 1000 }, ) export function getDataProcessPreview( diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 0998f04..ab5835d 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -93,6 +93,17 @@ export interface DataProcessSourceContent { total_chars?: number } +export interface DataProcessPdfPageRange { + page_number: number + source_start: number + source_end: number +} + +export interface DataProcessPdfPages { + page_count: number + pages: DataProcessPdfPageRange[] +} + export interface DataProcessExternalSourcePayload { type: 'postgresql' url: string @@ -129,7 +140,26 @@ export interface DataProcessPreviewBuildPayload { source_file_ids?: Array } -export type DataProcessPreviewBuildResult = DataProcessPage +export type DataProcessPreviewFileStatus = 'waiting' | 'processing' | 'success' | 'failed' + +export interface DataProcessPreviewBuildFileResult { + source_file_id: string | number + preview_count: number + status: 'completed' | 'empty' +} + +export interface DataProcessPreviewFileBuildProgress { + source_file_id: string | number + status: Exclude + progress: number + preview_count?: number + error?: string +} + +export interface DataProcessPreviewBuildResult extends DataProcessPage { + file_counts?: Record + files?: DataProcessPreviewBuildFileResult[] +} export interface DataProcessPreviewCreatePayload { source_file_id?: string | number | null diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 038da11..c381f91 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -15,29 +15,27 @@ import { createDefaultStructuredOptions, createDefaultUnstructuredOptions, } from './create/dataProcessCreateState' -import { - DATA_PROCESS_DRAFT_STORAGE_KEY, - useDataProcessDraft, -} from './create/useDataProcessDraft' import { useDataProcessGeneration } from './create/useDataProcessGeneration' +import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild' +import { + mapDataProcessSourceFile, + useDataProcessSourceUpload, + validateSourceFileSelection, +} from './create/useDataProcessSourceUpload' import { useModelsStore } from '@/stores/models' import { - buildDataProcessPreview, createDataProcessPreview, createDataProcessTask, deleteDataProcessPreview, deleteDataProcessSourceFile, getDataProcessPreview, getDataProcessSourceContent, - getDataProcessTask, pullDataProcessExternalSource, testDataProcessExternalSource, updateDataProcessPreview, updateDataProcessTask, - uploadDataProcessSourceFiles, type DataProcessExternalSourcePayload, type DataProcessPreviewItem, - type DataProcessSourceFile, } from '@/api/modules/dataProcess' import type { DataProcessConfig } from '@/types/dataProcess' import type { @@ -54,15 +52,12 @@ import type { const router = useRouter() const modelsStore = useModelsStore() const { list: modelList } = storeToRefs(modelsStore) -const generationModels = computed(() => modelList.value.filter((model) => ( - model.type === 'LLM' - && (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url)) -))) +// 候选范围与模型管理保持一致，不在数据处理页面重复定义模型过滤规则。 +const generationModels = computed(() => modelList.value) const taskSetupRef = ref>() const modelSelectionRef = ref>() const confirmDialogRef = ref>() -const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1' - +const PREVIEW_MODEL_VERSION = 'backend-pipeline-v2' const WIZARD_STEPS = [ { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' }, { id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' }, @@ -82,7 +77,8 @@ const modelSelectionOptions = computed(() => ( processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value )) const uploadedFiles = ref([]) - +const previewBuilding = ref(false) +const { buildPreviewsByFile } = useDataProcessPreviewBuild() const externalSource = reactive({ type: 'postgresql', url: '', @@ -95,7 +91,6 @@ const externalSource = reactive({ }) const externalPulling = ref(false) const externalConnected = ref(false) - const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', ')) const previewSignature = ref('') const previewItems = ref([]) @@ -103,7 +98,6 @@ const selectedPreviewFileId = ref(null) const selectedPreviewId = ref(null) const selectedPreviewIdsByFile = ref>({}) const dirty = ref(false) -const restoringDraft = ref(false) let allowLeave = false const { @@ -123,6 +117,19 @@ const { dirty, beforeGenerate: syncPreviewChanges, }) +const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({ + taskId, + uploadedFiles, + onUploaded(file) { + previewSignature.value = '' + resetDownstream() + dirty.value = true + ElMessage.success(`文件 ${file.name} 上传成功，等待切分`) + }, +}) +const hasUnfinishedUploads = computed(() => uploadedFiles.value.some((file) => ( + file.status !== 'ready' || !file.sourceFileId +))) const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length) const previewItemsByFile = computed(() => { @@ -149,7 +156,10 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => { const primaryActionLabel = computed(() => { if (currentStepId.value === 'create') return '继续：选择大模型' if (currentStepId.value === 'model') return '继续：上传文件' - if (currentStepId.value === 'upload') return '继续：数据预览' + if (currentStepId.value === 'upload') { + if (sourceUploading.value) return '正在上传' + return previewBuilding.value ? '正在切分' : '继续：数据预览' + } if (currentStepId.value === 'preview') return '确认预览并继续' if (currentStepId.value === 'results') return '保存任务' if (generation.status === 'running') return '正在生成' @@ -260,33 +270,6 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem { } } -function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile { - return { - uid: String(file.id), - sourceFileId: String(file.id), - name: file.name, - size: file.size_bytes, - count: file.record_count, - content, - fileFormat: file.file_format, - checksumSha256: file.checksum_sha256, - status: 'ready', - } -} - -const { persistDraft, restoreDraft } = useDataProcessDraft({ - taskId, - currentStepId, - task, - processType, - structuredOptions, - unstructuredOptions, - externalSource, - restoringDraft, - dirty, - goToStep, -}) - function previewAffectingOptions() { if (processType.value === 'structured') { return { @@ -368,92 +351,70 @@ function generationAffectingOptions() { const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions())) +function buildPreviewConfigSignature() { + return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}` +} + function buildPreviewSignature() { const filesSignature = uploadedFiles.value .map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`) .join('|') - return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}` + return `${buildPreviewConfigSignature()}:${filesSignature}` } watch( [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource], - () => { - if (!restoringDraft.value) dirty.value = true - }, + () => { dirty.value = true }, { deep: true }, ) watch(processType, (nextType, previousType) => { - if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return + if (nextType === previousType || uploadedFiles.value.length === 0) return resetSourceDataForProcessTypeChange() ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据') }) watch(generationOptionsSignature, (currentSignature, previousSignature) => { - if (restoringDraft.value || currentSignature === previousSignature) return + if (currentSignature === previousSignature) return resetDownstream() }) -watch( - [taskId, task, processType, structuredOptions, unstructuredOptions, externalSource], - persistDraft, - { deep: true }, -) - function scrollToStepTop() { document.querySelector('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' }) } watch(currentStep, () => nextTick(scrollToStepTop)) -async function handleFileChange(uploadFile: UploadFile) { +function handleFileChange(uploadFile: UploadFile) { const raw = uploadFile.raw if (!raw) return if (!taskId.value) { ElMessage.error('任务尚未创建，请返回模型选择步骤后重试') return } - if (raw.size > 200 * 1024 * 1024) { - ElMessage.warning('单文件不能超过 200MB') + const validation = validateSourceFileSelection(raw, processType.value, uploadedFiles.value) + if (!validation.valid) { + ElMessage[validation.severity](validation.message) return } - const extension = raw.name.split('.').pop()?.toLowerCase() ?? '' - const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv']) - if (!textExtensions.has(extension)) { - ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV；不会用示例内容替代无法解析的文件') - return - } - - if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) { - ElMessage.warning('同名且同大小的文件已经上传') - return - } - - let content = '' - try { - content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer()) - } catch { - ElMessage.error('文件不是有效的 UTF-8 文本，请转换编码后重试') - return - } - if (!content.trim()) { - ElMessage.warning('不能上传空文件') - return - } - - try { - const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw]) - const source = uploaded.files[0] - if (!source) throw new Error('后端未返回源文件记录') - uploadedFiles.value.push(mapSourceFile(source, content)) - previewSignature.value = '' - resetDownstream() - dirty.value = true - ElMessage.success(`文件 ${source.name} 上传成功`) - } catch { - // 请求层已展示后端的解析或格式错误。 - } + // 必须先同步插入文件行，再交给队列；这样选择完成后页面会立即展示全部文件。 + const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}` + uploadedFiles.value.push({ + uid: localUid, + rawFile: raw, + name: raw.name, + size: raw.size, + count: 0, + content: '', + fileFormat: validation.extension, + status: 'queued', + uploadProgress: 0, + previewStatus: 'waiting', + previewProgress: 0, + }) + dirty.value = true + enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension }) } async function useSampleFile() { @@ -462,32 +423,7 @@ async function useSampleFile() { return } const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' }) - await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile) -} - -async function restoreRegisteredSources() { - if (!taskId.value) return - try { - const savedTask = await getDataProcessTask(taskId.value) - const sources = savedTask.source_files || [] - const restoredFiles = await Promise.all(sources.map(async (file) => { - try { - const source = await getDataProcessSourceContent(taskId.value!, file.id, { - start_line: 1, - line_count: 5000, - }) - return mapSourceFile(file, source.content) - } catch { - return mapSourceFile(file) - } - })) - uploadedFiles.value = restoredFiles - if (restoredFiles.length) { - ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`) - } - } catch { - ElMessage.warning('草稿任务暂时无法从后端同步，请检查服务后重试') - } + handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile) } function updateExternalSource(value: ExternalDataSource) { @@ -539,7 +475,7 @@ async function handlePullData() { start_line: 1, line_count: 5000, }) - newFiles.push(mapSourceFile(file, source.content)) + newFiles.push(mapDataProcessSourceFile(file, source.content)) } uploadedFiles.value.push(...newFiles) externalConnected.value = true @@ -557,11 +493,19 @@ async function handlePullData() { async function handleRemoveFile(uid: string | number) { const index = uploadedFiles.value.findIndex(f => f.uid === uid) if (index < 0 || !taskId.value) return - try { - await deleteDataProcessSourceFile(taskId.value, uid) - } catch { + const file = uploadedFiles.value[index] + if (!file) return + if (file.status === 'uploading') { + ElMessage.warning('当前文件正在上传，请等待上传结束后再删除') return } + if (file.sourceFileId) { + try { + await deleteDataProcessSourceFile(taskId.value, file.sourceFileId) + } catch { + return + } + } uploadedFiles.value.splice(index, 1) previewSignature.value = '' previewItems.value = [] @@ -596,7 +540,6 @@ async function nextFromModel() { : await createDataProcessTask(taskPayload()) taskId.value = String(saved.id) dirty.value = true - persistDraft() goToStep('upload') } catch { // 请求层已展示名称冲突或配置非法等具体原因。 @@ -604,6 +547,7 @@ async function nextFromModel() { } async function nextFromUpload() { + if (previewBuilding.value || sourceUploading.value) return if (!taskId.value) { ElMessage.error('任务尚未创建，请返回模型选择步骤后重试') return @@ -612,13 +556,61 @@ async function nextFromUpload() { ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件') return } + const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed') + if (failedUploads.length) { + ElMessage.warning(`${failedUploads.length} 个文件上传失败，请删除后重新选择`) + return + } + if (hasUnfinishedUploads.value) { + ElMessage.warning('请等待所有文件上传完成后再开始切分') + return + } const signature = buildPreviewSignature() - if (signature !== previewSignature.value) { + const configSignature = buildPreviewConfigSignature() + const allFilesSucceeded = () => uploadedFiles.value.every((file) => ( + file.previewStatus === 'success' && file.previewConfigSignature === configSignature + )) + + if (signature === previewSignature.value && previewItems.value.length && allFilesSucceeded()) { + goToStep('preview') + return + } + + for (const file of uploadedFiles.value) { + if (file.previewConfigSignature === configSignature) continue + file.previewStatus = 'waiting' + file.previewProgress = 0 + file.previewError = undefined + file.previewCount = undefined + } + + previewBuilding.value = true + try { + const pendingFileIds = uploadedFiles.value + .filter((file) => file.previewStatus !== 'success' || file.previewConfigSignature !== configSignature) + .map((file) => file.sourceFileId) + .filter((fileId): fileId is string => Boolean(fileId)) + // 构建接口是同步响应：请求中只展示真实的“处理中”，响应成功后才记为 100%。 + await buildPreviewsByFile(taskId.value, pendingFileIds, (progress) => { + const file = uploadedFiles.value.find((item) => ( + String(item.sourceFileId) === String(progress.source_file_id) + )) + if (!file) return + file.previewStatus = progress.status + file.previewProgress = progress.progress + file.previewCount = progress.preview_count + file.previewError = progress.error + if (progress.status === 'success') file.previewConfigSignature = configSignature + }) + + const failedCount = uploadedFiles.value.filter((file) => file.previewStatus === 'failed').length + if (failedCount || !allFilesSucceeded()) { + ElMessage.warning(`${failedCount || 1} 个文件切分失败；修正问题后点击“继续：数据预览”即可重试`) + return + } + try { - await buildDataProcessPreview(taskId.value, { - source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid), - }) const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 }) const items = [...first.items] const pages = Math.ceil(first.total / first.page_size) @@ -628,6 +620,7 @@ async function nextFromUpload() { } previewItems.value = items.map(mapPreviewItem) } catch { + ElMessage.warning('文件切分已经完成，但预览加载失败；请再次点击继续重试加载') return } if (!previewItems.value.length) { @@ -641,8 +634,10 @@ async function nextFromUpload() { : {} previewSignature.value = signature resetDownstream() + goToStep('preview') + } finally { + previewBuilding.value = false } - goToStep('preview') } function selectPreviewFile(fileId: string) { @@ -770,6 +765,14 @@ async function handlePrimaryAction() { } function handleBack() { + if (sourceUploading.value) { + ElMessage.warning('请等待当前文件上传完成') + return + } + if (previewBuilding.value) { + ElMessage.warning('请等待当前文件切分完成') + return + } if (generation.status === 'running') { ElMessage.warning('请先停止当前生成任务') return @@ -793,7 +796,6 @@ async function saveTask() { return } dirty.value = false - localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY) allowLeave = true ElMessage.success('数据处理任务已保存') await router.push('/data-process') @@ -818,6 +820,10 @@ async function handleCancel() { } onBeforeRouteLeave(async () => { + if (sourceUploading.value) { + ElMessage.warning('请等待当前文件上传完成后再离开页面') + return false + } if (allowLeave || !dirty.value) return true const confirmed = await confirmDialogRef.value?.open({ title: '确认离开当前页面？', @@ -833,9 +839,9 @@ onBeforeRouteLeave(async () => { onBeforeUnmount(() => { stopGenerationTimer() }) -onMounted(async () => { - restoreDraft() - await Promise.all([modelsStore.load(), restoreRegisteredSources()]) +onMounted(() => { + localStorage.removeItem('yg-data-process-create-draft') + void modelsStore.load(true) }) @@ -899,6 +905,8 @@ onMounted(async () => { :external-source="externalSource" :external-pulling="externalPulling" :external-connected="externalConnected" + :preview-building="previewBuilding" + :source-uploading="sourceUploading" @update:external-source="updateExternalSource" @file-change="handleFileChange" @remove-file="handleRemoveFile" @@ -915,6 +923,8 @@ onMounted(async () => { :items="activePreviewItems" :process-type="processType" :file-name="activePreviewFile?.name ?? ''" + :task-id="taskId" + :source-file-id="activePreviewFile?.sourceFileId ?? activePreviewFile?.uid ?? null" :files="previewFiles" @update:selected-id="selectPreviewItem" @update:selected-file-id="selectPreviewFile" @@ -949,7 +959,7 @@ onMounted(async () => {