feat(data-process): 优化上传切分流程与PDF高亮预览
This commit is contained in:
caoxiaozhu
263
frontend/package-lock.json
generated
263
frontend/package-lock.json
generated
@@ -16,6 +16,7 @@
|
|||||||
"element-plus": "^2.9.1",
|
"element-plus": "^2.9.1",
|
||||||
"marked": "^15.0.5",
|
"marked": "^15.0.5",
|
||||||
"md-editor-v3": "^5.1.4",
|
"md-editor-v3": "^5.1.4",
|
||||||
|
"pdfjs-dist": "^6.1.200",
|
||||||
"pinia": "^2.3.0",
|
"pinia": "^2.3.0",
|
||||||
"vue": "^3.5.13",
|
"vue": "^3.5.13",
|
||||||
"vue-echarts": "^8.0.1",
|
"vue-echarts": "^8.0.1",
|
||||||
@@ -1213,6 +1214,256 @@
|
|||||||
"integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==",
|
"integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@napi-rs/canvas": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"workspaces": [
|
||||||
|
"e2e/*"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@napi-rs/canvas-android-arm64": "1.0.2",
|
||||||
|
"@napi-rs/canvas-darwin-arm64": "1.0.2",
|
||||||
|
"@napi-rs/canvas-darwin-x64": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-arm64-gnu": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-arm64-musl": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-riscv64-gnu": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-x64-gnu": "1.0.2",
|
||||||
|
"@napi-rs/canvas-linux-x64-musl": "1.0.2",
|
||||||
|
"@napi-rs/canvas-win32-arm64-msvc": "1.0.2",
|
||||||
|
"@napi-rs/canvas-win32-x64-msvc": "1.0.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"android"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==",
|
||||||
|
"cpu": [
|
||||||
|
"riscv64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@nodelib/fs.scandir": {
|
"node_modules/@nodelib/fs.scandir": {
|
||||||
"version": "2.1.5",
|
"version": "2.1.5",
|
||||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||||
@@ -3443,6 +3694,18 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/pdfjs-dist": {
|
||||||
|
"version": "6.1.200",
|
||||||
|
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.1.200.tgz",
|
||||||
|
"integrity": "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=22.13.0 || >=24"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@napi-rs/canvas": "^1.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/picocolors": {
|
"node_modules/picocolors": {
|
||||||
"version": "1.1.1",
|
"version": "1.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||||
|
|||||||
@@ -36,6 +36,7 @@
|
|||||||
"element-plus": "^2.9.1",
|
"element-plus": "^2.9.1",
|
||||||
"marked": "^15.0.5",
|
"marked": "^15.0.5",
|
||||||
"md-editor-v3": "^5.1.4",
|
"md-editor-v3": "^5.1.4",
|
||||||
|
"pdfjs-dist": "^6.1.200",
|
||||||
"pinia": "^2.3.0",
|
"pinia": "^2.3.0",
|
||||||
"vue": "^3.5.13",
|
"vue": "^3.5.13",
|
||||||
"vue-echarts": "^8.0.1",
|
"vue-echarts": "^8.0.1",
|
||||||
|
|||||||
@@ -12,17 +12,19 @@ const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmD
|
|||||||
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
|
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
|
||||||
const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts')
|
const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts')
|
||||||
const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts')
|
const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts')
|
||||||
|
const sourceUploadWorkerPath = path.join(createDir, 'useDataProcessSourceUpload.ts')
|
||||||
const viewSource = await readFile(viewPath, 'utf8')
|
const viewSource = await readFile(viewPath, 'utf8')
|
||||||
const layoutSource = await readFile(layoutPath, 'utf8')
|
const layoutSource = await readFile(layoutPath, 'utf8')
|
||||||
const [draftSource, stateSource, generationSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([
|
const [stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([
|
||||||
readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'),
|
|
||||||
readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
|
readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
|
||||||
readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
|
readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
|
||||||
|
readFile(path.join(createDir, 'useDataProcessPreviewBuild.ts'), 'utf8'),
|
||||||
|
readFile(sourceUploadWorkerPath, 'utf8'),
|
||||||
readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
|
readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
|
||||||
readFile(apiPath, 'utf8'),
|
readFile(apiPath, 'utf8'),
|
||||||
readFile(contractTypesPath, 'utf8'),
|
readFile(contractTypesPath, 'utf8'),
|
||||||
])
|
])
|
||||||
const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n')
|
const implementationSource = [viewSource, stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource].join('\n')
|
||||||
|
|
||||||
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
|
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
|
||||||
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
|
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
|
||||||
@@ -58,9 +60,13 @@ assert.match(
|
|||||||
/@media \(max-width: 1100px\)[\s\S]*?\.step-title\s*\{[\s\S]*?display:\s*none[\s\S]*?\.step-item\.is-active \.step-title\s*\{[\s\S]*?display:\s*block/,
|
/@media \(max-width: 1100px\)[\s\S]*?\.step-title\s*\{[\s\S]*?display:\s*none[\s\S]*?\.step-item\.is-active \.step-title\s*\{[\s\S]*?display:\s*block/,
|
||||||
'六步向导在中等宽度下没有收起非当前步骤标题',
|
'六步向导在中等宽度下没有收起非当前步骤标题',
|
||||||
)
|
)
|
||||||
assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化')
|
assert.equal(existsSync(path.join(createDir, 'useDataProcessDraft.ts')), false, '不应保留新建任务草稿模块')
|
||||||
assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取')
|
assert.doesNotMatch(
|
||||||
assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿')
|
viewSource,
|
||||||
|
/useDataProcessDraft|persistDraft|restoreDraft|restoringDraft|localStorage\.(?:setItem|getItem)/,
|
||||||
|
'新建任务不应保存或恢复草稿',
|
||||||
|
)
|
||||||
|
assert.match(viewSource, /localStorage\.removeItem\('yg-data-process-create-draft'\)/, '进入新建页时应清理遗留草稿')
|
||||||
assert.ok(viewSource.split('\n').length < 1000, 'DataProcessCreateView 拆分后仍超过 1000 行')
|
assert.ok(viewSource.split('\n').length < 1000, 'DataProcessCreateView 拆分后仍超过 1000 行')
|
||||||
assert.match(viewSource, /useDataProcessGeneration\(\{/, '生成流程没有拆分到独立 composable')
|
assert.match(viewSource, /useDataProcessGeneration\(\{/, '生成流程没有拆分到独立 composable')
|
||||||
|
|
||||||
@@ -79,13 +85,16 @@ for (const component of expectedComponents) {
|
|||||||
|
|
||||||
const typesPath = path.join(createDir, 'types.ts')
|
const typesPath = path.join(createDir, 'types.ts')
|
||||||
const modelPath = path.join(createDir, 'previewModel.ts')
|
const modelPath = path.join(createDir, 'previewModel.ts')
|
||||||
|
const pdfViewerPath = path.join(createDir, 'PdfSourceViewer.vue')
|
||||||
assert.ok(existsSync(typesPath), '缺少向导类型定义')
|
assert.ok(existsSync(typesPath), '缺少向导类型定义')
|
||||||
assert.ok(existsSync(modelPath), '缺少来源映射模型')
|
assert.ok(existsSync(modelPath), '缺少来源映射模型')
|
||||||
|
assert.ok(existsSync(pdfViewerPath), '缺少 PDF 原文件预览组件')
|
||||||
|
|
||||||
const [typesSource, modelSource, previewSource] = await Promise.all([
|
const [typesSource, modelSource, previewSource, pdfViewerSource] = await Promise.all([
|
||||||
readFile(typesPath, 'utf8'),
|
readFile(typesPath, 'utf8'),
|
||||||
readFile(modelPath, 'utf8'),
|
readFile(modelPath, 'utf8'),
|
||||||
readFile(path.join(createDir, 'PreviewCompareStep.vue'), 'utf8'),
|
readFile(path.join(createDir, 'PreviewCompareStep.vue'), 'utf8'),
|
||||||
|
readFile(pdfViewerPath, 'utf8'),
|
||||||
])
|
])
|
||||||
|
|
||||||
for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedContent']) {
|
for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedContent']) {
|
||||||
@@ -98,8 +107,8 @@ assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后
|
|||||||
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
||||||
assert.match(
|
assert.match(
|
||||||
viewSource,
|
viewSource,
|
||||||
/buildDataProcessPreview\(taskId\.value,\s*\{[\s\S]*?source_file_ids:\s*uploadedFiles\.value\.map/,
|
/const \{ buildPreviewsByFile \} = useDataProcessPreviewBuild\(\)/,
|
||||||
'预览没有通过后端按已上传源文件构建',
|
'父页面没有通过独立 composable 构建逐文件预览',
|
||||||
)
|
)
|
||||||
assert.match(viewSource, /getDataProcessPreview\(taskId\.value,\s*\{ page:\s*1, page_size:\s*500 \}\)/, '预览构建后没有分页读取后端数据')
|
assert.match(viewSource, /getDataProcessPreview\(taskId\.value,\s*\{ page:\s*1, page_size:\s*500 \}\)/, '预览构建后没有分页读取后端数据')
|
||||||
assert.doesNotMatch(viewSource, /buildPreviewItems\(/, '创建向导仍在本地构建集成预览数据')
|
assert.doesNotMatch(viewSource, /buildPreviewItems\(/, '创建向导仍在本地构建集成预览数据')
|
||||||
@@ -142,6 +151,21 @@ assert.match(previewSource, /\.editor-actions\s*\{[\s\S]*?justify-content:\s*fle
|
|||||||
assert.doesNotMatch(previewSource, /item-token|item-status|modifiedOnly|仅看已修改/, '切片列表不应再显示 Token 或修改状态')
|
assert.doesNotMatch(previewSource, /item-token|item-status|modifiedOnly|仅看已修改/, '切片列表不应再显示 Token 或修改状态')
|
||||||
assert.match(previewSource, /\.preview-editor\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?overflow-y:\s*auto/, '编辑模式必须占据右侧剩余区域并可滚动')
|
assert.match(previewSource, /\.preview-editor\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?overflow-y:\s*auto/, '编辑模式必须占据右侧剩余区域并可滚动')
|
||||||
assert.match(previewSource, /@media \(max-width: 900px\)/, '第四步缺少窄屏上下布局')
|
assert.match(previewSource, /@media \(max-width: 900px\)/, '第四步缺少窄屏上下布局')
|
||||||
|
assert.match(previewSource, /<PdfSourceViewer[\s\S]*v-if="isPdfSource"/, 'PDF 文件没有切换到原文件查看组件')
|
||||||
|
assert.match(previewSource, /:selected-item="selectedItem \?\? null"/, 'PDF 查看组件没有接收当前选中切片')
|
||||||
|
assert.match(previewSource, /:data-preview-id="item\.id"/, '切片行缺少稳定的交互定位标识')
|
||||||
|
assert.match(previewSource, /<div v-else ref="sourceViewerRef" class="source-viewer"/, '非 PDF 文件没有保留文本预览')
|
||||||
|
assert.match(pdfViewerSource, /getDataProcessSourceRawUrl/, 'PDF 查看组件没有使用受控原文件地址')
|
||||||
|
assert.match(pdfViewerSource, /getDataProcessPdfPages/, 'PDF 查看组件没有读取页码与全文偏移映射')
|
||||||
|
assert.match(pdfViewerSource, /pdfjs-dist/, 'PDF 查看组件没有使用可控的 PDF.js 渲染器')
|
||||||
|
assert.match(pdfViewerSource, /TextLayer/, 'PDF 查看组件没有渲染可定位的 PDF 文字层')
|
||||||
|
assert.match(pdfViewerSource, /is-slice-highlighted/, 'PDF 查看组件没有实现选中切片高亮')
|
||||||
|
assert.match(pdfViewerSource, /:data-page-number="currentPage"/, 'PDF 查看组件缺少当前物理页标识')
|
||||||
|
assert.doesNotMatch(pdfViewerSource, /<iframe/, 'PDF 查看组件不应继续使用无法控制高亮的浏览器 iframe')
|
||||||
|
assert.match(pdfViewerSource, /:aria-label="`PDF 预览：\$\{fileName\}`"/, 'PDF 查看器缺少可访问标题')
|
||||||
|
assert.match(apiSource, /getDataProcessSourceRawUrl/, '前端 API 缺少 PDF 原文件预览地址')
|
||||||
|
assert.match(apiSource, /getDataProcessPdfPages/, '前端 API 缺少 PDF 页码映射接口')
|
||||||
|
assert.match(apiSource, /source-files\/\$\{encodeURIComponent\(fileId\)\}\/pdf-pages/, 'PDF 页码映射接口地址不正确')
|
||||||
|
|
||||||
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
||||||
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
|
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
|
||||||
@@ -190,11 +214,11 @@ assert.match(viewSource, /<ModelSelectionStep\s+[\s\S]*?v-else-if="currentStepId
|
|||||||
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
|
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
|
||||||
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：选择大模型'/, '第一步主按钮没有指向大模型选择')
|
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：选择大模型'/, '第一步主按钮没有指向大模型选择')
|
||||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：上传文件'/, '第二步主按钮没有指向上传文件')
|
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：上传文件'/, '第二步主按钮没有指向上传文件')
|
||||||
assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续：数据预览'/, '第三步主按钮没有指向数据预览')
|
assert.match(
|
||||||
const draftVersionMatch = draftSource.match(/DATA_PROCESS_DRAFT_SCHEMA_VERSION\s*=\s*(\d+)/)
|
viewSource,
|
||||||
assert.ok(draftVersionMatch, '草稿缺少数字版本标识')
|
/if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续：数据预览'/,
|
||||||
assert.ok(Number(draftVersionMatch[1]) >= 7, '安全草稿格式版本不得低于 v7')
|
'第三步主按钮没有依次反映上传、切分状态并指向数据预览',
|
||||||
|
)
|
||||||
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
|
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
|
||||||
const nextFromModelStart = viewSource.indexOf('async function nextFromModel()', nextFromCreateStart)
|
const nextFromModelStart = viewSource.indexOf('async function nextFromModel()', nextFromCreateStart)
|
||||||
const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromModelStart)
|
const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromModelStart)
|
||||||
@@ -213,36 +237,95 @@ assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, '
|
|||||||
assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务')
|
assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务')
|
||||||
assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件')
|
assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件')
|
||||||
assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据')
|
assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据')
|
||||||
assert.match(nextFromUploadSource, /buildDataProcessPreview\(/, '上传步骤没有调用后端构建预览')
|
assert.match(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/const pendingFileIds = uploadedFiles\.value[\s\S]*?\.filter\(\(file\) => file\.previewStatus !== 'success' \|\| file\.previewConfigSignature !== configSignature\)[\s\S]*?\.map\(\(file\) => file\.sourceFileId\)[\s\S]*?\.filter\(\(fileId\): fileId is string => Boolean\(fileId\)\)/,
|
||||||
|
'上传步骤没有仅选择待处理或失败文件，无法跳过成功文件并重试失败文件',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/await buildPreviewsByFile\(taskId\.value, pendingFileIds, \(progress\) => \{/,
|
||||||
|
'上传步骤没有通过独立 composable 串行构建待处理文件',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/file\.previewStatus = progress\.status[\s\S]*?file\.previewProgress = progress\.progress[\s\S]*?file\.previewError = progress\.error/,
|
||||||
|
'上传步骤没有把单文件构建进度与错误回写到对应文件',
|
||||||
|
)
|
||||||
|
assert.doesNotMatch(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/source_file_ids:\s*uploadedFiles\.value\.map/,
|
||||||
|
'上传步骤仍一次性批量构建全部文件预览',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
previewBuildSource,
|
||||||
|
/for \(const sourceFileId of sourceFileIds\)[\s\S]*?onProgress\(\{ source_file_id: sourceFileId, status: 'processing', progress: 0 \}\)/,
|
||||||
|
'逐文件构建 composable 没有串行处理并先上报 processing 进度',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
previewBuildSource,
|
||||||
|
/buildDataProcessPreview\(taskId,\s*\{[\s\S]*?replace_existing:\s*true[\s\S]*?source_file_ids:\s*\[sourceFileId\]/,
|
||||||
|
'逐文件构建 composable 没有按单个源文件 ID 替换预览',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
previewBuildSource,
|
||||||
|
/status: 'success', progress: 100, preview_count: previewCount/,
|
||||||
|
'单文件构建成功后没有上报 100% 和预览数量',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
previewBuildSource,
|
||||||
|
/catch \(error\)[\s\S]*?status: 'failed'[\s\S]*?progress: 0[\s\S]*?error:/,
|
||||||
|
'单文件构建失败后没有保留可重试错误',
|
||||||
|
)
|
||||||
assert.match(nextFromUploadSource, /getDataProcessPreview\(/, '上传步骤没有读取后端预览结果')
|
assert.match(nextFromUploadSource, /getDataProcessPreview\(/, '上传步骤没有读取后端预览结果')
|
||||||
assert.match(nextFromUploadSource, /goToStep\('preview'\)/, '上传步骤完成后没有进入数据预览')
|
assert.match(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/const allFilesSucceeded = \(\) => uploadedFiles\.value\.every\(\(file\) => \([\s\S]*?file\.previewStatus === 'success'[\s\S]*?file\.previewConfigSignature === configSignature/,
|
||||||
|
'上传步骤缺少全部文件 success 且配置签名一致的完成判定',
|
||||||
|
)
|
||||||
|
const previewSuccessGateIndex = nextFromUploadSource.indexOf('if (failedCount || !allFilesSucceeded())')
|
||||||
|
const previewStepIndex = nextFromUploadSource.lastIndexOf("goToStep('preview')")
|
||||||
|
assert.ok(previewSuccessGateIndex >= 0, '上传步骤没有在任一文件未成功时停留当前步骤')
|
||||||
|
assert.match(
|
||||||
|
nextFromUploadSource,
|
||||||
|
/if \(signature === previewSignature\.value && previewItems\.value\.length && allFilesSucceeded\(\)\) \{\s*goToStep\('preview'\)/,
|
||||||
|
'缓存预览快速路径没有要求全部文件 success',
|
||||||
|
)
|
||||||
|
assert.equal(
|
||||||
|
(nextFromUploadSource.match(/goToStep\('preview'\)/g) || []).length,
|
||||||
|
2,
|
||||||
|
'上传步骤只能通过全成功缓存路径或本轮全成功路径进入预览',
|
||||||
|
)
|
||||||
|
assert.ok(
|
||||||
|
previewStepIndex > previewSuccessGateIndex,
|
||||||
|
'上传步骤必须在全部文件 success 后才能进入数据预览',
|
||||||
|
)
|
||||||
assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识')
|
assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识')
|
||||||
assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成')
|
assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成')
|
||||||
assert.match(viewSource, /generation\.status === 'success'[\s\S]*?goToStep\('results'\)/, '生成成功后没有进入结果编辑与保存')
|
assert.match(viewSource, /generation\.status === 'success'[\s\S]*?goToStep\('results'\)/, '生成成功后没有进入结果编辑与保存')
|
||||||
assert.match(draftSource, /currentStepId:\s*bindings\.currentStepId\.value/, '草稿没有保存稳定步骤标识')
|
|
||||||
const draftSnapshotSource = draftSource.slice(
|
|
||||||
draftSource.indexOf('function draftSnapshot()'),
|
|
||||||
draftSource.indexOf('function writeDraft'),
|
|
||||||
)
|
|
||||||
for (const forbiddenField of ['uploadedFiles', 'previewItems', 'results', 'password', 'token']) {
|
|
||||||
assert.ok(!draftSnapshotSource.includes(forbiddenField), `安全草稿不应持久化：${forbiddenField}`)
|
|
||||||
}
|
|
||||||
assert.match(draftSource, /writeDraft\(false\)/, '恢复旧草稿后没有立即覆盖潜在敏感数据')
|
|
||||||
assert.match(viewSource, /<ResultEditorStep\s+[\s\S]*?v-else-if="currentStepId === 'results'"/, '结果编辑器必须只在结果步骤渲染')
|
assert.match(viewSource, /<ResultEditorStep\s+[\s\S]*?v-else-if="currentStepId === 'results'"/, '结果编辑器必须只在结果步骤渲染')
|
||||||
assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTypeChange\(\)/, '切换处理类型后没有失效旧源数据')
|
assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTypeChange\(\)/, '切换处理类型后没有失效旧源数据')
|
||||||
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
|
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
|
||||||
|
|
||||||
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
||||||
for (const option of [
|
const expectedStructuredOptions = [
|
||||||
'清理无效数据',
|
['clean_invalid', '清理无效数据', '清理全空列，并剔除关键字段残缺的数据行'],
|
||||||
'识别表格结构',
|
['detect_structure', '识别表格结构', '识别多级表头与合并单元格，并将嵌套字段展平'],
|
||||||
'重复数据去重',
|
['deduplicate', '重复数据去重', '基于整行精确匹配和关键字段组合删除重复记录'],
|
||||||
'数据格式标准化',
|
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
|
||||||
'异常数据过滤',
|
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值，并过滤乱码等异常记录'],
|
||||||
'敏感信息脱敏',
|
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||||
]) {
|
]
|
||||||
assert.ok(structuredOptionsSource.includes(option), `结构化预处理缺少选项：${option}`)
|
for (const [value, label, description] of expectedStructuredOptions) {
|
||||||
|
assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值：${value}`)
|
||||||
|
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签：${label}`)
|
||||||
|
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确：${value}`)
|
||||||
}
|
}
|
||||||
|
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{ value: '([^']+)', label:/g)]
|
||||||
|
.map((match) => match[1])
|
||||||
|
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
|
||||||
|
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
|
||||||
|
assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重')
|
||||||
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
|
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
|
||||||
for (const splitName of ['训练集', '验证集', '测试集']) {
|
for (const splitName of ['训练集', '验证集', '测试集']) {
|
||||||
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`)
|
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`)
|
||||||
@@ -260,8 +343,6 @@ for (const splitField of ['train', 'validation', 'test']) {
|
|||||||
assert.match(structuredOptionsSource, /<el-input-number[\s\S]*options\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
|
assert.match(structuredOptionsSource, /<el-input-number[\s\S]*options\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
|
||||||
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
|
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
|
||||||
assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
|
assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
|
||||||
assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿')
|
|
||||||
assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
|
|
||||||
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
|
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
|
||||||
assert.match(generationSource, /generateDataProcess\(taskId\)/, '开始生成没有调用真实 API')
|
assert.match(generationSource, /generateDataProcess\(taskId\)/, '开始生成没有调用真实 API')
|
||||||
assert.match(generationSource, /getDataProcessProgress\(taskId\)/, '生成状态没有通过真实 API 轮询')
|
assert.match(generationSource, /getDataProcessProgress\(taskId\)/, '生成状态没有通过真实 API 轮询')
|
||||||
@@ -284,13 +365,39 @@ for (const apiName of [
|
|||||||
]) {
|
]) {
|
||||||
assert.match(apiSource, new RegExp(`export (?:const|async function|function) ${apiName}\\b`), `API 模块缺少 ${apiName}`)
|
assert.match(apiSource, new RegExp(`export (?:const|async function|function) ${apiName}\\b`), `API 模块缺少 ${apiName}`)
|
||||||
}
|
}
|
||||||
assert.match(viewSource, /uploadDataProcessSourceFiles\(taskId\.value,\s*\[raw\]\)/, '文件上传没有调用真实 API')
|
assert.match(sourceUploadWorkerSource, /uploadDataProcessSourceFiles\(currentTaskId,\s*\[job\.file\]/, '文件上传没有逐文件调用真实 API')
|
||||||
|
assert.match(sourceUploadWorkerSource, /STRUCTURED_FILE_EXTENSIONS = new Set\(\['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'\]\)/, '结构化文件扩展名白名单不完整')
|
||||||
|
for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson']) {
|
||||||
|
assert.ok(sourceUploadWorkerSource.includes(`'${extension}'`), `非结构化文件扩展名白名单缺少 ${extension}`)
|
||||||
|
}
|
||||||
|
assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别')
|
||||||
|
assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示')
|
||||||
|
assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验')
|
||||||
|
assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本')
|
||||||
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
|
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
|
||||||
|
assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%')
|
||||||
|
assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时')
|
||||||
assert.match(apiSource, /\/preview\/build/, 'API 模块缺少后端预览构建路径')
|
assert.match(apiSource, /\/preview\/build/, 'API 模块缺少后端预览构建路径')
|
||||||
|
assert.match(apiSource, /\/preview\/build`[\s\S]*?\{ timeout: 5 \* 60 \* 1000 \}/, '单文件切分请求缺少 5 分钟超时')
|
||||||
assert.match(apiSource, /\/progress`/, 'API 模块缺少生成进度路径')
|
assert.match(apiSource, /\/progress`/, 'API 模块缺少生成进度路径')
|
||||||
assert.match(apiSource, /\/results`/, 'API 模块缺少结果分页路径')
|
assert.match(apiSource, /\/results`/, 'API 模块缺少结果分页路径')
|
||||||
assert.match(apiSource, /\/publish`/, 'API 模块缺少数据集发布路径')
|
assert.match(apiSource, /\/publish`/, 'API 模块缺少数据集发布路径')
|
||||||
assert.match(contractTypesSource, /source_file_ids\?: Array<string \| number>/, '预览构建契约缺少源文件 ID 列表')
|
assert.match(contractTypesSource, /source_file_ids\?: Array<string \| number>/, '预览构建契约缺少源文件 ID 列表')
|
||||||
|
assert.match(
|
||||||
|
contractTypesSource,
|
||||||
|
/export type DataProcessPreviewFileStatus = 'waiting' \| 'processing' \| 'success' \| 'failed'/,
|
||||||
|
'文件预览状态契约不完整',
|
||||||
|
)
|
||||||
|
for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
|
||||||
|
assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段：${field}`)
|
||||||
|
}
|
||||||
|
assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整')
|
||||||
|
assert.match(viewSource, /uploadedFiles\.value\.push\([\s\S]*?status: 'queued'[\s\S]*?enqueueSourceUpload/, '文件选择后没有先进入列表再加入上传队列')
|
||||||
|
assert.match(sourceUploadWorkerSource, /while \(queue\.length\) \{[\s\S]*?await uploadOne\(job\)/, '多文件上传没有由单一队列逐个等待')
|
||||||
|
assert.doesNotMatch(sourceUploadWorkerSource, /Promise\.(?:all|allSettled)/, '上传队列不得并发消费文件')
|
||||||
|
assert.match(sourceUploadWorkerSource, /pending\.status = 'ready'[\s\S]*?pending\.uploadProgress = 100/, '服务端响应成功后没有将文件置为上传完成')
|
||||||
|
assert.match(viewSource, /failedUploads[\s\S]*?hasUnfinishedUploads[\s\S]*?buildPreviewsByFile/, '上传失败或未完成时没有阻断切分')
|
||||||
|
assert.doesNotMatch(viewSource, /file\.sourceFileId \|\| file\.uid/, '切分或删除仍可能把本地临时 UID 当成后端文件 ID')
|
||||||
assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段')
|
assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段')
|
||||||
|
|
||||||
for (const field of [
|
for (const field of [
|
||||||
@@ -302,7 +409,7 @@ for (const field of [
|
|||||||
'minOutputLength',
|
'minOutputLength',
|
||||||
]) {
|
]) {
|
||||||
assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`)
|
assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`)
|
||||||
assert.ok(implementationSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`)
|
assert.ok(implementationSource.includes(field), `父页面默认值缺少字段：${field}`)
|
||||||
}
|
}
|
||||||
assert.match(structuredOptionsSource, /GenerationOptionsPanel/, '结构化生成选项没有复用统一的质量筛选组件')
|
assert.match(structuredOptionsSource, /GenerationOptionsPanel/, '结构化生成选项没有复用统一的质量筛选组件')
|
||||||
assert.match(unstructuredOptionsSource, /GenerationOptionsPanel/, '非结构化生成选项没有复用统一的质量筛选组件')
|
assert.match(unstructuredOptionsSource, /GenerationOptionsPanel/, '非结构化生成选项没有复用统一的质量筛选组件')
|
||||||
@@ -336,10 +443,23 @@ assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最
|
|||||||
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
|
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
|
||||||
assert.match(taskSetupSource, /qualityValidationMessage/, '质量规则缺少继续前校验')
|
assert.match(taskSetupSource, /qualityValidationMessage/, '质量规则缺少继续前校验')
|
||||||
assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表')
|
assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表')
|
||||||
assert.match(viewSource, /model\.type === 'LLM'/, '数据生成模型列表没有排除非大模型')
|
const generationModelsStart = viewSource.indexOf('const generationModels')
|
||||||
|
const generationModelsEnd = viewSource.indexOf('const taskSetupRef', generationModelsStart)
|
||||||
|
const generationModelsSource = viewSource.slice(generationModelsStart, generationModelsEnd)
|
||||||
|
assert.match(generationModelsSource, /computed\(\(\) => modelList\.value\)/, '数据生成模型候选没有直接使用模型管理完整列表')
|
||||||
|
assert.doesNotMatch(generationModelsSource, /\.filter\(|model\.type|model\.model_source|model\.status|model\.purpose/, '数据生成模型候选仍按类型、来源、状态或用途静默过滤')
|
||||||
|
assert.match(viewSource, /modelsStore\.load\(true\)/, '进入创建向导时没有强制刷新模型管理列表')
|
||||||
assert.match(viewSource, /<ModelSelectionStep[\s\S]*?:models="generationModels"/, '创建页没有向独立大模型选择步骤传递模型列表')
|
assert.match(viewSource, /<ModelSelectionStep[\s\S]*?:models="generationModels"/, '创建页没有向独立大模型选择步骤传递模型列表')
|
||||||
|
assert.match(generationControlSource, /v-for="model in models \|\| \[\]"/, '模型下拉没有遍历完整候选列表')
|
||||||
|
assert.match(generationControlSource, /:value="model\.id"/, '模型下拉没有使用模型管理 ID 作为选中值')
|
||||||
|
assert.match(generationControlSource, /modelMeta\(model\)/, '模型下拉缺少来源和类型说明')
|
||||||
|
|
||||||
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
|
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
|
||||||
|
assert.match(
|
||||||
|
typesSource,
|
||||||
|
/export type ChunkMethod = 'structure' \| 'fixed' \| 'custom'/,
|
||||||
|
'非结构化切分方式类型必须只保留 structure、fixed 和 custom',
|
||||||
|
)
|
||||||
for (const field of [
|
for (const field of [
|
||||||
'preprocessOptions',
|
'preprocessOptions',
|
||||||
'chunkMethod',
|
'chunkMethod',
|
||||||
@@ -363,14 +483,54 @@ for (const removedField of ['contextScope', 'generationTypes', 'skipUnanswerable
|
|||||||
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
|
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
|
||||||
assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关')
|
assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关')
|
||||||
assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关')
|
assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关')
|
||||||
|
const expectedSmartPreprocessOptions = [
|
||||||
|
'clean_invalid_content',
|
||||||
|
'detect_document_structure',
|
||||||
|
'merge_short_content',
|
||||||
|
'filter_low_quality',
|
||||||
|
'deduplicate_content',
|
||||||
|
'preserve_context',
|
||||||
|
]
|
||||||
|
const smartOptionsStart = unstructuredOptionsSource.indexOf('const SMART_PREPROCESS_OPTIONS')
|
||||||
|
const smartOptionsEnd = unstructuredOptionsSource.indexOf('const CHUNK_METHODS', smartOptionsStart)
|
||||||
|
const smartOptionsSource = unstructuredOptionsSource.slice(smartOptionsStart, smartOptionsEnd)
|
||||||
|
const smartOptionValues = [...smartOptionsSource.matchAll(/^\s*'([^']+)',?$/gm)].map((match) => match[1])
|
||||||
|
assert.deepEqual(smartOptionValues, expectedSmartPreprocessOptions, '智能预处理内部值与后端语义不一致')
|
||||||
|
assert.equal(new Set(smartOptionValues).size, smartOptionValues.length, '非结构化智能预处理 value 必须唯一')
|
||||||
|
for (const descriptionPart of [
|
||||||
|
'清理无效内容',
|
||||||
|
'感知文档结构',
|
||||||
|
'合并短块',
|
||||||
|
'预过滤低质量内容',
|
||||||
|
'近重复去重',
|
||||||
|
'重叠保护上下文',
|
||||||
|
]) {
|
||||||
|
assert.ok(unstructuredOptionsSource.includes(descriptionPart), `智能预处理说明缺少语义：${descriptionPart}`)
|
||||||
|
}
|
||||||
|
assert.ok(unstructuredOptionsSource.includes('姓名、手机号、邮箱和身份证号'), '非结构化脱敏说明缺少完整字段范围')
|
||||||
|
assert.match(unstructuredOptionsSource, /Array\.from\(new Set\(props\.options\.preprocessOptions\.filter\(/, '非结构化预处理选中值没有去重')
|
||||||
assert.match(unstructuredOptionsSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项')
|
assert.match(unstructuredOptionsSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项')
|
||||||
assert.match(unstructuredOptionsSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑')
|
assert.match(unstructuredOptionsSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑')
|
||||||
assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑')
|
assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑')
|
||||||
|
|
||||||
assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
|
assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
|
||||||
for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) {
|
const expectedChunkMethods = [
|
||||||
assert.ok(unstructuredOptionsSource.includes(method), `切分方式缺少选项：${method}`)
|
['structure', '文档结构'],
|
||||||
|
['fixed', '固定 Token'],
|
||||||
|
['custom', '自定义分隔符'],
|
||||||
|
]
|
||||||
|
const chunkMethodsStart = unstructuredOptionsSource.indexOf('const CHUNK_METHODS')
|
||||||
|
const chunkMethodsEnd = unstructuredOptionsSource.indexOf('const UNSTRUCTURED_NUMBER_LIMITS', chunkMethodsStart)
|
||||||
|
const chunkMethodsSource = unstructuredOptionsSource.slice(chunkMethodsStart, chunkMethodsEnd)
|
||||||
|
const chunkMethodValues = [...chunkMethodsSource.matchAll(/value: '([^']+)'/g)].map((match) => match[1])
|
||||||
|
assert.deepEqual(chunkMethodValues, expectedChunkMethods.map(([value]) => value), '切分方式值集合不准确')
|
||||||
|
for (const [value, label] of expectedChunkMethods) {
|
||||||
|
assert.ok(chunkMethodsSource.includes(`value: '${value}', label: '${label}'`), `切分方式缺少选项：${label}`)
|
||||||
}
|
}
|
||||||
|
for (const removedMethod of ['semantic', 'heading']) {
|
||||||
|
assert.ok(!chunkMethodsSource.includes(`value: '${removedMethod}'`), `切分方式仍保留已移除值：${removedMethod}`)
|
||||||
|
}
|
||||||
|
assert.ok(unstructuredOptionsSource.includes('推荐使用文档结构'), '切分方式缺少文档结构默认推荐说明')
|
||||||
for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) {
|
for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) {
|
||||||
assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`)
|
assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`)
|
||||||
}
|
}
|
||||||
@@ -395,19 +555,87 @@ assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), '
|
|||||||
assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局')
|
assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局')
|
||||||
|
|
||||||
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
|
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
|
||||||
assert.match(stateSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分')
|
assert.match(stateSource, /chunkMethod:\s*'structure'/, '非结构化默认切分方式必须为文档结构')
|
||||||
assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
|
assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
|
||||||
assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
|
assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
|
||||||
assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
|
assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
|
||||||
assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
|
assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
|
||||||
assert.match(draftSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.bindings\.unstructuredOptions\.value/, '非结构化配置没有写入草稿')
|
|
||||||
assert.match(draftSource, /bindings\.unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复')
|
|
||||||
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
|
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
|
||||||
assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识')
|
|
||||||
assert.match(draftSource, /schemaVersion:\s*DATA_PROCESS_DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识')
|
|
||||||
assert.match(draftSource, /bindings\.goToStep\('create'\)/, '恢复配置后应回到安全的创建步骤')
|
|
||||||
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
|
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
|
||||||
|
|
||||||
|
function defaultPreprocessValues(functionName, nextFunctionName) {
|
||||||
|
const start = stateSource.indexOf(`export function ${functionName}`)
|
||||||
|
const end = nextFunctionName ? stateSource.indexOf(`export function ${nextFunctionName}`, start) : stateSource.length
|
||||||
|
const functionSource = stateSource.slice(start, end)
|
||||||
|
const match = functionSource.match(/preprocessOptions:\s*\[([\s\S]*?)\]/)
|
||||||
|
assert.ok(match, `${functionName} 缺少 preprocessOptions 默认值`)
|
||||||
|
return [...match[1].matchAll(/'([^']+)'/g)].map((item) => item[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultStructuredPreprocess = defaultPreprocessValues(
|
||||||
|
'createDefaultStructuredOptions',
|
||||||
|
'createDefaultUnstructuredOptions',
|
||||||
|
)
|
||||||
|
assert.deepEqual(
|
||||||
|
defaultStructuredPreprocess,
|
||||||
|
['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||||
|
'结构化默认预处理配置不准确',
|
||||||
|
)
|
||||||
|
assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复')
|
||||||
|
const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions')
|
||||||
|
assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整')
|
||||||
|
assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复')
|
||||||
|
|
||||||
|
const backendConfigStart = viewSource.indexOf('function toBackendConfig()')
|
||||||
|
const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart)
|
||||||
|
assert.ok(backendConfigStart >= 0 && backendConfigEnd > backendConfigStart, '缺少任务后端配置映射')
|
||||||
|
const backendConfigSource = viewSource.slice(backendConfigStart, backendConfigEnd)
|
||||||
|
assert.ok(backendConfigSource.includes('preprocess_options: [...options.preprocessOptions]'), '预处理选项没有完整传入任务配置')
|
||||||
|
assert.ok(backendConfigSource.includes('dataset_split: { ...options.datasetSplit }'), '数据集划分没有完整传入任务配置')
|
||||||
|
for (const [backendField, frontendField] of [
|
||||||
|
['semantic_enrichment', 'semanticEnrichment'],
|
||||||
|
['generation_model_id', 'generationModelId'],
|
||||||
|
['generation_prompt', 'generationPrompt'],
|
||||||
|
['temperature', 'temperature'],
|
||||||
|
['max_tokens', 'maxTokens'],
|
||||||
|
['json_mode', 'jsonMode'],
|
||||||
|
['quality_filter_enabled', 'qualityFilterEnabled'],
|
||||||
|
['filter_low_quality', 'filterLowQuality'],
|
||||||
|
['filter_short_content', 'filterShortContent'],
|
||||||
|
['min_output_length', 'minOutputLength'],
|
||||||
|
]) {
|
||||||
|
assert.ok(
|
||||||
|
backendConfigSource.includes(`${backendField}: options.${frontendField}`),
|
||||||
|
`公共配置 ${frontendField} 没有传入 task payload`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for (const [backendField, frontendField] of [
|
||||||
|
['chunk_method', 'chunkMethod'],
|
||||||
|
['chunk_size', 'chunkSize'],
|
||||||
|
['chunk_overlap', 'chunkOverlap'],
|
||||||
|
['min_chunk_size', 'minChunkSize'],
|
||||||
|
['custom_delimiter', 'customDelimiter'],
|
||||||
|
['preserve_tables', 'preserveTables'],
|
||||||
|
['preserve_code_blocks', 'preserveCodeBlocks'],
|
||||||
|
['preserve_lists', 'preserveLists'],
|
||||||
|
['qa_pairs_per_chunk', 'qaPairsPerChunk'],
|
||||||
|
]) {
|
||||||
|
assert.ok(
|
||||||
|
backendConfigSource.includes(`${backendField}: unstructuredOptions.value.${frontendField}`),
|
||||||
|
`非结构化配置 ${frontendField} 没有传入 task payload`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
assert.ok(
|
||||||
|
backendConfigSource.includes('qa_pairs_per_row: structuredOptions.value.qaPairsPerRow'),
|
||||||
|
'结构化每行生成数量没有传入 task payload',
|
||||||
|
)
|
||||||
|
const taskPayloadStart = viewSource.indexOf('function taskPayload()')
|
||||||
|
const taskPayloadEnd = viewSource.indexOf('function externalPayload()', taskPayloadStart)
|
||||||
|
const taskPayloadSource = viewSource.slice(taskPayloadStart, taskPayloadEnd)
|
||||||
|
for (const marker of ['name: task.name.trim()', 'description: task.description.trim()', 'process_type: processType.value', 'config: toBackendConfig()']) {
|
||||||
|
assert.ok(taskPayloadSource.includes(marker), `任务创建 payload 缺少：${marker}`)
|
||||||
|
}
|
||||||
|
|
||||||
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
|
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
|
||||||
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart)
|
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart)
|
||||||
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
|
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
|
||||||
@@ -667,7 +895,7 @@ for (const property of ['max-height', 'overflow', 'overflow-x', 'overflow-y']) {
|
|||||||
for (const marker of [
|
for (const marker of [
|
||||||
'uploaded-file-list-header',
|
'uploaded-file-list-header',
|
||||||
'uploaded-file-items',
|
'uploaded-file-items',
|
||||||
'已添加 {{ uploadedFiles.length }} 个文件',
|
'已选择 {{ uploadedFiles.length }} 个文件',
|
||||||
':title="file.name"',
|
':title="file.name"',
|
||||||
]) {
|
]) {
|
||||||
assert.ok(sourceUploadSource.includes(marker), `源数据文件列表缺少：${marker}`)
|
assert.ok(sourceUploadSource.includes(marker), `源数据文件列表缺少：${marker}`)
|
||||||
@@ -720,7 +948,7 @@ assert.match(
|
|||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
sourceUploadSource,
|
sourceUploadSource,
|
||||||
/<div\s+class="uploaded-file-list-header">\s*<span>已添加 \{\{ uploadedFiles\.length \}\} 个文件<\/span>/,
|
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?正在逐个上传/,
|
||||||
'文件列表标题结构或文件数量文案缺失',
|
'文件列表标题结构或文件数量文案缺失',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
@@ -730,17 +958,27 @@ assert.match(
|
|||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
sourceUploadSource,
|
sourceUploadSource,
|
||||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
||||||
'无文件时未保留原有大拖拽上传区或上传配置',
|
'无文件时未保留原有大拖拽上传区或上传配置',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
sourceUploadSource,
|
sourceUploadSource,
|
||||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>[\s\S]*?<template\s+#tip>\s*<div\s+class="el-upload__tip">\s*\{\{\s*processType === 'unstructured'\s*\? '支持 TXT、Markdown、PDF、Word、JSON、JSONL，单文件不超过 200MB'\s*:\s*'支持 JSON、JSONL、CSV、Excel，单文件不超过 200MB'\s*\}\}/,
|
/<el-upload\s+v-if="uploadedFiles\.length === 0"[\s\S]*?<template\s+#tip>[\s\S]*?processType === 'unstructured'/,
|
||||||
'无文件时大拖拽上传区缺少格式提示槽、处理类型分支或完整格式提示',
|
'无文件时大拖拽上传区缺少按处理类型展示的格式提示',
|
||||||
)
|
)
|
||||||
|
assert.ok(
|
||||||
|
sourceUploadSource.includes("? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'"),
|
||||||
|
'非结构化上传 accept 不完整',
|
||||||
|
)
|
||||||
|
assert.ok(
|
||||||
|
sourceUploadSource.includes(": '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx'"),
|
||||||
|
'结构化上传 accept 不完整',
|
||||||
|
)
|
||||||
|
assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构化格式提示没有说明旧版 DOC/PPT 需转换')
|
||||||
|
assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换')
|
||||||
assert.match(
|
assert.match(
|
||||||
sourceUploadSource,
|
sourceUploadSource,
|
||||||
/<div\s+class="uploaded-file-list-header">\s*<span>已添加 \{\{ uploadedFiles\.length \}\} 个文件<\/span>\s*<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
||||||
'有文件时缺少标题右侧的继续上传触发器或上传配置',
|
'有文件时缺少标题右侧的继续上传触发器或上传配置',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
@@ -754,7 +992,20 @@ assert.match(
|
|||||||
'继续上传未覆盖内层上传节点的宽度和顶部间距',
|
'继续上传未覆盖内层上传节点的宽度和顶部间距',
|
||||||
)
|
)
|
||||||
assert.match(sourceUploadSource, /\.uploaded-file\s*\{[^}]*min-height:\s*48px/, '文件行没有保持 48px 最小高度')
|
assert.match(sourceUploadSource, /\.uploaded-file\s*\{[^}]*min-height:\s*48px/, '文件行没有保持 48px 最小高度')
|
||||||
assert.match(sourceUploadSource, /<span class="file-status"><i class="fa fa-check-circle"[^>]*\/> 校验通过<\/span>/, '文件行缺少校验成功状态')
|
assert.match(sourceUploadSource, /success:\s*\{ label: '切分完成', icon: 'fa-check-circle' \}/, '文件行缺少切分成功状态')
|
||||||
|
assert.match(sourceUploadSource, /previewBuilding/, '上传组件没有接收逐文件预览构建状态')
|
||||||
|
assert.match(sourceUploadSource, /sourceUploading/, '上传组件没有接收串行上传状态')
|
||||||
|
assert.match(
|
||||||
|
sourceUploadSource,
|
||||||
|
/<el-progress[\s\S]*?:percentage="getFileBarPercentage\(file\)"[\s\S]*?:indeterminate="isFileProcessing\(file\)"/,
|
||||||
|
'文件行缺少上传与切分阶段的独立进度',
|
||||||
|
)
|
||||||
|
for (const label of ['等待上传', '正在上传', '上传失败', '等待切分', '正在切分', '切分完成', '切分失败']) {
|
||||||
|
assert.ok(sourceUploadSource.includes(`label: '${label}'`), `文件行缺少状态：${label}`)
|
||||||
|
}
|
||||||
|
assert.match(sourceUploadSource, /file\.uploadProgress/, '文件行没有使用真实上传进度')
|
||||||
|
assert.match(sourceUploadSource, /file\.uploadError \|\| file\.previewError/, '文件行没有按阶段展示上传或切分失败原因')
|
||||||
|
assert.match(sourceUploadSource, /file\.previewError/, '文件行没有展示逐文件失败原因以支持重试')
|
||||||
assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, '文件行缺少 remove-file 删除动作')
|
assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, '文件行缺少 remove-file 删除动作')
|
||||||
|
|
||||||
const { descriptor } = parseSfc(viewSource, { filename: viewPath })
|
const { descriptor } = parseSfc(viewSource, { filename: viewPath })
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type {
|
|||||||
DataProcessExternalSourcePayload,
|
DataProcessExternalSourcePayload,
|
||||||
DataProcessExternalTestResult,
|
DataProcessExternalTestResult,
|
||||||
DataProcessPage,
|
DataProcessPage,
|
||||||
|
DataProcessPdfPages,
|
||||||
DataProcessPreviewBuildPayload,
|
DataProcessPreviewBuildPayload,
|
||||||
DataProcessPreviewBuildResult,
|
DataProcessPreviewBuildResult,
|
||||||
DataProcessPreviewCreatePayload,
|
DataProcessPreviewCreatePayload,
|
||||||
@@ -27,8 +28,13 @@ export type {
|
|||||||
DataProcessExternalSourcePayload,
|
DataProcessExternalSourcePayload,
|
||||||
DataProcessExternalTestResult,
|
DataProcessExternalTestResult,
|
||||||
DataProcessPage,
|
DataProcessPage,
|
||||||
|
DataProcessPdfPageRange,
|
||||||
|
DataProcessPdfPages,
|
||||||
DataProcessPreviewBuildPayload,
|
DataProcessPreviewBuildPayload,
|
||||||
|
DataProcessPreviewBuildFileResult,
|
||||||
DataProcessPreviewBuildResult,
|
DataProcessPreviewBuildResult,
|
||||||
|
DataProcessPreviewFileBuildProgress,
|
||||||
|
DataProcessPreviewFileStatus,
|
||||||
DataProcessPreviewCreatePayload,
|
DataProcessPreviewCreatePayload,
|
||||||
DataProcessPreviewItem,
|
DataProcessPreviewItem,
|
||||||
DataProcessPreviewUpdatePayload,
|
DataProcessPreviewUpdatePayload,
|
||||||
@@ -71,13 +77,25 @@ export const updateDataProcessTask = (taskId: string | number, payload: DataProc
|
|||||||
export const deleteDataProcessTask = (taskId: string | number) =>
|
export const deleteDataProcessTask = (taskId: string | number) =>
|
||||||
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||||
|
|
||||||
export function uploadDataProcessSourceFiles(taskId: string | number, files: File[]) {
|
export function uploadDataProcessSourceFiles(
|
||||||
|
taskId: string | number,
|
||||||
|
files: File[],
|
||||||
|
onProgress?: (progress: number) => void,
|
||||||
|
) {
|
||||||
const formData = new FormData()
|
const formData = new FormData()
|
||||||
files.forEach((file) => formData.append('files', file))
|
files.forEach((file) => formData.append('files', file))
|
||||||
return post<{ files: DataProcessSourceFile[] }>(
|
return post<{ files: DataProcessSourceFile[] }>(
|
||||||
`/data-process/${encodeURIComponent(taskId)}/source-files`,
|
`/data-process/${encodeURIComponent(taskId)}/source-files`,
|
||||||
formData,
|
formData,
|
||||||
{ headers: { 'Content-Type': 'multipart/form-data' } },
|
{
|
||||||
|
timeout: 5 * 60 * 1000,
|
||||||
|
onUploadProgress: (event) => {
|
||||||
|
if (!event.total) return
|
||||||
|
// 发送完成不等于服务端解析完成；收到成功响应前最多展示 99%。
|
||||||
|
const progress = Math.round((event.loaded / event.total) * 100)
|
||||||
|
onProgress?.(Math.min(99, Math.max(0, progress)))
|
||||||
|
},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,6 +113,18 @@ export const getDataProcessSourceContent = (
|
|||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
export const getDataProcessSourceRawUrl = (
|
||||||
|
taskId: string | number,
|
||||||
|
fileId: string | number,
|
||||||
|
) => `/modelTF/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/raw`
|
||||||
|
|
||||||
|
export const getDataProcessPdfPages = (
|
||||||
|
taskId: string | number,
|
||||||
|
fileId: string | number,
|
||||||
|
) => get<DataProcessPdfPages>(
|
||||||
|
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/pdf-pages`,
|
||||||
|
)
|
||||||
|
|
||||||
export const testDataProcessExternalSource = (
|
export const testDataProcessExternalSource = (
|
||||||
taskId: string | number,
|
taskId: string | number,
|
||||||
payload: DataProcessExternalSourcePayload,
|
payload: DataProcessExternalSourcePayload,
|
||||||
@@ -120,6 +150,7 @@ export const buildDataProcessPreview = (
|
|||||||
) => post<DataProcessPreviewBuildResult>(
|
) => post<DataProcessPreviewBuildResult>(
|
||||||
`/data-process/${encodeURIComponent(taskId)}/preview/build`,
|
`/data-process/${encodeURIComponent(taskId)}/preview/build`,
|
||||||
payload,
|
payload,
|
||||||
|
{ timeout: 5 * 60 * 1000 },
|
||||||
)
|
)
|
||||||
|
|
||||||
export function getDataProcessPreview(
|
export function getDataProcessPreview(
|
||||||
|
|||||||
@@ -93,6 +93,17 @@ export interface DataProcessSourceContent {
|
|||||||
total_chars?: number
|
total_chars?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DataProcessPdfPageRange {
|
||||||
|
page_number: number
|
||||||
|
source_start: number
|
||||||
|
source_end: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessPdfPages {
|
||||||
|
page_count: number
|
||||||
|
pages: DataProcessPdfPageRange[]
|
||||||
|
}
|
||||||
|
|
||||||
export interface DataProcessExternalSourcePayload {
|
export interface DataProcessExternalSourcePayload {
|
||||||
type: 'postgresql'
|
type: 'postgresql'
|
||||||
url: string
|
url: string
|
||||||
@@ -129,7 +140,26 @@ export interface DataProcessPreviewBuildPayload {
|
|||||||
source_file_ids?: Array<string | number>
|
source_file_ids?: Array<string | number>
|
||||||
}
|
}
|
||||||
|
|
||||||
export type DataProcessPreviewBuildResult = DataProcessPage<DataProcessPreviewItem>
|
export type DataProcessPreviewFileStatus = 'waiting' | 'processing' | 'success' | 'failed'
|
||||||
|
|
||||||
|
export interface DataProcessPreviewBuildFileResult {
|
||||||
|
source_file_id: string | number
|
||||||
|
preview_count: number
|
||||||
|
status: 'completed' | 'empty'
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessPreviewFileBuildProgress {
|
||||||
|
source_file_id: string | number
|
||||||
|
status: Exclude<DataProcessPreviewFileStatus, 'waiting'>
|
||||||
|
progress: number
|
||||||
|
preview_count?: number
|
||||||
|
error?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessPreviewBuildResult extends DataProcessPage<DataProcessPreviewItem> {
|
||||||
|
file_counts?: Record<string, number>
|
||||||
|
files?: DataProcessPreviewBuildFileResult[]
|
||||||
|
}
|
||||||
|
|
||||||
export interface DataProcessPreviewCreatePayload {
|
export interface DataProcessPreviewCreatePayload {
|
||||||
source_file_id?: string | number | null
|
source_file_id?: string | number | null
|
||||||
|
|||||||
@@ -15,29 +15,27 @@ import {
|
|||||||
createDefaultStructuredOptions,
|
createDefaultStructuredOptions,
|
||||||
createDefaultUnstructuredOptions,
|
createDefaultUnstructuredOptions,
|
||||||
} from './create/dataProcessCreateState'
|
} from './create/dataProcessCreateState'
|
||||||
import {
|
|
||||||
DATA_PROCESS_DRAFT_STORAGE_KEY,
|
|
||||||
useDataProcessDraft,
|
|
||||||
} from './create/useDataProcessDraft'
|
|
||||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||||
|
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||||
|
import {
|
||||||
|
mapDataProcessSourceFile,
|
||||||
|
useDataProcessSourceUpload,
|
||||||
|
validateSourceFileSelection,
|
||||||
|
} from './create/useDataProcessSourceUpload'
|
||||||
import { useModelsStore } from '@/stores/models'
|
import { useModelsStore } from '@/stores/models'
|
||||||
import {
|
import {
|
||||||
buildDataProcessPreview,
|
|
||||||
createDataProcessPreview,
|
createDataProcessPreview,
|
||||||
createDataProcessTask,
|
createDataProcessTask,
|
||||||
deleteDataProcessPreview,
|
deleteDataProcessPreview,
|
||||||
deleteDataProcessSourceFile,
|
deleteDataProcessSourceFile,
|
||||||
getDataProcessPreview,
|
getDataProcessPreview,
|
||||||
getDataProcessSourceContent,
|
getDataProcessSourceContent,
|
||||||
getDataProcessTask,
|
|
||||||
pullDataProcessExternalSource,
|
pullDataProcessExternalSource,
|
||||||
testDataProcessExternalSource,
|
testDataProcessExternalSource,
|
||||||
updateDataProcessPreview,
|
updateDataProcessPreview,
|
||||||
updateDataProcessTask,
|
updateDataProcessTask,
|
||||||
uploadDataProcessSourceFiles,
|
|
||||||
type DataProcessExternalSourcePayload,
|
type DataProcessExternalSourcePayload,
|
||||||
type DataProcessPreviewItem,
|
type DataProcessPreviewItem,
|
||||||
type DataProcessSourceFile,
|
|
||||||
} from '@/api/modules/dataProcess'
|
} from '@/api/modules/dataProcess'
|
||||||
import type { DataProcessConfig } from '@/types/dataProcess'
|
import type { DataProcessConfig } from '@/types/dataProcess'
|
||||||
import type {
|
import type {
|
||||||
@@ -54,15 +52,12 @@ import type {
|
|||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
const modelsStore = useModelsStore()
|
const modelsStore = useModelsStore()
|
||||||
const { list: modelList } = storeToRefs(modelsStore)
|
const { list: modelList } = storeToRefs(modelsStore)
|
||||||
const generationModels = computed(() => modelList.value.filter((model) => (
|
// 候选范围与模型管理保持一致，不在数据处理页面重复定义模型过滤规则。
|
||||||
model.type === 'LLM'
|
const generationModels = computed(() => modelList.value)
|
||||||
&& (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
|
|
||||||
)))
|
|
||||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||||
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
|
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
|
||||||
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||||
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'
|
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v2'
|
||||||
|
|
||||||
const WIZARD_STEPS = [
|
const WIZARD_STEPS = [
|
||||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||||
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
|
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
|
||||||
@@ -82,7 +77,8 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
|
|||||||
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
|
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
|
||||||
))
|
))
|
||||||
const uploadedFiles = ref<UploadedDataFile[]>([])
|
const uploadedFiles = ref<UploadedDataFile[]>([])
|
||||||
|
const previewBuilding = ref(false)
|
||||||
|
const { buildPreviewsByFile } = useDataProcessPreviewBuild()
|
||||||
const externalSource = reactive<ExternalDataSource>({
|
const externalSource = reactive<ExternalDataSource>({
|
||||||
type: 'postgresql',
|
type: 'postgresql',
|
||||||
url: '',
|
url: '',
|
||||||
@@ -95,7 +91,6 @@ const externalSource = reactive<ExternalDataSource>({
|
|||||||
})
|
})
|
||||||
const externalPulling = ref(false)
|
const externalPulling = ref(false)
|
||||||
const externalConnected = ref(false)
|
const externalConnected = ref(false)
|
||||||
|
|
||||||
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
||||||
const previewSignature = ref('')
|
const previewSignature = ref('')
|
||||||
const previewItems = ref<PreviewItem[]>([])
|
const previewItems = ref<PreviewItem[]>([])
|
||||||
@@ -103,7 +98,6 @@ const selectedPreviewFileId = ref<string | null>(null)
|
|||||||
const selectedPreviewId = ref<string | null>(null)
|
const selectedPreviewId = ref<string | null>(null)
|
||||||
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
|
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
|
||||||
const dirty = ref(false)
|
const dirty = ref(false)
|
||||||
const restoringDraft = ref(false)
|
|
||||||
let allowLeave = false
|
let allowLeave = false
|
||||||
|
|
||||||
const {
|
const {
|
||||||
@@ -123,6 +117,19 @@ const {
|
|||||||
dirty,
|
dirty,
|
||||||
beforeGenerate: syncPreviewChanges,
|
beforeGenerate: syncPreviewChanges,
|
||||||
})
|
})
|
||||||
|
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
|
||||||
|
taskId,
|
||||||
|
uploadedFiles,
|
||||||
|
onUploaded(file) {
|
||||||
|
previewSignature.value = ''
|
||||||
|
resetDownstream()
|
||||||
|
dirty.value = true
|
||||||
|
ElMessage.success(`文件 ${file.name} 上传成功，等待切分`)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const hasUnfinishedUploads = computed(() => uploadedFiles.value.some((file) => (
|
||||||
|
file.status !== 'ready' || !file.sourceFileId
|
||||||
|
)))
|
||||||
|
|
||||||
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
|
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
|
||||||
const previewItemsByFile = computed(() => {
|
const previewItemsByFile = computed(() => {
|
||||||
@@ -149,7 +156,10 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
|
|||||||
const primaryActionLabel = computed(() => {
|
const primaryActionLabel = computed(() => {
|
||||||
if (currentStepId.value === 'create') return '继续：选择大模型'
|
if (currentStepId.value === 'create') return '继续：选择大模型'
|
||||||
if (currentStepId.value === 'model') return '继续：上传文件'
|
if (currentStepId.value === 'model') return '继续：上传文件'
|
||||||
if (currentStepId.value === 'upload') return '继续：数据预览'
|
if (currentStepId.value === 'upload') {
|
||||||
|
if (sourceUploading.value) return '正在上传'
|
||||||
|
return previewBuilding.value ? '正在切分' : '继续：数据预览'
|
||||||
|
}
|
||||||
if (currentStepId.value === 'preview') return '确认预览并继续'
|
if (currentStepId.value === 'preview') return '确认预览并继续'
|
||||||
if (currentStepId.value === 'results') return '保存任务'
|
if (currentStepId.value === 'results') return '保存任务'
|
||||||
if (generation.status === 'running') return '正在生成'
|
if (generation.status === 'running') return '正在生成'
|
||||||
@@ -260,33 +270,6 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
|
|
||||||
return {
|
|
||||||
uid: String(file.id),
|
|
||||||
sourceFileId: String(file.id),
|
|
||||||
name: file.name,
|
|
||||||
size: file.size_bytes,
|
|
||||||
count: file.record_count,
|
|
||||||
content,
|
|
||||||
fileFormat: file.file_format,
|
|
||||||
checksumSha256: file.checksum_sha256,
|
|
||||||
status: 'ready',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const { persistDraft, restoreDraft } = useDataProcessDraft({
|
|
||||||
taskId,
|
|
||||||
currentStepId,
|
|
||||||
task,
|
|
||||||
processType,
|
|
||||||
structuredOptions,
|
|
||||||
unstructuredOptions,
|
|
||||||
externalSource,
|
|
||||||
restoringDraft,
|
|
||||||
dirty,
|
|
||||||
goToStep,
|
|
||||||
})
|
|
||||||
|
|
||||||
function previewAffectingOptions() {
|
function previewAffectingOptions() {
|
||||||
if (processType.value === 'structured') {
|
if (processType.value === 'structured') {
|
||||||
return {
|
return {
|
||||||
@@ -368,92 +351,70 @@ function generationAffectingOptions() {
|
|||||||
|
|
||||||
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
|
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
|
||||||
|
|
||||||
|
function buildPreviewConfigSignature() {
|
||||||
|
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}`
|
||||||
|
}
|
||||||
|
|
||||||
function buildPreviewSignature() {
|
function buildPreviewSignature() {
|
||||||
const filesSignature = uploadedFiles.value
|
const filesSignature = uploadedFiles.value
|
||||||
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
|
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
|
||||||
.join('|')
|
.join('|')
|
||||||
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
|
return `${buildPreviewConfigSignature()}:${filesSignature}`
|
||||||
}
|
}
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||||
() => {
|
() => { dirty.value = true },
|
||||||
if (!restoringDraft.value) dirty.value = true
|
|
||||||
},
|
|
||||||
{ deep: true },
|
{ deep: true },
|
||||||
)
|
)
|
||||||
|
|
||||||
watch(processType, (nextType, previousType) => {
|
watch(processType, (nextType, previousType) => {
|
||||||
if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
|
if (nextType === previousType || uploadedFiles.value.length === 0) return
|
||||||
resetSourceDataForProcessTypeChange()
|
resetSourceDataForProcessTypeChange()
|
||||||
ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
|
ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||||
if (restoringDraft.value || currentSignature === previousSignature) return
|
if (currentSignature === previousSignature) return
|
||||||
resetDownstream()
|
resetDownstream()
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(
|
|
||||||
[taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
|
|
||||||
persistDraft,
|
|
||||||
{ deep: true },
|
|
||||||
)
|
|
||||||
|
|
||||||
function scrollToStepTop() {
|
function scrollToStepTop() {
|
||||||
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
|
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
|
||||||
}
|
}
|
||||||
|
|
||||||
watch(currentStep, () => nextTick(scrollToStepTop))
|
watch(currentStep, () => nextTick(scrollToStepTop))
|
||||||
|
|
||||||
async function handleFileChange(uploadFile: UploadFile) {
|
function handleFileChange(uploadFile: UploadFile) {
|
||||||
const raw = uploadFile.raw
|
const raw = uploadFile.raw
|
||||||
if (!raw) return
|
if (!raw) return
|
||||||
if (!taskId.value) {
|
if (!taskId.value) {
|
||||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (raw.size > 200 * 1024 * 1024) {
|
const validation = validateSourceFileSelection(raw, processType.value, uploadedFiles.value)
|
||||||
ElMessage.warning('单文件不能超过 200MB')
|
if (!validation.valid) {
|
||||||
|
ElMessage[validation.severity](validation.message)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
// 必须先同步插入文件行，再交给队列；这样选择完成后页面会立即展示全部文件。
|
||||||
const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
|
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
|
||||||
if (!textExtensions.has(extension)) {
|
uploadedFiles.value.push({
|
||||||
ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV；不会用示例内容替代无法解析的文件')
|
uid: localUid,
|
||||||
return
|
rawFile: raw,
|
||||||
}
|
name: raw.name,
|
||||||
|
size: raw.size,
|
||||||
if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
|
count: 0,
|
||||||
ElMessage.warning('同名且同大小的文件已经上传')
|
content: '',
|
||||||
return
|
fileFormat: validation.extension,
|
||||||
}
|
status: 'queued',
|
||||||
|
uploadProgress: 0,
|
||||||
let content = ''
|
previewStatus: 'waiting',
|
||||||
try {
|
previewProgress: 0,
|
||||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
|
})
|
||||||
} catch {
|
dirty.value = true
|
||||||
ElMessage.error('文件不是有效的 UTF-8 文本，请转换编码后重试')
|
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
|
||||||
return
|
|
||||||
}
|
|
||||||
if (!content.trim()) {
|
|
||||||
ElMessage.warning('不能上传空文件')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
|
|
||||||
const source = uploaded.files[0]
|
|
||||||
if (!source) throw new Error('后端未返回源文件记录')
|
|
||||||
uploadedFiles.value.push(mapSourceFile(source, content))
|
|
||||||
previewSignature.value = ''
|
|
||||||
resetDownstream()
|
|
||||||
dirty.value = true
|
|
||||||
ElMessage.success(`文件 ${source.name} 上传成功`)
|
|
||||||
} catch {
|
|
||||||
// 请求层已展示后端的解析或格式错误。
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function useSampleFile() {
|
async function useSampleFile() {
|
||||||
@@ -462,32 +423,7 @@ async function useSampleFile() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
|
const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
|
||||||
await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
|
handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
|
||||||
}
|
|
||||||
|
|
||||||
async function restoreRegisteredSources() {
|
|
||||||
if (!taskId.value) return
|
|
||||||
try {
|
|
||||||
const savedTask = await getDataProcessTask(taskId.value)
|
|
||||||
const sources = savedTask.source_files || []
|
|
||||||
const restoredFiles = await Promise.all(sources.map(async (file) => {
|
|
||||||
try {
|
|
||||||
const source = await getDataProcessSourceContent(taskId.value!, file.id, {
|
|
||||||
start_line: 1,
|
|
||||||
line_count: 5000,
|
|
||||||
})
|
|
||||||
return mapSourceFile(file, source.content)
|
|
||||||
} catch {
|
|
||||||
return mapSourceFile(file)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
uploadedFiles.value = restoredFiles
|
|
||||||
if (restoredFiles.length) {
|
|
||||||
ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
ElMessage.warning('草稿任务暂时无法从后端同步，请检查服务后重试')
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateExternalSource(value: ExternalDataSource) {
|
function updateExternalSource(value: ExternalDataSource) {
|
||||||
@@ -539,7 +475,7 @@ async function handlePullData() {
|
|||||||
start_line: 1,
|
start_line: 1,
|
||||||
line_count: 5000,
|
line_count: 5000,
|
||||||
})
|
})
|
||||||
newFiles.push(mapSourceFile(file, source.content))
|
newFiles.push(mapDataProcessSourceFile(file, source.content))
|
||||||
}
|
}
|
||||||
uploadedFiles.value.push(...newFiles)
|
uploadedFiles.value.push(...newFiles)
|
||||||
externalConnected.value = true
|
externalConnected.value = true
|
||||||
@@ -557,11 +493,19 @@ async function handlePullData() {
|
|||||||
async function handleRemoveFile(uid: string | number) {
|
async function handleRemoveFile(uid: string | number) {
|
||||||
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
|
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
|
||||||
if (index < 0 || !taskId.value) return
|
if (index < 0 || !taskId.value) return
|
||||||
try {
|
const file = uploadedFiles.value[index]
|
||||||
await deleteDataProcessSourceFile(taskId.value, uid)
|
if (!file) return
|
||||||
} catch {
|
if (file.status === 'uploading') {
|
||||||
|
ElMessage.warning('当前文件正在上传，请等待上传结束后再删除')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (file.sourceFileId) {
|
||||||
|
try {
|
||||||
|
await deleteDataProcessSourceFile(taskId.value, file.sourceFileId)
|
||||||
|
} catch {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
uploadedFiles.value.splice(index, 1)
|
uploadedFiles.value.splice(index, 1)
|
||||||
previewSignature.value = ''
|
previewSignature.value = ''
|
||||||
previewItems.value = []
|
previewItems.value = []
|
||||||
@@ -596,7 +540,6 @@ async function nextFromModel() {
|
|||||||
: await createDataProcessTask(taskPayload())
|
: await createDataProcessTask(taskPayload())
|
||||||
taskId.value = String(saved.id)
|
taskId.value = String(saved.id)
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
persistDraft()
|
|
||||||
goToStep('upload')
|
goToStep('upload')
|
||||||
} catch {
|
} catch {
|
||||||
// 请求层已展示名称冲突或配置非法等具体原因。
|
// 请求层已展示名称冲突或配置非法等具体原因。
|
||||||
@@ -604,6 +547,7 @@ async function nextFromModel() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function nextFromUpload() {
|
async function nextFromUpload() {
|
||||||
|
if (previewBuilding.value || sourceUploading.value) return
|
||||||
if (!taskId.value) {
|
if (!taskId.value) {
|
||||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||||
return
|
return
|
||||||
@@ -612,13 +556,61 @@ async function nextFromUpload() {
|
|||||||
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
|
||||||
|
if (failedUploads.length) {
|
||||||
|
ElMessage.warning(`${failedUploads.length} 个文件上传失败，请删除后重新选择`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (hasUnfinishedUploads.value) {
|
||||||
|
ElMessage.warning('请等待所有文件上传完成后再开始切分')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
const signature = buildPreviewSignature()
|
const signature = buildPreviewSignature()
|
||||||
if (signature !== previewSignature.value) {
|
const configSignature = buildPreviewConfigSignature()
|
||||||
|
const allFilesSucceeded = () => uploadedFiles.value.every((file) => (
|
||||||
|
file.previewStatus === 'success' && file.previewConfigSignature === configSignature
|
||||||
|
))
|
||||||
|
|
||||||
|
if (signature === previewSignature.value && previewItems.value.length && allFilesSucceeded()) {
|
||||||
|
goToStep('preview')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const file of uploadedFiles.value) {
|
||||||
|
if (file.previewConfigSignature === configSignature) continue
|
||||||
|
file.previewStatus = 'waiting'
|
||||||
|
file.previewProgress = 0
|
||||||
|
file.previewError = undefined
|
||||||
|
file.previewCount = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
previewBuilding.value = true
|
||||||
|
try {
|
||||||
|
const pendingFileIds = uploadedFiles.value
|
||||||
|
.filter((file) => file.previewStatus !== 'success' || file.previewConfigSignature !== configSignature)
|
||||||
|
.map((file) => file.sourceFileId)
|
||||||
|
.filter((fileId): fileId is string => Boolean(fileId))
|
||||||
|
// 构建接口是同步响应：请求中只展示真实的“处理中”，响应成功后才记为 100%。
|
||||||
|
await buildPreviewsByFile(taskId.value, pendingFileIds, (progress) => {
|
||||||
|
const file = uploadedFiles.value.find((item) => (
|
||||||
|
String(item.sourceFileId) === String(progress.source_file_id)
|
||||||
|
))
|
||||||
|
if (!file) return
|
||||||
|
file.previewStatus = progress.status
|
||||||
|
file.previewProgress = progress.progress
|
||||||
|
file.previewCount = progress.preview_count
|
||||||
|
file.previewError = progress.error
|
||||||
|
if (progress.status === 'success') file.previewConfigSignature = configSignature
|
||||||
|
})
|
||||||
|
|
||||||
|
const failedCount = uploadedFiles.value.filter((file) => file.previewStatus === 'failed').length
|
||||||
|
if (failedCount || !allFilesSucceeded()) {
|
||||||
|
ElMessage.warning(`${failedCount || 1} 个文件切分失败；修正问题后点击“继续：数据预览”即可重试`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await buildDataProcessPreview(taskId.value, {
|
|
||||||
source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
|
|
||||||
})
|
|
||||||
const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
|
const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
|
||||||
const items = [...first.items]
|
const items = [...first.items]
|
||||||
const pages = Math.ceil(first.total / first.page_size)
|
const pages = Math.ceil(first.total / first.page_size)
|
||||||
@@ -628,6 +620,7 @@ async function nextFromUpload() {
|
|||||||
}
|
}
|
||||||
previewItems.value = items.map(mapPreviewItem)
|
previewItems.value = items.map(mapPreviewItem)
|
||||||
} catch {
|
} catch {
|
||||||
|
ElMessage.warning('文件切分已经完成，但预览加载失败；请再次点击继续重试加载')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (!previewItems.value.length) {
|
if (!previewItems.value.length) {
|
||||||
@@ -641,8 +634,10 @@ async function nextFromUpload() {
|
|||||||
: {}
|
: {}
|
||||||
previewSignature.value = signature
|
previewSignature.value = signature
|
||||||
resetDownstream()
|
resetDownstream()
|
||||||
|
goToStep('preview')
|
||||||
|
} finally {
|
||||||
|
previewBuilding.value = false
|
||||||
}
|
}
|
||||||
goToStep('preview')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function selectPreviewFile(fileId: string) {
|
function selectPreviewFile(fileId: string) {
|
||||||
@@ -770,6 +765,14 @@ async function handlePrimaryAction() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function handleBack() {
|
function handleBack() {
|
||||||
|
if (sourceUploading.value) {
|
||||||
|
ElMessage.warning('请等待当前文件上传完成')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (previewBuilding.value) {
|
||||||
|
ElMessage.warning('请等待当前文件切分完成')
|
||||||
|
return
|
||||||
|
}
|
||||||
if (generation.status === 'running') {
|
if (generation.status === 'running') {
|
||||||
ElMessage.warning('请先停止当前生成任务')
|
ElMessage.warning('请先停止当前生成任务')
|
||||||
return
|
return
|
||||||
@@ -793,7 +796,6 @@ async function saveTask() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
dirty.value = false
|
dirty.value = false
|
||||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
|
||||||
allowLeave = true
|
allowLeave = true
|
||||||
ElMessage.success('数据处理任务已保存')
|
ElMessage.success('数据处理任务已保存')
|
||||||
await router.push('/data-process')
|
await router.push('/data-process')
|
||||||
@@ -818,6 +820,10 @@ async function handleCancel() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onBeforeRouteLeave(async () => {
|
onBeforeRouteLeave(async () => {
|
||||||
|
if (sourceUploading.value) {
|
||||||
|
ElMessage.warning('请等待当前文件上传完成后再离开页面')
|
||||||
|
return false
|
||||||
|
}
|
||||||
if (allowLeave || !dirty.value) return true
|
if (allowLeave || !dirty.value) return true
|
||||||
const confirmed = await confirmDialogRef.value?.open({
|
const confirmed = await confirmDialogRef.value?.open({
|
||||||
title: '确认离开当前页面？',
|
title: '确认离开当前页面？',
|
||||||
@@ -833,9 +839,9 @@ onBeforeRouteLeave(async () => {
|
|||||||
onBeforeUnmount(() => {
|
onBeforeUnmount(() => {
|
||||||
stopGenerationTimer()
|
stopGenerationTimer()
|
||||||
})
|
})
|
||||||
onMounted(async () => {
|
onMounted(() => {
|
||||||
restoreDraft()
|
localStorage.removeItem('yg-data-process-create-draft')
|
||||||
await Promise.all([modelsStore.load(), restoreRegisteredSources()])
|
void modelsStore.load(true)
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@@ -899,6 +905,8 @@ onMounted(async () => {
|
|||||||
:external-source="externalSource"
|
:external-source="externalSource"
|
||||||
:external-pulling="externalPulling"
|
:external-pulling="externalPulling"
|
||||||
:external-connected="externalConnected"
|
:external-connected="externalConnected"
|
||||||
|
:preview-building="previewBuilding"
|
||||||
|
:source-uploading="sourceUploading"
|
||||||
@update:external-source="updateExternalSource"
|
@update:external-source="updateExternalSource"
|
||||||
@file-change="handleFileChange"
|
@file-change="handleFileChange"
|
||||||
@remove-file="handleRemoveFile"
|
@remove-file="handleRemoveFile"
|
||||||
@@ -915,6 +923,8 @@ onMounted(async () => {
|
|||||||
:items="activePreviewItems"
|
:items="activePreviewItems"
|
||||||
:process-type="processType"
|
:process-type="processType"
|
||||||
:file-name="activePreviewFile?.name ?? ''"
|
:file-name="activePreviewFile?.name ?? ''"
|
||||||
|
:task-id="taskId"
|
||||||
|
:source-file-id="activePreviewFile?.sourceFileId ?? activePreviewFile?.uid ?? null"
|
||||||
:files="previewFiles"
|
:files="previewFiles"
|
||||||
@update:selected-id="selectPreviewItem"
|
@update:selected-id="selectPreviewItem"
|
||||||
@update:selected-file-id="selectPreviewFile"
|
@update:selected-file-id="selectPreviewFile"
|
||||||
@@ -949,7 +959,7 @@ onMounted(async () => {
|
|||||||
|
|
||||||
<footer class="wizard-footer">
|
<footer class="wizard-footer">
|
||||||
<div class="footer-left">
|
<div class="footer-left">
|
||||||
<el-button v-if="currentStep > 0" @click="handleBack">
|
<el-button v-if="currentStep > 0" :disabled="previewBuilding || sourceUploading" @click="handleBack">
|
||||||
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回：{{ previousStepLabel }}
|
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回：{{ previousStepLabel }}
|
||||||
</el-button>
|
</el-button>
|
||||||
<el-button v-else @click="handleCancel">取消</el-button>
|
<el-button v-else @click="handleCancel">取消</el-button>
|
||||||
@@ -960,8 +970,8 @@ onMounted(async () => {
|
|||||||
<el-button
|
<el-button
|
||||||
class="wizard-primary-action"
|
class="wizard-primary-action"
|
||||||
type="primary"
|
type="primary"
|
||||||
:loading="generation.status === 'running'"
|
:loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||||
:disabled="currentStepId === 'generate' && generation.status === 'running'"
|
:disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||||
@click="handlePrimaryAction"
|
@click="handlePrimaryAction"
|
||||||
>
|
>
|
||||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||||
|
|||||||
@@ -84,6 +84,12 @@ const configLabelMap: Record<string, string> = {
|
|||||||
preserve_lists: '保留列表',
|
preserve_lists: '保留列表',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const chunkMethodLabelMap: Record<string, string> = {
|
||||||
|
structure: '文档结构',
|
||||||
|
fixed: '固定 Token',
|
||||||
|
custom: '自定义分隔符',
|
||||||
|
}
|
||||||
|
|
||||||
function numeric(value: number | undefined) {
|
function numeric(value: number | undefined) {
|
||||||
return Number.isFinite(value) ? Number(value) : 0
|
return Number.isFinite(value) ? Number(value) : 0
|
||||||
}
|
}
|
||||||
@@ -135,6 +141,9 @@ const configRows = computed(() => Object.entries(detail.value?.config || {})
|
|||||||
})))
|
})))
|
||||||
|
|
||||||
function formatConfigValue(key: string, value: unknown) {
|
function formatConfigValue(key: string, value: unknown) {
|
||||||
|
if (key === 'chunk_method' && typeof value === 'string') {
|
||||||
|
return chunkMethodLabelMap[value] || value
|
||||||
|
}
|
||||||
if (key === 'dataset_split' && value && typeof value === 'object') {
|
if (key === 'dataset_split' && value && typeof value === 'object') {
|
||||||
const split = value as Partial<DataProcessDatasetSplit>
|
const split = value as Partial<DataProcessDatasetSplit>
|
||||||
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
||||||
|
|||||||
@@ -41,6 +41,15 @@ function sectionValidationMessage() {
|
|||||||
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
|
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
|
||||||
return isModelMessage ? '' : props.validationMessage
|
return isModelMessage ? '' : props.validationMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function modelMeta(model: ModelItem) {
|
||||||
|
const source = model.model_source === 'api' || model.model_source === 'online'
|
||||||
|
? '在线模型'
|
||||||
|
: model.model_source === 'local'
|
||||||
|
? '本地模型'
|
||||||
|
: model.model_source || '模型管理'
|
||||||
|
return model.type ? `${source} · ${model.type}` : source
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -66,11 +75,11 @@ function sectionValidationMessage() {
|
|||||||
>
|
>
|
||||||
<div class="model-option">
|
<div class="model-option">
|
||||||
<span>{{ model.name }}</span>
|
<span>{{ model.name }}</span>
|
||||||
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
|
<small>{{ modelMeta(model) }}</small>
|
||||||
</div>
|
</div>
|
||||||
</el-option>
|
</el-option>
|
||||||
<template #empty>
|
<template #empty>
|
||||||
<div class="model-empty">暂无可用的大模型，请先在模型管理中添加</div>
|
<div class="model-empty">暂无模型，请先在模型管理中添加</div>
|
||||||
</template>
|
</template>
|
||||||
</el-select>
|
</el-select>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
650
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
650
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
@@ -0,0 +1,650 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import {
|
||||||
|
GlobalWorkerOptions,
|
||||||
|
TextLayer,
|
||||||
|
getDocument,
|
||||||
|
type PDFDocumentLoadingTask,
|
||||||
|
type PDFDocumentProxy,
|
||||||
|
type RenderTask,
|
||||||
|
} from 'pdfjs-dist'
|
||||||
|
import pdfWorkerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url'
|
||||||
|
import { computed, nextTick, onBeforeUnmount, ref, shallowRef, watch } from 'vue'
|
||||||
|
import {
|
||||||
|
getDataProcessPdfPages,
|
||||||
|
getDataProcessSourceRawUrl,
|
||||||
|
type DataProcessPdfPageRange,
|
||||||
|
} from '@/api/modules/dataProcess'
|
||||||
|
import type { PreviewItem } from './types'
|
||||||
|
|
||||||
|
GlobalWorkerOptions.workerSrc = pdfWorkerUrl
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
taskId: string | number | null
|
||||||
|
sourceFileId: string | number | null
|
||||||
|
fileName: string
|
||||||
|
selectedItem: PreviewItem | null
|
||||||
|
}>()
|
||||||
|
|
||||||
|
const scrollRef = ref<HTMLElement | null>(null)
|
||||||
|
const pageRef = ref<HTMLElement | null>(null)
|
||||||
|
const canvasRef = ref<HTMLCanvasElement | null>(null)
|
||||||
|
const textLayerRef = ref<HTMLElement | null>(null)
|
||||||
|
const documentRef = shallowRef<PDFDocumentProxy | null>(null)
|
||||||
|
const pageRanges = ref<DataProcessPdfPageRange[]>([])
|
||||||
|
const currentPage = ref(1)
|
||||||
|
const pageCount = ref(0)
|
||||||
|
const zoom = ref(1)
|
||||||
|
const renderedScale = ref(1)
|
||||||
|
const loading = ref(false)
|
||||||
|
const rendering = ref(false)
|
||||||
|
const errorMessage = ref('')
|
||||||
|
const highlightState = ref<'idle' | 'highlighted' | 'unmatched' | 'manual'>('idle')
|
||||||
|
|
||||||
|
let loadingTask: PDFDocumentLoadingTask | null = null
|
||||||
|
let renderTask: RenderTask | null = null
|
||||||
|
let textLayer: TextLayer | null = null
|
||||||
|
let loadSequence = 0
|
||||||
|
let renderSequence = 0
|
||||||
|
let renderedPageNumber = 0
|
||||||
|
let resizeFrame = 0
|
||||||
|
|
||||||
|
const sourceUrl = computed(() => (
|
||||||
|
props.taskId != null && props.sourceFileId != null
|
||||||
|
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||||
|
: ''
|
||||||
|
))
|
||||||
|
|
||||||
|
const pageStyle = computed(() => ({
|
||||||
|
'--total-scale-factor': String(renderedScale.value),
|
||||||
|
}))
|
||||||
|
|
||||||
|
const locationText = computed(() => {
|
||||||
|
if (highlightState.value === 'highlighted') return `切片原文已在第 ${currentPage.value} 页高亮`
|
||||||
|
if (highlightState.value === 'unmatched') return `已定位第 ${currentPage.value} 页，未匹配到可高亮文字`
|
||||||
|
if (highlightState.value === 'manual') return '手动新增切片没有原文位置'
|
||||||
|
return pageCount.value ? `第 ${currentPage.value} / ${pageCount.value} 页` : '正在读取 PDF'
|
||||||
|
})
|
||||||
|
|
||||||
|
function pageForItem(item: PreviewItem | null) {
|
||||||
|
if (!item || item.sourceStart == null || item.sourceEnd == null) return null
|
||||||
|
return pageRanges.value.find((page) => (
|
||||||
|
item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end
|
||||||
|
)) ?? pageRanges.value.find((page) => (
|
||||||
|
item.sourceStart! < page.source_end && item.sourceEnd! > page.source_start
|
||||||
|
)) ?? null
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectedTextForPage(page: DataProcessPdfPageRange) {
|
||||||
|
const item = props.selectedItem
|
||||||
|
if (!item || item.sourceStart == null || item.sourceEnd == null) return ''
|
||||||
|
const intersectionStart = Math.max(item.sourceStart, page.source_start)
|
||||||
|
const intersectionEnd = Math.min(item.sourceEnd, page.source_end)
|
||||||
|
if (intersectionEnd <= intersectionStart) return ''
|
||||||
|
const relativeStart = Math.max(0, intersectionStart - item.sourceStart)
|
||||||
|
const relativeEnd = Math.max(relativeStart, intersectionEnd - item.sourceStart)
|
||||||
|
return item.originalContent.slice(relativeStart, relativeEnd)
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLocatorText(value: string) {
|
||||||
|
return value.normalize('NFKC').replace(/[^\p{L}\p{N}]+/gu, '').toLocaleLowerCase()
|
||||||
|
}
|
||||||
|
|
||||||
|
function nearestOccurrence(haystack: string, needle: string, expectedIndex: number) {
|
||||||
|
let nearest = -1
|
||||||
|
let nearestDistance = Number.POSITIVE_INFINITY
|
||||||
|
let cursor = haystack.indexOf(needle)
|
||||||
|
while (cursor >= 0) {
|
||||||
|
const distance = Math.abs(cursor - expectedIndex)
|
||||||
|
if (distance < nearestDistance) {
|
||||||
|
nearest = cursor
|
||||||
|
nearestDistance = distance
|
||||||
|
}
|
||||||
|
cursor = haystack.indexOf(needle, cursor + 1)
|
||||||
|
}
|
||||||
|
return nearest
|
||||||
|
}
|
||||||
|
|
||||||
|
function findHighlight(
|
||||||
|
itemStrings: string[],
|
||||||
|
selectedText: string,
|
||||||
|
page: DataProcessPdfPageRange,
|
||||||
|
) {
|
||||||
|
const itemRanges: Array<{ start: number; end: number }> = []
|
||||||
|
let pageText = ''
|
||||||
|
for (const item of itemStrings) {
|
||||||
|
const start = pageText.length
|
||||||
|
pageText += normalizeLocatorText(item)
|
||||||
|
itemRanges.push({ start, end: pageText.length })
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = normalizeLocatorText(selectedText)
|
||||||
|
if (!pageText || !target) return []
|
||||||
|
const itemStart = props.selectedItem?.sourceStart ?? page.source_start
|
||||||
|
const sourceLength = Math.max(1, page.source_end - page.source_start)
|
||||||
|
const expectedRatio = Math.min(1, Math.max(0, (itemStart - page.source_start) / sourceLength))
|
||||||
|
const expectedIndex = Math.round(pageText.length * expectedRatio)
|
||||||
|
const anchorLengths = [target.length, 120, 80, 48, 24, 12, 8]
|
||||||
|
.map((length) => Math.min(length, target.length))
|
||||||
|
.filter((length, index, values) => length >= 4 && values.indexOf(length) === index)
|
||||||
|
|
||||||
|
for (const anchorLength of anchorLengths) {
|
||||||
|
const anchor = target.slice(0, anchorLength)
|
||||||
|
const matchStart = nearestOccurrence(pageText, anchor, expectedIndex)
|
||||||
|
if (matchStart < 0) continue
|
||||||
|
const matchEnd = matchStart + anchor.length
|
||||||
|
return itemRanges.reduce<number[]>((matches, range, index) => {
|
||||||
|
if (range.end > matchStart && range.start < matchEnd) matches.push(index)
|
||||||
|
return matches
|
||||||
|
}, [])
|
||||||
|
}
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
function applySelectedHighlight(page: DataProcessPdfPageRange | null) {
|
||||||
|
const item = props.selectedItem
|
||||||
|
for (const element of textLayer?.textDivs ?? []) {
|
||||||
|
element.classList.remove('is-slice-highlighted')
|
||||||
|
}
|
||||||
|
if (!item || item.sourceStart == null || item.sourceEnd == null) {
|
||||||
|
highlightState.value = item ? 'manual' : 'idle'
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!page || page.page_number !== currentPage.value || !textLayer) {
|
||||||
|
highlightState.value = 'idle'
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const matches = findHighlight(
|
||||||
|
textLayer.textContentItemsStr,
|
||||||
|
selectedTextForPage(page),
|
||||||
|
page,
|
||||||
|
)
|
||||||
|
for (const index of matches) {
|
||||||
|
textLayer.textDivs[index]?.classList.add('is-slice-highlighted')
|
||||||
|
}
|
||||||
|
highlightState.value = matches.length ? 'highlighted' : 'unmatched'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderCurrentPage(force = false) {
|
||||||
|
const document = documentRef.value
|
||||||
|
const pageContainer = pageRef.value
|
||||||
|
const canvas = canvasRef.value
|
||||||
|
const layerContainer = textLayerRef.value
|
||||||
|
const scroller = scrollRef.value
|
||||||
|
if (!document || !pageContainer || !canvas || !layerContainer || !scroller) return
|
||||||
|
|
||||||
|
const selectedPage = pageForItem(props.selectedItem)
|
||||||
|
if (!force && renderedPageNumber === currentPage.value && textLayer) {
|
||||||
|
applySelectedHighlight(selectedPage)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const sequence = ++renderSequence
|
||||||
|
renderTask?.cancel()
|
||||||
|
textLayer?.cancel()
|
||||||
|
renderTask = null
|
||||||
|
textLayer = null
|
||||||
|
rendering.value = true
|
||||||
|
errorMessage.value = ''
|
||||||
|
|
||||||
|
try {
|
||||||
|
const page = await document.getPage(currentPage.value)
|
||||||
|
if (sequence !== renderSequence) return
|
||||||
|
const baseViewport = page.getViewport({ scale: 1 })
|
||||||
|
const availableWidth = Math.max(280, scroller.clientWidth - 36)
|
||||||
|
const scale = (availableWidth / baseViewport.width) * zoom.value
|
||||||
|
const viewport = page.getViewport({ scale })
|
||||||
|
const outputScale = Math.max(1, window.devicePixelRatio || 1)
|
||||||
|
const context = canvas.getContext('2d')
|
||||||
|
if (!context) throw new Error('浏览器无法创建 PDF 画布')
|
||||||
|
|
||||||
|
renderedScale.value = scale
|
||||||
|
pageContainer.style.width = `${viewport.width}px`
|
||||||
|
pageContainer.style.height = `${viewport.height}px`
|
||||||
|
canvas.width = Math.floor(viewport.width * outputScale)
|
||||||
|
canvas.height = Math.floor(viewport.height * outputScale)
|
||||||
|
canvas.style.width = `${viewport.width}px`
|
||||||
|
canvas.style.height = `${viewport.height}px`
|
||||||
|
layerContainer.replaceChildren()
|
||||||
|
|
||||||
|
const activeRenderTask = page.render({
|
||||||
|
canvas,
|
||||||
|
viewport,
|
||||||
|
transform: outputScale === 1 ? undefined : [outputScale, 0, 0, outputScale, 0, 0],
|
||||||
|
})
|
||||||
|
renderTask = activeRenderTask
|
||||||
|
// 立即挂接取消处理，避免 ResizeObserver 触发重绘时产生未处理的取消异常。
|
||||||
|
const canvasRenderPromise = activeRenderTask.promise.catch((error: unknown) => {
|
||||||
|
if (
|
||||||
|
sequence !== renderSequence
|
||||||
|
|| (error instanceof Error && error.name === 'RenderingCancelledException')
|
||||||
|
) return
|
||||||
|
throw error
|
||||||
|
})
|
||||||
|
const textContent = await page.getTextContent()
|
||||||
|
if (sequence !== renderSequence) return
|
||||||
|
const activeTextLayer = new TextLayer({
|
||||||
|
textContentSource: textContent,
|
||||||
|
container: layerContainer,
|
||||||
|
viewport,
|
||||||
|
})
|
||||||
|
textLayer = activeTextLayer
|
||||||
|
await Promise.all([canvasRenderPromise, activeTextLayer.render()])
|
||||||
|
if (sequence !== renderSequence) return
|
||||||
|
renderedPageNumber = currentPage.value
|
||||||
|
applySelectedHighlight(selectedPage)
|
||||||
|
await nextTick()
|
||||||
|
pageContainer.querySelector<HTMLElement>('.is-slice-highlighted')
|
||||||
|
?.scrollIntoView({ block: 'center', inline: 'center', behavior: 'smooth' })
|
||||||
|
} catch (error) {
|
||||||
|
if (sequence !== renderSequence) return
|
||||||
|
errorMessage.value = error instanceof Error ? error.message : 'PDF 页面渲染失败'
|
||||||
|
} finally {
|
||||||
|
if (sequence === renderSequence) rendering.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function locateSelectedItem() {
|
||||||
|
if (!documentRef.value) return
|
||||||
|
const item = props.selectedItem
|
||||||
|
if (!item || item.sourceStart == null || item.sourceEnd == null) {
|
||||||
|
applySelectedHighlight(null)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const page = pageForItem(item)
|
||||||
|
if (!page) {
|
||||||
|
highlightState.value = 'unmatched'
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (currentPage.value !== page.page_number) {
|
||||||
|
currentPage.value = page.page_number
|
||||||
|
await renderCurrentPage(true)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await renderCurrentPage(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadPdf() {
|
||||||
|
const sequence = ++loadSequence
|
||||||
|
++renderSequence
|
||||||
|
renderedPageNumber = 0
|
||||||
|
renderTask?.cancel()
|
||||||
|
textLayer?.cancel()
|
||||||
|
await loadingTask?.destroy()
|
||||||
|
loadingTask = null
|
||||||
|
documentRef.value = null
|
||||||
|
pageRanges.value = []
|
||||||
|
pageCount.value = 0
|
||||||
|
currentPage.value = 1
|
||||||
|
errorMessage.value = ''
|
||||||
|
highlightState.value = 'idle'
|
||||||
|
if (!sourceUrl.value || props.taskId == null || props.sourceFileId == null) return
|
||||||
|
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
const task = getDocument({ url: sourceUrl.value })
|
||||||
|
loadingTask = task
|
||||||
|
const [mapping, document] = await Promise.all([
|
||||||
|
getDataProcessPdfPages(props.taskId, props.sourceFileId),
|
||||||
|
task.promise,
|
||||||
|
])
|
||||||
|
if (sequence !== loadSequence) {
|
||||||
|
await task.destroy()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
documentRef.value = document
|
||||||
|
pageRanges.value = mapping.pages
|
||||||
|
pageCount.value = document.numPages
|
||||||
|
await nextTick()
|
||||||
|
await locateSelectedItem()
|
||||||
|
if (!props.selectedItem) await renderCurrentPage(true)
|
||||||
|
} catch (error) {
|
||||||
|
if (sequence !== loadSequence) return
|
||||||
|
errorMessage.value = error instanceof Error ? error.message : 'PDF 原文件加载失败'
|
||||||
|
} finally {
|
||||||
|
if (sequence === loadSequence) loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function changePage(delta: number) {
|
||||||
|
const nextPage = Math.min(pageCount.value, Math.max(1, currentPage.value + delta))
|
||||||
|
if (nextPage === currentPage.value) return
|
||||||
|
currentPage.value = nextPage
|
||||||
|
await renderCurrentPage(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function changeZoom(delta: number) {
|
||||||
|
const nextZoom = Math.min(2, Math.max(0.6, Number((zoom.value + delta).toFixed(1))))
|
||||||
|
if (nextZoom === zoom.value) return
|
||||||
|
zoom.value = nextZoom
|
||||||
|
await renderCurrentPage(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleResizeRender() {
|
||||||
|
window.cancelAnimationFrame(resizeFrame)
|
||||||
|
resizeFrame = window.requestAnimationFrame(() => {
|
||||||
|
if (documentRef.value) void renderCurrentPage(true)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const resizeObserver = new ResizeObserver(scheduleResizeRender)
|
||||||
|
watch(scrollRef, (element, previous) => {
|
||||||
|
if (previous) resizeObserver.unobserve(previous)
|
||||||
|
if (element) resizeObserver.observe(element)
|
||||||
|
})
|
||||||
|
watch(sourceUrl, () => void loadPdf(), { immediate: true })
|
||||||
|
watch(() => props.selectedItem?.id, () => void locateSelectedItem())
|
||||||
|
|
||||||
|
onBeforeUnmount(() => {
|
||||||
|
++loadSequence
|
||||||
|
++renderSequence
|
||||||
|
window.cancelAnimationFrame(resizeFrame)
|
||||||
|
resizeObserver.disconnect()
|
||||||
|
renderTask?.cancel()
|
||||||
|
textLayer?.cancel()
|
||||||
|
void loadingTask?.destroy()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="pdf-source-viewer" aria-label="PDF 原文件预览">
|
||||||
|
<div v-if="sourceUrl" class="pdf-toolbar">
|
||||||
|
<div class="toolbar-group">
|
||||||
|
<button type="button" aria-label="上一页" :disabled="currentPage <= 1 || loading" @click="changePage(-1)">
|
||||||
|
<i class="fa fa-chevron-left" aria-hidden="true" />
|
||||||
|
</button>
|
||||||
|
<span class="page-indicator">{{ currentPage }} / {{ pageCount || '—' }}</span>
|
||||||
|
<button type="button" aria-label="下一页" :disabled="currentPage >= pageCount || loading" @click="changePage(1)">
|
||||||
|
<i class="fa fa-chevron-right" aria-hidden="true" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<span class="location-state" :class="`is-${highlightState}`" role="status">
|
||||||
|
<i v-if="highlightState === 'highlighted'" class="fa fa-map-marker" aria-hidden="true" />
|
||||||
|
{{ locationText }}
|
||||||
|
</span>
|
||||||
|
<div class="toolbar-group">
|
||||||
|
<button type="button" aria-label="缩小" :disabled="zoom <= 0.6 || loading" @click="changeZoom(-0.1)">−</button>
|
||||||
|
<span class="zoom-indicator">{{ Math.round(zoom * 100) }}%</span>
|
||||||
|
<button type="button" aria-label="放大" :disabled="zoom >= 2 || loading" @click="changeZoom(0.1)">+</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="sourceUrl" ref="scrollRef" class="pdf-scroll" tabindex="0" :aria-label="`PDF 预览：${fileName}`">
|
||||||
|
<div
|
||||||
|
ref="pageRef"
|
||||||
|
class="pdf-page"
|
||||||
|
:style="pageStyle"
|
||||||
|
:data-page-number="currentPage"
|
||||||
|
>
|
||||||
|
<canvas ref="canvasRef" class="pdf-canvas" />
|
||||||
|
<div ref="textLayerRef" class="pdf-text-layer" />
|
||||||
|
</div>
|
||||||
|
<div v-if="loading || rendering" class="pdf-loading" role="status">
|
||||||
|
<i class="fa fa-spinner fa-spin" aria-hidden="true" />
|
||||||
|
{{ loading ? '正在加载 PDF…' : '正在渲染页面…' }}
|
||||||
|
</div>
|
||||||
|
<div v-if="errorMessage" class="pdf-error" role="alert">
|
||||||
|
<i class="fa fa-exclamation-circle" aria-hidden="true" />
|
||||||
|
<strong>PDF 预览失败</strong>
|
||||||
|
<span>{{ errorMessage }}</span>
|
||||||
|
<a :href="sourceUrl" target="_blank" rel="noopener noreferrer">在新窗口打开原件</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div v-else class="pdf-unavailable" role="status">
|
||||||
|
<i class="fa fa-file-pdf-o" aria-hidden="true" />
|
||||||
|
<strong>PDF 原文件暂不可预览</strong>
|
||||||
|
<span>请重新上传该文件后再试。</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.pdf-source-viewer {
|
||||||
|
display: flex;
|
||||||
|
min-height: 0;
|
||||||
|
flex: 1;
|
||||||
|
flex-direction: column;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #525659;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-toolbar {
|
||||||
|
display: grid;
|
||||||
|
min-height: 44px;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||||
|
align-items: center;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 0 12px;
|
||||||
|
color: #f2f4f7;
|
||||||
|
background: #323639;
|
||||||
|
border-bottom: 1px solid #1f2427;
|
||||||
|
}
|
||||||
|
|
||||||
|
.toolbar-group {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 5px;
|
||||||
|
|
||||||
|
button {
|
||||||
|
display: inline-flex;
|
||||||
|
width: 30px;
|
||||||
|
height: 30px;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
color: #f2f4f7;
|
||||||
|
font: inherit;
|
||||||
|
font-size: 16px;
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
|
||||||
|
&:hover:not(:disabled) {
|
||||||
|
background: rgb(255 255 255 / 12%);
|
||||||
|
}
|
||||||
|
|
||||||
|
&:disabled {
|
||||||
|
color: #7d8387;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.page-indicator,
|
||||||
|
.zoom-indicator {
|
||||||
|
min-width: 54px;
|
||||||
|
color: #e4e7ec;
|
||||||
|
font-size: 12px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.location-state {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
color: #d0d5dd;
|
||||||
|
font-size: 12px;
|
||||||
|
text-align: center;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
|
||||||
|
&.is-highlighted {
|
||||||
|
color: #ffd666;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-unmatched {
|
||||||
|
color: #fdb022;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-scroll {
|
||||||
|
position: relative;
|
||||||
|
display: flex;
|
||||||
|
min-height: 538px;
|
||||||
|
flex: 1;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: center;
|
||||||
|
overflow: auto;
|
||||||
|
padding: 18px;
|
||||||
|
outline: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-page {
|
||||||
|
position: relative;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #fff;
|
||||||
|
box-shadow: 0 2px 12px rgb(0 0 0 / 34%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-canvas {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer {
|
||||||
|
--min-font-size: 1;
|
||||||
|
--text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
|
||||||
|
--min-font-size-inv: calc(1 / var(--min-font-size));
|
||||||
|
|
||||||
|
position: absolute;
|
||||||
|
z-index: 1;
|
||||||
|
inset: 0;
|
||||||
|
overflow: clip;
|
||||||
|
color-scheme: only light;
|
||||||
|
line-height: 1;
|
||||||
|
letter-spacing: normal;
|
||||||
|
word-spacing: normal;
|
||||||
|
text-align: initial;
|
||||||
|
text-size-adjust: none;
|
||||||
|
forced-color-adjust: none;
|
||||||
|
transform-origin: 0 0;
|
||||||
|
caret-color: CanvasText;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer :deep(span),
|
||||||
|
.pdf-text-layer :deep(br) {
|
||||||
|
position: absolute;
|
||||||
|
color: transparent;
|
||||||
|
white-space: pre;
|
||||||
|
cursor: text;
|
||||||
|
user-select: text;
|
||||||
|
transform-origin: 0 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer > :deep(:not(.markedContent)),
|
||||||
|
.pdf-text-layer :deep(.markedContent span:not(.markedContent)) {
|
||||||
|
--font-height: 0;
|
||||||
|
--scale-x: 1;
|
||||||
|
--rotate: 0deg;
|
||||||
|
|
||||||
|
z-index: 1;
|
||||||
|
font-size: calc(var(--text-scale-factor) * var(--font-height));
|
||||||
|
transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer :deep(.markedContent) {
|
||||||
|
display: contents;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer :deep(.is-slice-highlighted) {
|
||||||
|
margin: -2px;
|
||||||
|
padding: 2px;
|
||||||
|
background: rgb(255 202 40 / 48%);
|
||||||
|
border-radius: 3px;
|
||||||
|
box-shadow: 0 0 0 1px rgb(245 158 11 / 38%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-text-layer :deep(::selection) {
|
||||||
|
color: transparent;
|
||||||
|
background: rgb(37 99 235 / 30%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-loading,
|
||||||
|
.pdf-error {
|
||||||
|
position: absolute;
|
||||||
|
z-index: 3;
|
||||||
|
top: 50%;
|
||||||
|
left: 50%;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
transform: translate(-50%, -50%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-loading {
|
||||||
|
gap: 8px;
|
||||||
|
padding: 10px 14px;
|
||||||
|
color: #f2f4f7;
|
||||||
|
font-size: 13px;
|
||||||
|
background: rgb(31 36 39 / 86%);
|
||||||
|
border-radius: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-error {
|
||||||
|
width: min(360px, calc(100% - 32px));
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 20px;
|
||||||
|
color: #667085;
|
||||||
|
text-align: center;
|
||||||
|
background: #fff;
|
||||||
|
border-radius: 8px;
|
||||||
|
box-shadow: 0 4px 20px rgb(0 0 0 / 22%);
|
||||||
|
|
||||||
|
i {
|
||||||
|
color: #d92d20;
|
||||||
|
font-size: 28px;
|
||||||
|
}
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: #344054;
|
||||||
|
}
|
||||||
|
|
||||||
|
span,
|
||||||
|
a {
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-unavailable {
|
||||||
|
display: flex;
|
||||||
|
width: 100%;
|
||||||
|
flex: 1;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 8px;
|
||||||
|
color: #667085;
|
||||||
|
background: #f8f9fb;
|
||||||
|
|
||||||
|
i {
|
||||||
|
color: #d92d20;
|
||||||
|
font-size: 36px;
|
||||||
|
}
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: #344054;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
span {
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 900px) {
|
||||||
|
.pdf-toolbar {
|
||||||
|
gap: 5px;
|
||||||
|
padding: 0 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.location-state {
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.pdf-scroll {
|
||||||
|
min-height: 420px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, nextTick, ref, watch } from 'vue'
|
import { computed, nextTick, ref, watch } from 'vue'
|
||||||
|
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||||
import { sourceLines } from './previewModel'
|
import { sourceLines } from './previewModel'
|
||||||
import type { PreviewItem, ProcessType } from './types'
|
import type { PreviewItem, ProcessType } from './types'
|
||||||
|
|
||||||
@@ -9,6 +10,8 @@ const props = defineProps<{
|
|||||||
selectedId: string | null
|
selectedId: string | null
|
||||||
processType: ProcessType
|
processType: ProcessType
|
||||||
fileName: string
|
fileName: string
|
||||||
|
taskId: string | number | null
|
||||||
|
sourceFileId: string | number | null
|
||||||
files: { id: string; name: string; count: number; modifiedCount: number }[]
|
files: { id: string; name: string; count: number; modifiedCount: number }[]
|
||||||
selectedFileId: string | null
|
selectedFileId: string | null
|
||||||
}>()
|
}>()
|
||||||
@@ -31,6 +34,7 @@ const editorDraft = ref('')
|
|||||||
const lines = computed(() => sourceLines(props.sourceText))
|
const lines = computed(() => sourceLines(props.sourceText))
|
||||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||||
|
const isPdfSource = computed(() => /\.pdf$/i.test(props.fileName))
|
||||||
|
|
||||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||||
const matchesSearch = !search.value.trim()
|
const matchesSearch = !search.value.trim()
|
||||||
@@ -102,7 +106,7 @@ watch(selectedItem, async (item) => {
|
|||||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
if (item.sourceStart == null) return
|
if (isPdfSource.value || item.sourceStart == null) return
|
||||||
await nextTick()
|
await nextTick()
|
||||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||||
@@ -160,7 +164,14 @@ function lineRange(item: PreviewItem) {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
<PdfSourceViewer
|
||||||
|
v-if="isPdfSource"
|
||||||
|
:task-id="taskId"
|
||||||
|
:source-file-id="sourceFileId"
|
||||||
|
:file-name="fileName"
|
||||||
|
:selected-item="selectedItem ?? null"
|
||||||
|
/>
|
||||||
|
<div v-else ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
||||||
<div
|
<div
|
||||||
v-for="line in lines"
|
v-for="line in lines"
|
||||||
:key="line.number"
|
:key="line.number"
|
||||||
@@ -198,6 +209,8 @@ function lineRange(item: PreviewItem) {
|
|||||||
:key="item.id"
|
:key="item.id"
|
||||||
class="preview-item"
|
class="preview-item"
|
||||||
:class="{ 'is-active': item.id === selectedItem?.id }"
|
:class="{ 'is-active': item.id === selectedItem?.id }"
|
||||||
|
:data-preview-id="item.id"
|
||||||
|
:aria-selected="item.id === selectedItem?.id"
|
||||||
role="button"
|
role="button"
|
||||||
tabindex="0"
|
tabindex="0"
|
||||||
@click="selectItem(item.id)"
|
@click="selectItem(item.id)"
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ const props = defineProps<{
|
|||||||
externalSource: ExternalDataSource
|
externalSource: ExternalDataSource
|
||||||
externalPulling: boolean
|
externalPulling: boolean
|
||||||
externalConnected: boolean
|
externalConnected: boolean
|
||||||
|
previewBuilding: boolean
|
||||||
|
sourceUploading: boolean
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
@@ -31,14 +33,23 @@ const AUTH_MODES = [
|
|||||||
|
|
||||||
const FILE_PAGE_SIZE = 10
|
const FILE_PAGE_SIZE = 10
|
||||||
const currentFilePage = ref(1)
|
const currentFilePage = ref(1)
|
||||||
|
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
|
||||||
|
|
||||||
|
const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
|
||||||
|
queued: { label: '等待上传', icon: 'fa-clock-o' },
|
||||||
|
uploading: { label: '正在上传', icon: 'fa-cloud-upload' },
|
||||||
|
waiting: { label: '等待切分', icon: 'fa-clock-o' },
|
||||||
|
processing: { label: '正在切分', icon: 'fa-spinner fa-spin' },
|
||||||
|
success: { label: '切分完成', icon: 'fa-check-circle' },
|
||||||
|
'upload-failed': { label: '上传失败', icon: 'fa-exclamation-circle' },
|
||||||
|
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
|
||||||
|
}
|
||||||
|
|
||||||
const isExternal = computed(() => props.processType === 'external')
|
const isExternal = computed(() => props.processType === 'external')
|
||||||
|
|
||||||
// 后端首版严格支持这些可验证的文本格式；不要把无法解析的二进制文档
|
|
||||||
// 静默替换成示例正文。
|
|
||||||
const uploadAccept = computed(() => props.processType === 'unstructured'
|
const uploadAccept = computed(() => props.processType === 'unstructured'
|
||||||
? '.txt,.md,.json,.jsonl'
|
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
|
||||||
: '.json,.jsonl,.csv,.txt,.md')
|
: '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx')
|
||||||
|
|
||||||
const pagedUploadedFiles = computed(() => {
|
const pagedUploadedFiles = computed(() => {
|
||||||
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
|
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
|
||||||
@@ -67,6 +78,48 @@ function formatSize(size: number) {
|
|||||||
if (size >= 1024 * 1024) return `${(size / 1024 / 1024).toFixed(1)} MB`
|
if (size >= 1024 * 1024) return `${(size / 1024 / 1024).toFixed(1)} MB`
|
||||||
return `${(size / 1024).toFixed(1)} KB`
|
return `${(size / 1024).toFixed(1)} KB`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getFileStage(file: UploadedDataFile): FileStage {
|
||||||
|
if (file.status === 'queued' || file.status === 'uploading') return file.status
|
||||||
|
if (file.status === 'failed') return 'upload-failed'
|
||||||
|
if (file.previewStatus === 'processing' || file.previewStatus === 'success') return file.previewStatus
|
||||||
|
if (file.previewStatus === 'failed') return 'preview-failed'
|
||||||
|
return 'waiting'
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFileProgress(file: UploadedDataFile) {
|
||||||
|
const stage = getFileStage(file)
|
||||||
|
const progress = stage === 'queued' || stage === 'uploading' || stage === 'upload-failed'
|
||||||
|
? file.uploadProgress
|
||||||
|
: stage === 'waiting' ? 100 : file.previewProgress ?? 0
|
||||||
|
return Math.min(100, Math.max(0, progress))
|
||||||
|
}
|
||||||
|
|
||||||
|
function isFileProcessing(file: UploadedDataFile) {
|
||||||
|
return getFileStage(file) === 'processing'
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFileBarPercentage(file: UploadedDataFile) {
|
||||||
|
// Element Plus 的不定进度动画需要非零宽度；这里不作为完成百分比展示。
|
||||||
|
return isFileProcessing(file) ? 100 : getFileProgress(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFileProgressStatus(file: UploadedDataFile): 'success' | 'exception' | undefined {
|
||||||
|
const stage = getFileStage(file)
|
||||||
|
if (stage === 'waiting' || stage === 'success') return 'success'
|
||||||
|
if (stage === 'upload-failed' || stage === 'preview-failed') return 'exception'
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFileProgressText(file: UploadedDataFile) {
|
||||||
|
if (isFileProcessing(file)) return '处理中'
|
||||||
|
if (getFileStage(file) === 'waiting') return '已上传'
|
||||||
|
return `${getFileProgress(file)}%`
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFileError(file: UploadedDataFile) {
|
||||||
|
return file.uploadError || file.previewError
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -167,7 +220,7 @@ function formatSize(size: number) {
|
|||||||
<div class="external-actions">
|
<div class="external-actions">
|
||||||
<el-button
|
<el-button
|
||||||
:loading="externalPulling && !externalConnected"
|
:loading="externalPulling && !externalConnected"
|
||||||
:disabled="externalPulling"
|
:disabled="externalPulling || previewBuilding"
|
||||||
plain
|
plain
|
||||||
@click="emit('test-connection')"
|
@click="emit('test-connection')"
|
||||||
>
|
>
|
||||||
@@ -176,7 +229,7 @@ function formatSize(size: number) {
|
|||||||
<el-button
|
<el-button
|
||||||
type="primary"
|
type="primary"
|
||||||
:loading="externalPulling"
|
:loading="externalPulling"
|
||||||
:disabled="externalPulling"
|
:disabled="externalPulling || previewBuilding"
|
||||||
@click="emit('pull-data')"
|
@click="emit('pull-data')"
|
||||||
>
|
>
|
||||||
拉取数据
|
拉取数据
|
||||||
@@ -200,10 +253,34 @@ function formatSize(size: number) {
|
|||||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 拉取成功</span>
|
<div
|
||||||
|
class="file-preview-progress"
|
||||||
|
:class="`is-${getFileStage(file)}`"
|
||||||
|
:title="getFileError(file)"
|
||||||
|
>
|
||||||
|
<div class="file-status" role="status" aria-live="polite">
|
||||||
|
<span>
|
||||||
|
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||||
|
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||||
|
</span>
|
||||||
|
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||||
|
</div>
|
||||||
|
<el-progress
|
||||||
|
:percentage="getFileBarPercentage(file)"
|
||||||
|
:indeterminate="isFileProcessing(file)"
|
||||||
|
:duration="1.5"
|
||||||
|
:stroke-width="5"
|
||||||
|
:show-text="false"
|
||||||
|
:status="getFileProgressStatus(file)"
|
||||||
|
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||||
|
:aria-valuetext="isFileProcessing(file) ? '正在切分，进度未知' : getFileProgressText(file)"
|
||||||
|
/>
|
||||||
|
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||||
|
</div>
|
||||||
<el-button
|
<el-button
|
||||||
link
|
link
|
||||||
type="danger"
|
type="danger"
|
||||||
|
:disabled="previewBuilding || file.status === 'uploading'"
|
||||||
:aria-label="`删除数据集 ${file.name}`"
|
:aria-label="`删除数据集 ${file.name}`"
|
||||||
@click="emit('remove-file', file.uid)"
|
@click="emit('remove-file', file.uid)"
|
||||||
>
|
>
|
||||||
@@ -233,7 +310,13 @@ function formatSize(size: number) {
|
|||||||
<h3 id="source-upload-title">源数据上传</h3>
|
<h3 id="source-upload-title">源数据上传</h3>
|
||||||
<p>上传后可在下一步检查内容和切分效果，支持同时添加多个文件</p>
|
<p>上传后可在下一步检查内容和切分效果，支持同时添加多个文件</p>
|
||||||
</div>
|
</div>
|
||||||
<el-button v-if="uploadedFiles.length === 0" link type="primary" @click="emit('use-sample')">
|
<el-button
|
||||||
|
v-if="uploadedFiles.length === 0"
|
||||||
|
link
|
||||||
|
type="primary"
|
||||||
|
:disabled="previewBuilding"
|
||||||
|
@click="emit('use-sample')"
|
||||||
|
>
|
||||||
使用示例数据
|
使用示例数据
|
||||||
</el-button>
|
</el-button>
|
||||||
</div>
|
</div>
|
||||||
@@ -243,6 +326,7 @@ function formatSize(size: number) {
|
|||||||
drag
|
drag
|
||||||
multiple
|
multiple
|
||||||
:accept="uploadAccept"
|
:accept="uploadAccept"
|
||||||
|
:disabled="previewBuilding"
|
||||||
:auto-upload="false"
|
:auto-upload="false"
|
||||||
:show-file-list="false"
|
:show-file-list="false"
|
||||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||||
@@ -253,25 +337,29 @@ function formatSize(size: number) {
|
|||||||
<template #tip>
|
<template #tip>
|
||||||
<div class="el-upload__tip">
|
<div class="el-upload__tip">
|
||||||
{{ processType === 'unstructured'
|
{{ processType === 'unstructured'
|
||||||
? '支持 TXT、Markdown、PDF、Word、JSON、JSONL，单文件不超过 200MB'
|
? '支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON；旧版 DOC/PPT 请先转换，单文件不超过 200MB'
|
||||||
: '支持 JSON、JSONL、CSV、Excel，单文件不超过 200MB' }}
|
: '支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX；旧版 XLS 请先转换，单文件不超过 200MB' }}
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
</el-upload>
|
</el-upload>
|
||||||
|
|
||||||
<section v-else class="uploaded-file-list" aria-label="已上传文件列表">
|
<section v-else class="uploaded-file-list" aria-label="已上传文件列表">
|
||||||
<div class="uploaded-file-list-header">
|
<div class="uploaded-file-list-header">
|
||||||
<span>已添加 {{ uploadedFiles.length }} 个文件</span>
|
<span>
|
||||||
|
已选择 {{ uploadedFiles.length }} 个文件
|
||||||
|
<small v-if="sourceUploading" class="upload-queue-status"> · 正在逐个上传</small>
|
||||||
|
</span>
|
||||||
<div class="continue-upload">
|
<div class="continue-upload">
|
||||||
<el-upload
|
<el-upload
|
||||||
multiple
|
multiple
|
||||||
:accept="uploadAccept"
|
:accept="uploadAccept"
|
||||||
|
:disabled="previewBuilding"
|
||||||
:auto-upload="false"
|
:auto-upload="false"
|
||||||
:show-file-list="false"
|
:show-file-list="false"
|
||||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||||
aria-label="继续添加源数据文件"
|
aria-label="继续添加源数据文件"
|
||||||
>
|
>
|
||||||
<el-button size="small" type="primary">继续上传</el-button>
|
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
|
||||||
</el-upload>
|
</el-upload>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -285,10 +373,34 @@ function formatSize(size: number) {
|
|||||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 校验通过</span>
|
<div
|
||||||
|
class="file-preview-progress"
|
||||||
|
:class="`is-${getFileStage(file)}`"
|
||||||
|
:title="getFileError(file)"
|
||||||
|
>
|
||||||
|
<div class="file-status" role="status" aria-live="polite">
|
||||||
|
<span>
|
||||||
|
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||||
|
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||||
|
</span>
|
||||||
|
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||||
|
</div>
|
||||||
|
<el-progress
|
||||||
|
:percentage="getFileBarPercentage(file)"
|
||||||
|
:indeterminate="isFileProcessing(file)"
|
||||||
|
:duration="1.5"
|
||||||
|
:stroke-width="5"
|
||||||
|
:show-text="false"
|
||||||
|
:status="getFileProgressStatus(file)"
|
||||||
|
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||||
|
:aria-valuetext="isFileProcessing(file) ? '正在切分，进度未知' : getFileProgressText(file)"
|
||||||
|
/>
|
||||||
|
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||||
|
</div>
|
||||||
<el-button
|
<el-button
|
||||||
link
|
link
|
||||||
type="danger"
|
type="danger"
|
||||||
|
:disabled="previewBuilding || file.status === 'uploading'"
|
||||||
:aria-label="`删除文件 ${file.name}`"
|
:aria-label="`删除文件 ${file.name}`"
|
||||||
@click="emit('remove-file', file.uid)"
|
@click="emit('remove-file', file.uid)"
|
||||||
>
|
>
|
||||||
@@ -441,6 +553,11 @@ function formatSize(size: number) {
|
|||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.upload-queue-status {
|
||||||
|
color: #5b50f2;
|
||||||
|
font-size: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
.uploaded-file-pagination {
|
.uploaded-file-pagination {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
@@ -495,9 +612,59 @@ function formatSize(size: number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.file-preview-progress {
|
||||||
|
display: flex;
|
||||||
|
flex: 0 1 220px;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 5px;
|
||||||
|
min-width: 150px;
|
||||||
|
|
||||||
|
&.is-queued .file-status,
|
||||||
|
&.is-waiting .file-status {
|
||||||
|
color: #8a93a3;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-uploading .file-status,
|
||||||
|
&.is-processing .file-status {
|
||||||
|
color: #5b50f2;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-success .file-status {
|
||||||
|
color: #2ca66a;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-upload-failed .file-status,
|
||||||
|
&.is-preview-failed .file-status,
|
||||||
|
.file-preview-error {
|
||||||
|
color: #d94b4b;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
.file-status {
|
.file-status {
|
||||||
color: #2ca66a;
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 8px;
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
|
|
||||||
|
> span:first-child {
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.file-progress-value {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
.file-preview-error {
|
||||||
|
overflow: hidden;
|
||||||
|
font-size: 11px;
|
||||||
|
line-height: 1.4;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.uploaded-file :deep(.el-button) {
|
.uploaded-file :deep(.el-button) {
|
||||||
@@ -514,8 +681,12 @@ function formatSize(size: number) {
|
|||||||
padding: 8px 10px;
|
padding: 8px 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.file-status {
|
.file-preview-progress {
|
||||||
flex: 0 1 auto;
|
flex: 0 1 auto;
|
||||||
|
min-width: 130px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.file-status {
|
||||||
line-height: 1.4;
|
line-height: 1.4;
|
||||||
white-space: normal;
|
white-space: normal;
|
||||||
}
|
}
|
||||||
@@ -545,7 +716,8 @@ function formatSize(size: number) {
|
|||||||
min-width: calc(100% - 40px);
|
min-width: calc(100% - 40px);
|
||||||
}
|
}
|
||||||
|
|
||||||
.file-status {
|
.file-preview-progress {
|
||||||
|
flex: 1 0 calc(100% - 38px);
|
||||||
margin-left: 38px;
|
margin-left: 38px;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,12 +22,12 @@ const PREPROCESS_OPTIONS: Array<{
|
|||||||
label: string
|
label: string
|
||||||
description: string
|
description: string
|
||||||
}> = [
|
}> = [
|
||||||
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列，并剔除关键字段残缺的数据行' },
|
||||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
{ value: 'detect_structure', label: '识别表格结构', description: '识别多级表头与合并单元格，并将嵌套字段展平' },
|
||||||
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
{ value: 'deduplicate', label: '重复数据去重', description: '基于整行精确匹配和关键字段组合删除重复记录' },
|
||||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一编码、空白、字段名和 JSON 序列化格式' },
|
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值，并过滤乱码等异常记录' },
|
||||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||||
]
|
]
|
||||||
|
|
||||||
function updateField<K extends keyof StructuredProcessOptions>(
|
function updateField<K extends keyof StructuredProcessOptions>(
|
||||||
@@ -43,9 +43,11 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
|||||||
|
|
||||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||||
const preprocessOptions = value.filter(
|
const preprocessOptions = Array.from(new Set(value.filter(
|
||||||
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
|
(option): option is PreprocessOption => (
|
||||||
)
|
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||||
|
),
|
||||||
|
)))
|
||||||
updateField('preprocessOptions', preprocessOptions)
|
updateField('preprocessOptions', preprocessOptions)
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -30,9 +30,8 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||||||
{ value: 'semantic', label: '自动语义切分' },
|
{ value: 'structure', label: '文档结构' },
|
||||||
{ value: 'heading', label: '按标题和段落' },
|
{ value: 'fixed', label: '固定 Token' },
|
||||||
{ value: 'fixed', label: '按固定长度' },
|
|
||||||
{ value: 'custom', label: '自定义分隔符' },
|
{ value: 'custom', label: '自定义分隔符' },
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -70,9 +69,9 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
|||||||
|
|
||||||
function updateSmartPreprocess(value: string | number | boolean) {
|
function updateSmartPreprocess(value: string | number | boolean) {
|
||||||
const enabled = Boolean(value)
|
const enabled = Boolean(value)
|
||||||
const remainingOptions = props.options.preprocessOptions.filter(
|
const remainingOptions = Array.from(new Set(props.options.preprocessOptions.filter(
|
||||||
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
||||||
)
|
)))
|
||||||
updateField(
|
updateField(
|
||||||
'preprocessOptions',
|
'preprocessOptions',
|
||||||
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
||||||
@@ -80,9 +79,11 @@ function updateSmartPreprocess(value: string | number | boolean) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function updateDesensitize(value: string | number | boolean) {
|
function updateDesensitize(value: string | number | boolean) {
|
||||||
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
|
const preprocessOptions: UnstructuredPreprocessOption[] = Array.from(new Set(
|
||||||
(option) => option !== 'desensitize',
|
props.options.preprocessOptions.filter(
|
||||||
)
|
(option) => option !== 'desensitize',
|
||||||
|
),
|
||||||
|
))
|
||||||
if (Boolean(value)) preprocessOptions.push('desensitize')
|
if (Boolean(value)) preprocessOptions.push('desensitize')
|
||||||
updateField('preprocessOptions', preprocessOptions)
|
updateField('preprocessOptions', preprocessOptions)
|
||||||
}
|
}
|
||||||
@@ -116,7 +117,7 @@ defineExpose({ revealValidation })
|
|||||||
<div class="section-title-row">
|
<div class="section-title-row">
|
||||||
<div>
|
<div>
|
||||||
<h3>预处理选项</h3>
|
<h3>预处理选项</h3>
|
||||||
<p>默认使用推荐策略，只需决定是否需要脱敏</p>
|
<p>默认启用结构感知的推荐策略，只需决定是否需要脱敏</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="preprocess-option-grid">
|
<div class="preprocess-option-grid">
|
||||||
@@ -127,7 +128,7 @@ defineExpose({ revealValidation })
|
|||||||
/>
|
/>
|
||||||
<span class="preprocess-option-copy">
|
<span class="preprocess-option-copy">
|
||||||
<strong>智能预处理</strong>
|
<strong>智能预处理</strong>
|
||||||
<small>自动清理、解析、去重及保留上下文</small>
|
<small>清理无效内容、感知文档结构、合并短块、预过滤低质量内容、近重复去重，并通过重叠保护上下文</small>
|
||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
@@ -138,7 +139,7 @@ defineExpose({ revealValidation })
|
|||||||
/>
|
/>
|
||||||
<span class="preprocess-option-copy">
|
<span class="preprocess-option-copy">
|
||||||
<strong>敏感信息脱敏</strong>
|
<strong>敏感信息脱敏</strong>
|
||||||
<small>处理姓名、手机号等隐私信息</small>
|
<small>识别并脱敏姓名、手机号、邮箱和身份证号</small>
|
||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
@@ -159,7 +160,7 @@ defineExpose({ revealValidation })
|
|||||||
<div class="section-title-row">
|
<div class="section-title-row">
|
||||||
<div>
|
<div>
|
||||||
<h3>切分选项</h3>
|
<h3>切分选项</h3>
|
||||||
<p>以语义完整为优先，将长文档拆成可独立生成问答的内容块</p>
|
<p>按文档结构、固定 Token 或自定义分隔符拆分长文档</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -178,7 +179,7 @@ defineExpose({ revealValidation })
|
|||||||
:value="method.value"
|
:value="method.value"
|
||||||
/>
|
/>
|
||||||
</el-select>
|
</el-select>
|
||||||
<small>推荐使用自动语义切分，在长度限制内优先保留完整句段</small>
|
<small>推荐使用文档结构，按标题、段落和句子边界保留完整内容块</small>
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
<label class="config-field">
|
<label class="config-field">
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
|||||||
'deduplicate_content',
|
'deduplicate_content',
|
||||||
'preserve_context',
|
'preserve_context',
|
||||||
],
|
],
|
||||||
chunkMethod: 'semantic',
|
chunkMethod: 'structure',
|
||||||
chunkSize: 800,
|
chunkSize: 800,
|
||||||
chunkOverlap: 100,
|
chunkOverlap: 100,
|
||||||
minChunkSize: 100,
|
minChunkSize: 100,
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import type { DataProcessPreviewFileStatus } from '@/types/dataProcess'
|
||||||
|
|
||||||
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||||
|
|
||||||
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||||
@@ -44,7 +46,7 @@ export type UnstructuredPreprocessOption =
|
|||||||
| 'desensitize'
|
| 'desensitize'
|
||||||
| 'preserve_context'
|
| 'preserve_context'
|
||||||
|
|
||||||
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
|
export type ChunkMethod = 'structure' | 'fixed' | 'custom'
|
||||||
|
|
||||||
export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||||
preprocessOptions: UnstructuredPreprocessOption[]
|
preprocessOptions: UnstructuredPreprocessOption[]
|
||||||
@@ -62,7 +64,7 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface ExternalDataSource {
|
export interface ExternalDataSource {
|
||||||
type: string
|
type: 'postgresql'
|
||||||
url: string
|
url: string
|
||||||
authMode: 'none' | 'basic'
|
authMode: 'none' | 'basic'
|
||||||
username?: string
|
username?: string
|
||||||
@@ -75,14 +77,21 @@ export interface ExternalDataSource {
|
|||||||
export interface UploadedDataFile {
|
export interface UploadedDataFile {
|
||||||
uid: string | number
|
uid: string | number
|
||||||
sourceFileId?: string
|
sourceFileId?: string
|
||||||
|
rawFile?: File
|
||||||
name: string
|
name: string
|
||||||
size: number
|
size: number
|
||||||
count: number
|
count: number
|
||||||
content: string
|
content: string
|
||||||
fileFormat?: string
|
fileFormat?: string
|
||||||
checksumSha256?: string
|
checksumSha256?: string
|
||||||
status?: 'uploading' | 'ready' | 'failed'
|
status: 'queued' | 'uploading' | 'ready' | 'failed'
|
||||||
error?: string
|
uploadProgress: number
|
||||||
|
uploadError?: string
|
||||||
|
previewStatus?: DataProcessPreviewFileStatus
|
||||||
|
previewProgress?: number
|
||||||
|
previewError?: string
|
||||||
|
previewCount?: number
|
||||||
|
previewConfigSignature?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SourceLine {
|
export interface SourceLine {
|
||||||
|
|||||||
@@ -1,152 +0,0 @@
|
|||||||
import { nextTick, type Reactive, type Ref } from 'vue'
|
|
||||||
import { ElMessage } from 'element-plus'
|
|
||||||
import type {
|
|
||||||
ExternalDataSource,
|
|
||||||
ProcessType,
|
|
||||||
StepId,
|
|
||||||
StructuredProcessOptions,
|
|
||||||
UnstructuredProcessOptions,
|
|
||||||
} from './types'
|
|
||||||
|
|
||||||
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
|
||||||
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 7
|
|
||||||
|
|
||||||
interface DraftSnapshot {
|
|
||||||
schemaVersion?: number
|
|
||||||
taskId?: string
|
|
||||||
currentStepId?: StepId
|
|
||||||
task?: { name?: string; description?: string }
|
|
||||||
processType?: ProcessType
|
|
||||||
structuredOptions?: Partial<StructuredProcessOptions>
|
|
||||||
unstructuredOptions?: Partial<UnstructuredProcessOptions>
|
|
||||||
externalSource?: Partial<ExternalDataSource>
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DraftBindings {
|
|
||||||
taskId: Ref<string | null>
|
|
||||||
currentStepId: Readonly<Ref<StepId>>
|
|
||||||
task: Reactive<{ name: string; description: string }>
|
|
||||||
processType: Ref<ProcessType>
|
|
||||||
structuredOptions: Ref<StructuredProcessOptions>
|
|
||||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
|
||||||
externalSource: Reactive<ExternalDataSource>
|
|
||||||
restoringDraft: Ref<boolean>
|
|
||||||
dirty: Ref<boolean>
|
|
||||||
goToStep: (stepId: StepId) => void
|
|
||||||
}
|
|
||||||
|
|
||||||
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
|
|
||||||
return {
|
|
||||||
type: typeof source.type === 'string' ? source.type : 'postgresql',
|
|
||||||
url: typeof source.url === 'string' ? source.url : '',
|
|
||||||
authMode: source.authMode === 'basic' ? 'basic' as const : 'none' as const,
|
|
||||||
username: typeof source.username === 'string' ? source.username : '',
|
|
||||||
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
|
|
||||||
query: typeof source.query === 'string' ? source.query : '',
|
|
||||||
fileName: typeof source.fileName === 'string' ? source.fileName : 'external-data.jsonl',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isDraftSnapshot(value: unknown): value is DraftSnapshot {
|
|
||||||
return Boolean(value && typeof value === 'object')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。
|
|
||||||
*/
|
|
||||||
export function useDataProcessDraft(bindings: DraftBindings) {
|
|
||||||
function draftSnapshot(): DraftSnapshot {
|
|
||||||
return {
|
|
||||||
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
|
|
||||||
taskId: bindings.taskId.value || undefined,
|
|
||||||
currentStepId: bindings.currentStepId.value,
|
|
||||||
task: { ...bindings.task },
|
|
||||||
processType: bindings.processType.value,
|
|
||||||
structuredOptions: {
|
|
||||||
...bindings.structuredOptions.value,
|
|
||||||
preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions],
|
|
||||||
datasetSplit: { ...bindings.structuredOptions.value.datasetSplit },
|
|
||||||
},
|
|
||||||
unstructuredOptions: {
|
|
||||||
...bindings.unstructuredOptions.value,
|
|
||||||
preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions],
|
|
||||||
datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit },
|
|
||||||
},
|
|
||||||
externalSource: sanitizeExternalSource(bindings.externalSource),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function writeDraft(showWarning: boolean) {
|
|
||||||
try {
|
|
||||||
localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
|
|
||||||
} catch {
|
|
||||||
if (showWarning) ElMessage.warning('草稿保存失败，请检查浏览器存储空间')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function persistDraft() {
|
|
||||||
if (!bindings.restoringDraft.value) writeDraft(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
function restoreDraft() {
|
|
||||||
try {
|
|
||||||
const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
|
||||||
if (!raw) return
|
|
||||||
const snapshot: unknown = JSON.parse(raw)
|
|
||||||
if (!isDraftSnapshot(snapshot)) return
|
|
||||||
|
|
||||||
bindings.restoringDraft.value = true
|
|
||||||
bindings.goToStep('create')
|
|
||||||
bindings.taskId.value = typeof snapshot.taskId === 'string' ? snapshot.taskId : null
|
|
||||||
bindings.task.name = snapshot.task?.name || ''
|
|
||||||
bindings.task.description = snapshot.task?.description || ''
|
|
||||||
bindings.processType.value = snapshot.processType === 'unstructured' || snapshot.processType === 'external'
|
|
||||||
? snapshot.processType
|
|
||||||
: 'structured'
|
|
||||||
|
|
||||||
if (snapshot.structuredOptions) {
|
|
||||||
bindings.structuredOptions.value = {
|
|
||||||
...bindings.structuredOptions.value,
|
|
||||||
...snapshot.structuredOptions,
|
|
||||||
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
|
|
||||||
? snapshot.structuredOptions.preprocessOptions
|
|
||||||
: bindings.structuredOptions.value.preprocessOptions,
|
|
||||||
datasetSplit: {
|
|
||||||
...bindings.structuredOptions.value.datasetSplit,
|
|
||||||
...snapshot.structuredOptions.datasetSplit,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (snapshot.unstructuredOptions) {
|
|
||||||
bindings.unstructuredOptions.value = {
|
|
||||||
...bindings.unstructuredOptions.value,
|
|
||||||
...snapshot.unstructuredOptions,
|
|
||||||
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
|
|
||||||
? snapshot.unstructuredOptions.preprocessOptions
|
|
||||||
: bindings.unstructuredOptions.value.preprocessOptions,
|
|
||||||
datasetSplit: {
|
|
||||||
...bindings.unstructuredOptions.value.datasetSplit,
|
|
||||||
...snapshot.unstructuredOptions.datasetSplit,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
|
|
||||||
password: '',
|
|
||||||
})
|
|
||||||
bindings.dirty.value = false
|
|
||||||
|
|
||||||
nextTick(() => {
|
|
||||||
bindings.restoringDraft.value = false
|
|
||||||
// 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。
|
|
||||||
writeDraft(false)
|
|
||||||
})
|
|
||||||
ElMessage.info('已恢复上次的任务配置')
|
|
||||||
} catch {
|
|
||||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { draftSnapshot, persistDraft, restoreDraft }
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
import { buildDataProcessPreview } from '@/api/modules/dataProcess'
|
||||||
|
import type { DataProcessPreviewFileBuildProgress } from '@/types/dataProcess'
|
||||||
|
|
||||||
|
export function useDataProcessPreviewBuild() {
|
||||||
|
async function buildPreviewsByFile(
|
||||||
|
taskId: string | number,
|
||||||
|
sourceFileIds: Array<string | number>,
|
||||||
|
onProgress: (progress: DataProcessPreviewFileBuildProgress) => void,
|
||||||
|
) {
|
||||||
|
const results: DataProcessPreviewFileBuildProgress[] = []
|
||||||
|
for (const sourceFileId of sourceFileIds) {
|
||||||
|
onProgress({ source_file_id: sourceFileId, status: 'processing', progress: 0 })
|
||||||
|
let progress: DataProcessPreviewFileBuildProgress
|
||||||
|
try {
|
||||||
|
const built = await buildDataProcessPreview(taskId, {
|
||||||
|
replace_existing: true,
|
||||||
|
source_file_ids: [sourceFileId],
|
||||||
|
})
|
||||||
|
const sourceId = String(sourceFileId)
|
||||||
|
const fileResult = built.files?.find((item) => String(item.source_file_id) === sourceId)
|
||||||
|
const previewCount = fileResult?.preview_count
|
||||||
|
?? built.file_counts?.[sourceId]
|
||||||
|
?? built.items.filter((item) => String(item.source_file_id) === sourceId).length
|
||||||
|
progress = previewCount > 0
|
||||||
|
? { source_file_id: sourceFileId, status: 'success', progress: 100, preview_count: previewCount }
|
||||||
|
: {
|
||||||
|
source_file_id: sourceFileId,
|
||||||
|
status: 'failed',
|
||||||
|
progress: 0,
|
||||||
|
preview_count: 0,
|
||||||
|
error: '该文件没有生成可用的预览条目，请检查内容或切分配置',
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
progress = {
|
||||||
|
source_file_id: sourceFileId,
|
||||||
|
status: 'failed',
|
||||||
|
progress: 0,
|
||||||
|
error: error instanceof Error && error.message ? error.message : '切分失败，请重试',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
results.push(progress)
|
||||||
|
onProgress(progress)
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
return { buildPreviewsByFile }
|
||||||
|
}
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
import { ref, type Ref } from 'vue'
|
||||||
|
import {
|
||||||
|
getDataProcessSourceContent,
|
||||||
|
uploadDataProcessSourceFiles,
|
||||||
|
type DataProcessSourceFile,
|
||||||
|
} from '@/api/modules/dataProcess'
|
||||||
|
import type { ProcessType, UploadedDataFile } from './types'
|
||||||
|
|
||||||
|
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||||
|
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||||
|
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||||
|
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||||
|
])
|
||||||
|
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||||
|
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||||
|
const MAX_SOURCE_FILE_COUNT = 20
|
||||||
|
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||||
|
|
||||||
|
interface SourceUploadJob {
|
||||||
|
uid: string
|
||||||
|
file: File
|
||||||
|
extension: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SourceUploadOptions {
|
||||||
|
taskId: Ref<string | null>
|
||||||
|
uploadedFiles: Ref<UploadedDataFile[]>
|
||||||
|
onUploaded: (file: UploadedDataFile) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
type SourceFileValidation =
|
||||||
|
| { valid: true; extension: string }
|
||||||
|
| { valid: false; severity: 'error' | 'warning'; message: string }
|
||||||
|
|
||||||
|
export function validateSourceFileSelection(
|
||||||
|
raw: File,
|
||||||
|
processType: ProcessType,
|
||||||
|
selectedFiles: UploadedDataFile[],
|
||||||
|
): SourceFileValidation {
|
||||||
|
if (raw.size > MAX_SOURCE_FILE_BYTES) {
|
||||||
|
return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
|
||||||
|
}
|
||||||
|
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||||
|
if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
severity: 'error',
|
||||||
|
message: '旧版 DOC、XLS、PPT 文件暂不支持，请分别转换为 DOCX、XLSX、PPTX 后上传',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const supportedExtensions = processType === 'unstructured'
|
||||||
|
? UNSTRUCTURED_FILE_EXTENSIONS
|
||||||
|
: STRUCTURED_FILE_EXTENSIONS
|
||||||
|
if (!supportedExtensions.has(extension)) {
|
||||||
|
return {
|
||||||
|
valid: false,
|
||||||
|
severity: 'error',
|
||||||
|
message: processType === 'unstructured'
|
||||||
|
? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
|
||||||
|
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||||
|
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||||
|
}
|
||||||
|
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||||
|
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||||
|
}
|
||||||
|
const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
|
||||||
|
if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
|
||||||
|
return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
|
||||||
|
}
|
||||||
|
return { valid: true, extension }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapDataProcessSourceFile(
|
||||||
|
file: DataProcessSourceFile,
|
||||||
|
content = '',
|
||||||
|
): UploadedDataFile {
|
||||||
|
return {
|
||||||
|
uid: String(file.id),
|
||||||
|
sourceFileId: String(file.id),
|
||||||
|
name: file.name,
|
||||||
|
size: file.size_bytes,
|
||||||
|
count: file.record_count,
|
||||||
|
content,
|
||||||
|
fileFormat: file.file_format,
|
||||||
|
checksumSha256: file.checksum_sha256,
|
||||||
|
status: 'ready',
|
||||||
|
uploadProgress: 100,
|
||||||
|
previewStatus: 'waiting',
|
||||||
|
previewProgress: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getUploadErrorMessage(error: unknown) {
|
||||||
|
const responseData = (error as {
|
||||||
|
response?: { data?: { detail?: string | { message?: string }; message?: string } }
|
||||||
|
})?.response?.data
|
||||||
|
const detail = responseData?.detail
|
||||||
|
if (typeof detail === 'string') return detail
|
||||||
|
if (detail?.message) return detail.message
|
||||||
|
if (responseData?.message) return responseData.message
|
||||||
|
if (error instanceof Error && error.message) return error.message
|
||||||
|
return '上传失败，请删除该文件后重新选择'
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||||
|
const sourceUploading = ref(false)
|
||||||
|
const queue: SourceUploadJob[] = []
|
||||||
|
let worker: Promise<void> | null = null
|
||||||
|
|
||||||
|
async function uploadOne(job: SourceUploadJob) {
|
||||||
|
const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
|
||||||
|
if (!pending) return
|
||||||
|
|
||||||
|
const currentTaskId = options.taskId.value
|
||||||
|
if (!currentTaskId) {
|
||||||
|
pending.status = 'failed'
|
||||||
|
pending.uploadError = '任务尚未创建，请返回模型选择步骤后重试'
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
pending.status = 'uploading'
|
||||||
|
pending.uploadProgress = 0
|
||||||
|
pending.uploadError = undefined
|
||||||
|
|
||||||
|
try {
|
||||||
|
let content = ''
|
||||||
|
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||||
|
try {
|
||||||
|
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||||
|
} catch {
|
||||||
|
throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
|
||||||
|
}
|
||||||
|
if (!content.trim()) throw new Error('不能上传空文件')
|
||||||
|
}
|
||||||
|
|
||||||
|
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||||
|
pending.uploadProgress = progress
|
||||||
|
})
|
||||||
|
const source = uploaded.files[0]
|
||||||
|
if (!source) throw new Error('后端未返回源文件记录')
|
||||||
|
|
||||||
|
// 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
|
||||||
|
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||||
|
rawFile: job.file,
|
||||||
|
status: 'uploading',
|
||||||
|
uploadProgress: 99,
|
||||||
|
})
|
||||||
|
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||||
|
try {
|
||||||
|
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||||
|
start_line: 1,
|
||||||
|
line_count: 10_000,
|
||||||
|
})
|
||||||
|
pending.content = parsed.content
|
||||||
|
} catch {
|
||||||
|
// 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pending.content = content
|
||||||
|
}
|
||||||
|
|
||||||
|
pending.status = 'ready'
|
||||||
|
pending.uploadProgress = 100
|
||||||
|
options.onUploaded(pending)
|
||||||
|
} catch (error) {
|
||||||
|
pending.status = 'failed'
|
||||||
|
pending.uploadError = getUploadErrorMessage(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function drainQueue() {
|
||||||
|
sourceUploading.value = true
|
||||||
|
try {
|
||||||
|
while (queue.length) {
|
||||||
|
const job = queue.shift()
|
||||||
|
if (job) await uploadOne(job)
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
sourceUploading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueSourceUpload(job: SourceUploadJob) {
|
||||||
|
queue.push(job)
|
||||||
|
if (worker) return
|
||||||
|
worker = drainQueue().finally(() => {
|
||||||
|
worker = null
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
enqueueSourceUpload,
|
||||||
|
sourceUploading,
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user