update
This commit is contained in:
wangjiming
263
frontend/package-lock.json
generated
263
frontend/package-lock.json
generated
@@ -16,6 +16,7 @@
|
||||
"element-plus": "^2.9.1",
|
||||
"marked": "^15.0.5",
|
||||
"md-editor-v3": "^5.1.4",
|
||||
"pdfjs-dist": "^6.2.108",
|
||||
"pinia": "^2.3.0",
|
||||
"vue": "^3.5.13",
|
||||
"vue-echarts": "^8.0.1",
|
||||
@@ -1213,6 +1214,256 @@
|
||||
"integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas/-/canvas-1.0.3.tgz",
|
||||
"integrity": "sha512-OlI657a5XXvKGFX7kNeIzJ8rO7IXt87Mqu2H8rXE46viAuOfum/JA7ysX7+eBhxNKznT+RCZh418mndlcFX3+w==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "1.0.3",
|
||||
"@napi-rs/canvas-darwin-arm64": "1.0.3",
|
||||
"@napi-rs/canvas-darwin-x64": "1.0.3",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "1.0.3",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "1.0.3",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "1.0.3",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "1.0.3",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "1.0.3",
|
||||
"@napi-rs/canvas-linux-x64-musl": "1.0.3",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "1.0.3",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.3.tgz",
|
||||
"integrity": "sha512-7kSCdUhoXiO+AaIMXdBGdtp6EctZNkmF62Rea/BmVQlwKaM3bBhOzyGUzxyxz9dv5vdBfpyAaxhSRSJF4kqK4A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.3.tgz",
|
||||
"integrity": "sha512-ds14V1BPagLszQyaDTeggny5fNeTCqsUQ5QhFj9VDxSEfzrVxXtdbR0LoFyKa0Siaaw8KvqSk4t7k/WoZJwvbg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.3.tgz",
|
||||
"integrity": "sha512-qof3LRAAycmkV2I1izZo9RoSHF8kCQr5O05sFwv0jK8rSdYV6KHVwimo6Qb7RxZj40WHKbLHm5JDaUF0o5XUAA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.3.tgz",
|
||||
"integrity": "sha512-FU2kKZLmolHA9+KcUA+l1+xH3WTLUUTQDU/kLv9SEUr2TrRPu94aytOeizFJDHPs/QBcw4QL1mCQhetQXYBbag==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.3.tgz",
|
||||
"integrity": "sha512-GVSjntxKeA+/y/ZKf1F+cmUw1WeIkE5aMRPqnZUlBTBvBcrvgWccJAWuYCKPX4QJQwZILIIwhgdAbl51yj6fpA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.3.tgz",
|
||||
"integrity": "sha512-J51oK/axyZ13kxycumSMfLiDZMdWdOVvqDFI28BpuViZHE3A0bQfr8B5vg8YnPEnqLD3BSn1hkdlh2buspEcNQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.3.tgz",
|
||||
"integrity": "sha512-CtQgQjoVTX67jS9XuCTtJ40Sl7wRLMguoFnnGnfDmCWf7kzKFZVwj5ynqUOIGKFMSB61ZCuQlwPvVNxYTTseaw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.3.tgz",
|
||||
"integrity": "sha512-jtfzAHFp+FRaR7zGT4jyCe6wUgAG/dVb5A4Apd8FY9jKarntDfUAlJXscugiH7ZF5kKnu7/lHFk9LaDPcrGEVQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.3.tgz",
|
||||
"integrity": "sha512-xTzaUCKUHTY4bCGadeeRZggbRVbGUT1petg7Z8r9AJR2+D9Bqu6nQAgqBGC6D47tA70LjaaaLTrJ7wNY1T74dg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.3.tgz",
|
||||
"integrity": "sha512-ktVLuBkI6QVOm5BwO/WbdGwxgeetAMJa7TTmR8qBarXF0OU2NKjvjUtPJAl2y8t+zBRczJl/1VOl9gua6WcK2g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.3.tgz",
|
||||
"integrity": "sha512-SGhlQ8bDjL1Cz2KnsKMasr/5sTcwG/SZkB6WCJxLsmSm/3aS2C+3p39bA7iZ2/94+NkVDySZfbiGoaSZSFHYxA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodelib/fs.scandir": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
@@ -3443,6 +3694,18 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "6.2.108",
|
||||
"resolved": "https://registry.npmmirror.com/pdfjs-dist/-/pdfjs-dist-6.2.108.tgz",
|
||||
"integrity": "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=22.13.0 || >=24"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
"element-plus": "^2.9.1",
|
||||
"marked": "^15.0.5",
|
||||
"md-editor-v3": "^5.1.4",
|
||||
"pdfjs-dist": "^6.2.108",
|
||||
"pinia": "^2.3.0",
|
||||
"vue": "^3.5.13",
|
||||
"vue-echarts": "^8.0.1",
|
||||
|
||||
306
frontend/src/api/modules/dataProcess.ts
Normal file
306
frontend/src/api/modules/dataProcess.ts
Normal file
@@ -0,0 +1,306 @@
|
||||
import { del, get, post, put } from '../request'
|
||||
import type {
|
||||
DataProcessExternalSourcePayload,
|
||||
DataProcessExternalTestResult,
|
||||
DataProcessPage,
|
||||
DataProcessOfficePreview,
|
||||
DataProcessPdfPages,
|
||||
DataProcessPreviewBuildPayload,
|
||||
DataProcessPreviewBuildResult,
|
||||
DataProcessPreviewProgress,
|
||||
DataProcessPreviewCreatePayload,
|
||||
DataProcessPreviewItem,
|
||||
DataProcessPreviewUpdatePayload,
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
DataProcessResult,
|
||||
DataProcessResultBatchRegeneratePayload,
|
||||
DataProcessResultBatchRegenerateResult,
|
||||
DataProcessResultRegeneratePayload,
|
||||
DataProcessResultUpdatePayload,
|
||||
DataProcessSourceContent,
|
||||
DataProcessSourceFile,
|
||||
DataProcessTask,
|
||||
DataProcessTaskCreatePayload,
|
||||
DataProcessTaskUpdatePayload,
|
||||
DataProcessWorkflowStep,
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
export type {
|
||||
DataProcessConfig,
|
||||
DataProcessDatasetSplit,
|
||||
DataProcessExternalSourcePayload,
|
||||
DataProcessExternalTestResult,
|
||||
DataProcessPage,
|
||||
DataProcessDocxParagraph,
|
||||
DataProcessDocxPreview,
|
||||
DataProcessDocxTable,
|
||||
DataProcessDocxTableRow,
|
||||
DataProcessOfficePreview,
|
||||
DataProcessPdfPageRange,
|
||||
DataProcessPdfPages,
|
||||
DataProcessPreviewBuildPayload,
|
||||
DataProcessPreviewBuildFileResult,
|
||||
DataProcessPreviewBuildResult,
|
||||
DataProcessPreviewProgress,
|
||||
DataProcessPreviewStatus,
|
||||
DataProcessPreviewFileBuildProgress,
|
||||
DataProcessPreviewFileStatus,
|
||||
DataProcessPreviewCreatePayload,
|
||||
DataProcessPreviewItem,
|
||||
DataProcessPreviewUpdatePayload,
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
DataProcessResult,
|
||||
DataProcessResultBatchRegenerateFailure,
|
||||
DataProcessResultBatchRegenerateItem,
|
||||
DataProcessResultBatchRegeneratePayload,
|
||||
DataProcessResultBatchRegenerateResult,
|
||||
DataProcessResultRegeneratePayload,
|
||||
DataProcessResultStatus,
|
||||
DataProcessResultUpdatePayload,
|
||||
DataProcessSplit,
|
||||
DataProcessSourceContent,
|
||||
DataProcessSourceFile,
|
||||
DataProcessStatus,
|
||||
DataProcessTask,
|
||||
DataProcessTaskCreatePayload,
|
||||
DataProcessTaskUpdatePayload,
|
||||
DataProcessType,
|
||||
DataProcessWorkflowStep,
|
||||
DataProcessXlsxActiveSheet,
|
||||
DataProcessXlsxPreview,
|
||||
DataProcessXlsxPreviewRow,
|
||||
DataProcessXlsxSheet,
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
export function getDataProcessTasks(params: {
|
||||
page?: number
|
||||
page_size?: number
|
||||
keyword?: string
|
||||
status?: string
|
||||
process_type?: string
|
||||
} = {}) {
|
||||
return get<DataProcessPage<DataProcessTask>>('/data-process', params)
|
||||
}
|
||||
|
||||
export const getDataProcessTask = (taskId: string | number) =>
|
||||
get<DataProcessTask>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||
|
||||
export const createDataProcessTask = (payload: DataProcessTaskCreatePayload) =>
|
||||
post<DataProcessTask>(`/data-process`, payload)
|
||||
|
||||
export const updateDataProcessTask = (taskId: string | number, payload: DataProcessTaskUpdatePayload) =>
|
||||
put<DataProcessTask>(`/data-process/${encodeURIComponent(taskId)}`, payload)
|
||||
|
||||
export const updateDataProcessWorkflowStep = (
|
||||
taskId: string | number,
|
||||
workflowStep: DataProcessWorkflowStep,
|
||||
) => put<DataProcessTask>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/workflow-step`,
|
||||
{ workflow_step: workflowStep },
|
||||
)
|
||||
|
||||
export const regenerateDataProcessTask = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessRegeneratePayload,
|
||||
) => post<DataProcessRegenerateResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/regenerate`,
|
||||
payload,
|
||||
)
|
||||
|
||||
export const deleteDataProcessTask = (taskId: string | number) =>
|
||||
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||
|
||||
export function uploadDataProcessSourceFiles(
|
||||
taskId: string | number,
|
||||
files: File[],
|
||||
onProgress?: (progress: number) => void,
|
||||
) {
|
||||
const formData = new FormData()
|
||||
files.forEach((file) => formData.append('files', file))
|
||||
return post<{ files: DataProcessSourceFile[] }>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files`,
|
||||
formData,
|
||||
{
|
||||
timeout: 5 * 60 * 1000,
|
||||
onUploadProgress: (event) => {
|
||||
if (!event.total) return
|
||||
// 发送完成不等于服务端解析完成；收到成功响应前最多展示 99%。
|
||||
const progress = Math.round((event.loaded / event.total) * 100)
|
||||
onProgress?.(Math.min(99, Math.max(0, progress)))
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
export const deleteDataProcessSourceFile = (taskId: string | number, fileId: string | number) =>
|
||||
del<{ deleted: string | number }>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}`,
|
||||
)
|
||||
|
||||
export const getDataProcessSourceContent = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
params: { start_line?: number; line_count?: number } = {},
|
||||
) => get<DataProcessSourceContent>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`,
|
||||
params,
|
||||
)
|
||||
|
||||
export const getDataProcessSourceRawUrl = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) => `/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/raw`
|
||||
|
||||
export const getDataProcessPdfPages = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) => get<DataProcessPdfPages>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/pdf-pages`,
|
||||
)
|
||||
|
||||
export const getDataProcessOfficePreview = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
params: { sheet_index?: number; offset?: number; limit?: number } = {},
|
||||
) => get<DataProcessOfficePreview>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/office-preview`,
|
||||
params,
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
|
||||
export const testDataProcessExternalSource = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessExternalSourcePayload,
|
||||
) => {
|
||||
const { query: _query, file_name: _fileName, ...connection } = payload
|
||||
return post<DataProcessExternalTestResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/external/test`,
|
||||
connection,
|
||||
)
|
||||
}
|
||||
|
||||
export const pullDataProcessExternalSource = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessExternalSourcePayload,
|
||||
) => post<{ files: DataProcessSourceFile[] }>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/external/pull`,
|
||||
payload,
|
||||
)
|
||||
|
||||
export const buildDataProcessPreview = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessPreviewBuildPayload = {},
|
||||
) => post<DataProcessPreviewBuildResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/build`,
|
||||
payload,
|
||||
{ timeout: 5 * 60 * 1000 },
|
||||
)
|
||||
|
||||
export const startDataProcessPreview = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessPreviewBuildPayload = {},
|
||||
) => post<DataProcessPreviewProgress>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/start`,
|
||||
payload,
|
||||
)
|
||||
|
||||
export const getDataProcessPreviewProgress = (taskId: string | number) =>
|
||||
get<DataProcessPreviewProgress>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/progress`,
|
||||
)
|
||||
|
||||
export function getDataProcessPreview(
|
||||
taskId: string | number,
|
||||
params: { source_file_id?: string | number; page?: number; page_size?: number; keyword?: string } = {},
|
||||
) {
|
||||
return get<DataProcessPage<DataProcessPreviewItem>>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview`,
|
||||
params,
|
||||
)
|
||||
}
|
||||
|
||||
export const updateDataProcessPreview = (
|
||||
taskId: string | number,
|
||||
previewId: string | number,
|
||||
payload: DataProcessPreviewUpdatePayload,
|
||||
) => put<DataProcessPreviewItem>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/${encodeURIComponent(previewId)}`,
|
||||
payload,
|
||||
)
|
||||
|
||||
export const createDataProcessPreview = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessPreviewCreatePayload,
|
||||
) => post<DataProcessPreviewItem>(`/data-process/${encodeURIComponent(taskId)}/preview`, payload)
|
||||
|
||||
export const deleteDataProcessPreview = (
|
||||
taskId: string | number,
|
||||
previewId: string | number,
|
||||
) => del<Record<string, never>>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/preview/${encodeURIComponent(previewId)}`,
|
||||
)
|
||||
|
||||
export const generateDataProcess = (taskId: string | number) =>
|
||||
post<DataProcessProgress>(`/data-process/${encodeURIComponent(taskId)}/generate`)
|
||||
|
||||
export const getDataProcessProgress = (taskId: string | number) =>
|
||||
get<DataProcessProgress>(`/data-process/${encodeURIComponent(taskId)}/progress`)
|
||||
|
||||
export const confirmDataProcessResults = (taskId: string | number) =>
|
||||
post<DataProcessTask>(`/data-process/${encodeURIComponent(taskId)}/confirm-results`)
|
||||
|
||||
export function getDataProcessResults(
|
||||
taskId: string | number,
|
||||
params: { page?: number; page_size?: number; keyword?: string; status?: string; split?: string } = {},
|
||||
) {
|
||||
return get<DataProcessPage<DataProcessResult>>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/results`,
|
||||
params,
|
||||
)
|
||||
}
|
||||
|
||||
export const updateDataProcessResult = (
|
||||
taskId: string | number,
|
||||
resultId: string | number,
|
||||
payload: DataProcessResultUpdatePayload,
|
||||
) => put<DataProcessResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/results/${encodeURIComponent(resultId)}`,
|
||||
payload,
|
||||
)
|
||||
|
||||
export const restoreDataProcessResult = (taskId: string | number, resultId: string | number) =>
|
||||
post<DataProcessResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/results/${encodeURIComponent(resultId)}/restore`,
|
||||
)
|
||||
|
||||
export const regenerateDataProcessResult = (
|
||||
taskId: string | number,
|
||||
resultId: string | number,
|
||||
payload: DataProcessResultRegeneratePayload,
|
||||
) => post<DataProcessResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/results/${encodeURIComponent(resultId)}/regenerate`,
|
||||
payload,
|
||||
{ timeout: 210_000 },
|
||||
)
|
||||
|
||||
export const regenerateDataProcessResults = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessResultBatchRegeneratePayload,
|
||||
) => post<DataProcessResultBatchRegenerateResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/results/regenerate-batch`,
|
||||
payload,
|
||||
{ timeout: 240_000 },
|
||||
)
|
||||
|
||||
export const publishDataProcess = (taskId: string | number, payload: DataProcessPublishPayload) =>
|
||||
post<DataProcessPublishResult>(`/data-process/${encodeURIComponent(taskId)}/publish`, payload)
|
||||
@@ -89,10 +89,10 @@ export const downloadFileUrl = (
|
||||
fileId: string | number,
|
||||
versionId?: string,
|
||||
) => {
|
||||
const baseUrl = `/modelTF/dataset-manage/download/${datasetId}/${fileId}`
|
||||
const baseUrl = `/dataset-manage/download/${datasetId}/${fileId}`
|
||||
return versionId ? `${baseUrl}?version_id=${encodeURIComponent(versionId)}` : baseUrl
|
||||
}
|
||||
|
||||
/** 打包下载数据集 URL */
|
||||
export const downloadDatasetUrl = (datasetId: string | number) =>
|
||||
`/modelTF/dataset-manage/download/${datasetId}`
|
||||
`/dataset-manage/download/${datasetId}`
|
||||
|
||||
@@ -45,4 +45,4 @@ export const mergeModel = (data: {
|
||||
|
||||
/** 导出已训练模型权重 */
|
||||
export const exportModelUrl = (modelName: string) =>
|
||||
`/modelTF/model-manage/trained-models/${encodeURIComponent(modelName)}/export`
|
||||
`/model-manage/trained-models/${encodeURIComponent(modelName)}/export`
|
||||
|
||||
@@ -144,7 +144,19 @@ const routes: RouteRecordRaw[] = [
|
||||
path: 'data-process/:id',
|
||||
name: 'data-process-detail',
|
||||
component: () => import('@/views/data-process/DataProcessDetailView.vue'),
|
||||
meta: { title: '数据处理详情' },
|
||||
meta: { title: '数据处理详情', backRouteName: 'data-process' },
|
||||
},
|
||||
{
|
||||
path: 'data-process/:id/regenerate',
|
||||
name: 'data-process-regenerate',
|
||||
component: () => import('@/views/data-process/DataProcessCreateView.vue'),
|
||||
meta: { title: '重新生成数据处理任务', pageSurface: 'self', backRouteName: 'data-process' },
|
||||
},
|
||||
{
|
||||
path: 'data-process/:id/workflow',
|
||||
name: 'data-process-workflow',
|
||||
component: () => import('@/views/data-process/DataProcessCreateView.vue'),
|
||||
meta: { title: '数据处理任务进度', pageSurface: 'self', backRouteName: 'data-process' },
|
||||
},
|
||||
// 数据集管理
|
||||
{
|
||||
|
||||
398
frontend/src/types/dataProcess.ts
Normal file
398
frontend/src/types/dataProcess.ts
Normal file
@@ -0,0 +1,398 @@
|
||||
/** 数据处理模块的前后端契约。API 字段统一使用 snake_case。 */
|
||||
|
||||
export type DataProcessStatus = 'pending' | 'running' | 'completed' | 'failed' | 'stopped'
|
||||
export type DataProcessType = 'structured' | 'unstructured' | 'external'
|
||||
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
|
||||
export type DataProcessSplit = 'train' | 'validation' | 'test'
|
||||
export type DataProcessOutputType = 'standard' | 'reasoning'
|
||||
export type DataProcessReasoningDetail = 'normal' | 'detailed'
|
||||
export type DataProcessWorkflowStep = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
export type DataProcessPreviewStatus = 'idle' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'
|
||||
|
||||
export interface DataProcessPage<T> {
|
||||
items: T[]
|
||||
total: number
|
||||
page: number
|
||||
page_size: number
|
||||
}
|
||||
|
||||
export interface DataProcessDatasetSplit {
|
||||
train: number
|
||||
validation: number
|
||||
test: number
|
||||
}
|
||||
|
||||
export type DataProcessConfig = Record<string, unknown> & {
|
||||
dataset_split?: DataProcessDatasetSplit
|
||||
output_type?: DataProcessOutputType
|
||||
reasoning_detail?: DataProcessReasoningDetail
|
||||
}
|
||||
|
||||
export interface DataProcessTask {
|
||||
id: string | number
|
||||
name: string
|
||||
description?: string
|
||||
status: DataProcessStatus
|
||||
process_type: DataProcessType
|
||||
config?: DataProcessConfig
|
||||
progress?: number
|
||||
source_dataset_id?: string | number | null
|
||||
source_dataset_name?: string | null
|
||||
source_dataset?: string | null
|
||||
output_dataset_id?: string | number | null
|
||||
output_dataset_name?: string | null
|
||||
output_dataset?: string | null
|
||||
output_datasets?: DataProcessOutputDataset[]
|
||||
source_file_count?: number
|
||||
preview_count?: number
|
||||
input_count?: number
|
||||
output_count?: number
|
||||
filtered_count?: number
|
||||
duplicate_count?: number
|
||||
error_count?: number
|
||||
creator_name?: string | null
|
||||
creator?: string | null
|
||||
created_by?: string | number | null
|
||||
create_time?: string
|
||||
created_at?: string
|
||||
start_time?: string | null
|
||||
started_at?: string | null
|
||||
complete_time?: string | null
|
||||
completed_at?: string | null
|
||||
updated_at: string
|
||||
duration?: string | null
|
||||
duration_seconds?: number | null
|
||||
failure_reason?: string | null
|
||||
results_confirmed?: boolean
|
||||
workflow_step?: DataProcessWorkflowStep
|
||||
preview_status?: DataProcessPreviewStatus
|
||||
preview_progress?: number
|
||||
preview_failure_reason?: string | null
|
||||
preview_total_files?: number
|
||||
preview_completed_files?: number
|
||||
preview_run_id?: string | null
|
||||
source_files?: DataProcessSourceFile[]
|
||||
}
|
||||
|
||||
export interface DataProcessOutputDataset {
|
||||
id: string | number
|
||||
name: string
|
||||
type: 'train' | 'val' | 'test' | string
|
||||
count?: number
|
||||
dataset_split?: DataProcessSplit
|
||||
}
|
||||
|
||||
export interface DataProcessTaskCreatePayload {
|
||||
name: string
|
||||
description?: string
|
||||
process_type: DataProcessType
|
||||
config: DataProcessConfig
|
||||
}
|
||||
|
||||
export interface DataProcessRegeneratePayload extends DataProcessTaskCreatePayload {
|
||||
expected_updated_at: string
|
||||
}
|
||||
|
||||
export interface DataProcessRegenerateResult {
|
||||
task: DataProcessTask
|
||||
preview_invalidated: boolean
|
||||
published_outputs_preserved: boolean
|
||||
}
|
||||
|
||||
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
|
||||
|
||||
export interface DataProcessSourceFile {
|
||||
id: string | number
|
||||
task_id?: string | number
|
||||
name: string
|
||||
size_bytes: number
|
||||
record_count: number
|
||||
file_format?: string
|
||||
checksum_sha256?: string
|
||||
status?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
export interface DataProcessSourceContent {
|
||||
file_id?: string | number
|
||||
file?: DataProcessSourceFile
|
||||
content: string
|
||||
start_line?: number
|
||||
end_line?: number
|
||||
line_count?: number
|
||||
total_lines?: number
|
||||
has_more?: boolean
|
||||
truncated?: boolean
|
||||
offset?: number
|
||||
limit?: number
|
||||
total_chars?: number
|
||||
}
|
||||
|
||||
export interface DataProcessPdfPageRange {
|
||||
page_number: number
|
||||
source_start: number
|
||||
source_end: number
|
||||
}
|
||||
|
||||
export interface DataProcessPdfPages {
|
||||
page_count: number
|
||||
pages: DataProcessPdfPageRange[]
|
||||
}
|
||||
|
||||
export interface DataProcessDocxParagraph {
|
||||
type: 'paragraph'
|
||||
text: string
|
||||
style: string
|
||||
heading_level: number | null
|
||||
alignment: 'left' | 'center' | 'right' | 'justify' | 'distribute'
|
||||
is_list: boolean
|
||||
source_start: number
|
||||
source_end: number
|
||||
}
|
||||
|
||||
export interface DataProcessDocxTableRow {
|
||||
cells: string[]
|
||||
source_start: number
|
||||
source_end: number
|
||||
}
|
||||
|
||||
export interface DataProcessDocxTable {
|
||||
type: 'table'
|
||||
rows: DataProcessDocxTableRow[]
|
||||
}
|
||||
|
||||
export interface DataProcessDocxPreview {
|
||||
format: 'docx'
|
||||
file_name: string
|
||||
blocks: Array<DataProcessDocxParagraph | DataProcessDocxTable>
|
||||
truncated: boolean
|
||||
}
|
||||
|
||||
export interface DataProcessXlsxSheet {
|
||||
index: number
|
||||
name: string
|
||||
state: string
|
||||
}
|
||||
|
||||
export interface DataProcessXlsxPreviewRow {
|
||||
row_number: number
|
||||
record_index: number
|
||||
values: unknown[]
|
||||
record: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface DataProcessXlsxActiveSheet {
|
||||
index: number
|
||||
name: string
|
||||
columns: string[]
|
||||
rows: DataProcessXlsxPreviewRow[]
|
||||
offset: number
|
||||
limit: number
|
||||
has_more: boolean
|
||||
}
|
||||
|
||||
export interface DataProcessXlsxPreview {
|
||||
format: 'xlsx'
|
||||
file_name: string
|
||||
sheets: DataProcessXlsxSheet[]
|
||||
active_sheet: DataProcessXlsxActiveSheet
|
||||
}
|
||||
|
||||
export type DataProcessOfficePreview = DataProcessDocxPreview | DataProcessXlsxPreview
|
||||
|
||||
export interface DataProcessExternalSourcePayload {
|
||||
type: 'postgresql'
|
||||
url: string
|
||||
auth_mode: 'none' | 'basic'
|
||||
username?: string
|
||||
password?: string
|
||||
limit: number
|
||||
query?: string
|
||||
file_name?: string
|
||||
}
|
||||
|
||||
export interface DataProcessExternalTestResult {
|
||||
connected: boolean
|
||||
latency_ms?: number
|
||||
message?: string
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewItem {
|
||||
id: string | number
|
||||
source_file_id: string | number
|
||||
original_content: string
|
||||
edited_content: string
|
||||
source_start: number | null
|
||||
source_end: number | null
|
||||
source_start_line: number | null
|
||||
source_end_line: number | null
|
||||
token_count: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
quality_score?: DataProcessQualityScore | null
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewBuildPayload {
|
||||
replace_existing?: true
|
||||
source_file_ids?: Array<string | number>
|
||||
}
|
||||
|
||||
export type DataProcessPreviewFileStatus = 'waiting' | 'processing' | 'success' | 'failed'
|
||||
|
||||
export interface DataProcessPreviewBuildFileResult {
|
||||
source_file_id: string | number
|
||||
preview_count: number
|
||||
status: 'completed' | 'empty'
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewFileBuildProgress {
|
||||
source_file_id: string | number
|
||||
status: Exclude<DataProcessPreviewFileStatus, 'waiting'>
|
||||
progress: number
|
||||
preview_count?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewBuildResult extends DataProcessPage<DataProcessPreviewItem> {
|
||||
file_counts?: Record<string, number>
|
||||
files?: DataProcessPreviewBuildFileResult[]
|
||||
}
|
||||
|
||||
/** 后台切分任务级进度。完成后应重新读取预览条目及源文件统计。 */
|
||||
export interface DataProcessPreviewProgress {
|
||||
task_id: string | number
|
||||
workflow_step: DataProcessWorkflowStep
|
||||
preview_status: DataProcessPreviewStatus
|
||||
preview_progress: number
|
||||
preview_failure_reason?: string | null
|
||||
preview_total_files?: number
|
||||
preview_completed_files?: number
|
||||
preview_run_id?: string | null
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewCreatePayload {
|
||||
source_file_id?: string | number | null
|
||||
original_content?: string
|
||||
edited_content: string
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
source_start_line?: number | null
|
||||
source_end_line?: number | null
|
||||
token_count?: number
|
||||
status?: DataProcessPreviewItem['status']
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewUpdatePayload {
|
||||
edited_content: string
|
||||
status?: DataProcessPreviewItem['status']
|
||||
expected_updated_at?: string
|
||||
}
|
||||
|
||||
export interface DataProcessProgress {
|
||||
task_id: string | number
|
||||
status: DataProcessStatus
|
||||
stage?: string
|
||||
progress: number
|
||||
message?: string
|
||||
processed_count?: number
|
||||
total_count?: number
|
||||
input_count?: number
|
||||
output_count?: number
|
||||
filtered_count?: number
|
||||
duplicate_count?: number
|
||||
error_count?: number
|
||||
failure_reason?: string | null
|
||||
results_confirmed?: boolean
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export interface DataProcessResult {
|
||||
id: string | number
|
||||
preview_item_id?: string | number | null
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
original_instruction?: string | null
|
||||
original_input?: string | null
|
||||
original_output?: string | null
|
||||
status: DataProcessResultStatus
|
||||
error?: string | null
|
||||
split?: DataProcessSplit | null
|
||||
quality_score?: DataProcessQualityScore | null
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export interface DataProcessResultUpdatePayload {
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
expected_updated_at?: string
|
||||
}
|
||||
|
||||
export interface DataProcessResultRegeneratePayload {
|
||||
expected_updated_at: string
|
||||
}
|
||||
|
||||
export interface DataProcessResultBatchRegenerateItem {
|
||||
result_id: string
|
||||
expected_updated_at: string
|
||||
}
|
||||
|
||||
export interface DataProcessResultBatchRegeneratePayload {
|
||||
items: DataProcessResultBatchRegenerateItem[]
|
||||
}
|
||||
|
||||
export interface DataProcessResultBatchRegenerateFailure {
|
||||
result_id: string
|
||||
code: 'conflict' | 'skipped' | 'generation_failed' | 'internal_error'
|
||||
message: string
|
||||
}
|
||||
|
||||
export interface DataProcessResultBatchRegenerateResult {
|
||||
batch_id: string
|
||||
total: number
|
||||
succeeded: number
|
||||
failed: number
|
||||
remaining_invalid_count: number
|
||||
duration_ms: number
|
||||
items: DataProcessResult[]
|
||||
failures: DataProcessResultBatchRegenerateFailure[]
|
||||
}
|
||||
|
||||
export interface DataProcessQualityScore {
|
||||
overall?: number
|
||||
completeness?: number
|
||||
length?: number
|
||||
readability?: number
|
||||
relevance?: number
|
||||
duplicate?: number
|
||||
is_valid?: boolean
|
||||
flags?: string[]
|
||||
fingerprint?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export interface DataProcessPublishPayload {
|
||||
dataset_name: string
|
||||
dataset_type: 'train' | 'test' | 'eval' | 'val' | 'other'
|
||||
storage_type: 'local'
|
||||
split: DataProcessDatasetSplit
|
||||
format: 'alpaca_jsonl' | 'jsonl'
|
||||
}
|
||||
|
||||
export interface DataProcessPublishResult {
|
||||
dataset_id?: string | number
|
||||
output_dataset_id?: string | number
|
||||
dataset_name?: string
|
||||
record_count?: number
|
||||
dataset?: {
|
||||
id: string | number
|
||||
name?: string
|
||||
record_count?: number
|
||||
count?: number
|
||||
[key: string]: unknown
|
||||
}
|
||||
datasets?: DataProcessOutputDataset[]
|
||||
output_datasets?: DataProcessOutputDataset[]
|
||||
created?: boolean
|
||||
}
|
||||
13
frontend/src/utils/dataProcessStatus.d.ts
vendored
Normal file
13
frontend/src/utils/dataProcessStatus.d.ts
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
import type { DataProcessStatus } from '@/types/dataProcess'
|
||||
|
||||
export type DataProcessStatusTagType = 'primary' | 'success' | 'warning' | 'danger' | 'info'
|
||||
|
||||
export interface DataProcessStatusLike {
|
||||
status?: DataProcessStatus
|
||||
output_dataset_id?: string | number | null
|
||||
}
|
||||
|
||||
export function dataProcessDisplayStatus(task: DataProcessStatusLike): {
|
||||
label: string
|
||||
type: DataProcessStatusTagType
|
||||
}
|
||||
24
frontend/src/utils/dataProcessStatus.js
Normal file
24
frontend/src/utils/dataProcessStatus.js
Normal file
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* 将数据处理任务的执行状态与当前轮发布指针合并为用户可理解的状态。
|
||||
* 上一轮保留的 output_datasets 不参与判断，避免将尚未重新发布的当前轮误标为已发布。
|
||||
*
|
||||
* @param {{ status?: string, preview_status?: string, output_dataset_id?: string | number | null, results_confirmed?: boolean }} task
|
||||
* @returns {{ label: string, type: 'primary' | 'success' | 'warning' | 'danger' | 'info' }}
|
||||
*/
|
||||
export function dataProcessDisplayStatus(task) {
|
||||
if (task.preview_status === 'running' || task.preview_status === 'queued') {
|
||||
return { label: '切分中', type: 'primary' }
|
||||
}
|
||||
if (task.preview_status === 'failed') return { label: '切分失败', type: 'danger' }
|
||||
if (task.preview_status === 'cancelled') return { label: '切分已取消', type: 'warning' }
|
||||
if (task.status === 'running') return { label: '生成中', type: 'primary' }
|
||||
if (task.status === 'completed') {
|
||||
if (task.results_confirmed === false) return { label: '待确认', type: 'warning' }
|
||||
return task.output_dataset_id
|
||||
? { label: '已发布', type: 'success' }
|
||||
: { label: '已生成', type: 'warning' }
|
||||
}
|
||||
if (task.status === 'failed') return { label: '生成失败', type: 'danger' }
|
||||
if (task.status === 'stopped') return { label: '已停止', type: 'info' }
|
||||
return { label: '待生成', type: 'info' }
|
||||
}
|
||||
@@ -53,6 +53,14 @@ const trainingTasks = ref<DashboardTask[]>([])
|
||||
const loginDurationStats = ref<LoginDurationStat[]>([])
|
||||
const recentLoginUsers = ref<RecentLoginUser[]>([])
|
||||
const training7d = ref<{ date: string; train: number; gpu: number; accuracy: number | null }[]>([])
|
||||
|
||||
const onlineServicesHint = computed(() => {
|
||||
if (onlineServices.value === 0) return '暂无在线服务'
|
||||
const abnormal = serviceStatuses.value.filter(
|
||||
(s) => s.state === 'busy' || s.state === 'error'
|
||||
).length
|
||||
return abnormal > 0 ? `${abnormal} 个异常` : '全部在线'
|
||||
})
|
||||
const operationDistribution = ref<{ name: string; value: number }[]>([])
|
||||
|
||||
const serviceIcon: Record<string, string> = {
|
||||
@@ -319,8 +327,8 @@ function viewTask(task: DashboardTask) {
|
||||
<div class="overview-metrics">
|
||||
<div class="overview-metric">
|
||||
<span>在线服务</span>
|
||||
<strong>{{ onlineServices }}</strong>
|
||||
<small>全部在线</small>
|
||||
<strong>{{ onlineServices }}</strong>
|
||||
<small>{{ onlineServicesHint }}</small>
|
||||
</div>
|
||||
<div class="overview-metric">
|
||||
<span>运行中任务</span>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,88 +1,108 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { deleteDataProcessTask, getDataProcessTasks } from '@/api/modules/dataProcess'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import type { DataProcessTask, DataProcessType } from '@/types/dataProcess'
|
||||
import { dataProcessDisplayStatus } from '@/utils/dataProcessStatus.js'
|
||||
|
||||
import type { ProcessType } from './create/types'
|
||||
|
||||
/** 数据处理任务类型 */
|
||||
interface DataProcessTask {
|
||||
id: number | string
|
||||
name: string
|
||||
status: string
|
||||
process_type: ProcessType
|
||||
source_dataset: string
|
||||
output_dataset?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
const processTypeMap: Record<ProcessType, string> = {
|
||||
const processTypeMap: Record<DataProcessType, string> = {
|
||||
structured: '结构化数据',
|
||||
unstructured: '非结构化数据',
|
||||
external: '外来数据源拉取',
|
||||
}
|
||||
|
||||
// TODO: 接入真实接口前，先用本地 mock 数据
|
||||
const router = useRouter()
|
||||
const dataList = ref<DataProcessTask[]>([
|
||||
{
|
||||
id: 183921,
|
||||
name: '客服问答数据清洗',
|
||||
status: 'completed',
|
||||
process_type: 'structured',
|
||||
source_dataset: '客服对话原始集',
|
||||
output_dataset: '客服对话清洗集',
|
||||
create_time: '2026-07-08 14:23:00',
|
||||
},
|
||||
{
|
||||
id: 492015,
|
||||
name: '指令微调数据构造',
|
||||
status: 'running',
|
||||
process_type: 'unstructured',
|
||||
source_dataset: '通用语料库',
|
||||
output_dataset: 'SFT 指令集',
|
||||
create_time: '2026-07-09 09:10:00',
|
||||
},
|
||||
{
|
||||
id: 731948,
|
||||
name: '敏感信息脱敏处理',
|
||||
status: 'pending',
|
||||
process_type: 'structured',
|
||||
source_dataset: '用户反馈数据',
|
||||
create_time: '2026-07-09 16:45:00',
|
||||
},
|
||||
{
|
||||
id: 582012,
|
||||
name: '多轮对话拼接',
|
||||
status: 'failed',
|
||||
process_type: 'structured',
|
||||
source_dataset: '单轮问答集',
|
||||
create_time: '2026-07-10 08:30:00',
|
||||
},
|
||||
])
|
||||
const dataList = ref<DataProcessTask[]>([])
|
||||
const loading = ref(false)
|
||||
const deletingId = ref<string | number | null>(null)
|
||||
const loadError = ref('')
|
||||
|
||||
/** 新建数据处理任务 */
|
||||
function handleCreate() {
|
||||
router.push('/data-process/create')
|
||||
async function loadData(silent = false) {
|
||||
if (!silent) loading.value = true
|
||||
loadError.value = ''
|
||||
try {
|
||||
const response = await getDataProcessTasks({ page: 1, page_size: 200 })
|
||||
dataList.value = response.items
|
||||
if (dataList.value.some((item) => (
|
||||
item.status === 'running'
|
||||
|| item.preview_status === 'running'
|
||||
|| item.preview_status === 'queued'
|
||||
))) startPolling()
|
||||
else stopPolling()
|
||||
} catch {
|
||||
loadError.value = '数据处理任务加载失败，请稍后重试。'
|
||||
} finally {
|
||||
if (!silent) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
/** 查看任务详情 */
|
||||
function viewDetail(row: unknown) {
|
||||
const taskId = (row as { id: string | number }).id
|
||||
router.push({ name: 'data-process-detail', params: { id: taskId } })
|
||||
const task = row as DataProcessTask
|
||||
const routeName = task.status === 'completed' && task.results_confirmed !== false
|
||||
? 'data-process-detail'
|
||||
: 'data-process-workflow'
|
||||
router.push({ name: routeName, params: { id: task.id } })
|
||||
}
|
||||
|
||||
/** 删除任务（功能开发中） */
|
||||
function handleDelete(_row: unknown) {
|
||||
ElMessage.info('删除功能开发中...')
|
||||
/** 删除由后端再次校验任务状态以及发布锁。 */
|
||||
async function handleDelete(row: DataProcessTask) {
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
`确定删除数据处理任务“${row.name}”吗？删除后无法恢复。`,
|
||||
'确认删除',
|
||||
{
|
||||
type: 'warning',
|
||||
confirmButtonText: '删除',
|
||||
cancelButtonText: '取消',
|
||||
confirmButtonClass: 'el-button--danger',
|
||||
},
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
deletingId.value = row.id
|
||||
try {
|
||||
await deleteDataProcessTask(row.id)
|
||||
dataList.value = dataList.value.filter((item) => item.id !== row.id)
|
||||
ElMessage.success('数据处理任务已删除')
|
||||
} catch {
|
||||
// 统一请求层已展示后端返回的失败原因。
|
||||
} finally {
|
||||
deletingId.value = null
|
||||
}
|
||||
}
|
||||
|
||||
function formatDateTime(value?: string) {
|
||||
if (!value) return '-'
|
||||
return new Date(value).toLocaleString('zh-CN', { hour12: false })
|
||||
const date = new Date(value)
|
||||
return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false })
|
||||
}
|
||||
|
||||
function safeCount(value: unknown) {
|
||||
const count = Number(value)
|
||||
return Number.isFinite(count) ? Math.max(0, Math.trunc(count)) : 0
|
||||
}
|
||||
|
||||
function documentCountLabel(task: DataProcessTask) {
|
||||
return `${safeCount(task.source_file_count)} 个`
|
||||
}
|
||||
|
||||
function generatedCountLabel(task: DataProcessTask) {
|
||||
return `${safeCount(task.output_count)} 条`
|
||||
}
|
||||
|
||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||
() => loadData(true),
|
||||
3000,
|
||||
{ immediate: false },
|
||||
)
|
||||
|
||||
onMounted(() => void loadData())
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -90,37 +110,43 @@ function formatDateTime(value?: string) {
|
||||
<DataTablePage
|
||||
title=""
|
||||
:data="dataList"
|
||||
:loading="loading"
|
||||
searchable
|
||||
:search-fields="['name']"
|
||||
create-text="新建数据处理"
|
||||
create-to="/data-process/create"
|
||||
row-key="id"
|
||||
:page-size="10"
|
||||
:empty-text="loadError || '暂无数据处理任务'"
|
||||
>
|
||||
<template #columns>
|
||||
<el-table-column label="任务ID" prop="id" align="center" width="100" />
|
||||
<el-table-column label="任务名称" prop="name" align="center" show-overflow-tooltip />
|
||||
<el-table-column label="任务状态" align="center" width="110">
|
||||
<template #default="{ row }">
|
||||
<ModelStatusTag :status="row.status" />
|
||||
<el-tag
|
||||
:type="dataProcessDisplayStatus(row as DataProcessTask).type"
|
||||
size="small"
|
||||
effect="light"
|
||||
>{{ dataProcessDisplayStatus(row as DataProcessTask).label }}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="处理类型" align="center" width="140">
|
||||
<template #default="{ row }">
|
||||
<el-tag v-if="row.process_type" size="small" type="info" effect="plain">
|
||||
{{ processTypeMap[row.process_type as ProcessType] || row.process_type }}
|
||||
{{ processTypeMap[row.process_type as DataProcessType] || row.process_type }}
|
||||
</el-tag>
|
||||
<span v-else>-</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="源数据集" align="center" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.source_dataset || '-' }}</template>
|
||||
<el-table-column label="文档数量" align="center" width="120">
|
||||
<template #default="{ row }">{{ documentCountLabel(row as DataProcessTask) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="输出数据集" align="center" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.output_dataset || '-' }}</template>
|
||||
<el-table-column label="生成个数" align="center" width="120">
|
||||
<template #default="{ row }">{{ generatedCountLabel(row as DataProcessTask) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="创建时间" align="center" width="190">
|
||||
<template #default="{ row }">{{ formatDateTime(row.create_time) }}</template>
|
||||
<template #default="{ row }">{{ formatDateTime(row.create_time || row.created_at) }}</template>
|
||||
</el-table-column>
|
||||
</template>
|
||||
|
||||
@@ -129,7 +155,13 @@ function formatDateTime(value?: string) {
|
||||
<el-button type="primary" link size="small" @click="viewDetail(row)">
|
||||
<i class="fa fa-file-text-o" style="margin-right: 4px" />详情
|
||||
</el-button>
|
||||
<el-button type="danger" link size="small" @click="handleDelete(row)">
|
||||
<el-button
|
||||
type="danger"
|
||||
link
|
||||
size="small"
|
||||
:loading="deletingId === row.id"
|
||||
@click="handleDelete(row as DataProcessTask)"
|
||||
>
|
||||
<i class="fa fa-trash-o" style="margin-right: 4px" />删除
|
||||
</el-button>
|
||||
</div>
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import { ref, onMounted, onBeforeUnmount } from 'vue'
|
||||
import type { ModelItem } from '@/types'
|
||||
import type { GenerationControlOptions } from './types'
|
||||
import {
|
||||
defaultGenerationPrompt,
|
||||
isBuiltInGenerationPrompt,
|
||||
} from './dataProcessCreateState'
|
||||
|
||||
const props = defineProps<{
|
||||
options: GenerationControlOptions
|
||||
@@ -14,6 +18,43 @@ const emit = defineEmits<{
|
||||
'update:options': [value: GenerationControlOptions]
|
||||
}>()
|
||||
|
||||
const promptInputRef = ref<any>(null)
|
||||
const promptHighlightRef = ref<HTMLElement | null>(null)
|
||||
|
||||
function splitPrompt(prompt: string) {
|
||||
const parts = (prompt || '').split('{{ content }}')
|
||||
const result: Array<{ text: string, isVar: boolean }> = []
|
||||
parts.forEach((part, index) => {
|
||||
if (part) {
|
||||
result.push({ text: part, isVar: false })
|
||||
}
|
||||
if (index < parts.length - 1) {
|
||||
result.push({ text: '{{ content }}', isVar: true })
|
||||
}
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
function syncScroll(e: Event) {
|
||||
if (promptHighlightRef.value) {
|
||||
promptHighlightRef.value.scrollTop = (e.target as HTMLElement).scrollTop
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
const textarea = promptInputRef.value?.textarea
|
||||
if (textarea) {
|
||||
textarea.addEventListener('scroll', syncScroll)
|
||||
}
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
const textarea = promptInputRef.value?.textarea
|
||||
if (textarea) {
|
||||
textarea.removeEventListener('scroll', syncScroll)
|
||||
}
|
||||
})
|
||||
|
||||
function updateField<K extends keyof GenerationControlOptions>(
|
||||
field: K,
|
||||
value: GenerationControlOptions[K],
|
||||
@@ -21,6 +62,10 @@ function updateField<K extends keyof GenerationControlOptions>(
|
||||
emit('update:options', { ...props.options, [field]: value })
|
||||
}
|
||||
|
||||
function resetPrompt() {
|
||||
updateField('generationPrompt', defaultGenerationPrompt(props.options.outputType))
|
||||
}
|
||||
|
||||
function updateQualityRules(value: Array<string | number>) {
|
||||
const rules = value.filter((item): item is string => typeof item === 'string')
|
||||
emit('update:options', {
|
||||
@@ -30,6 +75,21 @@ function updateQualityRules(value: Array<string | number>) {
|
||||
})
|
||||
}
|
||||
|
||||
function updateOutputType(value: string | number | boolean | undefined) {
|
||||
const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
|
||||
emit('update:options', {
|
||||
...props.options,
|
||||
outputType,
|
||||
generationPrompt: isBuiltInGenerationPrompt(props.options.generationPrompt)
|
||||
? defaultGenerationPrompt(outputType)
|
||||
: props.options.generationPrompt,
|
||||
})
|
||||
}
|
||||
|
||||
function updateReasoningDetail(value: string | number | boolean | undefined) {
|
||||
updateField('reasoningDetail', value === 'detailed' ? 'detailed' : 'normal')
|
||||
}
|
||||
|
||||
const selectedQualityRules = () => [
|
||||
props.options.filterLowQuality ? 'low_quality' : '',
|
||||
props.options.filterShortContent ? 'short_content' : '',
|
||||
@@ -41,6 +101,15 @@ function sectionValidationMessage() {
|
||||
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
|
||||
return isModelMessage ? '' : props.validationMessage
|
||||
}
|
||||
|
||||
function modelMeta(model: ModelItem) {
|
||||
const source = model.model_source === 'api' || model.model_source === 'online'
|
||||
? '在线模型'
|
||||
: model.model_source === 'local'
|
||||
? '本地模型'
|
||||
: model.model_source || '模型管理'
|
||||
return model.type ? `${source} · ${model.type}` : source
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -66,34 +135,50 @@ function sectionValidationMessage() {
|
||||
>
|
||||
<div class="model-option">
|
||||
<span>{{ model.name }}</span>
|
||||
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
|
||||
<small>{{ modelMeta(model) }}</small>
|
||||
</div>
|
||||
</el-option>
|
||||
<template #empty>
|
||||
<div class="model-empty">暂无可用的大模型，请先在模型管理中添加</div>
|
||||
<div class="model-empty">暂无模型，请先在模型管理中添加</div>
|
||||
</template>
|
||||
</el-select>
|
||||
</div>
|
||||
|
||||
<div class="model-field">
|
||||
<div class="field-copy">
|
||||
<strong>默认提示语</strong>
|
||||
<small>用于约束生成内容的格式、语气和完整性，可按任务需要修改</small>
|
||||
<div class="field-header" style="display: flex; justify-content: space-between; align-items: center; width: 100%;">
|
||||
<strong>默认提示语</strong>
|
||||
<el-button link type="primary" @click="resetPrompt" style="font-size: 12px; padding: 0;">恢复默认</el-button>
|
||||
</div>
|
||||
<small v-if="options.outputType === 'reasoning'">
|
||||
当前使用思维链专用提示语；系统还会按所选详细程度约束推理结构
|
||||
</small>
|
||||
<small v-else>当前使用标准回答提示语，只要求问题和最终答案</small>
|
||||
</div>
|
||||
<div class="prompt-input-wrapper">
|
||||
<div ref="promptHighlightRef" class="prompt-highlight-layer" aria-hidden="true">
|
||||
<template v-for="(part, index) in splitPrompt(options.generationPrompt)" :key="index">
|
||||
<span v-if="part.isVar" class="highlight-var">{{ part.text }}</span>
|
||||
<span v-else>{{ part.text }}</span>
|
||||
</template>
|
||||
</div>
|
||||
<el-input
|
||||
ref="promptInputRef"
|
||||
class="prompt-input"
|
||||
:model-value="options.generationPrompt"
|
||||
type="textarea"
|
||||
:rows="6"
|
||||
maxlength="2000"
|
||||
show-word-limit
|
||||
resize="vertical"
|
||||
placeholder="请输入模型生成内容时需要遵循的要求"
|
||||
aria-label="模型默认提示语"
|
||||
@update:model-value="updateField('generationPrompt', $event)"
|
||||
/>
|
||||
</div>
|
||||
<el-input
|
||||
class="prompt-input"
|
||||
:model-value="options.generationPrompt"
|
||||
type="textarea"
|
||||
:rows="4"
|
||||
maxlength="500"
|
||||
show-word-limit
|
||||
resize="vertical"
|
||||
placeholder="请输入模型生成内容时需要遵循的要求"
|
||||
aria-label="模型默认提示语"
|
||||
@update:model-value="updateField('generationPrompt', $event)"
|
||||
/>
|
||||
<div class="prompt-variables-hint">
|
||||
提示：可在文本中通过 <code v-text="'{{ content }}'" /> 引用当前正在处理的数据内容。
|
||||
默认提示语已包含 <code v-text="'{{ content }}'" />，运行时会替换为当前数据内容；
|
||||
可移动该占位符来调整内容在提示语中的位置。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -148,6 +233,49 @@ function sectionValidationMessage() {
|
||||
</div>
|
||||
|
||||
<div v-else class="generation-config-group quality-config-group">
|
||||
<div class="output-type-field">
|
||||
<div class="output-type-row">
|
||||
<div class="field-copy">
|
||||
<strong>输出类型</strong>
|
||||
<small>控制答案是否包含可用于推理模型训练的思维链内容</small>
|
||||
</div>
|
||||
<el-select
|
||||
class="output-type-select"
|
||||
:model-value="options.outputType"
|
||||
aria-label="输出类型"
|
||||
@update:model-value="updateOutputType"
|
||||
>
|
||||
<el-option label="标准回答" value="standard" />
|
||||
<el-option label="思维链回答" value="reasoning" />
|
||||
</el-select>
|
||||
</div>
|
||||
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
|
||||
<div class="field-copy">
|
||||
<strong>推理详细程度</strong>
|
||||
<small>控制推理过程保留关键步骤，或完整展开依据与中间推导</small>
|
||||
</div>
|
||||
<el-select
|
||||
class="output-type-select"
|
||||
:model-value="options.reasoningDetail"
|
||||
aria-label="推理详细程度"
|
||||
@update:model-value="updateReasoningDetail"
|
||||
>
|
||||
<el-option label="普通推理（推荐）" value="normal" />
|
||||
<el-option label="详细推理" value="detailed" />
|
||||
</el-select>
|
||||
</div>
|
||||
<p class="output-type-hint">
|
||||
<template v-if="options.outputType === 'reasoning'">
|
||||
<template v-if="options.reasoningDetail === 'detailed'">
|
||||
完整展开条件、来源依据、中间推导和结论核对，
|
||||
</template>
|
||||
<template v-else>保留关键依据与必要步骤，</template>
|
||||
最终按 <code><think>推理过程</think></code> 加最终答案保存。
|
||||
</template>
|
||||
<template v-else>仅保存最终答案，不包含推理过程。</template>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="quality-switch-row">
|
||||
<div class="config-group-heading">
|
||||
<div>
|
||||
@@ -279,6 +407,50 @@ function sectionValidationMessage() {
|
||||
|
||||
.model-select { width: 100%; }
|
||||
.prompt-input { width: 100%; }
|
||||
|
||||
.prompt-input-wrapper {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.prompt-highlight-layer {
|
||||
position: absolute;
|
||||
top: 1px;
|
||||
left: 1px;
|
||||
right: 1px;
|
||||
bottom: 1px;
|
||||
padding: 5px 11px;
|
||||
box-sizing: border-box;
|
||||
pointer-events: none;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
color: #344054;
|
||||
overflow-y: auto;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.prompt-highlight-layer::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.highlight-var {
|
||||
color: #5b50f2;
|
||||
background: #f0f0ff;
|
||||
border-radius: 4px;
|
||||
padding: 0 4px;
|
||||
}
|
||||
|
||||
.prompt-input :deep(.el-textarea__inner) {
|
||||
color: transparent !important;
|
||||
caret-color: #344054;
|
||||
background: transparent !important;
|
||||
z-index: 2;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.model-option { width: 100%; display: flex; align-items: center; justify-content: space-between; gap: 16px; }
|
||||
.model-option small { color: #98a2b3; font-size: 11px; }
|
||||
.model-empty { padding: 12px; color: #8a93a3; font-size: 12px; text-align: center; }
|
||||
@@ -295,6 +467,42 @@ function sectionValidationMessage() {
|
||||
}
|
||||
}
|
||||
|
||||
.output-type-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) 150px;
|
||||
align-items: center;
|
||||
gap: 24px;
|
||||
}
|
||||
|
||||
.output-type-select {
|
||||
width: 150px;
|
||||
}
|
||||
|
||||
.output-type-field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
margin-bottom: 14px;
|
||||
padding-bottom: 14px;
|
||||
border-bottom: 1px solid #e7eaf0;
|
||||
}
|
||||
|
||||
.output-type-hint {
|
||||
min-height: 20px;
|
||||
margin: 0;
|
||||
color: #667085;
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
|
||||
code {
|
||||
padding: 2px 5px;
|
||||
color: #5b50f2;
|
||||
background: #f0f0ff;
|
||||
border-radius: 4px;
|
||||
font-family: inherit;
|
||||
}
|
||||
}
|
||||
|
||||
.advanced-settings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
@@ -376,6 +584,8 @@ function sectionValidationMessage() {
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.advanced-settings-grid { grid-template-columns: 1fr; }
|
||||
.output-type-row { grid-template-columns: 1fr; gap: 10px; }
|
||||
.output-type-select { width: 100%; }
|
||||
.quality-rule-list { grid-template-columns: 1fr; }
|
||||
.minimum-length-field { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ defineProps<{
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
stop: []
|
||||
retry: []
|
||||
}>()
|
||||
</script>
|
||||
@@ -44,7 +43,7 @@ const emit = defineEmits<{
|
||||
{{ generation.status === 'idle' ? '准备开始处理'
|
||||
: generation.status === 'running' ? '正在生成数据'
|
||||
: generation.status === 'success' ? '数据生成完成'
|
||||
: '生成已停止' }}
|
||||
: '生成失败' }}
|
||||
</h3>
|
||||
<p>{{ generation.message }}</p>
|
||||
<el-progress
|
||||
@@ -58,9 +57,6 @@ const emit = defineEmits<{
|
||||
<span>应用预览修改</span>
|
||||
<span>生成标准结果</span>
|
||||
</div>
|
||||
<el-button v-if="generation.status === 'running'" plain type="warning" @click="emit('stop')">
|
||||
停止生成
|
||||
</el-button>
|
||||
<el-button v-if="generation.status === 'failed'" plain type="primary" @click="emit('retry')">
|
||||
重新生成
|
||||
</el-button>
|
||||
|
||||
622
frontend/src/views/data-process/create/OfficeSourceViewer.vue
Normal file
622
frontend/src/views/data-process/create/OfficeSourceViewer.vue
Normal file
@@ -0,0 +1,622 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, ref, shallowRef, watch } from 'vue'
|
||||
import {
|
||||
getDataProcessOfficePreview,
|
||||
getDataProcessSourceRawUrl,
|
||||
type DataProcessDocxPreview,
|
||||
type DataProcessDocxTableRow,
|
||||
type DataProcessOfficePreview,
|
||||
type DataProcessXlsxPreview,
|
||||
type DataProcessXlsxPreviewRow,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { PreviewItem } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
taskId: string | number | null
|
||||
sourceFileId: string | number | null
|
||||
fileName: string
|
||||
fileFormat?: string
|
||||
selectedItem: PreviewItem | null
|
||||
}>()
|
||||
|
||||
const XLSX_PAGE_SIZE = 100
|
||||
const scrollRef = ref<HTMLElement | null>(null)
|
||||
const preview = shallowRef<DataProcessOfficePreview | null>(null)
|
||||
const loading = ref(false)
|
||||
const errorMessage = ref('')
|
||||
const activeSheetIndex = ref(0)
|
||||
const pageOffset = ref(0)
|
||||
let loadSequence = 0
|
||||
const prefersReducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)').matches
|
||||
|
||||
const normalizedFormat = computed(() => (
|
||||
props.fileFormat?.toLowerCase().replace(/^\./, '')
|
||||
|| props.fileName.split('.').pop()?.toLowerCase()
|
||||
|| ''
|
||||
))
|
||||
const isDocx = computed(() => normalizedFormat.value === 'docx')
|
||||
const docxPreview = computed((): DataProcessDocxPreview | null => (
|
||||
preview.value?.format === 'docx' ? preview.value : null
|
||||
))
|
||||
const xlsxPreview = computed((): DataProcessXlsxPreview | null => (
|
||||
preview.value?.format === 'xlsx' ? preview.value : null
|
||||
))
|
||||
const sourceUrl = computed(() => (
|
||||
props.taskId != null && props.sourceFileId != null
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
const visibleRowRange = computed(() => {
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
|
||||
const start = sheet.offset + 1
|
||||
const end = sheet.offset + sheet.rows.length
|
||||
return `第 ${start}–${end} 条记录`
|
||||
})
|
||||
|
||||
function overlapsSelection(start: number, end: number) {
|
||||
const item = props.selectedItem
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
|
||||
return end > item.sourceStart && start < item.sourceEnd
|
||||
}
|
||||
|
||||
function tableRowHighlighted(row: DataProcessDocxTableRow) {
|
||||
return overlapsSelection(row.source_start, row.source_end)
|
||||
}
|
||||
|
||||
function stableValue(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(stableValue)
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, item]) => [key, stableValue(item)]),
|
||||
)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function recordKey(value: unknown) {
|
||||
try {
|
||||
return JSON.stringify(stableValue(value))
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
const selectedRecordKey = computed(() => {
|
||||
const content = props.selectedItem?.originalContent
|
||||
if (!content) return ''
|
||||
try {
|
||||
return recordKey(JSON.parse(content))
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
})
|
||||
|
||||
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
|
||||
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
|
||||
}
|
||||
|
||||
function displayCell(value: unknown) {
|
||||
if (value == null || value === '') return '—'
|
||||
if (typeof value === 'object') return JSON.stringify(value)
|
||||
return String(value)
|
||||
}
|
||||
|
||||
async function locateSelectedItem() {
|
||||
await nextTick()
|
||||
const selected = scrollRef.value?.querySelector<HTMLElement>(
|
||||
'.docx-block.is-highlighted, .docx-table-row.is-highlighted, .xlsx-row.is-highlighted',
|
||||
)
|
||||
selected?.scrollIntoView({
|
||||
block: 'center',
|
||||
inline: 'nearest',
|
||||
behavior: prefersReducedMotion ? 'auto' : 'smooth',
|
||||
})
|
||||
}
|
||||
|
||||
async function loadPreview(options: { reset?: boolean } = {}) {
|
||||
const sequence = ++loadSequence
|
||||
if (options.reset) {
|
||||
activeSheetIndex.value = 0
|
||||
pageOffset.value = 0
|
||||
preview.value = null
|
||||
}
|
||||
errorMessage.value = ''
|
||||
if (props.taskId == null || props.sourceFileId == null) {
|
||||
errorMessage.value = '缺少原文件标识，无法加载预览'
|
||||
return
|
||||
}
|
||||
|
||||
loading.value = true
|
||||
try {
|
||||
const result = await getDataProcessOfficePreview(
|
||||
props.taskId,
|
||||
props.sourceFileId,
|
||||
isDocx.value
|
||||
? {}
|
||||
: {
|
||||
sheet_index: activeSheetIndex.value,
|
||||
offset: pageOffset.value,
|
||||
limit: XLSX_PAGE_SIZE,
|
||||
},
|
||||
)
|
||||
if (sequence !== loadSequence) return
|
||||
preview.value = result
|
||||
if (result.format === 'xlsx') activeSheetIndex.value = result.active_sheet.index
|
||||
await locateSelectedItem()
|
||||
} catch (error) {
|
||||
if (sequence !== loadSequence) return
|
||||
errorMessage.value = error instanceof Error ? error.message : 'Office 原文件预览加载失败'
|
||||
} finally {
|
||||
if (sequence === loadSequence) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function changeSheet(value: string | number) {
|
||||
activeSheetIndex.value = Number(value)
|
||||
pageOffset.value = 0
|
||||
void loadPreview()
|
||||
}
|
||||
|
||||
function previousPage() {
|
||||
pageOffset.value = Math.max(0, pageOffset.value - XLSX_PAGE_SIZE)
|
||||
void loadPreview()
|
||||
}
|
||||
|
||||
function nextPage() {
|
||||
if (!xlsxPreview.value?.active_sheet.has_more) return
|
||||
pageOffset.value += XLSX_PAGE_SIZE
|
||||
void loadPreview()
|
||||
}
|
||||
|
||||
watch(
|
||||
() => [props.taskId, props.sourceFileId, normalizedFormat.value],
|
||||
() => void loadPreview({ reset: true }),
|
||||
{ immediate: true },
|
||||
)
|
||||
|
||||
watch(
|
||||
() => props.selectedItem?.id,
|
||||
() => void locateSelectedItem(),
|
||||
)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div
|
||||
class="office-source-viewer"
|
||||
:aria-label="`${normalizedFormat.toUpperCase()} 预览：${fileName}`"
|
||||
>
|
||||
<div class="office-toolbar">
|
||||
<template v-if="xlsxPreview">
|
||||
<div class="sheet-selector">
|
||||
<span>工作表</span>
|
||||
<el-select
|
||||
:model-value="activeSheetIndex"
|
||||
size="small"
|
||||
aria-label="选择 Excel 工作表"
|
||||
@update:model-value="changeSheet"
|
||||
>
|
||||
<el-option
|
||||
v-for="sheet in xlsxPreview.sheets"
|
||||
:key="sheet.index"
|
||||
:label="sheet.name"
|
||||
:value="sheet.index"
|
||||
/>
|
||||
</el-select>
|
||||
</div>
|
||||
<span>{{ visibleRowRange }}</span>
|
||||
</template>
|
||||
<template v-else>
|
||||
<span>Word 网页版式预览</span>
|
||||
<span v-if="docxPreview?.truncated">文档较长，仅展示前 2,000 个内容块</span>
|
||||
</template>
|
||||
<span v-if="loading && preview" class="toolbar-loading" role="status">
|
||||
<i class="fa fa-spinner fa-spin" /> 正在更新预览
|
||||
</span>
|
||||
<a
|
||||
v-if="sourceUrl"
|
||||
class="source-file-link"
|
||||
:href="sourceUrl"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<i class="fa fa-external-link" /> 打开原文件
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div v-if="loading && !preview" class="office-state" role="status">
|
||||
<i class="fa fa-spinner fa-spin" />
|
||||
<strong>正在加载原文件预览</strong>
|
||||
<span>{{ isDocx ? '正在还原 Word 文档结构' : '正在读取 Excel 工作表' }}</span>
|
||||
</div>
|
||||
|
||||
<div v-else-if="errorMessage" class="office-state is-error" role="alert">
|
||||
<i class="fa fa-exclamation-circle" />
|
||||
<strong>{{ isDocx ? 'Word 预览失败' : 'Excel 预览失败' }}</strong>
|
||||
<span>{{ errorMessage }}</span>
|
||||
<div>
|
||||
<el-button type="primary" size="small" @click="loadPreview()">重试</el-button>
|
||||
<el-button v-if="sourceUrl" tag="a" :href="sourceUrl" target="_blank" size="small">
|
||||
打开原文件
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="docxPreview" ref="scrollRef" class="docx-scroll">
|
||||
<article class="docx-page">
|
||||
<template v-for="(block, index) in docxPreview.blocks" :key="index">
|
||||
<component
|
||||
:is="block.heading_level ? `h${block.heading_level}` : 'p'"
|
||||
v-if="block.type === 'paragraph'"
|
||||
class="docx-block"
|
||||
:class="{
|
||||
'is-highlighted': overlapsSelection(block.source_start, block.source_end),
|
||||
'is-list': block.is_list,
|
||||
}"
|
||||
:style="{ textAlign: block.alignment }"
|
||||
:data-source-start="block.source_start"
|
||||
>
|
||||
{{ block.text }}
|
||||
</component>
|
||||
<div v-else class="docx-table-wrap">
|
||||
<table class="docx-table">
|
||||
<tbody>
|
||||
<tr
|
||||
v-for="(row, rowIndex) in block.rows"
|
||||
:key="rowIndex"
|
||||
class="docx-table-row"
|
||||
:class="{ 'is-highlighted': tableRowHighlighted(row) }"
|
||||
:data-source-start="row.source_start"
|
||||
>
|
||||
<td v-for="(cell, cellIndex) in row.cells" :key="cellIndex">{{ cell || ' ' }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</template>
|
||||
<div v-if="!docxPreview.blocks.length" class="office-empty">文档中没有可预览的正文</div>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<template v-else-if="xlsxPreview">
|
||||
<div ref="scrollRef" class="xlsx-scroll">
|
||||
<table v-if="xlsxPreview.active_sheet.columns.length" class="xlsx-grid">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="row-number-cell">#</th>
|
||||
<th
|
||||
v-for="column in xlsxPreview.active_sheet.columns"
|
||||
:key="column"
|
||||
:title="column"
|
||||
>
|
||||
{{ column }}
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr
|
||||
v-for="row in xlsxPreview.active_sheet.rows"
|
||||
:key="row.row_number"
|
||||
class="xlsx-row"
|
||||
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
|
||||
>
|
||||
<th class="row-number-cell">{{ row.row_number }}</th>
|
||||
<td
|
||||
v-for="(value, cellIndex) in row.values"
|
||||
:key="cellIndex"
|
||||
:title="displayCell(value)"
|
||||
>
|
||||
{{ displayCell(value) }}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div v-else class="office-empty">当前工作表没有可预览记录</div>
|
||||
</div>
|
||||
<div class="xlsx-pagination">
|
||||
<el-button
|
||||
size="small"
|
||||
:disabled="pageOffset === 0 || loading"
|
||||
aria-label="上一页工作表记录"
|
||||
@click="previousPage"
|
||||
>
|
||||
<i class="fa fa-angle-left" /> 上一页
|
||||
</el-button>
|
||||
<span>{{ visibleRowRange }}</span>
|
||||
<el-button
|
||||
size="small"
|
||||
:disabled="!xlsxPreview.active_sheet.has_more || loading"
|
||||
aria-label="下一页工作表记录"
|
||||
@click="nextPage"
|
||||
>
|
||||
下一页 <i class="fa fa-angle-right" />
|
||||
</el-button>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.office-source-viewer {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
background: #f4f6f9;
|
||||
}
|
||||
|
||||
.office-toolbar {
|
||||
display: flex;
|
||||
min-height: 42px;
|
||||
flex: none;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 6px 12px;
|
||||
color: #7d8798;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #e5e8ee;
|
||||
font-size: 11px;
|
||||
|
||||
}
|
||||
|
||||
.sheet-selector {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
|
||||
> span {
|
||||
flex: none;
|
||||
}
|
||||
|
||||
:deep(.el-select) {
|
||||
width: min(220px, 28vw);
|
||||
}
|
||||
}
|
||||
|
||||
.source-file-link {
|
||||
flex: none;
|
||||
margin-left: auto;
|
||||
color: #5147df;
|
||||
text-decoration: none;
|
||||
|
||||
&:hover,
|
||||
&:focus-visible {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
|
||||
.toolbar-loading {
|
||||
color: #5b50f2;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.office-state {
|
||||
display: flex;
|
||||
flex: 1;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
gap: 9px;
|
||||
padding: 28px;
|
||||
color: #667085;
|
||||
text-align: center;
|
||||
|
||||
> i {
|
||||
color: #5b50f2;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
> strong {
|
||||
color: #344054;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
> span {
|
||||
max-width: 420px;
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
&.is-error > i {
|
||||
color: #d92d20;
|
||||
}
|
||||
}
|
||||
|
||||
.docx-scroll {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
padding: 22px;
|
||||
overflow: auto;
|
||||
scroll-behavior: smooth;
|
||||
}
|
||||
|
||||
.docx-page {
|
||||
width: min(760px, 100%);
|
||||
min-height: calc(100% - 2px);
|
||||
padding: 54px clamp(30px, 7%, 68px);
|
||||
margin: 0 auto;
|
||||
color: #262b34;
|
||||
background: #fff;
|
||||
border: 1px solid #dfe3e9;
|
||||
box-shadow: 0 2px 10px rgb(15 23 42 / 8%);
|
||||
font-family: "Songti SC", SimSun, serif;
|
||||
font-size: 13px;
|
||||
line-height: 1.8;
|
||||
}
|
||||
|
||||
.docx-block {
|
||||
padding: 2px 6px;
|
||||
margin: 0 0 10px;
|
||||
border-radius: 3px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
transition: background-color 0.18s ease, box-shadow 0.18s ease;
|
||||
|
||||
&.is-list {
|
||||
padding-left: 22px;
|
||||
}
|
||||
|
||||
&.is-highlighted {
|
||||
background: #fff0b8;
|
||||
box-shadow: inset 3px 0 #f0b429;
|
||||
}
|
||||
}
|
||||
|
||||
h1.docx-block { font-size: 22px; }
|
||||
h2.docx-block { font-size: 19px; }
|
||||
h3.docx-block { font-size: 17px; }
|
||||
h4.docx-block,
|
||||
h5.docx-block,
|
||||
h6.docx-block { font-size: 15px; }
|
||||
|
||||
.docx-table-wrap {
|
||||
max-width: 100%;
|
||||
margin: 12px 0 18px;
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.docx-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
table-layout: fixed;
|
||||
|
||||
td {
|
||||
padding: 7px 9px;
|
||||
border: 1px solid #9da5b2;
|
||||
vertical-align: top;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
}
|
||||
|
||||
.docx-table-row.is-highlighted td {
|
||||
background: #fff0b8;
|
||||
}
|
||||
|
||||
.xlsx-scroll {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
background: #fff;
|
||||
scroll-behavior: smooth;
|
||||
}
|
||||
|
||||
.xlsx-grid {
|
||||
min-width: 100%;
|
||||
color: #344054;
|
||||
border-spacing: 0;
|
||||
border-collapse: separate;
|
||||
table-layout: auto;
|
||||
font-size: 11px;
|
||||
|
||||
th,
|
||||
td {
|
||||
min-width: 120px;
|
||||
max-width: 320px;
|
||||
height: 36px;
|
||||
padding: 7px 10px;
|
||||
overflow: hidden;
|
||||
text-align: left;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
border-right: 1px solid #e4e7ec;
|
||||
border-bottom: 1px solid #e4e7ec;
|
||||
}
|
||||
|
||||
thead th {
|
||||
position: sticky;
|
||||
z-index: 2;
|
||||
top: 0;
|
||||
color: #475467;
|
||||
background: #f2f4f7;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
tbody tr:hover td,
|
||||
tbody tr:hover th {
|
||||
background: #f9fafb;
|
||||
}
|
||||
}
|
||||
|
||||
.row-number-cell {
|
||||
position: sticky;
|
||||
z-index: 1;
|
||||
left: 0;
|
||||
min-width: 54px !important;
|
||||
width: 54px;
|
||||
color: #98a2b3;
|
||||
text-align: center !important;
|
||||
background: #f8fafc;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
thead .row-number-cell {
|
||||
z-index: 3;
|
||||
}
|
||||
|
||||
.xlsx-row.is-highlighted {
|
||||
td,
|
||||
th {
|
||||
background: #fff0b8;
|
||||
box-shadow: inset 0 2px #f0b429, inset 0 -2px #f0b429;
|
||||
}
|
||||
}
|
||||
|
||||
.xlsx-pagination {
|
||||
display: flex;
|
||||
min-height: 46px;
|
||||
flex: none;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 12px;
|
||||
padding: 7px 12px;
|
||||
color: #7d8798;
|
||||
background: #fff;
|
||||
border-top: 1px solid #e5e8ee;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.office-empty {
|
||||
display: flex;
|
||||
min-height: 180px;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #98a2b3;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.office-source-viewer {
|
||||
height: 360px;
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.office-toolbar {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.docx-scroll {
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.docx-page {
|
||||
padding: 34px 24px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.docx-scroll,
|
||||
.xlsx-scroll {
|
||||
scroll-behavior: auto;
|
||||
}
|
||||
|
||||
.docx-block {
|
||||
transition: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
667
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
667
frontend/src/views/data-process/create/PdfSourceViewer.vue
Normal file
@@ -0,0 +1,667 @@
|
||||
<script setup lang="ts">
|
||||
import {
|
||||
GlobalWorkerOptions,
|
||||
TextLayer,
|
||||
getDocument,
|
||||
type PDFDocumentLoadingTask,
|
||||
type PDFDocumentProxy,
|
||||
type RenderTask,
|
||||
} from 'pdfjs-dist'
|
||||
import pdfWorkerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url'
|
||||
import { computed, nextTick, onBeforeUnmount, ref, shallowRef, watch } from 'vue'
|
||||
import {
|
||||
getDataProcessPdfPages,
|
||||
getDataProcessSourceRawUrl,
|
||||
type DataProcessPdfPageRange,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { PreviewItem } from './types'
|
||||
|
||||
GlobalWorkerOptions.workerSrc = pdfWorkerUrl
|
||||
|
||||
const props = defineProps<{
|
||||
taskId: string | number | null
|
||||
sourceFileId: string | number | null
|
||||
fileName: string
|
||||
selectedItem: PreviewItem | null
|
||||
}>()
|
||||
|
||||
const scrollRef = ref<HTMLElement | null>(null)
|
||||
const pageRef = ref<HTMLElement | null>(null)
|
||||
const canvasRef = ref<HTMLCanvasElement | null>(null)
|
||||
const textLayerRef = ref<HTMLElement | null>(null)
|
||||
const documentRef = shallowRef<PDFDocumentProxy | null>(null)
|
||||
const pageRanges = ref<DataProcessPdfPageRange[]>([])
|
||||
const currentPage = ref(1)
|
||||
const pageCount = ref(0)
|
||||
const zoom = ref(1)
|
||||
const renderedScale = ref(1)
|
||||
const loading = ref(false)
|
||||
const rendering = ref(false)
|
||||
const errorMessage = ref('')
|
||||
const highlightState = ref<'idle' | 'highlighted' | 'unmatched' | 'manual'>('idle')
|
||||
|
||||
let loadingTask: PDFDocumentLoadingTask | null = null
|
||||
let renderTask: RenderTask | null = null
|
||||
let textLayer: TextLayer | null = null
|
||||
let loadSequence = 0
|
||||
let renderSequence = 0
|
||||
let renderedPageNumber = 0
|
||||
let resizeFrame = 0
|
||||
|
||||
const sourceUrl = computed(() => (
|
||||
props.taskId != null && props.sourceFileId != null
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
|
||||
const pageStyle = computed(() => ({
|
||||
'--total-scale-factor': String(renderedScale.value),
|
||||
}))
|
||||
|
||||
const locationText = computed(() => {
|
||||
if (highlightState.value === 'highlighted') return `切片原文已在第 ${currentPage.value} 页高亮`
|
||||
if (highlightState.value === 'unmatched') return `已定位第 ${currentPage.value} 页，未匹配到可高亮文字`
|
||||
if (highlightState.value === 'manual') return '手动新增切片没有原文位置'
|
||||
return pageCount.value ? `第 ${currentPage.value} / ${pageCount.value} 页` : '正在读取 PDF'
|
||||
})
|
||||
|
||||
function pageForItem(item: PreviewItem | null) {
|
||||
if (!item) return null
|
||||
if (item.sourceStart == null || item.sourceEnd == null) {
|
||||
const pageNumber = item.sourcePages?.[0]
|
||||
return pageNumber == null
|
||||
? null
|
||||
: pageRanges.value.find((page) => page.page_number === pageNumber) ?? null
|
||||
}
|
||||
return pageRanges.value.find((page) => (
|
||||
item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end
|
||||
)) ?? pageRanges.value.find((page) => (
|
||||
item.sourceStart! < page.source_end && item.sourceEnd! > page.source_start
|
||||
)) ?? null
|
||||
}
|
||||
|
||||
function selectedTextForPage(page: DataProcessPdfPageRange) {
|
||||
const item = props.selectedItem
|
||||
if (!item) return ''
|
||||
if (item.sourceStart == null || item.sourceEnd == null) {
|
||||
return item.sourcePages?.includes(page.page_number) ? item.originalContent : ''
|
||||
}
|
||||
const intersectionStart = Math.max(item.sourceStart, page.source_start)
|
||||
const intersectionEnd = Math.min(item.sourceEnd, page.source_end)
|
||||
if (intersectionEnd <= intersectionStart) return ''
|
||||
const relativeStart = Math.max(0, intersectionStart - item.sourceStart)
|
||||
const relativeEnd = Math.max(relativeStart, intersectionEnd - item.sourceStart)
|
||||
return item.originalContent.slice(relativeStart, relativeEnd)
|
||||
}
|
||||
|
||||
function normalizeLocatorText(value: string) {
|
||||
return value.normalize('NFKC').replace(/[^\p{L}\p{N}]+/gu, '').toLocaleLowerCase()
|
||||
}
|
||||
|
||||
function nearestOccurrence(haystack: string, needle: string, expectedIndex: number) {
|
||||
let nearest = -1
|
||||
let nearestDistance = Number.POSITIVE_INFINITY
|
||||
let cursor = haystack.indexOf(needle)
|
||||
while (cursor >= 0) {
|
||||
const distance = Math.abs(cursor - expectedIndex)
|
||||
if (distance < nearestDistance) {
|
||||
nearest = cursor
|
||||
nearestDistance = distance
|
||||
}
|
||||
cursor = haystack.indexOf(needle, cursor + 1)
|
||||
}
|
||||
return nearest
|
||||
}
|
||||
|
||||
function findHighlight(
|
||||
itemStrings: string[],
|
||||
selectedText: string,
|
||||
page: DataProcessPdfPageRange,
|
||||
) {
|
||||
const itemRanges: Array<{ start: number; end: number }> = []
|
||||
let pageText = ''
|
||||
for (const item of itemStrings) {
|
||||
const start = pageText.length
|
||||
pageText += normalizeLocatorText(item)
|
||||
itemRanges.push({ start, end: pageText.length })
|
||||
}
|
||||
|
||||
const target = normalizeLocatorText(selectedText)
|
||||
if (!pageText || !target) return []
|
||||
const itemStart = props.selectedItem?.sourceStart ?? page.source_start
|
||||
const sourceLength = Math.max(1, page.source_end - page.source_start)
|
||||
const expectedRatio = Math.min(1, Math.max(0, (itemStart - page.source_start) / sourceLength))
|
||||
const expectedIndex = Math.round(pageText.length * expectedRatio)
|
||||
const anchorLengths = [target.length, 120, 80, 48, 24, 12, 8]
|
||||
.map((length) => Math.min(length, target.length))
|
||||
.filter((length, index, values) => length >= 4 && values.indexOf(length) === index)
|
||||
|
||||
for (const anchorLength of anchorLengths) {
|
||||
const anchor = target.slice(0, anchorLength)
|
||||
const matchStart = nearestOccurrence(pageText, anchor, expectedIndex)
|
||||
if (matchStart < 0) continue
|
||||
const matchEnd = matchStart + anchor.length
|
||||
return itemRanges.reduce<number[]>((matches, range, index) => {
|
||||
if (range.end > matchStart && range.start < matchEnd) matches.push(index)
|
||||
return matches
|
||||
}, [])
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
function applySelectedHighlight(page: DataProcessPdfPageRange | null) {
|
||||
const item = props.selectedItem
|
||||
for (const element of textLayer?.textDivs ?? []) {
|
||||
element.classList.remove('is-slice-highlighted')
|
||||
}
|
||||
if (!item || (
|
||||
(item.sourceStart == null || item.sourceEnd == null)
|
||||
&& !item.sourcePages?.length
|
||||
)) {
|
||||
highlightState.value = item ? 'manual' : 'idle'
|
||||
return
|
||||
}
|
||||
if (!page || page.page_number !== currentPage.value || !textLayer) {
|
||||
highlightState.value = 'idle'
|
||||
return
|
||||
}
|
||||
const matches = findHighlight(
|
||||
textLayer.textContentItemsStr,
|
||||
selectedTextForPage(page),
|
||||
page,
|
||||
)
|
||||
for (const index of matches) {
|
||||
textLayer.textDivs[index]?.classList.add('is-slice-highlighted')
|
||||
}
|
||||
highlightState.value = matches.length ? 'highlighted' : 'unmatched'
|
||||
}
|
||||
|
||||
async function renderCurrentPage(force = false) {
|
||||
const document = documentRef.value
|
||||
const pageContainer = pageRef.value
|
||||
const canvas = canvasRef.value
|
||||
const layerContainer = textLayerRef.value
|
||||
const scroller = scrollRef.value
|
||||
if (!document || !pageContainer || !canvas || !layerContainer || !scroller) return
|
||||
|
||||
const selectedPage = pageForItem(props.selectedItem)
|
||||
if (!force && renderedPageNumber === currentPage.value && textLayer) {
|
||||
applySelectedHighlight(selectedPage)
|
||||
return
|
||||
}
|
||||
|
||||
const sequence = ++renderSequence
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
renderTask = null
|
||||
textLayer = null
|
||||
rendering.value = true
|
||||
errorMessage.value = ''
|
||||
|
||||
try {
|
||||
const page = await document.getPage(currentPage.value)
|
||||
if (sequence !== renderSequence) return
|
||||
const baseViewport = page.getViewport({ scale: 1 })
|
||||
const availableWidth = Math.max(280, scroller.clientWidth - 36)
|
||||
const scale = (availableWidth / baseViewport.width) * zoom.value
|
||||
const viewport = page.getViewport({ scale })
|
||||
const outputScale = Math.max(1, window.devicePixelRatio || 1)
|
||||
const context = canvas.getContext('2d')
|
||||
if (!context) throw new Error('浏览器无法创建 PDF 画布')
|
||||
|
||||
renderedScale.value = scale
|
||||
pageContainer.style.width = `${viewport.width}px`
|
||||
pageContainer.style.height = `${viewport.height}px`
|
||||
canvas.width = Math.floor(viewport.width * outputScale)
|
||||
canvas.height = Math.floor(viewport.height * outputScale)
|
||||
canvas.style.width = `${viewport.width}px`
|
||||
canvas.style.height = `${viewport.height}px`
|
||||
layerContainer.replaceChildren()
|
||||
|
||||
const activeRenderTask = page.render({
|
||||
canvas,
|
||||
viewport,
|
||||
transform: outputScale === 1 ? undefined : [outputScale, 0, 0, outputScale, 0, 0],
|
||||
})
|
||||
renderTask = activeRenderTask
|
||||
// 立即挂接取消处理，避免 ResizeObserver 触发重绘时产生未处理的取消异常。
|
||||
const canvasRenderPromise = activeRenderTask.promise.catch((error: unknown) => {
|
||||
if (
|
||||
sequence !== renderSequence
|
||||
|| (error instanceof Error && error.name === 'RenderingCancelledException')
|
||||
) return
|
||||
throw error
|
||||
})
|
||||
// WebKit 兼容：PDF.js 的 getTextContent() 依赖 ReadableStream 异步迭代，
|
||||
// 部分 Safari/WKWebView 未实现该接口。TextLayer 可直接通过 getReader() 消费文本流。
|
||||
const textContentSource = page.streamTextContent()
|
||||
if (sequence !== renderSequence) return
|
||||
const activeTextLayer = new TextLayer({
|
||||
textContentSource,
|
||||
container: layerContainer,
|
||||
viewport,
|
||||
})
|
||||
textLayer = activeTextLayer
|
||||
await Promise.all([canvasRenderPromise, activeTextLayer.render()])
|
||||
if (sequence !== renderSequence) return
|
||||
renderedPageNumber = currentPage.value
|
||||
applySelectedHighlight(selectedPage)
|
||||
await nextTick()
|
||||
pageContainer.querySelector<HTMLElement>('.is-slice-highlighted')
|
||||
?.scrollIntoView({ block: 'center', inline: 'center', behavior: 'smooth' })
|
||||
} catch (error) {
|
||||
if (sequence !== renderSequence) return
|
||||
errorMessage.value = error instanceof Error ? error.message : 'PDF 页面渲染失败'
|
||||
} finally {
|
||||
if (sequence === renderSequence) rendering.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function locateSelectedItem() {
|
||||
if (!documentRef.value) return
|
||||
const item = props.selectedItem
|
||||
if (!item || (
|
||||
(item.sourceStart == null || item.sourceEnd == null)
|
||||
&& !item.sourcePages?.length
|
||||
)) {
|
||||
applySelectedHighlight(null)
|
||||
return
|
||||
}
|
||||
const page = pageForItem(item)
|
||||
if (!page) {
|
||||
highlightState.value = 'unmatched'
|
||||
return
|
||||
}
|
||||
if (currentPage.value !== page.page_number) {
|
||||
currentPage.value = page.page_number
|
||||
await renderCurrentPage(true)
|
||||
return
|
||||
}
|
||||
await renderCurrentPage(false)
|
||||
}
|
||||
|
||||
async function loadPdf() {
|
||||
const sequence = ++loadSequence
|
||||
++renderSequence
|
||||
renderedPageNumber = 0
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
await loadingTask?.destroy()
|
||||
loadingTask = null
|
||||
documentRef.value = null
|
||||
pageRanges.value = []
|
||||
pageCount.value = 0
|
||||
currentPage.value = 1
|
||||
errorMessage.value = ''
|
||||
highlightState.value = 'idle'
|
||||
if (!sourceUrl.value || props.taskId == null || props.sourceFileId == null) return
|
||||
|
||||
loading.value = true
|
||||
try {
|
||||
const task = getDocument({ url: sourceUrl.value })
|
||||
loadingTask = task
|
||||
const [mapping, document] = await Promise.all([
|
||||
getDataProcessPdfPages(props.taskId, props.sourceFileId),
|
||||
task.promise,
|
||||
])
|
||||
if (sequence !== loadSequence) {
|
||||
await task.destroy()
|
||||
return
|
||||
}
|
||||
documentRef.value = document
|
||||
pageRanges.value = mapping.pages
|
||||
pageCount.value = document.numPages
|
||||
await nextTick()
|
||||
await locateSelectedItem()
|
||||
if (!props.selectedItem) await renderCurrentPage(true)
|
||||
} catch (error) {
|
||||
if (sequence !== loadSequence) return
|
||||
errorMessage.value = error instanceof Error ? error.message : 'PDF 原文件加载失败'
|
||||
} finally {
|
||||
if (sequence === loadSequence) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function changePage(delta: number) {
|
||||
const nextPage = Math.min(pageCount.value, Math.max(1, currentPage.value + delta))
|
||||
if (nextPage === currentPage.value) return
|
||||
currentPage.value = nextPage
|
||||
await renderCurrentPage(true)
|
||||
}
|
||||
|
||||
async function changeZoom(delta: number) {
|
||||
const nextZoom = Math.min(2, Math.max(0.6, Number((zoom.value + delta).toFixed(1))))
|
||||
if (nextZoom === zoom.value) return
|
||||
zoom.value = nextZoom
|
||||
await renderCurrentPage(true)
|
||||
}
|
||||
|
||||
function scheduleResizeRender() {
|
||||
window.cancelAnimationFrame(resizeFrame)
|
||||
resizeFrame = window.requestAnimationFrame(() => {
|
||||
if (documentRef.value) void renderCurrentPage(true)
|
||||
})
|
||||
}
|
||||
|
||||
const resizeObserver = new ResizeObserver(scheduleResizeRender)
|
||||
watch(scrollRef, (element, previous) => {
|
||||
if (previous) resizeObserver.unobserve(previous)
|
||||
if (element) resizeObserver.observe(element)
|
||||
})
|
||||
watch(sourceUrl, () => void loadPdf(), { immediate: true })
|
||||
watch(() => props.selectedItem?.id, () => void locateSelectedItem())
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
++loadSequence
|
||||
++renderSequence
|
||||
window.cancelAnimationFrame(resizeFrame)
|
||||
resizeObserver.disconnect()
|
||||
renderTask?.cancel()
|
||||
textLayer?.cancel()
|
||||
void loadingTask?.destroy()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="pdf-source-viewer" aria-label="PDF 原文件预览">
|
||||
<div v-if="sourceUrl" class="pdf-toolbar">
|
||||
<div class="toolbar-group">
|
||||
<button type="button" aria-label="上一页" :disabled="currentPage <= 1 || loading" @click="changePage(-1)">
|
||||
<i class="fa fa-chevron-left" aria-hidden="true" />
|
||||
</button>
|
||||
<span class="page-indicator">{{ currentPage }} / {{ pageCount || '—' }}</span>
|
||||
<button type="button" aria-label="下一页" :disabled="currentPage >= pageCount || loading" @click="changePage(1)">
|
||||
<i class="fa fa-chevron-right" aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
<span class="location-state" :class="`is-${highlightState}`" role="status">
|
||||
<i v-if="highlightState === 'highlighted'" class="fa fa-map-marker" aria-hidden="true" />
|
||||
{{ locationText }}
|
||||
</span>
|
||||
<div class="toolbar-group">
|
||||
<button type="button" aria-label="缩小" :disabled="zoom <= 0.6 || loading" @click="changeZoom(-0.1)">−</button>
|
||||
<span class="zoom-indicator">{{ Math.round(zoom * 100) }}%</span>
|
||||
<button type="button" aria-label="放大" :disabled="zoom >= 2 || loading" @click="changeZoom(0.1)">+</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="sourceUrl" ref="scrollRef" class="pdf-scroll" tabindex="0" :aria-label="`PDF 预览：${fileName}`">
|
||||
<div
|
||||
ref="pageRef"
|
||||
class="pdf-page"
|
||||
:style="pageStyle"
|
||||
:data-page-number="currentPage"
|
||||
>
|
||||
<canvas ref="canvasRef" class="pdf-canvas" />
|
||||
<div ref="textLayerRef" class="pdf-text-layer" />
|
||||
</div>
|
||||
<div v-if="loading || rendering" class="pdf-loading" role="status">
|
||||
<i class="fa fa-spinner fa-spin" aria-hidden="true" />
|
||||
{{ loading ? '正在加载 PDF…' : '正在渲染页面…' }}
|
||||
</div>
|
||||
<div v-if="errorMessage" class="pdf-error" role="alert">
|
||||
<i class="fa fa-exclamation-circle" aria-hidden="true" />
|
||||
<strong>PDF 预览失败</strong>
|
||||
<span>{{ errorMessage }}</span>
|
||||
<a :href="sourceUrl" target="_blank" rel="noopener noreferrer">在新窗口打开原件</a>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="pdf-unavailable" role="status">
|
||||
<i class="fa fa-file-pdf-o" aria-hidden="true" />
|
||||
<strong>PDF 原文件暂不可预览</strong>
|
||||
<span>请重新上传该文件后再试。</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.pdf-source-viewer {
|
||||
display: flex;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
background: #525659;
|
||||
}
|
||||
|
||||
.pdf-toolbar {
|
||||
display: grid;
|
||||
min-height: 44px;
|
||||
flex: 0 0 auto;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 0 12px;
|
||||
color: #f2f4f7;
|
||||
background: #323639;
|
||||
border-bottom: 1px solid #1f2427;
|
||||
}
|
||||
|
||||
.toolbar-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
|
||||
button {
|
||||
display: inline-flex;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #f2f4f7;
|
||||
font: inherit;
|
||||
font-size: 16px;
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
|
||||
&:hover:not(:disabled) {
|
||||
background: rgb(255 255 255 / 12%);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
color: #7d8387;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.page-indicator,
|
||||
.zoom-indicator {
|
||||
min-width: 54px;
|
||||
color: #e4e7ec;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.location-state {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
color: #d0d5dd;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
|
||||
&.is-highlighted {
|
||||
color: #ffd666;
|
||||
}
|
||||
|
||||
&.is-unmatched {
|
||||
color: #fdb022;
|
||||
}
|
||||
}
|
||||
|
||||
.pdf-scroll {
|
||||
position: relative;
|
||||
display: flex;
|
||||
min-height: 538px;
|
||||
flex: 1;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
overflow: auto;
|
||||
padding: 18px;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.pdf-page {
|
||||
position: relative;
|
||||
flex: 0 0 auto;
|
||||
overflow: hidden;
|
||||
background: #fff;
|
||||
box-shadow: 0 2px 12px rgb(0 0 0 / 34%);
|
||||
}
|
||||
|
||||
.pdf-canvas {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.pdf-text-layer {
|
||||
--min-font-size: 1;
|
||||
--text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
|
||||
--min-font-size-inv: calc(1 / var(--min-font-size));
|
||||
|
||||
position: absolute;
|
||||
z-index: 1;
|
||||
inset: 0;
|
||||
overflow: clip;
|
||||
color-scheme: only light;
|
||||
line-height: 1;
|
||||
letter-spacing: normal;
|
||||
word-spacing: normal;
|
||||
text-align: initial;
|
||||
text-size-adjust: none;
|
||||
forced-color-adjust: none;
|
||||
transform-origin: 0 0;
|
||||
caret-color: CanvasText;
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(span),
|
||||
.pdf-text-layer :deep(br) {
|
||||
position: absolute;
|
||||
color: transparent;
|
||||
white-space: pre;
|
||||
cursor: text;
|
||||
user-select: text;
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
.pdf-text-layer > :deep(:not(.markedContent)),
|
||||
.pdf-text-layer :deep(.markedContent span:not(.markedContent)) {
|
||||
--font-height: 0;
|
||||
--scale-x: 1;
|
||||
--rotate: 0deg;
|
||||
|
||||
z-index: 1;
|
||||
font-size: calc(var(--text-scale-factor) * var(--font-height));
|
||||
transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(.markedContent) {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(.is-slice-highlighted) {
|
||||
margin: -2px;
|
||||
padding: 2px;
|
||||
background: rgb(255 202 40 / 48%);
|
||||
border-radius: 3px;
|
||||
box-shadow: 0 0 0 1px rgb(245 158 11 / 38%);
|
||||
}
|
||||
|
||||
.pdf-text-layer :deep(::selection) {
|
||||
color: transparent;
|
||||
background: rgb(37 99 235 / 30%);
|
||||
}
|
||||
|
||||
.pdf-loading,
|
||||
.pdf-error {
|
||||
position: absolute;
|
||||
z-index: 3;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transform: translate(-50%, -50%);
|
||||
}
|
||||
|
||||
.pdf-loading {
|
||||
gap: 8px;
|
||||
padding: 10px 14px;
|
||||
color: #f2f4f7;
|
||||
font-size: 13px;
|
||||
background: rgb(31 36 39 / 86%);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.pdf-error {
|
||||
width: min(360px, calc(100% - 32px));
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 20px;
|
||||
color: #667085;
|
||||
text-align: center;
|
||||
background: #fff;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 20px rgb(0 0 0 / 22%);
|
||||
|
||||
i {
|
||||
color: #d92d20;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
}
|
||||
|
||||
span,
|
||||
a {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.pdf-unavailable {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
color: #667085;
|
||||
background: #f8f9fb;
|
||||
|
||||
i {
|
||||
color: #d92d20;
|
||||
font-size: 36px;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
span {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.pdf-toolbar {
|
||||
gap: 5px;
|
||||
padding: 0 6px;
|
||||
}
|
||||
|
||||
.location-state {
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.pdf-scroll {
|
||||
min-height: 420px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -1,5 +1,7 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
import OfficeSourceViewer from './OfficeSourceViewer.vue'
|
||||
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||
import { sourceLines } from './previewModel'
|
||||
import type { PreviewItem, ProcessType } from './types'
|
||||
|
||||
@@ -9,6 +11,9 @@ const props = defineProps<{
|
||||
selectedId: string | null
|
||||
processType: ProcessType
|
||||
fileName: string
|
||||
fileFormat?: string
|
||||
taskId: string | number | null
|
||||
sourceFileId: string | number | null
|
||||
files: { id: string; name: string; count: number; modifiedCount: number }[]
|
||||
selectedFileId: string | null
|
||||
}>()
|
||||
@@ -17,18 +22,27 @@ const emit = defineEmits<{
|
||||
'update:selectedId': [value: string]
|
||||
'update:selectedFileId': [value: string]
|
||||
'update:item-content': [id: string, value: string]
|
||||
'restore:item': [id: string]
|
||||
'add:item': []
|
||||
'remove:item': [id: string]
|
||||
}>()
|
||||
|
||||
const sourceViewerRef = ref<HTMLElement | null>(null)
|
||||
const search = ref('')
|
||||
const currentPage = ref(1)
|
||||
const PREVIEW_PAGE_SIZE = 6
|
||||
const PREVIEW_PAGE_SIZE = 10
|
||||
const editingItemId = ref<string | null>(null)
|
||||
const editorDraft = ref('')
|
||||
const lines = computed(() => sourceLines(props.sourceText))
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||
const normalizedFileFormat = computed(() => (
|
||||
props.fileFormat?.toLowerCase().replace(/^\./, '')
|
||||
|| props.fileName.split('.').pop()?.toLowerCase()
|
||||
|| ''
|
||||
))
|
||||
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
|
||||
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !search.value.trim()
|
||||
@@ -71,6 +85,12 @@ function saveEditor() {
|
||||
closeEditor()
|
||||
}
|
||||
|
||||
function restoreItem() {
|
||||
if (!editingItem.value) return
|
||||
emit('restore:item', editingItem.value.id)
|
||||
closeEditor()
|
||||
}
|
||||
|
||||
function removeItem(item: PreviewItem) {
|
||||
selectItem(item.id)
|
||||
emit('remove:item', item.id)
|
||||
@@ -94,7 +114,7 @@ watch(selectedItem, async (item) => {
|
||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||
}
|
||||
|
||||
if (item.sourceStart == null) return
|
||||
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
|
||||
await nextTick()
|
||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||
@@ -106,6 +126,11 @@ function itemNumber(item: PreviewItem) {
|
||||
}
|
||||
|
||||
function lineRange(item: PreviewItem) {
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
return first === last ? `来源：第 ${first} 页` : `来源：第 ${first}–${last} 页`
|
||||
}
|
||||
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增，无源文件定位'
|
||||
return item.sourceStartLine === item.sourceEndLine
|
||||
? `来源：第 ${item.sourceStartLine} 行`
|
||||
@@ -152,7 +177,22 @@ function lineRange(item: PreviewItem) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
||||
<PdfSourceViewer
|
||||
v-if="isPdfSource"
|
||||
:task-id="taskId"
|
||||
:source-file-id="sourceFileId"
|
||||
:file-name="fileName"
|
||||
:selected-item="selectedItem ?? null"
|
||||
/>
|
||||
<OfficeSourceViewer
|
||||
v-else-if="isOfficeSource"
|
||||
:task-id="taskId"
|
||||
:source-file-id="sourceFileId"
|
||||
:file-name="fileName"
|
||||
:file-format="normalizedFileFormat"
|
||||
:selected-item="selectedItem ?? null"
|
||||
/>
|
||||
<div v-else ref="sourceViewerRef" class="source-viewer" tabindex="0" aria-label="源文件内容">
|
||||
<div
|
||||
v-for="line in lines"
|
||||
:key="line.number"
|
||||
@@ -169,7 +209,12 @@ function lineRange(item: PreviewItem) {
|
||||
<div class="preview-pane">
|
||||
<div class="pane-header">
|
||||
<strong>{{ processType === 'unstructured' ? '切片内容' : '记录内容' }}</strong>
|
||||
<span>共 {{ items.length.toLocaleString() }} 条</span>
|
||||
<div class="pane-header-actions">
|
||||
<span>共 {{ items.length.toLocaleString() }} 条</span>
|
||||
<el-button link type="primary" @click="emit('add:item')">
|
||||
<i class="fa fa-plus" /> 手动新增
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template v-if="!editingItem">
|
||||
@@ -185,6 +230,8 @@ function lineRange(item: PreviewItem) {
|
||||
:key="item.id"
|
||||
class="preview-item"
|
||||
:class="{ 'is-active': item.id === selectedItem?.id }"
|
||||
:data-preview-id="item.id"
|
||||
:aria-selected="item.id === selectedItem?.id"
|
||||
role="button"
|
||||
tabindex="0"
|
||||
@click="selectItem(item.id)"
|
||||
@@ -234,6 +281,13 @@ function lineRange(item: PreviewItem) {
|
||||
resize="none"
|
||||
/>
|
||||
<div class="editor-actions">
|
||||
<el-button
|
||||
v-if="editingItem.sourceStart != null"
|
||||
link
|
||||
@click="restoreItem"
|
||||
>
|
||||
<i class="fa fa-undo" /> 恢复原始内容
|
||||
</el-button>
|
||||
<div>
|
||||
<el-button @click="closeEditor">取消</el-button>
|
||||
<el-button type="primary" @click="saveEditor">保存修改</el-button>
|
||||
@@ -288,6 +342,17 @@ function lineRange(item: PreviewItem) {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.pane-header-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
|
||||
> span {
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
}
|
||||
}
|
||||
|
||||
.file-option {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto auto;
|
||||
|
||||
@@ -1,22 +1,74 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref } from 'vue'
|
||||
import type { ResultItem } from './types'
|
||||
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
items: ResultItem[]
|
||||
previewItems: PreviewItem[]
|
||||
selectedId: string | null
|
||||
regeneratingResultId: string | null
|
||||
bulkRegeneration: BulkResultRegenerationState
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:selectedId': [value: string]
|
||||
'update:field': [id: string, field: 'instruction' | 'input' | 'output', value: string]
|
||||
'restore:item': [id: string]
|
||||
'regenerate:item': [id: string]
|
||||
'regenerate:all': []
|
||||
}>()
|
||||
|
||||
const search = ref('')
|
||||
const invalidOnly = ref(false)
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
|
||||
const invalidCount = computed(() => (
|
||||
props.items.filter((item) => item.savedStatus === 'invalid').length
|
||||
))
|
||||
const bulkRegenerationActive = computed(() => props.bulkRegeneration.status === 'running')
|
||||
const bulkRegenerationVisible = computed(() => (
|
||||
props.bulkRegeneration.status !== 'idle' && props.bulkRegeneration.total > 0
|
||||
))
|
||||
const bulkRegenerationPercentage = computed(() => (
|
||||
props.bulkRegeneration.total > 0
|
||||
? Math.round((props.bulkRegeneration.completed / props.bulkRegeneration.total) * 100)
|
||||
: 0
|
||||
))
|
||||
const itemRegenerating = (id: string) => (
|
||||
props.regeneratingResultId === id
|
||||
|| props.bulkRegeneration.targetIds.includes(id)
|
||||
)
|
||||
const selectedItemRegenerating = computed(() => (
|
||||
selectedItem.value ? itemRegenerating(selectedItem.value.id) : false
|
||||
))
|
||||
const selectedSource = computed(() => {
|
||||
const previewItemId = selectedItem.value?.previewItemId
|
||||
if (!previewItemId) return null
|
||||
return props.previewItems.find((item) => item.id === previewItemId) ?? null
|
||||
})
|
||||
const selectedSourceContent = computed(() => (
|
||||
selectedSource.value?.editedContent.trim()
|
||||
|| selectedSource.value?.originalContent.trim()
|
||||
|| ''
|
||||
))
|
||||
const selectedSourceWasPreprocessed = computed(() => Boolean(
|
||||
selectedSource.value
|
||||
&& selectedSource.value.editedContent.trim()
|
||||
&& selectedSource.value.editedContent !== selectedSource.value.originalContent
|
||||
))
|
||||
const selectedSourceMeta = computed(() => {
|
||||
const source = selectedSource.value
|
||||
if (!source) return ''
|
||||
const parts: string[] = []
|
||||
if (source.sourceStartLine != null) {
|
||||
parts.push(
|
||||
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
|
||||
? `第 ${source.sourceStartLine}–${source.sourceEndLine} 行`
|
||||
: `第 ${source.sourceStartLine} 行`,
|
||||
)
|
||||
}
|
||||
if (source.tokenCount > 0) parts.push(`${source.tokenCount} Token`)
|
||||
return parts.join(' · ')
|
||||
})
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const keyword = search.value.trim().toLowerCase()
|
||||
@@ -37,8 +89,34 @@ function selectRelative(offset: number) {
|
||||
<template>
|
||||
<section class="result-step">
|
||||
<div class="result-workspace">
|
||||
<aside class="result-list-pane">
|
||||
<div class="pane-header"><strong>生成结果</strong><span>共 {{ items.length }} 条</span></div>
|
||||
<aside class="result-list-pane" :class="{ 'has-bulk-progress': bulkRegenerationVisible }">
|
||||
<div class="pane-header result-list-header">
|
||||
<div class="result-list-title"><strong>生成结果</strong><span>共 {{ items.length }} 条</span></div>
|
||||
<el-button
|
||||
v-if="invalidCount > 0"
|
||||
size="small"
|
||||
plain
|
||||
type="primary"
|
||||
:loading="bulkRegenerationActive"
|
||||
:disabled="Boolean(regeneratingResultId) || bulkRegenerationActive"
|
||||
@click="emit('regenerate:all')"
|
||||
>
|
||||
<i v-if="!bulkRegenerationActive" class="fa fa-refresh" style="margin-right: 4px;" />
|
||||
{{ bulkRegenerationActive ? '重新生成中' : `全部重新生成（${invalidCount}）` }}
|
||||
</el-button>
|
||||
</div>
|
||||
<div v-if="bulkRegenerationVisible" class="bulk-regeneration-progress">
|
||||
<div>
|
||||
<span>已处理 {{ bulkRegeneration.completed }} / {{ bulkRegeneration.total }}</span>
|
||||
<span>成功 {{ bulkRegeneration.succeeded }} · 失败 {{ bulkRegeneration.failed }}</span>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="bulkRegenerationPercentage"
|
||||
:show-text="false"
|
||||
:stroke-width="5"
|
||||
:color="bulkRegeneration.failed > 0 ? '#d97706' : '#5b50f2'"
|
||||
/>
|
||||
</div>
|
||||
<div class="result-toolbar">
|
||||
<el-input v-model="search" clearable size="small" placeholder="搜索结果">
|
||||
<template #prefix><i class="fa fa-search" /></template>
|
||||
@@ -59,7 +137,12 @@ function selectRelative(offset: number) {
|
||||
<strong>{{ item.instruction || '未填写指令' }}</strong>
|
||||
<small>{{ item.output || '未填写输出' }}</small>
|
||||
</span>
|
||||
<i class="fa" :class="item.status === 'invalid' ? 'fa-exclamation-circle is-error' : 'fa-check-circle is-valid'" />
|
||||
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
|
||||
<i
|
||||
v-else
|
||||
class="fa"
|
||||
:class="item.status === 'invalid' ? 'fa-exclamation-circle is-error' : 'fa-check-circle is-valid'"
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
</aside>
|
||||
@@ -69,14 +152,46 @@ function selectRelative(offset: number) {
|
||||
<div>
|
||||
<strong>结果 #{{ String(selectedIndex + 1).padStart(3, '0') }}</strong>
|
||||
<span v-if="selectedItem.status === 'modified'" class="modified-label">已修改</span>
|
||||
<el-tag v-if="selectedItem.split" size="small" effect="plain">{{ selectedItem.split }}</el-tag>
|
||||
<el-tag
|
||||
v-if="selectedItem.qualityScore != null"
|
||||
size="small"
|
||||
:type="selectedItem.qualityScore >= 80 ? 'success' : selectedItem.qualityScore >= 60 ? 'warning' : 'danger'"
|
||||
>质量 {{ selectedItem.qualityScore.toFixed(1) }}</el-tag>
|
||||
</div>
|
||||
<div class="result-header-actions">
|
||||
<el-button
|
||||
v-if="selectedItem.savedStatus === 'invalid'"
|
||||
size="small"
|
||||
plain
|
||||
type="primary"
|
||||
:loading="selectedItemRegenerating"
|
||||
:disabled="bulkRegenerationActive || (Boolean(regeneratingResultId) && !selectedItemRegenerating)"
|
||||
@click="emit('regenerate:item', selectedItem.id)"
|
||||
>
|
||||
<i v-if="!selectedItemRegenerating" class="fa fa-refresh" style="margin-right: 4px;" /> 重新生成
|
||||
</el-button>
|
||||
</div>
|
||||
<el-button link @click="emit('restore:item', selectedItem.id)"><i class="fa fa-undo" /> 恢复生成结果</el-button>
|
||||
</div>
|
||||
|
||||
<article class="source-reference" aria-labelledby="source-reference-title">
|
||||
<div class="source-reference-heading">
|
||||
<div>
|
||||
<strong id="source-reference-title">原文参照</strong>
|
||||
<span v-if="selectedSourceMeta">{{ selectedSourceMeta }}</span>
|
||||
</div>
|
||||
<el-tag v-if="selectedSourceWasPreprocessed" size="small" effect="plain">已智能预处理</el-tag>
|
||||
</div>
|
||||
<pre v-if="selectedSourceContent">{{ selectedSourceContent }}</pre>
|
||||
<p v-else>当前结果没有关联到可用的原文切片。</p>
|
||||
<small v-if="selectedSourceWasPreprocessed">这里展示的是实际送入模型的预处理后原文，便于核对问题和答案是否有依据。</small>
|
||||
</article>
|
||||
|
||||
<div class="field-editor">
|
||||
<label>Instruction <em>必填</em></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.instruction"
|
||||
:disabled="selectedItemRegenerating"
|
||||
type="textarea"
|
||||
:rows="3"
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'instruction', $event)"
|
||||
@@ -86,6 +201,7 @@ function selectRelative(offset: number) {
|
||||
<label>Input <span>选填</span></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.input"
|
||||
:disabled="selectedItemRegenerating"
|
||||
type="textarea"
|
||||
:rows="2"
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'input', $event)"
|
||||
@@ -95,6 +211,7 @@ function selectRelative(offset: number) {
|
||||
<label>Output <em>必填</em></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.output"
|
||||
:disabled="selectedItemRegenerating"
|
||||
type="textarea"
|
||||
:rows="7"
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'output', $event)"
|
||||
@@ -136,6 +253,25 @@ function selectRelative(offset: number) {
|
||||
border-right: 1px solid #e5e8ee;
|
||||
}
|
||||
|
||||
.result-list-title {
|
||||
min-width: 0;
|
||||
|
||||
span {
|
||||
margin-left: 8px;
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
font-weight: 400;
|
||||
}
|
||||
}
|
||||
|
||||
.result-list-header {
|
||||
gap: 10px;
|
||||
|
||||
:deep(.el-button) {
|
||||
flex: none;
|
||||
}
|
||||
}
|
||||
|
||||
.pane-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -169,12 +305,37 @@ function selectRelative(offset: number) {
|
||||
border-bottom: 1px solid #edf0f5;
|
||||
}
|
||||
|
||||
.bulk-regeneration-progress {
|
||||
padding: 9px 12px 10px;
|
||||
background: #fafaff;
|
||||
border-bottom: 1px solid #e8e7ff;
|
||||
|
||||
> div {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
margin-bottom: 7px;
|
||||
color: #667085;
|
||||
font-size: 10px;
|
||||
}
|
||||
}
|
||||
|
||||
.result-list {
|
||||
height: 476px;
|
||||
padding: 7px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.has-bulk-progress .result-list {
|
||||
height: 420px;
|
||||
}
|
||||
|
||||
.result-header-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.result-item {
|
||||
display: grid;
|
||||
grid-template-columns: 46px minmax(0, 1fr) 18px;
|
||||
@@ -236,10 +397,80 @@ function selectRelative(offset: number) {
|
||||
color: #d97706;
|
||||
}
|
||||
|
||||
.css-spinner {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid rgba(91, 80, 242, 0.2);
|
||||
border-top-color: #5b50f2;
|
||||
border-radius: 50%;
|
||||
animation: css-spin 0.8s linear infinite;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
@keyframes css-spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.result-editor-pane {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.source-reference {
|
||||
margin: 14px 18px 2px;
|
||||
padding: 12px 14px;
|
||||
border: 1px solid #dfe4ec;
|
||||
border-radius: 7px;
|
||||
background: #f8fafc;
|
||||
|
||||
pre {
|
||||
max-height: 168px;
|
||||
margin: 10px 0 0;
|
||||
overflow: auto;
|
||||
color: #344054;
|
||||
font-family: inherit;
|
||||
font-size: 12px;
|
||||
line-height: 1.7;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 10px 0 0;
|
||||
color: #98a2b3;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
> small {
|
||||
display: block;
|
||||
margin-top: 8px;
|
||||
color: #8a93a3;
|
||||
font-size: 10px;
|
||||
}
|
||||
}
|
||||
|
||||
.source-reference-heading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
|
||||
> div {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
span {
|
||||
color: #8a93a3;
|
||||
font-size: 10px;
|
||||
}
|
||||
}
|
||||
|
||||
.field-editor {
|
||||
padding: 13px 18px 0;
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ const props = defineProps<{
|
||||
externalSource: ExternalDataSource
|
||||
externalPulling: boolean
|
||||
externalConnected: boolean
|
||||
previewBuilding: boolean
|
||||
sourceUploading: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -21,26 +23,33 @@ const emit = defineEmits<{
|
||||
}>()
|
||||
|
||||
const DATA_SOURCE_TYPES = [
|
||||
{ value: 'mysql', label: 'MySQL' },
|
||||
{ value: 'postgresql', label: 'PostgreSQL' },
|
||||
{ value: 'mongodb', label: 'MongoDB' },
|
||||
{ value: 'api', label: 'REST API' },
|
||||
]
|
||||
|
||||
const AUTH_MODES = [
|
||||
{ value: 'none', label: '免鉴权' },
|
||||
{ value: 'basic', label: '账号密码' },
|
||||
{ value: 'token', label: 'Token' },
|
||||
]
|
||||
|
||||
const FILE_PAGE_SIZE = 10
|
||||
const currentFilePage = ref(1)
|
||||
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
|
||||
|
||||
const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
|
||||
queued: { label: '等待上传', icon: 'fa-clock-o' },
|
||||
uploading: { label: '正在上传', icon: 'fa-cloud-upload' },
|
||||
waiting: { label: '等待切分', icon: 'fa-clock-o' },
|
||||
processing: { label: '正在切分', icon: 'fa-spinner fa-spin' },
|
||||
success: { label: '切分完成', icon: 'fa-check-circle' },
|
||||
'upload-failed': { label: '上传失败', icon: 'fa-exclamation-circle' },
|
||||
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
|
||||
}
|
||||
|
||||
const isExternal = computed(() => props.processType === 'external')
|
||||
|
||||
const uploadAccept = computed(() => props.processType === 'unstructured'
|
||||
? '.txt,.md,.pdf,.docx,.doc,.json,.jsonl'
|
||||
: '.json,.jsonl,.csv,.xlsx,.xls')
|
||||
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
|
||||
: '.json,.jsonl,.ndjson,.csv,.tsv,.xlsx')
|
||||
|
||||
const pagedUploadedFiles = computed(() => {
|
||||
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
|
||||
@@ -69,6 +78,48 @@ function formatSize(size: number) {
|
||||
if (size >= 1024 * 1024) return `${(size / 1024 / 1024).toFixed(1)} MB`
|
||||
return `${(size / 1024).toFixed(1)} KB`
|
||||
}
|
||||
|
||||
function getFileStage(file: UploadedDataFile): FileStage {
|
||||
if (file.status === 'queued' || file.status === 'uploading') return file.status
|
||||
if (file.status === 'failed') return 'upload-failed'
|
||||
if (file.previewStatus === 'processing' || file.previewStatus === 'success') return file.previewStatus
|
||||
if (file.previewStatus === 'failed') return 'preview-failed'
|
||||
return 'waiting'
|
||||
}
|
||||
|
||||
function getFileProgress(file: UploadedDataFile) {
|
||||
const stage = getFileStage(file)
|
||||
const progress = stage === 'queued' || stage === 'uploading' || stage === 'upload-failed'
|
||||
? file.uploadProgress
|
||||
: stage === 'waiting' ? 100 : file.previewProgress ?? 0
|
||||
return Math.min(100, Math.max(0, progress))
|
||||
}
|
||||
|
||||
function isFileProcessing(file: UploadedDataFile) {
|
||||
return getFileStage(file) === 'processing'
|
||||
}
|
||||
|
||||
function getFileBarPercentage(file: UploadedDataFile) {
|
||||
// Element Plus 的不定进度动画需要非零宽度；这里不作为完成百分比展示。
|
||||
return isFileProcessing(file) ? 100 : getFileProgress(file)
|
||||
}
|
||||
|
||||
function getFileProgressStatus(file: UploadedDataFile): 'success' | 'exception' | undefined {
|
||||
const stage = getFileStage(file)
|
||||
if (stage === 'waiting' || stage === 'success') return 'success'
|
||||
if (stage === 'upload-failed' || stage === 'preview-failed') return 'exception'
|
||||
return undefined
|
||||
}
|
||||
|
||||
function getFileProgressText(file: UploadedDataFile) {
|
||||
if (isFileProcessing(file)) return '处理中'
|
||||
if (getFileStage(file) === 'waiting') return '已上传'
|
||||
return `${getFileProgress(file)}%`
|
||||
}
|
||||
|
||||
function getFileError(file: UploadedDataFile) {
|
||||
return file.uploadError || file.previewError
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -101,7 +152,7 @@ function formatSize(size: number) {
|
||||
<el-form-item label="地址 / URL">
|
||||
<el-input
|
||||
:model-value="externalSource.url"
|
||||
placeholder="例如：mysql://host:3306/db 或 https://api.example.com/data"
|
||||
placeholder="例如：postgresql://db.example.com:5432/my_database"
|
||||
aria-label="数据源地址或 URL"
|
||||
@update:model-value="updateExternalField('url', $event)"
|
||||
/>
|
||||
@@ -140,17 +191,6 @@ function formatSize(size: number) {
|
||||
@update:model-value="updateExternalField('password', $event)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="externalSource.authMode === 'token'" label="Token">
|
||||
<el-input
|
||||
:model-value="externalSource.token"
|
||||
type="password"
|
||||
show-password
|
||||
autocomplete="off"
|
||||
placeholder="请输入访问 Token"
|
||||
aria-label="数据源访问 Token"
|
||||
@update:model-value="updateExternalField('token', $event)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="拉取条数">
|
||||
<el-input-number
|
||||
:model-value="externalSource.limit"
|
||||
@@ -162,12 +202,25 @@ function formatSize(size: number) {
|
||||
@update:model-value="updateExternalField('limit', Number($event) || 0)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="只读查询语句" class="external-query-field">
|
||||
<el-input
|
||||
:model-value="externalSource.query"
|
||||
type="textarea"
|
||||
:rows="4"
|
||||
maxlength="20000"
|
||||
show-word-limit
|
||||
placeholder="例如：SELECT question, answer FROM qa_data ORDER BY id"
|
||||
aria-label="外部数据源只读查询语句"
|
||||
@update:model-value="updateExternalField('query', $event)"
|
||||
/>
|
||||
<small>只允许单条 SELECT 或 WITH 查询；后端会拒绝写入、DDL 和多语句。</small>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
|
||||
<div class="external-actions">
|
||||
<el-button
|
||||
:loading="externalPulling && !externalConnected"
|
||||
:disabled="externalPulling"
|
||||
:disabled="externalPulling || previewBuilding"
|
||||
plain
|
||||
@click="emit('test-connection')"
|
||||
>
|
||||
@@ -176,7 +229,7 @@ function formatSize(size: number) {
|
||||
<el-button
|
||||
type="primary"
|
||||
:loading="externalPulling"
|
||||
:disabled="externalPulling"
|
||||
:disabled="externalPulling || previewBuilding"
|
||||
@click="emit('pull-data')"
|
||||
>
|
||||
拉取数据
|
||||
@@ -200,10 +253,34 @@ function formatSize(size: number) {
|
||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||
</span>
|
||||
</div>
|
||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 拉取成功</span>
|
||||
<div
|
||||
class="file-preview-progress"
|
||||
:class="`is-${getFileStage(file)}`"
|
||||
:title="getFileError(file)"
|
||||
>
|
||||
<div class="file-status" role="status" aria-live="polite">
|
||||
<span>
|
||||
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||
</span>
|
||||
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="getFileBarPercentage(file)"
|
||||
:indeterminate="isFileProcessing(file)"
|
||||
:duration="1.5"
|
||||
:stroke-width="5"
|
||||
:show-text="false"
|
||||
:status="getFileProgressStatus(file)"
|
||||
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||
:aria-valuetext="isFileProcessing(file) ? '正在切分，进度未知' : getFileProgressText(file)"
|
||||
/>
|
||||
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||
</div>
|
||||
<el-button
|
||||
link
|
||||
type="danger"
|
||||
:disabled="previewBuilding || file.status === 'uploading'"
|
||||
:aria-label="`删除数据集 ${file.name}`"
|
||||
@click="emit('remove-file', file.uid)"
|
||||
>
|
||||
@@ -233,7 +310,13 @@ function formatSize(size: number) {
|
||||
<h3 id="source-upload-title">源数据上传</h3>
|
||||
<p>上传后可在下一步检查内容和切分效果，支持同时添加多个文件</p>
|
||||
</div>
|
||||
<el-button v-if="uploadedFiles.length === 0" link type="primary" @click="emit('use-sample')">
|
||||
<el-button
|
||||
v-if="uploadedFiles.length === 0"
|
||||
link
|
||||
type="primary"
|
||||
:disabled="previewBuilding"
|
||||
@click="emit('use-sample')"
|
||||
>
|
||||
使用示例数据
|
||||
</el-button>
|
||||
</div>
|
||||
@@ -243,6 +326,7 @@ function formatSize(size: number) {
|
||||
drag
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
@@ -253,25 +337,29 @@ function formatSize(size: number) {
|
||||
<template #tip>
|
||||
<div class="el-upload__tip">
|
||||
{{ processType === 'unstructured'
|
||||
? '支持 TXT、Markdown、PDF、Word、JSON、JSONL，单文件不超过 200MB'
|
||||
: '支持 JSON、JSONL、CSV、Excel，单文件不超过 200MB' }}
|
||||
? '支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON；旧版 DOC/PPT 请先转换，单文件不超过 200MB'
|
||||
: '支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX；旧版 XLS 请先转换，单文件不超过 200MB' }}
|
||||
</div>
|
||||
</template>
|
||||
</el-upload>
|
||||
|
||||
<section v-else class="uploaded-file-list" aria-label="已上传文件列表">
|
||||
<div class="uploaded-file-list-header">
|
||||
<span>已添加 {{ uploadedFiles.length }} 个文件</span>
|
||||
<span>
|
||||
已选择 {{ uploadedFiles.length }} 个文件
|
||||
<small v-if="sourceUploading" class="upload-queue-status"> · 正在逐个上传</small>
|
||||
</span>
|
||||
<div class="continue-upload">
|
||||
<el-upload
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
aria-label="继续添加源数据文件"
|
||||
>
|
||||
<el-button size="small" type="primary">继续上传</el-button>
|
||||
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
|
||||
</el-upload>
|
||||
</div>
|
||||
</div>
|
||||
@@ -285,10 +373,34 @@ function formatSize(size: number) {
|
||||
<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template>
|
||||
</span>
|
||||
</div>
|
||||
<span class="file-status"><i class="fa fa-check-circle" aria-hidden="true" /> 校验通过</span>
|
||||
<div
|
||||
class="file-preview-progress"
|
||||
:class="`is-${getFileStage(file)}`"
|
||||
:title="getFileError(file)"
|
||||
>
|
||||
<div class="file-status" role="status" aria-live="polite">
|
||||
<span>
|
||||
<i class="fa" :class="FILE_STAGE_META[getFileStage(file)].icon" aria-hidden="true" />
|
||||
{{ FILE_STAGE_META[getFileStage(file)].label }}
|
||||
</span>
|
||||
<span class="file-progress-value">{{ getFileProgressText(file) }}</span>
|
||||
</div>
|
||||
<el-progress
|
||||
:percentage="getFileBarPercentage(file)"
|
||||
:indeterminate="isFileProcessing(file)"
|
||||
:duration="1.5"
|
||||
:stroke-width="5"
|
||||
:show-text="false"
|
||||
:status="getFileProgressStatus(file)"
|
||||
:aria-valuenow="isFileProcessing(file) ? undefined : getFileProgress(file)"
|
||||
:aria-valuetext="isFileProcessing(file) ? '正在切分，进度未知' : getFileProgressText(file)"
|
||||
/>
|
||||
<small v-if="getFileError(file)" class="file-preview-error">{{ getFileError(file) }}</small>
|
||||
</div>
|
||||
<el-button
|
||||
link
|
||||
type="danger"
|
||||
:disabled="previewBuilding || file.status === 'uploading'"
|
||||
:aria-label="`删除文件 ${file.name}`"
|
||||
@click="emit('remove-file', file.uid)"
|
||||
>
|
||||
@@ -441,6 +553,11 @@ function formatSize(size: number) {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.upload-queue-status {
|
||||
color: #5b50f2;
|
||||
font-size: inherit;
|
||||
}
|
||||
|
||||
.uploaded-file-pagination {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
@@ -495,9 +612,59 @@ function formatSize(size: number) {
|
||||
}
|
||||
}
|
||||
|
||||
.file-preview-progress {
|
||||
display: flex;
|
||||
flex: 0 1 220px;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
min-width: 150px;
|
||||
|
||||
&.is-queued .file-status,
|
||||
&.is-waiting .file-status {
|
||||
color: #8a93a3;
|
||||
}
|
||||
|
||||
&.is-uploading .file-status,
|
||||
&.is-processing .file-status {
|
||||
color: #5b50f2;
|
||||
}
|
||||
|
||||
&.is-success .file-status {
|
||||
color: #2ca66a;
|
||||
}
|
||||
|
||||
&.is-upload-failed .file-status,
|
||||
&.is-preview-failed .file-status,
|
||||
.file-preview-error {
|
||||
color: #d94b4b;
|
||||
}
|
||||
}
|
||||
|
||||
.file-status {
|
||||
color: #2ca66a;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
font-size: 12px;
|
||||
|
||||
> span:first-child {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.file-progress-value {
|
||||
flex: 0 0 auto;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.file-preview-error {
|
||||
overflow: hidden;
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.uploaded-file :deep(.el-button) {
|
||||
@@ -514,8 +681,12 @@ function formatSize(size: number) {
|
||||
padding: 8px 10px;
|
||||
}
|
||||
|
||||
.file-status {
|
||||
.file-preview-progress {
|
||||
flex: 0 1 auto;
|
||||
min-width: 130px;
|
||||
}
|
||||
|
||||
.file-status {
|
||||
line-height: 1.4;
|
||||
white-space: normal;
|
||||
}
|
||||
@@ -545,7 +716,8 @@ function formatSize(size: number) {
|
||||
min-width: calc(100% - 40px);
|
||||
}
|
||||
|
||||
.file-status {
|
||||
.file-preview-progress {
|
||||
flex: 1 0 calc(100% - 38px);
|
||||
margin-left: 38px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,10 @@ import type {
|
||||
PreprocessOption,
|
||||
StructuredProcessOptions,
|
||||
} from './types'
|
||||
import {
|
||||
normalizeQaPairsGenerationCount,
|
||||
QA_PAIRS_GENERATION_LIMITS,
|
||||
} from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
@@ -22,12 +26,20 @@ const PREPROCESS_OPTIONS: Array<{
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
||||
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列，并剔除关键字段残缺的数据行' },
|
||||
{
|
||||
value: 'detect_structure',
|
||||
label: '嵌套结构展平',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段；Excel 表头与合并单元格在上传时自动解析',
|
||||
},
|
||||
{
|
||||
value: 'deduplicate',
|
||||
label: '重复记录去重',
|
||||
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重，暂不支持自定义组合字段',
|
||||
},
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值，并过滤乱码等异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||
]
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
@@ -41,11 +53,17 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:options', { ...props.options, ...value })
|
||||
}
|
||||
|
||||
function updateQaPairsPerRow(value: number | undefined) {
|
||||
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = value.filter(
|
||||
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
|
||||
)
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
(option): option is PreprocessOption => (
|
||||
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||
),
|
||||
)))
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
</script>
|
||||
@@ -95,15 +113,16 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每行生成数量</strong>
|
||||
<small>每行结构化数据生成的问答对数量</small>
|
||||
<small>支持 1～50 条；数量越大，处理耗时和 Token 消耗越高</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerRow"
|
||||
:min="1"
|
||||
:max="5"
|
||||
:min="QA_PAIRS_GENERATION_LIMITS.min"
|
||||
:max="QA_PAIRS_GENERATION_LIMITS.max"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
|
||||
@update:model-value="updateQaPairsPerRow"
|
||||
/>
|
||||
</div>
|
||||
<DatasetSplitEditor
|
||||
|
||||
@@ -16,6 +16,7 @@ const props = defineProps<{
|
||||
processType: ProcessType
|
||||
structuredOptions: StructuredProcessOptions
|
||||
unstructuredOptions: UnstructuredProcessOptions
|
||||
processTypeLocked?: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -75,9 +76,6 @@ const chunkValidationMessage = computed(() => {
|
||||
) {
|
||||
return '重叠长度与最小切片长度之和不能大于切片长度'
|
||||
}
|
||||
if (props.unstructuredOptions.chunkMethod === 'custom' && !props.unstructuredOptions.customDelimiter.trim()) {
|
||||
return '请输入自定义分隔符'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
@@ -156,7 +154,8 @@ defineExpose({ validate })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>处理类型</h3>
|
||||
<p>类型会影响下一步支持的数据源格式和后续预览方式</p>
|
||||
<p v-if="processTypeLocked">重新生成沿用原任务处理类型，不可修改</p>
|
||||
<p v-else>类型会影响下一步支持的数据源格式和后续预览方式</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-form-item prop="processType" class="type-form-item">
|
||||
@@ -165,6 +164,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'structured' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'structured')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-table" /></span>
|
||||
@@ -178,6 +178,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'unstructured' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'unstructured')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-file-text-o" /></span>
|
||||
@@ -191,6 +192,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'external' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'external')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
|
||||
@@ -296,12 +298,25 @@ defineExpose({ validate })
|
||||
border-color: #a8a3ff;
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.58;
|
||||
}
|
||||
|
||||
&:disabled:hover {
|
||||
border-color: #dfe3ea;
|
||||
}
|
||||
|
||||
&.is-active {
|
||||
background: #fafaff;
|
||||
border-color: #5b50f2;
|
||||
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
|
||||
}
|
||||
|
||||
&.is-active:disabled {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
strong,
|
||||
small {
|
||||
display: block;
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import { QA_PAIRS_GENERATION_LIMITS } from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
@@ -30,17 +31,17 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
|
||||
]
|
||||
|
||||
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||||
{ value: 'semantic', label: '自动语义切分' },
|
||||
{ value: 'heading', label: '按标题和段落' },
|
||||
{ value: 'fixed', label: '按固定长度' },
|
||||
{ value: 'custom', label: '自定义分隔符' },
|
||||
{ value: 'layout_hybrid', label: '版面结构混合切分' },
|
||||
{ value: 'semantic', label: '语义切分' },
|
||||
{ value: 'fixed', label: '固定 Token 切分' },
|
||||
]
|
||||
|
||||
const UNSTRUCTURED_NUMBER_LIMITS = {
|
||||
chunkSize: { min: 200, max: 2000 },
|
||||
chunkOverlap: { min: 0, max: 500 },
|
||||
minChunkSize: { min: 20, max: 500 },
|
||||
qaPairsPerChunk: { min: 1, max: 3 },
|
||||
semanticBreakpointPercentile: { min: 1, max: 99 },
|
||||
qaPairsPerChunk: QA_PAIRS_GENERATION_LIMITS,
|
||||
} as const
|
||||
|
||||
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
|
||||
@@ -70,9 +71,9 @@ function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
|
||||
function updateSmartPreprocess(value: string | number | boolean) {
|
||||
const enabled = Boolean(value)
|
||||
const remainingOptions = props.options.preprocessOptions.filter(
|
||||
const remainingOptions = Array.from(new Set(props.options.preprocessOptions.filter(
|
||||
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
||||
)
|
||||
)))
|
||||
updateField(
|
||||
'preprocessOptions',
|
||||
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
||||
@@ -80,9 +81,11 @@ function updateSmartPreprocess(value: string | number | boolean) {
|
||||
}
|
||||
|
||||
function updateDesensitize(value: string | number | boolean) {
|
||||
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
|
||||
(option) => option !== 'desensitize',
|
||||
)
|
||||
const preprocessOptions: UnstructuredPreprocessOption[] = Array.from(new Set(
|
||||
props.options.preprocessOptions.filter(
|
||||
(option) => option !== 'desensitize',
|
||||
),
|
||||
))
|
||||
if (Boolean(value)) preprocessOptions.push('desensitize')
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
@@ -116,7 +119,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认使用推荐策略，只需决定是否需要脱敏</p>
|
||||
<p>默认启用结构感知的推荐策略，只需决定是否需要脱敏</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="preprocess-option-grid">
|
||||
@@ -127,7 +130,7 @@ defineExpose({ revealValidation })
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>智能预处理</strong>
|
||||
<small>自动清理、解析、去重及保留上下文</small>
|
||||
<small>清理页眉页脚、页码、目录和无效内容，感知文档结构、合并短块、预过滤低质量内容、近重复去重，并通过重叠保护上下文</small>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
@@ -138,7 +141,7 @@ defineExpose({ revealValidation })
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>敏感信息脱敏</strong>
|
||||
<small>处理姓名、手机号等隐私信息</small>
|
||||
<small>识别并脱敏姓名、手机号、邮箱和身份证号</small>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
@@ -159,7 +162,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>切分选项</h3>
|
||||
<p>以语义完整为优先，将长文档拆成可独立生成问答的内容块</p>
|
||||
<p>优先保留文档版面结构，也可按主题变化或固定 Token 切分</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -178,7 +181,9 @@ defineExpose({ revealValidation })
|
||||
:value="method.value"
|
||||
/>
|
||||
</el-select>
|
||||
<small>推荐使用自动语义切分，在长度限制内优先保留完整句段</small>
|
||||
<small v-if="options.chunkMethod === 'layout_hybrid'">默认推荐，按标题、段落、列表和表格结构切分，并合并过短内容块</small>
|
||||
<small v-else-if="options.chunkMethod === 'semantic'">根据相邻内容的语义变化寻找主题边界，适合长文章和知识材料</small>
|
||||
<small v-else>按句子边界控制固定 Token 长度，速度快且结果稳定</small>
|
||||
</label>
|
||||
|
||||
<label class="config-field">
|
||||
@@ -234,17 +239,22 @@ defineExpose({ revealValidation })
|
||||
<small>过短的尾部内容会尽量并入前一个切片</small>
|
||||
</label>
|
||||
|
||||
<label v-if="options.chunkMethod === 'custom'" class="config-field">
|
||||
<span class="config-field-label">自定义分隔符</span>
|
||||
<el-input
|
||||
:model-value="options.customDelimiter"
|
||||
maxlength="40"
|
||||
show-word-limit
|
||||
placeholder="例如：--- 或 ###"
|
||||
aria-label="自定义分隔符"
|
||||
@update:model-value="updateField('customDelimiter', $event)"
|
||||
/>
|
||||
<small>系统会优先在分隔符位置结束当前切片</small>
|
||||
<label v-if="options.chunkMethod === 'semantic'" class="config-field">
|
||||
<span class="config-field-label">语义断点百分位</span>
|
||||
<span class="unit-input">
|
||||
<el-input-number
|
||||
:model-value="options.semanticBreakpointPercentile"
|
||||
:min="1"
|
||||
:max="99"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="语义断点百分位"
|
||||
@update:model-value="updateUnstructuredNumber('semanticBreakpointPercentile', $event)"
|
||||
/>
|
||||
<span>%</span>
|
||||
</span>
|
||||
<small>百分位越低切片越多，推荐保持 95%</small>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
@@ -276,12 +286,12 @@ defineExpose({ revealValidation })
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每个切片生成数量</strong>
|
||||
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
|
||||
<small>支持 1～50 条；数量越大，处理耗时和 Token 消耗越高</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerChunk"
|
||||
:min="1"
|
||||
:max="3"
|
||||
:min="QA_PAIRS_GENERATION_LIMITS.min"
|
||||
:max="QA_PAIRS_GENERATION_LIMITS.max"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
|
||||
@@ -117,6 +117,12 @@
|
||||
min-height: 400px;
|
||||
}
|
||||
|
||||
.initialization-error {
|
||||
display: grid;
|
||||
justify-items: start;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
display: grid;
|
||||
flex-shrink: 0;
|
||||
|
||||
@@ -1,6 +1,99 @@
|
||||
import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types'
|
||||
import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
ProcessType,
|
||||
} from './types'
|
||||
import { normalizeQaPairsGenerationCount } from './types'
|
||||
|
||||
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
|
||||
const LEGACY_DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合所选输出类型和目标格式的内容，答案应事实清晰、语言自然，不要添加无关说明。'
|
||||
|
||||
const PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT = '你是一名专业的数据生成助手。请严格依据输入内容生成准确、完整、可直接用于监督微调的问答数据。只生成问题和最终答案，不输出分析、推理过程或来源中不存在的信息；答案应事实清晰、语言自然。'
|
||||
|
||||
const PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT = '你是一名专业的推理数据生成助手。请严格依据输入内容生成问题、可核验的推理过程和最终答案。推理需要说明关键依据与必要步骤，不得引入来源中不存在的事实；最终答案应准确、完整且语言自然。'
|
||||
|
||||
export const PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT_2 = `你是一名专业的数据生成助手。请根据下方来源内容生成可用于监督微调的问答数据。
|
||||
来源内容：
|
||||
{{ content }}
|
||||
要求：严格依据来源生成问题和最终答案，不输出分析、中间思考过程或来源中不存在的信息；答案应准确、完整、语言自然。`
|
||||
|
||||
export const PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_2 = `你是一名专业的推理数据生成助手。请根据下方来源内容生成可用于推理模型训练的问答数据。
|
||||
来源内容：
|
||||
{{ content }}
|
||||
要求：每条数据必须包含问题、可核验的中间思考过程和最终答案。中间思考过程需说明关键依据、必要步骤和中间计算，不得跳过推理只给结论，也不得引入来源中不存在的事实。`
|
||||
|
||||
export const PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT_3 = `你是一名专业的数据生成专家。你的任务是根据下方的来源内容，提取核心信息并生成高质量的问答数据，用于大模型的监督微调（SFT）。
|
||||
|
||||
来源内容：
|
||||
{{ content }}
|
||||
|
||||
任务要求：
|
||||
1. 问题构建：基于来源内容设计问题，问题应当具有实际意义，不仅限于表面的事实问答，可以包括对内容的主旨概括、具体细节的询问或应用场景的探讨。
|
||||
2. 答案生成：严格依据来源内容生成最终答案，答案需准确无误、逻辑连贯、完整且表达自然流畅。
|
||||
3. 忠于原文：不输出分析、中间思考过程或任何多余的解释，切勿引入来源内容中未提及的事实或主观臆造的信息。`
|
||||
|
||||
export const PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_3 = `你是一名具有深厚专业知识的推理数据生成专家。你的任务是根据给定的来源内容，生成高质量、逻辑严密且可用于训练推理大语言模型（如 O1 或 DeepSeek-R1）的问答数据。
|
||||
|
||||
来源内容：
|
||||
{{ content }}
|
||||
|
||||
任务要求：
|
||||
1. 多维度提问：从来源内容中提取核心概念或关键逻辑，构建具有挑战性的问题，避免简单的信息抽取，提问可以涉及因果分析、方案设计、逻辑推断或跨段落信息整合。
|
||||
2. 深度思维链（CoT）：
|
||||
- 必须提供详尽的中间思考过程。
|
||||
- 在思考过程中，需首先分析问题的核心需求，然后逐步拆解来源内容中的相关事实或数据。
|
||||
- 对于涉及计算或多步推导的问题，必须一步步展示计算或推导过程，并说明每一步的理论依据或事实来源。
|
||||
- 如果遇到矛盾或缺失的信息，需在思考过程中指出，并合理假设或推理，但最终答案不得编造来源中完全不相关的事实。
|
||||
3. 最终答案：在充分推理后给出明确的最终结论。结论应当是对问题的直接回应，语言自然、结构清晰，并且结论必须能从前面的思考过程中自然得出。
|
||||
|
||||
数据格式要求：
|
||||
请确保每条生成的数据清晰包含“问题”、“中间思考过程”和“最终答案”三个部分，且内容充实、专业。`
|
||||
|
||||
export const DEFAULT_STANDARD_GENERATION_PROMPT = `你是一名专业的数据生成专家。请基于下方来源内容，生成高质量、可直接用于大模型监督微调（SFT）的问答数据。
|
||||
|
||||
来源内容：
|
||||
{{ content }}
|
||||
|
||||
任务要求：
|
||||
1. 提问有价值：问题应聚焦来源内容的核心信息点，避免无意义的字面抄录或过于宽泛的提问；优先设计需要理解、归纳或推理才能回答的问题，例如主旨概括、细节追问、因果解释或场景应用。
|
||||
2. 答案忠于来源：严格依据来源内容作答，不得引入来源中不存在的事实、观点或数据；若来源信息不足以完整回答该问题，请如实说明不足，不要编造或臆测。
|
||||
3. 答案完整且自然：回答应直接回应问题，要点完整、逻辑清晰、语言流畅；涉及多项内容时建议分点陈述，但不必照搬原文措辞。`
|
||||
|
||||
export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名具有深厚专业知识的推理数据生成专家。请基于下方来源内容，生成包含完整思维链、可用于训练推理大语言模型（如 O1、DeepSeek-R1）的问答数据。
|
||||
|
||||
来源内容：
|
||||
{{ content }}
|
||||
|
||||
任务要求：
|
||||
1. 提问有挑战性：从来源内容中提炼需要多步思考才能回答的问题，避免可直接定位答案的简单事实抽取；可涉及因果分析、逻辑推断或跨信息整合。
|
||||
2. 推理可核验：逐步展开思考过程，每个关键结论都应标注其在来源中的依据或必要的中间推导；不得跳过关键步骤只给结论，也不得引入来源中完全不存在的事实。
|
||||
3. 信息不足时如实处理：遇到来源信息缺失或相互矛盾时，需在推理中明确指出，并基于来源做出合理假设，但不得编造与来源无关的内容。
|
||||
4. 最终答案须能从前面的推理过程中自然得出，语言简洁、结构清晰。`
|
||||
|
||||
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
|
||||
return outputType === 'reasoning'
|
||||
? DEFAULT_REASONING_GENERATION_PROMPT
|
||||
: DEFAULT_STANDARD_GENERATION_PROMPT
|
||||
}
|
||||
|
||||
export function isBuiltInGenerationPrompt(value: string) {
|
||||
const normalize = (s: string) => s.replace(/\r\n/g, '\n').trim()
|
||||
const normalizedValue = normalize(value)
|
||||
return [
|
||||
LEGACY_DEFAULT_GENERATION_PROMPT,
|
||||
PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||
PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT_2,
|
||||
PREVIOUS_DEFAULT_STANDARD_GENERATION_PROMPT_3,
|
||||
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT,
|
||||
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_2,
|
||||
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_3,
|
||||
DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||
DEFAULT_REASONING_GENERATION_PROMPT,
|
||||
].map(normalize).includes(normalizedValue)
|
||||
}
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
@@ -9,7 +102,9 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||
outputType: 'standard',
|
||||
reasoningDetail: 'normal',
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
@@ -30,11 +125,11 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
chunkMethod: 'semantic',
|
||||
chunkMethod: 'layout_hybrid',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
customDelimiter: '',
|
||||
semanticBreakpointPercentile: 95,
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
@@ -42,7 +137,9 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
generationPrompt: DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||
outputType: 'standard',
|
||||
reasoningDetail: 'normal',
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
@@ -52,3 +149,186 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
minOutputLength: 20,
|
||||
}
|
||||
}
|
||||
|
||||
function configValue<T>(config: DataProcessConfig, key: string, fallback: T): T {
|
||||
return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback
|
||||
}
|
||||
|
||||
function numberValue(config: DataProcessConfig, key: string, fallback: number): number {
|
||||
const value = Number(configValue(config, key, fallback))
|
||||
return Number.isFinite(value) ? value : fallback
|
||||
}
|
||||
|
||||
function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) {
|
||||
const value = config.dataset_split
|
||||
if (!value || typeof value !== 'object') return { ...fallback }
|
||||
const split = value as unknown as Record<string, unknown>
|
||||
const splitNumber = (key: keyof DataProcessDatasetSplit) => {
|
||||
const parsed = Number(split[key])
|
||||
return Number.isFinite(parsed) ? parsed : fallback[key]
|
||||
}
|
||||
return {
|
||||
train: splitNumber('train'),
|
||||
validation: splitNumber('validation'),
|
||||
test: splitNumber('test'),
|
||||
}
|
||||
}
|
||||
|
||||
function generationOptionsFromConfig(
|
||||
config: DataProcessConfig,
|
||||
defaults: GenerationControlOptions,
|
||||
): GenerationControlOptions {
|
||||
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
|
||||
? 'reasoning'
|
||||
: 'standard'
|
||||
const configuredPrompt = String(
|
||||
configValue(config, 'generation_prompt', defaults.generationPrompt),
|
||||
)
|
||||
return {
|
||||
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
|
||||
generationPrompt: isBuiltInGenerationPrompt(configuredPrompt)
|
||||
? defaultGenerationPrompt(outputType)
|
||||
: configuredPrompt,
|
||||
outputType,
|
||||
reasoningDetail: configValue(config, 'reasoning_detail', defaults.reasoningDetail) === 'detailed'
|
||||
? 'detailed'
|
||||
: 'normal',
|
||||
temperature: numberValue(config, 'temperature', defaults.temperature),
|
||||
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
|
||||
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
|
||||
qualityFilterEnabled: Boolean(configValue(
|
||||
config,
|
||||
'quality_filter_enabled',
|
||||
defaults.qualityFilterEnabled,
|
||||
)),
|
||||
filterLowQuality: Boolean(configValue(
|
||||
config,
|
||||
'filter_low_quality',
|
||||
defaults.filterLowQuality,
|
||||
)),
|
||||
filterShortContent: Boolean(configValue(
|
||||
config,
|
||||
'filter_short_content',
|
||||
defaults.filterShortContent,
|
||||
)),
|
||||
minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength),
|
||||
}
|
||||
}
|
||||
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerRow: normalizeQaPairsGenerationCount(
|
||||
numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
|
||||
defaults.qaPairsPerRow,
|
||||
),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
|
||||
export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions {
|
||||
const defaults = createDefaultUnstructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as UnstructuredPreprocessOption[]
|
||||
: defaults.preprocessOptions,
|
||||
chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod),
|
||||
chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize),
|
||||
chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap),
|
||||
minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize),
|
||||
semanticBreakpointPercentile: numberValue(
|
||||
config,
|
||||
'semantic_breakpoint_percentile',
|
||||
defaults.semanticBreakpointPercentile,
|
||||
),
|
||||
preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)),
|
||||
preserveCodeBlocks: Boolean(configValue(
|
||||
config,
|
||||
'preserve_code_blocks',
|
||||
defaults.preserveCodeBlocks,
|
||||
)),
|
||||
preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)),
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerChunk: normalizeQaPairsGenerationCount(
|
||||
numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
|
||||
defaults.qaPairsPerChunk,
|
||||
),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
|
||||
export function previewAffectingOptionsFor(
|
||||
processType: ProcessType,
|
||||
structured: StructuredProcessOptions,
|
||||
unstructured: UnstructuredProcessOptions,
|
||||
) {
|
||||
if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions }
|
||||
if (processType !== 'unstructured') return null
|
||||
const {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
} = unstructured
|
||||
return {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
}
|
||||
}
|
||||
|
||||
export function generationAffectingOptionsFor(
|
||||
processType: ProcessType,
|
||||
structured: StructuredProcessOptions,
|
||||
unstructured: UnstructuredProcessOptions,
|
||||
) {
|
||||
const options = processType === 'unstructured' ? unstructured : structured
|
||||
if (processType === 'external') return null
|
||||
const common = {
|
||||
semanticEnrichment: options.semanticEnrichment,
|
||||
datasetSplit: options.datasetSplit,
|
||||
generationModelId: options.generationModelId,
|
||||
generationPrompt: options.generationPrompt,
|
||||
outputType: options.outputType,
|
||||
reasoningDetail: options.reasoningDetail,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.maxTokens,
|
||||
jsonMode: options.jsonMode,
|
||||
qualityFilterEnabled: options.qualityFilterEnabled,
|
||||
filterLowQuality: options.filterLowQuality,
|
||||
filterShortContent: options.filterShortContent,
|
||||
minOutputLength: options.minOutputLength,
|
||||
}
|
||||
return processType === 'unstructured'
|
||||
? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk }
|
||||
: { ...common, qaPairsPerRow: structured.qaPairsPerRow }
|
||||
}
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
SourceLine,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import type { SourceLine } from './types'
|
||||
|
||||
/** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
'问：如何看待当前的通货膨胀风险？',
|
||||
'答：当前通胀水平总体可控，但仍需关注能源价格与供给扰动。',
|
||||
@@ -14,26 +8,13 @@ export const DEFAULT_SOURCE_TEXT = [
|
||||
'答：会议时间以美联储官方日历为准，市场会重点关注利率路径指引。',
|
||||
'问：人民币汇率未来走势如何？',
|
||||
'答：人民币汇率取决于中美利差、经济基本面与政策预期。',
|
||||
'问：银行理财产品收益率为何持续走低？',
|
||||
'答：主要与市场利率下行、资产端收益下降以及风险偏好变化有关。',
|
||||
'问：什么是复利？',
|
||||
'答：复利是指在计算利息时，将上一期利息加入本金，再计算下一期利息。',
|
||||
'问：如何评估股票的投资价值？',
|
||||
'答：评估股票投资价值可以从以下几个方面进行：',
|
||||
'1. 公司基本面：分析公司的财务状况、盈利能力、成长性等。',
|
||||
'2. 行业前景：考察公司所处行业的发展趋势和竞争格局。',
|
||||
'3. 估值水平：通过市盈率、市净率等指标判断估值是否合理。',
|
||||
'4. 财务健康：关注公司的负债情况、现金流状况等。',
|
||||
'5. 管理团队：评估管理层的能力和过往业绩。',
|
||||
'此外，还需要关注宏观经济环境、政策变化等因素对股票市场的影响。',
|
||||
'问：债券和股票的主要区别是什么？',
|
||||
'答：债券收益相对稳定但上行有限，股票波动更大且承担更高风险。',
|
||||
'问：什么是市盈率？',
|
||||
'答：市盈率是股票价格与每股收益的比值，常用于衡量估值水平。',
|
||||
'问：如何进行资产配置？',
|
||||
'答：应根据投资目标、风险承受能力和市场环境合理分配资产。',
|
||||
'答：复利是将上一期利息加入本金，再计算下一期利息。',
|
||||
].join('\n')
|
||||
|
||||
/**
|
||||
* 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。
|
||||
*/
|
||||
export function sourceLines(sourceText: string): SourceLine[] {
|
||||
const rawLines = sourceText.split('\n')
|
||||
let cursor = 0
|
||||
@@ -46,495 +27,7 @@ export function sourceLines(sourceText: string): SourceLine[] {
|
||||
})
|
||||
}
|
||||
|
||||
interface SourceRange {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
interface ProtectedRange extends SourceRange {
|
||||
kind: 'code' | 'table' | 'list'
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 800
|
||||
const DEFAULT_CHUNK_OVERLAP = 100
|
||||
const DEFAULT_MIN_CHUNK_SIZE = 100
|
||||
|
||||
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
|
||||
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
|
||||
}
|
||||
|
||||
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
|
||||
let nextStart = Math.max(0, start)
|
||||
let nextEnd = Math.min(sourceText.length, end)
|
||||
|
||||
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
|
||||
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
|
||||
|
||||
return { start: nextStart, end: nextEnd }
|
||||
}
|
||||
|
||||
function normalizeDelimiter(delimiter: string | undefined): string {
|
||||
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
|
||||
}
|
||||
|
||||
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
|
||||
return line.start < range.end && line.end > range.start
|
||||
}
|
||||
|
||||
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
|
||||
return ranges.some((range) => overlapsRange(line, range))
|
||||
}
|
||||
|
||||
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
let openFence: { start: number; marker: string; length: number } | null = null
|
||||
|
||||
for (const line of lines) {
|
||||
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
|
||||
if (!fence) continue
|
||||
|
||||
const marker = fence[1][0]
|
||||
if (!openFence) {
|
||||
openFence = { start: line.start, marker, length: fence[1].length }
|
||||
continue
|
||||
}
|
||||
|
||||
if (marker === openFence.marker && fence[1].length >= openFence.length) {
|
||||
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
|
||||
openFence = null
|
||||
}
|
||||
}
|
||||
|
||||
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isTableSeparator(content: string): boolean {
|
||||
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
|
||||
const cells = normalized.split('|').map((cell) => cell.trim())
|
||||
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||||
}
|
||||
|
||||
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length - 1; index += 1) {
|
||||
const header = lines[index]
|
||||
const separator = lines[index + 1]
|
||||
if (
|
||||
isLineProtected(header, codeRanges)
|
||||
|| isLineProtected(separator, codeRanges)
|
||||
|| !header.content.includes('|')
|
||||
|| !isTableSeparator(separator.content)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
let endIndex = index + 1
|
||||
while (
|
||||
endIndex + 1 < lines.length
|
||||
&& !isLineProtected(lines[endIndex + 1], codeRanges)
|
||||
&& lines[endIndex + 1].content.trim()
|
||||
&& lines[endIndex + 1].content.includes('|')
|
||||
) {
|
||||
endIndex += 1
|
||||
}
|
||||
|
||||
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
|
||||
index = endIndex
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isListItem(content: string): boolean {
|
||||
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
|
||||
}
|
||||
|
||||
function isListContinuation(content: string): boolean {
|
||||
return /^\s{2,}\S/.test(content)
|
||||
}
|
||||
|
||||
function detectListRanges(
|
||||
lines: SourceLine[],
|
||||
excludedRanges: SourceRange[],
|
||||
): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
|
||||
|
||||
let endIndex = index
|
||||
let itemCount = 1
|
||||
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
|
||||
const nextContent = lines[endIndex + 1].content
|
||||
if (isListItem(nextContent)) {
|
||||
itemCount += 1
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
if (isListContinuation(nextContent)) {
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (itemCount >= 2) {
|
||||
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
|
||||
index = endIndex
|
||||
}
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
|
||||
return ranges
|
||||
.sort((left, right) => left.start - right.start || left.end - right.end)
|
||||
.reduce<ProtectedRange[]>((merged, range) => {
|
||||
const previous = merged[merged.length - 1]
|
||||
if (previous && range.start < previous.end) {
|
||||
previous.end = Math.max(previous.end, range.end)
|
||||
return merged
|
||||
}
|
||||
merged.push({ ...range })
|
||||
return merged
|
||||
}, [])
|
||||
}
|
||||
|
||||
function protectedRangesForOptions(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): ProtectedRange[] {
|
||||
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
|
||||
|
||||
const lines = sourceLines(sourceText)
|
||||
const codeRanges = detectCodeBlockRanges(sourceText, lines)
|
||||
const tableRanges = detectTableRanges(lines, codeRanges)
|
||||
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
|
||||
const enabledRanges = [
|
||||
...(options?.preserveCodeBlocks ? codeRanges : []),
|
||||
...(options?.preserveTables ? tableRanges : []),
|
||||
...(options?.preserveLists ? listRanges : []),
|
||||
]
|
||||
|
||||
return mergeProtectedRanges(enabledRanges)
|
||||
}
|
||||
|
||||
function protectedRangeContaining(
|
||||
ranges: ProtectedRange[],
|
||||
offset: number,
|
||||
): ProtectedRange | undefined {
|
||||
return ranges.find((range) => range.start < offset && offset < range.end)
|
||||
}
|
||||
|
||||
function normalizeChunkStart(
|
||||
sourceText: string,
|
||||
cursor: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
let start = Math.max(0, Math.min(cursor, sourceText.length))
|
||||
const overlapBlock = protectedRangeContaining(protectedRanges, start)
|
||||
if (overlapBlock) start = overlapBlock.end
|
||||
|
||||
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
|
||||
|
||||
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
|
||||
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
|
||||
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
function protectChunkEnd(
|
||||
proposedEnd: number,
|
||||
start: number,
|
||||
minimumEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
|
||||
if (!splitBlock) return proposedEnd
|
||||
|
||||
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
|
||||
return splitBlock.start > start && splitBlock.start >= minimumEnd
|
||||
? splitBlock.start
|
||||
: splitBlock.end
|
||||
}
|
||||
|
||||
function restoreProtectedEdges(
|
||||
range: SourceRange,
|
||||
rawStart: number,
|
||||
rawEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): SourceRange {
|
||||
const nextRange = { ...range }
|
||||
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
|
||||
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
|
||||
|
||||
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
|
||||
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
|
||||
return nextRange
|
||||
}
|
||||
|
||||
function lastBoundaryInRange(
|
||||
sourceText: string,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const candidates: number[] = []
|
||||
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
|
||||
|
||||
boundaryTokens.forEach((token) => {
|
||||
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
|
||||
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
|
||||
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
|
||||
})
|
||||
|
||||
return candidates.length ? Math.max(...candidates) : null
|
||||
}
|
||||
|
||||
function lastHeadingBoundary(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const section = sourceText.slice(start, idealEnd)
|
||||
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
|
||||
let boundary: number | null = null
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = headingPattern.exec(section))) {
|
||||
const absoluteStart = start + match.index
|
||||
if (absoluteStart >= minimumEnd) boundary = absoluteStart
|
||||
}
|
||||
|
||||
return boundary
|
||||
}
|
||||
|
||||
function resolveChunkEnd(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
options: UnstructuredProcessOptions | undefined,
|
||||
): number {
|
||||
const method = options?.chunkMethod ?? 'semantic'
|
||||
|
||||
if (method === 'fixed') return idealEnd
|
||||
|
||||
if (method === 'custom') {
|
||||
const delimiter = normalizeDelimiter(options?.customDelimiter)
|
||||
if (!delimiter) return idealEnd
|
||||
|
||||
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
|
||||
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
|
||||
return boundary >= minimumEnd ? boundary : idealEnd
|
||||
}
|
||||
|
||||
if (method === 'heading') {
|
||||
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
|
||||
if (headingBoundary !== null) return headingBoundary
|
||||
}
|
||||
|
||||
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
|
||||
}
|
||||
|
||||
function buildUnstructuredRanges(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): SourceRange[] {
|
||||
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
|
||||
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
|
||||
const minimumCharacters = Math.min(
|
||||
targetCharacters,
|
||||
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
|
||||
)
|
||||
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
|
||||
const protectedRanges = protectedRangesForOptions(sourceText, options)
|
||||
const ranges: SourceRange[] = []
|
||||
let cursor = 0
|
||||
|
||||
while (cursor < sourceText.length) {
|
||||
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
|
||||
if (start >= sourceText.length) break
|
||||
|
||||
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
|
||||
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
|
||||
let end = idealEnd === sourceText.length
|
||||
? idealEnd
|
||||
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
|
||||
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
|
||||
|
||||
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
|
||||
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
|
||||
|
||||
let range = trimSourceRange(sourceText, start, end)
|
||||
range = restoreProtectedEdges(range, start, end, protectedRanges)
|
||||
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
|
||||
range.end = Math.min(end, range.start + minimumCharacters)
|
||||
}
|
||||
if (range.end <= range.start) {
|
||||
cursor = Math.max(cursor + 1, end)
|
||||
continue
|
||||
}
|
||||
|
||||
const isLastRange = end >= sourceText.length
|
||||
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
|
||||
ranges[ranges.length - 1].end = range.end
|
||||
break
|
||||
}
|
||||
|
||||
ranges.push(range)
|
||||
if (isLastRange) break
|
||||
|
||||
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
|
||||
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
|
||||
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
|
||||
const nextCursor = range.end - actualOverlap
|
||||
cursor = nextCursor > start ? nextCursor : range.end
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
|
||||
if (!lines.length) return null
|
||||
|
||||
let low = 0
|
||||
let high = lines.length - 1
|
||||
let result = 0
|
||||
|
||||
while (low <= high) {
|
||||
const middle = Math.floor((low + high) / 2)
|
||||
if (lines[middle].start <= offset) {
|
||||
result = middle
|
||||
low = middle + 1
|
||||
} else {
|
||||
high = middle - 1
|
||||
}
|
||||
}
|
||||
|
||||
return lines[result].number
|
||||
}
|
||||
|
||||
function previewItemFromRange(
|
||||
sourceText: string,
|
||||
lines: SourceLine[],
|
||||
range: SourceRange,
|
||||
sourceFileId: string,
|
||||
index: number,
|
||||
): PreviewItem {
|
||||
const content = sourceText.slice(range.start, range.end)
|
||||
|
||||
return {
|
||||
id: `preview-${sourceFileId}-${index + 1}`,
|
||||
sourceFileId,
|
||||
originalContent: content,
|
||||
editedContent: content,
|
||||
sourceStart: range.start,
|
||||
sourceEnd: range.end,
|
||||
sourceStartLine: lineNumberAtOffset(lines, range.start),
|
||||
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
|
||||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||
status: 'original',
|
||||
}
|
||||
}
|
||||
|
||||
export function buildPreviewItems(
|
||||
sourceText: string,
|
||||
processType: ProcessType,
|
||||
sourceFileId = 'default-source',
|
||||
unstructuredOptions?: UnstructuredProcessOptions,
|
||||
): PreviewItem[] {
|
||||
const lines = sourceLines(sourceText)
|
||||
|
||||
if (processType === 'unstructured') {
|
||||
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
|
||||
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
|
||||
))
|
||||
}
|
||||
|
||||
const meaningfulLines = lines.filter((line) => line.content.trim())
|
||||
const groupSize = processType === 'structured' ? 1 : 3
|
||||
const items: PreviewItem[] = []
|
||||
|
||||
for (let index = 0; index < meaningfulLines.length; index += groupSize) {
|
||||
const group = meaningfulLines.slice(index, index + groupSize)
|
||||
if (!group.length) continue
|
||||
|
||||
const sourceStart = group[0].start
|
||||
const sourceEnd = group[group.length - 1].end
|
||||
const content = sourceText.slice(sourceStart, sourceEnd)
|
||||
|
||||
items.push({
|
||||
id: `preview-${sourceFileId}-${items.length + 1}`,
|
||||
sourceFileId,
|
||||
originalContent: content,
|
||||
editedContent: content,
|
||||
sourceStart,
|
||||
sourceEnd,
|
||||
sourceStartLine: group[0].number,
|
||||
sourceEndLine: group[group.length - 1].number,
|
||||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||
status: 'original',
|
||||
})
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
const SEMANTIC_PREFIXES = [
|
||||
'请结合实际情况，说明一下：',
|
||||
'如果方便的话，请详细解答：',
|
||||
'请用通俗易懂的方式说明：',
|
||||
'请从实际应用角度说明：',
|
||||
'请简洁、自然地说明：',
|
||||
]
|
||||
|
||||
export function createResults(
|
||||
items: PreviewItem[],
|
||||
options?: StructuredProcessOptions | UnstructuredProcessOptions,
|
||||
): ResultItem[] {
|
||||
const resultCount = options && 'qaPairsPerChunk' in options
|
||||
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||||
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||||
|
||||
const generatedResults = items.flatMap((item, index) => {
|
||||
if (options?.qualityFilterEnabled && options.filterLowQuality) {
|
||||
if (item.status === 'invalid' || !item.editedContent.trim()) return []
|
||||
}
|
||||
|
||||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||||
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||
|
||||
return Array.from({ length: resultCount }, (_, variantIndex) => {
|
||||
const instruction = options?.semanticEnrichment
|
||||
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
|
||||
: variantIndex === 0
|
||||
? baseInstruction
|
||||
: `${baseInstruction}（问法 ${variantIndex + 1}）`
|
||||
|
||||
return {
|
||||
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
|
||||
instruction,
|
||||
input: '',
|
||||
output,
|
||||
originalInstruction: instruction,
|
||||
originalInput: '',
|
||||
originalOutput: output,
|
||||
status: 'valid' as const,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
if (!options?.qualityFilterEnabled) return generatedResults
|
||||
|
||||
return generatedResults.filter((result) => {
|
||||
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
|
||||
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
|
||||
return true
|
||||
})
|
||||
/** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */
|
||||
export function estimateTokenCount(text: string): number {
|
||||
return text.match(/[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)?.length ?? 0
|
||||
}
|
||||
|
||||
@@ -1,7 +1,24 @@
|
||||
import type {
|
||||
DataProcessOutputType,
|
||||
DataProcessPreviewFileStatus,
|
||||
DataProcessReasoningDetail,
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||
|
||||
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
|
||||
export const QA_PAIRS_GENERATION_LIMITS = { min: 1, max: 50 } as const
|
||||
|
||||
export function normalizeQaPairsGenerationCount(value: unknown, fallback = 1): number {
|
||||
const parsed = Number(value)
|
||||
const normalized = Number.isFinite(parsed) ? Math.trunc(parsed) : fallback
|
||||
return Math.min(
|
||||
QA_PAIRS_GENERATION_LIMITS.max,
|
||||
Math.max(QA_PAIRS_GENERATION_LIMITS.min, normalized),
|
||||
)
|
||||
}
|
||||
|
||||
export type PreprocessOption =
|
||||
| 'clean_invalid'
|
||||
| 'detect_structure'
|
||||
@@ -19,6 +36,8 @@ export interface DatasetSplitOptions {
|
||||
export interface GenerationControlOptions {
|
||||
generationModelId: string | number | ''
|
||||
generationPrompt: string
|
||||
outputType: DataProcessOutputType
|
||||
reasoningDetail: DataProcessReasoningDetail
|
||||
temperature: number
|
||||
maxTokens: number
|
||||
jsonMode: boolean
|
||||
@@ -44,7 +63,7 @@ export type UnstructuredPreprocessOption =
|
||||
| 'desensitize'
|
||||
| 'preserve_context'
|
||||
|
||||
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
|
||||
export type ChunkMethod = 'layout_hybrid' | 'semantic' | 'fixed'
|
||||
|
||||
export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
preprocessOptions: UnstructuredPreprocessOption[]
|
||||
@@ -52,7 +71,7 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
chunkSize: number
|
||||
chunkOverlap: number
|
||||
minChunkSize: number
|
||||
customDelimiter: string
|
||||
semanticBreakpointPercentile: number
|
||||
preserveTables: boolean
|
||||
preserveCodeBlocks: boolean
|
||||
preserveLists: boolean
|
||||
@@ -62,21 +81,34 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
}
|
||||
|
||||
export interface ExternalDataSource {
|
||||
type: string
|
||||
type: 'postgresql'
|
||||
url: string
|
||||
authMode: string
|
||||
authMode: 'none' | 'basic'
|
||||
username?: string
|
||||
password?: string
|
||||
token?: string
|
||||
limit: number
|
||||
query?: string
|
||||
fileName?: string
|
||||
}
|
||||
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
rawFile?: File
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
content: string
|
||||
fileFormat?: string
|
||||
checksumSha256?: string
|
||||
status: 'queued' | 'uploading' | 'ready' | 'failed'
|
||||
uploadProgress: number
|
||||
uploadError?: string
|
||||
previewStatus?: DataProcessPreviewFileStatus
|
||||
previewProgress?: number
|
||||
previewError?: string
|
||||
previewCount?: number
|
||||
previewConfigSignature?: string
|
||||
}
|
||||
|
||||
export interface SourceLine {
|
||||
@@ -91,12 +123,18 @@ export interface PreviewItem {
|
||||
sourceFileId: string
|
||||
originalContent: string
|
||||
editedContent: string
|
||||
savedEditedContent: string
|
||||
sourceStart: number | null
|
||||
sourceEnd: number | null
|
||||
sourceStartLine: number | null
|
||||
sourceEndLine: number | null
|
||||
sourcePages?: number[]
|
||||
tokenCount: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
qualityScore?: number
|
||||
qualityDetails?: Record<string, number>
|
||||
piiStats?: Record<string, number>
|
||||
updatedAt?: string
|
||||
}
|
||||
|
||||
export interface GenerationState {
|
||||
@@ -107,12 +145,31 @@ export interface GenerationState {
|
||||
|
||||
export interface ResultItem {
|
||||
id: string
|
||||
previewItemId: string | null
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
originalInstruction: string
|
||||
originalInput: string
|
||||
originalOutput: string
|
||||
savedInstruction: string
|
||||
savedInput: string
|
||||
savedOutput: string
|
||||
savedStatus: 'valid' | 'modified' | 'invalid'
|
||||
status: 'valid' | 'modified' | 'invalid'
|
||||
error?: string
|
||||
split?: 'train' | 'validation' | 'test'
|
||||
qualityScore?: number
|
||||
qualityDetails?: Record<string, number>
|
||||
updatedAt?: string
|
||||
}
|
||||
|
||||
export interface BulkResultRegenerationState {
|
||||
status: 'idle' | 'running' | 'completed' | 'partial' | 'failed'
|
||||
total: number
|
||||
completed: number
|
||||
succeeded: number
|
||||
failed: number
|
||||
targetIds: string[]
|
||||
failedIds: string[]
|
||||
}
|
||||
|
||||
@@ -1,35 +1,81 @@
|
||||
import { reactive, ref, type Ref } from 'vue'
|
||||
import { computed, reactive, ref, type Ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { createResults } from './previewModel'
|
||||
import type {
|
||||
GenerationState,
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import {
|
||||
generateDataProcess,
|
||||
getDataProcessProgress,
|
||||
getDataProcessResults,
|
||||
regenerateDataProcessResult,
|
||||
regenerateDataProcessResults,
|
||||
restoreDataProcessResult,
|
||||
updateDataProcessResult,
|
||||
type DataProcessProgress,
|
||||
type DataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { BulkResultRegenerationState, GenerationState, ResultItem } from './types'
|
||||
|
||||
interface GenerationBindings {
|
||||
previewItems: Ref<PreviewItem[]>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
taskId: Ref<string | null>
|
||||
dirty: Ref<boolean>
|
||||
beforeGenerate?: () => Promise<boolean | void>
|
||||
}
|
||||
|
||||
const RESULT_PAGE_SIZE = 500
|
||||
const POLL_INTERVAL_MS = 1500
|
||||
// 4 个后端 worker 可连续消费三轮，减少每 4 条等待最慢项造成的空闲；
|
||||
// 单条重生成最长 60 秒，因此 12 条仍处于批量接口 240 秒超时预算内。
|
||||
const BULK_REGENERATION_CHUNK_SIZE = 12
|
||||
|
||||
function mapResult(item: DataProcessResult): ResultItem {
|
||||
return {
|
||||
id: String(item.id),
|
||||
previewItemId: item.preview_item_id == null ? null : String(item.preview_item_id),
|
||||
instruction: item.instruction,
|
||||
input: item.input || '',
|
||||
output: item.output,
|
||||
originalInstruction: item.original_instruction ?? item.instruction,
|
||||
originalInput: item.original_input ?? item.input ?? '',
|
||||
originalOutput: item.original_output ?? item.output,
|
||||
savedInstruction: item.instruction,
|
||||
savedInput: item.input || '',
|
||||
savedOutput: item.output,
|
||||
savedStatus: item.status,
|
||||
status: item.status,
|
||||
error: item.error || undefined,
|
||||
split: item.split || undefined,
|
||||
qualityScore: item.quality_score?.overall,
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
|
||||
export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
const results = ref<ResultItem[]>([])
|
||||
const selectedResultId = ref<string | null>(null)
|
||||
const regeneratingResultId = ref<string | null>(null)
|
||||
const bulkRegeneration = reactive<BulkResultRegenerationState>({
|
||||
status: 'idle',
|
||||
total: 0,
|
||||
completed: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
targetIds: [],
|
||||
failedIds: [],
|
||||
})
|
||||
const resultRegenerationBusy = computed(() => (
|
||||
Boolean(regeneratingResultId.value) || bulkRegeneration.status === 'running'
|
||||
))
|
||||
const generation = reactive<GenerationState>({
|
||||
status: 'idle',
|
||||
progress: 0,
|
||||
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
|
||||
})
|
||||
let generationTimer: ReturnType<typeof setInterval> | null = null
|
||||
let generationTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let generationRun = 0
|
||||
let pollFailureCount = 0
|
||||
let generationStarting = false
|
||||
|
||||
function stopGenerationTimer() {
|
||||
if (generationTimer) clearInterval(generationTimer)
|
||||
generationRun += 1
|
||||
if (generationTimer) clearTimeout(generationTimer)
|
||||
generationTimer = null
|
||||
}
|
||||
|
||||
@@ -40,39 +86,157 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。'
|
||||
results.value = []
|
||||
selectedResultId.value = null
|
||||
regeneratingResultId.value = null
|
||||
Object.assign(bulkRegeneration, {
|
||||
status: 'idle',
|
||||
total: 0,
|
||||
completed: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
targetIds: [],
|
||||
failedIds: [],
|
||||
})
|
||||
}
|
||||
|
||||
function startGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'running'
|
||||
generation.progress = 0
|
||||
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
|
||||
function applyProgress(progress: DataProcessProgress) {
|
||||
generation.progress = Math.max(0, Math.min(100, Number(progress.progress) || 0))
|
||||
generation.message = progress.message || (
|
||||
progress.status === 'running'
|
||||
? '后端正在生成标准化结果并进行质量评分。'
|
||||
: progress.status === 'completed'
|
||||
? '数据处理已完成。'
|
||||
: progress.failure_reason || '任务已停止。'
|
||||
)
|
||||
}
|
||||
|
||||
generationTimer = setInterval(() => {
|
||||
generation.progress = Math.min(100, generation.progress + 8)
|
||||
if (generation.progress < 100) return
|
||||
async function loadAllResults(taskId: string) {
|
||||
const first = await getDataProcessResults(taskId, { page: 1, page_size: RESULT_PAGE_SIZE })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const next = await getDataProcessResults(taskId, { page, page_size: RESULT_PAGE_SIZE })
|
||||
items.push(...next.items)
|
||||
}
|
||||
results.value = items.map(mapResult)
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
}
|
||||
|
||||
stopGenerationTimer()
|
||||
async function finishFromProgress(progress: DataProcessProgress, notify = true) {
|
||||
pollFailureCount = 0
|
||||
applyProgress(progress)
|
||||
if (progress.status === 'completed') {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
await loadAllResults(taskId)
|
||||
generation.status = 'success'
|
||||
results.value = createResults(
|
||||
bindings.previewItems.value,
|
||||
bindings.processType.value === 'structured'
|
||||
? bindings.structuredOptions.value
|
||||
: bindings.processType.value === 'unstructured'
|
||||
? bindings.unstructuredOptions.value
|
||||
: undefined,
|
||||
)
|
||||
generation.progress = 100
|
||||
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
bindings.dirty.value = true
|
||||
ElMessage.success('数据处理完成')
|
||||
}, 180)
|
||||
if (notify) ElMessage.success('数据处理完成')
|
||||
return
|
||||
}
|
||||
if (progress.status === 'failed' || progress.status === 'stopped') {
|
||||
generation.status = 'failed'
|
||||
generation.message = progress.failure_reason || progress.message || (
|
||||
progress.status === 'stopped' ? '任务已停止，可以重新生成。' : '数据处理失败，请检查配置后重试。'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function stopGeneration() {
|
||||
async function pollGeneration(runId: number) {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId || runId !== generationRun || generation.status !== 'running') return
|
||||
|
||||
try {
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (runId !== generationRun) return
|
||||
pollFailureCount = 0
|
||||
if (progress.status === 'running' || progress.status === 'pending') {
|
||||
applyProgress(progress)
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
await finishFromProgress(progress)
|
||||
} catch (error) {
|
||||
if (runId !== generationRun) return
|
||||
pollFailureCount += 1
|
||||
if (pollFailureCount <= 3) {
|
||||
generation.message = `进度查询暂时失败，正在重试（${pollFailureCount}/3）…`
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。'
|
||||
}
|
||||
}
|
||||
|
||||
async function startGeneration() {
|
||||
if (generationStarting || generation.status === 'running') return false
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) {
|
||||
ElMessage.error('任务尚未创建，请返回上一步重试')
|
||||
return false
|
||||
}
|
||||
|
||||
generationStarting = true
|
||||
let runId: number | null = null
|
||||
try {
|
||||
const canStart = await bindings.beforeGenerate?.()
|
||||
if (canStart === false) return false
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
runId = activeRunId
|
||||
generation.status = 'running'
|
||||
pollFailureCount = 0
|
||||
generation.progress = 0
|
||||
generation.message = '正在同步预览修改并启动后端处理，请稍候。'
|
||||
const progress = await generateDataProcess(taskId)
|
||||
if (activeRunId !== generationRun) return false
|
||||
if (progress.status === 'completed' || progress.status === 'failed' || progress.status === 'stopped') {
|
||||
await finishFromProgress(progress)
|
||||
return true
|
||||
}
|
||||
applyProgress(progress)
|
||||
generationTimer = setTimeout(() => void pollGeneration(activeRunId), POLL_INTERVAL_MS)
|
||||
return true
|
||||
} catch (error) {
|
||||
if (runId !== null && runId !== generationRun) return
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '启动数据处理失败，请重试。'
|
||||
return false
|
||||
} finally {
|
||||
generationStarting = false
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
stopGenerationTimer()
|
||||
generation.status = 'failed'
|
||||
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
try {
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (activeRunId !== generationRun) return
|
||||
if (progress.status === 'running') {
|
||||
generation.status = 'running'
|
||||
applyProgress(progress)
|
||||
generationTimer = setTimeout(() => void pollGeneration(activeRunId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
if (progress.status === 'completed') {
|
||||
await finishFromProgress(progress, false)
|
||||
return
|
||||
}
|
||||
if (progress.status === 'failed' || progress.status === 'stopped') {
|
||||
await finishFromProgress(progress, false)
|
||||
return
|
||||
}
|
||||
generation.status = 'idle'
|
||||
applyProgress(progress)
|
||||
} catch (error) {
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。'
|
||||
}
|
||||
}
|
||||
|
||||
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
|
||||
@@ -88,22 +252,174 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
|
||||
function restoreResult(id: string) {
|
||||
|
||||
|
||||
async function regenerateResult(id: string) {
|
||||
const taskId = bindings.taskId.value
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item.instruction = item.originalInstruction
|
||||
item.input = item.originalInput
|
||||
item.output = item.originalOutput
|
||||
item.error = undefined
|
||||
item.status = 'valid'
|
||||
bindings.dirty.value = true
|
||||
if (!taskId || !item) return false
|
||||
if (resultRegenerationBusy.value) {
|
||||
ElMessage.warning('请等待当前失败结果重新生成完成')
|
||||
return false
|
||||
}
|
||||
if (item.savedStatus !== 'invalid') {
|
||||
ElMessage.warning('只有生成失败的结果可以重新生成')
|
||||
return false
|
||||
}
|
||||
if (!item.updatedAt) {
|
||||
ElMessage.error('结果版本信息缺失，请刷新页面后重试')
|
||||
return false
|
||||
}
|
||||
|
||||
regeneratingResultId.value = id
|
||||
try {
|
||||
const regenerated = await regenerateDataProcessResult(taskId, id, {
|
||||
expected_updated_at: item.updatedAt,
|
||||
})
|
||||
const index = results.value.findIndex((entry) => entry.id === id)
|
||||
if (index >= 0) results.value[index] = mapResult(regenerated)
|
||||
ElMessage.success('当前结果已重新生成')
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
} finally {
|
||||
regeneratingResultId.value = null
|
||||
}
|
||||
}
|
||||
|
||||
async function regenerateAllResults() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return false
|
||||
if (resultRegenerationBusy.value) {
|
||||
ElMessage.warning('请等待当前失败结果重新生成完成')
|
||||
return false
|
||||
}
|
||||
|
||||
const candidates = results.value.filter((item) => item.savedStatus === 'invalid')
|
||||
if (!candidates.length) {
|
||||
ElMessage.info('当前没有需要重新生成的失败结果')
|
||||
return false
|
||||
}
|
||||
const unsaved = candidates.find((item) => (
|
||||
item.instruction !== item.savedInstruction
|
||||
|| item.input !== item.savedInput
|
||||
|| item.output !== item.savedOutput
|
||||
))
|
||||
if (unsaved) {
|
||||
selectedResultId.value = unsaved.id
|
||||
ElMessage.warning('失败结果存在未保存修改，请先保存或恢复后再全部重新生成')
|
||||
return false
|
||||
}
|
||||
const missingVersion = candidates.find((item) => !item.updatedAt)
|
||||
if (missingVersion) {
|
||||
selectedResultId.value = missingVersion.id
|
||||
ElMessage.error('失败结果版本信息缺失，请刷新页面后重试')
|
||||
return false
|
||||
}
|
||||
|
||||
Object.assign(bulkRegeneration, {
|
||||
status: 'running',
|
||||
total: candidates.length,
|
||||
completed: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
targetIds: candidates.map((item) => item.id),
|
||||
failedIds: [],
|
||||
})
|
||||
|
||||
try {
|
||||
for (let offset = 0; offset < candidates.length; offset += BULK_REGENERATION_CHUNK_SIZE) {
|
||||
const chunk = candidates.slice(offset, offset + BULK_REGENERATION_CHUNK_SIZE)
|
||||
try {
|
||||
const regenerated = await regenerateDataProcessResults(taskId, {
|
||||
items: chunk.map((item) => ({
|
||||
result_id: item.id,
|
||||
expected_updated_at: item.updatedAt as string,
|
||||
})),
|
||||
})
|
||||
for (const item of regenerated.items) {
|
||||
const index = results.value.findIndex((entry) => entry.id === String(item.id))
|
||||
if (index >= 0) results.value[index] = mapResult(item)
|
||||
}
|
||||
bulkRegeneration.completed += regenerated.total
|
||||
bulkRegeneration.succeeded += regenerated.succeeded
|
||||
bulkRegeneration.failed += regenerated.failed
|
||||
bulkRegeneration.failedIds.push(
|
||||
...regenerated.failures.map((failure) => failure.result_id),
|
||||
)
|
||||
} catch {
|
||||
const remaining = candidates.slice(offset)
|
||||
bulkRegeneration.completed = bulkRegeneration.total
|
||||
bulkRegeneration.failed += remaining.length
|
||||
bulkRegeneration.failedIds.push(...remaining.map((item) => item.id))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (bulkRegeneration.failed === 0) {
|
||||
bulkRegeneration.status = 'completed'
|
||||
ElMessage.success(`已重新生成 ${bulkRegeneration.succeeded} 条失败结果`)
|
||||
} else if (bulkRegeneration.succeeded > 0) {
|
||||
bulkRegeneration.status = 'partial'
|
||||
ElMessage.warning(
|
||||
`重新生成完成：成功 ${bulkRegeneration.succeeded} 条，仍失败 ${bulkRegeneration.failed} 条`,
|
||||
)
|
||||
} else {
|
||||
bulkRegeneration.status = 'failed'
|
||||
ElMessage.error(`重新生成失败：${bulkRegeneration.failed} 条结果仍需重试`)
|
||||
}
|
||||
const firstFailed = bulkRegeneration.failedIds.find((id) => (
|
||||
results.value.find((item) => item.id === id)?.savedStatus === 'invalid'
|
||||
))
|
||||
if (firstFailed) selectedResultId.value = firstFailed
|
||||
return bulkRegeneration.failed === 0
|
||||
} catch {
|
||||
const remaining = Math.max(0, bulkRegeneration.total - bulkRegeneration.completed)
|
||||
const remainingIds = candidates
|
||||
.filter((candidate) => (
|
||||
results.value.find((item) => item.id === candidate.id)?.savedStatus === 'invalid'
|
||||
))
|
||||
.map((item) => item.id)
|
||||
bulkRegeneration.completed = bulkRegeneration.total
|
||||
bulkRegeneration.failed += remaining
|
||||
bulkRegeneration.failedIds = [...new Set([
|
||||
...bulkRegeneration.failedIds,
|
||||
...remainingIds,
|
||||
])]
|
||||
bulkRegeneration.status = 'failed'
|
||||
if (bulkRegeneration.failedIds[0]) selectedResultId.value = bulkRegeneration.failedIds[0]
|
||||
ElMessage.error('批量重新生成意外中断，尚未成功的结果保持原状')
|
||||
return false
|
||||
} finally {
|
||||
bulkRegeneration.targetIds = []
|
||||
}
|
||||
}
|
||||
|
||||
async function persistResultChanges() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) throw new Error('任务尚未创建')
|
||||
const changed = results.value.filter((item) => (
|
||||
item.instruction !== item.savedInstruction
|
||||
|| item.input !== item.savedInput
|
||||
|| item.output !== item.savedOutput
|
||||
))
|
||||
for (const item of changed) {
|
||||
const saved = await updateDataProcessResult(taskId, item.id, {
|
||||
instruction: item.instruction,
|
||||
input: item.input,
|
||||
output: item.output,
|
||||
expected_updated_at: item.updatedAt,
|
||||
})
|
||||
const index = results.value.findIndex((entry) => entry.id === item.id)
|
||||
if (index >= 0) results.value[index] = mapResult(saved)
|
||||
}
|
||||
}
|
||||
|
||||
function validateResults() {
|
||||
let firstInvalidId: string | null = null
|
||||
for (const item of results.value) {
|
||||
if (!item.instruction.trim() || !item.output.trim()) {
|
||||
item.error = 'Instruction 和 Output 不能为空'
|
||||
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
|
||||
item.error ||= '结果未通过后端质量校验，请修改后重新保存'
|
||||
item.status = 'invalid'
|
||||
firstInvalidId ??= item.id
|
||||
}
|
||||
@@ -113,13 +429,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
}
|
||||
|
||||
return {
|
||||
bulkRegeneration,
|
||||
generation,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
selectedResultId,
|
||||
persistResultChanges,
|
||||
resetDownstream,
|
||||
restoreResult,
|
||||
regenerateAllResults,
|
||||
regenerateResult,
|
||||
resumeGeneration,
|
||||
startGeneration,
|
||||
stopGeneration,
|
||||
stopGenerationTimer,
|
||||
updateResultField,
|
||||
validateResults,
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { onBeforeUnmount } from 'vue'
|
||||
import {
|
||||
getDataProcessPreviewProgress,
|
||||
startDataProcessPreview,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type {
|
||||
DataProcessPreviewFileBuildProgress,
|
||||
DataProcessPreviewProgress,
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
const POLL_INTERVAL_MS = 1500
|
||||
const MAX_CONSECUTIVE_FAILURES = 3
|
||||
|
||||
type PreviewProgressCallback = (progress: DataProcessPreviewProgress) => void
|
||||
|
||||
function isActive(progress: DataProcessPreviewProgress) {
|
||||
return progress.preview_status === 'queued' || progress.preview_status === 'running'
|
||||
}
|
||||
|
||||
function waitForNextPoll() {
|
||||
return new Promise<void>((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
|
||||
}
|
||||
|
||||
/**
|
||||
* 后台切分控制器。
|
||||
*
|
||||
* startPreviewBuild 只发起一次 POST，随后查询任务级进度；resumePreviewBuild
|
||||
* 不重复启动任务，仅接管已有切分。两者都在 completed/failed 时返回最终进度。
|
||||
* 响应不依赖逐文件明细，完成后由调用方重新读取预览及源文件统计。
|
||||
*/
|
||||
export function useDataProcessPreviewBuild() {
|
||||
let activePollRun = 0
|
||||
|
||||
function stopPreviewPolling() {
|
||||
activePollRun += 1
|
||||
}
|
||||
|
||||
async function pollUntilSettled(
|
||||
taskId: string | number,
|
||||
initialProgress: DataProcessPreviewProgress,
|
||||
pollRun: number,
|
||||
onProgress?: PreviewProgressCallback,
|
||||
) {
|
||||
let progress = initialProgress
|
||||
let consecutiveFailures = 0
|
||||
if (pollRun !== activePollRun) return progress
|
||||
onProgress?.(progress)
|
||||
|
||||
while (isActive(progress) && pollRun === activePollRun) {
|
||||
await waitForNextPoll()
|
||||
if (pollRun !== activePollRun) break
|
||||
try {
|
||||
progress = await getDataProcessPreviewProgress(taskId)
|
||||
consecutiveFailures = 0
|
||||
onProgress?.(progress)
|
||||
} catch (error) {
|
||||
consecutiveFailures += 1
|
||||
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) throw error
|
||||
}
|
||||
}
|
||||
return progress
|
||||
}
|
||||
|
||||
async function startPreviewBuild(
|
||||
taskId: string | number,
|
||||
sourceFileIds: Array<string | number>,
|
||||
onProgress?: PreviewProgressCallback,
|
||||
): Promise<DataProcessPreviewProgress> {
|
||||
stopPreviewPolling()
|
||||
const pollRun = activePollRun
|
||||
const progress = await startDataProcessPreview(taskId, {
|
||||
replace_existing: true,
|
||||
source_file_ids: sourceFileIds,
|
||||
})
|
||||
return pollUntilSettled(taskId, progress, pollRun, onProgress)
|
||||
}
|
||||
|
||||
async function resumePreviewBuild(
|
||||
taskId: string | number,
|
||||
onProgress?: PreviewProgressCallback,
|
||||
): Promise<DataProcessPreviewProgress> {
|
||||
stopPreviewPolling()
|
||||
const pollRun = activePollRun
|
||||
const progress = await getDataProcessPreviewProgress(taskId)
|
||||
return pollUntilSettled(taskId, progress, pollRun, onProgress)
|
||||
}
|
||||
|
||||
/**
|
||||
* 兼容现有上传步骤的逐文件展示契约；底层仍只启动一个后台切分任务。
|
||||
* 后端不返回 files[]，因此运行中按任务总进度同步展示，完成后调用方
|
||||
* 必须重新加载预览，不能把这里的可视状态当作文件统计来源。
|
||||
*/
|
||||
async function buildPreviewsByFile(
|
||||
taskId: string | number,
|
||||
sourceFileIds: Array<string | number>,
|
||||
onProgress: (progress: DataProcessPreviewFileBuildProgress) => void,
|
||||
): Promise<DataProcessPreviewFileBuildProgress[]> {
|
||||
const emitTaskProgress = (taskProgress: DataProcessPreviewProgress) => {
|
||||
for (const sourceFileId of sourceFileIds) {
|
||||
if (taskProgress.preview_status === 'completed') {
|
||||
onProgress({ source_file_id: sourceFileId, status: 'success', progress: 100 })
|
||||
} else if (taskProgress.preview_status === 'failed' || taskProgress.preview_status === 'cancelled') {
|
||||
onProgress({
|
||||
source_file_id: sourceFileId,
|
||||
status: 'failed',
|
||||
progress: 0,
|
||||
error: taskProgress.preview_failure_reason || '切分失败，请重试',
|
||||
})
|
||||
} else {
|
||||
onProgress({
|
||||
source_file_id: sourceFileId,
|
||||
status: 'processing',
|
||||
progress: taskProgress.preview_progress,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
const finalProgress = await startPreviewBuild(taskId, sourceFileIds, emitTaskProgress)
|
||||
return sourceFileIds.map((sourceFileId) => (
|
||||
finalProgress.preview_status === 'completed'
|
||||
? { source_file_id: sourceFileId, status: 'success', progress: 100 }
|
||||
: {
|
||||
source_file_id: sourceFileId,
|
||||
status: 'failed',
|
||||
progress: 0,
|
||||
error: finalProgress.preview_failure_reason || '切分失败，请重试',
|
||||
}
|
||||
))
|
||||
}
|
||||
|
||||
onBeforeUnmount(stopPreviewPolling)
|
||||
|
||||
return {
|
||||
startPreviewBuild,
|
||||
resumePreviewBuild,
|
||||
buildPreviewsByFile,
|
||||
stopPreviewPolling,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import {
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
regenerateDataProcessTask,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type {
|
||||
DataProcessPreviewItem,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessTask,
|
||||
} from '@/types/dataProcess'
|
||||
import {
|
||||
createStructuredOptionsFromConfig,
|
||||
createUnstructuredOptionsFromConfig,
|
||||
} from './dataProcessCreateState'
|
||||
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
UploadedDataFile,
|
||||
} from './types'
|
||||
|
||||
interface ConfirmOptions {
|
||||
title: string
|
||||
message: string
|
||||
confirmText: string
|
||||
cancelText: string
|
||||
tone: 'warning'
|
||||
}
|
||||
|
||||
interface RegenerationBindings {
|
||||
task: Reactive<{ name: string; description: string }>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
uploadedFiles: Ref<UploadedDataFile[]>
|
||||
previewItems: Ref<PreviewItem[]>
|
||||
selectedPreviewFileId: Ref<string | null>
|
||||
selectedPreviewId: Ref<string | null>
|
||||
selectedPreviewIdsByFile: Ref<Record<string, string>>
|
||||
previewSignature: Ref<string>
|
||||
dirty: Ref<boolean>
|
||||
buildPreviewConfigSignature: () => string
|
||||
buildPreviewSignature: () => string
|
||||
mapPreviewItem: (item: DataProcessPreviewItem) => PreviewItem
|
||||
resetDownstream: () => void
|
||||
}
|
||||
|
||||
async function loadSourceContent(taskId: string, fileId: string | number) {
|
||||
const chunks: string[] = []
|
||||
let startLine = 1
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
start_line: startLine,
|
||||
line_count: 10_000,
|
||||
})
|
||||
chunks.push(source.content || '')
|
||||
if (!source.has_more) break
|
||||
const nextLine = Number(source.end_line || startLine) + 1
|
||||
if (nextLine <= startLine) break
|
||||
startLine = nextLine
|
||||
}
|
||||
// source_content_lines 已保留原始换行；分页之间直接拼接，避免凭空增加空行并破坏偏移。
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
|
||||
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const next = await getDataProcessPreview(taskId, { page, page_size: 500 })
|
||||
items.push(...next.items)
|
||||
}
|
||||
return items.map(mapPreviewItem)
|
||||
}
|
||||
|
||||
export function useDataProcessRegeneration(bindings: RegenerationBindings) {
|
||||
const route = useRoute()
|
||||
const isRegeneration = computed(() => route.name === 'data-process-regenerate')
|
||||
const isWorkflowResume = computed(() => route.name === 'data-process-workflow')
|
||||
const sourceTaskId = computed(() => (
|
||||
isRegeneration.value || isWorkflowResume.value ? String(route.params.id || '') : ''
|
||||
))
|
||||
const originalProcessType = ref<ProcessType | null>(null)
|
||||
const originalTaskUpdatedAt = ref('')
|
||||
const regenerationPrepared = ref(false)
|
||||
const originalPreviewConfigSignature = ref('')
|
||||
const confirmedPreviewConfigSignature = ref('')
|
||||
const hydrating = ref(false)
|
||||
const initializationError = ref('')
|
||||
|
||||
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
|
||||
const taskId = String(task.id)
|
||||
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
|
||||
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
|
||||
)))
|
||||
bindings.previewItems.value = preservePreviews
|
||||
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
|
||||
: []
|
||||
|
||||
const configSignature = bindings.buildPreviewConfigSignature()
|
||||
const previewCounts = new Map<string, number>()
|
||||
for (const item of bindings.previewItems.value) {
|
||||
previewCounts.set(item.sourceFileId, (previewCounts.get(item.sourceFileId) || 0) + 1)
|
||||
}
|
||||
for (const file of bindings.uploadedFiles.value) {
|
||||
const count = previewCounts.get(String(file.sourceFileId)) || 0
|
||||
file.previewCount = count
|
||||
file.previewStatus = preservePreviews && count > 0 ? 'success' : 'waiting'
|
||||
file.previewProgress = preservePreviews && count > 0 ? 100 : 0
|
||||
file.previewConfigSignature = preservePreviews && count > 0 ? configSignature : undefined
|
||||
}
|
||||
|
||||
bindings.selectedPreviewFileId.value = String(bindings.uploadedFiles.value[0]?.uid ?? '') || null
|
||||
bindings.selectedPreviewId.value = bindings.selectedPreviewFileId.value
|
||||
? bindings.previewItems.value.find((item) => (
|
||||
item.sourceFileId === bindings.selectedPreviewFileId.value
|
||||
))?.id ?? null
|
||||
: null
|
||||
bindings.selectedPreviewIdsByFile.value = (
|
||||
bindings.selectedPreviewFileId.value && bindings.selectedPreviewId.value
|
||||
) ? { [bindings.selectedPreviewFileId.value]: bindings.selectedPreviewId.value } : {}
|
||||
bindings.previewSignature.value = preservePreviews && bindings.previewItems.value.length
|
||||
? bindings.buildPreviewSignature()
|
||||
: ''
|
||||
bindings.resetDownstream()
|
||||
}
|
||||
|
||||
async function loadSource() {
|
||||
if (!sourceTaskId.value) return
|
||||
hydrating.value = true
|
||||
initializationError.value = ''
|
||||
try {
|
||||
const sourceTask = await getDataProcessTask(sourceTaskId.value)
|
||||
const sourceType = sourceTask.process_type as ProcessType
|
||||
originalProcessType.value = isRegeneration.value ? sourceType : null
|
||||
originalTaskUpdatedAt.value = sourceTask.updated_at
|
||||
bindings.task.name = sourceTask.name
|
||||
bindings.task.description = sourceTask.description || ''
|
||||
bindings.processType.value = sourceType
|
||||
const config = sourceTask.config || {}
|
||||
bindings.structuredOptions.value = createStructuredOptionsFromConfig(config)
|
||||
bindings.unstructuredOptions.value = createUnstructuredOptionsFromConfig(config)
|
||||
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
|
||||
confirmedPreviewConfigSignature.value = ''
|
||||
await hydrateWorkspace(sourceTask, true)
|
||||
await nextTick()
|
||||
bindings.dirty.value = false
|
||||
return sourceTask
|
||||
} catch (error) {
|
||||
initializationError.value = error instanceof Error
|
||||
? error.message
|
||||
: '原数据处理任务加载失败，请返回详情页后重试'
|
||||
} finally {
|
||||
hydrating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmPreviewConfigChange(
|
||||
openConfirm: (options: ConfirmOptions) => Promise<boolean | undefined>,
|
||||
) {
|
||||
if (!isRegeneration.value) return true
|
||||
if (!bindings.previewItems.value.length) return true
|
||||
const currentSignature = bindings.buildPreviewConfigSignature()
|
||||
const changed = currentSignature !== originalPreviewConfigSignature.value
|
||||
if (!changed || currentSignature === confirmedPreviewConfigSignature.value) return true
|
||||
const confirmed = await openConfirm({
|
||||
title: '确认修改切分配置？',
|
||||
message: '修改预处理或切分配置后，将按新配置重新切分。在点击“开始生成”前，原生成结果和已发布数据会继续保留。',
|
||||
confirmText: '确认并继续',
|
||||
cancelText: '返回检查',
|
||||
tone: 'warning',
|
||||
})
|
||||
if (confirmed) confirmedPreviewConfigSignature.value = currentSignature
|
||||
return Boolean(confirmed)
|
||||
}
|
||||
|
||||
async function prepareRegeneration(
|
||||
payload: Omit<DataProcessRegeneratePayload, 'expected_updated_at'>,
|
||||
) {
|
||||
// 第一次 prepare 后，上传、切分、编辑预览和生成都会推进任务版本。
|
||||
// 再次提交前读取同一任务的最新版本，仍由后端事务处理读取后的并发竞争。
|
||||
if (regenerationPrepared.value) {
|
||||
const latestTask = await getDataProcessTask(sourceTaskId.value)
|
||||
originalTaskUpdatedAt.value = latestTask.updated_at || originalTaskUpdatedAt.value
|
||||
}
|
||||
const regenerated = await regenerateDataProcessTask(sourceTaskId.value, {
|
||||
...payload,
|
||||
expected_updated_at: originalTaskUpdatedAt.value,
|
||||
})
|
||||
originalTaskUpdatedAt.value = regenerated.task.updated_at || originalTaskUpdatedAt.value
|
||||
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
|
||||
confirmedPreviewConfigSignature.value = ''
|
||||
regenerationPrepared.value = true
|
||||
bindings.dirty.value = true
|
||||
try {
|
||||
const regeneratedTask = regenerated.task.source_files
|
||||
? regenerated.task
|
||||
: await getDataProcessTask(regenerated.task.id)
|
||||
originalTaskUpdatedAt.value = regeneratedTask.updated_at || originalTaskUpdatedAt.value
|
||||
hydrating.value = true
|
||||
try {
|
||||
await hydrateWorkspace(regeneratedTask, !regenerated.preview_invalidated)
|
||||
} finally {
|
||||
hydrating.value = false
|
||||
}
|
||||
} catch (error) {
|
||||
initializationError.value = error instanceof Error
|
||||
? `重新生成配置已保存，但工作区恢复失败：${error.message}`
|
||||
: '重新生成配置已保存，但工作区恢复失败，请重试加载原任务'
|
||||
throw error
|
||||
}
|
||||
return regenerated
|
||||
}
|
||||
|
||||
async function confirmStartGeneration(
|
||||
openConfirm: (options: ConfirmOptions) => Promise<boolean | undefined>,
|
||||
syncPreviewChanges: () => Promise<void>,
|
||||
) {
|
||||
if (isRegeneration.value) {
|
||||
const confirmed = await openConfirm({
|
||||
title: '开始重新生成？',
|
||||
message: '点击开始后，当前生成结果将被替换。已发布数据集会继续保留，直到重新发布。',
|
||||
confirmText: '开始生成',
|
||||
cancelText: '继续检查',
|
||||
tone: 'warning',
|
||||
})
|
||||
if (!confirmed) return false
|
||||
}
|
||||
await syncPreviewChanges()
|
||||
return true
|
||||
}
|
||||
|
||||
return {
|
||||
sourceTaskId,
|
||||
isRegeneration,
|
||||
isWorkflowResume,
|
||||
originalProcessType,
|
||||
originalTaskUpdatedAt,
|
||||
regenerationPrepared,
|
||||
hydrating,
|
||||
initializationError,
|
||||
hydrateWorkspace,
|
||||
loadSource,
|
||||
confirmPreviewConfigChange,
|
||||
prepareRegeneration,
|
||||
confirmStartGeneration,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import { ref, type Ref } from 'vue'
|
||||
import {
|
||||
getDataProcessSourceContent,
|
||||
uploadDataProcessSourceFiles,
|
||||
type DataProcessSourceFile,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
])
|
||||
const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
taskId: Ref<string | null>
|
||||
uploadedFiles: Ref<UploadedDataFile[]>
|
||||
onUploaded: (file: UploadedDataFile) => void
|
||||
}
|
||||
|
||||
type SourceFileValidation =
|
||||
| { valid: true; extension: string }
|
||||
| { valid: false; severity: 'error' | 'warning'; message: string }
|
||||
|
||||
export function validateSourceFileSelection(
|
||||
raw: File,
|
||||
processType: ProcessType,
|
||||
selectedFiles: UploadedDataFile[],
|
||||
): SourceFileValidation {
|
||||
if (raw.size > MAX_SOURCE_FILE_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '单文件不能超过 200MB' }
|
||||
}
|
||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||
if (LEGACY_OFFICE_EXTENSIONS.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: '旧版 DOC、XLS、PPT 文件暂不支持，请分别转换为 DOCX、XLSX、PPTX 后上传',
|
||||
}
|
||||
}
|
||||
const supportedExtensions = processType === 'unstructured'
|
||||
? UNSTRUCTURED_FILE_EXTENSIONS
|
||||
: STRUCTURED_FILE_EXTENSIONS
|
||||
if (!supportedExtensions.has(extension)) {
|
||||
return {
|
||||
valid: false,
|
||||
severity: 'error',
|
||||
message: processType === 'unstructured'
|
||||
? '非结构化数据支持 TXT、MD、MARKDOWN、PDF、DOCX、PPTX、JSON、JSONL、NDJSON'
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
const selectedBytes = selectedFiles.reduce((total, file) => total + file.size, 0)
|
||||
if (selectedBytes + raw.size > MAX_SOURCE_BATCH_BYTES) {
|
||||
return { valid: false, severity: 'warning', message: '当前任务选择的文件总大小不能超过 500MB' }
|
||||
}
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
): UploadedDataFile {
|
||||
return {
|
||||
uid: String(file.id),
|
||||
sourceFileId: String(file.id),
|
||||
name: file.name,
|
||||
size: file.size_bytes,
|
||||
count: file.record_count,
|
||||
content,
|
||||
fileFormat: file.file_format,
|
||||
checksumSha256: file.checksum_sha256,
|
||||
status: 'ready',
|
||||
uploadProgress: 100,
|
||||
previewStatus: 'waiting',
|
||||
previewProgress: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function getUploadErrorMessage(error: unknown) {
|
||||
const responseData = (error as {
|
||||
response?: { data?: { detail?: string | { message?: string }; message?: string } }
|
||||
})?.response?.data
|
||||
const detail = responseData?.detail
|
||||
if (typeof detail === 'string') return detail
|
||||
if (detail?.message) return detail.message
|
||||
if (responseData?.message) return responseData.message
|
||||
if (error instanceof Error && error.message) return error.message
|
||||
return '上传失败，请删除该文件后重新选择'
|
||||
}
|
||||
|
||||
export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
const sourceUploading = ref(false)
|
||||
const queue: SourceUploadJob[] = []
|
||||
let worker: Promise<void> | null = null
|
||||
|
||||
async function uploadOne(job: SourceUploadJob) {
|
||||
const pending = options.uploadedFiles.value.find((file) => String(file.uid) === job.uid)
|
||||
if (!pending) return
|
||||
|
||||
const currentTaskId = options.taskId.value
|
||||
if (!currentTaskId) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = '任务尚未创建，请返回模型选择步骤后重试'
|
||||
return
|
||||
}
|
||||
|
||||
pending.status = 'uploading'
|
||||
pending.uploadProgress = 0
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码，请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
const source = uploaded.files[0]
|
||||
if (!source) throw new Error('后端未返回源文件记录')
|
||||
|
||||
// 先登记后端 ID，确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库，正文稍后仍可由预览构建接口读取，不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
pending.uploadProgress = 100
|
||||
options.onUploaded(pending)
|
||||
} catch (error) {
|
||||
pending.status = 'failed'
|
||||
pending.uploadError = getUploadErrorMessage(error)
|
||||
}
|
||||
}
|
||||
|
||||
async function drainQueue() {
|
||||
sourceUploading.value = true
|
||||
try {
|
||||
while (queue.length) {
|
||||
const job = queue.shift()
|
||||
if (job) await uploadOne(job)
|
||||
}
|
||||
} finally {
|
||||
sourceUploading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueSourceUpload(job: SourceUploadJob) {
|
||||
queue.push(job)
|
||||
if (worker) return
|
||||
worker = drainQueue().finally(() => {
|
||||
worker = null
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
enqueueSourceUpload,
|
||||
sourceUploading,
|
||||
}
|
||||
}
|
||||
@@ -44,8 +44,8 @@ export default defineConfig({
|
||||
port: 16801,
|
||||
proxy: {
|
||||
// Frontend uses /modelTF and proxies to the local five-digit backend port.
|
||||
// 后端跑在 WSL2（networkingMode=mirrored）里，uvicorn 必须以
|
||||
// --host 0.0.0.0 启动，Windows 的 localhost 才能到达它。详见 UI测试手册第 0 节。
|
||||
// 后端跑在 WSL2 默认（NAT）网络下，Windows 的 localhost 经 localhostForwarding 转发到 WSL；
|
||||
// 后端以 --host 0.0.0.0 启动即可。不要用 networkingMode=mirrored（本机镜像回环对 WSL 服务失效）。
|
||||
'/modelTF': {
|
||||
target: 'http://localhost:17861',
|
||||
changeOrigin: true,
|
||||
|
||||
Reference in New Issue
Block a user