feat(data-process): 接入重新生成配置流程

This commit is contained in:
caoxiaozhu
2026-07-25 22:41:06 +08:00
parent 396d3f6f47
commit 06e7455630
12 changed files with 648 additions and 145 deletions

View File

@@ -22,6 +22,7 @@ assert.match(routerSource, /path:\s*['"]data-process\/:id['"]/, '缺少数据处
assert.match(routerSource, /name:\s*['"]data-process-detail['"]/, '数据处理详情路由缺少名称') assert.match(routerSource, /name:\s*['"]data-process-detail['"]/, '数据处理详情路由缺少名称')
assert.match(routerSource, /DataProcessDetailView\.vue/, '数据处理详情路由未加载详情页面') assert.match(routerSource, /DataProcessDetailView\.vue/, '数据处理详情路由未加载详情页面')
assert.match(routerSource, /title:\s*['"]数据处理详情['"]/, '数据处理详情路由标题不正确') assert.match(routerSource, /title:\s*['"]数据处理详情['"]/, '数据处理详情路由标题不正确')
assert.match(routerSource, /path:\s*['"]data-process\/:id\/regenerate['"][\s\S]*?name:\s*['"]data-process-regenerate['"][\s\S]*?DataProcessCreateView\.vue/, '重新生成路由未复用创建向导')
assert.match(listSource, /name:\s*['"]data-process-detail['"]/, '列表详情按钮未使用详情命名路由') assert.match(listSource, /name:\s*['"]data-process-detail['"]/, '列表详情按钮未使用详情命名路由')
assert.match(listSource, /params:\s*\{\s*id:\s*taskId\s*\}/, '列表详情按钮未传递任务 ID') assert.match(listSource, /params:\s*\{\s*id:\s*taskId\s*\}/, '列表详情按钮未传递任务 ID')
@@ -62,7 +63,11 @@ assert.match(detailSource, /\.output-dataset-links[\s\S]*?\.el-button[\s\S]*?wid
assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?display:\s*flex/, '输出数据集按钮内容没有撑满统一宽度') assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?display:\s*flex/, '输出数据集按钮内容没有撑满统一宽度')
assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐') assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐')
assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集') assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集')
assert.match(detailSource, /outputDatasetBaseName\.value/, '重新同步时未使用单一基础名称') assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由')
assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value/, '详情页没有覆盖可恢复的重新生成任务状态')
assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口')
assert.match(detailSource, /v-if="detail\.status === 'completed' && !outputDatasetId"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布完成任务没有保留首次发布入口')
assert.doesNotMatch(detailSource, /查看三个数据集|重新同步数据集|resyncPublishedDataset/, '已发布任务仍保留旧查看或同步按钮')
assert.doesNotMatch(detailSource, /label="数据集类型"/, '发布三个数据集时不应再选择单一数据集类型') assert.doesNotMatch(detailSource, /label="数据集类型"/, '发布三个数据集时不应再选择单一数据集类型')
assert.match(detailSource, /无法加载数据处理任务/, '未知任务或加载失败缺少明确错误状态') assert.match(detailSource, /无法加载数据处理任务/, '未知任务或加载失败缺少明确错误状态')
assert.match(detailSource, /结果明细加载失败/, '结果加载失败缺少明确错误状态') assert.match(detailSource, /结果明细加载失败/, '结果加载失败缺少明确错误状态')

View File

@@ -14,24 +14,28 @@ const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts')
const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts') const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts')
const requestPath = path.resolve(scriptDir, '../src/api/request.ts') const requestPath = path.resolve(scriptDir, '../src/api/request.ts')
const authStorePath = path.resolve(scriptDir, '../src/stores/auth.ts') const authStorePath = path.resolve(scriptDir, '../src/stores/auth.ts')
const modelsStorePath = path.resolve(scriptDir, '../src/stores/models.ts')
const routerPath = path.resolve(scriptDir, '../src/router/index.ts') const routerPath = path.resolve(scriptDir, '../src/router/index.ts')
const sessionActivityPath = path.resolve(scriptDir, '../src/utils/sessionActivity.ts') const sessionActivityPath = path.resolve(scriptDir, '../src/utils/sessionActivity.ts')
const sourceUploadWorkerPath = path.join(createDir, 'useDataProcessSourceUpload.ts') const sourceUploadWorkerPath = path.join(createDir, 'useDataProcessSourceUpload.ts')
const regenerationPath = path.join(createDir, 'useDataProcessRegeneration.ts')
const viewSource = await readFile(viewPath, 'utf8') const viewSource = await readFile(viewPath, 'utf8')
const layoutSource = await readFile(layoutPath, 'utf8') const layoutSource = await readFile(layoutPath, 'utf8')
const [stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([ const [stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource, regenerationSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([
readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'), readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'), readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
readFile(path.join(createDir, 'useDataProcessPreviewBuild.ts'), 'utf8'), readFile(path.join(createDir, 'useDataProcessPreviewBuild.ts'), 'utf8'),
readFile(sourceUploadWorkerPath, 'utf8'), readFile(sourceUploadWorkerPath, 'utf8'),
readFile(regenerationPath, 'utf8'),
readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'), readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
readFile(apiPath, 'utf8'), readFile(apiPath, 'utf8'),
readFile(contractTypesPath, 'utf8'), readFile(contractTypesPath, 'utf8'),
]) ])
const implementationSource = [viewSource, stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource].join('\n') const implementationSource = [viewSource, stateSource, generationSource, previewBuildSource, sourceUploadWorkerSource, regenerationSource].join('\n')
const [requestSource, authStoreSource, routerSource, sessionActivitySource] = await Promise.all([ const [requestSource, authStoreSource, modelsStoreSource, routerSource, sessionActivitySource] = await Promise.all([
readFile(requestPath, 'utf8'), readFile(requestPath, 'utf8'),
readFile(authStorePath, 'utf8'), readFile(authStorePath, 'utf8'),
readFile(modelsStorePath, 'utf8'),
readFile(routerPath, 'utf8'), readFile(routerPath, 'utf8'),
readFile(sessionActivityPath, 'utf8'), readFile(sessionActivityPath, 'utf8'),
]) ])
@@ -60,6 +64,9 @@ assert.match(viewSource, /import AppConfirmDialog from '@\/components\/AppConfir
assert.match(viewSource, /<AppConfirmDialog/, '创建页模板缺少公共确认弹窗实例') assert.match(viewSource, /<AppConfirmDialog/, '创建页模板缺少公共确认弹窗实例')
assert.match(viewSource, /onBeforeRouteLeave\(async \(\) =>/, '路由离开确认没有改为异步公共弹窗流程') assert.match(viewSource, /onBeforeRouteLeave\(async \(\) =>/, '路由离开确认没有改为异步公共弹窗流程')
assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框') assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框')
assert.match(routerSource, /path:\s*'data-process\/:id\/regenerate'[\s\S]*?name:\s*'data-process-regenerate'[\s\S]*?DataProcessCreateView\.vue/, '重新生成路由没有复用创建向导')
assert.match(regenerationSource, /route\.name === 'data-process-regenerate'[\s\S]*?route\.params\.id/, '创建向导没有从路由参数识别重新生成来源任务')
assert.match(viewSource, /const currentStep = ref\(0\)/, '重新生成必须从向导第一步开始')
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量') assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) { for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) {
@@ -257,6 +264,12 @@ assert.doesNotMatch(nextFromCreateSource, /uploadedFiles|buildDataProcessPreview
assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, '大模型选择步骤继续前没有校验模型配置') assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, '大模型选择步骤继续前没有校验模型配置')
assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务') assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务')
assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件') assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件')
assert.match(nextFromModelSource, /await modelsStore\.load\(true\)/, '提交任务前没有等待最新模型列表')
assert.ok(nextFromModelSource.indexOf('await modelsStore.load(true)') < nextFromModelSource.indexOf('prepareRegeneration(taskPayload())'), '重新生成可能在模型列表校验前修改服务端')
assert.ok(nextFromModelSource.indexOf('await modelsStore.load(true)') < nextFromModelSource.indexOf('updateDataProcessTask'), '普通任务可能在模型列表校验前修改服务端')
assert.ok(nextFromModelSource.includes('模型列表加载失败,未提交任何修改'), '模型列表加载失败没有明确阻断提交')
assert.ok(nextFromModelSource.includes('原任务使用的模型已删除'), '原任务模型已删除没有要求重新选择')
assert.match(modelsStoreSource, /catch \{[\s\S]*?list\.value = \[\][\s\S]*?loaded\.value = false/, '模型列表请求失败后 loaded 状态仍可能错误保留')
assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据') assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据')
assert.match( assert.match(
nextFromUploadSource, nextFromUploadSource,
@@ -383,6 +396,7 @@ assert.doesNotMatch(generationSource, /createResults\(/, '生成 composable 仍
for (const apiName of [ for (const apiName of [
'createDataProcessTask', 'createDataProcessTask',
'regenerateDataProcessTask',
'uploadDataProcessSourceFiles', 'uploadDataProcessSourceFiles',
'buildDataProcessPreview', 'buildDataProcessPreview',
'getDataProcessPreview', 'getDataProcessPreview',
@@ -428,6 +442,9 @@ assert.match(sourceUploadWorkerSource, /pending\.status = 'ready'[\s\S]*?pending
assert.match(viewSource, /failedUploads[\s\S]*?hasUnfinishedUploads[\s\S]*?buildPreviewsByFile/, '上传失败或未完成时没有阻断切分') assert.match(viewSource, /failedUploads[\s\S]*?hasUnfinishedUploads[\s\S]*?buildPreviewsByFile/, '上传失败或未完成时没有阻断切分')
assert.doesNotMatch(viewSource, /file\.sourceFileId \|\| file\.uid/, '切分或删除仍可能把本地临时 UID 当成后端文件 ID') assert.doesNotMatch(viewSource, /file\.sourceFileId \|\| file\.uid/, '切分或删除仍可能把本地临时 UID 当成后端文件 ID')
assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段') assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段')
assert.match(contractTypesSource, /export interface DataProcessRegeneratePayload extends DataProcessTaskCreatePayload[\s\S]*?expected_updated_at: string/, '重新生成 payload 的原任务并发版本必须为必填')
assert.match(contractTypesSource, /export interface DataProcessRegenerateResult[\s\S]*?preview_invalidated: boolean[\s\S]*?published_outputs_preserved: boolean/, '重新生成响应缺少预览失效或已发布数据保留状态')
assert.match(apiSource, /regenerateDataProcessTask[\s\S]*?\/regenerate`/, '重新生成 API 路径未接入')
for (const field of [ for (const field of [
'generationModelId', 'generationModelId',
@@ -510,6 +527,10 @@ for (const removedField of ['contextScope', 'generationTypes', 'skipUnanswerable
} }
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示') assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
assert.match(taskSetupSource, /processTypeLocked\?: boolean/, '任务配置缺少处理类型锁定状态')
assert.match(taskSetupSource, /:disabled="processTypeLocked"/, '重新生成时处理类型选项没有禁用')
assert.ok(taskSetupSource.includes('重新生成沿用原任务处理类型,不可修改'), '处理类型锁定缺少明确说明')
assert.match(viewSource, /:process-type-locked="isRegeneration"/, '创建向导没有向第一步传递重新生成锁定状态')
assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关') assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关')
assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关') assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关')
const expectedSmartPreprocessOptions = [ const expectedSmartPreprocessOptions = [
@@ -668,14 +689,51 @@ assert.ok(
const taskPayloadStart = viewSource.indexOf('function taskPayload()') const taskPayloadStart = viewSource.indexOf('function taskPayload()')
const taskPayloadEnd = viewSource.indexOf('function externalPayload()', taskPayloadStart) const taskPayloadEnd = viewSource.indexOf('function externalPayload()', taskPayloadStart)
const taskPayloadSource = viewSource.slice(taskPayloadStart, taskPayloadEnd) const taskPayloadSource = viewSource.slice(taskPayloadStart, taskPayloadEnd)
for (const marker of ['name: task.name.trim()', 'description: task.description.trim()', 'process_type: processType.value', 'config: toBackendConfig()']) { for (const marker of ['name: task.name.trim()', 'description: task.description.trim()', 'process_type: originalProcessType.value || processType.value', 'config: toBackendConfig()']) {
assert.ok(taskPayloadSource.includes(marker), `任务创建 payload 缺少:${marker}`) assert.ok(taskPayloadSource.includes(marker), `任务创建 payload 缺少:${marker}`)
} }
assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\)/, '配置反向映射没有区分缺失值与 false/0')
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /createUnstructuredOptionsFromConfig/, '非结构化配置缺少后端到表单的反向映射')
assert.match(stateSource, /configValue<unknown>\(config, 'preprocess_options', \[\]\)/, '历史任务缺少预处理配置时必须按后端空列表语义回填')
for (const [field, fallback] of [
['preserve_tables', 'preserveTables'],
['preserve_code_blocks', 'preserveCodeBlocks'],
['preserve_lists', 'preserveLists'],
]) {
assert.match(
stateSource,
new RegExp(`configValue\\(\\s*config,\\s*'${field}',[\\s\\S]*?defaults\\.${fallback}`),
`历史任务缺少 ${field} 时必须沿用表单默认值,同时保留显式 false`,
)
}
assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务')
assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文')
assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片')
assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用')
assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重')
assert.match(regenerationSource, /if \(!bindings\.previewItems\.value\.length\) return true/, '没有现有切片时仍会弹出删除切片警告')
assert.ok(regenerationSource.includes('删除现有切片和生成结果;已发布三数据集暂时保留'), '切分变更警告没有说明完整影响范围')
assert.match(regenerationSource, /regenerateDataProcessTask\(sourceTaskId\.value,[\s\S]*?expected_updated_at: originalTaskUpdatedAt\.value/, '第二步没有携带最新乐观并发版本重新生成任务')
assert.match(regenerationSource, /originalTaskUpdatedAt\.value = regenerated\.task\.updated_at/, '重新生成成功后没有更新下一次提交的并发版本')
assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?getDataProcessTask\(sourceTaskId\.value\)[\s\S]*?originalTaskUpdatedAt\.value = latestTask\.updated_at/, '服务端中间操作后再次提交没有刷新任务并发版本')
assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态')
assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片')
assert.match(regenerationSource, /任务已进入重新生成状态,但工作区恢复失败/, '重新生成已提交但水合失败时缺少可恢复错误状态')
assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容')
assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口')
assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新')
assert.ok(viewSource.includes('已提交到服务端的修改不会因离开页面而撤销'), '离开提示没有说明已提交服务端的修改不可撤销')
assert.match(viewSource, /regenerationPrepared\.value \? '离开页面' : '放弃修改'/, '服务端已提交后离开按钮仍使用误导性的放弃修改文案')
assert.match(viewSource, /v-if="initializationError"[\s\S]*?@click="loadRegenerationSource"[\s\S]*?重试加载原任务/, '原任务初始化失败后缺少安全重试入口')
assert.match(regenerationSource, /previewStatus = preservePreviews && count > 0 \? 'success' : 'waiting'/, '保留切片时没有跳过重切,或切片失效后未回到等待状态')
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()') const previewOptionsStart = stateSource.indexOf('export function previewAffectingOptionsFor(')
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart) const previewOptionsEnd = stateSource.indexOf('export function generationAffectingOptionsFor(', previewOptionsStart)
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数') assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
const previewOptionsSource = viewSource.slice(previewOptionsStart, previewOptionsEnd) const previewOptionsSource = stateSource.slice(previewOptionsStart, previewOptionsEnd)
for (const field of [ for (const field of [
'preprocessOptions', 'preprocessOptions',
'chunkMethod', 'chunkMethod',
@@ -693,10 +751,10 @@ for (const field of ['semanticEnrichment', 'qaPairsPerChunk', 'datasetSplit']) {
assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`) assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`)
} }
const generationOptionsStart = viewSource.indexOf('function generationAffectingOptions()') const generationOptionsStart = stateSource.indexOf('export function generationAffectingOptionsFor(')
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart) const generationOptionsEnd = stateSource.length
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数') assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd) const generationOptionsSource = stateSource.slice(generationOptionsStart, generationOptionsEnd)
for (const field of [ for (const field of [
'semanticEnrichment', 'semanticEnrichment',
'qaPairsPerChunk', 'qaPairsPerChunk',

View File

@@ -10,6 +10,8 @@ import type {
DataProcessPreviewItem, DataProcessPreviewItem,
DataProcessPreviewUpdatePayload, DataProcessPreviewUpdatePayload,
DataProcessProgress, DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessPublishPayload, DataProcessPublishPayload,
DataProcessPublishResult, DataProcessPublishResult,
DataProcessQualityScore, DataProcessQualityScore,
@@ -39,6 +41,8 @@ export type {
DataProcessPreviewItem, DataProcessPreviewItem,
DataProcessPreviewUpdatePayload, DataProcessPreviewUpdatePayload,
DataProcessProgress, DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessPublishPayload, DataProcessPublishPayload,
DataProcessPublishResult, DataProcessPublishResult,
DataProcessQualityScore, DataProcessQualityScore,
@@ -74,6 +78,14 @@ export const createDataProcessTask = (payload: DataProcessTaskCreatePayload) =>
export const updateDataProcessTask = (taskId: string | number, payload: DataProcessTaskUpdatePayload) => export const updateDataProcessTask = (taskId: string | number, payload: DataProcessTaskUpdatePayload) =>
put<DataProcessTask>(`/data-process/${encodeURIComponent(taskId)}`, payload) put<DataProcessTask>(`/data-process/${encodeURIComponent(taskId)}`, payload)
export const regenerateDataProcessTask = (
taskId: string | number,
payload: DataProcessRegeneratePayload,
) => post<DataProcessRegenerateResult>(
`/data-process/${encodeURIComponent(taskId)}/regenerate`,
payload,
)
export const deleteDataProcessTask = (taskId: string | number) => export const deleteDataProcessTask = (taskId: string | number) =>
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`) del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)

View File

@@ -140,6 +140,12 @@ const routes: RouteRecordRaw[] = [
component: () => import('@/views/data-process/DataProcessCreateView.vue'), component: () => import('@/views/data-process/DataProcessCreateView.vue'),
meta: { title: '新建数据处理任务', pageSurface: 'self' }, meta: { title: '新建数据处理任务', pageSurface: 'self' },
}, },
{
path: 'data-process/:id/regenerate',
name: 'data-process-regenerate',
component: () => import('@/views/data-process/DataProcessCreateView.vue'),
meta: { title: '重新生成数据处理任务', pageSurface: 'self' },
},
{ {
path: 'data-process/:id', path: 'data-process/:id',
name: 'data-process-detail', name: 'data-process-detail',

View File

@@ -14,7 +14,7 @@ export const useModelsStore = defineStore('models', () => {
async function load(force = false) { async function load(force = false) {
if (loaded.value && !force) return if (loaded.value && !force) return
if (pendingLoad && !force) return pendingLoad if (pendingLoad) return pendingLoad
pendingLoad = (async () => { pendingLoad = (async () => {
try { try {
@@ -22,6 +22,7 @@ export const useModelsStore = defineStore('models', () => {
loaded.value = true loaded.value = true
} catch { } catch {
list.value = [] list.value = []
loaded.value = false
} finally { } finally {
pendingLoad = null pendingLoad = null
} }

View File

@@ -53,6 +53,7 @@ export interface DataProcessTask {
started_at?: string | null started_at?: string | null
complete_time?: string | null complete_time?: string | null
completed_at?: string | null completed_at?: string | null
updated_at: string
duration?: string | null duration?: string | null
duration_seconds?: number | null duration_seconds?: number | null
failure_reason?: string | null failure_reason?: string | null
@@ -74,6 +75,16 @@ export interface DataProcessTaskCreatePayload {
config: DataProcessConfig config: DataProcessConfig
} }
export interface DataProcessRegeneratePayload extends DataProcessTaskCreatePayload {
expected_updated_at: string
}
export interface DataProcessRegenerateResult {
task: DataProcessTask
preview_invalidated: boolean
published_outputs_preserved: boolean
}
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload> export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
export interface DataProcessSourceFile { export interface DataProcessSourceFile {

View File

@@ -14,9 +14,12 @@ import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
import { import {
createDefaultStructuredOptions, createDefaultStructuredOptions,
createDefaultUnstructuredOptions, createDefaultUnstructuredOptions,
generationAffectingOptionsFor,
previewAffectingOptionsFor,
} from './create/dataProcessCreateState' } from './create/dataProcessCreateState'
import { useDataProcessGeneration } from './create/useDataProcessGeneration' import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild' import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import { import {
mapDataProcessSourceFile, mapDataProcessSourceFile,
useDataProcessSourceUpload, useDataProcessSourceUpload,
@@ -51,7 +54,7 @@ import type {
const router = useRouter() const router = useRouter()
const modelsStore = useModelsStore() const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore) const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
// 候选范围与模型管理保持一致,不在数据处理页面重复定义模型过滤规则。 // 候选范围与模型管理保持一致,不在数据处理页面重复定义模型过滤规则。
const generationModels = computed(() => modelList.value) const generationModels = computed(() => modelList.value)
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>() const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
@@ -98,6 +101,7 @@ const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null) const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({}) const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const dirty = ref(false) const dirty = ref(false)
const modelSubmitLoading = ref(false)
let allowLeave = false let allowLeave = false
const { const {
@@ -236,7 +240,7 @@ function taskPayload() {
return { return {
name: task.name.trim(), name: task.name.trim(),
description: task.description.trim(), description: task.description.trim(),
process_type: processType.value, process_type: originalProcessType.value || processType.value,
config: toBackendConfig(), config: toBackendConfig(),
} }
} }
@@ -274,82 +278,11 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
} }
function previewAffectingOptions() { function previewAffectingOptions() {
if (processType.value === 'structured') { return previewAffectingOptionsFor(processType.value, structuredOptions.value, unstructuredOptions.value)
return {
preprocessOptions: structuredOptions.value.preprocessOptions,
}
}
if (processType.value === 'unstructured') {
const {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
semanticBreakpointPercentile,
preserveTables,
preserveCodeBlocks,
preserveLists,
} = unstructuredOptions.value
return {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
semanticBreakpointPercentile,
preserveTables,
preserveCodeBlocks,
preserveLists,
}
}
return null
} }
function generationAffectingOptions() { function generationAffectingOptions() {
if (processType.value === 'structured') { return generationAffectingOptionsFor(processType.value, structuredOptions.value, unstructuredOptions.value)
const {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
} = structuredOptions.value
return {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
}
}
if (processType.value === 'unstructured') {
const {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
} = unstructuredOptions.value
return {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
}
}
return null
} }
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions())) const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
@@ -365,19 +298,58 @@ function buildPreviewSignature() {
return `${buildPreviewConfigSignature()}:${filesSignature}` return `${buildPreviewConfigSignature()}:${filesSignature}`
} }
const {
isRegeneration,
originalProcessType,
regenerationPrepared,
hydrating,
initializationError,
loadSource: loadRegenerationSource,
confirmPreviewConfigChange,
prepareRegeneration,
} = useDataProcessRegeneration({
task,
processType,
structuredOptions,
unstructuredOptions,
uploadedFiles,
previewItems,
selectedPreviewFileId,
selectedPreviewId,
selectedPreviewIdsByFile,
previewSignature,
dirty,
buildPreviewConfigSignature,
buildPreviewSignature,
mapPreviewItem,
resetDownstream,
})
const leaveWarningMessage = computed(() => regenerationPrepared.value
? '仍有尚未完成的本地操作;已提交到服务端的修改不会因离开页面而撤销。'
: '当前存在未保存修改,离开后这些修改将不会保留。')
const leaveConfirmText = computed(() => regenerationPrepared.value ? '离开页面' : '放弃修改')
watch( watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource], [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => { dirty.value = true }, () => {
if (!hydrating.value) dirty.value = true
},
{ deep: true }, { deep: true },
) )
watch(processType, (nextType, previousType) => { watch(processType, (nextType, previousType) => {
if (hydrating.value) return
if (isRegeneration.value && originalProcessType.value && nextType !== originalProcessType.value) {
processType.value = originalProcessType.value
return
}
if (nextType === previousType || uploadedFiles.value.length === 0) return if (nextType === previousType || uploadedFiles.value.length === 0) return
resetSourceDataForProcessTypeChange() resetSourceDataForProcessTypeChange()
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据') ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
}) })
watch(generationOptionsSignature, (currentSignature, previousSignature) => { watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (hydrating.value) return
if (currentSignature === previousSignature) return if (currentSignature === previousSignature) return
resetDownstream() resetDownstream()
}) })
@@ -531,21 +503,50 @@ function resetSourceDataForProcessTypeChange() {
async function nextFromCreate() { async function nextFromCreate() {
const valid = await taskSetupRef.value?.validate() const valid = await taskSetupRef.value?.validate()
if (!valid) return if (!valid) return
const confirmed = await confirmPreviewConfigChange((options) => (
confirmDialogRef.value?.open(options) ?? Promise.resolve(false)
))
if (!confirmed) return
goToStep('model') goToStep('model')
} }
async function nextFromModel() { async function nextFromModel() {
const valid = await modelSelectionRef.value?.validate() if (modelSubmitLoading.value) return
if (!valid) return modelSubmitLoading.value = true
try { try {
const saved = taskId.value await modelsStore.load(true)
? await updateDataProcessTask(taskId.value, taskPayload()) if (!modelsLoaded.value) {
: await createDataProcessTask(taskPayload()) ElMessage.error('模型列表加载失败,未提交任何修改,请稍后重试')
taskId.value = String(saved.id) return
}
const modelId = modelSelectionOptions.value.generationModelId
const modelExists = generationModels.value.some(model => String(model.id) === String(modelId))
if (modelId !== '' && !modelExists) {
ElMessage.error(isRegeneration.value ? '原任务使用的模型已删除,请重新选择模型' : '所选模型已删除,请重新选择')
return
}
const valid = await modelSelectionRef.value?.validate()
if (!valid) return
if (isRegeneration.value) {
const regenerated = await prepareRegeneration(taskPayload())
taskId.value = String(regenerated.task.id)
if (regenerated.preview_invalidated) {
ElMessage.info('切分配置已变化,现有源文件将在下一步按新配置重新切分')
} else if (regenerated.published_outputs_preserved) {
ElMessage.info('已保留原切片;已发布的三个数据集将在重新发布前保持不变')
}
} else {
const saved = taskId.value
? await updateDataProcessTask(taskId.value, taskPayload())
: await createDataProcessTask(taskPayload())
taskId.value = String(saved.id)
}
dirty.value = true dirty.value = true
goToStep('upload') goToStep('upload')
} catch { } catch {
// 请求层已展示名称冲突或配置非法等具体原因。 // 请求层已展示名称冲突或配置非法等具体原因。
} finally {
modelSubmitLoading.value = false
} }
} }
@@ -812,8 +813,8 @@ async function handleCancel() {
} }
const confirmed = await confirmDialogRef.value?.open({ const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?', title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。', message: leaveWarningMessage.value,
confirmText: '放弃修改', confirmText: leaveConfirmText.value,
cancelText: '继续编辑', cancelText: '继续编辑',
tone: 'danger', tone: 'danger',
}) })
@@ -830,8 +831,8 @@ onBeforeRouteLeave(async () => {
if (allowLeave || !dirty.value) return true if (allowLeave || !dirty.value) return true
const confirmed = await confirmDialogRef.value?.open({ const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?', title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。', message: leaveWarningMessage.value,
confirmText: '放弃修改', confirmText: leaveConfirmText.value,
cancelText: '继续编辑', cancelText: '继续编辑',
tone: 'danger', tone: 'danger',
}) })
@@ -845,6 +846,7 @@ onBeforeUnmount(() => {
onMounted(() => { onMounted(() => {
localStorage.removeItem('yg-data-process-create-draft') localStorage.removeItem('yg-data-process-create-draft')
void modelsStore.load(true) void modelsStore.load(true)
if (isRegeneration.value) void loadRegenerationSource()
}) })
</script> </script>
@@ -882,15 +884,20 @@ onMounted(() => {
</div> </div>
</div> </div>
<div class="wizard-content"> <div class="wizard-content" v-loading="hydrating">
<div v-if="initializationError" class="initialization-error" role="alert">
<el-alert type="error" :closable="false" :title="initializationError" show-icon />
<el-button type="primary" :loading="hydrating" @click="loadRegenerationSource">重试加载原任务</el-button>
</div>
<TaskSetupStep <TaskSetupStep
v-if="currentStepId === 'create'" v-else-if="currentStepId === 'create'"
ref="taskSetupRef" ref="taskSetupRef"
v-model:name="task.name" v-model:name="task.name"
v-model:description="task.description" v-model:description="task.description"
v-model:process-type="processType" v-model:process-type="processType"
v-model:structured-options="structuredOptions" v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions" v-model:unstructured-options="unstructuredOptions"
:process-type-locked="isRegeneration"
/> />
<ModelSelectionStep <ModelSelectionStep
@@ -974,8 +981,8 @@ onMounted(() => {
<el-button <el-button
class="wizard-primary-action" class="wizard-primary-action"
type="primary" type="primary"
:loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))" :loading="modelSubmitLoading || generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)" :disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction" @click="handlePrimaryAction"
> >
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" /> {{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

View File

@@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue' import { computed, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import { ElMessage, ElMessageBox } from 'element-plus' import { ElMessage } from 'element-plus'
import PageCard from '@/components/PageCard.vue' import PageCard from '@/components/PageCard.vue'
import ModelStatusTag from '@/components/ModelStatusTag.vue' import ModelStatusTag from '@/components/ModelStatusTag.vue'
import { usePolling } from '@/composables/usePolling' import { usePolling } from '@/composables/usePolling'
@@ -167,11 +167,14 @@ const outputDatasets = computed(() => {
const outputDatasetName = computed(() => ( const outputDatasetName = computed(() => (
outputDatasets.value.map((item) => item.name).join('、') outputDatasets.value.map((item) => item.name).join('、')
)) ))
const outputDatasetBaseName = computed(() => {
const name = String(outputDatasets.value[0]?.name || '').trim()
return name.replace(/-(?:训练集|验证集|测试集)$/, '')
})
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null) const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
const canRegenerate = computed(() => {
const status = detail.value?.status
return status === 'pending'
|| status === 'failed'
|| status === 'stopped'
|| (status === 'completed' && Boolean(outputDatasetId.value))
})
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-') const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at) const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at) const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
@@ -428,10 +431,6 @@ function configuredSplit(): DataProcessDatasetSplit {
function openPublishDialog() { function openPublishDialog() {
if (!detail.value) return if (!detail.value) return
if (outputDatasetId.value) {
void router.push({ path: '/dataset', query: { tab: 'task', task_id: taskId.value } })
return
}
publishForm.dataset_name = `${detail.value.name}-数据集` publishForm.dataset_name = `${detail.value.name}-数据集`
publishForm.split = configuredSplit() publishForm.split = configuredSplit()
publishDialogVisible.value = true publishDialogVisible.value = true
@@ -461,31 +460,8 @@ async function publishDataset() {
} }
} }
async function resyncPublishedDataset() { function startRegeneration() {
if (!detail.value || !outputDatasetId.value || publishing.value) return void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
try {
await ElMessageBox.confirm(
'将按当前 8:1:1 配置同步为训练集、验证集、测试集三个独立数据集。',
'重新同步三个数据集?',
{ type: 'warning', confirmButtonText: '重新同步', cancelButtonText: '取消' },
)
} catch {
return
}
publishing.value = true
try {
const published = await publishDataProcess(taskId.value, {
...publishForm,
dataset_name: outputDatasetBaseName.value || `${detail.value.name}-数据集`,
split: configuredSplit(),
})
const datasets = published.datasets || published.output_datasets || []
ElMessage.success(`训练集、验证集和测试集已同步,共 ${datasets.length || 3} 个数据集`)
await router.push({ path: '/dataset', query: { tab: 'task', task_id: taskId.value } })
} finally {
publishing.value = false
}
} }
watch([currentPage, pageSize], () => void loadResults()) watch([currentPage, pageSize], () => void loadResults())
@@ -504,20 +480,20 @@ onBeforeUnmount(() => {
<h1>{{ detail.name }}</h1> <h1>{{ detail.name }}</h1>
<ModelStatusTag :status="detail.status" /> <ModelStatusTag :status="detail.status" />
<el-button <el-button
v-if="detail.status === 'completed'" v-if="detail.status === 'completed' && !outputDatasetId"
class="publish-button" class="publish-button"
type="primary" type="primary"
@click="openPublishDialog" @click="openPublishDialog"
> >
<i class="fa" :class="outputDatasetId ? 'fa-external-link' : 'fa-database'" /> <i class="fa fa-database" />发布为三个数据集
{{ outputDatasetId ? '查看三个数据集' : '发布为三个数据集' }}
</el-button> </el-button>
<el-button <el-button
v-if="detail.status === 'completed' && outputDatasetId" v-if="canRegenerate"
:loading="publishing" class="publish-button"
@click="resyncPublishedDataset" type="primary"
@click="startRegeneration"
> >
<i class="fa fa-refresh" />重新同步数据集 <i class="fa fa-refresh" />重新生成
</el-button> </el-button>
</div> </div>
<p>{{ detail.description || '暂无任务描述' }}</p> <p>{{ detail.description || '暂无任务描述' }}</p>

View File

@@ -16,6 +16,7 @@ const props = defineProps<{
processType: ProcessType processType: ProcessType
structuredOptions: StructuredProcessOptions structuredOptions: StructuredProcessOptions
unstructuredOptions: UnstructuredProcessOptions unstructuredOptions: UnstructuredProcessOptions
processTypeLocked?: boolean
}>() }>()
const emit = defineEmits<{ const emit = defineEmits<{
@@ -153,7 +154,8 @@ defineExpose({ validate })
<div class="section-title-row"> <div class="section-title-row">
<div> <div>
<h3>处理类型</h3> <h3>处理类型</h3>
<p>类型会影响下一步支持的数据源格式和后续预览方式</p> <p v-if="processTypeLocked">重新生成沿用原任务处理类型不可修改</p>
<p v-else>类型会影响下一步支持的数据源格式和后续预览方式</p>
</div> </div>
</div> </div>
<el-form-item prop="processType" class="type-form-item"> <el-form-item prop="processType" class="type-form-item">
@@ -162,6 +164,7 @@ defineExpose({ validate })
type="button" type="button"
class="type-option" class="type-option"
:class="{ 'is-active': processType === 'structured' }" :class="{ 'is-active': processType === 'structured' }"
:disabled="processTypeLocked"
@click="emit('update:processType', 'structured')" @click="emit('update:processType', 'structured')"
> >
<span class="type-icon"><i class="fa fa-table" /></span> <span class="type-icon"><i class="fa fa-table" /></span>
@@ -175,6 +178,7 @@ defineExpose({ validate })
type="button" type="button"
class="type-option" class="type-option"
:class="{ 'is-active': processType === 'unstructured' }" :class="{ 'is-active': processType === 'unstructured' }"
:disabled="processTypeLocked"
@click="emit('update:processType', 'unstructured')" @click="emit('update:processType', 'unstructured')"
> >
<span class="type-icon"><i class="fa fa-file-text-o" /></span> <span class="type-icon"><i class="fa fa-file-text-o" /></span>
@@ -188,6 +192,7 @@ defineExpose({ validate })
type="button" type="button"
class="type-option" class="type-option"
:class="{ 'is-active': processType === 'external' }" :class="{ 'is-active': processType === 'external' }"
:disabled="processTypeLocked"
@click="emit('update:processType', 'external')" @click="emit('update:processType', 'external')"
> >
<span class="type-icon"><i class="fa fa-cloud-download" /></span> <span class="type-icon"><i class="fa fa-cloud-download" /></span>
@@ -293,12 +298,25 @@ defineExpose({ validate })
border-color: #a8a3ff; border-color: #a8a3ff;
} }
&:disabled {
cursor: not-allowed;
opacity: 0.58;
}
&:disabled:hover {
border-color: #dfe3ea;
}
&.is-active { &.is-active {
background: #fafaff; background: #fafaff;
border-color: #5b50f2; border-color: #5b50f2;
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08); box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
} }
&.is-active:disabled {
opacity: 1;
}
strong, strong,
small { small {
display: block; display: block;

View File

@@ -117,6 +117,12 @@
min-height: 400px; min-height: 400px;
} }
.initialization-error {
display: grid;
justify-items: start;
gap: 16px;
}
.wizard-footer { .wizard-footer {
display: grid; display: grid;
flex-shrink: 0; flex-shrink: 0;

View File

@@ -1,4 +1,12 @@
import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types' import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess'
import type {
GenerationControlOptions,
PreprocessOption,
StructuredProcessOptions,
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
ProcessType,
} from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。' export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
@@ -52,3 +60,166 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
minOutputLength: 20, minOutputLength: 20,
} }
} }
function configValue<T>(config: DataProcessConfig, key: string, fallback: T): T {
return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback
}
function numberValue(config: DataProcessConfig, key: string, fallback: number): number {
const value = Number(configValue(config, key, fallback))
return Number.isFinite(value) ? value : fallback
}
function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) {
const value = config.dataset_split
if (!value || typeof value !== 'object') return { ...fallback }
const split = value as unknown as Record<string, unknown>
const splitNumber = (key: keyof DataProcessDatasetSplit) => {
const parsed = Number(split[key])
return Number.isFinite(parsed) ? parsed : fallback[key]
}
return {
train: splitNumber('train'),
validation: splitNumber('validation'),
test: splitNumber('test'),
}
}
function generationOptionsFromConfig(
config: DataProcessConfig,
defaults: GenerationControlOptions,
): GenerationControlOptions {
return {
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
generationPrompt: String(configValue(config, 'generation_prompt', defaults.generationPrompt)),
temperature: numberValue(config, 'temperature', defaults.temperature),
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
qualityFilterEnabled: Boolean(configValue(
config,
'quality_filter_enabled',
defaults.qualityFilterEnabled,
)),
filterLowQuality: Boolean(configValue(
config,
'filter_low_quality',
defaults.filterLowQuality,
)),
filterShortContent: Boolean(configValue(
config,
'filter_short_content',
defaults.filterShortContent,
)),
minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength),
}
}
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
const defaults = createDefaultStructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as PreprocessOption[]
: defaults.preprocessOptions,
semanticEnrichment: Boolean(configValue(
config,
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}
export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions {
const defaults = createDefaultUnstructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as UnstructuredPreprocessOption[]
: defaults.preprocessOptions,
chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod),
chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize),
chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap),
minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize),
semanticBreakpointPercentile: numberValue(
config,
'semantic_breakpoint_percentile',
defaults.semanticBreakpointPercentile,
),
preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)),
preserveCodeBlocks: Boolean(configValue(
config,
'preserve_code_blocks',
defaults.preserveCodeBlocks,
)),
preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)),
semanticEnrichment: Boolean(configValue(
config,
'semantic_enrichment',
defaults.semanticEnrichment,
)),
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
}
}
export function previewAffectingOptionsFor(
processType: ProcessType,
structured: StructuredProcessOptions,
unstructured: UnstructuredProcessOptions,
) {
if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions }
if (processType !== 'unstructured') return null
const {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
semanticBreakpointPercentile,
preserveTables,
preserveCodeBlocks,
preserveLists,
} = unstructured
return {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
semanticBreakpointPercentile,
preserveTables,
preserveCodeBlocks,
preserveLists,
}
}
export function generationAffectingOptionsFor(
processType: ProcessType,
structured: StructuredProcessOptions,
unstructured: UnstructuredProcessOptions,
) {
const options = processType === 'unstructured' ? unstructured : structured
if (processType === 'external') return null
const common = {
semanticEnrichment: options.semanticEnrichment,
datasetSplit: options.datasetSplit,
generationModelId: options.generationModelId,
generationPrompt: options.generationPrompt,
temperature: options.temperature,
maxTokens: options.maxTokens,
jsonMode: options.jsonMode,
qualityFilterEnabled: options.qualityFilterEnabled,
filterLowQuality: options.filterLowQuality,
filterShortContent: options.filterShortContent,
minOutputLength: options.minOutputLength,
}
return processType === 'unstructured'
? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk }
: { ...common, qaPairsPerRow: structured.qaPairsPerRow }
}

View File

@@ -0,0 +1,232 @@
import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
import { useRoute } from 'vue-router'
import {
getDataProcessPreview,
getDataProcessSourceContent,
getDataProcessTask,
regenerateDataProcessTask,
} from '@/api/modules/dataProcess'
import type {
DataProcessPreviewItem,
DataProcessRegeneratePayload,
DataProcessTask,
} from '@/types/dataProcess'
import {
createStructuredOptionsFromConfig,
createUnstructuredOptionsFromConfig,
} from './dataProcessCreateState'
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
import type {
PreviewItem,
ProcessType,
StructuredProcessOptions,
UnstructuredProcessOptions,
UploadedDataFile,
} from './types'
interface ConfirmOptions {
title: string
message: string
confirmText: string
cancelText: string
tone: 'warning'
}
interface RegenerationBindings {
task: Reactive<{ name: string; description: string }>
processType: Ref<ProcessType>
structuredOptions: Ref<StructuredProcessOptions>
unstructuredOptions: Ref<UnstructuredProcessOptions>
uploadedFiles: Ref<UploadedDataFile[]>
previewItems: Ref<PreviewItem[]>
selectedPreviewFileId: Ref<string | null>
selectedPreviewId: Ref<string | null>
selectedPreviewIdsByFile: Ref<Record<string, string>>
previewSignature: Ref<string>
dirty: Ref<boolean>
buildPreviewConfigSignature: () => string
buildPreviewSignature: () => string
mapPreviewItem: (item: DataProcessPreviewItem) => PreviewItem
resetDownstream: () => void
}
async function loadSourceContent(taskId: string, fileId: string | number) {
const chunks: string[] = []
let startLine = 1
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
start_line: startLine,
line_count: 10_000,
})
chunks.push(source.content || '')
if (!source.has_more) break
const nextLine = Number(source.end_line || startLine) + 1
if (nextLine <= startLine) break
startLine = nextLine
}
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
return chunks.join('')
}
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
const items = [...first.items]
const pages = Math.ceil(first.total / first.page_size)
for (let page = 2; page <= pages; page += 1) {
const next = await getDataProcessPreview(taskId, { page, page_size: 500 })
items.push(...next.items)
}
return items.map(mapPreviewItem)
}
export function useDataProcessRegeneration(bindings: RegenerationBindings) {
const route = useRoute()
const sourceTaskId = computed(() => (
route.name === 'data-process-regenerate' ? String(route.params.id || '') : ''
))
const isRegeneration = computed(() => Boolean(sourceTaskId.value))
const originalProcessType = ref<ProcessType | null>(null)
const originalTaskUpdatedAt = ref('')
const regenerationPrepared = ref(false)
const originalPreviewConfigSignature = ref('')
const confirmedPreviewConfigSignature = ref('')
const hydrating = ref(false)
const initializationError = ref('')
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
const taskId = String(task.id)
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
)))
bindings.previewItems.value = preservePreviews
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
: []
const configSignature = bindings.buildPreviewConfigSignature()
const previewCounts = new Map<string, number>()
for (const item of bindings.previewItems.value) {
previewCounts.set(item.sourceFileId, (previewCounts.get(item.sourceFileId) || 0) + 1)
}
for (const file of bindings.uploadedFiles.value) {
const count = previewCounts.get(String(file.sourceFileId)) || 0
file.previewCount = count
file.previewStatus = preservePreviews && count > 0 ? 'success' : 'waiting'
file.previewProgress = preservePreviews && count > 0 ? 100 : 0
file.previewConfigSignature = preservePreviews && count > 0 ? configSignature : undefined
}
bindings.selectedPreviewFileId.value = String(bindings.uploadedFiles.value[0]?.uid ?? '') || null
bindings.selectedPreviewId.value = bindings.selectedPreviewFileId.value
? bindings.previewItems.value.find((item) => (
item.sourceFileId === bindings.selectedPreviewFileId.value
))?.id ?? null
: null
bindings.selectedPreviewIdsByFile.value = (
bindings.selectedPreviewFileId.value && bindings.selectedPreviewId.value
) ? { [bindings.selectedPreviewFileId.value]: bindings.selectedPreviewId.value } : {}
bindings.previewSignature.value = preservePreviews && bindings.previewItems.value.length
? bindings.buildPreviewSignature()
: ''
bindings.resetDownstream()
}
async function loadSource() {
if (!sourceTaskId.value) return
hydrating.value = true
initializationError.value = ''
try {
const sourceTask = await getDataProcessTask(sourceTaskId.value)
const sourceType = sourceTask.process_type as ProcessType
originalProcessType.value = sourceType
originalTaskUpdatedAt.value = sourceTask.updated_at
bindings.task.name = sourceTask.name
bindings.task.description = sourceTask.description || ''
bindings.processType.value = sourceType
const config = sourceTask.config || {}
bindings.structuredOptions.value = createStructuredOptionsFromConfig(config)
bindings.unstructuredOptions.value = createUnstructuredOptionsFromConfig(config)
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
confirmedPreviewConfigSignature.value = ''
await hydrateWorkspace(sourceTask, true)
await nextTick()
bindings.dirty.value = false
} catch (error) {
initializationError.value = error instanceof Error
? error.message
: '原数据处理任务加载失败,请返回详情页后重试'
} finally {
hydrating.value = false
}
}
async function confirmPreviewConfigChange(
openConfirm: (options: ConfirmOptions) => Promise<boolean | undefined>,
) {
if (!isRegeneration.value) return true
if (!bindings.previewItems.value.length) return true
const currentSignature = bindings.buildPreviewConfigSignature()
const changed = currentSignature !== originalPreviewConfigSignature.value
if (!changed || currentSignature === confirmedPreviewConfigSignature.value) return true
const confirmed = await openConfirm({
title: '确认修改切分配置?',
message: '修改预处理或切分配置将删除现有切片和生成结果;已发布三数据集暂时保留,直到重新发布后才会更新。',
confirmText: '确认并继续',
cancelText: '返回检查',
tone: 'warning',
})
if (confirmed) confirmedPreviewConfigSignature.value = currentSignature
return Boolean(confirmed)
}
async function prepareRegeneration(
payload: Omit<DataProcessRegeneratePayload, 'expected_updated_at'>,
) {
// 第一次 prepare 后,上传、切分、编辑预览和生成都会推进任务版本。
// 再次提交前读取同一任务的最新版本,仍由后端事务处理读取后的并发竞争。
if (regenerationPrepared.value) {
const latestTask = await getDataProcessTask(sourceTaskId.value)
originalTaskUpdatedAt.value = latestTask.updated_at || originalTaskUpdatedAt.value
}
const regenerated = await regenerateDataProcessTask(sourceTaskId.value, {
...payload,
expected_updated_at: originalTaskUpdatedAt.value,
})
originalTaskUpdatedAt.value = regenerated.task.updated_at || originalTaskUpdatedAt.value
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
confirmedPreviewConfigSignature.value = ''
regenerationPrepared.value = true
bindings.dirty.value = true
try {
const regeneratedTask = regenerated.task.source_files
? regenerated.task
: await getDataProcessTask(regenerated.task.id)
originalTaskUpdatedAt.value = regeneratedTask.updated_at || originalTaskUpdatedAt.value
hydrating.value = true
try {
await hydrateWorkspace(regeneratedTask, !regenerated.preview_invalidated)
} finally {
hydrating.value = false
}
} catch (error) {
initializationError.value = error instanceof Error
? `任务已进入重新生成状态,但工作区恢复失败:${error.message}`
: '任务已进入重新生成状态,但工作区恢复失败,请重试加载原任务'
throw error
}
return regenerated
}
return {
sourceTaskId,
isRegeneration,
originalProcessType,
originalTaskUpdatedAt,
regenerationPrepared,
hydrating,
initializationError,
hydrateWorkspace,
loadSource,
confirmPreviewConfigChange,
prepareRegeneration,
}
}