diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index 6cc7e0b..57d2320 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -4,14 +4,35 @@ import { readFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' import path from 'node:path' import { parse as parseSfc } from '@vue/compiler-sfc' +import ts from 'typescript' const scriptDir = path.dirname(fileURLToPath(import.meta.url)) const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue') const createDir = path.resolve(scriptDir, '../src/views/data-process/create') +const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue') const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue') const viewSource = await readFile(viewPath, 'utf8') const layoutSource = await readFile(layoutPath, 'utf8') +assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog') +const confirmDialogSource = await readFile(confirmDialogPath, 'utf8') +for (const marker of ['', 'role="alertdialog"', ':aria-modal="true"', 'handleKeydown', 'Escape']) { + assert.ok(confirmDialogSource.includes(marker), `公共确认弹窗缺少可访问性能力：${marker}`) +} +assert.match(confirmDialogSource, /min-height:\s*44px/, '公共确认弹窗按钮触控区域不足 44px') +assert.match(confirmDialogSource, /focus\(\)/, '公共确认弹窗打开后没有管理键盘焦点') +assert.match(confirmDialogSource, /defineExpose\(\{ open \}\)/, '公共确认弹窗没有暴露 Promise 式 open API') +assert.match(confirmDialogSource, /width:\s*min\(480px,\s*100%\)/, '企业级确认弹窗宽度应保持紧凑的 480px') +assert.match(confirmDialogSource, /border-radius:\s*8px/, '企业级确认弹窗应使用克制的 8px 圆角') +assert.doesNotMatch(confirmDialogSource, /backdrop-filter/, '企业级确认弹窗不应使用装饰性背景模糊') +assert.ok(confirmDialogSource.includes('app-confirm-header'), '企业级确认弹窗缺少独立标题栏') +assert.match(confirmDialogSource, /\.app-confirm-button\s*\{[\s\S]*?height:\s*34px/, '桌面端操作按钮应使用紧凑的 34px 高度') +assert.match(confirmDialogSource, /@media \(max-width: 520px\)[\s\S]*?\.app-confirm-button\s*\{[\s\S]*?min-height:\s*44px/, '移动端操作按钮仍需保留 44px 触控高度') +assert.match(viewSource, /import AppConfirmDialog from '@\/components\/AppConfirmDialog\.vue'/, '创建页没有接入公共确认弹窗') +assert.match(viewSource, //, '路由离开确认没有改为异步公共弹窗流程') +assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框') + assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量') for (const title of ['创建任务', '数据预览', '开始生成', '结果编辑与保存']) { assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤：${title}`) @@ -53,8 +74,8 @@ assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件 assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态') assert.match( viewSource, - /buildPreviewItems\(file\.content, processType\.value, String\(file\.uid\)\)/, - '预览没有按文件分别生成', + /buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/, + '预览没有按文件分别生成或未传入非结构化切分配置', ) for (const marker of [ @@ -99,6 +120,304 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第二步缺少窄 const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') const taskSetupSource = await readFile(taskSetupPath, 'utf8') +assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示') +for (const option of [ + '清理无效数据', + '识别表格结构', + '重复数据去重', + '数据格式标准化', + '异常数据过滤', + '敏感信息脱敏', +]) { + assert.ok(taskSetupSource.includes(option), `结构化预处理缺少选项：${option}`) +} +assert.ok(taskSetupSource.includes('生成选项'), '结构化配置缺少生成选项分类') +assert.ok(taskSetupSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关') +assert.ok(taskSetupSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明') +for (const splitName of ['训练集', '验证集', '测试集']) { + assert.ok(taskSetupSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`) +} +assert.match(taskSetupSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算') +assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验') +assert.ok(taskSetupSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示') +for (const splitField of ['train', 'validation', 'test']) { + assert.match( + taskSetupSource, + new RegExp(`structuredOptions\\.datasetSplit\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`), + `数据集划分字段 ${splitField} 缺少 0～100 的整数限制`, + ) +} +assert.match(taskSetupSource, //, '父页面缺少结构化配置状态') +assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10') +assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿') +assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复') +assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置') +assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑') + +assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型') +for (const field of [ + 'preprocessOptions', + 'chunkMethod', + 'chunkSize', + 'chunkOverlap', + 'minChunkSize', + 'customDelimiter', + 'preserveTables', + 'preserveCodeBlocks', + 'preserveLists', + 'semanticEnrichment', + 'qaPairsPerChunk', + 'contextScope', + 'generationTypes', + 'skipUnanswerable', + 'datasetSplit', +]) { + assert.ok(typesSource.includes(field), `非结构化处理选项缺少字段：${field}`) +} + +assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示') +for (const option of [ + '清理无效内容', + '识别文档结构', + '合并过短内容', + '过滤低质量内容', + '重复内容去重', + '敏感信息脱敏', + '保留上下文信息', +]) { + assert.ok(taskSetupSource.includes(option), `非结构化预处理缺少选项：${option}`) +} + +assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类') +for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) { + assert.ok(taskSetupSource.includes(method), `切分方式缺少选项：${method}`) +} +for (const label of ['切片长度', '重叠长度', '最小切片长度', '完整保留表格', '完整保留代码块', '完整保留列表']) { + assert.ok(taskSetupSource.includes(label), `切分选项缺少配置：${label}`) +} +assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token') +assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token') +assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token') + +for (const label of [ + '每个切片生成数量', + '上下文范围', + '当前切片', + '相邻切片', + '当前章节', + '问题类型', + '事实问答', + '概念解释', + '操作步骤', + '原因分析', + '综合问答', + '跳过无法回答的内容', +]) { + assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少：${label}`) +} +assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3') +assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验') +assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界') +assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明') + +assert.match(viewSource, /const unstructuredOptions = ref/, '父页面缺少非结构化配置状态') +assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分') +assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') +assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') +assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') +assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') +assert.match(viewSource, /contextScope:\s*'adjacent'/, '默认上下文范围必须为相邻切片') +assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿') +assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复') +assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置') +assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识') +assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识') +assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览') +assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断') + +const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()') +const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart) +assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数') +const previewOptionsSource = viewSource.slice(previewOptionsStart, previewOptionsEnd) +for (const field of [ + 'preprocessOptions', + 'chunkMethod', + 'chunkSize', + 'chunkOverlap', + 'minChunkSize', + 'customDelimiter', + 'preserveTables', + 'preserveCodeBlocks', + 'preserveLists', +]) { + assert.ok(previewOptionsSource.includes(field), `预览签名缺少切分影响字段：${field}`) +} +for (const field of [ + 'semanticEnrichment', + 'qaPairsPerChunk', + 'contextScope', + 'generationTypes', + 'skipUnanswerable', + 'datasetSplit', +]) { + assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`) +} + +const generationOptionsStart = viewSource.indexOf('function generationAffectingOptions()') +const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart) +assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数') +const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd) +for (const field of [ + 'semanticEnrichment', + 'qaPairsPerChunk', + 'contextScope', + 'generationTypes', + 'skipUnanswerable', + 'datasetSplit', +]) { + assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段：${field}`) +} +assert.match( + viewSource, + /watch\(generationOptionsSignature,[\s\S]*?resetDownstream\(\)/, + '生成配置变化后没有仅失效下游结果', +) +for (const mutationFunction of [ + 'updatePreviewContent', + 'restorePreviewItem', + 'addPreviewItem', + 'removePreviewItem', +]) { + const mutationStart = viewSource.indexOf(`function ${mutationFunction}`) + const mutationEnd = viewSource.indexOf('\nfunction ', mutationStart + 1) + const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd) + assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`) +} +assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置') +assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑') + +const transpiledModel = ts.transpileModule(modelSource, { + compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 }, +}).outputText +const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`) +const longDocument = Array.from( + { length: 180 }, + (_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`, +).join('\n') +const baseUnstructuredOptions = { + preprocessOptions: [], + chunkMethod: 'semantic', + chunkSize: 200, + chunkOverlap: 50, + minChunkSize: 50, + customDelimiter: '', + preserveTables: false, + preserveCodeBlocks: false, + preserveLists: false, + semanticEnrichment: false, + qaPairsPerChunk: 3, + contextScope: 'adjacent', + generationTypes: ['factual'], + skipUnanswerable: true, + datasetSplit: { train: 80, validation: 10, test: 10 }, +} + +for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) { + const options = { + ...baseUnstructuredOptions, + chunkMethod, + customDelimiter: chunkMethod === 'custom' ? '\\n' : '', + } + const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options) + assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`) + assert.ok( + previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent), + `${chunkMethod} 切分方式的来源偏移不准确`, + ) + assert.ok( + previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine), + `${chunkMethod} 切分方式的来源行号不准确`, + ) +} + +const overlapDocument = '甲'.repeat(1200) +const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', { + ...baseUnstructuredOptions, + chunkMethod: 'fixed', +}) +assert.equal( + overlapItems[0].sourceEnd - overlapItems[1].sourceStart, + 100, + '固定长度切分没有按配置保留 50 个估算 Token 的重叠内容', +) + +const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}` +const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', { + ...baseUnstructuredOptions, + chunkMethod: 'heading', + chunkOverlap: 0, +}) +assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片') +assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片') + +const customDocument = `${'甲'.repeat(150)}${'乙'.repeat(600)}` +const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', { + ...baseUnstructuredOptions, + chunkMethod: 'custom', + chunkOverlap: 0, + customDelimiter: '', +}) +assert.ok(customItems[0].originalContent.endsWith(''), '自定义切分未在指定分隔符处结束切片') + +function assertProtectedContent(optionField, block, label) { + const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}` + const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, { + ...baseUnstructuredOptions, + chunkMethod: 'fixed', + chunkOverlap: 0, + preserveTables: false, + preserveCodeBlocks: false, + preserveLists: false, + [optionField]: true, + }) + const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, { + ...baseUnstructuredOptions, + chunkMethod: 'fixed', + chunkOverlap: 0, + preserveTables: false, + preserveCodeBlocks: false, + preserveLists: false, + }) + assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`) + assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`) +} + +const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n') +const tableBlock = [ + '| 字段 | 说明 |', + '| --- | --- |', + ...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`), +].join('\n') +const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}：这是需要完整保留的内容。`).join('\n') +assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块') +assertProtectedContent('preserveTables', tableBlock, '表格') +assertProtectedContent('preserveLists', listBlock, '列表') + +const samplePreviewItems = previewModelModule.buildPreviewItems( + longDocument, + 'unstructured', + 'generation-check', + baseUnstructuredOptions, +) +assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片') +const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions) +assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片') + +const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check') +assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏') + function findNextStyleBlockStart(source, startIndex) { let quote = null diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 22bd596..9d559bc 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -1,17 +1,30 @@