feat: 数据处理向导配置体系扩展

新增结构化与非结构化处理选项类型,TaskSetupStep 增加预处理、切分方法、数据集划分等配置 UI,previewModel 实现语义切分与受保护区间算法,CreateView 接入配置状态与草稿持久化并替换为 AppConfirmDialog,回归脚本扩充配置与弹窗断言。
This commit is contained in:
caoxiaozhu
2026-07-11 14:49:10 +08:00
parent 9e16f6358d
commit ddf47eb8fe
5 changed files with 1941 additions and 66 deletions

View File

@@ -4,14 +4,35 @@ import { readFile } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
import { parse as parseSfc } from '@vue/compiler-sfc'
import ts from 'typescript'
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue')
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
const viewSource = await readFile(viewPath, 'utf8')
const layoutSource = await readFile(layoutPath, 'utf8')
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
for (const marker of ['<Teleport to="body">', 'role="alertdialog"', ':aria-modal="true"', 'handleKeydown', 'Escape']) {
assert.ok(confirmDialogSource.includes(marker), `公共确认弹窗缺少可访问性能力:${marker}`)
}
assert.match(confirmDialogSource, /min-height:\s*44px/, '公共确认弹窗按钮触控区域不足 44px')
assert.match(confirmDialogSource, /focus\(\)/, '公共确认弹窗打开后没有管理键盘焦点')
assert.match(confirmDialogSource, /defineExpose\(\{ open \}\)/, '公共确认弹窗没有暴露 Promise 式 open API')
assert.match(confirmDialogSource, /width:\s*min\(480px,\s*100%\)/, '企业级确认弹窗宽度应保持紧凑的 480px')
assert.match(confirmDialogSource, /border-radius:\s*8px/, '企业级确认弹窗应使用克制的 8px 圆角')
assert.doesNotMatch(confirmDialogSource, /backdrop-filter/, '企业级确认弹窗不应使用装饰性背景模糊')
assert.ok(confirmDialogSource.includes('app-confirm-header'), '企业级确认弹窗缺少独立标题栏')
assert.match(confirmDialogSource, /\.app-confirm-button\s*\{[\s\S]*?height:\s*34px/, '桌面端操作按钮应使用紧凑的 34px 高度')
assert.match(confirmDialogSource, /@media \(max-width: 520px\)[\s\S]*?\.app-confirm-button\s*\{[\s\S]*?min-height:\s*44px/, '移动端操作按钮仍需保留 44px 触控高度')
assert.match(viewSource, /import AppConfirmDialog from '@\/components\/AppConfirmDialog\.vue'/, '创建页没有接入公共确认弹窗')
assert.match(viewSource, /<AppConfirmDialog/, '创建页模板缺少公共确认弹窗实例')
assert.match(viewSource, /onBeforeRouteLeave\(async \(\) =>/, '路由离开确认没有改为异步公共弹窗流程')
assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框')
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
for (const title of ['创建任务', '数据预览', '开始生成', '结果编辑与保存']) {
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
@@ -53,8 +74,8 @@ assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
assert.match(
viewSource,
/buildPreviewItems\(file\.content, processType\.value, String\(file\.uid\)\)/,
'预览没有按文件分别生成',
/buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/,
'预览没有按文件分别生成或未传入非结构化切分配置',
)
for (const marker of [
@@ -99,6 +120,304 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第二步缺少窄
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const taskSetupSource = await readFile(taskSetupPath, 'utf8')
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
for (const option of [
'清理无效数据',
'识别表格结构',
'重复数据去重',
'数据格式标准化',
'异常数据过滤',
'敏感信息脱敏',
]) {
assert.ok(taskSetupSource.includes(option), `结构化预处理缺少选项:${option}`)
}
assert.ok(taskSetupSource.includes('生成选项'), '结构化配置缺少生成选项分类')
assert.ok(taskSetupSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关')
assert.ok(taskSetupSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明')
for (const splitName of ['训练集', '验证集', '测试集']) {
assert.ok(taskSetupSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
}
assert.match(taskSetupSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算')
assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验')
assert.ok(taskSetupSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示')
for (const splitField of ['train', 'validation', 'test']) {
assert.match(
taskSetupSource,
new RegExp(`structuredOptions\\.datasetSplit\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`),
`数据集划分字段 ${splitField} 缺少 0100 的整数限制`,
)
}
assert.match(taskSetupSource, /<el-switch[\s\S]*structuredOptions\.semanticEnrichment/, '语义丰富表达必须使用开关控件')
assert.match(taskSetupSource, /<el-input-number[\s\S]*structuredOptions\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿')
assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
for (const field of [
'preprocessOptions',
'chunkMethod',
'chunkSize',
'chunkOverlap',
'minChunkSize',
'customDelimiter',
'preserveTables',
'preserveCodeBlocks',
'preserveLists',
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(typesSource.includes(field), `非结构化处理选项缺少字段:${field}`)
}
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
for (const option of [
'清理无效内容',
'识别文档结构',
'合并过短内容',
'过滤低质量内容',
'重复内容去重',
'敏感信息脱敏',
'保留上下文信息',
]) {
assert.ok(taskSetupSource.includes(option), `非结构化预处理缺少选项:${option}`)
}
assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) {
assert.ok(taskSetupSource.includes(method), `切分方式缺少选项:${method}`)
}
for (const label of ['切片长度', '重叠长度', '最小切片长度', '完整保留表格', '完整保留代码块', '完整保留列表']) {
assert.ok(taskSetupSource.includes(label), `切分选项缺少配置:${label}`)
}
assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token')
assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token')
assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token')
for (const label of [
'每个切片生成数量',
'上下文范围',
'当前切片',
'相邻切片',
'当前章节',
'问题类型',
'事实问答',
'概念解释',
'操作步骤',
'原因分析',
'综合问答',
'跳过无法回答的内容',
]) {
assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少:${label}`)
}
assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3')
assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验')
assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界')
assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明')
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分')
assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
assert.match(viewSource, /contextScope:\s*'adjacent'/, '默认上下文范围必须为相邻切片')
assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿')
assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识')
assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识')
assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览')
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart)
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
const previewOptionsSource = viewSource.slice(previewOptionsStart, previewOptionsEnd)
for (const field of [
'preprocessOptions',
'chunkMethod',
'chunkSize',
'chunkOverlap',
'minChunkSize',
'customDelimiter',
'preserveTables',
'preserveCodeBlocks',
'preserveLists',
]) {
assert.ok(previewOptionsSource.includes(field), `预览签名缺少切分影响字段:${field}`)
}
for (const field of [
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`)
}
const generationOptionsStart = viewSource.indexOf('function generationAffectingOptions()')
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart)
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd)
for (const field of [
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`)
}
assert.match(
viewSource,
/watch\(generationOptionsSignature,[\s\S]*?resetDownstream\(\)/,
'生成配置变化后没有仅失效下游结果',
)
for (const mutationFunction of [
'updatePreviewContent',
'restorePreviewItem',
'addPreviewItem',
'removePreviewItem',
]) {
const mutationStart = viewSource.indexOf(`function ${mutationFunction}`)
const mutationEnd = viewSource.indexOf('\nfunction ', mutationStart + 1)
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
}
assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置')
assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑')
const transpiledModel = ts.transpileModule(modelSource, {
compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
}).outputText
const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`)
const longDocument = Array.from(
{ length: 180 },
(_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`,
).join('\n')
const baseUnstructuredOptions = {
preprocessOptions: [],
chunkMethod: 'semantic',
chunkSize: 200,
chunkOverlap: 50,
minChunkSize: 50,
customDelimiter: '',
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 3,
contextScope: 'adjacent',
generationTypes: ['factual'],
skipUnanswerable: true,
datasetSplit: { train: 80, validation: 10, test: 10 },
}
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
const options = {
...baseUnstructuredOptions,
chunkMethod,
customDelimiter: chunkMethod === 'custom' ? '\\n' : '',
}
const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options)
assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`)
assert.ok(
previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent),
`${chunkMethod} 切分方式的来源偏移不准确`,
)
assert.ok(
previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine),
`${chunkMethod} 切分方式的来源行号不准确`,
)
}
const overlapDocument = '甲'.repeat(1200)
const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
})
assert.equal(
overlapItems[0].sourceEnd - overlapItems[1].sourceStart,
100,
'固定长度切分没有按配置保留 50 个估算 Token 的重叠内容',
)
const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}`
const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', {
...baseUnstructuredOptions,
chunkMethod: 'heading',
chunkOverlap: 0,
})
assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片')
assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片')
const customDocument = `${'甲'.repeat(150)}<CUT>${'乙'.repeat(600)}`
const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', {
...baseUnstructuredOptions,
chunkMethod: 'custom',
chunkOverlap: 0,
customDelimiter: '<CUT>',
})
assert.ok(customItems[0].originalContent.endsWith('<CUT>'), '自定义切分未在指定分隔符处结束切片')
function assertProtectedContent(optionField, block, label) {
const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}`
const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
chunkOverlap: 0,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
[optionField]: true,
})
const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
chunkOverlap: 0,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
})
assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`)
assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`)
}
const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n')
const tableBlock = [
'| 字段 | 说明 |',
'| --- | --- |',
...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`),
].join('\n')
const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}:这是需要完整保留的内容。`).join('\n')
assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块')
assertProtectedContent('preserveTables', tableBlock, '表格')
assertProtectedContent('preserveLists', listBlock, '列表')
const samplePreviewItems = previewModelModule.buildPreviewItems(
longDocument,
'unstructured',
'generation-check',
baseUnstructuredOptions,
)
assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片')
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
function findNextStyleBlockStart(source, startIndex) {
let quote = null