feat: 完成数据处理接口与前端接入
This commit is contained in:
caoxiaozhu
@@ -6,10 +6,12 @@ import { parse as parseSfc } from '@vue/compiler-sfc'
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const sourceRoot = path.resolve(scriptDir, '../src')
|
||||
const [detailSource, listSource, routerSource] = await Promise.all([
|
||||
const [detailSource, listSource, routerSource, apiSource, typesSource] = await Promise.all([
|
||||
readFile(path.join(sourceRoot, 'views/data-process/DataProcessDetailView.vue'), 'utf8'),
|
||||
readFile(path.join(sourceRoot, 'views/data-process/DataProcessListView.vue'), 'utf8'),
|
||||
readFile(path.join(sourceRoot, 'router/index.ts'), 'utf8'),
|
||||
readFile(path.join(sourceRoot, 'api/modules/dataProcess.ts'), 'utf8'),
|
||||
readFile(path.join(sourceRoot, 'types/dataProcess.ts'), 'utf8'),
|
||||
])
|
||||
|
||||
const { descriptor, errors } = parseSfc(detailSource, { filename: 'DataProcessDetailView.vue' })
|
||||
@@ -38,18 +40,41 @@ for (const requiredCopy of [
|
||||
assert.match(detailSource, new RegExp(requiredCopy), `详情页缺少必要信息：${requiredCopy}`)
|
||||
}
|
||||
|
||||
for (const status of ['completed', 'running', 'pending', 'failed']) {
|
||||
assert.match(detailSource, new RegExp(`status:\\s*['"]${status}['"]`), `详情 Mock 缺少 ${status} 状态`)
|
||||
}
|
||||
|
||||
assert.match(detailSource, /const completedResults:\s*ResultRow\[\]/, '完成任务缺少结果明细 Mock')
|
||||
assert.match(detailSource, /:data="paginatedResults"/, '结果表格未绑定分页后的处理结果')
|
||||
assert.match(typesSource, /export type DataProcessStatus = 'pending' \| 'running' \| 'completed' \| 'failed' \| 'stopped'/, '任务状态契约不完整')
|
||||
assert.match(detailSource, /getDataProcessTask\(taskId\.value\)/, '详情页没有通过真实 API 加载任务')
|
||||
assert.match(detailSource, /getDataProcessResults\(taskId\.value,\s*\{[\s\S]*?page:[\s\S]*?page_size:[\s\S]*?keyword:[\s\S]*?status:/, '结果列表没有接入服务端分页、搜索和状态筛选')
|
||||
assert.match(detailSource, /getDataProcessProgress\(taskId\.value\)/, '运行中任务没有查询真实进度')
|
||||
assert.match(detailSource, /usePolling\(refreshRuntime,\s*3000/, '运行中任务没有启用进度轮询')
|
||||
assert.match(detailSource, /:data="results"/, '结果表格未绑定服务端结果数据')
|
||||
assert.match(detailSource, /v-model="keyword"/, '结果明细缺少搜索能力')
|
||||
assert.match(detailSource, /v-model="statusFilter"/, '结果明细缺少状态筛选')
|
||||
assert.match(detailSource, /router\.push\(`\/dataset\/\$\{detail\.outputDatasetId\}\/preview`\)/, '输出数据集未接入预览入口')
|
||||
assert.match(detailSource, /未找到数据处理任务/, '未知任务 ID 缺少明确空状态')
|
||||
assert.match(detailSource, /updateDataProcessResult\(taskId\.value,[\s\S]*?expected_updated_at:/, '结果编辑没有携带并发版本时间')
|
||||
assert.match(detailSource, /restoreDataProcessResult\(taskId\.value,\s*result\.id\)/, '结果恢复没有调用真实 API')
|
||||
assert.match(detailSource, /publishDataProcess\(taskId\.value,/, '发布数据集没有调用真实 API')
|
||||
assert.match(detailSource, /router\.push\(`\/dataset\/\$\{datasetId\}\/preview`\)/, '发布成功后未进入数据集预览')
|
||||
assert.match(detailSource, /无法加载数据处理任务/, '未知任务或加载失败缺少明确错误状态')
|
||||
assert.match(detailSource, /结果明细加载失败/, '结果加载失败缺少明确错误状态')
|
||||
assert.match(detailSource, /width:\s*100%/, '详情页没有铺满内容区域')
|
||||
assert.doesNotMatch(detailSource, /^\s*max-width:\s*\d+px/m, '详情页不应使用固定最大宽度')
|
||||
assert.doesNotMatch(detailSource, /\b(?:password|secret|token)\b/i, '详情页不得展示敏感凭据字段')
|
||||
assert.match(detailSource, /!\/\(\?:password\|secret\|token\|api_key\)\/i\.test\(key\)/, '处理配置没有过滤敏感凭据字段')
|
||||
assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据')
|
||||
|
||||
console.log('数据处理任务详情 UI 回归检查通过')
|
||||
for (const apiName of [
|
||||
'getDataProcessTask',
|
||||
'getDataProcessProgress',
|
||||
'getDataProcessResults',
|
||||
'updateDataProcessResult',
|
||||
'restoreDataProcessResult',
|
||||
'publishDataProcess',
|
||||
]) {
|
||||
assert.match(
|
||||
apiSource,
|
||||
new RegExp(`export (?:const|(?:async )?function) ${apiName}\\b`),
|
||||
`API 模块缺少 ${apiName}`,
|
||||
)
|
||||
}
|
||||
assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数')
|
||||
assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID')
|
||||
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确')
|
||||
|
||||
console.log('数据处理任务详情真实 API 回归检查通过')
|
||||
|
||||
@@ -5,8 +5,12 @@ import path from 'node:path'
|
||||
import { parse as parseSfc } from '@vue/compiler-sfc'
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessListView.vue')
|
||||
const source = await readFile(viewPath, 'utf8')
|
||||
const sourceRoot = path.resolve(scriptDir, '../src')
|
||||
const viewPath = path.join(sourceRoot, 'views/data-process/DataProcessListView.vue')
|
||||
const [source, apiSource] = await Promise.all([
|
||||
readFile(viewPath, 'utf8'),
|
||||
readFile(path.join(sourceRoot, 'api/modules/dataProcess.ts'), 'utf8'),
|
||||
])
|
||||
const { descriptor, errors } = parseSfc(source, { filename: viewPath })
|
||||
|
||||
assert.equal(errors.length, 0, `数据处理任务列表模板无法解析：${errors[0]}`)
|
||||
@@ -15,5 +19,17 @@ assert.match(source, /:data="dataList"/, '任务表格必须直接展示完整
|
||||
assert.doesNotMatch(source, /activeTab|filteredDataList/, '不应保留状态切换筛选逻辑')
|
||||
assert.doesNotMatch(source, /全部任务|处理中|已完成/, '不应保留状态切换按钮文案')
|
||||
assert.doesNotMatch(source, /capsule-tabs|capsule-tab-item/, '不应保留状态切换专用样式')
|
||||
assert.ok(source.includes("getDataProcessTasks({ page: 1, page_size: 200"), '列表没有通过真实 API 分页加载任务')
|
||||
assert.doesNotMatch(source, /usePolling|refreshActiveTasks|startPolling/, '列表不应自动轮询刷新')
|
||||
assert.doesNotMatch(source, /toolbar-extra|refreshData|refreshing|fa-refresh/, '列表不应显示手动刷新按钮')
|
||||
assert.ok(source.includes('ElMessageBox.confirm('), '删除任务前缺少二次确认')
|
||||
assert.ok(source.includes('await deleteDataProcessTask(row.id)'), '删除操作没有调用真实 API')
|
||||
assert.ok(!source.includes('TODO: 接入真实接口'), '列表仍包含本地 Mock 任务')
|
||||
assert.ok(source.includes('const dataList = ref<DataProcessTask[]>([])'), '任务列表必须以空数组初始化并等待真实 API 数据')
|
||||
|
||||
console.log('数据处理任务列表状态切换移除回归检查通过')
|
||||
assert.match(apiSource, /export (?:async )?function getDataProcessTasks/, 'API 模块缺少任务列表方法')
|
||||
assert.match(apiSource, /export const deleteDataProcessTask/, 'API 模块缺少任务删除方法')
|
||||
assert.ok(apiSource.includes("get<DataProcessPage<DataProcessTask>>('/data-process', params)"), '任务列表 API 路径或分页契约不正确')
|
||||
assert.ok(apiSource.includes('`/data-process/${encodeURIComponent(taskId)}`'), '任务详情资源路径没有安全编码任务 ID')
|
||||
|
||||
console.log('数据处理任务列表真实 API 回归检查通过')
|
||||
|
||||
@@ -4,20 +4,23 @@ import { readFile } from 'node:fs/promises'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import path from 'node:path'
|
||||
import { parse as parseSfc } from '@vue/compiler-sfc'
|
||||
import ts from 'typescript'
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
|
||||
const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
|
||||
const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue')
|
||||
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
|
||||
const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts')
|
||||
const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts')
|
||||
const viewSource = await readFile(viewPath, 'utf8')
|
||||
const layoutSource = await readFile(layoutPath, 'utf8')
|
||||
const [draftSource, stateSource, generationSource, viewStyleSource] = await Promise.all([
|
||||
const [draftSource, stateSource, generationSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([
|
||||
readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'),
|
||||
readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
|
||||
readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
|
||||
readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
|
||||
readFile(apiPath, 'utf8'),
|
||||
readFile(contractTypesPath, 'utf8'),
|
||||
])
|
||||
const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n')
|
||||
|
||||
@@ -58,7 +61,8 @@ assert.match(
|
||||
assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化')
|
||||
assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取')
|
||||
assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿')
|
||||
assert.ok(viewSource.split('\n').length < 800, 'DataProcessCreateView 拆分后仍超过 800 行')
|
||||
assert.ok(viewSource.split('\n').length < 1000, 'DataProcessCreateView 拆分后仍超过 1000 行')
|
||||
assert.match(viewSource, /useDataProcessGeneration\(\{/, '生成流程没有拆分到独立 composable')
|
||||
|
||||
const expectedComponents = [
|
||||
'TaskSetupStep.vue',
|
||||
@@ -89,15 +93,16 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
|
||||
}
|
||||
assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
|
||||
assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
|
||||
assert.match(modelSource, /export function buildPreviewItems/, '缺少切片来源映射生成函数')
|
||||
assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
|
||||
assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件标识')
|
||||
assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
|
||||
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
||||
assert.match(
|
||||
viewSource,
|
||||
/buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/,
|
||||
'预览没有按文件分别生成或未传入非结构化切分配置',
|
||||
/buildDataProcessPreview\(taskId\.value,\s*\{[\s\S]*?source_file_ids:\s*uploadedFiles\.value\.map/,
|
||||
'预览没有通过后端按已上传源文件构建',
|
||||
)
|
||||
assert.match(viewSource, /getDataProcessPreview\(taskId\.value,\s*\{ page:\s*1, page_size:\s*500 \}\)/, '预览构建后没有分页读取后端数据')
|
||||
assert.doesNotMatch(viewSource, /buildPreviewItems\(/, '创建向导仍在本地构建集成预览数据')
|
||||
|
||||
for (const marker of [
|
||||
'preview-workspace',
|
||||
@@ -186,7 +191,9 @@ assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId =
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：选择大模型'/, '第一步主按钮没有指向大模型选择')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：上传文件'/, '第二步主按钮没有指向上传文件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续：数据预览'/, '第三步主按钮没有指向数据预览')
|
||||
assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6/, '安全草稿格式必须升级到 v6')
|
||||
const draftVersionMatch = draftSource.match(/DATA_PROCESS_DRAFT_SCHEMA_VERSION\s*=\s*(\d+)/)
|
||||
assert.ok(draftVersionMatch, '草稿缺少数字版本标识')
|
||||
assert.ok(Number(draftVersionMatch[1]) >= 7, '安全草稿格式版本不得低于 v7')
|
||||
|
||||
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
|
||||
const nextFromModelStart = viewSource.indexOf('async function nextFromModel()', nextFromCreateStart)
|
||||
@@ -201,11 +208,13 @@ const nextFromModelSource = viewSource.slice(nextFromModelStart, nextFromUploadS
|
||||
const nextFromUploadSource = viewSource.slice(nextFromUploadStart, selectPreviewFileStart)
|
||||
assert.match(nextFromCreateSource, /taskSetupRef\.value\?\.validate\(\)/, '创建步骤继续前没有校验任务配置')
|
||||
assert.match(nextFromCreateSource, /goToStep\('model'\)/, '创建步骤校验通过后没有进入大模型选择')
|
||||
assert.doesNotMatch(nextFromCreateSource, /uploadedFiles|buildPreviewItems/, '创建步骤仍在校验文件或提前生成预览')
|
||||
assert.doesNotMatch(nextFromCreateSource, /uploadedFiles|buildDataProcessPreview/, '创建步骤仍在校验文件或提前生成预览')
|
||||
assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, '大模型选择步骤继续前没有校验模型配置')
|
||||
assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务')
|
||||
assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件')
|
||||
assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据')
|
||||
assert.match(nextFromUploadSource, /buildPreviewItems\(/, '上传步骤没有在进入预览前生成预览数据')
|
||||
assert.match(nextFromUploadSource, /buildDataProcessPreview\(/, '上传步骤没有调用后端构建预览')
|
||||
assert.match(nextFromUploadSource, /getDataProcessPreview\(/, '上传步骤没有读取后端预览结果')
|
||||
assert.match(nextFromUploadSource, /goToStep\('preview'\)/, '上传步骤完成后没有进入数据预览')
|
||||
assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识')
|
||||
assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成')
|
||||
@@ -254,7 +263,35 @@ assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 1
|
||||
assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿')
|
||||
assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
|
||||
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
|
||||
assert.match(generationSource, /createResults\([\s\S]*bindings\.structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
|
||||
assert.match(generationSource, /generateDataProcess\(taskId\)/, '开始生成没有调用真实 API')
|
||||
assert.match(generationSource, /getDataProcessProgress\(taskId\)/, '生成状态没有通过真实 API 轮询')
|
||||
assert.match(generationSource, /getDataProcessResults\(taskId,[\s\S]*?page:[\s\S]*?page_size:/, '生成完成后没有分页加载真实结果')
|
||||
assert.match(generationSource, /updateDataProcessResult\(taskId,\s*item\.id,[\s\S]*?expected_updated_at:/, '结果保存没有调用真实 API 或缺少并发版本')
|
||||
assert.ok(generationSource.includes('item.quality_score?.overall'), '结果映射没有读取质量总分 overall')
|
||||
assert.ok(generationSource.includes('item.quality_score?.flags || []'), '结果映射没有读取质量标记 flags')
|
||||
assert.doesNotMatch(generationSource, /createResults\(/, '生成 composable 仍在本地伪造处理结果')
|
||||
|
||||
for (const apiName of [
|
||||
'createDataProcessTask',
|
||||
'uploadDataProcessSourceFiles',
|
||||
'buildDataProcessPreview',
|
||||
'getDataProcessPreview',
|
||||
'generateDataProcess',
|
||||
'getDataProcessProgress',
|
||||
'getDataProcessResults',
|
||||
'updateDataProcessResult',
|
||||
'publishDataProcess',
|
||||
]) {
|
||||
assert.match(apiSource, new RegExp(`export (?:const|async function|function) ${apiName}\\b`), `API 模块缺少 ${apiName}`)
|
||||
}
|
||||
assert.match(viewSource, /uploadDataProcessSourceFiles\(taskId\.value,\s*\[raw\]\)/, '文件上传没有调用真实 API')
|
||||
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
|
||||
assert.match(apiSource, /\/preview\/build/, 'API 模块缺少后端预览构建路径')
|
||||
assert.match(apiSource, /\/progress`/, 'API 模块缺少生成进度路径')
|
||||
assert.match(apiSource, /\/results`/, 'API 模块缺少结果分页路径')
|
||||
assert.match(apiSource, /\/publish`/, 'API 模块缺少数据集发布路径')
|
||||
assert.match(contractTypesSource, /source_file_ids\?: Array<string \| number>/, '预览构建契约缺少源文件 ID 列表')
|
||||
assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段')
|
||||
|
||||
for (const field of [
|
||||
'generationModelId',
|
||||
@@ -425,156 +462,7 @@ for (const mutationFunction of [
|
||||
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
|
||||
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
|
||||
}
|
||||
assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置')
|
||||
assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑')
|
||||
|
||||
const transpiledModel = ts.transpileModule(modelSource, {
|
||||
compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText
|
||||
const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`)
|
||||
const longDocument = Array.from(
|
||||
{ length: 180 },
|
||||
(_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`,
|
||||
).join('\n')
|
||||
const baseUnstructuredOptions = {
|
||||
preprocessOptions: [],
|
||||
chunkMethod: 'semantic',
|
||||
chunkSize: 200,
|
||||
chunkOverlap: 50,
|
||||
minChunkSize: 50,
|
||||
customDelimiter: '',
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 3,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: 1,
|
||||
generationPrompt: '仅输出问答对',
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
}
|
||||
|
||||
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
|
||||
const options = {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod,
|
||||
customDelimiter: chunkMethod === 'custom' ? '\\n' : '',
|
||||
}
|
||||
const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options)
|
||||
assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`)
|
||||
assert.ok(
|
||||
previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent),
|
||||
`${chunkMethod} 切分方式的来源偏移不准确`,
|
||||
)
|
||||
assert.ok(
|
||||
previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine),
|
||||
`${chunkMethod} 切分方式的来源行号不准确`,
|
||||
)
|
||||
}
|
||||
|
||||
const overlapDocument = '甲'.repeat(1200)
|
||||
const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod: 'fixed',
|
||||
})
|
||||
assert.equal(
|
||||
overlapItems[0].sourceEnd - overlapItems[1].sourceStart,
|
||||
100,
|
||||
'固定长度切分没有按配置保留 50 个估算 Token 的重叠内容',
|
||||
)
|
||||
|
||||
const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}`
|
||||
const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod: 'heading',
|
||||
chunkOverlap: 0,
|
||||
})
|
||||
assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片')
|
||||
assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片')
|
||||
|
||||
const customDocument = `${'甲'.repeat(150)}<CUT>${'乙'.repeat(600)}`
|
||||
const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod: 'custom',
|
||||
chunkOverlap: 0,
|
||||
customDelimiter: '<CUT>',
|
||||
})
|
||||
assert.ok(customItems[0].originalContent.endsWith('<CUT>'), '自定义切分未在指定分隔符处结束切片')
|
||||
|
||||
function assertProtectedContent(optionField, block, label) {
|
||||
const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}`
|
||||
const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod: 'fixed',
|
||||
chunkOverlap: 0,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
[optionField]: true,
|
||||
})
|
||||
const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, {
|
||||
...baseUnstructuredOptions,
|
||||
chunkMethod: 'fixed',
|
||||
chunkOverlap: 0,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
})
|
||||
assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`)
|
||||
assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`)
|
||||
}
|
||||
|
||||
const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n')
|
||||
const tableBlock = [
|
||||
'| 字段 | 说明 |',
|
||||
'| --- | --- |',
|
||||
...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`),
|
||||
].join('\n')
|
||||
const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}：这是需要完整保留的内容。`).join('\n')
|
||||
assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块')
|
||||
assertProtectedContent('preserveTables', tableBlock, '表格')
|
||||
assertProtectedContent('preserveLists', listBlock, '列表')
|
||||
|
||||
const samplePreviewItems = previewModelModule.buildPreviewItems(
|
||||
longDocument,
|
||||
'unstructured',
|
||||
'generation-check',
|
||||
baseUnstructuredOptions,
|
||||
)
|
||||
assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片')
|
||||
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
|
||||
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
|
||||
|
||||
const shortContentItems = [{
|
||||
...samplePreviewItems[0],
|
||||
editedContent: '问：示例\n短回答',
|
||||
}]
|
||||
const filteredShortResults = previewModelModule.createResults(shortContentItems, {
|
||||
...baseUnstructuredOptions,
|
||||
qualityFilterEnabled: true,
|
||||
filterLowQuality: false,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
assert.equal(filteredShortResults.length, 0, '开启过短内容过滤后仍保留低于最少字数的结果')
|
||||
|
||||
const invalidContentItems = [{
|
||||
...samplePreviewItems[0],
|
||||
status: 'invalid',
|
||||
}]
|
||||
const filteredInvalidResults = previewModelModule.createResults(invalidContentItems, {
|
||||
...baseUnstructuredOptions,
|
||||
qualityFilterEnabled: true,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: false,
|
||||
})
|
||||
assert.equal(filteredInvalidResults.length, 0, '开启低质量过滤后仍保留标记为无效的结果')
|
||||
|
||||
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
|
||||
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
|
||||
assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
|
||||
|
||||
function findNextStyleBlockStart(source, startIndex) {
|
||||
let quote = null
|
||||
@@ -872,7 +760,7 @@ assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, '
|
||||
const { descriptor } = parseSfc(viewSource, { filename: viewPath })
|
||||
const template = descriptor.template?.content || ''
|
||||
assert.equal((template.match(/class="wizard-primary-action"/g) || []).length, 1, '页面必须只有一个主操作入口')
|
||||
assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?clearTimeout\(connectionTimer\)[\s\S]*?clearTimeout\(pullTimer\)/, '生成与外部数据源计时器没有在卸载时清理')
|
||||
assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?\}\)/, '生成轮询计时器没有在卸载时清理')
|
||||
assert.match(viewSource, /function scrollToStepTop/, '步骤切换后没有恢复页面顶部上下文')
|
||||
assert.match(viewSource, /nextTick\(scrollToStepTop\)/, '步骤切换没有触发页面滚动复位')
|
||||
assert.match(viewStyleSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏')
|
||||
|
||||
Reference in New Issue
Block a user