feat: 完成数据处理接口与前端接入

frontend/scripts/regression-data-process-wizard.mjs

@@ -4,20 +4,23 @@ import { readFile } from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import path from 'node:path'
 import { parse as parseSfc } from '@vue/compiler-sfc'
-import ts from 'typescript'
 
 const scriptDir = path.dirname(fileURLToPath(import.meta.url))
 const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
 const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
 const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue')
 const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
+const apiPath = path.resolve(scriptDir, '../src/api/modules/dataProcess.ts')
+const contractTypesPath = path.resolve(scriptDir, '../src/types/dataProcess.ts')
 const viewSource = await readFile(viewPath, 'utf8')
 const layoutSource = await readFile(layoutPath, 'utf8')
-const [draftSource, stateSource, generationSource, viewStyleSource] = await Promise.all([
+const [draftSource, stateSource, generationSource, viewStyleSource, apiSource, contractTypesSource] = await Promise.all([
   readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'),
   readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
   readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
   readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
+  readFile(apiPath, 'utf8'),
+  readFile(contractTypesPath, 'utf8'),
 ])
 const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n')
 
@@ -58,7 +61,8 @@ assert.match(
 assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化')
 assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取')
 assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿')
-assert.ok(viewSource.split('\n').length < 800, 'DataProcessCreateView 拆分后仍超过 800 行')
+assert.ok(viewSource.split('\n').length < 1000, 'DataProcessCreateView 拆分后仍超过 1000 行')
+assert.match(viewSource, /useDataProcessGeneration\(\{/, '生成流程没有拆分到独立 composable')
 
 const expectedComponents = [
   'TaskSetupStep.vue',
@@ -89,15 +93,16 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
 }
 assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
 assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
-assert.match(modelSource, /export function buildPreviewItems/, '缺少切片来源映射生成函数')
 assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
-assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件标识')
+assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
 assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
 assert.match(
   viewSource,
-  /buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/,
-  '预览没有按文件分别生成或未传入非结构化切分配置',
+  /buildDataProcessPreview\(taskId\.value,\s*\{[\s\S]*?source_file_ids:\s*uploadedFiles\.value\.map/,
+  '预览没有通过后端按已上传源文件构建',
 )
+assert.match(viewSource, /getDataProcessPreview\(taskId\.value,\s*\{ page:\s*1, page_size:\s*500 \}\)/, '预览构建后没有分页读取后端数据')
+assert.doesNotMatch(viewSource, /buildPreviewItems\(/, '创建向导仍在本地构建集成预览数据')
 
 for (const marker of [
   'preview-workspace',
@@ -186,7 +191,9 @@ assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId =
 assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：选择大模型'/, '第一步主按钮没有指向大模型选择')
 assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续：上传文件'/, '第二步主按钮没有指向上传文件')
 assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续：数据预览'/, '第三步主按钮没有指向数据预览')
-assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6/, '安全草稿格式必须升级到 v6')
+const draftVersionMatch = draftSource.match(/DATA_PROCESS_DRAFT_SCHEMA_VERSION\s*=\s*(\d+)/)
+assert.ok(draftVersionMatch, '草稿缺少数字版本标识')
+assert.ok(Number(draftVersionMatch[1]) >= 7, '安全草稿格式版本不得低于 v7')
 
 const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
 const nextFromModelStart = viewSource.indexOf('async function nextFromModel()', nextFromCreateStart)
@@ -201,11 +208,13 @@ const nextFromModelSource = viewSource.slice(nextFromModelStart, nextFromUploadS
 const nextFromUploadSource = viewSource.slice(nextFromUploadStart, selectPreviewFileStart)
 assert.match(nextFromCreateSource, /taskSetupRef\.value\?\.validate\(\)/, '创建步骤继续前没有校验任务配置')
 assert.match(nextFromCreateSource, /goToStep\('model'\)/, '创建步骤校验通过后没有进入大模型选择')
-assert.doesNotMatch(nextFromCreateSource, /uploadedFiles|buildPreviewItems/, '创建步骤仍在校验文件或提前生成预览')
+assert.doesNotMatch(nextFromCreateSource, /uploadedFiles|buildDataProcessPreview/, '创建步骤仍在校验文件或提前生成预览')
 assert.match(nextFromModelSource, /modelSelectionRef\.value\?\.validate\(\)/, '大模型选择步骤继续前没有校验模型配置')
+assert.match(nextFromModelSource, /createDataProcessTask\(taskPayload\(\)\)/, '大模型选择完成后没有通过真实 API 创建任务')
 assert.match(nextFromModelSource, /goToStep\('upload'\)/, '大模型选择完成后没有进入上传文件')
 assert.match(nextFromUploadSource, /uploadedFiles\.value\.length === 0/, '上传步骤继续前没有校验源数据')
-assert.match(nextFromUploadSource, /buildPreviewItems\(/, '上传步骤没有在进入预览前生成预览数据')
+assert.match(nextFromUploadSource, /buildDataProcessPreview\(/, '上传步骤没有调用后端构建预览')
+assert.match(nextFromUploadSource, /getDataProcessPreview\(/, '上传步骤没有读取后端预览结果')
 assert.match(nextFromUploadSource, /goToStep\('preview'\)/, '上传步骤完成后没有进入数据预览')
 assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识')
 assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成')
@@ -254,7 +263,35 @@ assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 1
 assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿')
 assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
 assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
-assert.match(generationSource, /createResults\([\s\S]*bindings\.structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
+assert.match(generationSource, /generateDataProcess\(taskId\)/, '开始生成没有调用真实 API')
+assert.match(generationSource, /getDataProcessProgress\(taskId\)/, '生成状态没有通过真实 API 轮询')
+assert.match(generationSource, /getDataProcessResults\(taskId,[\s\S]*?page:[\s\S]*?page_size:/, '生成完成后没有分页加载真实结果')
+assert.match(generationSource, /updateDataProcessResult\(taskId,\s*item\.id,[\s\S]*?expected_updated_at:/, '结果保存没有调用真实 API 或缺少并发版本')
+assert.ok(generationSource.includes('item.quality_score?.overall'), '结果映射没有读取质量总分 overall')
+assert.ok(generationSource.includes('item.quality_score?.flags || []'), '结果映射没有读取质量标记 flags')
+assert.doesNotMatch(generationSource, /createResults\(/, '生成 composable 仍在本地伪造处理结果')
+
+for (const apiName of [
+  'createDataProcessTask',
+  'uploadDataProcessSourceFiles',
+  'buildDataProcessPreview',
+  'getDataProcessPreview',
+  'generateDataProcess',
+  'getDataProcessProgress',
+  'getDataProcessResults',
+  'updateDataProcessResult',
+  'publishDataProcess',
+]) {
+  assert.match(apiSource, new RegExp(`export (?:const|async function|function) ${apiName}\\b`), `API 模块缺少 ${apiName}`)
+}
+assert.match(viewSource, /uploadDataProcessSourceFiles\(taskId\.value,\s*\[raw\]\)/, '文件上传没有调用真实 API')
+assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
+assert.match(apiSource, /\/preview\/build/, 'API 模块缺少后端预览构建路径')
+assert.match(apiSource, /\/progress`/, 'API 模块缺少生成进度路径')
+assert.match(apiSource, /\/results`/, 'API 模块缺少结果分页路径')
+assert.match(apiSource, /\/publish`/, 'API 模块缺少数据集发布路径')
+assert.match(contractTypesSource, /source_file_ids\?: Array<string \| number>/, '预览构建契约缺少源文件 ID 列表')
+assert.match(contractTypesSource, /expected_updated_at\?: string/, '编辑契约缺少乐观并发版本字段')
 
 for (const field of [
   'generationModelId',
@@ -425,156 +462,7 @@ for (const mutationFunction of [
   const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
   assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
 }
-assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置')
-assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑')
-
-const transpiledModel = ts.transpileModule(modelSource, {
-  compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
-}).outputText
-const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`)
-const longDocument = Array.from(
-  { length: 180 },
-  (_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`,
-).join('\n')
-const baseUnstructuredOptions = {
-  preprocessOptions: [],
-  chunkMethod: 'semantic',
-  chunkSize: 200,
-  chunkOverlap: 50,
-  minChunkSize: 50,
-  customDelimiter: '',
-  preserveTables: false,
-  preserveCodeBlocks: false,
-  preserveLists: false,
-  semanticEnrichment: false,
-  qaPairsPerChunk: 3,
-  datasetSplit: { train: 80, validation: 10, test: 10 },
-  generationModelId: 1,
-  generationPrompt: '仅输出问答对',
-  qualityFilterEnabled: false,
-  filterLowQuality: true,
-  filterShortContent: true,
-  minOutputLength: 20,
-}
-
-for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
-  const options = {
-    ...baseUnstructuredOptions,
-    chunkMethod,
-    customDelimiter: chunkMethod === 'custom' ? '\\n' : '',
-  }
-  const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options)
-  assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`)
-  assert.ok(
-    previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent),
-    `${chunkMethod} 切分方式的来源偏移不准确`,
-  )
-  assert.ok(
-    previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine),
-    `${chunkMethod} 切分方式的来源行号不准确`,
-  )
-}
-
-const overlapDocument = '甲'.repeat(1200)
-const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', {
-  ...baseUnstructuredOptions,
-  chunkMethod: 'fixed',
-})
-assert.equal(
-  overlapItems[0].sourceEnd - overlapItems[1].sourceStart,
-  100,
-  '固定长度切分没有按配置保留 50 个估算 Token 的重叠内容',
-)
-
-const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}`
-const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', {
-  ...baseUnstructuredOptions,
-  chunkMethod: 'heading',
-  chunkOverlap: 0,
-})
-assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片')
-assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片')
-
-const customDocument = `${'甲'.repeat(150)}<CUT>${'乙'.repeat(600)}`
-const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', {
-  ...baseUnstructuredOptions,
-  chunkMethod: 'custom',
-  chunkOverlap: 0,
-  customDelimiter: '<CUT>',
-})
-assert.ok(customItems[0].originalContent.endsWith('<CUT>'), '自定义切分未在指定分隔符处结束切片')
-
-function assertProtectedContent(optionField, block, label) {
-  const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}`
-  const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, {
-    ...baseUnstructuredOptions,
-    chunkMethod: 'fixed',
-    chunkOverlap: 0,
-    preserveTables: false,
-    preserveCodeBlocks: false,
-    preserveLists: false,
-    [optionField]: true,
-  })
-  const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, {
-    ...baseUnstructuredOptions,
-    chunkMethod: 'fixed',
-    chunkOverlap: 0,
-    preserveTables: false,
-    preserveCodeBlocks: false,
-    preserveLists: false,
-  })
-  assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`)
-  assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`)
-}
-
-const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n')
-const tableBlock = [
-  '| 字段 | 说明 |',
-  '| --- | --- |',
-  ...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`),
-].join('\n')
-const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}：这是需要完整保留的内容。`).join('\n')
-assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块')
-assertProtectedContent('preserveTables', tableBlock, '表格')
-assertProtectedContent('preserveLists', listBlock, '列表')
-
-const samplePreviewItems = previewModelModule.buildPreviewItems(
-  longDocument,
-  'unstructured',
-  'generation-check',
-  baseUnstructuredOptions,
-)
-assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片')
-const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
-assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
-
-const shortContentItems = [{
-  ...samplePreviewItems[0],
-  editedContent: '问：示例\n短回答',
-}]
-const filteredShortResults = previewModelModule.createResults(shortContentItems, {
-  ...baseUnstructuredOptions,
-  qualityFilterEnabled: true,
-  filterLowQuality: false,
-  filterShortContent: true,
-  minOutputLength: 20,
-})
-assert.equal(filteredShortResults.length, 0, '开启过短内容过滤后仍保留低于最少字数的结果')
-
-const invalidContentItems = [{
-  ...samplePreviewItems[0],
-  status: 'invalid',
-}]
-const filteredInvalidResults = previewModelModule.createResults(invalidContentItems, {
-  ...baseUnstructuredOptions,
-  qualityFilterEnabled: true,
-  filterLowQuality: true,
-  filterShortContent: false,
-})
-assert.equal(filteredInvalidResults.length, 0, '开启低质量过滤后仍保留标记为无效的结果')
-
-const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
-assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
+assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
 
 function findNextStyleBlockStart(source, startIndex) {
   let quote = null
@@ -872,7 +760,7 @@ assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, '
 const { descriptor } = parseSfc(viewSource, { filename: viewPath })
 const template = descriptor.template?.content || ''
 assert.equal((template.match(/class="wizard-primary-action"/g) || []).length, 1, '页面必须只有一个主操作入口')
-assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?clearTimeout\(connectionTimer\)[\s\S]*?clearTimeout\(pullTimer\)/, '生成与外部数据源计时器没有在卸载时清理')
+assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?\}\)/, '生成轮询计时器没有在卸载时清理')
 assert.match(viewSource, /function scrollToStepTop/, '步骤切换后没有恢复页面顶部上下文')
 assert.match(viewSource, /nextTick\(scrollToStepTop\)/, '步骤切换没有触发页面滚动复位')
 assert.match(viewStyleSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏')