fix: 完善数据预处理与 JSON 上传链路

This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions

View File

@@ -83,6 +83,11 @@ assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?d
assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐')
assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集')
assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由')
assert.match(detailSource, /const canRepeatGeneration = computed[\s\S]*?status === 'completed'[\s\S]*?results_confirmed !== false[\s\S]*?previewCount\.value > 0/, '已完成任务缺少再次生成资格判断')
assert.match(detailSource, /repeatDataProcessTask\(taskId\.value,[\s\S]*?expected_updated_at: detail\.value\.updated_at[\s\S]*?request_id: repeatRequestId\.value/, '再次生成没有携带源任务版本和幂等请求 ID')
assert.match(detailSource, /name: 'data-process-workflow'[\s\S]*?params: \{ id: repeated\.task\.id \}/, '再次生成成功后没有进入新任务工作流')
assert.match(detailSource, /原任务和原结果不会被修改/, '再次生成确认提示没有说明原任务保持不变')
assert.match(detailSource, /v-if="canRepeatGeneration"[\s\S]*?@click="repeatGeneration"[\s\S]*?按原配置再生成一批/, '已完成任务详情缺少再次生成新批次入口')
assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value[\s\S]*?hasPublishedOutputs\.value/, '详情页没有覆盖指针已清空但旧发布数据集仍存在的重新生成任务')
assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口')
assert.match(detailSource, /v-if="detail\.status === 'completed' && !hasCurrentPublishedDataset"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布或发布指针失效的完成任务没有保留发布入口')
@@ -115,6 +120,11 @@ assert.match(detailSource, /inputMetricCount\.toLocaleString\(\) \}\} \{\{ input
assert.match(detailSource, /sourceFileCount\.toLocaleString\(\) \}\} 个/, '源文件数量缺少个数单位')
assert.match(detailSource, /<span>生成结果<\/span><strong>\{\{ numeric\(detail\.output_count\)\.toLocaleString\(\) \}\} 条<\/strong>/, '生成结果数量缺少条数单位或仍误称成功输出')
assert.match(detailSource, /const configExpanded = ref\(false\)/, '处理配置没有默认收起')
assert.match(detailSource, /appendGroup\(\['clean_invalid', 'deduplicate'\], '数据清洗'\)/, '详情页没有将完整清洗配置合并为数据清洗')
assert.match(detailSource, /appendGroup\(\['detect_structure', 'normalize_format'\], '结构标准化'\)/, '详情页没有将完整结构配置合并为结构标准化')
assert.match(detailSource, /历史部分配置/, '详情页没有标识旧任务的半组选项')
assert.match(detailSource, /异常数据过滤(历史规则)/, '详情页没有标识已停用的历史异常过滤规则')
assert.match(detailSource, /new Set\(value\.map/, '详情页没有去除历史预处理配置中的重复值')
assert.match(detailSource, /:aria-expanded="configExpanded"/, '处理配置折叠按钮缺少无障碍状态')
assert.match(detailSource, /<el-collapse-transition>[\s\S]*?v-show="configExpanded"/, '处理配置没有折叠过渡或内容状态')
assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据')
@@ -126,6 +136,7 @@ for (const apiName of [
'updateDataProcessResult',
'restoreDataProcessResult',
'publishDataProcess',
'repeatDataProcessTask',
]) {
assert.match(
apiSource,
@@ -136,5 +147,8 @@ for (const apiName of [
assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数')
assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID')
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确')
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/repeat`/, '再次生成 API 路径不正确')
assert.match(typesSource, /interface DataProcessRepeatPayload[\s\S]*?expected_updated_at: string[\s\S]*?request_id: string/, '再次生成请求契约不完整')
assert.match(typesSource, /interface DataProcessRepeatResult[\s\S]*?task: DataProcessTask[\s\S]*?source_task_id: string[\s\S]*?created: boolean/, '再次生成响应契约不完整')
console.log('数据处理任务详情真实 API 回归检查通过')

View File

@@ -183,8 +183,36 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
assert.ok(typesSource.includes(field), `PreviewItem 缺少字段:${field}`)
}
assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
assert.match(typesSource, /sourceLocator\?: PreviewSourceLocator/, 'PreviewItem 缺少结构化来源定位契约')
assert.match(typesSource, /headingPath\?: string\[\]/, 'PreviewItem 缺少非结构化标题路径')
assert.match(typesSource, /PreviewSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, '前端来源定位 kind 未使用明确联合类型')
assert.match(contractTypesSource, /DataProcessSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, 'API 来源定位 kind 未使用明确联合类型')
for (const field of ['kind', 'record_index', 'start_line', 'end_line', 'source_start', 'source_end', 'json_pointer', 'sheet_index', 'sheet_name', 'row_number', 'sheet_record_index']) {
assert.ok(typesSource.includes(field), `PreviewSourceLocator 缺少字段:${field}`)
assert.ok(contractTypesSource.includes(field), `后端来源定位契约缺少字段:${field}`)
}
assert.match(contractTypesSource, /source_locator\?: DataProcessSourceLocator/, '质量信息缺少来源定位契约')
assert.match(contractTypesSource, /heading_path\?: string\[\]/, '质量信息缺少标题路径契约')
assert.match(viewSource, /const sourceLocator = item\.quality_score\?\.source_locator/, '预览映射丢失来源定位')
assert.match(viewSource, /sourceStart:\s*item\.source_start\s*\?\?\s*sourceLocator\?\.source_start/, 'JSON locator 的字符起点没有映射到预览项')
assert.match(viewSource, /sourceEnd:\s*item\.source_end\s*\?\?\s*sourceLocator\?\.source_end/, 'JSON locator 的字符终点没有映射到预览项')
assert.match(viewSource, /sourceStartLine:\s*item\.source_start_line\s*\?\?\s*sourceLocator\?\.start_line/, 'JSON locator 的起始行没有映射到预览项')
assert.match(viewSource, /sourceEndLine:\s*item\.source_end_line\s*\?\?\s*sourceLocator\?\.end_line/, 'JSON locator 的结束行没有映射到预览项')
assert.match(viewSource, /headingPath:[\s\S]*?item\.quality_score\?\.heading_path/, '预览映射丢失标题路径')
assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
assert.match(modelSource, /export function sourceLineWindow/, '缺少有界源文件行窗口函数')
assert.match(modelSource, /maxLines:\s*number/, '源文件行窗口缺少最大渲染行数参数')
assert.doesNotMatch(modelSource, /\.split\(\s*['"]\\n['"]\s*\)/, '源文件行窗口仍会先对全文 split')
assert.match(modelSource, /lines\.length < limit/, '源文件行扫描没有受最大行数约束')
assert.match(modelSource, /unicodeCodePointLength/, '源文件字符偏移未与后端 Unicode code point 计数保持一致')
assert.match(modelSource, /export function sourceLineNumberAtOffset/, '字符偏移缺少无数组的行号解析函数')
const manualPreviewHelperStart = modelSource.indexOf('export function isManualPreviewItem(')
const manualPreviewHelperEnd = modelSource.indexOf('\n}', manualPreviewHelperStart)
assert.ok(manualPreviewHelperStart >= 0, '缺少统一的手动预览项判定函数')
const manualPreviewHelperSource = modelSource.slice(manualPreviewHelperStart, manualPreviewHelperEnd + 2)
for (const field of ['status', 'originalContent', 'sourceStart', 'sourceEnd', 'sourceStartLine', 'sourceEndLine', 'sourcePages', 'sourceLocator']) {
assert.ok(manualPreviewHelperSource.includes(field), `手动预览项判定缺少来源字段:${field}`)
}
assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
const previewBuildBindingStart = viewSource.indexOf('useDataProcessPreviewBuild()')
@@ -210,6 +238,30 @@ for (const marker of [
}
assert.match(previewSource, /sourceStart/, '第四步未使用来源起始偏移')
assert.match(previewSource, /sourceEnd/, '第四步未使用来源结束偏移')
const lineRangeStart = previewSource.indexOf('function lineRange(item: PreviewItem)')
const lineRangeEnd = previewSource.indexOf('\n}', lineRangeStart)
const lineRangeSource = previewSource.slice(lineRangeStart, lineRangeEnd + 2)
assert.match(lineRangeSource, /isManualPreviewItem\(item\)[\s\S]*?手动新增,无源文件定位/, '来源标签仍会把缺少行偏移的正常记录误判为手动新增')
assert.match(lineRangeSource, /props\.processType === 'unstructured'[\s\S]*?来源:源文件记录/, '结构化来源记录缺少无行偏移时的准确标签')
assert.doesNotMatch(lineRangeSource, /sourceStartLine == null[^\n]*手动新增/, '来源标签仍直接以缺少行号判定手动新增')
assert.match(lineRangeSource, /sheet_name[\s\S]*?row_number[\s\S]*?来源:\$\{sheet\} · 第 \$\{locator\.row_number\} 行/, 'XLSX 来源标签没有展示工作表和物理行号')
assert.match(lineRangeSource, /json_pointer[\s\S]*?JSON 路径/, 'JSON 来源标签没有展示 JSON 路径')
assert.match(lineRangeSource, /locator\?\.kind === 'json'[\s\S]*?JSON 根对象/, 'JSON 根对象来源标签被空 JSON Pointer 错误降级')
assert.match(lineRangeSource, /locatedLines[\s\S]*?第 \$\{locatedLines\.start\}[\s\S]*?locatedLines\.end/, 'JSONL/CSV 来源标签没有展示行范围')
assert.match(lineRangeSource, /headingPath[\s\S]*?章节:/, '非结构化来源标签没有合并标题路径')
assert.match(previewSource, /sourceLocator\?\.start_line[\s\S]*?sourceLocator\?\.end_line/, '文本预览没有优先使用后端行号定位')
assert.match(previewSource, /sourceLocator\?\.source_start\s*\?\?\s*item\.sourceStart/, '文本预览没有优先使用 locator 字符起点')
assert.match(previewSource, /sourceLocator\?\.source_end\s*\?\?\s*item\.sourceEnd/, '文本预览没有优先使用 locator 字符终点')
assert.match(previewSource, /data-line-number="line\.number"/, '文本预览行缺少稳定行号定位标识')
assert.match(previewSource, /isLineHighlighted\(line\.number, line\.start, line\.end\)/, '文本预览没有按物理行号高亮')
assert.match(previewSource, /querySelector<HTMLElement>\(`\[data-line-number=/, '选中记录后没有按物理行号滚动定位')
assert.match(previewSource, /const SOURCE_LINE_RENDER_LIMIT = 240/, '源文件查看器缺少安全渲染上限')
assert.match(previewSource, /const SOURCE_LINE_CHARACTER_LIMIT = 4_000/, '源文件查看器缺少单行字符渲染上限')
assert.match(previewSource, /sourceLineWindow\([\s\S]*?SOURCE_LINE_RENDER_LIMIT/, '源文件查看器没有使用有界行窗口')
assert.match(previewSource, /SOURCE_LINE_RENDER_LIMIT,[\s\S]*?SOURCE_LINE_CHARACTER_LIMIT,[\s\S]*?selectedSourceLine\.value,[\s\S]*?selectedSourceOffset\.value/, '单行超大 JSON 没有围绕选中来源构建字符窗口')
assert.match(previewSource, /sourceWindowStartLine/, '源文件查看器缺少窗口起始行状态')
assert.match(previewSource, /showPreviousSourceWindow[\s\S]*?showNextSourceWindow/, '源文件查看器缺少前后窗口导航')
assert.match(previewSource, /sourceLineNumberAtOffset\(props\.sourceText/, '仅有字符偏移时没有解析目标物理行')
assert.match(previewSource, /filterable/, '文件选择器必须可搜索')
assert.match(previewSource, /当前文件/, '预览缺少当前文件切换器')
assert.doesNotMatch(previewSource, /located-badge|sync-label|已定位到/, '源文件栏不应显示冗余定位提示')
@@ -269,6 +321,18 @@ for (const marker of [
]) {
assert.ok(officeViewerSource.includes(marker), `Word/XLSX 预览缺少结构或行为:${marker}`)
}
assert.match(officeViewerSource, /const selectedXlsxLocator = computed/, 'XLSX 查看器没有读取精确来源定位')
assert.match(officeViewerSource, /row\.row_number === locator\.row_number/, 'XLSX 查看器没有按物理行号精确高亮')
assert.match(officeViewerSource, /row\.record_index === locator\.sheet_record_index/, 'XLSX 查看器没有按工作表记录序号精确高亮')
assert.match(officeViewerSource, /Math\.floor\(locator\.sheet_record_index \/ XLSX_PAGE_SIZE\) \* XLSX_PAGE_SIZE/, 'XLSX 查看器没有按记录序号自动计算分页')
assert.match(officeViewerSource, /activeSheetIndex\.value = targetSheet[\s\S]*?pageOffset\.value = targetOffset[\s\S]*?loadPreview\(\)/, '切换记录时 XLSX 查看器没有自动切工作表和分页')
const xlsxHighlightStart = officeViewerSource.indexOf('function xlsxRowHighlighted(')
const xlsxHighlightEnd = officeViewerSource.indexOf('\n}', xlsxHighlightStart)
const xlsxHighlightSource = officeViewerSource.slice(xlsxHighlightStart, xlsxHighlightEnd + 2)
assert.ok(
xlsxHighlightSource.indexOf('locator.row_number') < xlsxHighlightSource.indexOf('selectedRecordKey.value'),
'XLSX 查看器没有把精确定位放在原内容比对 fallback 之前',
)
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
@@ -454,7 +518,7 @@ assert.match(
)
assert.match(
workflowInitializationSource,
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?goToStep\(resumeStep\)[\s\S]*?resumeGeneration/,
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)/,
'生成运行中时没有强制回到第五步并接管后台进度',
)
const startGenerationHandler = viewSource.slice(
@@ -463,7 +527,35 @@ const startGenerationHandler = viewSource.slice(
)
assert.match(startGenerationHandler, /await persistWorkflowStep\('generate'\)[\s\S]*?await startGeneration\(\)[\s\S]*?dirty\.value = false/, '开始生成没有持久化第五步或启动真实后台任务')
assert.doesNotMatch(startGenerationHandler, /router\.(?:push|replace)|allowLeave\s*=\s*true/, '开始生成后应停留在第五步,不得自动跳回列表')
assert.match(viewSource, /:disabled="currentStepId === 'generate' \|\| previewBuilding \|\| sourceUploading"/, '第五步底部返回按钮没有固定禁用')
assert.match(
generationSource,
/const canReturnFromGeneration = computed\(\(\) => \([\s\S]*?generation\.status === 'idle'[\s\S]*?!generationStarting\.value[\s\S]*?!generationRestoring\.value/,
'第五步返回权限没有区分未启动、启动中和恢复中状态',
)
assert.match(
viewSource,
/:disabled="\(currentStepId === 'generate' && !canReturnFromGeneration\) \|\| previewBuilding \|\| sourceUploading"/,
'第五步尚未启动生成时返回按钮仍被禁用',
)
const handleBackStart = viewSource.indexOf('async function handleBack()')
const handleBackEnd = viewSource.indexOf('\n}', handleBackStart)
const handleBackSource = viewSource.slice(handleBackStart, handleBackEnd + 2)
assert.match(
handleBackSource,
/currentStepId\.value === 'generate' && !canReturnFromGeneration\.value/,
'第五步处理函数仍无条件拦截返回',
)
assert.match(
generationSource,
/async function resumeGeneration\(\)[\s\S]*?generationRestoring\.value = true[\s\S]*?await getDataProcessProgress\(taskId\)[\s\S]*?generationRestoring\.value = false/,
'恢复已启动任务时存在短暂可返回的 idle 窗口',
)
assert.match(viewSource, /const resume = resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)[\s\S]*?await resume/, '第五步展示时未先启动恢复锁')
assert.match(
generationSource,
/const generationStarting = ref\(false\)[\s\S]*?generationStarting\.value = true[\s\S]*?generationStarting\.value = false/,
'点击开始生成后到请求启动前没有锁定返回状态',
)
assert.match(
viewSource,
/generation\.status === 'success'[\s\S]*?persistWorkflowStep\('results'\)/,
@@ -506,32 +598,34 @@ assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTy
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
const expectedStructuredOptions = [
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
const expectedStructuredGroups = [
[
'detect_structure',
'嵌套结构展平',
'展平嵌套对象和可解析的 JSON 字段Excel 表头与合并单元格在上传时自动解析',
"values: ['clean_invalid', 'deduplicate']",
'数据清洗',
'清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
],
[
'deduplicate',
'重复记录去重',
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
"values: ['detect_structure', 'normalize_format']",
'结构标准化',
'展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
],
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
["values: ['desensitize']", '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
]
for (const [value, label, description] of expectedStructuredOptions) {
assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值${value}`)
for (const [values, label, description] of expectedStructuredGroups) {
assert.ok(structuredOptionsSource.includes(values), `结构化预处理组合值不准确${label}`)
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${label}`)
}
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
.map((match) => match[1])
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重')
assert.equal(expectedStructuredGroups.length, 3, '结构化预处理应收敛为 3 项')
const preprocessGroupsSource = structuredOptionsSource.slice(
structuredOptionsSource.indexOf('const PREPROCESS_GROUPS'),
structuredOptionsSource.indexOf('const legacyAnomalyFilterEnabled'),
)
assert.doesNotMatch(preprocessGroupsSource, /异常数据过滤|filter_anomaly|IQR/, '结构化新任务仍暴露异常数据过滤')
assert.match(structuredOptionsSource, /:indeterminate="groupIndeterminate\(group\.values\)"/, '历史部分选中的组合项没有半选回显')
assert.match(structuredOptionsSource, /function updatePreprocessGroup\([\s\S]*?new Set\(props\.options\.preprocessOptions\)[\s\S]*?next\.add\(value\)[\s\S]*?next\.delete\(value\)[\s\S]*?\[\.\.\.next\]/, '结构化预处理组合开关没有原子化更新或去重内部选项')
assert.match(typesSource, /仅用于恢复历史任务[\s\S]*?\| 'filter_anomaly'/, '异常数据过滤缺少历史兼容类型')
assert.match(structuredOptionsSource, /legacyAnomalyFilterEnabled[\s\S]*?历史任务[\s\S]*?结果可复现/, '历史异常过滤配置没有透明提示')
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
for (const splitName of ['训练集', '验证集', '测试集']) {
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
@@ -585,8 +679,20 @@ for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json',
}
assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别')
assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示')
assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验')
assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本')
const sourceValidationStart = sourceUploadWorkerSource.indexOf('export function validateSourceFileSelection(')
const sourceValidationEnd = sourceUploadWorkerSource.indexOf('\n}\n\nfunction unicodeCodePointLength', sourceValidationStart)
assert.ok(sourceValidationStart >= 0 && sourceValidationEnd > sourceValidationStart, '无法定位源文件选择校验函数')
const sourceValidationSource = sourceUploadWorkerSource.slice(sourceValidationStart, sourceValidationEnd + 2)
assert.doesNotMatch(sourceValidationSource, /file\.name === raw\.name[\s\S]{0,160}file\.size === raw\.size|同名且同大小/, '不同内容但同名同大小的文件仍会被前端误拒绝')
assert.match(sourceValidationSource, /selectedFiles\.length >= MAX_SOURCE_FILE_COUNT/, '移除伪重复校验时误删了文件数量限制')
assert.match(sourceValidationSource, /selectedBytes \+ raw\.size > MAX_SOURCE_BATCH_BYTES/, '移除伪重复校验时误删了批次大小限制')
assert.doesNotMatch(sourceUploadWorkerSource, /job\.file\.arrayBuffer\(|new TextDecoder/, '上传前仍把整个文本文件读入浏览器内存')
assert.match(sourceUploadWorkerSource, /export async function loadCanonicalSourceContent[\s\S]*?offset,[\s\S]*?limit: SOURCE_CONTENT_PAGE_CHARS/, '服务端 canonical content 没有按有界字符窗口读取')
assert.match(sourceUploadWorkerSource, /pending\.content = await loadCanonicalSourceContent\(currentTaskId, source\.id\)/, '上传成功后没有统一使用服务端 canonical content')
assert.doesNotMatch(sourceUploadWorkerSource, /\brawFile:\s*job\.file\b/, '上传成功状态仍长期保留原始 File')
assert.doesNotMatch(typesSource, /\brawFile\??:\s*File\b/, '上传状态类型仍长期持有原始 File')
assert.doesNotMatch(viewSource, /\brawFile:\s*raw\b/, '待上传列表仍复制保存原始 File')
assert.match(apiSource, /params:\s*\{[\s\S]*?offset\?: number[\s\S]*?limit\?: number[\s\S]*?\}/, '正文 API 前端契约缺少字符窗口参数')
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%')
assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时')
@@ -609,7 +715,7 @@ assert.match(
/export interface DataProcessPreviewProgress[\s\S]*?workflow_step: DataProcessWorkflowStep[\s\S]*?preview_status: DataProcessPreviewStatus[\s\S]*?preview_progress: number[\s\S]*?preview_run_id/,
'后台切分进度契约缺少步骤、状态、进度或任务代次',
)
for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
for (const field of ['status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段:${field}`)
}
assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整')
@@ -853,13 +959,23 @@ const defaultStructuredPreprocess = defaultPreprocessValues(
)
assert.deepEqual(
defaultStructuredPreprocess,
['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
'结构化默认预处理配置不准确',
[],
'结构化新任务不应默认勾选预处理',
)
assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复')
const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions')
assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整')
assert.deepEqual(defaultUnstructuredPreprocess, [], '非结构化新任务不应默认勾选预处理')
assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复')
for (const field of ['preserveTables', 'preserveCodeBlocks', 'preserveLists']) {
assert.match(
stateSource,
new RegExp(`${field}:\\s*false`),
`非结构化预处理选项 ${field} 不应默认开启`,
)
}
assert.match(structuredOptionsSource, /默认不执行预处理,请按数据情况自行选择/, '结构化预处理缺少默认不勾选说明')
assert.match(unstructuredOptionsSource, /默认不执行预处理,请按文档情况自行选择/, '非结构化预处理缺少默认不勾选说明')
assert.doesNotMatch(unstructuredOptionsSource, /默认启用结构感知/, '非结构化预处理仍保留默认启用的误导文案')
const backendConfigStart = viewSource.indexOf('function toBackendConfig()')
const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart)
@@ -934,7 +1050,7 @@ for (const [field, fallback] of [
)
}
assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务')
assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文')
assert.match(regenerationSource, /loadCanonicalSourceContent\(taskId, file\.id\)/, '重新生成没有复用分页 canonical 正文加载器')
assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片')
assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用')
assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重')
@@ -948,7 +1064,7 @@ assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?g
assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态')
assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片')
assert.match(regenerationSource, /重新生成配置已保存,但工作区恢复失败/, '重新生成配置已保存但水合失败时缺少可恢复错误状态')
assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
assert.match(sourceUploadWorkerSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容')
assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口')
assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新')
@@ -1017,6 +1133,17 @@ for (const mutationFunction of [
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
}
const updatePreviewContentStart = viewSource.indexOf('function updatePreviewContent(')
const updatePreviewContentEnd = viewSource.indexOf('\n}', updatePreviewContentStart)
const updatePreviewContentSource = viewSource.slice(updatePreviewContentStart, updatePreviewContentEnd + 2)
assert.match(updatePreviewContentSource, /isManualPreviewItem\(item\)/, '编辑预览内容仍未按稳定来源信息区分手动项')
assert.doesNotMatch(updatePreviewContentSource, /sourceStart == null/, '结构化来源记录编辑后仍会被误标为手动项')
const restorePreviewItemStart = viewSource.indexOf('function restorePreviewItem(')
const restorePreviewItemEnd = viewSource.indexOf('\n}', restorePreviewItemStart)
const restorePreviewItemSource = viewSource.slice(restorePreviewItemStart, restorePreviewItemEnd + 2)
assert.match(restorePreviewItemSource, /isManualPreviewItem\(item\)/, '恢复预览内容没有使用统一的手动项判定')
assert.doesNotMatch(restorePreviewItemSource, /sourceStart == null/, '结构化来源记录仍因缺少字符偏移而无法恢复')
assert.match(previewSource, /v-if="!isManualPreviewItem\(editingItem\)"/, '结构化来源记录的恢复原文按钮仍被错误隐藏')
assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
function findNextStyleBlockStart(source, startIndex) {

View File

@@ -14,6 +14,8 @@ import type {
DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessRepeatPayload,
DataProcessRepeatResult,
DataProcessPublishPayload,
DataProcessPublishResult,
DataProcessQualityScore,
@@ -56,6 +58,8 @@ export type {
DataProcessProgress,
DataProcessRegeneratePayload,
DataProcessRegenerateResult,
DataProcessRepeatPayload,
DataProcessRepeatResult,
DataProcessPublishPayload,
DataProcessPublishResult,
DataProcessQualityScore,
@@ -117,6 +121,15 @@ export const regenerateDataProcessTask = (
payload,
)
export const repeatDataProcessTask = (
taskId: string | number,
payload: DataProcessRepeatPayload,
) => post<DataProcessRepeatResult>(
`/data-process/${encodeURIComponent(taskId)}/repeat`,
payload,
{ timeout: 5 * 60 * 1000 },
)
export const deleteDataProcessTask = (taskId: string | number) =>
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
@@ -150,7 +163,12 @@ export const deleteDataProcessSourceFile = (taskId: string | number, fileId: str
export const getDataProcessSourceContent = (
taskId: string | number,
fileId: string | number,
params: { start_line?: number; line_count?: number } = {},
params: {
start_line?: number
line_count?: number
offset?: number
limit?: number
} = {},
) => get<DataProcessSourceContent>(
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`,
params,

View File

@@ -99,6 +99,20 @@ export interface DataProcessRegenerateResult {
published_outputs_preserved: boolean
}
export interface DataProcessRepeatPayload {
expected_updated_at: string
request_id: string
}
export interface DataProcessRepeatResult {
task: DataProcessTask
source_task_id: string
created: boolean
copied_source_file_count: number
copied_preview_count: number
progress: DataProcessProgress
}
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
export interface DataProcessSourceFile {
@@ -232,6 +246,22 @@ export interface DataProcessPreviewItem {
updated_at?: string
}
export type DataProcessSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
export interface DataProcessSourceLocator {
kind: DataProcessSourceLocatorKind
record_index?: number | null
start_line?: number | null
end_line?: number | null
source_start?: number | null
source_end?: number | null
json_pointer?: string | null
sheet_index?: number | null
sheet_name?: string | null
row_number?: number | null
sheet_record_index?: number | null
}
export interface DataProcessPreviewBuildPayload {
replace_existing?: true
source_file_ids?: Array<string | number>
@@ -369,6 +399,9 @@ export interface DataProcessQualityScore {
is_valid?: boolean
flags?: string[]
fingerprint?: string
source_pages?: number[]
heading_path?: string[]
source_locator?: DataProcessSourceLocator
[key: string]: unknown
}

View File

@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
useDataProcessSourceUpload,
validateSourceFileSelection,
@@ -33,7 +34,6 @@ import {
deleteDataProcessPreview,
deleteDataProcessSourceFile,
getDataProcessPreview,
getDataProcessSourceContent,
pullDataProcessExternalSource,
testDataProcessExternalSource,
updateDataProcessPreview,
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
let allowLeave = false
const {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
return 'fa-arrow-right'
})
const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title
: '')
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
}
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
const sourceLocator = item.quality_score?.source_locator
return {
id: String(item.id),
sourceFileId: String(item.source_file_id),
originalContent: item.original_content,
editedContent: item.edited_content,
savedEditedContent: item.edited_content,
sourceStart: item.source_start,
sourceEnd: item.source_end,
sourceStartLine: item.source_start_line,
sourceEndLine: item.source_end_line,
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
tokenCount: item.token_count,
status: item.status,
sourcePages: Array.isArray(item.quality_score?.source_pages)
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
: [],
sourceLocator,
headingPath: Array.isArray(item.quality_score?.heading_path)
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
: [],
updatedAt: item.updated_at,
}
}
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
uploadedFiles.value.push({
uid: localUid,
rawFile: raw,
name: raw.name,
size: raw.size,
count: 0,
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
previewProgress: 0,
})
dirty.value = true
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
enqueueSourceUpload({ uid: localUid, file: raw })
}
async function useSampleFile() {
@@ -488,11 +493,8 @@ async function handlePullData() {
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
const newFiles: UploadedDataFile[] = []
for (const file of response.files) {
const source = await getDataProcessSourceContent(taskId.value, file.id, {
start_line: 1,
line_count: 5000,
})
newFiles.push(mapDataProcessSourceFile(file, source.content))
const content = await loadCanonicalSourceContent(taskId.value, file.id)
newFiles.push(mapDataProcessSourceFile(file, content))
}
uploadedFiles.value.push(...newFiles)
externalConnected.value = true
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
function updatePreviewContent(id: string, value: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item) return
const isManual = isManualPreviewItem(item)
item.editedContent = value
item.tokenCount = estimateTokenCount(value)
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
item.status = !value.trim()
? 'invalid'
: value === item.originalContent
? 'original'
: isManual ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
function restorePreviewItem(id: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item || item.sourceStart == null) return
if (!item || isManualPreviewItem(item)) return
item.editedContent = item.originalContent
item.tokenCount = estimateTokenCount(item.originalContent)
item.status = 'original'
@@ -897,7 +904,7 @@ async function handleBack() {
ElMessage.warning('请等待当前文件切分完成')
return
}
if (currentStepId.value === 'generate') return
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
if (currentStep.value > 0) {
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
if (!targetStep) return
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
if (sourceTask.status === 'running') resumeStep = 'generate'
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
if (resumeStep === 'generate' || resumeStep === 'results') {
const resume = resumeGeneration()
goToStep(resumeStep)
await resume
return
}
goToStep(resumeStep)
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
}
onBeforeUnmount(() => {
@@ -1154,7 +1166,7 @@ onMounted(() => {
<div class="footer-left">
<el-button
v-if="currentStep > 0"
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
@click="handleBack"
>
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回{{ previousStepLabel }}
@@ -1167,8 +1179,8 @@ onMounted(() => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

View File

@@ -9,6 +9,7 @@ import {
getDataProcessResults,
getDataProcessTask,
publishDataProcess,
repeatDataProcessTask,
restoreDataProcessResult,
updateDataProcessResult,
} from '@/api/modules/dataProcess'
@@ -42,6 +43,8 @@ const savingResult = ref(false)
const restoringResultId = ref<string | number | null>(null)
const publishDialogVisible = ref(false)
const publishing = ref(false)
const repeatGenerating = ref(false)
const repeatRequestId = ref('')
const configExpanded = ref(false)
const resultCellTooltipOptions = {
popperClass: 'data-process-result-tooltip',
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
preserve_context: '保留上下文',
}
const structuredPreprocessOptionKeys = new Set([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
function formatStructuredPreprocessOptions(value: unknown[]) {
const options = [...new Set(value.map((item) => String(item)))]
const selected = new Set(options)
const consumed = new Set<string>()
const labels: string[] = []
function appendGroup(values: string[], groupLabel: string) {
const selectedValues = values.filter((item) => selected.has(item))
selectedValues.forEach((item) => consumed.add(item))
if (selectedValues.length === values.length) {
labels.push(groupLabel)
return
}
selectedValues.forEach((item) => {
labels.push(`${preprocessOptionLabelMap[item] || item}(历史部分配置)`)
})
}
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
if (selected.has('desensitize')) {
consumed.add('desensitize')
labels.push('敏感信息脱敏')
}
if (selected.has('filter_anomaly')) {
consumed.add('filter_anomaly')
labels.push('异常数据过滤(历史规则)')
}
options.forEach((item) => {
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
})
return labels.length ? labels.join('、') : '-'
}
function numeric(value: unknown) {
const parsed = typeof value === 'number' ? value : Number(value)
return Number.isFinite(parsed) ? parsed : 0
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|| status === 'stopped'
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
})
const canRepeatGeneration = computed(() => (
detail.value?.status === 'completed'
&& detail.value.results_confirmed !== false
&& previewCount.value > 0
))
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
}
if (Array.isArray(value)) {
if (key === 'preprocess_options') {
return value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
const containsStructuredOption = value.some((item) => (
structuredPreprocessOptionKeys.has(String(item))
))
return containsStructuredOption
? formatStructuredPreprocessOptions(value)
: value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
}
return value.length ? value.join('、') : '-'
}
@@ -503,6 +561,48 @@ function startRegeneration() {
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
}
function createRepeatRequestId() {
if (typeof globalThis.crypto?.randomUUID === 'function') {
return globalThis.crypto.randomUUID()
}
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
}
async function repeatGeneration() {
if (!detail.value?.updated_at || repeatGenerating.value) return
try {
await ElMessageBox.confirm(
'系统会复制当前配置、源文件和切分结果,创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
'按原配置再生成一批?',
{
confirmButtonText: '创建并开始生成',
cancelButtonText: '取消',
type: 'info',
},
)
} catch {
return
}
repeatGenerating.value = true
repeatRequestId.value ||= createRepeatRequestId()
try {
const repeated = await repeatDataProcessTask(taskId.value, {
expected_updated_at: detail.value.updated_at,
request_id: repeatRequestId.value,
})
ElMessage.success(repeated.created ? '已创建新任务,正在后台生成' : '已恢复此前创建的新任务')
await router.push({
name: 'data-process-workflow',
params: { id: repeated.task.id },
})
} catch {
// 保留幂等请求 ID网络超时后再次点击不会重复创建任务。
} finally {
repeatGenerating.value = false
}
}
watch([currentPage, pageSize], () => void loadResults())
onMounted(loadPage)
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
<el-tag :type="displayStatus.type" size="small" effect="light">
{{ displayStatus.label }}
</el-tag>
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
class="publish-button"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRegenerate"
class="publish-button"
type="primary"
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
</el-button>
<div class="heading-actions">
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRepeatGeneration"
type="primary"
:loading="repeatGenerating"
:disabled="repeatGenerating"
@click="repeatGeneration"
>
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
</el-button>
<el-button
v-if="canRegenerate"
type="warning"
plain
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
</el-button>
</div>
</div>
<p>{{ detail.description || '暂无任务描述' }}</p>
<dl class="heading-meta">
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
}
.publish-button { margin-left: auto; }
.heading-actions {
margin-left: auto;
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 8px;
:deep(.el-button + .el-button) { margin-left: 0; }
}
.load-state-actions { display: flex; gap: 10px; }
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
@media (max-width: 720px) {
.metric-grid, .config-grid { grid-template-columns: 1fr; }
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
.publish-button { width: 100%; margin-left: 0; }
.heading-actions { width: 100%; margin-left: 0; }
.heading-actions :deep(.el-button) { width: 100%; }
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
.result-toolbar { align-items: stretch; flex-direction: column; }
.result-filters { padding: 0 16px 16px; flex-direction: column; }

View File

@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
: ''
))
const selectedXlsxLocator = computed(() => {
const locator = props.selectedItem?.sourceLocator
if (!locator) return null
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
const hasRow = locator.row_number != null || locator.sheet_record_index != null
return hasSheet && hasRow ? locator : null
})
const visibleRowRange = computed(() => {
const sheet = xlsxPreview.value?.active_sheet
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
})
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
const locator = selectedXlsxLocator.value
const sheet = xlsxPreview.value?.active_sheet
if (locator && sheet) {
const sheetMatches = locator.sheet_index != null
? sheet.index === locator.sheet_index
: sheet.name === locator.sheet_name
if (!sheetMatches) return false
if (locator.row_number != null) return row.row_number === locator.row_number
return row.record_index === locator.sheet_record_index
}
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
}
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
async function loadPreview(options: { reset?: boolean } = {}) {
const sequence = ++loadSequence
if (options.reset) {
activeSheetIndex.value = 0
pageOffset.value = 0
const locator = selectedXlsxLocator.value
activeSheetIndex.value = locator?.sheet_index ?? 0
pageOffset.value = locator?.sheet_record_index == null
? 0
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
preview.value = null
}
errorMessage.value = ''
@@ -178,8 +198,34 @@ watch(
)
watch(
() => props.selectedItem?.id,
() => void locateSelectedItem(),
() => [
props.selectedItem?.id,
props.selectedItem?.sourceLocator?.sheet_index,
props.selectedItem?.sourceLocator?.sheet_record_index,
props.selectedItem?.sourceLocator?.row_number,
],
() => {
const locator = selectedXlsxLocator.value
if (!locator || isDocx.value) {
void locateSelectedItem()
return
}
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
const targetOffset = locator.sheet_record_index == null
? pageOffset.value
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
const activeSheet = xlsxPreview.value?.active_sheet
if (
activeSheet?.index === targetSheet
&& activeSheet.offset === targetOffset
) {
void locateSelectedItem()
return
}
activeSheetIndex.value = targetSheet
pageOffset.value = targetOffset
void loadPreview()
},
)
</script>
@@ -301,6 +347,8 @@ watch(
:key="row.row_number"
class="xlsx-row"
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
:data-row-number="row.row_number"
:data-record-index="row.record_index"
>
<th class="row-number-cell">{{ row.row_number }}</th>
<td

View File

@@ -2,7 +2,11 @@
import { computed, nextTick, ref, watch } from 'vue'
import OfficeSourceViewer from './OfficeSourceViewer.vue'
import PdfSourceViewer from './PdfSourceViewer.vue'
import { sourceLines } from './previewModel'
import {
isManualPreviewItem,
sourceLineNumberAtOffset,
sourceLineWindow,
} from './previewModel'
import type { PreviewItem, ProcessType } from './types'
const props = defineProps<{
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
const search = ref('')
const currentPage = ref(1)
const PREVIEW_PAGE_SIZE = 10
const SOURCE_LINE_RENDER_LIMIT = 240
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
const sourceWindowStartLine = ref(1)
const editingItemId = ref<string | null>(null)
const editorDraft = ref('')
const lines = computed(() => sourceLines(props.sourceText))
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
const normalizedFileFormat = computed(() => (
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
))
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
const selectedSourceOffset = computed(() => {
const item = selectedItem.value
return item ? sourceOffsetRange(item)?.start ?? null : null
})
const selectedSourceLine = computed(() => {
const item = selectedItem.value
if (!item) return null
return sourceLineRange(item)?.start
?? (selectedSourceOffset.value == null
? null
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
})
const visibleSourceWindow = computed(() => sourceLineWindow(
props.sourceText,
sourceWindowStartLine.value,
SOURCE_LINE_RENDER_LIMIT,
SOURCE_LINE_CHARACTER_LIMIT,
selectedSourceLine.value,
selectedSourceOffset.value,
))
const lines = computed(() => visibleSourceWindow.value.lines)
const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !search.value.trim()
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
function isLineHighlighted(lineStart: number, lineEnd: number) {
function sourceLineRange(item: PreviewItem) {
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
return start == null ? null : { start, end: end ?? start }
}
function sourceOffsetRange(item: PreviewItem) {
const start = item.sourceLocator?.source_start ?? item.sourceStart
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
return start == null ? null : { start, end: Math.max(start, end ?? start) }
}
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
const item = selectedItem.value
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
if (!item) return false
const lineRange = sourceLineRange(item)
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
const offsetRange = sourceOffsetRange(item)
if (!offsetRange) return false
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
}
function selectItem(id: string) {
@@ -107,34 +151,88 @@ watch(search, () => {
watch(() => props.selectedFileId, closeEditor)
watch(selectedItem, async (item) => {
watch([selectedItem, () => props.sourceText], async ([item]) => {
if (!item) return
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
if (visibleIndex >= 0) {
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
}
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
if (isPdfSource.value || isOfficeSource.value) return
const itemLineRange = sourceLineRange(item)
const itemOffsetRange = sourceOffsetRange(item)
if (!itemLineRange && !itemOffsetRange) {
sourceWindowStartLine.value = 1
return
}
const targetLine = selectedSourceLine.value
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
await nextTick()
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
const exactTarget = sourceViewerRef.value
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
const target = exactTarget
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
}, { immediate: true })
async function showPreviousSourceWindow() {
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
async function showNextSourceWindow() {
if (!visibleSourceWindow.value.hasMore) return
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
function itemNumber(item: PreviewItem) {
return props.items.findIndex((entry) => entry.id === item.id) + 1
}
function lineRange(item: PreviewItem) {
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
return first === last ? `来源:第 ${first}` : `来源:第 ${first}${last}`
if (isManualPreviewItem(item)) return '手动新增,无源文件定位'
const locator = item.sourceLocator
const locatedLines = sourceLineRange(item)
if (props.processType === 'unstructured') {
const parts: string[] = []
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
parts.push(first === last ? `${first}` : `${first}${last}`)
}
if (locatedLines) {
parts.push(
locatedLines.start === locatedLines.end
? `${locatedLines.start}`
: `${locatedLines.start}${locatedLines.end}`,
)
}
if (item.headingPath?.length) parts.push(`章节:${item.headingPath.join(' / ')}`)
return parts.length ? `来源:${parts.join(' · ')}` : '来源:源文件内容(无精确定位)'
}
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增,无源文件定位'
return item.sourceStartLine === item.sourceEndLine
? `来源:第 ${item.sourceStartLine}`
: `来源:第 ${item.sourceStartLine}${item.sourceEndLine}`
if (locator?.kind === 'xlsx') {
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
return locator.row_number != null
? `来源:${sheet} · 第 ${locator.row_number}`
: `来源:${sheet}`
}
if (locator?.kind === 'json') {
return locator.json_pointer
? `来源JSON 路径 ${locator.json_pointer}`
: '来源JSON 根对象'
}
if (locatedLines) {
return locatedLines.start === locatedLines.end
? `来源:第 ${locatedLines.start}`
: `来源:第 ${locatedLines.start}${locatedLines.end}`
}
return '来源:源文件记录'
}
</script>
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
<div>
<strong>源文件 · {{ fileName }}</strong>
</div>
<div
v-if="!isPdfSource && !isOfficeSource && lines.length"
class="source-window-controls"
aria-label="源文件行窗口"
>
<span> {{ visibleSourceWindow.startLine }}{{ visibleSourceWindow.endLine }} </span>
<el-button
link
size="small"
aria-label="查看上一段源文件"
:disabled="!visibleSourceWindow.hasPrevious"
@click="showPreviousSourceWindow"
>
上一段
</el-button>
<el-button
link
size="small"
aria-label="查看下一段源文件"
:disabled="!visibleSourceWindow.hasMore"
@click="showNextSourceWindow"
>
下一段
</el-button>
</div>
</div>
<PdfSourceViewer
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
v-for="line in lines"
:key="line.number"
class="source-line"
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
:data-source-start="line.start"
:data-line-number="line.number"
>
<span class="line-number">{{ line.number }}</span>
<span class="line-content">{{ line.content || ' ' }}</span>
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
/>
<div class="editor-actions">
<el-button
v-if="editingItem.sourceStart != null"
v-if="!isManualPreviewItem(editingItem)"
link
@click="restoreItem"
>
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
}
}
.source-window-controls {
flex: none;
gap: 2px !important;
> span {
margin-right: 4px;
color: #8a93a3;
font-size: 11px;
white-space: nowrap;
}
:deep(.el-button) {
margin-left: 0;
}
}
.source-viewer {
flex: 1;
height: 538px;

View File

@@ -1,4 +1,5 @@
<script setup lang="ts">
import { computed } from 'vue'
import type {
GenerationControlOptions,
PreprocessOption,
@@ -21,27 +22,32 @@ const emit = defineEmits<{
'update:options': [value: StructuredProcessOptions]
}>()
const PREPROCESS_OPTIONS: Array<{
value: PreprocessOption
const PREPROCESS_GROUPS: Array<{
values: PreprocessOption[]
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
{
value: 'detect_structure',
label: '嵌套结构展平',
description: '展平嵌套对象和可解析的 JSON 字段Excel 表头与合并单元格在上传时自动解析',
values: ['clean_invalid', 'deduplicate'],
label: '数据清洗',
description: '清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
},
{
value: 'deduplicate',
label: '重复记录去重',
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
values: ['detect_structure', 'normalize_format'],
label: '结构标准化',
description: '展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
},
{
values: ['desensitize'],
label: '敏感信息脱敏',
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
},
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
]
const legacyAnomalyFilterEnabled = computed(() => (
props.options.preprocessOptions.includes('filter_anomaly')
))
function updateField<K extends keyof StructuredProcessOptions>(
field: K,
value: StructuredProcessOptions[K],
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = Array.from(new Set(value.filter(
(option): option is PreprocessOption => (
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
),
)))
updateField('preprocessOptions', preprocessOptions)
function selectedCount(values: PreprocessOption[]) {
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
}
function groupSelected(values: PreprocessOption[]) {
return selectedCount(values) === values.length
}
function groupIndeterminate(values: PreprocessOption[]) {
const count = selectedCount(values)
return count > 0 && count < values.length
}
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
const next = new Set(props.options.preprocessOptions)
values.forEach((value) => {
if (Boolean(checked)) next.add(value)
else next.delete(value)
})
updateField('preprocessOptions', [...next])
}
</script>
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>选择在生成问答对之前需要执行的数据处理方式</p>
<p>默认不执行预处理请按数据情况自行选择</p>
</div>
</div>
<el-checkbox-group
:model-value="options.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updatePreprocessOptions"
>
<el-checkbox
v-for="option in PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
<div class="preprocess-option-grid">
<label
v-for="group in PREPROCESS_GROUPS"
:key="group.label"
class="preprocess-option"
:class="{ 'is-checked': groupSelected(group.values) }"
>
<el-checkbox
:model-value="groupSelected(group.values)"
:indeterminate="groupIndeterminate(group.values)"
@update:model-value="updatePreprocessGroup(group.values, $event)"
/>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
<strong>{{ group.label }}</strong>
<small>{{ group.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</label>
</div>
<el-alert
v-if="legacyAnomalyFilterEnabled"
class="legacy-preprocess-alert"
type="warning"
:closable="false"
title="该历史任务仍启用了已停用的“异常数据过滤”;为保证结果可复现,本次继续保留"
/>
</div>
<div class="form-section generation-options-section">

View File

@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>默认启用结构感知的推荐策略只需决定是否需要脱敏</p>
<p>默认不执行预处理请按文档情况自行选择</p>
</div>
</div>
<div class="preprocess-option-grid">

View File

@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
export function createDefaultStructuredOptions(): StructuredProcessOptions {
return {
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
preprocessOptions: [],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
return {
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
preprocessOptions: [],
chunkMethod: 'layout_hybrid',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
semanticBreakpointPercentile: 95,
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
const defaults = createDefaultStructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
const supportedPreprocessOptions = new Set<PreprocessOption>([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as PreprocessOption[]
? Array.from(new Set(preprocessOptions.map(String).filter(
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
)))
: defaults.preprocessOptions,
semanticEnrichment: Boolean(configValue(
config,

View File

@@ -1,4 +1,4 @@
import type { SourceLine } from './types'
import type { PreviewItem, SourceLine } from './types'
/** 仅用于“使用示例”上传;正式预览和切片全部由后端生成。 */
export const DEFAULT_SOURCE_TEXT = [
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
'答:复利是将上一期利息加入本金,再计算下一期利息。',
].join('\n')
/**
* 把后端返回的字符偏移映射为源文件行,仅负责界面高亮,不参与切片。
*/
export function sourceLines(sourceText: string): SourceLine[] {
const rawLines = sourceText.split('\n')
let cursor = 0
export interface SourceLineWindow {
lines: SourceLine[]
startLine: number
endLine: number
hasPrevious: boolean
hasMore: boolean
}
return rawLines.map((content, index) => {
const start = cursor
const end = start + content.length
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
return { number: index + 1, content, start, end }
})
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
let length = 0
let index = start
while (index < end) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
length += 1
}
return length
}
function advanceCodePoints(value: string, start: number, end: number, count: number) {
let index = start
let remaining = Math.max(0, count)
while (index < end && remaining > 0) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
remaining -= 1
}
return index
}
/**
* 只扫描并返回当前可见行窗口,不对全文 split避免大文件生成巨量字符串数组。
* 字符定位场景可开启 code point 偏移,以与后端 Python 的字符计数保持一致。
*/
export function sourceLineWindow(
sourceText: string,
requestedStartLine: number,
maxLines: number,
maxCharactersPerLine: number,
focusLine: number | null = null,
focusOffset: number | null = null,
): SourceLineWindow {
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
const limit = Math.max(1, Math.trunc(maxLines) || 1)
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
const trackUnicodeOffsets = focusOffset != null
const lines: SourceLine[] = []
let lineNumber = 1
let jsCursor = 0
let sourceCursor = 0
while (jsCursor <= sourceText.length && lineNumber < startLine) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
sourceCursor = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
while (jsCursor <= sourceText.length && lines.length < limit) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
const fullSourceEnd = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
: jsEnd
const focusedStart = focusLine === lineNumber && focusOffset != null
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
: sourceCursor
const segmentSourceStart = Math.min(
focusedStart,
Math.max(sourceCursor, fullSourceEnd - characterLimit),
)
const relativeSegmentStart = trackUnicodeOffsets
? segmentSourceStart - sourceCursor
: Math.max(0, segmentSourceStart - jsCursor)
const segmentJsStart = advanceCodePoints(
sourceText,
jsCursor,
jsEnd,
relativeSegmentStart,
)
const segmentJsEnd = advanceCodePoints(
sourceText,
segmentJsStart,
jsEnd,
characterLimit,
)
const segmentLength = trackUnicodeOffsets
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
: segmentJsEnd - segmentJsStart
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
const end = start + segmentLength
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
lines.push({ number: lineNumber, content, start, end })
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
return {
lines,
startLine: lines[0]?.number ?? startLine,
endLine: lines[lines.length - 1]?.number ?? startLine,
hasPrevious: startLine > 1,
hasMore: jsCursor <= sourceText.length,
}
}
/** 根据后端 code point 偏移查找物理行号,不构建全文行数组。 */
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
let offset = 0
let lineNumber = 1
for (const character of sourceText) {
if (offset >= normalizedOffset) break
if (character === '\n') lineNumber += 1
offset += 1
}
return lineNumber
}
/**
* 手动新增项可能先以空内容保存为 invalid编辑后又由后端标记为 modified
* 因此不能只依赖可变的 status空原文且完全没有来源定位才是稳定兜底。
*/
export function isManualPreviewItem(item: PreviewItem): boolean {
const hasSourceLocation = item.sourceStart != null
|| item.sourceEnd != null
|| item.sourceStartLine != null
|| item.sourceEndLine != null
|| Boolean(item.sourcePages?.length)
|| Boolean(item.sourceLocator)
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
}
/** 与后端预览 token 估算规则一致,仅用于编辑中的即时计数。 */

View File

@@ -24,6 +24,7 @@ export type PreprocessOption =
| 'detect_structure'
| 'deduplicate'
| 'normalize_format'
/** 仅用于恢复历史任务,新任务界面不再提供。 */
| 'filter_anomaly'
| 'desensitize'
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
export interface UploadedDataFile {
uid: string | number
sourceFileId?: string
rawFile?: File
name: string
size: number
count: number
@@ -118,6 +118,22 @@ export interface SourceLine {
end: number
}
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
export interface PreviewSourceLocator {
kind: PreviewSourceLocatorKind
record_index?: number | null
start_line?: number | null
end_line?: number | null
source_start?: number | null
source_end?: number | null
json_pointer?: string | null
sheet_index?: number | null
sheet_name?: string | null
row_number?: number | null
sheet_record_index?: number | null
}
export interface PreviewItem {
id: string
sourceFileId: string
@@ -129,6 +145,8 @@ export interface PreviewItem {
sourceStartLine: number | null
sourceEndLine: number | null
sourcePages?: number[]
sourceLocator?: PreviewSourceLocator
headingPath?: string[]
tokenCount: number
status: 'original' | 'modified' | 'manual' | 'invalid'
qualityScore?: number

View File

@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
let generationTimer: ReturnType<typeof setTimeout> | null = null
let generationRun = 0
let pollFailureCount = 0
let generationStarting = false
const generationStarting = ref(false)
const generationRestoring = ref(false)
const canReturnFromGeneration = computed(() => (
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
))
function stopGenerationTimer() {
generationRun += 1
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
}
async function startGeneration() {
if (generationStarting || generation.status === 'running') return false
if (generationStarting.value || generation.status === 'running') return false
const taskId = bindings.taskId.value
if (!taskId) {
ElMessage.error('任务尚未创建,请返回上一步重试')
return false
}
generationStarting = true
generationStarting.value = true
let runId: number | null = null
try {
const canStart = await bindings.beforeGenerate?.()
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
generation.message = error instanceof Error ? error.message : '启动数据处理失败,请重试。'
return false
} finally {
generationStarting = false
generationStarting.value = false
}
}
async function resumeGeneration() {
const taskId = bindings.taskId.value
if (!taskId) return
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
generationRestoring.value = true
try {
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
const progress = await getDataProcessProgress(taskId)
if (activeRunId !== generationRun) return
if (progress.status === 'running') {
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
} catch (error) {
generation.status = 'failed'
generation.message = error instanceof Error ? error.message : '查询任务进度失败,请重试。'
} finally {
generationRestoring.value = false
}
}
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
return {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,

View File

@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
import { useRoute } from 'vue-router'
import {
getDataProcessPreview,
getDataProcessSourceContent,
getDataProcessTask,
regenerateDataProcessTask,
} from '@/api/modules/dataProcess'
@@ -15,7 +14,10 @@ import {
createStructuredOptionsFromConfig,
createUnstructuredOptionsFromConfig,
} from './dataProcessCreateState'
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
} from './useDataProcessSourceUpload'
import type {
PreviewItem,
ProcessType,
@@ -50,24 +52,6 @@ interface RegenerationBindings {
resetDownstream: () => void
}
async function loadSourceContent(taskId: string, fileId: string | number) {
const chunks: string[] = []
let startLine = 1
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
start_line: startLine,
line_count: 10_000,
})
chunks.push(source.content || '')
if (!source.has_more) break
const nextLine = Number(source.end_line || startLine) + 1
if (nextLine <= startLine) break
startLine = nextLine
}
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
return chunks.join('')
}
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
const items = [...first.items]
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
const taskId = String(task.id)
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
)))
bindings.previewItems.value = preservePreviews
? await loadAllPreviews(taskId, bindings.mapPreviewItem)

View File

@@ -6,7 +6,6 @@ import {
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
interface SourceUploadJob {
uid: string
file: File
extension: string
}
interface SourceUploadOptions {
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
}
}
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
}
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
}
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
return { valid: true, extension }
}
function unicodeCodePointLength(value: string) {
let length = 0
for (const _character of value) length += 1
return length
}
/** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */
export async function loadCanonicalSourceContent(
taskId: string | number,
fileId: string | number,
) {
const chunks: string[] = []
let offset = 0
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
offset,
limit: SOURCE_CONTENT_PAGE_CHARS,
})
const content = source.content || ''
chunks.push(content)
if (!source.has_more) break
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试')
offset = nextOffset
}
return chunks.join('')
}
export function mapDataProcessSourceFile(
file: DataProcessSourceFile,
content = '',
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
pending.uploadError = undefined
try {
let content = ''
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
} catch {
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
}
if (!content.trim()) throw new Error('不能上传空文件')
}
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
pending.uploadProgress = progress
})
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
// 先登记后端 ID确保正文读取失败时仍可正确删除已落库的文件。
Object.assign(pending, mapDataProcessSourceFile(source), {
rawFile: job.file,
status: 'uploading',
uploadProgress: 99,
})
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
start_line: 1,
line_count: 10_000,
})
pending.content = parsed.content
} catch {
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
}
} else {
pending.content = content
try {
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
} catch {
throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试')
}
pending.status = 'ready'