fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
@@ -83,6 +83,11 @@ assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?d
|
||||
assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐')
|
||||
assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集')
|
||||
assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由')
|
||||
assert.match(detailSource, /const canRepeatGeneration = computed[\s\S]*?status === 'completed'[\s\S]*?results_confirmed !== false[\s\S]*?previewCount\.value > 0/, '已完成任务缺少再次生成资格判断')
|
||||
assert.match(detailSource, /repeatDataProcessTask\(taskId\.value,[\s\S]*?expected_updated_at: detail\.value\.updated_at[\s\S]*?request_id: repeatRequestId\.value/, '再次生成没有携带源任务版本和幂等请求 ID')
|
||||
assert.match(detailSource, /name: 'data-process-workflow'[\s\S]*?params: \{ id: repeated\.task\.id \}/, '再次生成成功后没有进入新任务工作流')
|
||||
assert.match(detailSource, /原任务和原结果不会被修改/, '再次生成确认提示没有说明原任务保持不变')
|
||||
assert.match(detailSource, /v-if="canRepeatGeneration"[\s\S]*?@click="repeatGeneration"[\s\S]*?按原配置再生成一批/, '已完成任务详情缺少再次生成新批次入口')
|
||||
assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value[\s\S]*?hasPublishedOutputs\.value/, '详情页没有覆盖指针已清空但旧发布数据集仍存在的重新生成任务')
|
||||
assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口')
|
||||
assert.match(detailSource, /v-if="detail\.status === 'completed' && !hasCurrentPublishedDataset"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布或发布指针失效的完成任务没有保留发布入口')
|
||||
@@ -115,6 +120,11 @@ assert.match(detailSource, /inputMetricCount\.toLocaleString\(\) \}\} \{\{ input
|
||||
assert.match(detailSource, /sourceFileCount\.toLocaleString\(\) \}\} 个/, '源文件数量缺少个数单位')
|
||||
assert.match(detailSource, /<span>生成结果<\/span><strong>\{\{ numeric\(detail\.output_count\)\.toLocaleString\(\) \}\} 条<\/strong>/, '生成结果数量缺少条数单位或仍误称成功输出')
|
||||
assert.match(detailSource, /const configExpanded = ref\(false\)/, '处理配置没有默认收起')
|
||||
assert.match(detailSource, /appendGroup\(\['clean_invalid', 'deduplicate'\], '数据清洗'\)/, '详情页没有将完整清洗配置合并为数据清洗')
|
||||
assert.match(detailSource, /appendGroup\(\['detect_structure', 'normalize_format'\], '结构标准化'\)/, '详情页没有将完整结构配置合并为结构标准化')
|
||||
assert.match(detailSource, /历史部分配置/, '详情页没有标识旧任务的半组选项')
|
||||
assert.match(detailSource, /异常数据过滤(历史规则)/, '详情页没有标识已停用的历史异常过滤规则')
|
||||
assert.match(detailSource, /new Set\(value\.map/, '详情页没有去除历史预处理配置中的重复值')
|
||||
assert.match(detailSource, /:aria-expanded="configExpanded"/, '处理配置折叠按钮缺少无障碍状态')
|
||||
assert.match(detailSource, /<el-collapse-transition>[\s\S]*?v-show="configExpanded"/, '处理配置没有折叠过渡或内容状态')
|
||||
assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据')
|
||||
@@ -126,6 +136,7 @@ for (const apiName of [
|
||||
'updateDataProcessResult',
|
||||
'restoreDataProcessResult',
|
||||
'publishDataProcess',
|
||||
'repeatDataProcessTask',
|
||||
]) {
|
||||
assert.match(
|
||||
apiSource,
|
||||
@@ -136,5 +147,8 @@ for (const apiName of [
|
||||
assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数')
|
||||
assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID')
|
||||
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确')
|
||||
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/repeat`/, '再次生成 API 路径不正确')
|
||||
assert.match(typesSource, /interface DataProcessRepeatPayload[\s\S]*?expected_updated_at: string[\s\S]*?request_id: string/, '再次生成请求契约不完整')
|
||||
assert.match(typesSource, /interface DataProcessRepeatResult[\s\S]*?task: DataProcessTask[\s\S]*?source_task_id: string[\s\S]*?created: boolean/, '再次生成响应契约不完整')
|
||||
|
||||
console.log('数据处理任务详情真实 API 回归检查通过')
|
||||
|
||||
@@ -183,8 +183,36 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
|
||||
assert.ok(typesSource.includes(field), `PreviewItem 缺少字段:${field}`)
|
||||
}
|
||||
assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
|
||||
assert.match(typesSource, /sourceLocator\?: PreviewSourceLocator/, 'PreviewItem 缺少结构化来源定位契约')
|
||||
assert.match(typesSource, /headingPath\?: string\[\]/, 'PreviewItem 缺少非结构化标题路径')
|
||||
assert.match(typesSource, /PreviewSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, '前端来源定位 kind 未使用明确联合类型')
|
||||
assert.match(contractTypesSource, /DataProcessSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, 'API 来源定位 kind 未使用明确联合类型')
|
||||
for (const field of ['kind', 'record_index', 'start_line', 'end_line', 'source_start', 'source_end', 'json_pointer', 'sheet_index', 'sheet_name', 'row_number', 'sheet_record_index']) {
|
||||
assert.ok(typesSource.includes(field), `PreviewSourceLocator 缺少字段:${field}`)
|
||||
assert.ok(contractTypesSource.includes(field), `后端来源定位契约缺少字段:${field}`)
|
||||
}
|
||||
assert.match(contractTypesSource, /source_locator\?: DataProcessSourceLocator/, '质量信息缺少来源定位契约')
|
||||
assert.match(contractTypesSource, /heading_path\?: string\[\]/, '质量信息缺少标题路径契约')
|
||||
assert.match(viewSource, /const sourceLocator = item\.quality_score\?\.source_locator/, '预览映射丢失来源定位')
|
||||
assert.match(viewSource, /sourceStart:\s*item\.source_start\s*\?\?\s*sourceLocator\?\.source_start/, 'JSON locator 的字符起点没有映射到预览项')
|
||||
assert.match(viewSource, /sourceEnd:\s*item\.source_end\s*\?\?\s*sourceLocator\?\.source_end/, 'JSON locator 的字符终点没有映射到预览项')
|
||||
assert.match(viewSource, /sourceStartLine:\s*item\.source_start_line\s*\?\?\s*sourceLocator\?\.start_line/, 'JSON locator 的起始行没有映射到预览项')
|
||||
assert.match(viewSource, /sourceEndLine:\s*item\.source_end_line\s*\?\?\s*sourceLocator\?\.end_line/, 'JSON locator 的结束行没有映射到预览项')
|
||||
assert.match(viewSource, /headingPath:[\s\S]*?item\.quality_score\?\.heading_path/, '预览映射丢失标题路径')
|
||||
assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
|
||||
assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
|
||||
assert.match(modelSource, /export function sourceLineWindow/, '缺少有界源文件行窗口函数')
|
||||
assert.match(modelSource, /maxLines:\s*number/, '源文件行窗口缺少最大渲染行数参数')
|
||||
assert.doesNotMatch(modelSource, /\.split\(\s*['"]\\n['"]\s*\)/, '源文件行窗口仍会先对全文 split')
|
||||
assert.match(modelSource, /lines\.length < limit/, '源文件行扫描没有受最大行数约束')
|
||||
assert.match(modelSource, /unicodeCodePointLength/, '源文件字符偏移未与后端 Unicode code point 计数保持一致')
|
||||
assert.match(modelSource, /export function sourceLineNumberAtOffset/, '字符偏移缺少无数组的行号解析函数')
|
||||
const manualPreviewHelperStart = modelSource.indexOf('export function isManualPreviewItem(')
|
||||
const manualPreviewHelperEnd = modelSource.indexOf('\n}', manualPreviewHelperStart)
|
||||
assert.ok(manualPreviewHelperStart >= 0, '缺少统一的手动预览项判定函数')
|
||||
const manualPreviewHelperSource = modelSource.slice(manualPreviewHelperStart, manualPreviewHelperEnd + 2)
|
||||
for (const field of ['status', 'originalContent', 'sourceStart', 'sourceEnd', 'sourceStartLine', 'sourceEndLine', 'sourcePages', 'sourceLocator']) {
|
||||
assert.ok(manualPreviewHelperSource.includes(field), `手动预览项判定缺少来源字段:${field}`)
|
||||
}
|
||||
assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
|
||||
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
||||
const previewBuildBindingStart = viewSource.indexOf('useDataProcessPreviewBuild()')
|
||||
@@ -210,6 +238,30 @@ for (const marker of [
|
||||
}
|
||||
assert.match(previewSource, /sourceStart/, '第四步未使用来源起始偏移')
|
||||
assert.match(previewSource, /sourceEnd/, '第四步未使用来源结束偏移')
|
||||
const lineRangeStart = previewSource.indexOf('function lineRange(item: PreviewItem)')
|
||||
const lineRangeEnd = previewSource.indexOf('\n}', lineRangeStart)
|
||||
const lineRangeSource = previewSource.slice(lineRangeStart, lineRangeEnd + 2)
|
||||
assert.match(lineRangeSource, /isManualPreviewItem\(item\)[\s\S]*?手动新增,无源文件定位/, '来源标签仍会把缺少行偏移的正常记录误判为手动新增')
|
||||
assert.match(lineRangeSource, /props\.processType === 'unstructured'[\s\S]*?来源:源文件记录/, '结构化来源记录缺少无行偏移时的准确标签')
|
||||
assert.doesNotMatch(lineRangeSource, /sourceStartLine == null[^\n]*手动新增/, '来源标签仍直接以缺少行号判定手动新增')
|
||||
assert.match(lineRangeSource, /sheet_name[\s\S]*?row_number[\s\S]*?来源:\$\{sheet\} · 第 \$\{locator\.row_number\} 行/, 'XLSX 来源标签没有展示工作表和物理行号')
|
||||
assert.match(lineRangeSource, /json_pointer[\s\S]*?JSON 路径/, 'JSON 来源标签没有展示 JSON 路径')
|
||||
assert.match(lineRangeSource, /locator\?\.kind === 'json'[\s\S]*?JSON 根对象/, 'JSON 根对象来源标签被空 JSON Pointer 错误降级')
|
||||
assert.match(lineRangeSource, /locatedLines[\s\S]*?第 \$\{locatedLines\.start\}[\s\S]*?locatedLines\.end/, 'JSONL/CSV 来源标签没有展示行范围')
|
||||
assert.match(lineRangeSource, /headingPath[\s\S]*?章节:/, '非结构化来源标签没有合并标题路径')
|
||||
assert.match(previewSource, /sourceLocator\?\.start_line[\s\S]*?sourceLocator\?\.end_line/, '文本预览没有优先使用后端行号定位')
|
||||
assert.match(previewSource, /sourceLocator\?\.source_start\s*\?\?\s*item\.sourceStart/, '文本预览没有优先使用 locator 字符起点')
|
||||
assert.match(previewSource, /sourceLocator\?\.source_end\s*\?\?\s*item\.sourceEnd/, '文本预览没有优先使用 locator 字符终点')
|
||||
assert.match(previewSource, /data-line-number="line\.number"/, '文本预览行缺少稳定行号定位标识')
|
||||
assert.match(previewSource, /isLineHighlighted\(line\.number, line\.start, line\.end\)/, '文本预览没有按物理行号高亮')
|
||||
assert.match(previewSource, /querySelector<HTMLElement>\(`\[data-line-number=/, '选中记录后没有按物理行号滚动定位')
|
||||
assert.match(previewSource, /const SOURCE_LINE_RENDER_LIMIT = 240/, '源文件查看器缺少安全渲染上限')
|
||||
assert.match(previewSource, /const SOURCE_LINE_CHARACTER_LIMIT = 4_000/, '源文件查看器缺少单行字符渲染上限')
|
||||
assert.match(previewSource, /sourceLineWindow\([\s\S]*?SOURCE_LINE_RENDER_LIMIT/, '源文件查看器没有使用有界行窗口')
|
||||
assert.match(previewSource, /SOURCE_LINE_RENDER_LIMIT,[\s\S]*?SOURCE_LINE_CHARACTER_LIMIT,[\s\S]*?selectedSourceLine\.value,[\s\S]*?selectedSourceOffset\.value/, '单行超大 JSON 没有围绕选中来源构建字符窗口')
|
||||
assert.match(previewSource, /sourceWindowStartLine/, '源文件查看器缺少窗口起始行状态')
|
||||
assert.match(previewSource, /showPreviousSourceWindow[\s\S]*?showNextSourceWindow/, '源文件查看器缺少前后窗口导航')
|
||||
assert.match(previewSource, /sourceLineNumberAtOffset\(props\.sourceText/, '仅有字符偏移时没有解析目标物理行')
|
||||
assert.match(previewSource, /filterable/, '文件选择器必须可搜索')
|
||||
assert.match(previewSource, /当前文件/, '预览缺少当前文件切换器')
|
||||
assert.doesNotMatch(previewSource, /located-badge|sync-label|已定位到/, '源文件栏不应显示冗余定位提示')
|
||||
@@ -269,6 +321,18 @@ for (const marker of [
|
||||
]) {
|
||||
assert.ok(officeViewerSource.includes(marker), `Word/XLSX 预览缺少结构或行为:${marker}`)
|
||||
}
|
||||
assert.match(officeViewerSource, /const selectedXlsxLocator = computed/, 'XLSX 查看器没有读取精确来源定位')
|
||||
assert.match(officeViewerSource, /row\.row_number === locator\.row_number/, 'XLSX 查看器没有按物理行号精确高亮')
|
||||
assert.match(officeViewerSource, /row\.record_index === locator\.sheet_record_index/, 'XLSX 查看器没有按工作表记录序号精确高亮')
|
||||
assert.match(officeViewerSource, /Math\.floor\(locator\.sheet_record_index \/ XLSX_PAGE_SIZE\) \* XLSX_PAGE_SIZE/, 'XLSX 查看器没有按记录序号自动计算分页')
|
||||
assert.match(officeViewerSource, /activeSheetIndex\.value = targetSheet[\s\S]*?pageOffset\.value = targetOffset[\s\S]*?loadPreview\(\)/, '切换记录时 XLSX 查看器没有自动切工作表和分页')
|
||||
const xlsxHighlightStart = officeViewerSource.indexOf('function xlsxRowHighlighted(')
|
||||
const xlsxHighlightEnd = officeViewerSource.indexOf('\n}', xlsxHighlightStart)
|
||||
const xlsxHighlightSource = officeViewerSource.slice(xlsxHighlightStart, xlsxHighlightEnd + 2)
|
||||
assert.ok(
|
||||
xlsxHighlightSource.indexOf('locator.row_number') < xlsxHighlightSource.indexOf('selectedRecordKey.value'),
|
||||
'XLSX 查看器没有把精确定位放在原内容比对 fallback 之前',
|
||||
)
|
||||
|
||||
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
||||
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
|
||||
@@ -454,7 +518,7 @@ assert.match(
|
||||
)
|
||||
assert.match(
|
||||
workflowInitializationSource,
|
||||
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?goToStep\(resumeStep\)[\s\S]*?resumeGeneration/,
|
||||
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)/,
|
||||
'生成运行中时没有强制回到第五步并接管后台进度',
|
||||
)
|
||||
const startGenerationHandler = viewSource.slice(
|
||||
@@ -463,7 +527,35 @@ const startGenerationHandler = viewSource.slice(
|
||||
)
|
||||
assert.match(startGenerationHandler, /await persistWorkflowStep\('generate'\)[\s\S]*?await startGeneration\(\)[\s\S]*?dirty\.value = false/, '开始生成没有持久化第五步或启动真实后台任务')
|
||||
assert.doesNotMatch(startGenerationHandler, /router\.(?:push|replace)|allowLeave\s*=\s*true/, '开始生成后应停留在第五步,不得自动跳回列表')
|
||||
assert.match(viewSource, /:disabled="currentStepId === 'generate' \|\| previewBuilding \|\| sourceUploading"/, '第五步底部返回按钮没有固定禁用')
|
||||
assert.match(
|
||||
generationSource,
|
||||
/const canReturnFromGeneration = computed\(\(\) => \([\s\S]*?generation\.status === 'idle'[\s\S]*?!generationStarting\.value[\s\S]*?!generationRestoring\.value/,
|
||||
'第五步返回权限没有区分未启动、启动中和恢复中状态',
|
||||
)
|
||||
assert.match(
|
||||
viewSource,
|
||||
/:disabled="\(currentStepId === 'generate' && !canReturnFromGeneration\) \|\| previewBuilding \|\| sourceUploading"/,
|
||||
'第五步尚未启动生成时返回按钮仍被禁用',
|
||||
)
|
||||
const handleBackStart = viewSource.indexOf('async function handleBack()')
|
||||
const handleBackEnd = viewSource.indexOf('\n}', handleBackStart)
|
||||
const handleBackSource = viewSource.slice(handleBackStart, handleBackEnd + 2)
|
||||
assert.match(
|
||||
handleBackSource,
|
||||
/currentStepId\.value === 'generate' && !canReturnFromGeneration\.value/,
|
||||
'第五步处理函数仍无条件拦截返回',
|
||||
)
|
||||
assert.match(
|
||||
generationSource,
|
||||
/async function resumeGeneration\(\)[\s\S]*?generationRestoring\.value = true[\s\S]*?await getDataProcessProgress\(taskId\)[\s\S]*?generationRestoring\.value = false/,
|
||||
'恢复已启动任务时存在短暂可返回的 idle 窗口',
|
||||
)
|
||||
assert.match(viewSource, /const resume = resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)[\s\S]*?await resume/, '第五步展示时未先启动恢复锁')
|
||||
assert.match(
|
||||
generationSource,
|
||||
/const generationStarting = ref\(false\)[\s\S]*?generationStarting\.value = true[\s\S]*?generationStarting\.value = false/,
|
||||
'点击开始生成后到请求启动前没有锁定返回状态',
|
||||
)
|
||||
assert.match(
|
||||
viewSource,
|
||||
/generation\.status === 'success'[\s\S]*?persistWorkflowStep\('results'\)/,
|
||||
@@ -506,32 +598,34 @@ assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTy
|
||||
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
|
||||
|
||||
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
||||
const expectedStructuredOptions = [
|
||||
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
|
||||
const expectedStructuredGroups = [
|
||||
[
|
||||
'detect_structure',
|
||||
'嵌套结构展平',
|
||||
'展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||
"values: ['clean_invalid', 'deduplicate']",
|
||||
'数据清洗',
|
||||
'清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
|
||||
],
|
||||
[
|
||||
'deduplicate',
|
||||
'重复记录去重',
|
||||
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||
"values: ['detect_structure', 'normalize_format']",
|
||||
'结构标准化',
|
||||
'展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
|
||||
],
|
||||
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
|
||||
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
|
||||
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||
["values: ['desensitize']", '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||
]
|
||||
for (const [value, label, description] of expectedStructuredOptions) {
|
||||
assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值:${value}`)
|
||||
for (const [values, label, description] of expectedStructuredGroups) {
|
||||
assert.ok(structuredOptionsSource.includes(values), `结构化预处理组合值不准确:${label}`)
|
||||
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
|
||||
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
|
||||
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${label}`)
|
||||
}
|
||||
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
|
||||
.map((match) => match[1])
|
||||
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
|
||||
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
|
||||
assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重')
|
||||
assert.equal(expectedStructuredGroups.length, 3, '结构化预处理应收敛为 3 项')
|
||||
const preprocessGroupsSource = structuredOptionsSource.slice(
|
||||
structuredOptionsSource.indexOf('const PREPROCESS_GROUPS'),
|
||||
structuredOptionsSource.indexOf('const legacyAnomalyFilterEnabled'),
|
||||
)
|
||||
assert.doesNotMatch(preprocessGroupsSource, /异常数据过滤|filter_anomaly|IQR/, '结构化新任务仍暴露异常数据过滤')
|
||||
assert.match(structuredOptionsSource, /:indeterminate="groupIndeterminate\(group\.values\)"/, '历史部分选中的组合项没有半选回显')
|
||||
assert.match(structuredOptionsSource, /function updatePreprocessGroup\([\s\S]*?new Set\(props\.options\.preprocessOptions\)[\s\S]*?next\.add\(value\)[\s\S]*?next\.delete\(value\)[\s\S]*?\[\.\.\.next\]/, '结构化预处理组合开关没有原子化更新或去重内部选项')
|
||||
assert.match(typesSource, /仅用于恢复历史任务[\s\S]*?\| 'filter_anomaly'/, '异常数据过滤缺少历史兼容类型')
|
||||
assert.match(structuredOptionsSource, /legacyAnomalyFilterEnabled[\s\S]*?历史任务[\s\S]*?结果可复现/, '历史异常过滤配置没有透明提示')
|
||||
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
|
||||
for (const splitName of ['训练集', '验证集', '测试集']) {
|
||||
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
|
||||
@@ -585,8 +679,20 @@ for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json',
|
||||
}
|
||||
assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别')
|
||||
assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示')
|
||||
assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验')
|
||||
assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本')
|
||||
const sourceValidationStart = sourceUploadWorkerSource.indexOf('export function validateSourceFileSelection(')
|
||||
const sourceValidationEnd = sourceUploadWorkerSource.indexOf('\n}\n\nfunction unicodeCodePointLength', sourceValidationStart)
|
||||
assert.ok(sourceValidationStart >= 0 && sourceValidationEnd > sourceValidationStart, '无法定位源文件选择校验函数')
|
||||
const sourceValidationSource = sourceUploadWorkerSource.slice(sourceValidationStart, sourceValidationEnd + 2)
|
||||
assert.doesNotMatch(sourceValidationSource, /file\.name === raw\.name[\s\S]{0,160}file\.size === raw\.size|同名且同大小/, '不同内容但同名同大小的文件仍会被前端误拒绝')
|
||||
assert.match(sourceValidationSource, /selectedFiles\.length >= MAX_SOURCE_FILE_COUNT/, '移除伪重复校验时误删了文件数量限制')
|
||||
assert.match(sourceValidationSource, /selectedBytes \+ raw\.size > MAX_SOURCE_BATCH_BYTES/, '移除伪重复校验时误删了批次大小限制')
|
||||
assert.doesNotMatch(sourceUploadWorkerSource, /job\.file\.arrayBuffer\(|new TextDecoder/, '上传前仍把整个文本文件读入浏览器内存')
|
||||
assert.match(sourceUploadWorkerSource, /export async function loadCanonicalSourceContent[\s\S]*?offset,[\s\S]*?limit: SOURCE_CONTENT_PAGE_CHARS/, '服务端 canonical content 没有按有界字符窗口读取')
|
||||
assert.match(sourceUploadWorkerSource, /pending\.content = await loadCanonicalSourceContent\(currentTaskId, source\.id\)/, '上传成功后没有统一使用服务端 canonical content')
|
||||
assert.doesNotMatch(sourceUploadWorkerSource, /\brawFile:\s*job\.file\b/, '上传成功状态仍长期保留原始 File')
|
||||
assert.doesNotMatch(typesSource, /\brawFile\??:\s*File\b/, '上传状态类型仍长期持有原始 File')
|
||||
assert.doesNotMatch(viewSource, /\brawFile:\s*raw\b/, '待上传列表仍复制保存原始 File')
|
||||
assert.match(apiSource, /params:\s*\{[\s\S]*?offset\?: number[\s\S]*?limit\?: number[\s\S]*?\}/, '正文 API 前端契约缺少字符窗口参数')
|
||||
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
|
||||
assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%')
|
||||
assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时')
|
||||
@@ -609,7 +715,7 @@ assert.match(
|
||||
/export interface DataProcessPreviewProgress[\s\S]*?workflow_step: DataProcessWorkflowStep[\s\S]*?preview_status: DataProcessPreviewStatus[\s\S]*?preview_progress: number[\s\S]*?preview_run_id/,
|
||||
'后台切分进度契约缺少步骤、状态、进度或任务代次',
|
||||
)
|
||||
for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
|
||||
for (const field of ['status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
|
||||
assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段:${field}`)
|
||||
}
|
||||
assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整')
|
||||
@@ -853,13 +959,23 @@ const defaultStructuredPreprocess = defaultPreprocessValues(
|
||||
)
|
||||
assert.deepEqual(
|
||||
defaultStructuredPreprocess,
|
||||
['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
'结构化默认预处理配置不准确',
|
||||
[],
|
||||
'结构化新任务不应默认勾选预处理',
|
||||
)
|
||||
assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复')
|
||||
const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions')
|
||||
assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整')
|
||||
assert.deepEqual(defaultUnstructuredPreprocess, [], '非结构化新任务不应默认勾选预处理')
|
||||
assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复')
|
||||
for (const field of ['preserveTables', 'preserveCodeBlocks', 'preserveLists']) {
|
||||
assert.match(
|
||||
stateSource,
|
||||
new RegExp(`${field}:\\s*false`),
|
||||
`非结构化预处理选项 ${field} 不应默认开启`,
|
||||
)
|
||||
}
|
||||
assert.match(structuredOptionsSource, /默认不执行预处理,请按数据情况自行选择/, '结构化预处理缺少默认不勾选说明')
|
||||
assert.match(unstructuredOptionsSource, /默认不执行预处理,请按文档情况自行选择/, '非结构化预处理缺少默认不勾选说明')
|
||||
assert.doesNotMatch(unstructuredOptionsSource, /默认启用结构感知/, '非结构化预处理仍保留默认启用的误导文案')
|
||||
|
||||
const backendConfigStart = viewSource.indexOf('function toBackendConfig()')
|
||||
const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart)
|
||||
@@ -934,7 +1050,7 @@ for (const [field, fallback] of [
|
||||
)
|
||||
}
|
||||
assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务')
|
||||
assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文')
|
||||
assert.match(regenerationSource, /loadCanonicalSourceContent\(taskId, file\.id\)/, '重新生成没有复用分页 canonical 正文加载器')
|
||||
assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片')
|
||||
assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用')
|
||||
assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重')
|
||||
@@ -948,7 +1064,7 @@ assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?g
|
||||
assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态')
|
||||
assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片')
|
||||
assert.match(regenerationSource, /重新生成配置已保存,但工作区恢复失败/, '重新生成配置已保存但水合失败时缺少可恢复错误状态')
|
||||
assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
|
||||
assert.match(sourceUploadWorkerSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
|
||||
assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容')
|
||||
assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口')
|
||||
assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新')
|
||||
@@ -1017,6 +1133,17 @@ for (const mutationFunction of [
|
||||
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
|
||||
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
|
||||
}
|
||||
const updatePreviewContentStart = viewSource.indexOf('function updatePreviewContent(')
|
||||
const updatePreviewContentEnd = viewSource.indexOf('\n}', updatePreviewContentStart)
|
||||
const updatePreviewContentSource = viewSource.slice(updatePreviewContentStart, updatePreviewContentEnd + 2)
|
||||
assert.match(updatePreviewContentSource, /isManualPreviewItem\(item\)/, '编辑预览内容仍未按稳定来源信息区分手动项')
|
||||
assert.doesNotMatch(updatePreviewContentSource, /sourceStart == null/, '结构化来源记录编辑后仍会被误标为手动项')
|
||||
const restorePreviewItemStart = viewSource.indexOf('function restorePreviewItem(')
|
||||
const restorePreviewItemEnd = viewSource.indexOf('\n}', restorePreviewItemStart)
|
||||
const restorePreviewItemSource = viewSource.slice(restorePreviewItemStart, restorePreviewItemEnd + 2)
|
||||
assert.match(restorePreviewItemSource, /isManualPreviewItem\(item\)/, '恢复预览内容没有使用统一的手动项判定')
|
||||
assert.doesNotMatch(restorePreviewItemSource, /sourceStart == null/, '结构化来源记录仍因缺少字符偏移而无法恢复')
|
||||
assert.match(previewSource, /v-if="!isManualPreviewItem\(editingItem\)"/, '结构化来源记录的恢复原文按钮仍被错误隐藏')
|
||||
assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
|
||||
|
||||
function findNextStyleBlockStart(source, startIndex) {
|
||||
|
||||
@@ -14,6 +14,8 @@ import type {
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessRepeatPayload,
|
||||
DataProcessRepeatResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
@@ -56,6 +58,8 @@ export type {
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessRepeatPayload,
|
||||
DataProcessRepeatResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
@@ -117,6 +121,15 @@ export const regenerateDataProcessTask = (
|
||||
payload,
|
||||
)
|
||||
|
||||
export const repeatDataProcessTask = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessRepeatPayload,
|
||||
) => post<DataProcessRepeatResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/repeat`,
|
||||
payload,
|
||||
{ timeout: 5 * 60 * 1000 },
|
||||
)
|
||||
|
||||
export const deleteDataProcessTask = (taskId: string | number) =>
|
||||
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||
|
||||
@@ -150,7 +163,12 @@ export const deleteDataProcessSourceFile = (taskId: string | number, fileId: str
|
||||
export const getDataProcessSourceContent = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
params: { start_line?: number; line_count?: number } = {},
|
||||
params: {
|
||||
start_line?: number
|
||||
line_count?: number
|
||||
offset?: number
|
||||
limit?: number
|
||||
} = {},
|
||||
) => get<DataProcessSourceContent>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`,
|
||||
params,
|
||||
|
||||
@@ -99,6 +99,20 @@ export interface DataProcessRegenerateResult {
|
||||
published_outputs_preserved: boolean
|
||||
}
|
||||
|
||||
export interface DataProcessRepeatPayload {
|
||||
expected_updated_at: string
|
||||
request_id: string
|
||||
}
|
||||
|
||||
export interface DataProcessRepeatResult {
|
||||
task: DataProcessTask
|
||||
source_task_id: string
|
||||
created: boolean
|
||||
copied_source_file_count: number
|
||||
copied_preview_count: number
|
||||
progress: DataProcessProgress
|
||||
}
|
||||
|
||||
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
|
||||
|
||||
export interface DataProcessSourceFile {
|
||||
@@ -232,6 +246,22 @@ export interface DataProcessPreviewItem {
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export type DataProcessSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
|
||||
|
||||
export interface DataProcessSourceLocator {
|
||||
kind: DataProcessSourceLocatorKind
|
||||
record_index?: number | null
|
||||
start_line?: number | null
|
||||
end_line?: number | null
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
json_pointer?: string | null
|
||||
sheet_index?: number | null
|
||||
sheet_name?: string | null
|
||||
row_number?: number | null
|
||||
sheet_record_index?: number | null
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewBuildPayload {
|
||||
replace_existing?: true
|
||||
source_file_ids?: Array<string | number>
|
||||
@@ -369,6 +399,9 @@ export interface DataProcessQualityScore {
|
||||
is_valid?: boolean
|
||||
flags?: string[]
|
||||
fingerprint?: string
|
||||
source_pages?: number[]
|
||||
heading_path?: string[]
|
||||
source_locator?: DataProcessSourceLocator
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
useDataProcessSourceUpload,
|
||||
validateSourceFileSelection,
|
||||
@@ -33,7 +34,6 @@ import {
|
||||
deleteDataProcessPreview,
|
||||
deleteDataProcessSourceFile,
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
pullDataProcessExternalSource,
|
||||
testDataProcessExternalSource,
|
||||
updateDataProcessPreview,
|
||||
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
|
||||
let allowLeave = false
|
||||
const {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
|
||||
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
|
||||
return 'fa-arrow-right'
|
||||
})
|
||||
|
||||
const previousStepLabel = computed(() => currentStep.value > 0
|
||||
? WIZARD_STEPS[currentStep.value - 1].title
|
||||
: '')
|
||||
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
|
||||
}
|
||||
|
||||
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
const sourceLocator = item.quality_score?.source_locator
|
||||
return {
|
||||
id: String(item.id),
|
||||
sourceFileId: String(item.source_file_id),
|
||||
originalContent: item.original_content,
|
||||
editedContent: item.edited_content,
|
||||
savedEditedContent: item.edited_content,
|
||||
sourceStart: item.source_start,
|
||||
sourceEnd: item.source_end,
|
||||
sourceStartLine: item.source_start_line,
|
||||
sourceEndLine: item.source_end_line,
|
||||
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
|
||||
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
|
||||
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
|
||||
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
|
||||
tokenCount: item.token_count,
|
||||
status: item.status,
|
||||
sourcePages: Array.isArray(item.quality_score?.source_pages)
|
||||
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
|
||||
: [],
|
||||
sourceLocator,
|
||||
headingPath: Array.isArray(item.quality_score?.heading_path)
|
||||
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
|
||||
uploadedFiles.value.push({
|
||||
uid: localUid,
|
||||
rawFile: raw,
|
||||
name: raw.name,
|
||||
size: raw.size,
|
||||
count: 0,
|
||||
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
previewProgress: 0,
|
||||
})
|
||||
dirty.value = true
|
||||
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
|
||||
enqueueSourceUpload({ uid: localUid, file: raw })
|
||||
}
|
||||
|
||||
async function useSampleFile() {
|
||||
@@ -488,11 +493,8 @@ async function handlePullData() {
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
|
||||
const newFiles: UploadedDataFile[] = []
|
||||
for (const file of response.files) {
|
||||
const source = await getDataProcessSourceContent(taskId.value, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
newFiles.push(mapDataProcessSourceFile(file, source.content))
|
||||
const content = await loadCanonicalSourceContent(taskId.value, file.id)
|
||||
newFiles.push(mapDataProcessSourceFile(file, content))
|
||||
}
|
||||
uploadedFiles.value.push(...newFiles)
|
||||
externalConnected.value = true
|
||||
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
|
||||
function updatePreviewContent(id: string, value: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
const isManual = isManualPreviewItem(item)
|
||||
item.editedContent = value
|
||||
item.tokenCount = estimateTokenCount(value)
|
||||
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
||||
item.status = !value.trim()
|
||||
? 'invalid'
|
||||
: value === item.originalContent
|
||||
? 'original'
|
||||
: isManual ? 'manual' : 'modified'
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
|
||||
|
||||
function restorePreviewItem(id: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item || item.sourceStart == null) return
|
||||
if (!item || isManualPreviewItem(item)) return
|
||||
item.editedContent = item.originalContent
|
||||
item.tokenCount = estimateTokenCount(item.originalContent)
|
||||
item.status = 'original'
|
||||
@@ -897,7 +904,7 @@ async function handleBack() {
|
||||
ElMessage.warning('请等待当前文件切分完成')
|
||||
return
|
||||
}
|
||||
if (currentStepId.value === 'generate') return
|
||||
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
|
||||
if (currentStep.value > 0) {
|
||||
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
|
||||
if (!targetStep) return
|
||||
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
|
||||
if (sourceTask.status === 'running') resumeStep = 'generate'
|
||||
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
|
||||
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') {
|
||||
const resume = resumeGeneration()
|
||||
goToStep(resumeStep)
|
||||
await resume
|
||||
return
|
||||
}
|
||||
goToStep(resumeStep)
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
@@ -1154,7 +1166,7 @@ onMounted(() => {
|
||||
<div class="footer-left">
|
||||
<el-button
|
||||
v-if="currentStep > 0"
|
||||
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
|
||||
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
|
||||
@click="handleBack"
|
||||
>
|
||||
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回:{{ previousStepLabel }}
|
||||
@@ -1167,8 +1179,8 @@ onMounted(() => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
getDataProcessResults,
|
||||
getDataProcessTask,
|
||||
publishDataProcess,
|
||||
repeatDataProcessTask,
|
||||
restoreDataProcessResult,
|
||||
updateDataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -42,6 +43,8 @@ const savingResult = ref(false)
|
||||
const restoringResultId = ref<string | number | null>(null)
|
||||
const publishDialogVisible = ref(false)
|
||||
const publishing = ref(false)
|
||||
const repeatGenerating = ref(false)
|
||||
const repeatRequestId = ref('')
|
||||
const configExpanded = ref(false)
|
||||
const resultCellTooltipOptions = {
|
||||
popperClass: 'data-process-result-tooltip',
|
||||
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
|
||||
preserve_context: '保留上下文',
|
||||
}
|
||||
|
||||
const structuredPreprocessOptionKeys = new Set([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
|
||||
function formatStructuredPreprocessOptions(value: unknown[]) {
|
||||
const options = [...new Set(value.map((item) => String(item)))]
|
||||
const selected = new Set(options)
|
||||
const consumed = new Set<string>()
|
||||
const labels: string[] = []
|
||||
|
||||
function appendGroup(values: string[], groupLabel: string) {
|
||||
const selectedValues = values.filter((item) => selected.has(item))
|
||||
selectedValues.forEach((item) => consumed.add(item))
|
||||
if (selectedValues.length === values.length) {
|
||||
labels.push(groupLabel)
|
||||
return
|
||||
}
|
||||
selectedValues.forEach((item) => {
|
||||
labels.push(`${preprocessOptionLabelMap[item] || item}(历史部分配置)`)
|
||||
})
|
||||
}
|
||||
|
||||
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
|
||||
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
|
||||
|
||||
if (selected.has('desensitize')) {
|
||||
consumed.add('desensitize')
|
||||
labels.push('敏感信息脱敏')
|
||||
}
|
||||
if (selected.has('filter_anomaly')) {
|
||||
consumed.add('filter_anomaly')
|
||||
labels.push('异常数据过滤(历史规则)')
|
||||
}
|
||||
|
||||
options.forEach((item) => {
|
||||
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
|
||||
})
|
||||
return labels.length ? labels.join('、') : '-'
|
||||
}
|
||||
|
||||
function numeric(value: unknown) {
|
||||
const parsed = typeof value === 'number' ? value : Number(value)
|
||||
return Number.isFinite(parsed) ? parsed : 0
|
||||
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|
||||
|| status === 'stopped'
|
||||
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
|
||||
})
|
||||
const canRepeatGeneration = computed(() => (
|
||||
detail.value?.status === 'completed'
|
||||
&& detail.value.results_confirmed !== false
|
||||
&& previewCount.value > 0
|
||||
))
|
||||
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
|
||||
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
|
||||
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
|
||||
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (key === 'preprocess_options') {
|
||||
return value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
const containsStructuredOption = value.some((item) => (
|
||||
structuredPreprocessOptionKeys.has(String(item))
|
||||
))
|
||||
return containsStructuredOption
|
||||
? formatStructuredPreprocessOptions(value)
|
||||
: value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
}
|
||||
return value.length ? value.join('、') : '-'
|
||||
}
|
||||
@@ -503,6 +561,48 @@ function startRegeneration() {
|
||||
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
|
||||
}
|
||||
|
||||
function createRepeatRequestId() {
|
||||
if (typeof globalThis.crypto?.randomUUID === 'function') {
|
||||
return globalThis.crypto.randomUUID()
|
||||
}
|
||||
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
|
||||
}
|
||||
|
||||
async function repeatGeneration() {
|
||||
if (!detail.value?.updated_at || repeatGenerating.value) return
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
'系统会复制当前配置、源文件和切分结果,创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
|
||||
'按原配置再生成一批?',
|
||||
{
|
||||
confirmButtonText: '创建并开始生成',
|
||||
cancelButtonText: '取消',
|
||||
type: 'info',
|
||||
},
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
repeatGenerating.value = true
|
||||
repeatRequestId.value ||= createRepeatRequestId()
|
||||
try {
|
||||
const repeated = await repeatDataProcessTask(taskId.value, {
|
||||
expected_updated_at: detail.value.updated_at,
|
||||
request_id: repeatRequestId.value,
|
||||
})
|
||||
ElMessage.success(repeated.created ? '已创建新任务,正在后台生成' : '已恢复此前创建的新任务')
|
||||
await router.push({
|
||||
name: 'data-process-workflow',
|
||||
params: { id: repeated.task.id },
|
||||
})
|
||||
} catch {
|
||||
// 保留幂等请求 ID;网络超时后再次点击不会重复创建任务。
|
||||
} finally {
|
||||
repeatGenerating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch([currentPage, pageSize], () => void loadResults())
|
||||
|
||||
onMounted(loadPage)
|
||||
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
|
||||
<el-tag :type="displayStatus.type" size="small" effect="light">
|
||||
{{ displayStatus.label }}
|
||||
</el-tag>
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
|
||||
</el-button>
|
||||
<div class="heading-actions">
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRepeatGeneration"
|
||||
type="primary"
|
||||
:loading="repeatGenerating"
|
||||
:disabled="repeatGenerating"
|
||||
@click="repeatGeneration"
|
||||
>
|
||||
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
type="warning"
|
||||
plain
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<p>{{ detail.description || '暂无任务描述' }}</p>
|
||||
<dl class="heading-meta">
|
||||
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
|
||||
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
|
||||
}
|
||||
|
||||
.publish-button { margin-left: auto; }
|
||||
.heading-actions {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
|
||||
:deep(.el-button + .el-button) { margin-left: 0; }
|
||||
}
|
||||
.load-state-actions { display: flex; gap: 10px; }
|
||||
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
|
||||
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
|
||||
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
|
||||
@media (max-width: 720px) {
|
||||
.metric-grid, .config-grid { grid-template-columns: 1fr; }
|
||||
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
|
||||
.publish-button { width: 100%; margin-left: 0; }
|
||||
.heading-actions { width: 100%; margin-left: 0; }
|
||||
.heading-actions :deep(.el-button) { width: 100%; }
|
||||
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
|
||||
.result-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.result-filters { padding: 0 16px 16px; flex-direction: column; }
|
||||
|
||||
@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
const selectedXlsxLocator = computed(() => {
|
||||
const locator = props.selectedItem?.sourceLocator
|
||||
if (!locator) return null
|
||||
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
|
||||
const hasRow = locator.row_number != null || locator.sheet_record_index != null
|
||||
return hasSheet && hasRow ? locator : null
|
||||
})
|
||||
const visibleRowRange = computed(() => {
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
|
||||
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
|
||||
})
|
||||
|
||||
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
|
||||
const locator = selectedXlsxLocator.value
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (locator && sheet) {
|
||||
const sheetMatches = locator.sheet_index != null
|
||||
? sheet.index === locator.sheet_index
|
||||
: sheet.name === locator.sheet_name
|
||||
if (!sheetMatches) return false
|
||||
if (locator.row_number != null) return row.row_number === locator.row_number
|
||||
return row.record_index === locator.sheet_record_index
|
||||
}
|
||||
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
|
||||
}
|
||||
|
||||
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
|
||||
async function loadPreview(options: { reset?: boolean } = {}) {
|
||||
const sequence = ++loadSequence
|
||||
if (options.reset) {
|
||||
activeSheetIndex.value = 0
|
||||
pageOffset.value = 0
|
||||
const locator = selectedXlsxLocator.value
|
||||
activeSheetIndex.value = locator?.sheet_index ?? 0
|
||||
pageOffset.value = locator?.sheet_record_index == null
|
||||
? 0
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
preview.value = null
|
||||
}
|
||||
errorMessage.value = ''
|
||||
@@ -178,8 +198,34 @@ watch(
|
||||
)
|
||||
|
||||
watch(
|
||||
() => props.selectedItem?.id,
|
||||
() => void locateSelectedItem(),
|
||||
() => [
|
||||
props.selectedItem?.id,
|
||||
props.selectedItem?.sourceLocator?.sheet_index,
|
||||
props.selectedItem?.sourceLocator?.sheet_record_index,
|
||||
props.selectedItem?.sourceLocator?.row_number,
|
||||
],
|
||||
() => {
|
||||
const locator = selectedXlsxLocator.value
|
||||
if (!locator || isDocx.value) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
|
||||
const targetOffset = locator.sheet_record_index == null
|
||||
? pageOffset.value
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
const activeSheet = xlsxPreview.value?.active_sheet
|
||||
if (
|
||||
activeSheet?.index === targetSheet
|
||||
&& activeSheet.offset === targetOffset
|
||||
) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
activeSheetIndex.value = targetSheet
|
||||
pageOffset.value = targetOffset
|
||||
void loadPreview()
|
||||
},
|
||||
)
|
||||
</script>
|
||||
|
||||
@@ -301,6 +347,8 @@ watch(
|
||||
:key="row.row_number"
|
||||
class="xlsx-row"
|
||||
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
|
||||
:data-row-number="row.row_number"
|
||||
:data-record-index="row.record_index"
|
||||
>
|
||||
<th class="row-number-cell">{{ row.row_number }}</th>
|
||||
<td
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
import OfficeSourceViewer from './OfficeSourceViewer.vue'
|
||||
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||
import { sourceLines } from './previewModel'
|
||||
import {
|
||||
isManualPreviewItem,
|
||||
sourceLineNumberAtOffset,
|
||||
sourceLineWindow,
|
||||
} from './previewModel'
|
||||
import type { PreviewItem, ProcessType } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
|
||||
const search = ref('')
|
||||
const currentPage = ref(1)
|
||||
const PREVIEW_PAGE_SIZE = 10
|
||||
const SOURCE_LINE_RENDER_LIMIT = 240
|
||||
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
|
||||
const sourceWindowStartLine = ref(1)
|
||||
const editingItemId = ref<string | null>(null)
|
||||
const editorDraft = ref('')
|
||||
const lines = computed(() => sourceLines(props.sourceText))
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||
const normalizedFileFormat = computed(() => (
|
||||
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
|
||||
))
|
||||
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
|
||||
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
|
||||
const selectedSourceOffset = computed(() => {
|
||||
const item = selectedItem.value
|
||||
return item ? sourceOffsetRange(item)?.start ?? null : null
|
||||
})
|
||||
const selectedSourceLine = computed(() => {
|
||||
const item = selectedItem.value
|
||||
if (!item) return null
|
||||
return sourceLineRange(item)?.start
|
||||
?? (selectedSourceOffset.value == null
|
||||
? null
|
||||
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
|
||||
})
|
||||
const visibleSourceWindow = computed(() => sourceLineWindow(
|
||||
props.sourceText,
|
||||
sourceWindowStartLine.value,
|
||||
SOURCE_LINE_RENDER_LIMIT,
|
||||
SOURCE_LINE_CHARACTER_LIMIT,
|
||||
selectedSourceLine.value,
|
||||
selectedSourceOffset.value,
|
||||
))
|
||||
const lines = computed(() => visibleSourceWindow.value.lines)
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !search.value.trim()
|
||||
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
|
||||
|
||||
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
|
||||
|
||||
function isLineHighlighted(lineStart: number, lineEnd: number) {
|
||||
function sourceLineRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
|
||||
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
|
||||
return start == null ? null : { start, end: end ?? start }
|
||||
}
|
||||
|
||||
function sourceOffsetRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.source_start ?? item.sourceStart
|
||||
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
|
||||
return start == null ? null : { start, end: Math.max(start, end ?? start) }
|
||||
}
|
||||
|
||||
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
|
||||
const item = selectedItem.value
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
|
||||
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
|
||||
if (!item) return false
|
||||
const lineRange = sourceLineRange(item)
|
||||
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
|
||||
const offsetRange = sourceOffsetRange(item)
|
||||
if (!offsetRange) return false
|
||||
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
|
||||
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
|
||||
}
|
||||
|
||||
function selectItem(id: string) {
|
||||
@@ -107,34 +151,88 @@ watch(search, () => {
|
||||
|
||||
watch(() => props.selectedFileId, closeEditor)
|
||||
|
||||
watch(selectedItem, async (item) => {
|
||||
watch([selectedItem, () => props.sourceText], async ([item]) => {
|
||||
if (!item) return
|
||||
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
|
||||
if (visibleIndex >= 0) {
|
||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||
}
|
||||
|
||||
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
|
||||
if (isPdfSource.value || isOfficeSource.value) return
|
||||
const itemLineRange = sourceLineRange(item)
|
||||
const itemOffsetRange = sourceOffsetRange(item)
|
||||
if (!itemLineRange && !itemOffsetRange) {
|
||||
sourceWindowStartLine.value = 1
|
||||
return
|
||||
}
|
||||
const targetLine = selectedSourceLine.value
|
||||
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
|
||||
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
|
||||
await nextTick()
|
||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||
const exactTarget = sourceViewerRef.value
|
||||
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
|
||||
const target = exactTarget
|
||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
|
||||
}, { immediate: true })
|
||||
|
||||
async function showPreviousSourceWindow() {
|
||||
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
async function showNextSourceWindow() {
|
||||
if (!visibleSourceWindow.value.hasMore) return
|
||||
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
function itemNumber(item: PreviewItem) {
|
||||
return props.items.findIndex((entry) => entry.id === item.id) + 1
|
||||
}
|
||||
|
||||
function lineRange(item: PreviewItem) {
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
return first === last ? `来源:第 ${first} 页` : `来源:第 ${first}–${last} 页`
|
||||
if (isManualPreviewItem(item)) return '手动新增,无源文件定位'
|
||||
|
||||
const locator = item.sourceLocator
|
||||
const locatedLines = sourceLineRange(item)
|
||||
if (props.processType === 'unstructured') {
|
||||
const parts: string[] = []
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
|
||||
}
|
||||
if (locatedLines) {
|
||||
parts.push(
|
||||
locatedLines.start === locatedLines.end
|
||||
? `第 ${locatedLines.start} 行`
|
||||
: `第 ${locatedLines.start}–${locatedLines.end} 行`,
|
||||
)
|
||||
}
|
||||
if (item.headingPath?.length) parts.push(`章节:${item.headingPath.join(' / ')}`)
|
||||
return parts.length ? `来源:${parts.join(' · ')}` : '来源:源文件内容(无精确定位)'
|
||||
}
|
||||
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增,无源文件定位'
|
||||
return item.sourceStartLine === item.sourceEndLine
|
||||
? `来源:第 ${item.sourceStartLine} 行`
|
||||
: `来源:第 ${item.sourceStartLine}–${item.sourceEndLine} 行`
|
||||
|
||||
if (locator?.kind === 'xlsx') {
|
||||
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
|
||||
return locator.row_number != null
|
||||
? `来源:${sheet} · 第 ${locator.row_number} 行`
|
||||
: `来源:${sheet}`
|
||||
}
|
||||
if (locator?.kind === 'json') {
|
||||
return locator.json_pointer
|
||||
? `来源:JSON 路径 ${locator.json_pointer}`
|
||||
: '来源:JSON 根对象'
|
||||
}
|
||||
if (locatedLines) {
|
||||
return locatedLines.start === locatedLines.end
|
||||
? `来源:第 ${locatedLines.start} 行`
|
||||
: `来源:第 ${locatedLines.start}–${locatedLines.end} 行`
|
||||
}
|
||||
return '来源:源文件记录'
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
|
||||
<div>
|
||||
<strong>源文件 · {{ fileName }}</strong>
|
||||
</div>
|
||||
<div
|
||||
v-if="!isPdfSource && !isOfficeSource && lines.length"
|
||||
class="source-window-controls"
|
||||
aria-label="源文件行窗口"
|
||||
>
|
||||
<span>第 {{ visibleSourceWindow.startLine }}–{{ visibleSourceWindow.endLine }} 行</span>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看上一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasPrevious"
|
||||
@click="showPreviousSourceWindow"
|
||||
>
|
||||
上一段
|
||||
</el-button>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看下一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasMore"
|
||||
@click="showNextSourceWindow"
|
||||
>
|
||||
下一段
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<PdfSourceViewer
|
||||
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
|
||||
v-for="line in lines"
|
||||
:key="line.number"
|
||||
class="source-line"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
|
||||
:data-source-start="line.start"
|
||||
:data-line-number="line.number"
|
||||
>
|
||||
<span class="line-number">{{ line.number }}</span>
|
||||
<span class="line-content">{{ line.content || ' ' }}</span>
|
||||
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
|
||||
/>
|
||||
<div class="editor-actions">
|
||||
<el-button
|
||||
v-if="editingItem.sourceStart != null"
|
||||
v-if="!isManualPreviewItem(editingItem)"
|
||||
link
|
||||
@click="restoreItem"
|
||||
>
|
||||
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
|
||||
}
|
||||
}
|
||||
|
||||
.source-window-controls {
|
||||
flex: none;
|
||||
gap: 2px !important;
|
||||
|
||||
> span {
|
||||
margin-right: 4px;
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
:deep(.el-button) {
|
||||
margin-left: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.source-viewer {
|
||||
flex: 1;
|
||||
height: 538px;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
@@ -21,27 +22,32 @@ const emit = defineEmits<{
|
||||
'update:options': [value: StructuredProcessOptions]
|
||||
}>()
|
||||
|
||||
const PREPROCESS_OPTIONS: Array<{
|
||||
value: PreprocessOption
|
||||
const PREPROCESS_GROUPS: Array<{
|
||||
values: PreprocessOption[]
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
||||
{
|
||||
value: 'detect_structure',
|
||||
label: '嵌套结构展平',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||
values: ['clean_invalid', 'deduplicate'],
|
||||
label: '数据清洗',
|
||||
description: '清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
|
||||
},
|
||||
{
|
||||
value: 'deduplicate',
|
||||
label: '重复记录去重',
|
||||
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||
values: ['detect_structure', 'normalize_format'],
|
||||
label: '结构标准化',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
|
||||
},
|
||||
{
|
||||
values: ['desensitize'],
|
||||
label: '敏感信息脱敏',
|
||||
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
|
||||
},
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||
]
|
||||
|
||||
const legacyAnomalyFilterEnabled = computed(() => (
|
||||
props.options.preprocessOptions.includes('filter_anomaly')
|
||||
))
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
field: K,
|
||||
value: StructuredProcessOptions[K],
|
||||
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
|
||||
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
(option): option is PreprocessOption => (
|
||||
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||
),
|
||||
)))
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
function selectedCount(values: PreprocessOption[]) {
|
||||
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
|
||||
}
|
||||
|
||||
function groupSelected(values: PreprocessOption[]) {
|
||||
return selectedCount(values) === values.length
|
||||
}
|
||||
|
||||
function groupIndeterminate(values: PreprocessOption[]) {
|
||||
const count = selectedCount(values)
|
||||
return count > 0 && count < values.length
|
||||
}
|
||||
|
||||
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
|
||||
const next = new Set(props.options.preprocessOptions)
|
||||
values.forEach((value) => {
|
||||
if (Boolean(checked)) next.add(value)
|
||||
else next.delete(value)
|
||||
})
|
||||
updateField('preprocessOptions', [...next])
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>选择在生成问答对之前需要执行的数据处理方式</p>
|
||||
<p>默认不执行预处理,请按数据情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-checkbox-group
|
||||
:model-value="options.preprocessOptions"
|
||||
class="preprocess-option-grid"
|
||||
@update:model-value="updatePreprocessOptions"
|
||||
>
|
||||
<el-checkbox
|
||||
v-for="option in PREPROCESS_OPTIONS"
|
||||
:key="option.value"
|
||||
:value="option.value"
|
||||
<div class="preprocess-option-grid">
|
||||
<label
|
||||
v-for="group in PREPROCESS_GROUPS"
|
||||
:key="group.label"
|
||||
class="preprocess-option"
|
||||
:class="{ 'is-checked': groupSelected(group.values) }"
|
||||
>
|
||||
<el-checkbox
|
||||
:model-value="groupSelected(group.values)"
|
||||
:indeterminate="groupIndeterminate(group.values)"
|
||||
@update:model-value="updatePreprocessGroup(group.values, $event)"
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>{{ option.label }}</strong>
|
||||
<small>{{ option.description }}</small>
|
||||
<strong>{{ group.label }}</strong>
|
||||
<small>{{ group.description }}</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</label>
|
||||
</div>
|
||||
<el-alert
|
||||
v-if="legacyAnomalyFilterEnabled"
|
||||
class="legacy-preprocess-alert"
|
||||
type="warning"
|
||||
:closable="false"
|
||||
title="该历史任务仍启用了已停用的“异常数据过滤”;为保证结果可复现,本次继续保留"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="form-section generation-options-section">
|
||||
|
||||
@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认启用结构感知的推荐策略,只需决定是否需要脱敏</p>
|
||||
<p>默认不执行预处理,请按文档情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="preprocess-option-grid">
|
||||
|
||||
@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
preprocessOptions: [],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
|
||||
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
preprocessOptions: [],
|
||||
chunkMethod: 'layout_hybrid',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
semanticBreakpointPercentile: 95,
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
const supportedPreprocessOptions = new Set<PreprocessOption>([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
? Array.from(new Set(preprocessOptions.map(String).filter(
|
||||
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
|
||||
)))
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { SourceLine } from './types'
|
||||
import type { PreviewItem, SourceLine } from './types'
|
||||
|
||||
/** 仅用于“使用示例”上传;正式预览和切片全部由后端生成。 */
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
|
||||
'答:复利是将上一期利息加入本金,再计算下一期利息。',
|
||||
].join('\n')
|
||||
|
||||
/**
|
||||
* 把后端返回的字符偏移映射为源文件行,仅负责界面高亮,不参与切片。
|
||||
*/
|
||||
export function sourceLines(sourceText: string): SourceLine[] {
|
||||
const rawLines = sourceText.split('\n')
|
||||
let cursor = 0
|
||||
export interface SourceLineWindow {
|
||||
lines: SourceLine[]
|
||||
startLine: number
|
||||
endLine: number
|
||||
hasPrevious: boolean
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
return rawLines.map((content, index) => {
|
||||
const start = cursor
|
||||
const end = start + content.length
|
||||
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
|
||||
return { number: index + 1, content, start, end }
|
||||
})
|
||||
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
|
||||
let length = 0
|
||||
let index = start
|
||||
while (index < end) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
length += 1
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
function advanceCodePoints(value: string, start: number, end: number, count: number) {
|
||||
let index = start
|
||||
let remaining = Math.max(0, count)
|
||||
while (index < end && remaining > 0) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
remaining -= 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* 只扫描并返回当前可见行窗口,不对全文 split,避免大文件生成巨量字符串数组。
|
||||
* 字符定位场景可开启 code point 偏移,以与后端 Python 的字符计数保持一致。
|
||||
*/
|
||||
export function sourceLineWindow(
|
||||
sourceText: string,
|
||||
requestedStartLine: number,
|
||||
maxLines: number,
|
||||
maxCharactersPerLine: number,
|
||||
focusLine: number | null = null,
|
||||
focusOffset: number | null = null,
|
||||
): SourceLineWindow {
|
||||
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
|
||||
const limit = Math.max(1, Math.trunc(maxLines) || 1)
|
||||
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
|
||||
const trackUnicodeOffsets = focusOffset != null
|
||||
const lines: SourceLine[] = []
|
||||
let lineNumber = 1
|
||||
let jsCursor = 0
|
||||
let sourceCursor = 0
|
||||
|
||||
while (jsCursor <= sourceText.length && lineNumber < startLine) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
sourceCursor = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
|
||||
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
while (jsCursor <= sourceText.length && lines.length < limit) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
const fullSourceEnd = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
|
||||
: jsEnd
|
||||
const focusedStart = focusLine === lineNumber && focusOffset != null
|
||||
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
|
||||
: sourceCursor
|
||||
const segmentSourceStart = Math.min(
|
||||
focusedStart,
|
||||
Math.max(sourceCursor, fullSourceEnd - characterLimit),
|
||||
)
|
||||
const relativeSegmentStart = trackUnicodeOffsets
|
||||
? segmentSourceStart - sourceCursor
|
||||
: Math.max(0, segmentSourceStart - jsCursor)
|
||||
const segmentJsStart = advanceCodePoints(
|
||||
sourceText,
|
||||
jsCursor,
|
||||
jsEnd,
|
||||
relativeSegmentStart,
|
||||
)
|
||||
const segmentJsEnd = advanceCodePoints(
|
||||
sourceText,
|
||||
segmentJsStart,
|
||||
jsEnd,
|
||||
characterLimit,
|
||||
)
|
||||
const segmentLength = trackUnicodeOffsets
|
||||
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
|
||||
: segmentJsEnd - segmentJsStart
|
||||
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
|
||||
const end = start + segmentLength
|
||||
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
|
||||
lines.push({ number: lineNumber, content, start, end })
|
||||
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
return {
|
||||
lines,
|
||||
startLine: lines[0]?.number ?? startLine,
|
||||
endLine: lines[lines.length - 1]?.number ?? startLine,
|
||||
hasPrevious: startLine > 1,
|
||||
hasMore: jsCursor <= sourceText.length,
|
||||
}
|
||||
}
|
||||
|
||||
/** 根据后端 code point 偏移查找物理行号,不构建全文行数组。 */
|
||||
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
|
||||
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
|
||||
let offset = 0
|
||||
let lineNumber = 1
|
||||
for (const character of sourceText) {
|
||||
if (offset >= normalizedOffset) break
|
||||
if (character === '\n') lineNumber += 1
|
||||
offset += 1
|
||||
}
|
||||
return lineNumber
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动新增项可能先以空内容保存为 invalid,编辑后又由后端标记为 modified,
|
||||
* 因此不能只依赖可变的 status;空原文且完全没有来源定位才是稳定兜底。
|
||||
*/
|
||||
export function isManualPreviewItem(item: PreviewItem): boolean {
|
||||
const hasSourceLocation = item.sourceStart != null
|
||||
|| item.sourceEnd != null
|
||||
|| item.sourceStartLine != null
|
||||
|| item.sourceEndLine != null
|
||||
|| Boolean(item.sourcePages?.length)
|
||||
|| Boolean(item.sourceLocator)
|
||||
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
|
||||
}
|
||||
|
||||
/** 与后端预览 token 估算规则一致,仅用于编辑中的即时计数。 */
|
||||
|
||||
@@ -24,6 +24,7 @@ export type PreprocessOption =
|
||||
| 'detect_structure'
|
||||
| 'deduplicate'
|
||||
| 'normalize_format'
|
||||
/** 仅用于恢复历史任务,新任务界面不再提供。 */
|
||||
| 'filter_anomaly'
|
||||
| 'desensitize'
|
||||
|
||||
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
rawFile?: File
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
@@ -118,6 +118,22 @@ export interface SourceLine {
|
||||
end: number
|
||||
}
|
||||
|
||||
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
|
||||
|
||||
export interface PreviewSourceLocator {
|
||||
kind: PreviewSourceLocatorKind
|
||||
record_index?: number | null
|
||||
start_line?: number | null
|
||||
end_line?: number | null
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
json_pointer?: string | null
|
||||
sheet_index?: number | null
|
||||
sheet_name?: string | null
|
||||
row_number?: number | null
|
||||
sheet_record_index?: number | null
|
||||
}
|
||||
|
||||
export interface PreviewItem {
|
||||
id: string
|
||||
sourceFileId: string
|
||||
@@ -129,6 +145,8 @@ export interface PreviewItem {
|
||||
sourceStartLine: number | null
|
||||
sourceEndLine: number | null
|
||||
sourcePages?: number[]
|
||||
sourceLocator?: PreviewSourceLocator
|
||||
headingPath?: string[]
|
||||
tokenCount: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
qualityScore?: number
|
||||
|
||||
@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
let generationTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let generationRun = 0
|
||||
let pollFailureCount = 0
|
||||
let generationStarting = false
|
||||
const generationStarting = ref(false)
|
||||
const generationRestoring = ref(false)
|
||||
const canReturnFromGeneration = computed(() => (
|
||||
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
|
||||
))
|
||||
|
||||
function stopGenerationTimer() {
|
||||
generationRun += 1
|
||||
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
}
|
||||
|
||||
async function startGeneration() {
|
||||
if (generationStarting || generation.status === 'running') return false
|
||||
if (generationStarting.value || generation.status === 'running') return false
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) {
|
||||
ElMessage.error('任务尚未创建,请返回上一步重试')
|
||||
return false
|
||||
}
|
||||
|
||||
generationStarting = true
|
||||
generationStarting.value = true
|
||||
let runId: number | null = null
|
||||
try {
|
||||
const canStart = await bindings.beforeGenerate?.()
|
||||
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
generation.message = error instanceof Error ? error.message : '启动数据处理失败,请重试。'
|
||||
return false
|
||||
} finally {
|
||||
generationStarting = false
|
||||
generationStarting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
generationRestoring.value = true
|
||||
try {
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (activeRunId !== generationRun) return
|
||||
if (progress.status === 'running') {
|
||||
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
} catch (error) {
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败,请重试。'
|
||||
} finally {
|
||||
generationRestoring.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
|
||||
return {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
|
||||
@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import {
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
regenerateDataProcessTask,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -15,7 +14,10 @@ import {
|
||||
createStructuredOptionsFromConfig,
|
||||
createUnstructuredOptionsFromConfig,
|
||||
} from './dataProcessCreateState'
|
||||
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
} from './useDataProcessSourceUpload'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
@@ -50,24 +52,6 @@ interface RegenerationBindings {
|
||||
resetDownstream: () => void
|
||||
}
|
||||
|
||||
async function loadSourceContent(taskId: string, fileId: string | number) {
|
||||
const chunks: string[] = []
|
||||
let startLine = 1
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
start_line: startLine,
|
||||
line_count: 10_000,
|
||||
})
|
||||
chunks.push(source.content || '')
|
||||
if (!source.has_more) break
|
||||
const nextLine = Number(source.end_line || startLine) + 1
|
||||
if (nextLine <= startLine) break
|
||||
startLine = nextLine
|
||||
}
|
||||
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
|
||||
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
|
||||
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
|
||||
const taskId = String(task.id)
|
||||
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
|
||||
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
|
||||
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
|
||||
)))
|
||||
bindings.previewItems.value = preservePreviews
|
||||
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
function unicodeCodePointLength(value: string) {
|
||||
let length = 0
|
||||
for (const _character of value) length += 1
|
||||
return length
|
||||
}
|
||||
|
||||
/** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */
|
||||
export async function loadCanonicalSourceContent(
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) {
|
||||
const chunks: string[] = []
|
||||
let offset = 0
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
offset,
|
||||
limit: SOURCE_CONTENT_PAGE_CHARS,
|
||||
})
|
||||
const content = source.content || ''
|
||||
chunks.push(content)
|
||||
if (!source.has_more) break
|
||||
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
|
||||
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试')
|
||||
offset = nextOffset
|
||||
}
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
|
||||
// 先登记后端 ID,确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
try {
|
||||
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
|
||||
} catch {
|
||||
throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试')
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
|
||||
Reference in New Issue
Block a user