feat: 数据处理向导配置体系扩展

新增结构化与非结构化处理选项类型,TaskSetupStep 增加预处理、切分方法、数据集划分等配置 UI,previewModel 实现语义切分与受保护区间算法,CreateView 接入配置状态与草稿持久化并替换为 AppConfirmDialog,回归脚本扩充配置与弹窗断言。
This commit is contained in:
caoxiaozhu
2026-07-11 14:49:10 +08:00
parent 9e16f6358d
commit ddf47eb8fe
5 changed files with 1941 additions and 66 deletions

View File

@@ -4,14 +4,35 @@ import { readFile } from 'node:fs/promises'
import { fileURLToPath } from 'node:url'
import path from 'node:path'
import { parse as parseSfc } from '@vue/compiler-sfc'
import ts from 'typescript'
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue')
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
const viewSource = await readFile(viewPath, 'utf8')
const layoutSource = await readFile(layoutPath, 'utf8')
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
for (const marker of ['<Teleport to="body">', 'role="alertdialog"', ':aria-modal="true"', 'handleKeydown', 'Escape']) {
assert.ok(confirmDialogSource.includes(marker), `公共确认弹窗缺少可访问性能力:${marker}`)
}
assert.match(confirmDialogSource, /min-height:\s*44px/, '公共确认弹窗按钮触控区域不足 44px')
assert.match(confirmDialogSource, /focus\(\)/, '公共确认弹窗打开后没有管理键盘焦点')
assert.match(confirmDialogSource, /defineExpose\(\{ open \}\)/, '公共确认弹窗没有暴露 Promise 式 open API')
assert.match(confirmDialogSource, /width:\s*min\(480px,\s*100%\)/, '企业级确认弹窗宽度应保持紧凑的 480px')
assert.match(confirmDialogSource, /border-radius:\s*8px/, '企业级确认弹窗应使用克制的 8px 圆角')
assert.doesNotMatch(confirmDialogSource, /backdrop-filter/, '企业级确认弹窗不应使用装饰性背景模糊')
assert.ok(confirmDialogSource.includes('app-confirm-header'), '企业级确认弹窗缺少独立标题栏')
assert.match(confirmDialogSource, /\.app-confirm-button\s*\{[\s\S]*?height:\s*34px/, '桌面端操作按钮应使用紧凑的 34px 高度')
assert.match(confirmDialogSource, /@media \(max-width: 520px\)[\s\S]*?\.app-confirm-button\s*\{[\s\S]*?min-height:\s*44px/, '移动端操作按钮仍需保留 44px 触控高度')
assert.match(viewSource, /import AppConfirmDialog from '@\/components\/AppConfirmDialog\.vue'/, '创建页没有接入公共确认弹窗')
assert.match(viewSource, /<AppConfirmDialog/, '创建页模板缺少公共确认弹窗实例')
assert.match(viewSource, /onBeforeRouteLeave\(async \(\) =>/, '路由离开确认没有改为异步公共弹窗流程')
assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框')
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
for (const title of ['创建任务', '数据预览', '开始生成', '结果编辑与保存']) {
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
@@ -53,8 +74,8 @@ assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
assert.match(
viewSource,
/buildPreviewItems\(file\.content, processType\.value, String\(file\.uid\)\)/,
'预览没有按文件分别生成',
/buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/,
'预览没有按文件分别生成或未传入非结构化切分配置',
)
for (const marker of [
@@ -99,6 +120,304 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第二步缺少窄
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const taskSetupSource = await readFile(taskSetupPath, 'utf8')
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
for (const option of [
'清理无效数据',
'识别表格结构',
'重复数据去重',
'数据格式标准化',
'异常数据过滤',
'敏感信息脱敏',
]) {
assert.ok(taskSetupSource.includes(option), `结构化预处理缺少选项:${option}`)
}
assert.ok(taskSetupSource.includes('生成选项'), '结构化配置缺少生成选项分类')
assert.ok(taskSetupSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关')
assert.ok(taskSetupSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明')
for (const splitName of ['训练集', '验证集', '测试集']) {
assert.ok(taskSetupSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
}
assert.match(taskSetupSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算')
assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验')
assert.ok(taskSetupSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示')
for (const splitField of ['train', 'validation', 'test']) {
assert.match(
taskSetupSource,
new RegExp(`structuredOptions\\.datasetSplit\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`),
`数据集划分字段 ${splitField} 缺少 0100 的整数限制`,
)
}
assert.match(taskSetupSource, /<el-switch[\s\S]*structuredOptions\.semanticEnrichment/, '语义丰富表达必须使用开关控件')
assert.match(taskSetupSource, /<el-input-number[\s\S]*structuredOptions\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿')
assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
for (const field of [
'preprocessOptions',
'chunkMethod',
'chunkSize',
'chunkOverlap',
'minChunkSize',
'customDelimiter',
'preserveTables',
'preserveCodeBlocks',
'preserveLists',
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(typesSource.includes(field), `非结构化处理选项缺少字段:${field}`)
}
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
for (const option of [
'清理无效内容',
'识别文档结构',
'合并过短内容',
'过滤低质量内容',
'重复内容去重',
'敏感信息脱敏',
'保留上下文信息',
]) {
assert.ok(taskSetupSource.includes(option), `非结构化预处理缺少选项:${option}`)
}
assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) {
assert.ok(taskSetupSource.includes(method), `切分方式缺少选项:${method}`)
}
for (const label of ['切片长度', '重叠长度', '最小切片长度', '完整保留表格', '完整保留代码块', '完整保留列表']) {
assert.ok(taskSetupSource.includes(label), `切分选项缺少配置:${label}`)
}
assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token')
assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token')
assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token')
for (const label of [
'每个切片生成数量',
'上下文范围',
'当前切片',
'相邻切片',
'当前章节',
'问题类型',
'事实问答',
'概念解释',
'操作步骤',
'原因分析',
'综合问答',
'跳过无法回答的内容',
]) {
assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少:${label}`)
}
assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3')
assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验')
assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界')
assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明')
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分')
assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
assert.match(viewSource, /contextScope:\s*'adjacent'/, '默认上下文范围必须为相邻切片')
assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿')
assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识')
assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识')
assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览')
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart)
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
const previewOptionsSource = viewSource.slice(previewOptionsStart, previewOptionsEnd)
for (const field of [
'preprocessOptions',
'chunkMethod',
'chunkSize',
'chunkOverlap',
'minChunkSize',
'customDelimiter',
'preserveTables',
'preserveCodeBlocks',
'preserveLists',
]) {
assert.ok(previewOptionsSource.includes(field), `预览签名缺少切分影响字段:${field}`)
}
for (const field of [
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`)
}
const generationOptionsStart = viewSource.indexOf('function generationAffectingOptions()')
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart)
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd)
for (const field of [
'semanticEnrichment',
'qaPairsPerChunk',
'contextScope',
'generationTypes',
'skipUnanswerable',
'datasetSplit',
]) {
assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`)
}
assert.match(
viewSource,
/watch\(generationOptionsSignature,[\s\S]*?resetDownstream\(\)/,
'生成配置变化后没有仅失效下游结果',
)
for (const mutationFunction of [
'updatePreviewContent',
'restorePreviewItem',
'addPreviewItem',
'removePreviewItem',
]) {
const mutationStart = viewSource.indexOf(`function ${mutationFunction}`)
const mutationEnd = viewSource.indexOf('\nfunction ', mutationStart + 1)
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
}
assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置')
assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑')
const transpiledModel = ts.transpileModule(modelSource, {
compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
}).outputText
const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`)
const longDocument = Array.from(
{ length: 180 },
(_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`,
).join('\n')
const baseUnstructuredOptions = {
preprocessOptions: [],
chunkMethod: 'semantic',
chunkSize: 200,
chunkOverlap: 50,
minChunkSize: 50,
customDelimiter: '',
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 3,
contextScope: 'adjacent',
generationTypes: ['factual'],
skipUnanswerable: true,
datasetSplit: { train: 80, validation: 10, test: 10 },
}
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
const options = {
...baseUnstructuredOptions,
chunkMethod,
customDelimiter: chunkMethod === 'custom' ? '\\n' : '',
}
const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options)
assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`)
assert.ok(
previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent),
`${chunkMethod} 切分方式的来源偏移不准确`,
)
assert.ok(
previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine),
`${chunkMethod} 切分方式的来源行号不准确`,
)
}
const overlapDocument = '甲'.repeat(1200)
const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
})
assert.equal(
overlapItems[0].sourceEnd - overlapItems[1].sourceStart,
100,
'固定长度切分没有按配置保留 50 个估算 Token 的重叠内容',
)
const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}`
const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', {
...baseUnstructuredOptions,
chunkMethod: 'heading',
chunkOverlap: 0,
})
assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片')
assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片')
const customDocument = `${'甲'.repeat(150)}<CUT>${'乙'.repeat(600)}`
const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', {
...baseUnstructuredOptions,
chunkMethod: 'custom',
chunkOverlap: 0,
customDelimiter: '<CUT>',
})
assert.ok(customItems[0].originalContent.endsWith('<CUT>'), '自定义切分未在指定分隔符处结束切片')
function assertProtectedContent(optionField, block, label) {
const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}`
const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
chunkOverlap: 0,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
[optionField]: true,
})
const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, {
...baseUnstructuredOptions,
chunkMethod: 'fixed',
chunkOverlap: 0,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
})
assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`)
assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`)
}
const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n')
const tableBlock = [
'| 字段 | 说明 |',
'| --- | --- |',
...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`),
].join('\n')
const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}:这是需要完整保留的内容。`).join('\n')
assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块')
assertProtectedContent('preserveTables', tableBlock, '表格')
assertProtectedContent('preserveLists', listBlock, '列表')
const samplePreviewItems = previewModelModule.buildPreviewItems(
longDocument,
'unstructured',
'generation-check',
baseUnstructuredOptions,
)
assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片')
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
function findNextStyleBlockStart(source, startIndex) {
let quote = null

View File

@@ -1,17 +1,30 @@
<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, ElMessageBox, type UploadFile } from 'element-plus'
import { ElMessage, type UploadFile } from 'element-plus'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
import type {
ExternalDataSource,
GenerationState,
PreviewItem,
ProcessType,
ResultItem,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './create/types'
const router = useRouter()
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const DRAFT_SCHEMA_VERSION = 2
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与上传源数据' },
@@ -24,6 +37,36 @@ const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('unstructured')
const structuredOptions = ref<StructuredProcessOptions>({
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
})
const unstructuredOptions = ref<UnstructuredProcessOptions>({
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
chunkMethod: 'semantic',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
customDelimiter: '',
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
semanticEnrichment: false,
qaPairsPerChunk: 1,
contextScope: 'adjacent',
generationTypes: ['factual', 'concept', 'comprehensive'],
skipUnanswerable: true,
datasetSplit: { train: 80, validation: 10, test: 10 },
})
interface UploadedDataFile {
uid: number | string
name: string
@@ -103,9 +146,21 @@ const previousStepLabel = computed(() => currentStep.value > 0
function draftSnapshot() {
return {
schemaVersion: DRAFT_SCHEMA_VERSION,
currentStep: currentStep.value,
task: { ...task },
processType: processType.value,
structuredOptions: {
...structuredOptions.value,
preprocessOptions: [...structuredOptions.value.preprocessOptions],
datasetSplit: { ...structuredOptions.value.datasetSplit },
},
unstructuredOptions: {
...unstructuredOptions.value,
preprocessOptions: [...unstructuredOptions.value.preprocessOptions],
generationTypes: [...unstructuredOptions.value.generationTypes],
datasetSplit: { ...unstructuredOptions.value.datasetSplit },
},
uploadedFiles: uploadedFiles.value,
externalSource: { ...externalSource },
previewSignature: previewSignature.value,
@@ -119,6 +174,74 @@ function draftSnapshot() {
}
}
function previewAffectingOptions() {
if (processType.value === 'structured') {
return {
preprocessOptions: structuredOptions.value.preprocessOptions,
}
}
if (processType.value === 'unstructured') {
const {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter,
preserveTables,
preserveCodeBlocks,
preserveLists,
} = unstructuredOptions.value
return {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
preserveTables,
preserveCodeBlocks,
preserveLists,
}
}
return null
}
function generationAffectingOptions() {
if (processType.value === 'structured') {
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value
return { semanticEnrichment, qaPairsPerRow, datasetSplit }
}
if (processType.value === 'unstructured') {
const {
semanticEnrichment,
qaPairsPerChunk,
contextScope,
generationTypes,
skipUnanswerable,
datasetSplit,
} = unstructuredOptions.value
return {
semanticEnrichment,
qaPairsPerChunk,
contextScope,
generationTypes,
skipUnanswerable,
datasetSplit,
}
}
return null
}
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
function buildPreviewSignature() {
const filesSignature = uploadedFiles.value
.map((file) => `${file.uid}:${file.content}`)
.join('|')
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
}
function persistDraft() {
if (restoringDraft.value) return
try {
@@ -128,7 +251,7 @@ function persistDraft() {
}
}
type DraftSnapshot = ReturnType<typeof draftSnapshot> & {
type DraftSnapshot = Partial<ReturnType<typeof draftSnapshot>> & {
fileName?: string
fileSize?: number
fileCount?: number
@@ -141,14 +264,46 @@ function restoreDraft() {
if (!raw) return
const snapshot = JSON.parse(raw) as DraftSnapshot
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
const requiresPreviewMigration = snapshot.schemaVersion !== DRAFT_SCHEMA_VERSION
restoringDraft.value = true
currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
currentStep.value = requiresPreviewMigration
? 0
: Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
task.name = snapshot.task?.name || ''
task.description = snapshot.task?.description || ''
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.structuredOptions) {
structuredOptions.value = {
...structuredOptions.value,
...snapshot.structuredOptions,
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
? snapshot.structuredOptions.preprocessOptions
: structuredOptions.value.preprocessOptions,
datasetSplit: {
...structuredOptions.value.datasetSplit,
...snapshot.structuredOptions.datasetSplit,
},
}
}
if (snapshot.unstructuredOptions) {
unstructuredOptions.value = {
...unstructuredOptions.value,
...snapshot.unstructuredOptions,
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
? snapshot.unstructuredOptions.preprocessOptions
: unstructuredOptions.value.preprocessOptions,
generationTypes: Array.isArray(snapshot.unstructuredOptions.generationTypes)
? snapshot.unstructuredOptions.generationTypes
: unstructuredOptions.value.generationTypes,
datasetSplit: {
...unstructuredOptions.value.datasetSplit,
...snapshot.unstructuredOptions.datasetSplit,
},
}
}
if (snapshot.uploadedFiles) {
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
@@ -163,41 +318,50 @@ function restoreDraft() {
}]
}
previewSignature.value = snapshot.previewSignature || ''
previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || ''
if (snapshot.externalSource) {
Object.assign(externalSource, snapshot.externalSource)
}
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
previewItems.value = Array.isArray(snapshot.previewItems)
previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems)
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
: []
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
selectedPreviewIdsByFile.value = snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = snapshot.selectedResultId || results.value[0]?.id || null
Object.assign(generation, snapshot.generation || {})
selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = requiresPreviewMigration
? null
: snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null
if (!requiresPreviewMigration) Object.assign(generation, snapshot.generation || {})
dirty.value = false
nextTick(() => { restoringDraft.value = false })
ElMessage.info('已恢复上次保存的草稿')
ElMessage.info(requiresPreviewMigration
? '已恢复任务信息,切分规则已更新,请重新生成预览'
: '已恢复上次保存的草稿')
} catch {
localStorage.removeItem(DRAFT_STORAGE_KEY)
}
}
watch(
[() => task.name, () => task.description, processType, externalSource],
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
{ deep: true },
)
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (restoringDraft.value || currentSignature === previousSignature) return
resetDownstream()
})
watch(
[currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
[currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature,
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
results, selectedResultId, generation],
persistDraft,
@@ -330,10 +494,15 @@ async function nextFromCreate() {
return
}
const signature = `${processType.value}:${uploadedFiles.value.map((file) => `${file.uid}:${file.content}`).join('|')}`
const signature = buildPreviewSignature()
if (signature !== previewSignature.value) {
previewItems.value = uploadedFiles.value.flatMap((file) =>
buildPreviewItems(file.content, processType.value, String(file.uid)),
buildPreviewItems(
file.content,
processType.value,
String(file.uid),
processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
),
)
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
@@ -367,6 +536,7 @@ function updatePreviewContent(id: string, value: string) {
item.editedContent = value
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
@@ -376,6 +546,7 @@ function restorePreviewItem(id: string) {
item.editedContent = item.originalContent
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
item.status = 'original'
resetDownstream()
dirty.value = true
}
@@ -395,19 +566,19 @@ function addPreviewItem() {
status: 'manual',
})
selectPreviewItem(id)
resetDownstream()
dirty.value = true
}
async function removePreviewItem(id: string) {
try {
await ElMessageBox.confirm('删除只影响本次处理,不会修改源文件。确认删除吗?', '删除预览内容', {
confirmButtonText: '删除',
cancelButtonText: '取消',
type: 'warning',
})
} catch {
return
}
const confirmed = await confirmDialogRef.value?.open({
title: '删除预览内容',
message: '删除只影响本次处理,不会修改源文件。删除后可重新从源文件生成预览。',
confirmText: '删除',
cancelText: '取消',
tone: 'danger',
})
if (!confirmed) return
const index = previewItems.value.findIndex((item) => item.id === id)
if (index < 0) return
@@ -416,6 +587,7 @@ async function removePreviewItem(id: string) {
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
resetDownstream()
dirty.value = true
}
@@ -436,8 +608,15 @@ function startGeneration() {
stopGenerationTimer()
generation.status = 'success'
generation.message = `已完成 ${previewItems.value.length.toLocaleString()} 条数据处理,可进入结果页检查。`
results.value = createResults(previewItems.value)
results.value = createResults(
previewItems.value,
processType.value === 'structured'
? structuredOptions.value
: processType.value === 'unstructured'
? unstructuredOptions.value
: undefined,
)
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理,可进入结果页检查。`
selectedResultId.value = results.value[0]?.id ?? null
dirty.value = true
ElMessage.success('数据处理完成')
@@ -538,25 +717,29 @@ async function handleCancel() {
router.back()
return
}
try {
await ElMessageBox.confirm('当前存在未保存修改,确定离开吗?', '离开创建任务', {
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
type: 'warning',
})
allowLeave = true
router.back()
} catch {
// 用户继续编辑。
}
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (!confirmed) return
allowLeave = true
router.back()
}
onBeforeRouteLeave((_to, _from, next) => {
if (allowLeave || !dirty.value) {
next()
return
}
next(window.confirm('当前存在未保存修改,确定离开吗?'))
onBeforeRouteLeave(async () => {
if (allowLeave || !dirty.value) return true
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (confirmed) allowLeave = true
return Boolean(confirmed)
})
onBeforeUnmount(stopGenerationTimer)
@@ -599,6 +782,8 @@ onMounted(restoreDraft)
v-model:name="task.name"
v-model:description="task.description"
v-model:process-type="processType"
v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
@@ -673,6 +858,8 @@ onMounted(restoreDraft)
</div>
</footer>
</div>
<AppConfirmDialog ref="confirmDialogRef" />
</template>
<style scoped lang="scss">

View File

@@ -1,7 +1,18 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import type { FormInstance, FormRules, UploadFile } from 'element-plus'
import type { ExternalDataSource, ProcessType } from './types'
import type {
ChunkMethod,
DatasetSplitOptions,
ExternalDataSource,
GenerationContextScope,
PreprocessOption,
ProcessType,
QuestionGenerationType,
StructuredProcessOptions,
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
} from './types'
const props = defineProps<{
name: string
@@ -11,6 +22,8 @@ const props = defineProps<{
externalSource: ExternalDataSource
externalPulling: boolean
externalConnected: boolean
structuredOptions: StructuredProcessOptions
unstructuredOptions: UnstructuredProcessOptions
}>()
const emit = defineEmits<{
@@ -18,6 +31,8 @@ const emit = defineEmits<{
'update:description': [value: string]
'update:processType': [value: ProcessType]
'update:externalSource': [value: ExternalDataSource]
'update:structuredOptions': [value: StructuredProcessOptions]
'update:unstructuredOptions': [value: UnstructuredProcessOptions]
'file-change': [file: UploadFile]
'remove-file': [uid: string | number]
'use-sample': []
@@ -38,16 +53,169 @@ const AUTH_MODES = [
{ value: 'token', label: 'Token' },
]
const PREPROCESS_OPTIONS: Array<{
value: PreprocessOption
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
]
const UNSTRUCTURED_PREPROCESS_OPTIONS: Array<{
value: UnstructuredPreprocessOption
label: string
description: string
}> = [
{ value: 'clean_invalid_content', label: '清理无效内容', description: '清除空段、乱码、页眉页脚和多余空白' },
{ value: 'detect_document_structure', label: '识别文档结构', description: '识别标题、章节、段落及特殊内容块' },
{ value: 'merge_short_content', label: '合并过短内容', description: '将信息不完整的过短段落并入上下文' },
{ value: 'filter_low_quality', label: '过滤低质量内容', description: '过滤广告、导航、无意义重复及信息过少内容' },
{ value: 'deduplicate_content', label: '重复内容去重', description: '识别完全重复或高度相似的段落' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱和证件号等信息' },
{ value: 'preserve_context', label: '保留上下文信息', description: '为切片保留所属文档、章节和标题信息' },
]
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
{ value: 'semantic', label: '自动语义切分' },
{ value: 'heading', label: '按标题和段落' },
{ value: 'fixed', label: '按固定长度' },
{ value: 'custom', label: '自定义分隔符' },
]
const CONTEXT_SCOPES: Array<{ value: GenerationContextScope; label: string }> = [
{ value: 'current', label: '当前切片' },
{ value: 'adjacent', label: '相邻切片' },
{ value: 'section', label: '当前章节' },
]
const GENERATION_TYPES: Array<{ value: QuestionGenerationType; label: string }> = [
{ value: 'factual', label: '事实问答' },
{ value: 'concept', label: '概念解释' },
{ value: 'procedure', label: '操作步骤' },
{ value: 'reasoning', label: '原因分析' },
{ value: 'comprehensive', label: '综合问答' },
]
function updateExternalField<K extends keyof ExternalDataSource>(field: K, value: ExternalDataSource[K]) {
emit('update:externalSource', { ...props.externalSource, [field]: value })
}
function updateStructuredField<K extends keyof StructuredProcessOptions>(
field: K,
value: StructuredProcessOptions[K],
) {
emit('update:structuredOptions', { ...props.structuredOptions, [field]: value })
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter(
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
)
updateStructuredField('preprocessOptions', preprocessOptions)
}
function updateSemanticEnrichment(value: string | number | boolean) {
updateStructuredField('semanticEnrichment', Boolean(value))
}
function updateDatasetSplit(field: keyof DatasetSplitOptions, value: number | undefined) {
updateStructuredField('datasetSplit', {
...props.structuredOptions.datasetSplit,
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
})
}
function updateUnstructuredField<K extends keyof UnstructuredProcessOptions>(
field: K,
value: UnstructuredProcessOptions[K],
) {
emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value })
}
function updateUnstructuredPreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(UNSTRUCTURED_PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter(
(option): option is UnstructuredPreprocessOption => (
typeof option === 'string' && allowedValues.has(option as UnstructuredPreprocessOption)
),
)
updateUnstructuredField('preprocessOptions', preprocessOptions)
}
function updateGenerationTypes(value: Array<string | number | boolean>) {
const allowedValues = new Set(GENERATION_TYPES.map((option) => option.value))
const generationTypes = value.filter(
(option): option is QuestionGenerationType => (
typeof option === 'string' && allowedValues.has(option as QuestionGenerationType)
),
)
updateUnstructuredField('generationTypes', generationTypes)
}
const UNSTRUCTURED_NUMBER_LIMITS = {
chunkSize: { min: 200, max: 2000 },
chunkOverlap: { min: 0, max: 500 },
minChunkSize: { min: 20, max: 500 },
qaPairsPerChunk: { min: 1, max: 3 },
} as const
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
function updateUnstructuredNumber(field: UnstructuredNumberField, value: number | undefined) {
const limits = UNSTRUCTURED_NUMBER_LIMITS[field]
const parsedValue = Number(value)
const nextValue = Number.isFinite(parsedValue) ? parsedValue : limits.min
updateUnstructuredField(field, Math.min(limits.max, Math.max(limits.min, nextValue)))
}
function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value: number | undefined) {
updateUnstructuredField('datasetSplit', {
...props.unstructuredOptions.datasetSplit,
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
})
}
const formRef = ref<FormInstance>()
const formModel = computed(() => ({
name: props.name,
processType: props.processType,
}))
const splitTotal = computed(() => {
const { train, validation, test } = props.structuredOptions.datasetSplit
return train + validation + test
})
const unstructuredSplitTotal = computed(() => {
const { train, validation, test } = props.unstructuredOptions.datasetSplit
return train + validation + test
})
const chunkValidationMessage = computed(() => {
if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) {
return '重叠长度必须小于切片长度'
}
if (props.unstructuredOptions.minChunkSize > props.unstructuredOptions.chunkSize) {
return '最小切片长度不能大于切片长度'
}
if (
props.unstructuredOptions.chunkOverlap + props.unstructuredOptions.minChunkSize
> props.unstructuredOptions.chunkSize
) {
return '重叠长度与最小切片长度之和不能大于切片长度'
}
if (props.unstructuredOptions.chunkMethod === 'custom' && !props.unstructuredOptions.customDelimiter.trim()) {
return '请输入自定义分隔符'
}
return ''
})
const rules: FormRules = {
name: [
{ required: true, message: '请输入任务名称', trigger: 'blur' },
@@ -88,6 +256,12 @@ async function validate() {
if (!formRef.value) return false
try {
await formRef.value.validate()
if (props.processType === 'structured' && splitTotal.value !== 100) return false
if (props.processType === 'unstructured') {
if (unstructuredSplitTotal.value !== 100) return false
if (chunkValidationMessage.value) return false
if (props.unstructuredOptions.generationTypes.length === 0) return false
}
return true
} catch {
return false
@@ -178,6 +352,461 @@ defineExpose({ validate })
</el-form-item>
</div>
<template v-if="processType === 'structured'">
<div class="form-section structured-options-section">
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>选择在生成问答对之前需要执行的数据处理方式</p>
</div>
</div>
<el-checkbox-group
:model-value="structuredOptions.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updatePreprocessOptions"
>
<el-checkbox
v-for="option in PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
class="preprocess-option"
>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</div>
<div class="form-section generation-options-section">
<div class="section-title-row">
<div>
<h3>生成选项</h3>
<p>配置每条结构化记录生成问答对的方式和数量</p>
</div>
</div>
<div class="generation-option-list">
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
<small>使用大模型将问答表述得更自然柔和</small>
</div>
<el-switch
:model-value="structuredOptions.semanticEnrichment"
inline-prompt
active-text=""
inactive-text=""
@update:model-value="updateSemanticEnrichment"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每行生成数量</strong>
<small>每行结构化数据生成的问答对数量</small>
</div>
<el-input-number
:model-value="structuredOptions.qaPairsPerRow"
:min="1"
:max="5"
:step="1"
controls-position="right"
@update:model-value="updateStructuredField('qaPairsPerRow', Number($event) || 1)"
/>
</div>
<div class="generation-option-row dataset-split-row">
<div class="generation-option-copy">
<strong>数据集划分</strong>
<small>按比例将生成后的问答对随机划分为训练集验证集和测试集</small>
</div>
<div class="dataset-split-config">
<div class="dataset-split-grid">
<label class="dataset-split-field">
<span>训练集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="structuredOptions.datasetSplit.train"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="训练集比例"
@update:model-value="updateDatasetSplit('train', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>验证集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="structuredOptions.datasetSplit.validation"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="验证集比例"
@update:model-value="updateDatasetSplit('validation', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>测试集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="structuredOptions.datasetSplit.test"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="测试集比例"
@update:model-value="updateDatasetSplit('test', $event)"
/>
<span>%</span>
</span>
</label>
</div>
<div class="dataset-split-summary" :class="{ 'is-invalid': splitTotal !== 100 }">
<span>总计 {{ splitTotal }}%</span>
<span v-if="splitTotal !== 100" role="alert">训练集验证集和测试集比例总和必须为 100%</span>
</div>
</div>
</div>
</div>
</div>
</template>
<template v-if="processType === 'unstructured'">
<div class="form-section unstructured-options-section">
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>先清理和补全文档语义再进入切分和问答生成</p>
</div>
</div>
<el-checkbox-group
:model-value="unstructuredOptions.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updateUnstructuredPreprocessOptions"
>
<el-checkbox
v-for="option in UNSTRUCTURED_PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
class="preprocess-option"
>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</div>
<div class="form-section chunk-options-section">
<div class="section-title-row">
<div>
<h3>切分选项</h3>
<p>以语义完整为优先将长文档拆成可独立生成问答的内容块</p>
</div>
</div>
<div class="chunk-settings-grid">
<label class="config-field">
<span class="config-field-label">切分方式</span>
<el-select
:model-value="unstructuredOptions.chunkMethod"
aria-label="切分方式"
@update:model-value="updateUnstructuredField('chunkMethod', $event)"
>
<el-option
v-for="method in CHUNK_METHODS"
:key="method.value"
:label="method.label"
:value="method.value"
/>
</el-select>
<small>推荐使用自动语义切分在长度限制内优先保留完整句段</small>
</label>
<label class="config-field">
<span class="config-field-label">切片长度</span>
<span class="unit-input">
<el-input-number
:model-value="unstructuredOptions.chunkSize"
:min="200"
:max="2000"
:step="50"
:precision="0"
controls-position="right"
aria-label="切片长度"
@update:model-value="updateUnstructuredNumber('chunkSize', $event)"
/>
<span>Token</span>
</span>
<small>单个切片的目标上限默认 800 Token</small>
</label>
<label class="config-field">
<span class="config-field-label">重叠长度</span>
<span class="unit-input">
<el-input-number
:model-value="unstructuredOptions.chunkOverlap"
:min="0"
:max="500"
:step="10"
:precision="0"
controls-position="right"
aria-label="重叠长度"
@update:model-value="updateUnstructuredNumber('chunkOverlap', $event)"
/>
<span>Token</span>
</span>
<small>在相邻切片中最多重复保留的上下文默认 100 Token</small>
</label>
<label class="config-field">
<span class="config-field-label">最小切片长度</span>
<span class="unit-input">
<el-input-number
:model-value="unstructuredOptions.minChunkSize"
:min="20"
:max="500"
:step="10"
:precision="0"
controls-position="right"
aria-label="最小切片长度"
@update:model-value="updateUnstructuredNumber('minChunkSize', $event)"
/>
<span>Token</span>
</span>
<small>过短的尾部内容会尽量并入前一个切片</small>
</label>
<label
v-if="unstructuredOptions.chunkMethod === 'custom'"
class="config-field is-full-width"
>
<span class="config-field-label">自定义分隔符</span>
<el-input
:model-value="unstructuredOptions.customDelimiter"
maxlength="40"
show-word-limit
placeholder="例如:--- 或 ###"
aria-label="自定义分隔符"
@update:model-value="updateUnstructuredField('customDelimiter', $event)"
/>
<small>系统会优先在分隔符位置结束当前切片</small>
</label>
</div>
<p class="chunk-estimation-note">
Token 数为轻量估算值实际长度以训练使用的模型分词器为准
</p>
<p v-if="chunkValidationMessage" class="option-validation-message" role="alert">
{{ chunkValidationMessage }}
</p>
<div class="preserve-options-panel">
<div class="generation-option-copy">
<strong>特殊内容保护</strong>
<small>避免切分点破坏表格代码或列表的完整性</small>
</div>
<div class="preserve-option-grid">
<el-checkbox
:model-value="unstructuredOptions.preserveTables"
@update:model-value="updateUnstructuredField('preserveTables', Boolean($event))"
>
完整保留表格
</el-checkbox>
<el-checkbox
:model-value="unstructuredOptions.preserveCodeBlocks"
@update:model-value="updateUnstructuredField('preserveCodeBlocks', Boolean($event))"
>
完整保留代码块
</el-checkbox>
<el-checkbox
:model-value="unstructuredOptions.preserveLists"
@update:model-value="updateUnstructuredField('preserveLists', Boolean($event))"
>
完整保留列表
</el-checkbox>
</div>
</div>
</div>
<div class="form-section generation-options-section">
<div class="section-title-row">
<div>
<h3>生成选项</h3>
<p>配置每个切片的问答生成方式范围和输出数量</p>
</div>
</div>
<div class="generation-option-list">
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
<small>使用大模型将问答表述得更自然柔和</small>
</div>
<el-switch
:model-value="unstructuredOptions.semanticEnrichment"
inline-prompt
active-text=""
inactive-text=""
@update:model-value="updateUnstructuredField('semanticEnrichment', Boolean($event))"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每个切片生成数量</strong>
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
</div>
<el-input-number
:model-value="unstructuredOptions.qaPairsPerChunk"
:min="1"
:max="3"
:step="1"
:precision="0"
controls-position="right"
@update:model-value="updateUnstructuredNumber('qaPairsPerChunk', $event)"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>上下文范围</strong>
<small>生成问答时可参考的文档范围</small>
</div>
<el-select
:model-value="unstructuredOptions.contextScope"
class="compact-select"
aria-label="上下文范围"
@update:model-value="updateUnstructuredField('contextScope', $event)"
>
<el-option
v-for="scope in CONTEXT_SCOPES"
:key="scope.value"
:label="scope.label"
:value="scope.value"
/>
</el-select>
</div>
<div class="generation-option-row is-stacked">
<div class="generation-option-copy">
<strong>问题类型</strong>
<small>按内容特点选择要生成的问答角度至少保留一项</small>
</div>
<el-checkbox-group
:model-value="unstructuredOptions.generationTypes"
class="generation-type-grid"
@update:model-value="updateGenerationTypes"
>
<el-checkbox
v-for="type in GENERATION_TYPES"
:key="type.value"
:value="type.value"
>
{{ type.label }}
</el-checkbox>
</el-checkbox-group>
<span
v-if="unstructuredOptions.generationTypes.length === 0"
class="option-validation-message"
role="alert"
>
请至少选择一种问题类型
</span>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>跳过无法回答的内容</strong>
<small>当切片缺少完整信息时不强行编造问答对</small>
</div>
<el-switch
:model-value="unstructuredOptions.skipUnanswerable"
inline-prompt
active-text=""
inactive-text=""
@update:model-value="updateUnstructuredField('skipUnanswerable', Boolean($event))"
/>
</div>
<div class="generation-option-row dataset-split-row">
<div class="generation-option-copy">
<strong>数据集划分</strong>
<small>按比例将生成后的问答对随机划分为训练集验证集和测试集</small>
</div>
<div class="dataset-split-config">
<div class="dataset-split-grid">
<label class="dataset-split-field">
<span>训练集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="unstructuredOptions.datasetSplit.train"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="非结构化训练集比例"
@update:model-value="updateUnstructuredDatasetSplit('train', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>验证集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="unstructuredOptions.datasetSplit.validation"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="非结构化验证集比例"
@update:model-value="updateUnstructuredDatasetSplit('validation', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>测试集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="unstructuredOptions.datasetSplit.test"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
aria-label="非结构化测试集比例"
@update:model-value="updateUnstructuredDatasetSplit('test', $event)"
/>
<span>%</span>
</span>
</label>
</div>
<div class="dataset-split-summary" :class="{ 'is-invalid': unstructuredSplitTotal !== 100 }">
<span>总计 {{ unstructuredSplitTotal }}%</span>
<span v-if="unstructuredSplitTotal !== 100" role="alert">
训练集验证集和测试集比例总和必须为 100%
</span>
</div>
</div>
</div>
</div>
</div>
</template>
<div v-if="isExternal" class="form-section external-section">
<div class="section-title-row">
<div>
@@ -405,6 +1034,222 @@ defineExpose({ validate })
gap: 20px;
}
.preprocess-option-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
margin-top: 16px;
}
.preprocess-option {
width: 100%;
min-height: 64px;
margin-right: 0;
padding: 12px 14px;
box-sizing: border-box;
align-items: flex-start;
border: 1px solid #e2e5ec;
border-radius: 8px;
transition: border-color 0.18s ease, background-color 0.18s ease;
&:hover {
border-color: #b7b2f7;
}
&.is-checked {
background: #fafaff;
border-color: #8b82f4;
}
:deep(.el-checkbox__input) {
margin-top: 3px;
}
:deep(.el-checkbox__label) {
min-width: 0;
padding-left: 10px;
white-space: normal;
}
}
.preprocess-option-copy,
.generation-option-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
}
.generation-option-list {
display: grid;
gap: 12px;
margin-top: 16px;
}
.chunk-settings-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
margin-top: 16px;
}
.config-field {
display: grid;
align-content: start;
gap: 8px;
min-width: 0;
padding: 14px;
color: #344054;
background: #fbfcfe;
border: 1px solid #e2e5ec;
border-radius: 8px;
&.is-full-width {
grid-column: 1 / -1;
}
> small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
:deep(.el-select),
:deep(.el-input) {
width: 100%;
}
}
.config-field-label {
font-size: 13px;
font-weight: 600;
}
.unit-input {
display: flex;
align-items: center;
gap: 8px;
color: #7b8495;
font-size: 12px;
:deep(.el-input-number) {
width: 100%;
}
}
.option-validation-message {
display: block;
margin: 8px 0 0;
color: #dc2626;
font-size: 12px;
line-height: 1.5;
}
.chunk-estimation-note {
margin: 9px 0 0;
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
.preserve-options-panel {
display: grid;
gap: 12px;
margin-top: 12px;
padding: 14px;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.preserve-option-grid,
.generation-type-grid {
display: flex;
flex-wrap: wrap;
gap: 8px 24px;
:deep(.el-checkbox) {
margin-right: 0;
}
}
.generation-option-row {
display: flex;
min-height: 64px;
align-items: center;
justify-content: space-between;
gap: 24px;
padding: 12px 14px;
box-sizing: border-box;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.generation-option-row.is-stacked {
align-items: stretch;
flex-direction: column;
gap: 12px;
}
.compact-select {
width: 220px;
max-width: 100%;
}
.dataset-split-row {
align-items: stretch;
flex-direction: column;
}
.dataset-split-config {
display: grid;
gap: 10px;
}
.dataset-split-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 16px;
}
.dataset-split-field {
display: grid;
gap: 7px;
color: #5f6878;
font-size: 12px;
}
.dataset-split-input {
display: flex;
align-items: center;
gap: 8px;
:deep(.el-input-number) {
width: 100%;
}
}
.dataset-split-summary {
display: flex;
justify-content: space-between;
gap: 12px;
color: #2ca66a;
font-size: 12px;
&.is-invalid {
color: #dc2626;
}
}
.section-title-row {
display: flex;
align-items: flex-start;
@@ -650,6 +1495,32 @@ defineExpose({ validate })
grid-template-columns: minmax(0, 1fr);
}
.preprocess-option-grid {
grid-template-columns: minmax(0, 1fr);
}
.chunk-settings-grid {
grid-template-columns: minmax(0, 1fr);
}
.config-field.is-full-width {
grid-column: auto;
}
.dataset-split-grid {
grid-template-columns: minmax(0, 1fr);
}
.dataset-split-summary {
align-items: flex-start;
flex-direction: column;
}
.generation-option-row:not(.dataset-split-row):not(.is-stacked) {
align-items: flex-start;
flex-direction: column;
}
.external-section .external-grid {
grid-template-columns: minmax(0, 1fr);
}

View File

@@ -1,4 +1,11 @@
import type { PreviewItem, ProcessType, ResultItem, SourceLine } from './types'
import type {
PreviewItem,
ProcessType,
ResultItem,
SourceLine,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
export const DEFAULT_SOURCE_TEXT = [
'问:如何看待当前的通货膨胀风险?',
@@ -39,8 +46,417 @@ export function sourceLines(sourceText: string): SourceLine[] {
})
}
export function buildPreviewItems(sourceText: string, processType: ProcessType, sourceFileId = 'default-source'): PreviewItem[] {
const meaningfulLines = sourceLines(sourceText).filter((line) => line.content.trim())
interface SourceRange {
start: number
end: number
}
interface ProtectedRange extends SourceRange {
kind: 'code' | 'table' | 'list'
}
const DEFAULT_CHUNK_SIZE = 800
const DEFAULT_CHUNK_OVERLAP = 100
const DEFAULT_MIN_CHUNK_SIZE = 100
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
}
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
let nextStart = Math.max(0, start)
let nextEnd = Math.min(sourceText.length, end)
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
return { start: nextStart, end: nextEnd }
}
function normalizeDelimiter(delimiter: string | undefined): string {
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
}
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
return line.start < range.end && line.end > range.start
}
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
return ranges.some((range) => overlapsRange(line, range))
}
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
let openFence: { start: number; marker: string; length: number } | null = null
for (const line of lines) {
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
if (!fence) continue
const marker = fence[1][0]
if (!openFence) {
openFence = { start: line.start, marker, length: fence[1].length }
continue
}
if (marker === openFence.marker && fence[1].length >= openFence.length) {
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
openFence = null
}
}
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
return ranges
}
function isTableSeparator(content: string): boolean {
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
const cells = normalized.split('|').map((cell) => cell.trim())
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
}
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length - 1; index += 1) {
const header = lines[index]
const separator = lines[index + 1]
if (
isLineProtected(header, codeRanges)
|| isLineProtected(separator, codeRanges)
|| !header.content.includes('|')
|| !isTableSeparator(separator.content)
) {
continue
}
let endIndex = index + 1
while (
endIndex + 1 < lines.length
&& !isLineProtected(lines[endIndex + 1], codeRanges)
&& lines[endIndex + 1].content.trim()
&& lines[endIndex + 1].content.includes('|')
) {
endIndex += 1
}
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
index = endIndex
}
return ranges
}
function isListItem(content: string): boolean {
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
}
function isListContinuation(content: string): boolean {
return /^\s{2,}\S/.test(content)
}
function detectListRanges(
lines: SourceLine[],
excludedRanges: SourceRange[],
): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length; index += 1) {
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
let endIndex = index
let itemCount = 1
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
const nextContent = lines[endIndex + 1].content
if (isListItem(nextContent)) {
itemCount += 1
endIndex += 1
continue
}
if (isListContinuation(nextContent)) {
endIndex += 1
continue
}
break
}
if (itemCount >= 2) {
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
index = endIndex
}
}
return ranges
}
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
return ranges
.sort((left, right) => left.start - right.start || left.end - right.end)
.reduce<ProtectedRange[]>((merged, range) => {
const previous = merged[merged.length - 1]
if (previous && range.start < previous.end) {
previous.end = Math.max(previous.end, range.end)
return merged
}
merged.push({ ...range })
return merged
}, [])
}
function protectedRangesForOptions(
sourceText: string,
options?: UnstructuredProcessOptions,
): ProtectedRange[] {
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
const lines = sourceLines(sourceText)
const codeRanges = detectCodeBlockRanges(sourceText, lines)
const tableRanges = detectTableRanges(lines, codeRanges)
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
const enabledRanges = [
...(options?.preserveCodeBlocks ? codeRanges : []),
...(options?.preserveTables ? tableRanges : []),
...(options?.preserveLists ? listRanges : []),
]
return mergeProtectedRanges(enabledRanges)
}
function protectedRangeContaining(
ranges: ProtectedRange[],
offset: number,
): ProtectedRange | undefined {
return ranges.find((range) => range.start < offset && offset < range.end)
}
function normalizeChunkStart(
sourceText: string,
cursor: number,
protectedRanges: ProtectedRange[],
): number {
let start = Math.max(0, Math.min(cursor, sourceText.length))
const overlapBlock = protectedRangeContaining(protectedRanges, start)
if (overlapBlock) start = overlapBlock.end
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
// 去除块前空白时可能进入缩进代码块/列表;此时恢复到完整块起点。
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
return start
}
function protectChunkEnd(
proposedEnd: number,
start: number,
minimumEnd: number,
protectedRanges: ProtectedRange[],
): number {
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
if (!splitBlock) return proposedEnd
// 优先在块前结束;块前不足最小切片长度时,将整个块收入当前切片。
return splitBlock.start > start && splitBlock.start >= minimumEnd
? splitBlock.start
: splitBlock.end
}
function restoreProtectedEdges(
range: SourceRange,
rawStart: number,
rawEnd: number,
protectedRanges: ProtectedRange[],
): SourceRange {
const nextRange = { ...range }
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
return nextRange
}
function lastBoundaryInRange(
sourceText: string,
idealEnd: number,
minimumEnd: number,
): number | null {
const candidates: number[] = []
const boundaryTokens = ['\n\n', '\n', '。', '', '', '', '.', '!', '?', ';']
boundaryTokens.forEach((token) => {
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
})
return candidates.length ? Math.max(...candidates) : null
}
function lastHeadingBoundary(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
): number | null {
const section = sourceText.slice(start, idealEnd)
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
let boundary: number | null = null
let match: RegExpExecArray | null
while ((match = headingPattern.exec(section))) {
const absoluteStart = start + match.index
if (absoluteStart >= minimumEnd) boundary = absoluteStart
}
return boundary
}
function resolveChunkEnd(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
options: UnstructuredProcessOptions | undefined,
): number {
const method = options?.chunkMethod ?? 'semantic'
if (method === 'fixed') return idealEnd
if (method === 'custom') {
const delimiter = normalizeDelimiter(options?.customDelimiter)
if (!delimiter) return idealEnd
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
return boundary >= minimumEnd ? boundary : idealEnd
}
if (method === 'heading') {
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
if (headingBoundary !== null) return headingBoundary
}
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
}
function buildUnstructuredRanges(
sourceText: string,
options?: UnstructuredProcessOptions,
): SourceRange[] {
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算避免引入分词器依赖。
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
const minimumCharacters = Math.min(
targetCharacters,
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
)
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
const protectedRanges = protectedRangesForOptions(sourceText, options)
const ranges: SourceRange[] = []
let cursor = 0
while (cursor < sourceText.length) {
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
if (start >= sourceText.length) break
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
let end = idealEnd === sourceText.length
? idealEnd
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
// 所有自定义边界都必须向前推进;异常配置回退到固定长度切分。
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
let range = trimSourceRange(sourceText, start, end)
range = restoreProtectedEdges(range, start, end, protectedRanges)
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
range.end = Math.min(end, range.start + minimumCharacters)
}
if (range.end <= range.start) {
cursor = Math.max(cursor + 1, end)
continue
}
const isLastRange = end >= sourceText.length
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
ranges[ranges.length - 1].end = range.end
break
}
ranges.push(range)
if (isLastRange) break
// overlap 是允许的最大重叠量;按当前切片动态收缩,保证每轮至少推进最小切片长度。
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
const nextCursor = range.end - actualOverlap
cursor = nextCursor > start ? nextCursor : range.end
}
return ranges
}
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
if (!lines.length) return null
let low = 0
let high = lines.length - 1
let result = 0
while (low <= high) {
const middle = Math.floor((low + high) / 2)
if (lines[middle].start <= offset) {
result = middle
low = middle + 1
} else {
high = middle - 1
}
}
return lines[result].number
}
function previewItemFromRange(
sourceText: string,
lines: SourceLine[],
range: SourceRange,
sourceFileId: string,
index: number,
): PreviewItem {
const content = sourceText.slice(range.start, range.end)
return {
id: `preview-${sourceFileId}-${index + 1}`,
sourceFileId,
originalContent: content,
editedContent: content,
sourceStart: range.start,
sourceEnd: range.end,
sourceStartLine: lineNumberAtOffset(lines, range.start),
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
status: 'original',
}
}
export function buildPreviewItems(
sourceText: string,
processType: ProcessType,
sourceFileId = 'default-source',
unstructuredOptions?: UnstructuredProcessOptions,
): PreviewItem[] {
const lines = sourceLines(sourceText)
if (processType === 'unstructured') {
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
))
}
const meaningfulLines = lines.filter((line) => line.content.trim())
const groupSize = processType === 'structured' ? 1 : 3
const items: PreviewItem[] = []
@@ -69,21 +485,44 @@ export function buildPreviewItems(sourceText: string, processType: ProcessType,
return items
}
export function createResults(items: PreviewItem[]): ResultItem[] {
return items.slice(0, 12).map((item, index) => {
const SEMANTIC_PREFIXES = [
'请结合实际情况,说明一下:',
'如果方便的话,请详细解答:',
'请用通俗易懂的方式说明:',
'请从实际应用角度说明:',
'请简洁、自然地说明:',
]
export function createResults(
items: PreviewItem[],
options?: StructuredProcessOptions | UnstructuredProcessOptions,
): ResultItem[] {
const resultCount = options && 'qaPairsPerChunk' in options
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
return items.flatMap((item, index) => {
const [firstLine = '', ...rest] = item.editedContent.split('\n')
const output = rest.join('\n').trim() || item.editedContent.trim()
const instruction = firstLine.replace(/^问[:]\s*/, '').trim() || `数据条目 ${index + 1}`
const baseInstruction = firstLine.replace(/^问[:]\s*/, '').trim() || `数据条目 ${index + 1}`
return {
id: `result-${index + 1}`,
instruction,
input: '',
output,
originalInstruction: instruction,
originalInput: '',
originalOutput: output,
status: 'valid',
}
return Array.from({ length: resultCount }, (_, variantIndex) => {
const instruction = options?.semanticEnrichment
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
: variantIndex === 0
? baseInstruction
: `${baseInstruction}(问法 ${variantIndex + 1}`
return {
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
instruction,
input: '',
output,
originalInstruction: instruction,
originalInput: '',
originalOutput: output,
status: 'valid' as const,
}
})
})
}

View File

@@ -2,6 +2,65 @@ export type ProcessType = 'structured' | 'unstructured' | 'external'
export type StepId = 'create' | 'preview' | 'generate' | 'results'
export type PreprocessOption =
| 'clean_invalid'
| 'detect_structure'
| 'deduplicate'
| 'normalize_format'
| 'filter_anomaly'
| 'desensitize'
export interface DatasetSplitOptions {
train: number
validation: number
test: number
}
export interface StructuredProcessOptions {
preprocessOptions: PreprocessOption[]
semanticEnrichment: boolean
qaPairsPerRow: number
datasetSplit: DatasetSplitOptions
}
export type UnstructuredPreprocessOption =
| 'clean_invalid_content'
| 'detect_document_structure'
| 'merge_short_content'
| 'filter_low_quality'
| 'deduplicate_content'
| 'desensitize'
| 'preserve_context'
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
export type GenerationContextScope = 'current' | 'adjacent' | 'section'
export type QuestionGenerationType =
| 'factual'
| 'concept'
| 'procedure'
| 'reasoning'
| 'comprehensive'
export interface UnstructuredProcessOptions {
preprocessOptions: UnstructuredPreprocessOption[]
chunkMethod: ChunkMethod
chunkSize: number
chunkOverlap: number
minChunkSize: number
customDelimiter: string
preserveTables: boolean
preserveCodeBlocks: boolean
preserveLists: boolean
semanticEnrichment: boolean
qaPairsPerChunk: number
contextScope: GenerationContextScope
generationTypes: QuestionGenerationType[]
skipUnanswerable: boolean
datasetSplit: DatasetSplitOptions
}
export interface ExternalDataSource {
type: string
url: string