feat: 数据处理向导配置体系扩展
新增结构化与非结构化处理选项类型,TaskSetupStep 增加预处理、切分方法、数据集划分等配置 UI,previewModel 实现语义切分与受保护区间算法,CreateView 接入配置状态与草稿持久化并替换为 AppConfirmDialog,回归脚本扩充配置与弹窗断言。
This commit is contained in:
@@ -4,14 +4,35 @@ import { readFile } from 'node:fs/promises'
|
|||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
import path from 'node:path'
|
import path from 'node:path'
|
||||||
import { parse as parseSfc } from '@vue/compiler-sfc'
|
import { parse as parseSfc } from '@vue/compiler-sfc'
|
||||||
|
import ts from 'typescript'
|
||||||
|
|
||||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
||||||
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
|
const viewPath = path.resolve(scriptDir, '../src/views/data-process/DataProcessCreateView.vue')
|
||||||
const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
|
const createDir = path.resolve(scriptDir, '../src/views/data-process/create')
|
||||||
|
const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmDialog.vue')
|
||||||
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
|
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
|
||||||
const viewSource = await readFile(viewPath, 'utf8')
|
const viewSource = await readFile(viewPath, 'utf8')
|
||||||
const layoutSource = await readFile(layoutPath, 'utf8')
|
const layoutSource = await readFile(layoutPath, 'utf8')
|
||||||
|
|
||||||
|
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
|
||||||
|
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
|
||||||
|
for (const marker of ['<Teleport to="body">', 'role="alertdialog"', ':aria-modal="true"', 'handleKeydown', 'Escape']) {
|
||||||
|
assert.ok(confirmDialogSource.includes(marker), `公共确认弹窗缺少可访问性能力:${marker}`)
|
||||||
|
}
|
||||||
|
assert.match(confirmDialogSource, /min-height:\s*44px/, '公共确认弹窗按钮触控区域不足 44px')
|
||||||
|
assert.match(confirmDialogSource, /focus\(\)/, '公共确认弹窗打开后没有管理键盘焦点')
|
||||||
|
assert.match(confirmDialogSource, /defineExpose\(\{ open \}\)/, '公共确认弹窗没有暴露 Promise 式 open API')
|
||||||
|
assert.match(confirmDialogSource, /width:\s*min\(480px,\s*100%\)/, '企业级确认弹窗宽度应保持紧凑的 480px')
|
||||||
|
assert.match(confirmDialogSource, /border-radius:\s*8px/, '企业级确认弹窗应使用克制的 8px 圆角')
|
||||||
|
assert.doesNotMatch(confirmDialogSource, /backdrop-filter/, '企业级确认弹窗不应使用装饰性背景模糊')
|
||||||
|
assert.ok(confirmDialogSource.includes('app-confirm-header'), '企业级确认弹窗缺少独立标题栏')
|
||||||
|
assert.match(confirmDialogSource, /\.app-confirm-button\s*\{[\s\S]*?height:\s*34px/, '桌面端操作按钮应使用紧凑的 34px 高度')
|
||||||
|
assert.match(confirmDialogSource, /@media \(max-width: 520px\)[\s\S]*?\.app-confirm-button\s*\{[\s\S]*?min-height:\s*44px/, '移动端操作按钮仍需保留 44px 触控高度')
|
||||||
|
assert.match(viewSource, /import AppConfirmDialog from '@\/components\/AppConfirmDialog\.vue'/, '创建页没有接入公共确认弹窗')
|
||||||
|
assert.match(viewSource, /<AppConfirmDialog/, '创建页模板缺少公共确认弹窗实例')
|
||||||
|
assert.match(viewSource, /onBeforeRouteLeave\(async \(\) =>/, '路由离开确认没有改为异步公共弹窗流程')
|
||||||
|
assert.doesNotMatch(viewSource, /window\.confirm|ElMessageBox/, '创建页仍在使用系统或 Element Plus 确认框')
|
||||||
|
|
||||||
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
|
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
|
||||||
for (const title of ['创建任务', '数据预览', '开始生成', '结果编辑与保存']) {
|
for (const title of ['创建任务', '数据预览', '开始生成', '结果编辑与保存']) {
|
||||||
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
|
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
|
||||||
@@ -53,8 +74,8 @@ assert.match(modelSource, /sourceFileId/, '切片生成没有写入来源文件
|
|||||||
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
||||||
assert.match(
|
assert.match(
|
||||||
viewSource,
|
viewSource,
|
||||||
/buildPreviewItems\(file\.content, processType\.value, String\(file\.uid\)\)/,
|
/buildPreviewItems\([\s\S]*?file\.content,[\s\S]*?processType\.value,[\s\S]*?String\(file\.uid\),[\s\S]*?unstructuredOptions\.value/,
|
||||||
'预览没有按文件分别生成',
|
'预览没有按文件分别生成或未传入非结构化切分配置',
|
||||||
)
|
)
|
||||||
|
|
||||||
for (const marker of [
|
for (const marker of [
|
||||||
@@ -99,6 +120,304 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第二步缺少窄
|
|||||||
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
||||||
const taskSetupSource = await readFile(taskSetupPath, 'utf8')
|
const taskSetupSource = await readFile(taskSetupPath, 'utf8')
|
||||||
|
|
||||||
|
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
||||||
|
for (const option of [
|
||||||
|
'清理无效数据',
|
||||||
|
'识别表格结构',
|
||||||
|
'重复数据去重',
|
||||||
|
'数据格式标准化',
|
||||||
|
'异常数据过滤',
|
||||||
|
'敏感信息脱敏',
|
||||||
|
]) {
|
||||||
|
assert.ok(taskSetupSource.includes(option), `结构化预处理缺少选项:${option}`)
|
||||||
|
}
|
||||||
|
assert.ok(taskSetupSource.includes('生成选项'), '结构化配置缺少生成选项分类')
|
||||||
|
assert.ok(taskSetupSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关')
|
||||||
|
assert.ok(taskSetupSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明')
|
||||||
|
for (const splitName of ['训练集', '验证集', '测试集']) {
|
||||||
|
assert.ok(taskSetupSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
|
||||||
|
}
|
||||||
|
assert.match(taskSetupSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算')
|
||||||
|
assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验')
|
||||||
|
assert.ok(taskSetupSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示')
|
||||||
|
for (const splitField of ['train', 'validation', 'test']) {
|
||||||
|
assert.match(
|
||||||
|
taskSetupSource,
|
||||||
|
new RegExp(`structuredOptions\\.datasetSplit\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`),
|
||||||
|
`数据集划分字段 ${splitField} 缺少 0~100 的整数限制`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
assert.match(taskSetupSource, /<el-switch[\s\S]*structuredOptions\.semanticEnrichment/, '语义丰富表达必须使用开关控件')
|
||||||
|
assert.match(taskSetupSource, /<el-input-number[\s\S]*structuredOptions\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
|
||||||
|
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
|
||||||
|
assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
|
||||||
|
assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿')
|
||||||
|
assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
|
||||||
|
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
|
||||||
|
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
|
||||||
|
|
||||||
|
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
|
||||||
|
for (const field of [
|
||||||
|
'preprocessOptions',
|
||||||
|
'chunkMethod',
|
||||||
|
'chunkSize',
|
||||||
|
'chunkOverlap',
|
||||||
|
'minChunkSize',
|
||||||
|
'customDelimiter',
|
||||||
|
'preserveTables',
|
||||||
|
'preserveCodeBlocks',
|
||||||
|
'preserveLists',
|
||||||
|
'semanticEnrichment',
|
||||||
|
'qaPairsPerChunk',
|
||||||
|
'contextScope',
|
||||||
|
'generationTypes',
|
||||||
|
'skipUnanswerable',
|
||||||
|
'datasetSplit',
|
||||||
|
]) {
|
||||||
|
assert.ok(typesSource.includes(field), `非结构化处理选项缺少字段:${field}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
|
||||||
|
for (const option of [
|
||||||
|
'清理无效内容',
|
||||||
|
'识别文档结构',
|
||||||
|
'合并过短内容',
|
||||||
|
'过滤低质量内容',
|
||||||
|
'重复内容去重',
|
||||||
|
'敏感信息脱敏',
|
||||||
|
'保留上下文信息',
|
||||||
|
]) {
|
||||||
|
assert.ok(taskSetupSource.includes(option), `非结构化预处理缺少选项:${option}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
|
||||||
|
for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) {
|
||||||
|
assert.ok(taskSetupSource.includes(method), `切分方式缺少选项:${method}`)
|
||||||
|
}
|
||||||
|
for (const label of ['切片长度', '重叠长度', '最小切片长度', '完整保留表格', '完整保留代码块', '完整保留列表']) {
|
||||||
|
assert.ok(taskSetupSource.includes(label), `切分选项缺少配置:${label}`)
|
||||||
|
}
|
||||||
|
assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token')
|
||||||
|
assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token')
|
||||||
|
assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token')
|
||||||
|
|
||||||
|
for (const label of [
|
||||||
|
'每个切片生成数量',
|
||||||
|
'上下文范围',
|
||||||
|
'当前切片',
|
||||||
|
'相邻切片',
|
||||||
|
'当前章节',
|
||||||
|
'问题类型',
|
||||||
|
'事实问答',
|
||||||
|
'概念解释',
|
||||||
|
'操作步骤',
|
||||||
|
'原因分析',
|
||||||
|
'综合问答',
|
||||||
|
'跳过无法回答的内容',
|
||||||
|
]) {
|
||||||
|
assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少:${label}`)
|
||||||
|
}
|
||||||
|
assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3')
|
||||||
|
assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验')
|
||||||
|
assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界')
|
||||||
|
assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明')
|
||||||
|
|
||||||
|
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
|
||||||
|
assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分')
|
||||||
|
assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
|
||||||
|
assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
|
||||||
|
assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
|
||||||
|
assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
|
||||||
|
assert.match(viewSource, /contextScope:\s*'adjacent'/, '默认上下文范围必须为相邻切片')
|
||||||
|
assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿')
|
||||||
|
assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复')
|
||||||
|
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
|
||||||
|
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识')
|
||||||
|
assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识')
|
||||||
|
assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览')
|
||||||
|
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
|
||||||
|
|
||||||
|
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
|
||||||
|
const previewOptionsEnd = viewSource.indexOf('function generationAffectingOptions()', previewOptionsStart)
|
||||||
|
assert.ok(previewOptionsStart >= 0 && previewOptionsEnd > previewOptionsStart, '缺少预览影响配置签名函数')
|
||||||
|
const previewOptionsSource = viewSource.slice(previewOptionsStart, previewOptionsEnd)
|
||||||
|
for (const field of [
|
||||||
|
'preprocessOptions',
|
||||||
|
'chunkMethod',
|
||||||
|
'chunkSize',
|
||||||
|
'chunkOverlap',
|
||||||
|
'minChunkSize',
|
||||||
|
'customDelimiter',
|
||||||
|
'preserveTables',
|
||||||
|
'preserveCodeBlocks',
|
||||||
|
'preserveLists',
|
||||||
|
]) {
|
||||||
|
assert.ok(previewOptionsSource.includes(field), `预览签名缺少切分影响字段:${field}`)
|
||||||
|
}
|
||||||
|
for (const field of [
|
||||||
|
'semanticEnrichment',
|
||||||
|
'qaPairsPerChunk',
|
||||||
|
'contextScope',
|
||||||
|
'generationTypes',
|
||||||
|
'skipUnanswerable',
|
||||||
|
'datasetSplit',
|
||||||
|
]) {
|
||||||
|
assert.ok(!previewOptionsSource.includes(field), `生成字段 ${field} 不应导致预览重建并丢失编辑`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const generationOptionsStart = viewSource.indexOf('function generationAffectingOptions()')
|
||||||
|
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart)
|
||||||
|
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
|
||||||
|
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd)
|
||||||
|
for (const field of [
|
||||||
|
'semanticEnrichment',
|
||||||
|
'qaPairsPerChunk',
|
||||||
|
'contextScope',
|
||||||
|
'generationTypes',
|
||||||
|
'skipUnanswerable',
|
||||||
|
'datasetSplit',
|
||||||
|
]) {
|
||||||
|
assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`)
|
||||||
|
}
|
||||||
|
assert.match(
|
||||||
|
viewSource,
|
||||||
|
/watch\(generationOptionsSignature,[\s\S]*?resetDownstream\(\)/,
|
||||||
|
'生成配置变化后没有仅失效下游结果',
|
||||||
|
)
|
||||||
|
for (const mutationFunction of [
|
||||||
|
'updatePreviewContent',
|
||||||
|
'restorePreviewItem',
|
||||||
|
'addPreviewItem',
|
||||||
|
'removePreviewItem',
|
||||||
|
]) {
|
||||||
|
const mutationStart = viewSource.indexOf(`function ${mutationFunction}`)
|
||||||
|
const mutationEnd = viewSource.indexOf('\nfunction ', mutationStart + 1)
|
||||||
|
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
|
||||||
|
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
|
||||||
|
}
|
||||||
|
assert.match(modelSource, /unstructuredOptions\?: UnstructuredProcessOptions/, '切片预览没有接收非结构化配置')
|
||||||
|
assert.match(modelSource, /qaPairsPerChunk/, '每个切片生成数量没有接入结果生成逻辑')
|
||||||
|
|
||||||
|
const transpiledModel = ts.transpileModule(modelSource, {
|
||||||
|
compilerOptions: { module: ts.ModuleKind.ES2022, target: ts.ScriptTarget.ES2022 },
|
||||||
|
}).outputText
|
||||||
|
const previewModelModule = await import(`data:text/javascript;base64,${Buffer.from(transpiledModel).toString('base64')}`)
|
||||||
|
const longDocument = Array.from(
|
||||||
|
{ length: 180 },
|
||||||
|
(_, index) => `${index + 1}. 这是用于验证非结构化切分边界的完整文本段落。`,
|
||||||
|
).join('\n')
|
||||||
|
const baseUnstructuredOptions = {
|
||||||
|
preprocessOptions: [],
|
||||||
|
chunkMethod: 'semantic',
|
||||||
|
chunkSize: 200,
|
||||||
|
chunkOverlap: 50,
|
||||||
|
minChunkSize: 50,
|
||||||
|
customDelimiter: '',
|
||||||
|
preserveTables: false,
|
||||||
|
preserveCodeBlocks: false,
|
||||||
|
preserveLists: false,
|
||||||
|
semanticEnrichment: false,
|
||||||
|
qaPairsPerChunk: 3,
|
||||||
|
contextScope: 'adjacent',
|
||||||
|
generationTypes: ['factual'],
|
||||||
|
skipUnanswerable: true,
|
||||||
|
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
|
||||||
|
const options = {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod,
|
||||||
|
customDelimiter: chunkMethod === 'custom' ? '\\n' : '',
|
||||||
|
}
|
||||||
|
const previewItems = previewModelModule.buildPreviewItems(longDocument, 'unstructured', chunkMethod, options)
|
||||||
|
assert.ok(previewItems.length > 1, `${chunkMethod} 切分方式未生成多个切片`)
|
||||||
|
assert.ok(
|
||||||
|
previewItems.every((item) => longDocument.slice(item.sourceStart, item.sourceEnd) === item.originalContent),
|
||||||
|
`${chunkMethod} 切分方式的来源偏移不准确`,
|
||||||
|
)
|
||||||
|
assert.ok(
|
||||||
|
previewItems.every((item) => item.sourceStartLine <= item.sourceEndLine),
|
||||||
|
`${chunkMethod} 切分方式的来源行号不准确`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const overlapDocument = '甲'.repeat(1200)
|
||||||
|
const overlapItems = previewModelModule.buildPreviewItems(overlapDocument, 'unstructured', 'overlap-check', {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod: 'fixed',
|
||||||
|
})
|
||||||
|
assert.equal(
|
||||||
|
overlapItems[0].sourceEnd - overlapItems[1].sourceStart,
|
||||||
|
100,
|
||||||
|
'固定长度切分没有按配置保留 50 个估算 Token 的重叠内容',
|
||||||
|
)
|
||||||
|
|
||||||
|
const headingDocument = `${'甲'.repeat(150)}\n# 第二章\n${'乙'.repeat(600)}`
|
||||||
|
const headingItems = previewModelModule.buildPreviewItems(headingDocument, 'unstructured', 'heading-check', {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod: 'heading',
|
||||||
|
chunkOverlap: 0,
|
||||||
|
})
|
||||||
|
assert.ok(!headingItems[0].originalContent.includes('# 第二章'), '按标题切分未在新标题前结束上一切片')
|
||||||
|
assert.ok(headingItems[1].originalContent.startsWith('# 第二章'), '按标题切分未从新标题开始下一切片')
|
||||||
|
|
||||||
|
const customDocument = `${'甲'.repeat(150)}<CUT>${'乙'.repeat(600)}`
|
||||||
|
const customItems = previewModelModule.buildPreviewItems(customDocument, 'unstructured', 'custom-check', {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod: 'custom',
|
||||||
|
chunkOverlap: 0,
|
||||||
|
customDelimiter: '<CUT>',
|
||||||
|
})
|
||||||
|
assert.ok(customItems[0].originalContent.endsWith('<CUT>'), '自定义切分未在指定分隔符处结束切片')
|
||||||
|
|
||||||
|
function assertProtectedContent(optionField, block, label) {
|
||||||
|
const document = `${'前言。'.repeat(50)}\n${block}\n${'结尾。'.repeat(100)}`
|
||||||
|
const enabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-on`, {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod: 'fixed',
|
||||||
|
chunkOverlap: 0,
|
||||||
|
preserveTables: false,
|
||||||
|
preserveCodeBlocks: false,
|
||||||
|
preserveLists: false,
|
||||||
|
[optionField]: true,
|
||||||
|
})
|
||||||
|
const disabledItems = previewModelModule.buildPreviewItems(document, 'unstructured', `${optionField}-off`, {
|
||||||
|
...baseUnstructuredOptions,
|
||||||
|
chunkMethod: 'fixed',
|
||||||
|
chunkOverlap: 0,
|
||||||
|
preserveTables: false,
|
||||||
|
preserveCodeBlocks: false,
|
||||||
|
preserveLists: false,
|
||||||
|
})
|
||||||
|
assert.ok(enabledItems.some((item) => item.originalContent.includes(block)), `${label}开启后仍被从内部切断`)
|
||||||
|
assert.ok(!disabledItems.some((item) => item.originalContent.includes(block)), `${label}关闭后的对照用例未命中切分边界`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const codeBlock = ['```ts', ...Array.from({ length: 36 }, (_, index) => `const value${index} = ${index};`), '```'].join('\n')
|
||||||
|
const tableBlock = [
|
||||||
|
'| 字段 | 说明 |',
|
||||||
|
'| --- | --- |',
|
||||||
|
...Array.from({ length: 36 }, (_, index) => `| field_${index} | 字段说明 ${index} |`),
|
||||||
|
].join('\n')
|
||||||
|
const listBlock = Array.from({ length: 42 }, (_, index) => `- 列表项 ${index + 1}:这是需要完整保留的内容。`).join('\n')
|
||||||
|
assertProtectedContent('preserveCodeBlocks', codeBlock, '代码块')
|
||||||
|
assertProtectedContent('preserveTables', tableBlock, '表格')
|
||||||
|
assertProtectedContent('preserveLists', listBlock, '列表')
|
||||||
|
|
||||||
|
const samplePreviewItems = previewModelModule.buildPreviewItems(
|
||||||
|
longDocument,
|
||||||
|
'unstructured',
|
||||||
|
'generation-check',
|
||||||
|
baseUnstructuredOptions,
|
||||||
|
)
|
||||||
|
assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片')
|
||||||
|
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
|
||||||
|
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
|
||||||
|
|
||||||
|
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
|
||||||
|
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
|
||||||
|
|
||||||
function findNextStyleBlockStart(source, startIndex) {
|
function findNextStyleBlockStart(source, startIndex) {
|
||||||
let quote = null
|
let quote = null
|
||||||
|
|
||||||
|
|||||||
@@ -1,17 +1,30 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
|
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
|
||||||
import { onBeforeRouteLeave, useRouter } from 'vue-router'
|
import { onBeforeRouteLeave, useRouter } from 'vue-router'
|
||||||
import { ElMessage, ElMessageBox, type UploadFile } from 'element-plus'
|
import { ElMessage, type UploadFile } from 'element-plus'
|
||||||
|
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
|
||||||
import TaskSetupStep from './create/TaskSetupStep.vue'
|
import TaskSetupStep from './create/TaskSetupStep.vue'
|
||||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||||
import GenerationStep from './create/GenerationStep.vue'
|
import GenerationStep from './create/GenerationStep.vue'
|
||||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||||
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
||||||
import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
|
import type {
|
||||||
|
ExternalDataSource,
|
||||||
|
GenerationState,
|
||||||
|
PreviewItem,
|
||||||
|
ProcessType,
|
||||||
|
ResultItem,
|
||||||
|
StepId,
|
||||||
|
StructuredProcessOptions,
|
||||||
|
UnstructuredProcessOptions,
|
||||||
|
} from './create/types'
|
||||||
|
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||||
|
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||||
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||||
|
const DRAFT_SCHEMA_VERSION = 2
|
||||||
|
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
|
||||||
|
|
||||||
const WIZARD_STEPS = [
|
const WIZARD_STEPS = [
|
||||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与上传源数据' },
|
{ id: 'create', title: '创建任务', desc: '填写任务信息与上传源数据' },
|
||||||
@@ -24,6 +37,36 @@ const currentStep = ref(0)
|
|||||||
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
|
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
|
||||||
const task = reactive({ name: '', description: '' })
|
const task = reactive({ name: '', description: '' })
|
||||||
const processType = ref<ProcessType>('unstructured')
|
const processType = ref<ProcessType>('unstructured')
|
||||||
|
const structuredOptions = ref<StructuredProcessOptions>({
|
||||||
|
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||||
|
semanticEnrichment: false,
|
||||||
|
qaPairsPerRow: 1,
|
||||||
|
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||||
|
})
|
||||||
|
const unstructuredOptions = ref<UnstructuredProcessOptions>({
|
||||||
|
preprocessOptions: [
|
||||||
|
'clean_invalid_content',
|
||||||
|
'detect_document_structure',
|
||||||
|
'merge_short_content',
|
||||||
|
'filter_low_quality',
|
||||||
|
'deduplicate_content',
|
||||||
|
'preserve_context',
|
||||||
|
],
|
||||||
|
chunkMethod: 'semantic',
|
||||||
|
chunkSize: 800,
|
||||||
|
chunkOverlap: 100,
|
||||||
|
minChunkSize: 100,
|
||||||
|
customDelimiter: '',
|
||||||
|
preserveTables: true,
|
||||||
|
preserveCodeBlocks: true,
|
||||||
|
preserveLists: true,
|
||||||
|
semanticEnrichment: false,
|
||||||
|
qaPairsPerChunk: 1,
|
||||||
|
contextScope: 'adjacent',
|
||||||
|
generationTypes: ['factual', 'concept', 'comprehensive'],
|
||||||
|
skipUnanswerable: true,
|
||||||
|
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||||
|
})
|
||||||
interface UploadedDataFile {
|
interface UploadedDataFile {
|
||||||
uid: number | string
|
uid: number | string
|
||||||
name: string
|
name: string
|
||||||
@@ -103,9 +146,21 @@ const previousStepLabel = computed(() => currentStep.value > 0
|
|||||||
|
|
||||||
function draftSnapshot() {
|
function draftSnapshot() {
|
||||||
return {
|
return {
|
||||||
|
schemaVersion: DRAFT_SCHEMA_VERSION,
|
||||||
currentStep: currentStep.value,
|
currentStep: currentStep.value,
|
||||||
task: { ...task },
|
task: { ...task },
|
||||||
processType: processType.value,
|
processType: processType.value,
|
||||||
|
structuredOptions: {
|
||||||
|
...structuredOptions.value,
|
||||||
|
preprocessOptions: [...structuredOptions.value.preprocessOptions],
|
||||||
|
datasetSplit: { ...structuredOptions.value.datasetSplit },
|
||||||
|
},
|
||||||
|
unstructuredOptions: {
|
||||||
|
...unstructuredOptions.value,
|
||||||
|
preprocessOptions: [...unstructuredOptions.value.preprocessOptions],
|
||||||
|
generationTypes: [...unstructuredOptions.value.generationTypes],
|
||||||
|
datasetSplit: { ...unstructuredOptions.value.datasetSplit },
|
||||||
|
},
|
||||||
uploadedFiles: uploadedFiles.value,
|
uploadedFiles: uploadedFiles.value,
|
||||||
externalSource: { ...externalSource },
|
externalSource: { ...externalSource },
|
||||||
previewSignature: previewSignature.value,
|
previewSignature: previewSignature.value,
|
||||||
@@ -119,6 +174,74 @@ function draftSnapshot() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function previewAffectingOptions() {
|
||||||
|
if (processType.value === 'structured') {
|
||||||
|
return {
|
||||||
|
preprocessOptions: structuredOptions.value.preprocessOptions,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (processType.value === 'unstructured') {
|
||||||
|
const {
|
||||||
|
preprocessOptions,
|
||||||
|
chunkMethod,
|
||||||
|
chunkSize,
|
||||||
|
chunkOverlap,
|
||||||
|
minChunkSize,
|
||||||
|
customDelimiter,
|
||||||
|
preserveTables,
|
||||||
|
preserveCodeBlocks,
|
||||||
|
preserveLists,
|
||||||
|
} = unstructuredOptions.value
|
||||||
|
return {
|
||||||
|
preprocessOptions,
|
||||||
|
chunkMethod,
|
||||||
|
chunkSize,
|
||||||
|
chunkOverlap,
|
||||||
|
minChunkSize,
|
||||||
|
customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
|
||||||
|
preserveTables,
|
||||||
|
preserveCodeBlocks,
|
||||||
|
preserveLists,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
function generationAffectingOptions() {
|
||||||
|
if (processType.value === 'structured') {
|
||||||
|
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value
|
||||||
|
return { semanticEnrichment, qaPairsPerRow, datasetSplit }
|
||||||
|
}
|
||||||
|
if (processType.value === 'unstructured') {
|
||||||
|
const {
|
||||||
|
semanticEnrichment,
|
||||||
|
qaPairsPerChunk,
|
||||||
|
contextScope,
|
||||||
|
generationTypes,
|
||||||
|
skipUnanswerable,
|
||||||
|
datasetSplit,
|
||||||
|
} = unstructuredOptions.value
|
||||||
|
return {
|
||||||
|
semanticEnrichment,
|
||||||
|
qaPairsPerChunk,
|
||||||
|
contextScope,
|
||||||
|
generationTypes,
|
||||||
|
skipUnanswerable,
|
||||||
|
datasetSplit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
|
||||||
|
|
||||||
|
function buildPreviewSignature() {
|
||||||
|
const filesSignature = uploadedFiles.value
|
||||||
|
.map((file) => `${file.uid}:${file.content}`)
|
||||||
|
.join('|')
|
||||||
|
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
|
||||||
|
}
|
||||||
|
|
||||||
function persistDraft() {
|
function persistDraft() {
|
||||||
if (restoringDraft.value) return
|
if (restoringDraft.value) return
|
||||||
try {
|
try {
|
||||||
@@ -128,7 +251,7 @@ function persistDraft() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type DraftSnapshot = ReturnType<typeof draftSnapshot> & {
|
type DraftSnapshot = Partial<ReturnType<typeof draftSnapshot>> & {
|
||||||
fileName?: string
|
fileName?: string
|
||||||
fileSize?: number
|
fileSize?: number
|
||||||
fileCount?: number
|
fileCount?: number
|
||||||
@@ -141,14 +264,46 @@ function restoreDraft() {
|
|||||||
if (!raw) return
|
if (!raw) return
|
||||||
const snapshot = JSON.parse(raw) as DraftSnapshot
|
const snapshot = JSON.parse(raw) as DraftSnapshot
|
||||||
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
|
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
|
||||||
|
const requiresPreviewMigration = snapshot.schemaVersion !== DRAFT_SCHEMA_VERSION
|
||||||
|
|
||||||
restoringDraft.value = true
|
restoringDraft.value = true
|
||||||
currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
|
currentStep.value = requiresPreviewMigration
|
||||||
|
? 0
|
||||||
|
: Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
|
||||||
task.name = snapshot.task?.name || ''
|
task.name = snapshot.task?.name || ''
|
||||||
task.description = snapshot.task?.description || ''
|
task.description = snapshot.task?.description || ''
|
||||||
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
|
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
|
||||||
? snapshot.processType
|
? snapshot.processType
|
||||||
: 'unstructured'
|
: 'unstructured'
|
||||||
|
if (snapshot.structuredOptions) {
|
||||||
|
structuredOptions.value = {
|
||||||
|
...structuredOptions.value,
|
||||||
|
...snapshot.structuredOptions,
|
||||||
|
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
|
||||||
|
? snapshot.structuredOptions.preprocessOptions
|
||||||
|
: structuredOptions.value.preprocessOptions,
|
||||||
|
datasetSplit: {
|
||||||
|
...structuredOptions.value.datasetSplit,
|
||||||
|
...snapshot.structuredOptions.datasetSplit,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (snapshot.unstructuredOptions) {
|
||||||
|
unstructuredOptions.value = {
|
||||||
|
...unstructuredOptions.value,
|
||||||
|
...snapshot.unstructuredOptions,
|
||||||
|
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
|
||||||
|
? snapshot.unstructuredOptions.preprocessOptions
|
||||||
|
: unstructuredOptions.value.preprocessOptions,
|
||||||
|
generationTypes: Array.isArray(snapshot.unstructuredOptions.generationTypes)
|
||||||
|
? snapshot.unstructuredOptions.generationTypes
|
||||||
|
: unstructuredOptions.value.generationTypes,
|
||||||
|
datasetSplit: {
|
||||||
|
...unstructuredOptions.value.datasetSplit,
|
||||||
|
...snapshot.unstructuredOptions.datasetSplit,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (snapshot.uploadedFiles) {
|
if (snapshot.uploadedFiles) {
|
||||||
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
|
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
|
||||||
@@ -163,41 +318,50 @@ function restoreDraft() {
|
|||||||
}]
|
}]
|
||||||
}
|
}
|
||||||
|
|
||||||
previewSignature.value = snapshot.previewSignature || ''
|
previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || ''
|
||||||
if (snapshot.externalSource) {
|
if (snapshot.externalSource) {
|
||||||
Object.assign(externalSource, snapshot.externalSource)
|
Object.assign(externalSource, snapshot.externalSource)
|
||||||
}
|
}
|
||||||
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
|
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
|
||||||
previewItems.value = Array.isArray(snapshot.previewItems)
|
previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems)
|
||||||
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
|
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
|
||||||
: []
|
: []
|
||||||
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
|
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
|
||||||
selectedPreviewIdsByFile.value = snapshot.selectedPreviewIdsByFile || {}
|
selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {}
|
||||||
selectedPreviewId.value = snapshot.selectedPreviewId
|
selectedPreviewId.value = requiresPreviewMigration
|
||||||
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|
? null
|
||||||
|| activePreviewItems.value[0]?.id
|
: snapshot.selectedPreviewId
|
||||||
|| null
|
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|
||||||
results.value = Array.isArray(snapshot.results) ? snapshot.results : []
|
|| activePreviewItems.value[0]?.id
|
||||||
selectedResultId.value = snapshot.selectedResultId || results.value[0]?.id || null
|
|| null
|
||||||
Object.assign(generation, snapshot.generation || {})
|
results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : []
|
||||||
|
selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null
|
||||||
|
if (!requiresPreviewMigration) Object.assign(generation, snapshot.generation || {})
|
||||||
dirty.value = false
|
dirty.value = false
|
||||||
nextTick(() => { restoringDraft.value = false })
|
nextTick(() => { restoringDraft.value = false })
|
||||||
ElMessage.info('已恢复上次保存的草稿')
|
ElMessage.info(requiresPreviewMigration
|
||||||
|
? '已恢复任务信息,切分规则已更新,请重新生成预览'
|
||||||
|
: '已恢复上次保存的草稿')
|
||||||
} catch {
|
} catch {
|
||||||
localStorage.removeItem(DRAFT_STORAGE_KEY)
|
localStorage.removeItem(DRAFT_STORAGE_KEY)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
[() => task.name, () => task.description, processType, externalSource],
|
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||||
() => {
|
() => {
|
||||||
if (!restoringDraft.value) dirty.value = true
|
if (!restoringDraft.value) dirty.value = true
|
||||||
},
|
},
|
||||||
{ deep: true },
|
{ deep: true },
|
||||||
)
|
)
|
||||||
|
|
||||||
|
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||||
|
if (restoringDraft.value || currentSignature === previousSignature) return
|
||||||
|
resetDownstream()
|
||||||
|
})
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
[currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
|
[currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature,
|
||||||
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
|
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
|
||||||
results, selectedResultId, generation],
|
results, selectedResultId, generation],
|
||||||
persistDraft,
|
persistDraft,
|
||||||
@@ -330,10 +494,15 @@ async function nextFromCreate() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const signature = `${processType.value}:${uploadedFiles.value.map((file) => `${file.uid}:${file.content}`).join('|')}`
|
const signature = buildPreviewSignature()
|
||||||
if (signature !== previewSignature.value) {
|
if (signature !== previewSignature.value) {
|
||||||
previewItems.value = uploadedFiles.value.flatMap((file) =>
|
previewItems.value = uploadedFiles.value.flatMap((file) =>
|
||||||
buildPreviewItems(file.content, processType.value, String(file.uid)),
|
buildPreviewItems(
|
||||||
|
file.content,
|
||||||
|
processType.value,
|
||||||
|
String(file.uid),
|
||||||
|
processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
|
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
|
||||||
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
|
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
|
||||||
@@ -367,6 +536,7 @@ function updatePreviewContent(id: string, value: string) {
|
|||||||
item.editedContent = value
|
item.editedContent = value
|
||||||
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
|
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
|
||||||
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
||||||
|
resetDownstream()
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -376,6 +546,7 @@ function restorePreviewItem(id: string) {
|
|||||||
item.editedContent = item.originalContent
|
item.editedContent = item.originalContent
|
||||||
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
|
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
|
||||||
item.status = 'original'
|
item.status = 'original'
|
||||||
|
resetDownstream()
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -395,19 +566,19 @@ function addPreviewItem() {
|
|||||||
status: 'manual',
|
status: 'manual',
|
||||||
})
|
})
|
||||||
selectPreviewItem(id)
|
selectPreviewItem(id)
|
||||||
|
resetDownstream()
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removePreviewItem(id: string) {
|
async function removePreviewItem(id: string) {
|
||||||
try {
|
const confirmed = await confirmDialogRef.value?.open({
|
||||||
await ElMessageBox.confirm('删除只影响本次处理,不会修改源文件。确认删除吗?', '删除预览内容', {
|
title: '删除预览内容?',
|
||||||
confirmButtonText: '删除',
|
message: '删除只影响本次处理,不会修改源文件。删除后可重新从源文件生成预览。',
|
||||||
cancelButtonText: '取消',
|
confirmText: '删除',
|
||||||
type: 'warning',
|
cancelText: '取消',
|
||||||
})
|
tone: 'danger',
|
||||||
} catch {
|
})
|
||||||
return
|
if (!confirmed) return
|
||||||
}
|
|
||||||
|
|
||||||
const index = previewItems.value.findIndex((item) => item.id === id)
|
const index = previewItems.value.findIndex((item) => item.id === id)
|
||||||
if (index < 0) return
|
if (index < 0) return
|
||||||
@@ -416,6 +587,7 @@ async function removePreviewItem(id: string) {
|
|||||||
if (selectedPreviewFileId.value && selectedPreviewId.value) {
|
if (selectedPreviewFileId.value && selectedPreviewId.value) {
|
||||||
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
|
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
|
||||||
}
|
}
|
||||||
|
resetDownstream()
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,8 +608,15 @@ function startGeneration() {
|
|||||||
|
|
||||||
stopGenerationTimer()
|
stopGenerationTimer()
|
||||||
generation.status = 'success'
|
generation.status = 'success'
|
||||||
generation.message = `已完成 ${previewItems.value.length.toLocaleString()} 条数据处理,可进入结果页检查。`
|
results.value = createResults(
|
||||||
results.value = createResults(previewItems.value)
|
previewItems.value,
|
||||||
|
processType.value === 'structured'
|
||||||
|
? structuredOptions.value
|
||||||
|
: processType.value === 'unstructured'
|
||||||
|
? unstructuredOptions.value
|
||||||
|
: undefined,
|
||||||
|
)
|
||||||
|
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理,可进入结果页检查。`
|
||||||
selectedResultId.value = results.value[0]?.id ?? null
|
selectedResultId.value = results.value[0]?.id ?? null
|
||||||
dirty.value = true
|
dirty.value = true
|
||||||
ElMessage.success('数据处理完成')
|
ElMessage.success('数据处理完成')
|
||||||
@@ -538,25 +717,29 @@ async function handleCancel() {
|
|||||||
router.back()
|
router.back()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
try {
|
const confirmed = await confirmDialogRef.value?.open({
|
||||||
await ElMessageBox.confirm('当前存在未保存修改,确定离开吗?', '离开创建任务', {
|
title: '确认离开当前页面?',
|
||||||
confirmButtonText: '放弃修改',
|
message: '当前存在未保存修改,离开后这些修改将不会保留。',
|
||||||
cancelButtonText: '继续编辑',
|
confirmText: '放弃修改',
|
||||||
type: 'warning',
|
cancelText: '继续编辑',
|
||||||
})
|
tone: 'danger',
|
||||||
allowLeave = true
|
})
|
||||||
router.back()
|
if (!confirmed) return
|
||||||
} catch {
|
allowLeave = true
|
||||||
// 用户继续编辑。
|
router.back()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
onBeforeRouteLeave((_to, _from, next) => {
|
onBeforeRouteLeave(async () => {
|
||||||
if (allowLeave || !dirty.value) {
|
if (allowLeave || !dirty.value) return true
|
||||||
next()
|
const confirmed = await confirmDialogRef.value?.open({
|
||||||
return
|
title: '确认离开当前页面?',
|
||||||
}
|
message: '当前存在未保存修改,离开后这些修改将不会保留。',
|
||||||
next(window.confirm('当前存在未保存修改,确定离开吗?'))
|
confirmText: '放弃修改',
|
||||||
|
cancelText: '继续编辑',
|
||||||
|
tone: 'danger',
|
||||||
|
})
|
||||||
|
if (confirmed) allowLeave = true
|
||||||
|
return Boolean(confirmed)
|
||||||
})
|
})
|
||||||
|
|
||||||
onBeforeUnmount(stopGenerationTimer)
|
onBeforeUnmount(stopGenerationTimer)
|
||||||
@@ -599,6 +782,8 @@ onMounted(restoreDraft)
|
|||||||
v-model:name="task.name"
|
v-model:name="task.name"
|
||||||
v-model:description="task.description"
|
v-model:description="task.description"
|
||||||
v-model:process-type="processType"
|
v-model:process-type="processType"
|
||||||
|
v-model:structured-options="structuredOptions"
|
||||||
|
v-model:unstructured-options="unstructuredOptions"
|
||||||
:uploaded-files="uploadedFiles"
|
:uploaded-files="uploadedFiles"
|
||||||
:external-source="externalSource"
|
:external-source="externalSource"
|
||||||
:external-pulling="externalPulling"
|
:external-pulling="externalPulling"
|
||||||
@@ -673,6 +858,8 @@ onMounted(restoreDraft)
|
|||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<AppConfirmDialog ref="confirmDialogRef" />
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped lang="scss">
|
<style scoped lang="scss">
|
||||||
|
|||||||
@@ -1,7 +1,18 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, ref, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
import type { FormInstance, FormRules, UploadFile } from 'element-plus'
|
import type { FormInstance, FormRules, UploadFile } from 'element-plus'
|
||||||
import type { ExternalDataSource, ProcessType } from './types'
|
import type {
|
||||||
|
ChunkMethod,
|
||||||
|
DatasetSplitOptions,
|
||||||
|
ExternalDataSource,
|
||||||
|
GenerationContextScope,
|
||||||
|
PreprocessOption,
|
||||||
|
ProcessType,
|
||||||
|
QuestionGenerationType,
|
||||||
|
StructuredProcessOptions,
|
||||||
|
UnstructuredPreprocessOption,
|
||||||
|
UnstructuredProcessOptions,
|
||||||
|
} from './types'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
name: string
|
name: string
|
||||||
@@ -11,6 +22,8 @@ const props = defineProps<{
|
|||||||
externalSource: ExternalDataSource
|
externalSource: ExternalDataSource
|
||||||
externalPulling: boolean
|
externalPulling: boolean
|
||||||
externalConnected: boolean
|
externalConnected: boolean
|
||||||
|
structuredOptions: StructuredProcessOptions
|
||||||
|
unstructuredOptions: UnstructuredProcessOptions
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
@@ -18,6 +31,8 @@ const emit = defineEmits<{
|
|||||||
'update:description': [value: string]
|
'update:description': [value: string]
|
||||||
'update:processType': [value: ProcessType]
|
'update:processType': [value: ProcessType]
|
||||||
'update:externalSource': [value: ExternalDataSource]
|
'update:externalSource': [value: ExternalDataSource]
|
||||||
|
'update:structuredOptions': [value: StructuredProcessOptions]
|
||||||
|
'update:unstructuredOptions': [value: UnstructuredProcessOptions]
|
||||||
'file-change': [file: UploadFile]
|
'file-change': [file: UploadFile]
|
||||||
'remove-file': [uid: string | number]
|
'remove-file': [uid: string | number]
|
||||||
'use-sample': []
|
'use-sample': []
|
||||||
@@ -38,16 +53,169 @@ const AUTH_MODES = [
|
|||||||
{ value: 'token', label: 'Token' },
|
{ value: 'token', label: 'Token' },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const PREPROCESS_OPTIONS: Array<{
|
||||||
|
value: PreprocessOption
|
||||||
|
label: string
|
||||||
|
description: string
|
||||||
|
}> = [
|
||||||
|
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
||||||
|
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
||||||
|
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
||||||
|
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
|
||||||
|
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
||||||
|
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const UNSTRUCTURED_PREPROCESS_OPTIONS: Array<{
|
||||||
|
value: UnstructuredPreprocessOption
|
||||||
|
label: string
|
||||||
|
description: string
|
||||||
|
}> = [
|
||||||
|
{ value: 'clean_invalid_content', label: '清理无效内容', description: '清除空段、乱码、页眉页脚和多余空白' },
|
||||||
|
{ value: 'detect_document_structure', label: '识别文档结构', description: '识别标题、章节、段落及特殊内容块' },
|
||||||
|
{ value: 'merge_short_content', label: '合并过短内容', description: '将信息不完整的过短段落并入上下文' },
|
||||||
|
{ value: 'filter_low_quality', label: '过滤低质量内容', description: '过滤广告、导航、无意义重复及信息过少内容' },
|
||||||
|
{ value: 'deduplicate_content', label: '重复内容去重', description: '识别完全重复或高度相似的段落' },
|
||||||
|
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱和证件号等信息' },
|
||||||
|
{ value: 'preserve_context', label: '保留上下文信息', description: '为切片保留所属文档、章节和标题信息' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||||||
|
{ value: 'semantic', label: '自动语义切分' },
|
||||||
|
{ value: 'heading', label: '按标题和段落' },
|
||||||
|
{ value: 'fixed', label: '按固定长度' },
|
||||||
|
{ value: 'custom', label: '自定义分隔符' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const CONTEXT_SCOPES: Array<{ value: GenerationContextScope; label: string }> = [
|
||||||
|
{ value: 'current', label: '当前切片' },
|
||||||
|
{ value: 'adjacent', label: '相邻切片' },
|
||||||
|
{ value: 'section', label: '当前章节' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const GENERATION_TYPES: Array<{ value: QuestionGenerationType; label: string }> = [
|
||||||
|
{ value: 'factual', label: '事实问答' },
|
||||||
|
{ value: 'concept', label: '概念解释' },
|
||||||
|
{ value: 'procedure', label: '操作步骤' },
|
||||||
|
{ value: 'reasoning', label: '原因分析' },
|
||||||
|
{ value: 'comprehensive', label: '综合问答' },
|
||||||
|
]
|
||||||
|
|
||||||
function updateExternalField<K extends keyof ExternalDataSource>(field: K, value: ExternalDataSource[K]) {
|
function updateExternalField<K extends keyof ExternalDataSource>(field: K, value: ExternalDataSource[K]) {
|
||||||
emit('update:externalSource', { ...props.externalSource, [field]: value })
|
emit('update:externalSource', { ...props.externalSource, [field]: value })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateStructuredField<K extends keyof StructuredProcessOptions>(
|
||||||
|
field: K,
|
||||||
|
value: StructuredProcessOptions[K],
|
||||||
|
) {
|
||||||
|
emit('update:structuredOptions', { ...props.structuredOptions, [field]: value })
|
||||||
|
}
|
||||||
|
|
||||||
|
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||||
|
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||||
|
const preprocessOptions = value.filter(
|
||||||
|
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
|
||||||
|
)
|
||||||
|
updateStructuredField('preprocessOptions', preprocessOptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateSemanticEnrichment(value: string | number | boolean) {
|
||||||
|
updateStructuredField('semanticEnrichment', Boolean(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateDatasetSplit(field: keyof DatasetSplitOptions, value: number | undefined) {
|
||||||
|
updateStructuredField('datasetSplit', {
|
||||||
|
...props.structuredOptions.datasetSplit,
|
||||||
|
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateUnstructuredField<K extends keyof UnstructuredProcessOptions>(
|
||||||
|
field: K,
|
||||||
|
value: UnstructuredProcessOptions[K],
|
||||||
|
) {
|
||||||
|
emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value })
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateUnstructuredPreprocessOptions(value: Array<string | number | boolean>) {
|
||||||
|
const allowedValues = new Set(UNSTRUCTURED_PREPROCESS_OPTIONS.map((option) => option.value))
|
||||||
|
const preprocessOptions = value.filter(
|
||||||
|
(option): option is UnstructuredPreprocessOption => (
|
||||||
|
typeof option === 'string' && allowedValues.has(option as UnstructuredPreprocessOption)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
updateUnstructuredField('preprocessOptions', preprocessOptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateGenerationTypes(value: Array<string | number | boolean>) {
|
||||||
|
const allowedValues = new Set(GENERATION_TYPES.map((option) => option.value))
|
||||||
|
const generationTypes = value.filter(
|
||||||
|
(option): option is QuestionGenerationType => (
|
||||||
|
typeof option === 'string' && allowedValues.has(option as QuestionGenerationType)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
updateUnstructuredField('generationTypes', generationTypes)
|
||||||
|
}
|
||||||
|
|
||||||
|
const UNSTRUCTURED_NUMBER_LIMITS = {
|
||||||
|
chunkSize: { min: 200, max: 2000 },
|
||||||
|
chunkOverlap: { min: 0, max: 500 },
|
||||||
|
minChunkSize: { min: 20, max: 500 },
|
||||||
|
qaPairsPerChunk: { min: 1, max: 3 },
|
||||||
|
} as const
|
||||||
|
|
||||||
|
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
|
||||||
|
|
||||||
|
function updateUnstructuredNumber(field: UnstructuredNumberField, value: number | undefined) {
|
||||||
|
const limits = UNSTRUCTURED_NUMBER_LIMITS[field]
|
||||||
|
const parsedValue = Number(value)
|
||||||
|
const nextValue = Number.isFinite(parsedValue) ? parsedValue : limits.min
|
||||||
|
updateUnstructuredField(field, Math.min(limits.max, Math.max(limits.min, nextValue)))
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value: number | undefined) {
|
||||||
|
updateUnstructuredField('datasetSplit', {
|
||||||
|
...props.unstructuredOptions.datasetSplit,
|
||||||
|
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const formRef = ref<FormInstance>()
|
const formRef = ref<FormInstance>()
|
||||||
const formModel = computed(() => ({
|
const formModel = computed(() => ({
|
||||||
name: props.name,
|
name: props.name,
|
||||||
processType: props.processType,
|
processType: props.processType,
|
||||||
}))
|
}))
|
||||||
|
|
||||||
|
const splitTotal = computed(() => {
|
||||||
|
const { train, validation, test } = props.structuredOptions.datasetSplit
|
||||||
|
return train + validation + test
|
||||||
|
})
|
||||||
|
|
||||||
|
const unstructuredSplitTotal = computed(() => {
|
||||||
|
const { train, validation, test } = props.unstructuredOptions.datasetSplit
|
||||||
|
return train + validation + test
|
||||||
|
})
|
||||||
|
|
||||||
|
const chunkValidationMessage = computed(() => {
|
||||||
|
if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) {
|
||||||
|
return '重叠长度必须小于切片长度'
|
||||||
|
}
|
||||||
|
if (props.unstructuredOptions.minChunkSize > props.unstructuredOptions.chunkSize) {
|
||||||
|
return '最小切片长度不能大于切片长度'
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
props.unstructuredOptions.chunkOverlap + props.unstructuredOptions.minChunkSize
|
||||||
|
> props.unstructuredOptions.chunkSize
|
||||||
|
) {
|
||||||
|
return '重叠长度与最小切片长度之和不能大于切片长度'
|
||||||
|
}
|
||||||
|
if (props.unstructuredOptions.chunkMethod === 'custom' && !props.unstructuredOptions.customDelimiter.trim()) {
|
||||||
|
return '请输入自定义分隔符'
|
||||||
|
}
|
||||||
|
return ''
|
||||||
|
})
|
||||||
|
|
||||||
const rules: FormRules = {
|
const rules: FormRules = {
|
||||||
name: [
|
name: [
|
||||||
{ required: true, message: '请输入任务名称', trigger: 'blur' },
|
{ required: true, message: '请输入任务名称', trigger: 'blur' },
|
||||||
@@ -88,6 +256,12 @@ async function validate() {
|
|||||||
if (!formRef.value) return false
|
if (!formRef.value) return false
|
||||||
try {
|
try {
|
||||||
await formRef.value.validate()
|
await formRef.value.validate()
|
||||||
|
if (props.processType === 'structured' && splitTotal.value !== 100) return false
|
||||||
|
if (props.processType === 'unstructured') {
|
||||||
|
if (unstructuredSplitTotal.value !== 100) return false
|
||||||
|
if (chunkValidationMessage.value) return false
|
||||||
|
if (props.unstructuredOptions.generationTypes.length === 0) return false
|
||||||
|
}
|
||||||
return true
|
return true
|
||||||
} catch {
|
} catch {
|
||||||
return false
|
return false
|
||||||
@@ -178,6 +352,461 @@ defineExpose({ validate })
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<template v-if="processType === 'structured'">
|
||||||
|
<div class="form-section structured-options-section">
|
||||||
|
<div class="section-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>预处理选项</h3>
|
||||||
|
<p>选择在生成问答对之前需要执行的数据处理方式</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<el-checkbox-group
|
||||||
|
:model-value="structuredOptions.preprocessOptions"
|
||||||
|
class="preprocess-option-grid"
|
||||||
|
@update:model-value="updatePreprocessOptions"
|
||||||
|
>
|
||||||
|
<el-checkbox
|
||||||
|
v-for="option in PREPROCESS_OPTIONS"
|
||||||
|
:key="option.value"
|
||||||
|
:value="option.value"
|
||||||
|
class="preprocess-option"
|
||||||
|
>
|
||||||
|
<span class="preprocess-option-copy">
|
||||||
|
<strong>{{ option.label }}</strong>
|
||||||
|
<small>{{ option.description }}</small>
|
||||||
|
</span>
|
||||||
|
</el-checkbox>
|
||||||
|
</el-checkbox-group>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-section generation-options-section">
|
||||||
|
<div class="section-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>生成选项</h3>
|
||||||
|
<p>配置每条结构化记录生成问答对的方式和数量</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="generation-option-list">
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>语义丰富表达</strong>
|
||||||
|
<small>使用大模型将问答表述得更自然、柔和</small>
|
||||||
|
</div>
|
||||||
|
<el-switch
|
||||||
|
:model-value="structuredOptions.semanticEnrichment"
|
||||||
|
inline-prompt
|
||||||
|
active-text="开"
|
||||||
|
inactive-text="关"
|
||||||
|
@update:model-value="updateSemanticEnrichment"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>每行生成数量</strong>
|
||||||
|
<small>每行结构化数据生成的问答对数量</small>
|
||||||
|
</div>
|
||||||
|
<el-input-number
|
||||||
|
:model-value="structuredOptions.qaPairsPerRow"
|
||||||
|
:min="1"
|
||||||
|
:max="5"
|
||||||
|
:step="1"
|
||||||
|
controls-position="right"
|
||||||
|
@update:model-value="updateStructuredField('qaPairsPerRow', Number($event) || 1)"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="generation-option-row dataset-split-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>数据集划分</strong>
|
||||||
|
<small>按比例将生成后的问答对随机划分为训练集、验证集和测试集</small>
|
||||||
|
</div>
|
||||||
|
<div class="dataset-split-config">
|
||||||
|
<div class="dataset-split-grid">
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>训练集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="structuredOptions.datasetSplit.train"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="训练集比例"
|
||||||
|
@update:model-value="updateDatasetSplit('train', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>验证集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="structuredOptions.datasetSplit.validation"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="验证集比例"
|
||||||
|
@update:model-value="updateDatasetSplit('validation', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>测试集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="structuredOptions.datasetSplit.test"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="测试集比例"
|
||||||
|
@update:model-value="updateDatasetSplit('test', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="dataset-split-summary" :class="{ 'is-invalid': splitTotal !== 100 }">
|
||||||
|
<span>总计 {{ splitTotal }}%</span>
|
||||||
|
<span v-if="splitTotal !== 100" role="alert">训练集、验证集和测试集比例总和必须为 100%</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<template v-if="processType === 'unstructured'">
|
||||||
|
<div class="form-section unstructured-options-section">
|
||||||
|
<div class="section-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>预处理选项</h3>
|
||||||
|
<p>先清理和补全文档语义,再进入切分和问答生成</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<el-checkbox-group
|
||||||
|
:model-value="unstructuredOptions.preprocessOptions"
|
||||||
|
class="preprocess-option-grid"
|
||||||
|
@update:model-value="updateUnstructuredPreprocessOptions"
|
||||||
|
>
|
||||||
|
<el-checkbox
|
||||||
|
v-for="option in UNSTRUCTURED_PREPROCESS_OPTIONS"
|
||||||
|
:key="option.value"
|
||||||
|
:value="option.value"
|
||||||
|
class="preprocess-option"
|
||||||
|
>
|
||||||
|
<span class="preprocess-option-copy">
|
||||||
|
<strong>{{ option.label }}</strong>
|
||||||
|
<small>{{ option.description }}</small>
|
||||||
|
</span>
|
||||||
|
</el-checkbox>
|
||||||
|
</el-checkbox-group>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-section chunk-options-section">
|
||||||
|
<div class="section-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>切分选项</h3>
|
||||||
|
<p>以语义完整为优先,将长文档拆成可独立生成问答的内容块</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="chunk-settings-grid">
|
||||||
|
<label class="config-field">
|
||||||
|
<span class="config-field-label">切分方式</span>
|
||||||
|
<el-select
|
||||||
|
:model-value="unstructuredOptions.chunkMethod"
|
||||||
|
aria-label="切分方式"
|
||||||
|
@update:model-value="updateUnstructuredField('chunkMethod', $event)"
|
||||||
|
>
|
||||||
|
<el-option
|
||||||
|
v-for="method in CHUNK_METHODS"
|
||||||
|
:key="method.value"
|
||||||
|
:label="method.label"
|
||||||
|
:value="method.value"
|
||||||
|
/>
|
||||||
|
</el-select>
|
||||||
|
<small>推荐使用自动语义切分,在长度限制内优先保留完整句段</small>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<label class="config-field">
|
||||||
|
<span class="config-field-label">切片长度</span>
|
||||||
|
<span class="unit-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.chunkSize"
|
||||||
|
:min="200"
|
||||||
|
:max="2000"
|
||||||
|
:step="50"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="切片长度"
|
||||||
|
@update:model-value="updateUnstructuredNumber('chunkSize', $event)"
|
||||||
|
/>
|
||||||
|
<span>Token</span>
|
||||||
|
</span>
|
||||||
|
<small>单个切片的目标上限,默认 800 Token</small>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<label class="config-field">
|
||||||
|
<span class="config-field-label">重叠长度</span>
|
||||||
|
<span class="unit-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.chunkOverlap"
|
||||||
|
:min="0"
|
||||||
|
:max="500"
|
||||||
|
:step="10"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="重叠长度"
|
||||||
|
@update:model-value="updateUnstructuredNumber('chunkOverlap', $event)"
|
||||||
|
/>
|
||||||
|
<span>Token</span>
|
||||||
|
</span>
|
||||||
|
<small>在相邻切片中最多重复保留的上下文,默认 100 Token</small>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<label class="config-field">
|
||||||
|
<span class="config-field-label">最小切片长度</span>
|
||||||
|
<span class="unit-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.minChunkSize"
|
||||||
|
:min="20"
|
||||||
|
:max="500"
|
||||||
|
:step="10"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="最小切片长度"
|
||||||
|
@update:model-value="updateUnstructuredNumber('minChunkSize', $event)"
|
||||||
|
/>
|
||||||
|
<span>Token</span>
|
||||||
|
</span>
|
||||||
|
<small>过短的尾部内容会尽量并入前一个切片</small>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<label
|
||||||
|
v-if="unstructuredOptions.chunkMethod === 'custom'"
|
||||||
|
class="config-field is-full-width"
|
||||||
|
>
|
||||||
|
<span class="config-field-label">自定义分隔符</span>
|
||||||
|
<el-input
|
||||||
|
:model-value="unstructuredOptions.customDelimiter"
|
||||||
|
maxlength="40"
|
||||||
|
show-word-limit
|
||||||
|
placeholder="例如:--- 或 ###"
|
||||||
|
aria-label="自定义分隔符"
|
||||||
|
@update:model-value="updateUnstructuredField('customDelimiter', $event)"
|
||||||
|
/>
|
||||||
|
<small>系统会优先在分隔符位置结束当前切片</small>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="chunk-estimation-note">
|
||||||
|
Token 数为轻量估算值,实际长度以训练使用的模型分词器为准。
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<p v-if="chunkValidationMessage" class="option-validation-message" role="alert">
|
||||||
|
{{ chunkValidationMessage }}
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="preserve-options-panel">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>特殊内容保护</strong>
|
||||||
|
<small>避免切分点破坏表格、代码或列表的完整性</small>
|
||||||
|
</div>
|
||||||
|
<div class="preserve-option-grid">
|
||||||
|
<el-checkbox
|
||||||
|
:model-value="unstructuredOptions.preserveTables"
|
||||||
|
@update:model-value="updateUnstructuredField('preserveTables', Boolean($event))"
|
||||||
|
>
|
||||||
|
完整保留表格
|
||||||
|
</el-checkbox>
|
||||||
|
<el-checkbox
|
||||||
|
:model-value="unstructuredOptions.preserveCodeBlocks"
|
||||||
|
@update:model-value="updateUnstructuredField('preserveCodeBlocks', Boolean($event))"
|
||||||
|
>
|
||||||
|
完整保留代码块
|
||||||
|
</el-checkbox>
|
||||||
|
<el-checkbox
|
||||||
|
:model-value="unstructuredOptions.preserveLists"
|
||||||
|
@update:model-value="updateUnstructuredField('preserveLists', Boolean($event))"
|
||||||
|
>
|
||||||
|
完整保留列表
|
||||||
|
</el-checkbox>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-section generation-options-section">
|
||||||
|
<div class="section-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>生成选项</h3>
|
||||||
|
<p>配置每个切片的问答生成方式、范围和输出数量</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="generation-option-list">
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>语义丰富表达</strong>
|
||||||
|
<small>使用大模型将问答表述得更自然、柔和</small>
|
||||||
|
</div>
|
||||||
|
<el-switch
|
||||||
|
:model-value="unstructuredOptions.semanticEnrichment"
|
||||||
|
inline-prompt
|
||||||
|
active-text="开"
|
||||||
|
inactive-text="关"
|
||||||
|
@update:model-value="updateUnstructuredField('semanticEnrichment', Boolean($event))"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>每个切片生成数量</strong>
|
||||||
|
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
|
||||||
|
</div>
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.qaPairsPerChunk"
|
||||||
|
:min="1"
|
||||||
|
:max="3"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
@update:model-value="updateUnstructuredNumber('qaPairsPerChunk', $event)"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>上下文范围</strong>
|
||||||
|
<small>生成问答时可参考的文档范围</small>
|
||||||
|
</div>
|
||||||
|
<el-select
|
||||||
|
:model-value="unstructuredOptions.contextScope"
|
||||||
|
class="compact-select"
|
||||||
|
aria-label="上下文范围"
|
||||||
|
@update:model-value="updateUnstructuredField('contextScope', $event)"
|
||||||
|
>
|
||||||
|
<el-option
|
||||||
|
v-for="scope in CONTEXT_SCOPES"
|
||||||
|
:key="scope.value"
|
||||||
|
:label="scope.label"
|
||||||
|
:value="scope.value"
|
||||||
|
/>
|
||||||
|
</el-select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="generation-option-row is-stacked">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>问题类型</strong>
|
||||||
|
<small>按内容特点选择要生成的问答角度,至少保留一项</small>
|
||||||
|
</div>
|
||||||
|
<el-checkbox-group
|
||||||
|
:model-value="unstructuredOptions.generationTypes"
|
||||||
|
class="generation-type-grid"
|
||||||
|
@update:model-value="updateGenerationTypes"
|
||||||
|
>
|
||||||
|
<el-checkbox
|
||||||
|
v-for="type in GENERATION_TYPES"
|
||||||
|
:key="type.value"
|
||||||
|
:value="type.value"
|
||||||
|
>
|
||||||
|
{{ type.label }}
|
||||||
|
</el-checkbox>
|
||||||
|
</el-checkbox-group>
|
||||||
|
<span
|
||||||
|
v-if="unstructuredOptions.generationTypes.length === 0"
|
||||||
|
class="option-validation-message"
|
||||||
|
role="alert"
|
||||||
|
>
|
||||||
|
请至少选择一种问题类型
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="generation-option-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>跳过无法回答的内容</strong>
|
||||||
|
<small>当切片缺少完整信息时,不强行编造问答对</small>
|
||||||
|
</div>
|
||||||
|
<el-switch
|
||||||
|
:model-value="unstructuredOptions.skipUnanswerable"
|
||||||
|
inline-prompt
|
||||||
|
active-text="开"
|
||||||
|
inactive-text="关"
|
||||||
|
@update:model-value="updateUnstructuredField('skipUnanswerable', Boolean($event))"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="generation-option-row dataset-split-row">
|
||||||
|
<div class="generation-option-copy">
|
||||||
|
<strong>数据集划分</strong>
|
||||||
|
<small>按比例将生成后的问答对随机划分为训练集、验证集和测试集</small>
|
||||||
|
</div>
|
||||||
|
<div class="dataset-split-config">
|
||||||
|
<div class="dataset-split-grid">
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>训练集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.datasetSplit.train"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="非结构化训练集比例"
|
||||||
|
@update:model-value="updateUnstructuredDatasetSplit('train', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>验证集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.datasetSplit.validation"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="非结构化验证集比例"
|
||||||
|
@update:model-value="updateUnstructuredDatasetSplit('validation', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<label class="dataset-split-field">
|
||||||
|
<span>测试集</span>
|
||||||
|
<span class="dataset-split-input">
|
||||||
|
<el-input-number
|
||||||
|
:model-value="unstructuredOptions.datasetSplit.test"
|
||||||
|
:min="0"
|
||||||
|
:max="100"
|
||||||
|
:step="1"
|
||||||
|
:precision="0"
|
||||||
|
controls-position="right"
|
||||||
|
aria-label="非结构化测试集比例"
|
||||||
|
@update:model-value="updateUnstructuredDatasetSplit('test', $event)"
|
||||||
|
/>
|
||||||
|
<span>%</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="dataset-split-summary" :class="{ 'is-invalid': unstructuredSplitTotal !== 100 }">
|
||||||
|
<span>总计 {{ unstructuredSplitTotal }}%</span>
|
||||||
|
<span v-if="unstructuredSplitTotal !== 100" role="alert">
|
||||||
|
训练集、验证集和测试集比例总和必须为 100%
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
<div v-if="isExternal" class="form-section external-section">
|
<div v-if="isExternal" class="form-section external-section">
|
||||||
<div class="section-title-row">
|
<div class="section-title-row">
|
||||||
<div>
|
<div>
|
||||||
@@ -405,6 +1034,222 @@ defineExpose({ validate })
|
|||||||
gap: 20px;
|
gap: 20px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.preprocess-option-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.preprocess-option {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 64px;
|
||||||
|
margin-right: 0;
|
||||||
|
padding: 12px 14px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
align-items: flex-start;
|
||||||
|
border: 1px solid #e2e5ec;
|
||||||
|
border-radius: 8px;
|
||||||
|
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
border-color: #b7b2f7;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-checked {
|
||||||
|
background: #fafaff;
|
||||||
|
border-color: #8b82f4;
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.el-checkbox__input) {
|
||||||
|
margin-top: 3px;
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.el-checkbox__label) {
|
||||||
|
min-width: 0;
|
||||||
|
padding-left: 10px;
|
||||||
|
white-space: normal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.preprocess-option-copy,
|
||||||
|
.generation-option-copy {
|
||||||
|
display: grid;
|
||||||
|
gap: 4px;
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: #344054;
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
small {
|
||||||
|
color: #8a93a3;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.generation-option-list {
|
||||||
|
display: grid;
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-settings-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.config-field {
|
||||||
|
display: grid;
|
||||||
|
align-content: start;
|
||||||
|
gap: 8px;
|
||||||
|
min-width: 0;
|
||||||
|
padding: 14px;
|
||||||
|
color: #344054;
|
||||||
|
background: #fbfcfe;
|
||||||
|
border: 1px solid #e2e5ec;
|
||||||
|
border-radius: 8px;
|
||||||
|
|
||||||
|
&.is-full-width {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
> small {
|
||||||
|
color: #8a93a3;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.el-select),
|
||||||
|
:deep(.el-input) {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.config-field-label {
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.unit-input {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
color: #7b8495;
|
||||||
|
font-size: 12px;
|
||||||
|
|
||||||
|
:deep(.el-input-number) {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.option-validation-message {
|
||||||
|
display: block;
|
||||||
|
margin: 8px 0 0;
|
||||||
|
color: #dc2626;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-estimation-note {
|
||||||
|
margin: 9px 0 0;
|
||||||
|
color: #8a93a3;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.preserve-options-panel {
|
||||||
|
display: grid;
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 12px;
|
||||||
|
padding: 14px;
|
||||||
|
border: 1px solid #e2e5ec;
|
||||||
|
border-radius: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.preserve-option-grid,
|
||||||
|
.generation-type-grid {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px 24px;
|
||||||
|
|
||||||
|
:deep(.el-checkbox) {
|
||||||
|
margin-right: 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.generation-option-row {
|
||||||
|
display: flex;
|
||||||
|
min-height: 64px;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 24px;
|
||||||
|
padding: 12px 14px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
border: 1px solid #e2e5ec;
|
||||||
|
border-radius: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.generation-option-row.is-stacked {
|
||||||
|
align-items: stretch;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.compact-select {
|
||||||
|
width: 220px;
|
||||||
|
max-width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-row {
|
||||||
|
align-items: stretch;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-config {
|
||||||
|
display: grid;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-field {
|
||||||
|
display: grid;
|
||||||
|
gap: 7px;
|
||||||
|
color: #5f6878;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-input {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
|
||||||
|
:deep(.el-input-number) {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-summary {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
color: #2ca66a;
|
||||||
|
font-size: 12px;
|
||||||
|
|
||||||
|
&.is-invalid {
|
||||||
|
color: #dc2626;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
.section-title-row {
|
.section-title-row {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: flex-start;
|
align-items: flex-start;
|
||||||
@@ -650,6 +1495,32 @@ defineExpose({ validate })
|
|||||||
grid-template-columns: minmax(0, 1fr);
|
grid-template-columns: minmax(0, 1fr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.preprocess-option-grid {
|
||||||
|
grid-template-columns: minmax(0, 1fr);
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-settings-grid {
|
||||||
|
grid-template-columns: minmax(0, 1fr);
|
||||||
|
}
|
||||||
|
|
||||||
|
.config-field.is-full-width {
|
||||||
|
grid-column: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-grid {
|
||||||
|
grid-template-columns: minmax(0, 1fr);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataset-split-summary {
|
||||||
|
align-items: flex-start;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.generation-option-row:not(.dataset-split-row):not(.is-stacked) {
|
||||||
|
align-items: flex-start;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
.external-section .external-grid {
|
.external-section .external-grid {
|
||||||
grid-template-columns: minmax(0, 1fr);
|
grid-template-columns: minmax(0, 1fr);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,11 @@
|
|||||||
import type { PreviewItem, ProcessType, ResultItem, SourceLine } from './types'
|
import type {
|
||||||
|
PreviewItem,
|
||||||
|
ProcessType,
|
||||||
|
ResultItem,
|
||||||
|
SourceLine,
|
||||||
|
StructuredProcessOptions,
|
||||||
|
UnstructuredProcessOptions,
|
||||||
|
} from './types'
|
||||||
|
|
||||||
export const DEFAULT_SOURCE_TEXT = [
|
export const DEFAULT_SOURCE_TEXT = [
|
||||||
'问:如何看待当前的通货膨胀风险?',
|
'问:如何看待当前的通货膨胀风险?',
|
||||||
@@ -39,8 +46,417 @@ export function sourceLines(sourceText: string): SourceLine[] {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildPreviewItems(sourceText: string, processType: ProcessType, sourceFileId = 'default-source'): PreviewItem[] {
|
interface SourceRange {
|
||||||
const meaningfulLines = sourceLines(sourceText).filter((line) => line.content.trim())
|
start: number
|
||||||
|
end: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProtectedRange extends SourceRange {
|
||||||
|
kind: 'code' | 'table' | 'list'
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_CHUNK_SIZE = 800
|
||||||
|
const DEFAULT_CHUNK_OVERLAP = 100
|
||||||
|
const DEFAULT_MIN_CHUNK_SIZE = 100
|
||||||
|
|
||||||
|
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
|
||||||
|
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
|
||||||
|
let nextStart = Math.max(0, start)
|
||||||
|
let nextEnd = Math.min(sourceText.length, end)
|
||||||
|
|
||||||
|
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
|
||||||
|
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
|
||||||
|
|
||||||
|
return { start: nextStart, end: nextEnd }
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDelimiter(delimiter: string | undefined): string {
|
||||||
|
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
|
||||||
|
}
|
||||||
|
|
||||||
|
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
|
||||||
|
return line.start < range.end && line.end > range.start
|
||||||
|
}
|
||||||
|
|
||||||
|
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
|
||||||
|
return ranges.some((range) => overlapsRange(line, range))
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
|
||||||
|
const ranges: ProtectedRange[] = []
|
||||||
|
let openFence: { start: number; marker: string; length: number } | null = null
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
|
||||||
|
if (!fence) continue
|
||||||
|
|
||||||
|
const marker = fence[1][0]
|
||||||
|
if (!openFence) {
|
||||||
|
openFence = { start: line.start, marker, length: fence[1].length }
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (marker === openFence.marker && fence[1].length >= openFence.length) {
|
||||||
|
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
|
||||||
|
openFence = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
|
||||||
|
return ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTableSeparator(content: string): boolean {
|
||||||
|
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
|
||||||
|
const cells = normalized.split('|').map((cell) => cell.trim())
|
||||||
|
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
|
||||||
|
const ranges: ProtectedRange[] = []
|
||||||
|
|
||||||
|
for (let index = 0; index < lines.length - 1; index += 1) {
|
||||||
|
const header = lines[index]
|
||||||
|
const separator = lines[index + 1]
|
||||||
|
if (
|
||||||
|
isLineProtected(header, codeRanges)
|
||||||
|
|| isLineProtected(separator, codeRanges)
|
||||||
|
|| !header.content.includes('|')
|
||||||
|
|| !isTableSeparator(separator.content)
|
||||||
|
) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let endIndex = index + 1
|
||||||
|
while (
|
||||||
|
endIndex + 1 < lines.length
|
||||||
|
&& !isLineProtected(lines[endIndex + 1], codeRanges)
|
||||||
|
&& lines[endIndex + 1].content.trim()
|
||||||
|
&& lines[endIndex + 1].content.includes('|')
|
||||||
|
) {
|
||||||
|
endIndex += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
|
||||||
|
index = endIndex
|
||||||
|
}
|
||||||
|
|
||||||
|
return ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
function isListItem(content: string): boolean {
|
||||||
|
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
|
||||||
|
}
|
||||||
|
|
||||||
|
function isListContinuation(content: string): boolean {
|
||||||
|
return /^\s{2,}\S/.test(content)
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectListRanges(
|
||||||
|
lines: SourceLine[],
|
||||||
|
excludedRanges: SourceRange[],
|
||||||
|
): ProtectedRange[] {
|
||||||
|
const ranges: ProtectedRange[] = []
|
||||||
|
|
||||||
|
for (let index = 0; index < lines.length; index += 1) {
|
||||||
|
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
|
||||||
|
|
||||||
|
let endIndex = index
|
||||||
|
let itemCount = 1
|
||||||
|
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
|
||||||
|
const nextContent = lines[endIndex + 1].content
|
||||||
|
if (isListItem(nextContent)) {
|
||||||
|
itemCount += 1
|
||||||
|
endIndex += 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (isListContinuation(nextContent)) {
|
||||||
|
endIndex += 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if (itemCount >= 2) {
|
||||||
|
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
|
||||||
|
index = endIndex
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
|
||||||
|
return ranges
|
||||||
|
.sort((left, right) => left.start - right.start || left.end - right.end)
|
||||||
|
.reduce<ProtectedRange[]>((merged, range) => {
|
||||||
|
const previous = merged[merged.length - 1]
|
||||||
|
if (previous && range.start < previous.end) {
|
||||||
|
previous.end = Math.max(previous.end, range.end)
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
merged.push({ ...range })
|
||||||
|
return merged
|
||||||
|
}, [])
|
||||||
|
}
|
||||||
|
|
||||||
|
function protectedRangesForOptions(
|
||||||
|
sourceText: string,
|
||||||
|
options?: UnstructuredProcessOptions,
|
||||||
|
): ProtectedRange[] {
|
||||||
|
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
|
||||||
|
|
||||||
|
const lines = sourceLines(sourceText)
|
||||||
|
const codeRanges = detectCodeBlockRanges(sourceText, lines)
|
||||||
|
const tableRanges = detectTableRanges(lines, codeRanges)
|
||||||
|
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
|
||||||
|
const enabledRanges = [
|
||||||
|
...(options?.preserveCodeBlocks ? codeRanges : []),
|
||||||
|
...(options?.preserveTables ? tableRanges : []),
|
||||||
|
...(options?.preserveLists ? listRanges : []),
|
||||||
|
]
|
||||||
|
|
||||||
|
return mergeProtectedRanges(enabledRanges)
|
||||||
|
}
|
||||||
|
|
||||||
|
function protectedRangeContaining(
|
||||||
|
ranges: ProtectedRange[],
|
||||||
|
offset: number,
|
||||||
|
): ProtectedRange | undefined {
|
||||||
|
return ranges.find((range) => range.start < offset && offset < range.end)
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeChunkStart(
|
||||||
|
sourceText: string,
|
||||||
|
cursor: number,
|
||||||
|
protectedRanges: ProtectedRange[],
|
||||||
|
): number {
|
||||||
|
let start = Math.max(0, Math.min(cursor, sourceText.length))
|
||||||
|
const overlapBlock = protectedRangeContaining(protectedRanges, start)
|
||||||
|
if (overlapBlock) start = overlapBlock.end
|
||||||
|
|
||||||
|
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
|
||||||
|
|
||||||
|
// 去除块前空白时可能进入缩进代码块/列表;此时恢复到完整块起点。
|
||||||
|
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
|
||||||
|
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
|
||||||
|
|
||||||
|
return start
|
||||||
|
}
|
||||||
|
|
||||||
|
function protectChunkEnd(
|
||||||
|
proposedEnd: number,
|
||||||
|
start: number,
|
||||||
|
minimumEnd: number,
|
||||||
|
protectedRanges: ProtectedRange[],
|
||||||
|
): number {
|
||||||
|
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
|
||||||
|
if (!splitBlock) return proposedEnd
|
||||||
|
|
||||||
|
// 优先在块前结束;块前不足最小切片长度时,将整个块收入当前切片。
|
||||||
|
return splitBlock.start > start && splitBlock.start >= minimumEnd
|
||||||
|
? splitBlock.start
|
||||||
|
: splitBlock.end
|
||||||
|
}
|
||||||
|
|
||||||
|
function restoreProtectedEdges(
|
||||||
|
range: SourceRange,
|
||||||
|
rawStart: number,
|
||||||
|
rawEnd: number,
|
||||||
|
protectedRanges: ProtectedRange[],
|
||||||
|
): SourceRange {
|
||||||
|
const nextRange = { ...range }
|
||||||
|
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
|
||||||
|
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
|
||||||
|
|
||||||
|
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
|
||||||
|
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
|
||||||
|
return nextRange
|
||||||
|
}
|
||||||
|
|
||||||
|
function lastBoundaryInRange(
|
||||||
|
sourceText: string,
|
||||||
|
idealEnd: number,
|
||||||
|
minimumEnd: number,
|
||||||
|
): number | null {
|
||||||
|
const candidates: number[] = []
|
||||||
|
const boundaryTokens = ['\n\n', '\n', '。', '!', '?', ';', '.', '!', '?', ';']
|
||||||
|
|
||||||
|
boundaryTokens.forEach((token) => {
|
||||||
|
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
|
||||||
|
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
|
||||||
|
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
|
||||||
|
})
|
||||||
|
|
||||||
|
return candidates.length ? Math.max(...candidates) : null
|
||||||
|
}
|
||||||
|
|
||||||
|
function lastHeadingBoundary(
|
||||||
|
sourceText: string,
|
||||||
|
start: number,
|
||||||
|
idealEnd: number,
|
||||||
|
minimumEnd: number,
|
||||||
|
): number | null {
|
||||||
|
const section = sourceText.slice(start, idealEnd)
|
||||||
|
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
|
||||||
|
let boundary: number | null = null
|
||||||
|
let match: RegExpExecArray | null
|
||||||
|
|
||||||
|
while ((match = headingPattern.exec(section))) {
|
||||||
|
const absoluteStart = start + match.index
|
||||||
|
if (absoluteStart >= minimumEnd) boundary = absoluteStart
|
||||||
|
}
|
||||||
|
|
||||||
|
return boundary
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveChunkEnd(
|
||||||
|
sourceText: string,
|
||||||
|
start: number,
|
||||||
|
idealEnd: number,
|
||||||
|
minimumEnd: number,
|
||||||
|
options: UnstructuredProcessOptions | undefined,
|
||||||
|
): number {
|
||||||
|
const method = options?.chunkMethod ?? 'semantic'
|
||||||
|
|
||||||
|
if (method === 'fixed') return idealEnd
|
||||||
|
|
||||||
|
if (method === 'custom') {
|
||||||
|
const delimiter = normalizeDelimiter(options?.customDelimiter)
|
||||||
|
if (!delimiter) return idealEnd
|
||||||
|
|
||||||
|
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
|
||||||
|
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
|
||||||
|
return boundary >= minimumEnd ? boundary : idealEnd
|
||||||
|
}
|
||||||
|
|
||||||
|
if (method === 'heading') {
|
||||||
|
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
|
||||||
|
if (headingBoundary !== null) return headingBoundary
|
||||||
|
}
|
||||||
|
|
||||||
|
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildUnstructuredRanges(
|
||||||
|
sourceText: string,
|
||||||
|
options?: UnstructuredProcessOptions,
|
||||||
|
): SourceRange[] {
|
||||||
|
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算,避免引入分词器依赖。
|
||||||
|
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
|
||||||
|
const minimumCharacters = Math.min(
|
||||||
|
targetCharacters,
|
||||||
|
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
|
||||||
|
)
|
||||||
|
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
|
||||||
|
const protectedRanges = protectedRangesForOptions(sourceText, options)
|
||||||
|
const ranges: SourceRange[] = []
|
||||||
|
let cursor = 0
|
||||||
|
|
||||||
|
while (cursor < sourceText.length) {
|
||||||
|
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
|
||||||
|
if (start >= sourceText.length) break
|
||||||
|
|
||||||
|
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
|
||||||
|
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
|
||||||
|
let end = idealEnd === sourceText.length
|
||||||
|
? idealEnd
|
||||||
|
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
|
||||||
|
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
|
||||||
|
|
||||||
|
// 所有自定义边界都必须向前推进;异常配置回退到固定长度切分。
|
||||||
|
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
|
||||||
|
|
||||||
|
let range = trimSourceRange(sourceText, start, end)
|
||||||
|
range = restoreProtectedEdges(range, start, end, protectedRanges)
|
||||||
|
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
|
||||||
|
range.end = Math.min(end, range.start + minimumCharacters)
|
||||||
|
}
|
||||||
|
if (range.end <= range.start) {
|
||||||
|
cursor = Math.max(cursor + 1, end)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
const isLastRange = end >= sourceText.length
|
||||||
|
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
|
||||||
|
ranges[ranges.length - 1].end = range.end
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
ranges.push(range)
|
||||||
|
if (isLastRange) break
|
||||||
|
|
||||||
|
// overlap 是允许的最大重叠量;按当前切片动态收缩,保证每轮至少推进最小切片长度。
|
||||||
|
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
|
||||||
|
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
|
||||||
|
const nextCursor = range.end - actualOverlap
|
||||||
|
cursor = nextCursor > start ? nextCursor : range.end
|
||||||
|
}
|
||||||
|
|
||||||
|
return ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
|
||||||
|
if (!lines.length) return null
|
||||||
|
|
||||||
|
let low = 0
|
||||||
|
let high = lines.length - 1
|
||||||
|
let result = 0
|
||||||
|
|
||||||
|
while (low <= high) {
|
||||||
|
const middle = Math.floor((low + high) / 2)
|
||||||
|
if (lines[middle].start <= offset) {
|
||||||
|
result = middle
|
||||||
|
low = middle + 1
|
||||||
|
} else {
|
||||||
|
high = middle - 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines[result].number
|
||||||
|
}
|
||||||
|
|
||||||
|
function previewItemFromRange(
|
||||||
|
sourceText: string,
|
||||||
|
lines: SourceLine[],
|
||||||
|
range: SourceRange,
|
||||||
|
sourceFileId: string,
|
||||||
|
index: number,
|
||||||
|
): PreviewItem {
|
||||||
|
const content = sourceText.slice(range.start, range.end)
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: `preview-${sourceFileId}-${index + 1}`,
|
||||||
|
sourceFileId,
|
||||||
|
originalContent: content,
|
||||||
|
editedContent: content,
|
||||||
|
sourceStart: range.start,
|
||||||
|
sourceEnd: range.end,
|
||||||
|
sourceStartLine: lineNumberAtOffset(lines, range.start),
|
||||||
|
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
|
||||||
|
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||||
|
status: 'original',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildPreviewItems(
|
||||||
|
sourceText: string,
|
||||||
|
processType: ProcessType,
|
||||||
|
sourceFileId = 'default-source',
|
||||||
|
unstructuredOptions?: UnstructuredProcessOptions,
|
||||||
|
): PreviewItem[] {
|
||||||
|
const lines = sourceLines(sourceText)
|
||||||
|
|
||||||
|
if (processType === 'unstructured') {
|
||||||
|
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
|
||||||
|
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
const meaningfulLines = lines.filter((line) => line.content.trim())
|
||||||
const groupSize = processType === 'structured' ? 1 : 3
|
const groupSize = processType === 'structured' ? 1 : 3
|
||||||
const items: PreviewItem[] = []
|
const items: PreviewItem[] = []
|
||||||
|
|
||||||
@@ -69,21 +485,44 @@ export function buildPreviewItems(sourceText: string, processType: ProcessType,
|
|||||||
return items
|
return items
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createResults(items: PreviewItem[]): ResultItem[] {
|
const SEMANTIC_PREFIXES = [
|
||||||
return items.slice(0, 12).map((item, index) => {
|
'请结合实际情况,说明一下:',
|
||||||
|
'如果方便的话,请详细解答:',
|
||||||
|
'请用通俗易懂的方式说明:',
|
||||||
|
'请从实际应用角度说明:',
|
||||||
|
'请简洁、自然地说明:',
|
||||||
|
]
|
||||||
|
|
||||||
|
export function createResults(
|
||||||
|
items: PreviewItem[],
|
||||||
|
options?: StructuredProcessOptions | UnstructuredProcessOptions,
|
||||||
|
): ResultItem[] {
|
||||||
|
const resultCount = options && 'qaPairsPerChunk' in options
|
||||||
|
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||||||
|
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||||||
|
|
||||||
|
return items.flatMap((item, index) => {
|
||||||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||||||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||||||
const instruction = firstLine.replace(/^问[::]\s*/, '').trim() || `数据条目 ${index + 1}`
|
const baseInstruction = firstLine.replace(/^问[::]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||||
|
|
||||||
return {
|
return Array.from({ length: resultCount }, (_, variantIndex) => {
|
||||||
id: `result-${index + 1}`,
|
const instruction = options?.semanticEnrichment
|
||||||
instruction,
|
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
|
||||||
input: '',
|
: variantIndex === 0
|
||||||
output,
|
? baseInstruction
|
||||||
originalInstruction: instruction,
|
: `${baseInstruction}(问法 ${variantIndex + 1})`
|
||||||
originalInput: '',
|
|
||||||
originalOutput: output,
|
return {
|
||||||
status: 'valid',
|
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
|
||||||
}
|
instruction,
|
||||||
|
input: '',
|
||||||
|
output,
|
||||||
|
originalInstruction: instruction,
|
||||||
|
originalInput: '',
|
||||||
|
originalOutput: output,
|
||||||
|
status: 'valid' as const,
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,65 @@ export type ProcessType = 'structured' | 'unstructured' | 'external'
|
|||||||
|
|
||||||
export type StepId = 'create' | 'preview' | 'generate' | 'results'
|
export type StepId = 'create' | 'preview' | 'generate' | 'results'
|
||||||
|
|
||||||
|
export type PreprocessOption =
|
||||||
|
| 'clean_invalid'
|
||||||
|
| 'detect_structure'
|
||||||
|
| 'deduplicate'
|
||||||
|
| 'normalize_format'
|
||||||
|
| 'filter_anomaly'
|
||||||
|
| 'desensitize'
|
||||||
|
|
||||||
|
export interface DatasetSplitOptions {
|
||||||
|
train: number
|
||||||
|
validation: number
|
||||||
|
test: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StructuredProcessOptions {
|
||||||
|
preprocessOptions: PreprocessOption[]
|
||||||
|
semanticEnrichment: boolean
|
||||||
|
qaPairsPerRow: number
|
||||||
|
datasetSplit: DatasetSplitOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
export type UnstructuredPreprocessOption =
|
||||||
|
| 'clean_invalid_content'
|
||||||
|
| 'detect_document_structure'
|
||||||
|
| 'merge_short_content'
|
||||||
|
| 'filter_low_quality'
|
||||||
|
| 'deduplicate_content'
|
||||||
|
| 'desensitize'
|
||||||
|
| 'preserve_context'
|
||||||
|
|
||||||
|
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
|
||||||
|
|
||||||
|
export type GenerationContextScope = 'current' | 'adjacent' | 'section'
|
||||||
|
|
||||||
|
export type QuestionGenerationType =
|
||||||
|
| 'factual'
|
||||||
|
| 'concept'
|
||||||
|
| 'procedure'
|
||||||
|
| 'reasoning'
|
||||||
|
| 'comprehensive'
|
||||||
|
|
||||||
|
export interface UnstructuredProcessOptions {
|
||||||
|
preprocessOptions: UnstructuredPreprocessOption[]
|
||||||
|
chunkMethod: ChunkMethod
|
||||||
|
chunkSize: number
|
||||||
|
chunkOverlap: number
|
||||||
|
minChunkSize: number
|
||||||
|
customDelimiter: string
|
||||||
|
preserveTables: boolean
|
||||||
|
preserveCodeBlocks: boolean
|
||||||
|
preserveLists: boolean
|
||||||
|
semanticEnrichment: boolean
|
||||||
|
qaPairsPerChunk: number
|
||||||
|
contextScope: GenerationContextScope
|
||||||
|
generationTypes: QuestionGenerationType[]
|
||||||
|
skipUnanswerable: boolean
|
||||||
|
datasetSplit: DatasetSplitOptions
|
||||||
|
}
|
||||||
|
|
||||||
export interface ExternalDataSource {
|
export interface ExternalDataSource {
|
||||||
type: string
|
type: string
|
||||||
url: string
|
url: string
|
||||||
|
|||||||
Reference in New Issue
Block a user