diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index 4dffb35..0d298fe 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -470,8 +470,8 @@ assert.match(generationControlSource, /modelMeta\(model\)/, '模型下拉缺少 assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型') assert.match( typesSource, - /export type ChunkMethod = 'structure' \| 'fixed' \| 'custom'/, - '非结构化切分方式类型必须只保留 structure、fixed 和 custom', + /export type ChunkMethod = 'layout_hybrid' \| 'semantic' \| 'fixed'/, + '非结构化切分方式类型必须只保留 layout_hybrid、semantic 和 fixed', ) for (const field of [ 'preprocessOptions', @@ -479,7 +479,7 @@ for (const field of [ 'chunkSize', 'chunkOverlap', 'minChunkSize', - 'customDelimiter', + 'semanticBreakpointPercentile', 'preserveTables', 'preserveCodeBlocks', 'preserveLists', @@ -528,9 +528,9 @@ assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开 assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类') const expectedChunkMethods = [ - ['structure', '文档结构'], - ['fixed', '固定 Token'], - ['custom', '自定义分隔符'], + ['layout_hybrid', '版面结构混合切分'], + ['semantic', '语义切分'], + ['fixed', '固定 Token 切分'], ] const chunkMethodsStart = unstructuredOptionsSource.indexOf('const CHUNK_METHODS') const chunkMethodsEnd = unstructuredOptionsSource.indexOf('const UNSTRUCTURED_NUMBER_LIMITS', chunkMethodsStart) @@ -540,10 +540,14 @@ assert.deepEqual(chunkMethodValues, expectedChunkMethods.map(([value]) => value) for (const [value, label] of expectedChunkMethods) { assert.ok(chunkMethodsSource.includes(`value: '${value}', label: '${label}'`), `切分方式缺少选项：${label}`) } -for (const removedMethod of ['semantic', 'heading']) { +for (const removedMethod of ['structure', 'custom', 'heading']) { assert.ok(!chunkMethodsSource.includes(`value: '${removedMethod}'`), `切分方式仍保留已移除值：${removedMethod}`) } -assert.ok(unstructuredOptionsSource.includes('推荐使用文档结构'), '切分方式缺少文档结构默认推荐说明') +assert.ok(unstructuredOptionsSource.includes('默认推荐，按标题、段落、列表和表格结构切分'), '版面结构混合切分缺少推荐说明') +assert.ok(unstructuredOptionsSource.includes('根据相邻内容的语义变化寻找主题边界'), '语义切分缺少适用场景说明') +assert.ok(unstructuredOptionsSource.includes('按句子边界控制固定 Token 长度'), '固定 Token 切分缺少稳定性说明') +assert.doesNotMatch(unstructuredOptionsSource, /自定义分隔符/, '切分选项仍保留自定义分隔符') +assert.doesNotMatch(taskSetupSource, /customDelimiter|自定义分隔符/, '任务配置仍校验已移除的自定义分隔符') for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) { assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`) } @@ -554,6 +558,8 @@ assert.match(unstructuredOptionsSource, /function updateSpecialContentProtection assert.match(unstructuredOptionsSource, /options\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token') assert.match(unstructuredOptionsSource, /options\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token') assert.match(unstructuredOptionsSource, /options\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token') +assert.match(unstructuredOptionsSource, /options\.semanticBreakpointPercentile[\s\S]*?:min="1"[\s\S]*?:max="99"/, '语义断点百分位必须限制在 1 到 99') +assert.match(unstructuredOptionsSource, /v-if="options\.chunkMethod === 'semantic'"[\s\S]*?options\.semanticBreakpointPercentile/, '语义断点百分位必须仅在语义切分时显示') for (const label of ['每个切片生成数量', '数据集划分']) { assert.ok(taskSetupFeatureSource.includes(label), `非结构化生成选项缺少：${label}`) @@ -568,10 +574,11 @@ assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), ' assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局') assert.match(viewSource, /const unstructuredOptions = ref/, '父页面缺少非结构化配置状态') -assert.match(stateSource, /chunkMethod:\s*'structure'/, '非结构化默认切分方式必须为文档结构') +assert.match(stateSource, /chunkMethod:\s*'layout_hybrid'/, '非结构化默认切分方式必须为版面结构混合切分') assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') +assert.match(stateSource, /semanticBreakpointPercentile:\s*95/, '默认语义断点百分位必须为 95') assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置') assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断') @@ -627,7 +634,7 @@ for (const [backendField, frontendField] of [ ['chunk_size', 'chunkSize'], ['chunk_overlap', 'chunkOverlap'], ['min_chunk_size', 'minChunkSize'], - ['custom_delimiter', 'customDelimiter'], + ['semantic_breakpoint_percentile', 'semanticBreakpointPercentile'], ['preserve_tables', 'preserveTables'], ['preserve_code_blocks', 'preserveCodeBlocks'], ['preserve_lists', 'preserveLists'], @@ -659,7 +666,7 @@ for (const field of [ 'chunkSize', 'chunkOverlap', 'minChunkSize', - 'customDelimiter', + 'semanticBreakpointPercentile', 'preserveTables', 'preserveCodeBlocks', 'preserveLists', diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 3d7e46f..15b6a9c 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -142,6 +142,7 @@ export interface DataProcessPreviewItem { source_end_line: number | null token_count: number status: 'original' | 'modified' | 'manual' | 'invalid' + quality_score?: DataProcessQualityScore | null updated_at?: string } diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 542af93..c627147 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -218,7 +218,7 @@ function toBackendConfig(): DataProcessConfig { chunk_size: unstructuredOptions.value.chunkSize, chunk_overlap: unstructuredOptions.value.chunkOverlap, min_chunk_size: unstructuredOptions.value.minChunkSize, - custom_delimiter: unstructuredOptions.value.customDelimiter, + semantic_breakpoint_percentile: unstructuredOptions.value.semanticBreakpointPercentile, preserve_tables: unstructuredOptions.value.preserveTables, preserve_code_blocks: unstructuredOptions.value.preserveCodeBlocks, preserve_lists: unstructuredOptions.value.preserveLists, @@ -266,6 +266,9 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem { sourceEndLine: item.source_end_line, tokenCount: item.token_count, status: item.status, + sourcePages: Array.isArray(item.quality_score?.source_pages) + ? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number') + : [], updatedAt: item.updated_at, } } @@ -283,7 +286,7 @@ function previewAffectingOptions() { chunkSize, chunkOverlap, minChunkSize, - customDelimiter, + semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, @@ -294,7 +297,7 @@ function previewAffectingOptions() { chunkSize, chunkOverlap, minChunkSize, - customDelimiter: chunkMethod === 'custom' ? customDelimiter : '', + semanticBreakpointPercentile, preserveTables, preserveCodeBlocks, preserveLists, diff --git a/frontend/src/views/data-process/DataProcessDetailView.vue b/frontend/src/views/data-process/DataProcessDetailView.vue index 418a02e..cb3e79a 100644 --- a/frontend/src/views/data-process/DataProcessDetailView.vue +++ b/frontend/src/views/data-process/DataProcessDetailView.vue @@ -79,16 +79,16 @@ const configLabelMap: Record = { chunk_size: '切片长度', chunk_overlap: '重叠长度', min_chunk_size: '最小切片长度', - custom_delimiter: '自定义分隔符', + semantic_breakpoint_percentile: '语义断点百分位', preserve_tables: '保留表格', preserve_code_blocks: '保留代码块', preserve_lists: '保留列表', } const chunkMethodLabelMap: Record = { - structure: '文档结构', + layout_hybrid: '版面结构混合切分', + semantic: '语义切分', fixed: '固定 Token', - custom: '自定义分隔符', } const preprocessOptionLabelMap: Record = { diff --git a/frontend/src/views/data-process/create/PdfSourceViewer.vue b/frontend/src/views/data-process/create/PdfSourceViewer.vue index 8c8cf08..d567fb4 100644 --- a/frontend/src/views/data-process/create/PdfSourceViewer.vue +++ b/frontend/src/views/data-process/create/PdfSourceViewer.vue @@ -66,7 +66,13 @@ const locationText = computed(() => { }) function pageForItem(item: PreviewItem | null) { - if (!item || item.sourceStart == null || item.sourceEnd == null) return null + if (!item) return null + if (item.sourceStart == null || item.sourceEnd == null) { + const pageNumber = item.sourcePages?.[0] + return pageNumber == null + ? null + : pageRanges.value.find((page) => page.page_number === pageNumber) ?? null + } return pageRanges.value.find((page) => ( item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end )) ?? pageRanges.value.find((page) => ( @@ -76,7 +82,10 @@ function pageForItem(item: PreviewItem | null) { function selectedTextForPage(page: DataProcessPdfPageRange) { const item = props.selectedItem - if (!item || item.sourceStart == null || item.sourceEnd == null) return '' + if (!item) return '' + if (item.sourceStart == null || item.sourceEnd == null) { + return item.sourcePages?.includes(page.page_number) ? item.originalContent : '' + } const intersectionStart = Math.max(item.sourceStart, page.source_start) const intersectionEnd = Math.min(item.sourceEnd, page.source_end) if (intersectionEnd <= intersectionStart) return '' @@ -145,7 +154,10 @@ function applySelectedHighlight(page: DataProcessPdfPageRange | null) { for (const element of textLayer?.textDivs ?? []) { element.classList.remove('is-slice-highlighted') } - if (!item || item.sourceStart == null || item.sourceEnd == null) { + if (!item || ( + (item.sourceStart == null || item.sourceEnd == null) + && !item.sourcePages?.length + )) { highlightState.value = item ? 'manual' : 'idle' return } @@ -248,7 +260,10 @@ async function renderCurrentPage(force = false) { async function locateSelectedItem() { if (!documentRef.value) return const item = props.selectedItem - if (!item || item.sourceStart == null || item.sourceEnd == null) { + if (!item || ( + (item.sourceStart == null || item.sourceEnd == null) + && !item.sourcePages?.length + )) { applySelectedHighlight(null) return } diff --git a/frontend/src/views/data-process/create/PreviewCompareStep.vue b/frontend/src/views/data-process/create/PreviewCompareStep.vue index b3f69df..a543ee5 100644 --- a/frontend/src/views/data-process/create/PreviewCompareStep.vue +++ b/frontend/src/views/data-process/create/PreviewCompareStep.vue @@ -118,6 +118,11 @@ function itemNumber(item: PreviewItem) { } function lineRange(item: PreviewItem) { + if (item.sourcePages?.length) { + const first = item.sourcePages[0] + const last = item.sourcePages[item.sourcePages.length - 1] + return first === last ? `来源：第 ${first} 页` : `来源：第 ${first}–${last} 页` + } if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增，无源文件定位' return item.sourceStartLine === item.sourceEndLine ? `来源：第 ${item.sourceStartLine} 行` diff --git a/frontend/src/views/data-process/create/TaskSetupStep.vue b/frontend/src/views/data-process/create/TaskSetupStep.vue index 6770512..6cb2977 100644 --- a/frontend/src/views/data-process/create/TaskSetupStep.vue +++ b/frontend/src/views/data-process/create/TaskSetupStep.vue @@ -75,9 +75,6 @@ const chunkValidationMessage = computed(() => { ) { return '重叠长度与最小切片长度之和不能大于切片长度' } - if (props.unstructuredOptions.chunkMethod === 'custom' && !props.unstructuredOptions.customDelimiter.trim()) { - return '请输入自定义分隔符' - } return '' }) diff --git a/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue b/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue index 0b56558..4b21400 100644 --- a/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue +++ b/frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue @@ -30,15 +30,16 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [ ] const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [ - { value: 'structure', label: '文档结构' }, - { value: 'fixed', label: '固定 Token' }, - { value: 'custom', label: '自定义分隔符' }, + { value: 'layout_hybrid', label: '版面结构混合切分' }, + { value: 'semantic', label: '语义切分' }, + { value: 'fixed', label: '固定 Token 切分' }, ] const UNSTRUCTURED_NUMBER_LIMITS = { chunkSize: { min: 200, max: 2000 }, chunkOverlap: { min: 0, max: 500 }, minChunkSize: { min: 20, max: 500 }, + semanticBreakpointPercentile: { min: 1, max: 99 }, qaPairsPerChunk: { min: 1, max: 3 }, } as const @@ -160,7 +161,7 @@ defineExpose({ revealValidation })

切分选项

-

按文档结构、固定 Token 或自定义分隔符拆分长文档

+

优先保留文档版面结构，也可按主题变化或固定 Token 切分

@@ -179,7 +180,9 @@ defineExpose({ revealValidation }) :value="method.value" /> - 推荐使用文档结构，按标题、段落和句子边界保留完整内容块 + 默认推荐，按标题、段落、列表和表格结构切分，并合并过短内容块 + 根据相邻内容的语义变化寻找主题边界，适合长文章和知识材料 + 按句子边界控制固定 Token 长度，速度快且结果稳定 -