feat(frontend): 更新三种切分模式与PDF定位

frontend/scripts/regression-data-process-wizard.mjs

@@ -470,8 +470,8 @@ assert.match(generationControlSource, /modelMeta\(model\)/, '模型下拉缺少
 assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
 assert.match(
   typesSource,
-  /export type ChunkMethod = 'structure' \| 'fixed' \| 'custom'/,
-  '非结构化切分方式类型必须只保留 structure、fixed 和 custom',
+  /export type ChunkMethod = 'layout_hybrid' \| 'semantic' \| 'fixed'/,
+  '非结构化切分方式类型必须只保留 layout_hybrid、semantic 和 fixed',
 )
 for (const field of [
   'preprocessOptions',
@@ -479,7 +479,7 @@ for (const field of [
   'chunkSize',
   'chunkOverlap',
   'minChunkSize',
-  'customDelimiter',
+  'semanticBreakpointPercentile',
   'preserveTables',
   'preserveCodeBlocks',
   'preserveLists',
@@ -528,9 +528,9 @@ assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开
 
 assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
 const expectedChunkMethods = [
-  ['structure', '文档结构'],
-  ['fixed', '固定 Token'],
-  ['custom', '自定义分隔符'],
+  ['layout_hybrid', '版面结构混合切分'],
+  ['semantic', '语义切分'],
+  ['fixed', '固定 Token 切分'],
 ]
 const chunkMethodsStart = unstructuredOptionsSource.indexOf('const CHUNK_METHODS')
 const chunkMethodsEnd = unstructuredOptionsSource.indexOf('const UNSTRUCTURED_NUMBER_LIMITS', chunkMethodsStart)
@@ -540,10 +540,14 @@ assert.deepEqual(chunkMethodValues, expectedChunkMethods.map(([value]) => value)
 for (const [value, label] of expectedChunkMethods) {
   assert.ok(chunkMethodsSource.includes(`value: '${value}', label: '${label}'`), `切分方式缺少选项：${label}`)
 }
-for (const removedMethod of ['semantic', 'heading']) {
+for (const removedMethod of ['structure', 'custom', 'heading']) {
   assert.ok(!chunkMethodsSource.includes(`value: '${removedMethod}'`), `切分方式仍保留已移除值：${removedMethod}`)
 }
-assert.ok(unstructuredOptionsSource.includes('推荐使用文档结构'), '切分方式缺少文档结构默认推荐说明')
+assert.ok(unstructuredOptionsSource.includes('默认推荐，按标题、段落、列表和表格结构切分'), '版面结构混合切分缺少推荐说明')
+assert.ok(unstructuredOptionsSource.includes('根据相邻内容的语义变化寻找主题边界'), '语义切分缺少适用场景说明')
+assert.ok(unstructuredOptionsSource.includes('按句子边界控制固定 Token 长度'), '固定 Token 切分缺少稳定性说明')
+assert.doesNotMatch(unstructuredOptionsSource, /自定义分隔符/, '切分选项仍保留自定义分隔符')
+assert.doesNotMatch(taskSetupSource, /customDelimiter|自定义分隔符/, '任务配置仍校验已移除的自定义分隔符')
 for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) {
   assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`)
 }
@@ -554,6 +558,8 @@ assert.match(unstructuredOptionsSource, /function updateSpecialContentProtection
 assert.match(unstructuredOptionsSource, /options\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token')
 assert.match(unstructuredOptionsSource, /options\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token')
 assert.match(unstructuredOptionsSource, /options\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token')
+assert.match(unstructuredOptionsSource, /options\.semanticBreakpointPercentile[\s\S]*?:min="1"[\s\S]*?:max="99"/, '语义断点百分位必须限制在 1 到 99')
+assert.match(unstructuredOptionsSource, /v-if="options\.chunkMethod === 'semantic'"[\s\S]*?options\.semanticBreakpointPercentile/, '语义断点百分位必须仅在语义切分时显示')
 
 for (const label of ['每个切片生成数量', '数据集划分']) {
   assert.ok(taskSetupFeatureSource.includes(label), `非结构化生成选项缺少：${label}`)
@@ -568,10 +574,11 @@ assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), '
 assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局')
 
 assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
-assert.match(stateSource, /chunkMethod:\s*'structure'/, '非结构化默认切分方式必须为文档结构')
+assert.match(stateSource, /chunkMethod:\s*'layout_hybrid'/, '非结构化默认切分方式必须为版面结构混合切分')
 assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
 assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
 assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
+assert.match(stateSource, /semanticBreakpointPercentile:\s*95/, '默认语义断点百分位必须为 95')
 assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
 assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
 assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
@@ -627,7 +634,7 @@ for (const [backendField, frontendField] of [
   ['chunk_size', 'chunkSize'],
   ['chunk_overlap', 'chunkOverlap'],
   ['min_chunk_size', 'minChunkSize'],
-  ['custom_delimiter', 'customDelimiter'],
+  ['semantic_breakpoint_percentile', 'semanticBreakpointPercentile'],
   ['preserve_tables', 'preserveTables'],
   ['preserve_code_blocks', 'preserveCodeBlocks'],
   ['preserve_lists', 'preserveLists'],
@@ -659,7 +666,7 @@ for (const field of [
   'chunkSize',
   'chunkOverlap',
   'minChunkSize',
-  'customDelimiter',
+  'semanticBreakpointPercentile',
   'preserveTables',
   'preserveCodeBlocks',
   'preserveLists',

frontend/src/types/dataProcess.ts

@@ -142,6 +142,7 @@ export interface DataProcessPreviewItem {
   source_end_line: number | null
   token_count: number
   status: 'original' | 'modified' | 'manual' | 'invalid'
+  quality_score?: DataProcessQualityScore | null
   updated_at?: string
 }
 

frontend/src/views/data-process/DataProcessCreateView.vue

@@ -218,7 +218,7 @@ function toBackendConfig(): DataProcessConfig {
       chunk_size: unstructuredOptions.value.chunkSize,
       chunk_overlap: unstructuredOptions.value.chunkOverlap,
       min_chunk_size: unstructuredOptions.value.minChunkSize,
-      custom_delimiter: unstructuredOptions.value.customDelimiter,
+      semantic_breakpoint_percentile: unstructuredOptions.value.semanticBreakpointPercentile,
       preserve_tables: unstructuredOptions.value.preserveTables,
       preserve_code_blocks: unstructuredOptions.value.preserveCodeBlocks,
       preserve_lists: unstructuredOptions.value.preserveLists,
@@ -266,6 +266,9 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
     sourceEndLine: item.source_end_line,
     tokenCount: item.token_count,
     status: item.status,
+    sourcePages: Array.isArray(item.quality_score?.source_pages)
+      ? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
+      : [],
     updatedAt: item.updated_at,
   }
 }
@@ -283,7 +286,7 @@ function previewAffectingOptions() {
       chunkSize,
       chunkOverlap,
       minChunkSize,
-      customDelimiter,
+      semanticBreakpointPercentile,
       preserveTables,
       preserveCodeBlocks,
       preserveLists,
@@ -294,7 +297,7 @@ function previewAffectingOptions() {
       chunkSize,
       chunkOverlap,
       minChunkSize,
-      customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
+      semanticBreakpointPercentile,
       preserveTables,
       preserveCodeBlocks,
       preserveLists,

frontend/src/views/data-process/DataProcessDetailView.vue

@@ -79,16 +79,16 @@ const configLabelMap: Record<string, string> = {
   chunk_size: '切片长度',
   chunk_overlap: '重叠长度',
   min_chunk_size: '最小切片长度',
-  custom_delimiter: '自定义分隔符',
+  semantic_breakpoint_percentile: '语义断点百分位',
   preserve_tables: '保留表格',
   preserve_code_blocks: '保留代码块',
   preserve_lists: '保留列表',
 }
 
 const chunkMethodLabelMap: Record<string, string> = {
-  structure: '文档结构',
+  layout_hybrid: '版面结构混合切分',
+  semantic: '语义切分',
   fixed: '固定 Token',
-  custom: '自定义分隔符',
 }
 
 const preprocessOptionLabelMap: Record<string, string> = {

frontend/src/views/data-process/create/PdfSourceViewer.vue

@@ -66,7 +66,13 @@ const locationText = computed(() => {
 })
 
 function pageForItem(item: PreviewItem | null) {
-  if (!item || item.sourceStart == null || item.sourceEnd == null) return null
+  if (!item) return null
+  if (item.sourceStart == null || item.sourceEnd == null) {
+    const pageNumber = item.sourcePages?.[0]
+    return pageNumber == null
+      ? null
+      : pageRanges.value.find((page) => page.page_number === pageNumber) ?? null
+  }
   return pageRanges.value.find((page) => (
     item.sourceStart! >= page.source_start && item.sourceStart! < page.source_end
   )) ?? pageRanges.value.find((page) => (
@@ -76,7 +82,10 @@ function pageForItem(item: PreviewItem | null) {
 
 function selectedTextForPage(page: DataProcessPdfPageRange) {
   const item = props.selectedItem
-  if (!item || item.sourceStart == null || item.sourceEnd == null) return ''
+  if (!item) return ''
+  if (item.sourceStart == null || item.sourceEnd == null) {
+    return item.sourcePages?.includes(page.page_number) ? item.originalContent : ''
+  }
   const intersectionStart = Math.max(item.sourceStart, page.source_start)
   const intersectionEnd = Math.min(item.sourceEnd, page.source_end)
   if (intersectionEnd <= intersectionStart) return ''
@@ -145,7 +154,10 @@ function applySelectedHighlight(page: DataProcessPdfPageRange | null) {
   for (const element of textLayer?.textDivs ?? []) {
     element.classList.remove('is-slice-highlighted')
   }
-  if (!item || item.sourceStart == null || item.sourceEnd == null) {
+  if (!item || (
+    (item.sourceStart == null || item.sourceEnd == null)
+    && !item.sourcePages?.length
+  )) {
     highlightState.value = item ? 'manual' : 'idle'
     return
   }
@@ -248,7 +260,10 @@ async function renderCurrentPage(force = false) {
 async function locateSelectedItem() {
   if (!documentRef.value) return
   const item = props.selectedItem
-  if (!item || item.sourceStart == null || item.sourceEnd == null) {
+  if (!item || (
+    (item.sourceStart == null || item.sourceEnd == null)
+    && !item.sourcePages?.length
+  )) {
     applySelectedHighlight(null)
     return
   }

frontend/src/views/data-process/create/PreviewCompareStep.vue

@@ -118,6 +118,11 @@ function itemNumber(item: PreviewItem) {
 }
 
 function lineRange(item: PreviewItem) {
+  if (item.sourcePages?.length) {
+    const first = item.sourcePages[0]
+    const last = item.sourcePages[item.sourcePages.length - 1]
+    return first === last ? `来源：第 ${first} 页` : `来源：第 ${first}–${last} 页`
+  }
   if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增，无源文件定位'
   return item.sourceStartLine === item.sourceEndLine
     ? `来源：第 ${item.sourceStartLine} 行`

frontend/src/views/data-process/create/TaskSetupStep.vue

@@ -75,9 +75,6 @@ const chunkValidationMessage = computed(() => {
   ) {
     return '重叠长度与最小切片长度之和不能大于切片长度'
   }
-  if (props.unstructuredOptions.chunkMethod === 'custom' && !props.unstructuredOptions.customDelimiter.trim()) {
-    return '请输入自定义分隔符'
-  }
   return ''
 })
 

frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue

@@ -30,15 +30,16 @@ const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
 ]
 
 const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
-  { value: 'structure', label: '文档结构' },
-  { value: 'fixed', label: '固定 Token' },
-  { value: 'custom', label: '自定义分隔符' },
+  { value: 'layout_hybrid', label: '版面结构混合切分' },
+  { value: 'semantic', label: '语义切分' },
+  { value: 'fixed', label: '固定 Token 切分' },
 ]
 
 const UNSTRUCTURED_NUMBER_LIMITS = {
   chunkSize: { min: 200, max: 2000 },
   chunkOverlap: { min: 0, max: 500 },
   minChunkSize: { min: 20, max: 500 },
+  semanticBreakpointPercentile: { min: 1, max: 99 },
   qaPairsPerChunk: { min: 1, max: 3 },
 } as const
 
@@ -160,7 +161,7 @@ defineExpose({ revealValidation })
     <div class="section-title-row">
       <div>
         <h3>切分选项</h3>
-        <p>按文档结构、固定 Token 或自定义分隔符拆分长文档</p>
+        <p>优先保留文档版面结构，也可按主题变化或固定 Token 切分</p>
       </div>
     </div>
 
@@ -179,7 +180,9 @@ defineExpose({ revealValidation })
             :value="method.value"
           />
         </el-select>
-        <small>推荐使用文档结构，按标题、段落和句子边界保留完整内容块</small>
+        <small v-if="options.chunkMethod === 'layout_hybrid'">默认推荐，按标题、段落、列表和表格结构切分，并合并过短内容块</small>
+        <small v-else-if="options.chunkMethod === 'semantic'">根据相邻内容的语义变化寻找主题边界，适合长文章和知识材料</small>
+        <small v-else>按句子边界控制固定 Token 长度，速度快且结果稳定</small>
       </label>
 
       <label class="config-field">
@@ -235,17 +238,22 @@ defineExpose({ revealValidation })
           <small>过短的尾部内容会尽量并入前一个切片</small>
         </label>
 
-        <label v-if="options.chunkMethod === 'custom'" class="config-field">
-          <span class="config-field-label">自定义分隔符</span>
-          <el-input
-            :model-value="options.customDelimiter"
-            maxlength="40"
-            show-word-limit
-            placeholder="例如：--- 或 ###"
-            aria-label="自定义分隔符"
-            @update:model-value="updateField('customDelimiter', $event)"
-          />
-          <small>系统会优先在分隔符位置结束当前切片</small>
+        <label v-if="options.chunkMethod === 'semantic'" class="config-field">
+          <span class="config-field-label">语义断点百分位</span>
+          <span class="unit-input">
+            <el-input-number
+              :model-value="options.semanticBreakpointPercentile"
+              :min="1"
+              :max="99"
+              :step="1"
+              :precision="0"
+              controls-position="right"
+              aria-label="语义断点百分位"
+              @update:model-value="updateUnstructuredNumber('semanticBreakpointPercentile', $event)"
+            />
+            <span>%</span>
+          </span>
+          <small>百分位越低切片越多，推荐保持 95%</small>
         </label>
     </div>
 

frontend/src/views/data-process/create/dataProcessCreateState.ts

@@ -30,11 +30,11 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
       'deduplicate_content',
       'preserve_context',
     ],
-    chunkMethod: 'structure',
+    chunkMethod: 'layout_hybrid',
     chunkSize: 800,
     chunkOverlap: 100,
     minChunkSize: 100,
-    customDelimiter: '',
+    semanticBreakpointPercentile: 95,
     preserveTables: true,
     preserveCodeBlocks: true,
     preserveLists: true,

frontend/src/views/data-process/create/types.ts

@@ -46,7 +46,7 @@ export type UnstructuredPreprocessOption =
   | 'desensitize'
   | 'preserve_context'
 
-export type ChunkMethod = 'structure' | 'fixed' | 'custom'
+export type ChunkMethod = 'layout_hybrid' | 'semantic' | 'fixed'
 
 export interface UnstructuredProcessOptions extends GenerationControlOptions {
   preprocessOptions: UnstructuredPreprocessOption[]
@@ -54,7 +54,7 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
   chunkSize: number
   chunkOverlap: number
   minChunkSize: number
-  customDelimiter: string
+  semanticBreakpointPercentile: number
   preserveTables: boolean
   preserveCodeBlocks: boolean
   preserveLists: boolean
@@ -110,6 +110,7 @@ export interface PreviewItem {
   sourceEnd: number | null
   sourceStartLine: number | null
   sourceEndLine: number | null
+  sourcePages?: number[]
   tokenCount: number
   status: 'original' | 'modified' | 'manual' | 'invalid'
   qualityScore?: number