523 lines
14 KiB
Vue
523 lines
14 KiB
Vue
<script setup lang="ts">
|
||
import { computed, ref, watch } from 'vue'
|
||
import type {
|
||
ChunkMethod,
|
||
GenerationControlOptions,
|
||
UnstructuredPreprocessOption,
|
||
UnstructuredProcessOptions,
|
||
} from './types'
|
||
import { QA_PAIRS_GENERATION_LIMITS } from './types'
|
||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||
|
||
const props = defineProps<{
|
||
options: UnstructuredProcessOptions
|
||
validationAttempted: boolean
|
||
qualityValidationMessage: string
|
||
chunkValidationMessage: string
|
||
}>()
|
||
|
||
const emit = defineEmits<{
|
||
'update:options': [value: UnstructuredProcessOptions]
|
||
}>()
|
||
|
||
const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
|
||
'clean_invalid_content',
|
||
'detect_document_structure',
|
||
'merge_short_content',
|
||
'filter_low_quality',
|
||
'deduplicate_content',
|
||
'preserve_context',
|
||
]
|
||
|
||
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||
{ value: 'layout_hybrid', label: '版面结构混合切分' },
|
||
{ value: 'semantic', label: '语义切分' },
|
||
{ value: 'fixed', label: '固定 Token 切分' },
|
||
]
|
||
|
||
const UNSTRUCTURED_NUMBER_LIMITS = {
|
||
chunkSize: { min: 200, max: 2000 },
|
||
chunkOverlap: { min: 0, max: 500 },
|
||
minChunkSize: { min: 20, max: 500 },
|
||
semanticBreakpointPercentile: { min: 1, max: 99 },
|
||
qaPairsPerChunk: QA_PAIRS_GENERATION_LIMITS,
|
||
} as const
|
||
|
||
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
|
||
|
||
const smartPreprocessEnabled = computed(() => SMART_PREPROCESS_OPTIONS.every(
|
||
(option) => props.options.preprocessOptions.includes(option),
|
||
))
|
||
|
||
const desensitizeEnabled = computed(() => props.options.preprocessOptions.includes('desensitize'))
|
||
|
||
const preserveSpecialContentEnabled = computed(() => (
|
||
props.options.preserveTables
|
||
&& props.options.preserveCodeBlocks
|
||
&& props.options.preserveLists
|
||
))
|
||
|
||
function updateField<K extends keyof UnstructuredProcessOptions>(
|
||
field: K,
|
||
value: UnstructuredProcessOptions[K],
|
||
) {
|
||
emit('update:options', { ...props.options, [field]: value })
|
||
}
|
||
|
||
function updateGenerationOptions(value: GenerationControlOptions) {
|
||
emit('update:options', { ...props.options, ...value })
|
||
}
|
||
|
||
function updateSmartPreprocess(value: string | number | boolean) {
|
||
const enabled = Boolean(value)
|
||
const remainingOptions = Array.from(new Set(props.options.preprocessOptions.filter(
|
||
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
||
)))
|
||
updateField(
|
||
'preprocessOptions',
|
||
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
||
)
|
||
}
|
||
|
||
function updateDesensitize(value: string | number | boolean) {
|
||
const preprocessOptions: UnstructuredPreprocessOption[] = Array.from(new Set(
|
||
props.options.preprocessOptions.filter(
|
||
(option) => option !== 'desensitize',
|
||
),
|
||
))
|
||
if (Boolean(value)) preprocessOptions.push('desensitize')
|
||
updateField('preprocessOptions', preprocessOptions)
|
||
}
|
||
|
||
function updateSpecialContentProtection(value: string | number | boolean) {
|
||
const enabled = Boolean(value)
|
||
emit('update:options', {
|
||
...props.options,
|
||
preserveTables: enabled,
|
||
preserveCodeBlocks: enabled,
|
||
preserveLists: enabled,
|
||
})
|
||
}
|
||
|
||
function updateUnstructuredNumber(field: UnstructuredNumberField, value: number | undefined) {
|
||
const limits = UNSTRUCTURED_NUMBER_LIMITS[field]
|
||
const parsedValue = Number(value)
|
||
const nextValue = Number.isFinite(parsedValue) ? parsedValue : limits.min
|
||
updateField(field, Math.min(limits.max, Math.max(limits.min, nextValue)))
|
||
}
|
||
|
||
function revealValidation() {
|
||
// Advanced settings are now flattened, no need to open toggle
|
||
}
|
||
|
||
defineExpose({ revealValidation })
|
||
</script>
|
||
|
||
<template>
|
||
<div class="form-section unstructured-options-section">
|
||
<div class="section-title-row">
|
||
<div>
|
||
<h3>预处理选项</h3>
|
||
<p>默认不执行预处理，请按文档情况自行选择</p>
|
||
</div>
|
||
</div>
|
||
<div class="preprocess-option-grid">
|
||
<label class="preprocess-option" :class="{ 'is-checked': smartPreprocessEnabled }">
|
||
<el-checkbox
|
||
:model-value="smartPreprocessEnabled"
|
||
@update:model-value="updateSmartPreprocess"
|
||
/>
|
||
<span class="preprocess-option-copy">
|
||
<strong>智能预处理</strong>
|
||
<small>清理页眉页脚、页码、目录和无效内容，感知文档结构、合并短块、预过滤低质量内容、近重复去重，并通过重叠保护上下文</small>
|
||
</span>
|
||
</label>
|
||
|
||
<label class="preprocess-option" :class="{ 'is-checked': desensitizeEnabled }">
|
||
<el-checkbox
|
||
:model-value="desensitizeEnabled"
|
||
@update:model-value="updateDesensitize"
|
||
/>
|
||
<span class="preprocess-option-copy">
|
||
<strong>敏感信息脱敏</strong>
|
||
<small>识别并脱敏姓名、手机号、邮箱和身份证号</small>
|
||
</span>
|
||
</label>
|
||
|
||
<label class="preprocess-option" :class="{ 'is-checked': preserveSpecialContentEnabled }">
|
||
<el-checkbox
|
||
:model-value="preserveSpecialContentEnabled"
|
||
@update:model-value="updateSpecialContentProtection"
|
||
/>
|
||
<span class="preprocess-option-copy">
|
||
<strong>保护表格、代码和列表</strong>
|
||
<small>避免切分点破坏特殊内容块的完整性</small>
|
||
</span>
|
||
</label>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="form-section chunk-options-section">
|
||
<div class="section-title-row">
|
||
<div>
|
||
<h3>切分选项</h3>
|
||
<p>优先保留文档版面结构，也可按主题变化或固定 Token 切分</p>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="chunk-settings-grid is-basic">
|
||
<label class="config-field">
|
||
<span class="config-field-label">切分方式</span>
|
||
<el-select
|
||
:model-value="options.chunkMethod"
|
||
aria-label="切分方式"
|
||
@update:model-value="updateField('chunkMethod', $event)"
|
||
>
|
||
<el-option
|
||
v-for="method in CHUNK_METHODS"
|
||
:key="method.value"
|
||
:label="method.label"
|
||
:value="method.value"
|
||
/>
|
||
</el-select>
|
||
<small v-if="options.chunkMethod === 'layout_hybrid'">默认推荐，按标题、段落、列表和表格结构切分，并合并过短内容块</small>
|
||
<small v-else-if="options.chunkMethod === 'semantic'">根据相邻内容的语义变化寻找主题边界，适合长文章和知识材料</small>
|
||
<small v-else>按句子边界控制固定 Token 长度，速度快且结果稳定</small>
|
||
</label>
|
||
|
||
<label class="config-field">
|
||
<span class="config-field-label">切片长度</span>
|
||
<span class="unit-input">
|
||
<el-input-number
|
||
:model-value="options.chunkSize"
|
||
:min="200"
|
||
:max="2000"
|
||
:step="50"
|
||
:precision="0"
|
||
controls-position="right"
|
||
aria-label="切片长度"
|
||
@update:model-value="updateUnstructuredNumber('chunkSize', $event)"
|
||
/>
|
||
<span>Token</span>
|
||
</span>
|
||
<small>单个切片的目标上限，默认 800 Token</small>
|
||
</label>
|
||
|
||
<label class="config-field">
|
||
<span class="config-field-label">重叠长度</span>
|
||
<span class="unit-input">
|
||
<el-input-number
|
||
:model-value="options.chunkOverlap"
|
||
:min="0"
|
||
:max="500"
|
||
:step="10"
|
||
:precision="0"
|
||
controls-position="right"
|
||
aria-label="重叠长度"
|
||
@update:model-value="updateUnstructuredNumber('chunkOverlap', $event)"
|
||
/>
|
||
<span>Token</span>
|
||
</span>
|
||
<small>在相邻切片中最多重复保留的上下文，默认 100 Token</small>
|
||
</label>
|
||
<label class="config-field">
|
||
<span class="config-field-label">最小切片长度</span>
|
||
<span class="unit-input">
|
||
<el-input-number
|
||
:model-value="options.minChunkSize"
|
||
:min="20"
|
||
:max="500"
|
||
:step="10"
|
||
:precision="0"
|
||
controls-position="right"
|
||
aria-label="最小切片长度"
|
||
@update:model-value="updateUnstructuredNumber('minChunkSize', $event)"
|
||
/>
|
||
<span>Token</span>
|
||
</span>
|
||
<small>过短的尾部内容会尽量并入前一个切片</small>
|
||
</label>
|
||
|
||
<label v-if="options.chunkMethod === 'semantic'" class="config-field">
|
||
<span class="config-field-label">语义断点百分位</span>
|
||
<span class="unit-input">
|
||
<el-input-number
|
||
:model-value="options.semanticBreakpointPercentile"
|
||
:min="1"
|
||
:max="99"
|
||
:step="1"
|
||
:precision="0"
|
||
controls-position="right"
|
||
aria-label="语义断点百分位"
|
||
@update:model-value="updateUnstructuredNumber('semanticBreakpointPercentile', $event)"
|
||
/>
|
||
<span>%</span>
|
||
</span>
|
||
<small>百分位越低切片越多，推荐保持 95%</small>
|
||
</label>
|
||
</div>
|
||
|
||
<p class="chunk-estimation-note">
|
||
Token 数为轻量估算值，实际长度以训练使用的模型分词器为准。
|
||
</p>
|
||
|
||
<p v-if="chunkValidationMessage" class="option-validation-message" role="alert">
|
||
{{ chunkValidationMessage }}
|
||
</p>
|
||
</div>
|
||
|
||
<div class="form-section generation-options-section">
|
||
<div class="section-title-row">
|
||
<div>
|
||
<h3>生成选项</h3>
|
||
<p>只保留会直接影响输出的核心设置</p>
|
||
</div>
|
||
</div>
|
||
<div class="generation-option-list">
|
||
<GenerationOptionsPanel
|
||
:options="options"
|
||
section="quality"
|
||
:validation-message="validationAttempted ? qualityValidationMessage : ''"
|
||
@update:options="updateGenerationOptions"
|
||
/>
|
||
|
||
|
||
<div class="generation-option-row">
|
||
<div class="generation-option-copy">
|
||
<strong>每个切片生成数量</strong>
|
||
<small>支持 1～50 条；数量越大，处理耗时和 Token 消耗越高</small>
|
||
</div>
|
||
<el-input-number
|
||
:model-value="options.qaPairsPerChunk"
|
||
:min="QA_PAIRS_GENERATION_LIMITS.min"
|
||
:max="QA_PAIRS_GENERATION_LIMITS.max"
|
||
:step="1"
|
||
:precision="0"
|
||
controls-position="right"
|
||
aria-label="每个切片生成数量"
|
||
@update:model-value="updateUnstructuredNumber('qaPairsPerChunk', $event)"
|
||
/>
|
||
</div>
|
||
|
||
<DatasetSplitEditor
|
||
:model-value="options.datasetSplit"
|
||
aria-label-prefix="非结构化"
|
||
@update:model-value="updateField('datasetSplit', $event)"
|
||
/>
|
||
</div>
|
||
</div>
|
||
|
||
</template>
|
||
|
||
<style scoped lang="scss">
|
||
.form-section {
|
||
padding: 0 0 26px;
|
||
margin-bottom: 26px;
|
||
border-bottom: 1px solid #edf0f5;
|
||
|
||
&:last-child {
|
||
margin-bottom: 0;
|
||
border-bottom: 0;
|
||
}
|
||
|
||
h3 {
|
||
margin: 0 0 16px;
|
||
color: #2f3747;
|
||
font-size: 15px;
|
||
font-weight: 650;
|
||
}
|
||
}
|
||
|
||
.section-title-row {
|
||
display: flex;
|
||
align-items: flex-start;
|
||
justify-content: space-between;
|
||
gap: 16px;
|
||
|
||
h3 {
|
||
margin-bottom: 5px;
|
||
}
|
||
|
||
p {
|
||
margin: 0;
|
||
color: #8a93a3;
|
||
font-size: 12px;
|
||
}
|
||
}
|
||
|
||
.generation-option-list {
|
||
display: grid;
|
||
gap: 12px;
|
||
margin-top: 16px;
|
||
}
|
||
|
||
.preprocess-option-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||
gap: 12px;
|
||
margin-top: 16px;
|
||
}
|
||
|
||
.preprocess-option {
|
||
display: flex;
|
||
gap: 10px;
|
||
width: 100%;
|
||
min-height: 64px;
|
||
margin-right: 0;
|
||
padding: 12px 14px;
|
||
box-sizing: border-box;
|
||
align-items: flex-start;
|
||
border: 1px solid #e2e5ec;
|
||
border-radius: 8px;
|
||
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||
|
||
&:hover {
|
||
border-color: #b7b2f7;
|
||
}
|
||
|
||
&.is-checked {
|
||
background: #fafaff;
|
||
border-color: #8b82f4;
|
||
}
|
||
|
||
:deep(.el-checkbox__input) {
|
||
margin-top: 3px;
|
||
}
|
||
|
||
:deep(.el-checkbox__label) {
|
||
min-width: 0;
|
||
padding-left: 10px;
|
||
white-space: normal;
|
||
}
|
||
}
|
||
|
||
.preprocess-option-copy {
|
||
display: grid;
|
||
gap: 4px;
|
||
|
||
strong {
|
||
color: #344054;
|
||
font-size: 13px;
|
||
font-weight: 600;
|
||
}
|
||
|
||
small {
|
||
color: #8a93a3;
|
||
font-size: 12px;
|
||
line-height: 1.5;
|
||
}
|
||
}
|
||
|
||
.generation-option-row {
|
||
display: flex;
|
||
min-height: 64px;
|
||
align-items: center;
|
||
justify-content: space-between;
|
||
gap: 24px;
|
||
padding: 12px 14px;
|
||
box-sizing: border-box;
|
||
border: 1px solid #e2e5ec;
|
||
border-radius: 8px;
|
||
}
|
||
|
||
.generation-option-copy {
|
||
display: grid;
|
||
gap: 4px;
|
||
|
||
strong {
|
||
color: #344054;
|
||
font-size: 13px;
|
||
font-weight: 600;
|
||
}
|
||
|
||
small {
|
||
color: #8a93a3;
|
||
font-size: 12px;
|
||
line-height: 1.5;
|
||
}
|
||
}
|
||
|
||
.chunk-settings-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||
gap: 12px;
|
||
margin-top: 16px;
|
||
}
|
||
|
||
.config-field {
|
||
display: grid;
|
||
align-content: start;
|
||
gap: 8px;
|
||
min-width: 0;
|
||
padding: 14px;
|
||
color: #344054;
|
||
background: #fff;
|
||
border: 1px solid #e2e5ec;
|
||
border-radius: 8px;
|
||
|
||
> small {
|
||
color: #8a93a3;
|
||
font-size: 12px;
|
||
line-height: 1.5;
|
||
}
|
||
|
||
:deep(.el-select),
|
||
:deep(.el-input) {
|
||
width: 100%;
|
||
}
|
||
}
|
||
|
||
.config-field-label {
|
||
font-size: 13px;
|
||
font-weight: 600;
|
||
}
|
||
|
||
.unit-input {
|
||
display: flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
color: #7b8495;
|
||
font-size: 12px;
|
||
|
||
:deep(.el-input-number) {
|
||
width: 100%;
|
||
}
|
||
}
|
||
|
||
.option-validation-message {
|
||
display: block;
|
||
margin: 8px 0 0;
|
||
color: #dc2626;
|
||
font-size: 12px;
|
||
line-height: 1.5;
|
||
}
|
||
|
||
.chunk-estimation-note {
|
||
margin: 9px 0 0;
|
||
color: #8a93a3;
|
||
font-size: 12px;
|
||
line-height: 1.5;
|
||
}
|
||
|
||
@media (max-width: 900px) {
|
||
.chunk-settings-grid {
|
||
grid-template-columns: minmax(0, 1fr);
|
||
}
|
||
|
||
.generation-option-row {
|
||
align-items: flex-start;
|
||
flex-direction: column;
|
||
}
|
||
|
||
.compact-option-list .generation-option-row {
|
||
align-items: center;
|
||
flex-direction: row;
|
||
}
|
||
|
||
.preprocess-option-grid {
|
||
grid-template-columns: minmax(0, 1fr);
|
||
}
|
||
}
|
||
</style>
|