feat: 数据处理向导新增生成控制选项

提取 GenerationOptionsPanel 组件统一管理生成模型、温度、最大长度、JSON 模式与质量过滤配置,StructuredProcessOptions 与 UnstructuredProcessOptions 继承 GenerationControlOptions,CreateView、TaskSetupStep 及各子步骤同步接入,详情/列表小幅调整,回归脚本扩充生成选项断言。
This commit is contained in:
caoxiaozhu
2026-07-13 11:47:33 +08:00
parent b64d730616
commit 5c469a1778
12 changed files with 744 additions and 30 deletions

View File

@@ -2,6 +2,7 @@
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, type UploadFile } from 'element-plus'
import { storeToRefs } from 'pinia'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue'
import SourceUploadStep from './create/SourceUploadStep.vue'
@@ -9,6 +10,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import { useModelsStore } from '@/stores/models'
import type {
ExternalDataSource,
GenerationState,
@@ -21,11 +23,15 @@ import type {
} from './create/types'
const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const DRAFT_SCHEMA_VERSION = 4
const DRAFT_SCHEMA_VERSION = 5
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
@@ -45,6 +51,15 @@ const structuredOptions = ref<StructuredProcessOptions>({
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
})
const unstructuredOptions = ref<UnstructuredProcessOptions>({
preprocessOptions: [
@@ -66,6 +81,15 @@ const unstructuredOptions = ref<UnstructuredProcessOptions>({
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
})
interface UploadedDataFile {
uid: number | string
@@ -219,19 +243,45 @@ function previewAffectingOptions() {
function generationAffectingOptions() {
if (processType.value === 'structured') {
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value
return { semanticEnrichment, qaPairsPerRow, datasetSplit }
const {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
} = structuredOptions.value
return {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
}
}
if (processType.value === 'unstructured') {
const {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
} = unstructuredOptions.value
return {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
}
}
return null
@@ -337,6 +387,31 @@ function restoreDraft() {
qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk)
? restoredUnstructuredOptions.qaPairsPerChunk
: unstructuredOptions.value.qaPairsPerChunk,
generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId,
generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string'
? restoredUnstructuredOptions.generationPrompt
: unstructuredOptions.value.generationPrompt,
temperature: Number.isFinite(restoredUnstructuredOptions.temperature)
? restoredUnstructuredOptions.temperature
: unstructuredOptions.value.temperature,
maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens)
? restoredUnstructuredOptions.maxTokens
: unstructuredOptions.value.maxTokens,
jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean'
? restoredUnstructuredOptions.jsonMode
: unstructuredOptions.value.jsonMode,
qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean'
? restoredUnstructuredOptions.qualityFilterEnabled
: unstructuredOptions.value.qualityFilterEnabled,
filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean'
? restoredUnstructuredOptions.filterLowQuality
: unstructuredOptions.value.filterLowQuality,
filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean'
? restoredUnstructuredOptions.filterShortContent
: unstructuredOptions.value.filterShortContent,
minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength)
? restoredUnstructuredOptions.minOutputLength
: unstructuredOptions.value.minOutputLength,
datasetSplit: {
train: Number.isFinite(restoredDatasetSplit?.train)
? restoredDatasetSplit.train
@@ -819,7 +894,10 @@ onBeforeRouteLeave(async () => {
})
onBeforeUnmount(stopGenerationTimer)
onMounted(restoreDraft)
onMounted(() => {
restoreDraft()
modelsStore.load()
})
</script>
<template>
@@ -861,6 +939,7 @@ onMounted(restoreDraft)
v-model:process-type="processType"
v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions"
:generation-models="generationModels"
/>
<SourceUploadStep

View File

@@ -18,7 +18,6 @@ interface ProcessDetail {
status: string
description: string
processType: string
dataType: string
sourceDataset: string
outputDataset?: string
outputDatasetId?: number
@@ -75,8 +74,7 @@ const detailMap: Record<string, ProcessDetail> = {
name: '客服问答数据清洗',
status: 'completed',
description: '清洗客服对话中的无效记录、重复问答和格式异常内容,生成可用于模型训练的标准数据集。',
processType: '去重清洗',
dataType: '结构化数据',
processType: '结构化数据',
sourceDataset: '客服对话原始集',
outputDataset: '客服对话清洗集',
outputDatasetId: 7,
@@ -101,7 +99,7 @@ const detailMap: Record<string, ProcessDetail> = {
},
'492015': {
id: '492015', name: '指令微调数据构造', status: 'running',
description: '从通用语料中构造指令微调训练样本。', processType: '指令构造', dataType: '非结构化数据',
description: '从通用语料中构造指令微调训练样本。', processType: '非结构化数据',
sourceDataset: '通用语料库', outputDataset: 'SFT 指令集', outputDatasetId: 8, creator: '管理员',
createTime: '2026-07-09 09:10:00', startTime: '2026-07-09 09:10:21', duration: '处理中', progress: 68,
inputCount: 18500, outputCount: 8568, filteredCount: 425, duplicateCount: 192, errorCount: 8,
@@ -112,7 +110,7 @@ const detailMap: Record<string, ProcessDetail> = {
},
'731948': {
id: '731948', name: '敏感信息脱敏处理', status: 'pending', description: '识别并脱敏用户反馈数据中的敏感字段。',
processType: '脱敏处理', dataType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
processType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
outputDatasetId: 9, creator: '管理员', createTime: '2026-07-09 16:45:00', duration: '等待执行', progress: 0,
inputCount: 9340, outputCount: 0, filteredCount: 0, duplicateCount: 0, errorCount: 0,
config: [
@@ -122,7 +120,7 @@ const detailMap: Record<string, ProcessDetail> = {
},
'582012': {
id: '582012', name: '多轮对话拼接', status: 'failed', description: '将单轮问答按会话标识拼接为多轮对话数据。',
processType: '格式转换', dataType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
processType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
createTime: '2026-07-10 08:30:00', startTime: '2026-07-10 08:30:16', completeTime: '2026-07-10 08:31:04',
duration: '48 秒', progress: 37, inputCount: 7520, outputCount: 2780, filteredCount: 24, duplicateCount: 0, errorCount: 1,
config: [
@@ -186,7 +184,6 @@ function resetPage() {
<dl class="heading-meta">
<div><dt>任务 ID</dt><dd>{{ detail.id }}</dd></div>
<div><dt>处理类型</dt><dd>{{ detail.processType }}</dd></div>
<div><dt>数据类型</dt><dd>{{ detail.dataType }}</dd></div>
<div><dt>创建人</dt><dd>{{ detail.creator }}</dd></div>
</dl>
</div>

View File

@@ -5,17 +5,25 @@ import { ElMessage } from 'element-plus'
import DataTablePage from '@/components/DataTablePage.vue'
import ModelStatusTag from '@/components/ModelStatusTag.vue'
import type { ProcessType } from './create/types'
/** 数据处理任务类型 */
interface DataProcessTask {
id: number | string
name: string
status: string
process_type: string
process_type: ProcessType
source_dataset: string
output_dataset?: string
create_time?: string
}
const processTypeMap: Record<ProcessType, string> = {
structured: '结构化数据',
unstructured: '非结构化数据',
external: '外来数据源拉取',
}
// TODO: 接入真实接口前,先用本地 mock 数据
const router = useRouter()
const dataList = ref<DataProcessTask[]>([
@@ -23,7 +31,7 @@ const dataList = ref<DataProcessTask[]>([
id: 183921,
name: '客服问答数据清洗',
status: 'completed',
process_type: '去重清洗',
process_type: 'structured',
source_dataset: '客服对话原始集',
output_dataset: '客服对话清洗集',
create_time: '2026-07-08 14:23:00',
@@ -32,7 +40,7 @@ const dataList = ref<DataProcessTask[]>([
id: 492015,
name: '指令微调数据构造',
status: 'running',
process_type: '指令构造',
process_type: 'unstructured',
source_dataset: '通用语料库',
output_dataset: 'SFT 指令集',
create_time: '2026-07-09 09:10:00',
@@ -41,7 +49,7 @@ const dataList = ref<DataProcessTask[]>([
id: 731948,
name: '敏感信息脱敏处理',
status: 'pending',
process_type: '脱敏处理',
process_type: 'structured',
source_dataset: '用户反馈数据',
create_time: '2026-07-09 16:45:00',
},
@@ -49,7 +57,7 @@ const dataList = ref<DataProcessTask[]>([
id: 582012,
name: '多轮对话拼接',
status: 'failed',
process_type: '格式转换',
process_type: 'structured',
source_dataset: '单轮问答集',
create_time: '2026-07-10 08:30:00',
},
@@ -98,7 +106,12 @@ function formatDateTime(value?: string) {
</template>
</el-table-column>
<el-table-column label="处理类型" align="center" width="140">
<template #default="{ row }">{{ row.process_type || '-' }}</template>
<template #default="{ row }">
<el-tag v-if="row.process_type" size="small" type="info" effect="plain">
{{ processTypeMap[row.process_type as ProcessType] || row.process_type }}
</el-tag>
<span v-else>-</span>
</template>
</el-table-column>
<el-table-column label="源数据集" align="center" show-overflow-tooltip>
<template #default="{ row }">{{ row.source_dataset || '-' }}</template>

View File

@@ -0,0 +1,439 @@
<script setup lang="ts">
import { ref } from 'vue'
import type { ModelItem } from '@/types'
import type { GenerationControlOptions } from './types'
const props = defineProps<{
options: GenerationControlOptions
models: ModelItem[]
section: 'model' | 'quality'
validationMessage?: string
}>()
const advancedModelSettingsOpen = ref(false)
const emit = defineEmits<{
'update:options': [value: GenerationControlOptions]
}>()
function updateField<K extends keyof GenerationControlOptions>(
field: K,
value: GenerationControlOptions[K],
) {
emit('update:options', { ...props.options, [field]: value })
}
function updateQualityRules(value: Array<string | number>) {
const rules = value.filter((item): item is string => typeof item === 'string')
emit('update:options', {
...props.options,
filterLowQuality: rules.includes('low_quality'),
filterShortContent: rules.includes('short_content'),
})
}
const selectedQualityRules = () => [
props.options.filterLowQuality ? 'low_quality' : '',
props.options.filterShortContent ? 'short_content' : '',
].filter(Boolean)
function sectionValidationMessage() {
if (!props.validationMessage) return ''
const isModelMessage = props.validationMessage.includes('数据生成模型')
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
return isModelMessage ? '' : props.validationMessage
}
</script>
<template>
<div v-if="section === 'model'" class="generation-config-group model-config-group">
<div class="model-field">
<div class="field-copy">
<strong>数据生成模型 <em>*</em></strong>
<small>选择负责生成问答指令或结构化内容的大模型</small>
</div>
<el-select
class="model-select"
:model-value="options.generationModelId"
filterable
placeholder="请选择用于生成数据的模型"
aria-label="数据生成模型"
@update:model-value="updateField('generationModelId', $event)"
>
<el-option
v-for="model in models"
:key="model.id"
:label="model.name"
:value="model.id"
>
<div class="model-option">
<span>{{ model.name }}</span>
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
</div>
</el-option>
<template #empty>
<div class="model-empty">暂无可用的大模型请先在模型管理中添加</div>
</template>
</el-select>
</div>
<div class="model-field">
<div class="field-copy">
<strong>默认提示语</strong>
<small>用于约束生成内容的格式语气和完整性可按任务需要修改</small>
</div>
<el-input
class="prompt-input"
:model-value="options.generationPrompt"
type="textarea"
:rows="4"
maxlength="500"
show-word-limit
resize="vertical"
placeholder="请输入模型生成内容时需要遵循的要求"
aria-label="模型默认提示语"
@update:model-value="updateField('generationPrompt', $event)"
/>
<div class="prompt-variables-hint">
提示可在文本中通过 <code>{{ content }}</code> 引用当前正在处理的数据内容
</div>
</div>
<button
type="button"
class="advanced-settings-toggle"
:aria-expanded="advancedModelSettingsOpen"
@click="advancedModelSettingsOpen = !advancedModelSettingsOpen"
>
<span>
<strong>高级设置</strong>
<small>温度最大输出长度与格式约束</small>
</span>
<i class="fa" :class="advancedModelSettingsOpen ? 'fa-chevron-up' : 'fa-chevron-down'" />
</button>
<div v-if="advancedModelSettingsOpen" class="advanced-settings-panel">
<div class="advanced-settings-grid">
<label class="config-field">
<span class="config-field-label">生成温度 (Temperature)</span>
<el-slider
:model-value="options.temperature"
:min="0"
:max="1"
:step="0.1"
show-input
@update:model-value="updateField('temperature', Number($event) || 0)"
/>
<small>较低值输出更确定保守较高值输出更多样有创造性</small>
</label>
<label class="config-field">
<span class="config-field-label">最大输出长度 (Max Tokens)</span>
<span class="unit-input">
<el-input-number
:model-value="options.maxTokens"
:min="100"
:max="8192"
:step="256"
controls-position="right"
@update:model-value="updateField('maxTokens', Number($event) || 1024)"
/>
<span>Token</span>
</span>
<small>限制单次生成的最大内容长度防止过度发散</small>
</label>
</div>
<div class="json-mode-row">
<div class="field-copy">
<strong>强制 JSON 格式输出</strong>
<small>要求大模型返回纯 JSON 对象有助于提高下游结构化提取的稳定性</small>
</div>
<el-switch
:model-value="options.jsonMode"
inline-prompt
active-text=""
inactive-text=""
@update:model-value="updateField('jsonMode', Boolean($event))"
/>
</div>
</div>
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
{{ sectionValidationMessage() }}
</p>
</div>
<div v-else class="generation-config-group quality-config-group">
<div class="quality-switch-row">
<div class="config-group-heading">
<div>
<h4>质量筛选</h4>
<p>生成后自动过滤不符合要求的内容不影响源数据预处理</p>
</div>
</div>
<el-switch
:model-value="options.qualityFilterEnabled"
inline-prompt
active-text=""
inactive-text=""
aria-label="质量筛选"
@update:model-value="updateField('qualityFilterEnabled', Boolean($event))"
/>
</div>
<div v-if="options.qualityFilterEnabled" class="quality-rule-panel">
<el-checkbox-group
:model-value="selectedQualityRules()"
class="quality-rule-list"
@update:model-value="updateQualityRules"
>
<el-checkbox value="low_quality">
<span class="rule-copy">
<strong>过滤低质量内容</strong>
<small>按相关性完整性和可读性过滤明显不合格的生成结果</small>
</span>
</el-checkbox>
<el-checkbox value="short_content">
<span class="rule-copy">
<strong>过滤过短内容</strong>
<small>过滤正文长度低于指定字数的生成结果</small>
</span>
</el-checkbox>
</el-checkbox-group>
<label v-if="options.filterShortContent" class="minimum-length-field">
<span>最少字数</span>
<span class="minimum-length-input">
<el-input-number
:model-value="options.minOutputLength"
:min="1"
:max="1000"
:step="5"
:precision="0"
controls-position="right"
aria-label="生成内容最少字数"
@update:model-value="updateField('minOutputLength', Number($event) || 1)"
/>
<span></span>
</span>
<small>少于该字数的生成结果将被过滤</small>
</label>
<p v-if="!options.filterLowQuality && !options.filterShortContent" class="quality-rule-warning" role="alert">
请至少选择一项质量筛选规则
</p>
</div>
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
{{ sectionValidationMessage() }}
</p>
</div>
</template>
<style scoped lang="scss">
.generation-config-group {
padding: 14px;
border: 1px solid #e2e5ec;
border-radius: 8px;
background: #fff;
}
.config-group-heading {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
h4 { margin: 0; color: #344054; font-size: 13px; font-weight: 600; }
p { margin: 4px 0 0; color: #8a93a3; font-size: 12px; line-height: 1.5; }
}
.model-config-group {
display: flex;
flex-direction: column;
gap: 16px;
}
.model-field {
display: flex;
flex-direction: column;
gap: 8px;
}
.field-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
em { color: var(--el-color-danger); font-style: normal; }
}
.model-select { width: 100%; }
.prompt-input { width: 100%; }
.model-option { width: 100%; display: flex; align-items: center; justify-content: space-between; gap: 16px; }
.model-option small { color: #98a2b3; font-size: 11px; }
.model-empty { padding: 12px; color: #8a93a3; font-size: 12px; text-align: center; }
.prompt-variables-hint {
color: #8a93a3;
font-size: 12px;
code {
padding: 2px 4px;
color: #5b50f2;
background: #f0f0ff;
border-radius: 4px;
font-family: inherit;
}
}
.advanced-settings-toggle {
display: flex;
width: 100%;
min-height: 48px;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-top: 4px;
padding: 10px 14px;
color: #344054;
text-align: left;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
cursor: pointer;
transition: border-color 0.18s ease, background-color 0.18s ease;
&:hover {
background: #f8f7ff;
border-color: #b7b2f7;
}
> span {
display: grid;
gap: 3px;
}
strong {
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
i {
flex: 0 0 auto;
color: #7b8495;
font-size: 12px;
}
}
.advanced-settings-panel {
display: grid;
gap: 16px;
margin-top: -8px;
padding: 16px;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.advanced-settings-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
}
.config-field {
display: grid;
align-content: start;
gap: 8px;
min-width: 0;
padding: 14px;
color: #344054;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
> small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
:deep(.el-slider) {
margin: 0 8px;
}
}
.config-field-label {
font-size: 13px;
font-weight: 600;
}
.unit-input {
display: flex;
align-items: center;
gap: 8px;
color: #7b8495;
font-size: 12px;
:deep(.el-input-number) {
width: 100%;
}
}
.json-mode-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 20px;
padding: 14px;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.quality-switch-row { display: flex; align-items: center; justify-content: space-between; gap: 20px; }
.quality-rule-panel { margin-top: 12px; padding-top: 12px; border-top: 1px solid #e7eaf0; }
.quality-rule-list { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
.quality-rule-list :deep(.el-checkbox) {
min-height: 62px; height: auto; margin-right: 0; padding: 10px 12px;
align-items: flex-start; border: 1px solid #e2e5ec; border-radius: 7px; background: #fff;
}
.quality-rule-list :deep(.el-checkbox__input) { margin-top: 3px; }
.quality-rule-list :deep(.el-checkbox__label) { min-width: 0; padding-left: 9px; white-space: normal; }
.rule-copy { display: grid; gap: 3px; }
.rule-copy strong { color: #344054; font-size: 12px; font-weight: 600; }
.rule-copy small { color: #8a93a3; font-size: 11px; line-height: 1.5; }
.minimum-length-field {
margin-top: 12px; display: grid; grid-template-columns: 110px 220px minmax(0, 1fr);
align-items: center; gap: 12px; color: #475467; font-size: 12px;
}
.minimum-length-input { display: flex; align-items: center; gap: 8px; }
.minimum-length-input :deep(.el-input-number) { width: 180px; }
.minimum-length-field > small { color: #8a93a3; }
.quality-rule-warning { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
.generation-validation-message { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
@media (max-width: 900px) {
.advanced-settings-grid { grid-template-columns: 1fr; }
.quality-rule-list { grid-template-columns: 1fr; }
.minimum-length-field { grid-template-columns: 1fr; }
}
</style>

View File

@@ -97,7 +97,7 @@ const emit = defineEmits<{
align-items: center;
justify-content: space-between;
padding: 15px 17px;
background: #fbfcfe;
background: #fff;
border-bottom: 1px solid #e8ebf0;
strong {

View File

@@ -350,7 +350,7 @@ function lineRange(item: PreviewItem) {
min-height: 52px;
padding: 0 15px;
color: #313949;
background: #fbfcfe;
background: #fff;
border-bottom: 1px solid #e8ebf0;
font-size: 13px;

View File

@@ -143,7 +143,7 @@ function selectRelative(offset: number) {
min-height: 52px;
padding: 0 15px;
color: #344054;
background: #fbfcfe;
background: #fff;
border-bottom: 1px solid #e8ebf0;
font-size: 13px;

View File

@@ -397,7 +397,7 @@ function formatSize(size: number) {
width: 100%;
min-height: 154px;
padding: 32px 20px;
background: #fbfcfe;
background: #fff;
border-color: #dfe3ea;
transition: border-color 0.18s ease, background-color 0.18s ease;
@@ -435,7 +435,7 @@ function formatSize(size: number) {
padding: 10px 14px;
color: #5f6878;
font-size: 12px;
background: #fbfcfe;
background: #fff;
border-bottom: 1px solid #edf0f5;
}

View File

@@ -4,12 +4,15 @@ import type { FormInstance, FormRules } from 'element-plus'
import type {
ChunkMethod,
DatasetSplitOptions,
GenerationControlOptions,
PreprocessOption,
ProcessType,
StructuredProcessOptions,
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
} from './types'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
import type { ModelItem } from '@/types'
const props = defineProps<{
name: string
@@ -17,6 +20,7 @@ const props = defineProps<{
processType: ProcessType
structuredOptions: StructuredProcessOptions
unstructuredOptions: UnstructuredProcessOptions
generationModels: ModelItem[]
}>()
const emit = defineEmits<{
@@ -63,6 +67,10 @@ function updateStructuredField<K extends keyof StructuredProcessOptions>(
emit('update:structuredOptions', { ...props.structuredOptions, [field]: value })
}
function updateStructuredGenerationOptions(value: GenerationControlOptions) {
emit('update:structuredOptions', { ...props.structuredOptions, ...value })
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter(
@@ -89,6 +97,10 @@ function updateUnstructuredField<K extends keyof UnstructuredProcessOptions>(
emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value })
}
function updateUnstructuredGenerationOptions(value: GenerationControlOptions) {
emit('update:unstructuredOptions', { ...props.unstructuredOptions, ...value })
}
function updateSmartPreprocess(value: string | number | boolean) {
const enabled = Boolean(value)
const remainingOptions = props.unstructuredOptions.preprocessOptions.filter(
@@ -143,6 +155,7 @@ function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value:
const formRef = ref<FormInstance>()
const advancedChunkSettingsOpen = ref(props.unstructuredOptions.chunkMethod === 'custom')
const validationAttempted = ref(false)
const formModel = computed(() => ({
name: props.name,
processType: props.processType,
@@ -172,6 +185,25 @@ const preserveSpecialContentEnabled = computed(() => (
&& props.unstructuredOptions.preserveLists
))
const activeGenerationOptions = computed<GenerationControlOptions | null>(() => {
if (props.processType === 'structured') return props.structuredOptions
if (props.processType === 'unstructured') return props.unstructuredOptions
return null
})
const generationValidationMessage = computed(() => {
const options = activeGenerationOptions.value
if (!options) return ''
if (options.generationModelId === '') return '请选择数据生成模型'
if (options.qualityFilterEnabled && !options.filterLowQuality && !options.filterShortContent) {
return '开启质量筛选后,请至少选择一项筛选规则'
}
if (options.qualityFilterEnabled && options.filterShortContent && options.minOutputLength < 1) {
return '最少字数必须大于 0'
}
return ''
})
const chunkValidationMessage = computed(() => {
if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) {
return '重叠长度必须小于切片长度'
@@ -208,6 +240,7 @@ const rules: FormRules = {
}
async function validate() {
validationAttempted.value = true
if (!formRef.value) return false
try {
await formRef.value.validate()
@@ -219,6 +252,7 @@ async function validate() {
return false
}
}
if (generationValidationMessage.value) return false
return true
} catch {
return false
@@ -344,6 +378,13 @@ defineExpose({ validate })
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="structuredOptions"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateStructuredGenerationOptions"
/>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
@@ -435,6 +476,24 @@ defineExpose({ validate })
</div>
</div>
</div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="structuredOptions"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateStructuredGenerationOptions"
/>
</div>
</div>
</template>
<template v-if="processType === 'unstructured'">
@@ -625,6 +684,13 @@ defineExpose({ validate })
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="unstructuredOptions"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateUnstructuredGenerationOptions"
/>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
@@ -723,6 +789,24 @@ defineExpose({ validate })
</div>
</div>
</div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="unstructuredOptions"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateUnstructuredGenerationOptions"
/>
</div>
</div>
</template>
</el-form>
@@ -843,7 +927,7 @@ defineExpose({ validate })
min-width: 0;
padding: 14px;
color: #344054;
background: #fbfcfe;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
@@ -902,7 +986,7 @@ defineExpose({ validate })
padding: 10px 14px;
color: #344054;
text-align: left;
background: #fbfcfe;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
cursor: pointer;
@@ -946,7 +1030,7 @@ defineExpose({ validate })
gap: 12px;
margin-top: 8px;
padding: 12px;
background: #fbfcfe;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
@@ -1154,3 +1238,4 @@ defineExpose({ validate })
}
</style>
GenerationControlOptions,

View File

@@ -501,7 +501,11 @@ export function createResults(
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
return items.flatMap((item, index) => {
const generatedResults = items.flatMap((item, index) => {
if (options?.qualityFilterEnabled && options.filterLowQuality) {
if (item.status === 'invalid' || !item.editedContent.trim()) return []
}
const [firstLine = '', ...rest] = item.editedContent.split('\n')
const output = rest.join('\n').trim() || item.editedContent.trim()
const baseInstruction = firstLine.replace(/^问[:]\s*/, '').trim() || `数据条目 ${index + 1}`
@@ -525,4 +529,12 @@ export function createResults(
}
})
})
if (!options?.qualityFilterEnabled) return generatedResults
return generatedResults.filter((result) => {
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
return true
})
}

View File

@@ -16,7 +16,19 @@ export interface DatasetSplitOptions {
test: number
}
export interface StructuredProcessOptions {
export interface GenerationControlOptions {
generationModelId: string | number | ''
generationPrompt: string
temperature: number
maxTokens: number
jsonMode: boolean
qualityFilterEnabled: boolean
filterLowQuality: boolean
filterShortContent: boolean
minOutputLength: number
}
export interface StructuredProcessOptions extends GenerationControlOptions {
preprocessOptions: PreprocessOption[]
semanticEnrichment: boolean
qaPairsPerRow: number
@@ -34,7 +46,7 @@ export type UnstructuredPreprocessOption =
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
export interface UnstructuredProcessOptions {
export interface UnstructuredProcessOptions extends GenerationControlOptions {
preprocessOptions: UnstructuredPreprocessOption[]
chunkMethod: ChunkMethod
chunkSize: number