From 5c469a1778e86114fc7484e4cd25f895f033db4d Mon Sep 17 00:00:00 2001 From: caoxiaozhu Date: Mon, 13 Jul 2026 11:47:33 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=90=91=E5=AF=BC=E6=96=B0=E5=A2=9E=E7=94=9F=E6=88=90=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 提取 GenerationOptionsPanel 组件统一管理生成模型、温度、最大长度、JSON 模式与质量过滤配置，StructuredProcessOptions 与 UnstructuredProcessOptions 继承 GenerationControlOptions，CreateView、TaskSetupStep 及各子步骤同步接入，详情/列表小幅调整，回归脚本扩充生成选项断言。 --- .../regression-data-process-wizard.mjs | 81 +++- .../data-process/DataProcessCreateView.vue | 87 +++- .../data-process/DataProcessDetailView.vue | 11 +- .../data-process/DataProcessListView.vue | 25 +- .../create/GenerationOptionsPanel.vue | 439 ++++++++++++++++++ .../data-process/create/GenerationStep.vue | 2 +- .../create/PreviewCompareStep.vue | 2 +- .../data-process/create/ResultEditorStep.vue | 2 +- .../data-process/create/SourceUploadStep.vue | 4 +- .../data-process/create/TaskSetupStep.vue | 91 +++- .../views/data-process/create/previewModel.ts | 14 +- .../src/views/data-process/create/types.ts | 16 +- 12 files changed, 744 insertions(+), 30 deletions(-) create mode 100644 frontend/src/views/data-process/create/GenerationOptionsPanel.vue diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index 4ca2013..f5300c9 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -126,6 +126,8 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄 const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') const taskSetupSource = await readFile(taskSetupPath, 'utf8') +const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue') +const generationControlSource = await readFile(generationOptionsPath, 'utf8') const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue') const sourceUploadSource = await readFile(sourceUploadPath, 'utf8') @@ -135,7 +137,7 @@ for (const marker of ['大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类') +assert.match(taskSetupSource, /section="model"/, '独立大模型分类没有挂载模型配置') +assert.match(taskSetupSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') +assert.doesNotMatch(generationControlSource, /

大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类') +for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { + assert.ok(generationControlSource.includes(label), `生成控制界面缺少：${label}`) +} +assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索') +assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制') +assert.match(generationControlSource, /\.model-option-row\s*\{[\s\S]*?display:\s*flex[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有与上方生成选项使用一致的配置行样式') +assert.match(viewSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') +assert.equal((viewSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') +assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示') +assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示') +assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') +assert.match(taskSetupSource, /generationValidationMessage/, '生成模型与质量规则缺少继续前校验') +assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表') +assert.match(viewSource, /model\.type === 'LLM'/, '数据生成模型列表没有排除非大模型') +assert.match(viewSource, /:generation-models="generationModels"/, '创建页没有向生成选项传递模型列表') + assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型') for (const field of [ 'preprocessOptions', @@ -298,7 +334,17 @@ const generationOptionsStart = viewSource.indexOf('function generationAffectingO const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart) assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数') const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd) -for (const field of ['semanticEnrichment', 'qaPairsPerChunk', 'datasetSplit']) { +for (const field of [ + 'semanticEnrichment', + 'qaPairsPerChunk', + 'datasetSplit', + 'generationModelId', + 'generationPrompt', + 'qualityFilterEnabled', + 'filterLowQuality', + 'filterShortContent', + 'minOutputLength', +]) { assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段：${field}`) } assert.match( @@ -341,6 +387,12 @@ const baseUnstructuredOptions = { semanticEnrichment: false, qaPairsPerChunk: 3, datasetSplit: { train: 80, validation: 10, test: 10 }, + generationModelId: 1, + generationPrompt: '仅输出问答对', + qualityFilterEnabled: false, + filterLowQuality: true, + filterShortContent: true, + minOutputLength: 20, } for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) { @@ -434,6 +486,31 @@ assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片' const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions) assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片') +const shortContentItems = [{ + ...samplePreviewItems[0], + editedContent: '问：示例\n短回答', +}] +const filteredShortResults = previewModelModule.createResults(shortContentItems, { + ...baseUnstructuredOptions, + qualityFilterEnabled: true, + filterLowQuality: false, + filterShortContent: true, + minOutputLength: 20, +}) +assert.equal(filteredShortResults.length, 0, '开启过短内容过滤后仍保留低于最少字数的结果') + +const invalidContentItems = [{ + ...samplePreviewItems[0], + status: 'invalid', +}] +const filteredInvalidResults = previewModelModule.createResults(invalidContentItems, { + ...baseUnstructuredOptions, + qualityFilterEnabled: true, + filterLowQuality: true, + filterShortContent: false, +}) +assert.equal(filteredInvalidResults.length, 0, '开启低质量过滤后仍保留标记为无效的结果') + const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check') assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏') diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 7ee04a3..379b582 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -2,6 +2,7 @@ import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue' import { onBeforeRouteLeave, useRouter } from 'vue-router' import { ElMessage, type UploadFile } from 'element-plus' +import { storeToRefs } from 'pinia' import AppConfirmDialog from '@/components/AppConfirmDialog.vue' import TaskSetupStep from './create/TaskSetupStep.vue' import SourceUploadStep from './create/SourceUploadStep.vue' @@ -9,6 +10,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue' import GenerationStep from './create/GenerationStep.vue' import ResultEditorStep from './create/ResultEditorStep.vue' import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel' +import { useModelsStore } from '@/stores/models' import type { ExternalDataSource, GenerationState, @@ -21,11 +23,15 @@ import type { } from './create/types' const router = useRouter() +const modelsStore = useModelsStore() +const { list: modelList } = storeToRefs(modelsStore) +const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM')) const taskSetupRef = ref>() const confirmDialogRef = ref>() const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft' -const DRAFT_SCHEMA_VERSION = 4 +const DRAFT_SCHEMA_VERSION = 5 const PREVIEW_MODEL_VERSION = 'document-chunk-v2' +const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。' const WIZARD_STEPS = [ { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' }, @@ -45,6 +51,15 @@ const structuredOptions = ref({ semanticEnrichment: false, qaPairsPerRow: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, + generationModelId: '', + generationPrompt: DEFAULT_GENERATION_PROMPT, + temperature: 0.7, + maxTokens: 1024, + jsonMode: false, + qualityFilterEnabled: false, + filterLowQuality: true, + filterShortContent: true, + minOutputLength: 20, }) const unstructuredOptions = ref({ preprocessOptions: [ @@ -66,6 +81,15 @@ const unstructuredOptions = ref({ semanticEnrichment: false, qaPairsPerChunk: 1, datasetSplit: { train: 80, validation: 10, test: 10 }, + generationModelId: '', + generationPrompt: DEFAULT_GENERATION_PROMPT, + temperature: 0.7, + maxTokens: 1024, + jsonMode: false, + qualityFilterEnabled: false, + filterLowQuality: true, + filterShortContent: true, + minOutputLength: 20, }) interface UploadedDataFile { uid: number | string @@ -219,19 +243,45 @@ function previewAffectingOptions() { function generationAffectingOptions() { if (processType.value === 'structured') { - const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value - return { semanticEnrichment, qaPairsPerRow, datasetSplit } + const { + semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt, + temperature, maxTokens, jsonMode, + qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength, + } = structuredOptions.value + return { + semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt, + temperature, maxTokens, jsonMode, + qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength, + } } if (processType.value === 'unstructured') { const { semanticEnrichment, qaPairsPerChunk, datasetSplit, + generationModelId, + generationPrompt, + temperature, + maxTokens, + jsonMode, + qualityFilterEnabled, + filterLowQuality, + filterShortContent, + minOutputLength, } = unstructuredOptions.value return { semanticEnrichment, qaPairsPerChunk, datasetSplit, + generationModelId, + generationPrompt, + temperature, + maxTokens, + jsonMode, + qualityFilterEnabled, + filterLowQuality, + filterShortContent, + minOutputLength, } } return null @@ -337,6 +387,31 @@ function restoreDraft() { qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk) ? restoredUnstructuredOptions.qaPairsPerChunk : unstructuredOptions.value.qaPairsPerChunk, + generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId, + generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string' + ? restoredUnstructuredOptions.generationPrompt + : unstructuredOptions.value.generationPrompt, + temperature: Number.isFinite(restoredUnstructuredOptions.temperature) + ? restoredUnstructuredOptions.temperature + : unstructuredOptions.value.temperature, + maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens) + ? restoredUnstructuredOptions.maxTokens + : unstructuredOptions.value.maxTokens, + jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean' + ? restoredUnstructuredOptions.jsonMode + : unstructuredOptions.value.jsonMode, + qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean' + ? restoredUnstructuredOptions.qualityFilterEnabled + : unstructuredOptions.value.qualityFilterEnabled, + filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean' + ? restoredUnstructuredOptions.filterLowQuality + : unstructuredOptions.value.filterLowQuality, + filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean' + ? restoredUnstructuredOptions.filterShortContent + : unstructuredOptions.value.filterShortContent, + minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength) + ? restoredUnstructuredOptions.minOutputLength + : unstructuredOptions.value.minOutputLength, datasetSplit: { train: Number.isFinite(restoredDatasetSplit?.train) ? restoredDatasetSplit.train @@ -819,7 +894,10 @@ onBeforeRouteLeave(async () => { }) onBeforeUnmount(stopGenerationTimer) -onMounted(restoreDraft) +onMounted(() => { + restoreDraft() + modelsStore.load() +}) - + diff --git a/frontend/src/views/data-process/create/GenerationOptionsPanel.vue b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue new file mode 100644 index 0000000..466906f --- /dev/null +++ b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue @@ -0,0 +1,439 @@ + + + + + diff --git a/frontend/src/views/data-process/create/GenerationStep.vue b/frontend/src/views/data-process/create/GenerationStep.vue index c94966a..4bb7517 100644 --- a/frontend/src/views/data-process/create/GenerationStep.vue +++ b/frontend/src/views/data-process/create/GenerationStep.vue @@ -97,7 +97,7 @@ const emit = defineEmits<{ align-items: center; justify-content: space-between; padding: 15px 17px; - background: #fbfcfe; + background: #fff; border-bottom: 1px solid #e8ebf0; strong { diff --git a/frontend/src/views/data-process/create/PreviewCompareStep.vue b/frontend/src/views/data-process/create/PreviewCompareStep.vue index 1e318ed..7c9be74 100644 --- a/frontend/src/views/data-process/create/PreviewCompareStep.vue +++ b/frontend/src/views/data-process/create/PreviewCompareStep.vue @@ -350,7 +350,7 @@ function lineRange(item: PreviewItem) { min-height: 52px; padding: 0 15px; color: #313949; - background: #fbfcfe; + background: #fff; border-bottom: 1px solid #e8ebf0; font-size: 13px; diff --git a/frontend/src/views/data-process/create/ResultEditorStep.vue b/frontend/src/views/data-process/create/ResultEditorStep.vue index 77ddc86..6f37823 100644 --- a/frontend/src/views/data-process/create/ResultEditorStep.vue +++ b/frontend/src/views/data-process/create/ResultEditorStep.vue @@ -143,7 +143,7 @@ function selectRelative(offset: number) { min-height: 52px; padding: 0 15px; color: #344054; - background: #fbfcfe; + background: #fff; border-bottom: 1px solid #e8ebf0; font-size: 13px; diff --git a/frontend/src/views/data-process/create/SourceUploadStep.vue b/frontend/src/views/data-process/create/SourceUploadStep.vue index 74bbf99..d5a2283 100644 --- a/frontend/src/views/data-process/create/SourceUploadStep.vue +++ b/frontend/src/views/data-process/create/SourceUploadStep.vue @@ -397,7 +397,7 @@ function formatSize(size: number) { width: 100%; min-height: 154px; padding: 32px 20px; - background: #fbfcfe; + background: #fff; border-color: #dfe3ea; transition: border-color 0.18s ease, background-color 0.18s ease; @@ -435,7 +435,7 @@ function formatSize(size: number) { padding: 10px 14px; color: #5f6878; font-size: 12px; - background: #fbfcfe; + background: #fff; border-bottom: 1px solid #edf0f5; } diff --git a/frontend/src/views/data-process/create/TaskSetupStep.vue b/frontend/src/views/data-process/create/TaskSetupStep.vue index f03f6b2..ccd84db 100644 --- a/frontend/src/views/data-process/create/TaskSetupStep.vue +++ b/frontend/src/views/data-process/create/TaskSetupStep.vue @@ -4,12 +4,15 @@ import type { FormInstance, FormRules } from 'element-plus' import type { ChunkMethod, DatasetSplitOptions, + GenerationControlOptions, PreprocessOption, ProcessType, StructuredProcessOptions, UnstructuredPreprocessOption, UnstructuredProcessOptions, } from './types' +import GenerationOptionsPanel from './GenerationOptionsPanel.vue' +import type { ModelItem } from '@/types' const props = defineProps<{ name: string @@ -17,6 +20,7 @@ const props = defineProps<{ processType: ProcessType structuredOptions: StructuredProcessOptions unstructuredOptions: UnstructuredProcessOptions + generationModels: ModelItem[] }>() const emit = defineEmits<{ @@ -63,6 +67,10 @@ function updateStructuredField( emit('update:structuredOptions', { ...props.structuredOptions, [field]: value }) } +function updateStructuredGenerationOptions(value: GenerationControlOptions) { + emit('update:structuredOptions', { ...props.structuredOptions, ...value }) +} + function updatePreprocessOptions(value: Array) { const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value)) const preprocessOptions = value.filter( @@ -89,6 +97,10 @@ function updateUnstructuredField( emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value }) } +function updateUnstructuredGenerationOptions(value: GenerationControlOptions) { + emit('update:unstructuredOptions', { ...props.unstructuredOptions, ...value }) +} + function updateSmartPreprocess(value: string | number | boolean) { const enabled = Boolean(value) const remainingOptions = props.unstructuredOptions.preprocessOptions.filter( @@ -143,6 +155,7 @@ function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value: const formRef = ref() const advancedChunkSettingsOpen = ref(props.unstructuredOptions.chunkMethod === 'custom') +const validationAttempted = ref(false) const formModel = computed(() => ({ name: props.name, processType: props.processType, @@ -172,6 +185,25 @@ const preserveSpecialContentEnabled = computed(() => ( && props.unstructuredOptions.preserveLists )) +const activeGenerationOptions = computed(() => { + if (props.processType === 'structured') return props.structuredOptions + if (props.processType === 'unstructured') return props.unstructuredOptions + return null +}) + +const generationValidationMessage = computed(() => { + const options = activeGenerationOptions.value + if (!options) return '' + if (options.generationModelId === '') return '请选择数据生成模型' + if (options.qualityFilterEnabled && !options.filterLowQuality && !options.filterShortContent) { + return '开启质量筛选后，请至少选择一项筛选规则' + } + if (options.qualityFilterEnabled && options.filterShortContent && options.minOutputLength < 1) { + return '最少字数必须大于 0' + } + return '' +}) + const chunkValidationMessage = computed(() => { if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) { return '重叠长度必须小于切片长度' @@ -208,6 +240,7 @@ const rules: FormRules = { } async function validate() { + validationAttempted.value = true if (!formRef.value) return false try { await formRef.value.validate() @@ -219,6 +252,7 @@ async function validate() { return false } } + if (generationValidationMessage.value) return false return true } catch { return false @@ -344,6 +378,13 @@ defineExpose({ validate })
+
语义丰富表达 @@ -435,6 +476,24 @@ defineExpose({ validate })
+ +
+
+
+

大模型

+

选择数据生成模型，并设置模型输出内容的要求

+
+
+
+ +
+
@@ -843,7 +927,7 @@ defineExpose({ validate }) min-width: 0; padding: 14px; color: #344054; - background: #fbfcfe; + background: #fff; border: 1px solid #e2e5ec; border-radius: 8px; @@ -902,7 +986,7 @@ defineExpose({ validate }) padding: 10px 14px; color: #344054; text-align: left; - background: #fbfcfe; + background: #fff; border: 1px solid #e2e5ec; border-radius: 8px; cursor: pointer; @@ -946,7 +1030,7 @@ defineExpose({ validate }) gap: 12px; margin-top: 8px; padding: 12px; - background: #fbfcfe; + background: #fff; border: 1px solid #e2e5ec; border-radius: 8px; @@ -1154,3 +1238,4 @@ defineExpose({ validate }) } + GenerationControlOptions, diff --git a/frontend/src/views/data-process/create/previewModel.ts b/frontend/src/views/data-process/create/previewModel.ts index 7a3f074..dc265e0 100644 --- a/frontend/src/views/data-process/create/previewModel.ts +++ b/frontend/src/views/data-process/create/previewModel.ts @@ -501,7 +501,11 @@ export function createResults( ? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1)) : Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1)) - return items.flatMap((item, index) => { + const generatedResults = items.flatMap((item, index) => { + if (options?.qualityFilterEnabled && options.filterLowQuality) { + if (item.status === 'invalid' || !item.editedContent.trim()) return [] + } + const [firstLine = '', ...rest] = item.editedContent.split('\n') const output = rest.join('\n').trim() || item.editedContent.trim() const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}` @@ -525,4 +529,12 @@ export function createResults( } }) }) + + if (!options?.qualityFilterEnabled) return generatedResults + + return generatedResults.filter((result) => { + if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false + if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false + return true + }) } diff --git a/frontend/src/views/data-process/create/types.ts b/frontend/src/views/data-process/create/types.ts index 7a4ff51..0e6a3d0 100644 --- a/frontend/src/views/data-process/create/types.ts +++ b/frontend/src/views/data-process/create/types.ts @@ -16,7 +16,19 @@ export interface DatasetSplitOptions { test: number } -export interface StructuredProcessOptions { +export interface GenerationControlOptions { + generationModelId: string | number | '' + generationPrompt: string + temperature: number + maxTokens: number + jsonMode: boolean + qualityFilterEnabled: boolean + filterLowQuality: boolean + filterShortContent: boolean + minOutputLength: number +} + +export interface StructuredProcessOptions extends GenerationControlOptions { preprocessOptions: PreprocessOption[] semanticEnrichment: boolean qaPairsPerRow: number @@ -34,7 +46,7 @@ export type UnstructuredPreprocessOption = export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom' -export interface UnstructuredProcessOptions { +export interface UnstructuredProcessOptions extends GenerationControlOptions { preprocessOptions: UnstructuredPreprocessOption[] chunkMethod: ChunkMethod chunkSize: number