feat: 数据处理向导新增生成控制选项

提取 GenerationOptionsPanel 组件统一管理生成模型、温度、最大长度、JSON 模式与质量过滤配置,StructuredProcessOptions 与 UnstructuredProcessOptions 继承 GenerationControlOptions,CreateView、TaskSetupStep 及各子步骤同步接入,详情/列表小幅调整,回归脚本扩充生成选项断言。
This commit is contained in:
caoxiaozhu
2026-07-13 11:47:33 +08:00
parent b64d730616
commit 5c469a1778
12 changed files with 744 additions and 30 deletions

View File

@@ -126,6 +126,8 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const taskSetupSource = await readFile(taskSetupPath, 'utf8') const taskSetupSource = await readFile(taskSetupPath, 'utf8')
const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue')
const generationControlSource = await readFile(generationOptionsPath, 'utf8')
const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue') const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue')
const sourceUploadSource = await readFile(sourceUploadPath, 'utf8') const sourceUploadSource = await readFile(sourceUploadPath, 'utf8')
@@ -135,7 +137,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第二步没有挂载独立上传组件') assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第二步没有挂载独立上传组件')
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:上传文件'/, '第一步主按钮没有指向上传文件') assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:上传文件'/, '第一步主按钮没有指向上传文件')
assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续:数据预览'/, '第二步主按钮没有指向数据预览') assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续:数据预览'/, '第二步主按钮没有指向数据预览')
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = 4/, '五步索引变更后必须升级草稿版本') assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = 5/, '生成配置扩展后必须升级草稿版本')
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()') const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromCreateStart) const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromCreateStart)
@@ -195,6 +197,40 @@ assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.st
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置') assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑') assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
for (const field of [
'generationModelId',
'generationPrompt',
'qualityFilterEnabled',
'filterLowQuality',
'filterShortContent',
'minOutputLength',
]) {
assert.ok(typesSource.includes(field), `生成控制配置缺少字段:${field}`)
assert.ok(viewSource.includes(field), `父页面默认值或草稿状态缺少字段:${field}`)
}
assert.match(taskSetupSource, /GenerationOptionsPanel/, '生成选项没有复用统一的大模型与质量筛选组件')
assert.match(taskSetupSource, /:options="structuredOptions"/, '结构化生成选项未接入统一配置组件')
assert.match(taskSetupSource, /:options="unstructuredOptions"/, '非结构化生成选项未接入统一配置组件')
assert.match(taskSetupSource, /<h3>大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类')
assert.match(taskSetupSource, /section="model"/, '独立大模型分类没有挂载模型配置')
assert.match(taskSetupSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`)
}
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
assert.match(generationControlSource, /\.model-option-row\s*\{[\s\S]*?display:\s*flex[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有与上方生成选项使用一致的配置行样式')
assert.match(viewSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语')
assert.equal((viewSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语')
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
assert.match(taskSetupSource, /generationValidationMessage/, '生成模型与质量规则缺少继续前校验')
assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表')
assert.match(viewSource, /model\.type === 'LLM'/, '数据生成模型列表没有排除非大模型')
assert.match(viewSource, /:generation-models="generationModels"/, '创建页没有向生成选项传递模型列表')
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型') assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
for (const field of [ for (const field of [
'preprocessOptions', 'preprocessOptions',
@@ -298,7 +334,17 @@ const generationOptionsStart = viewSource.indexOf('function generationAffectingO
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart) const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart)
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数') assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd) const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd)
for (const field of ['semanticEnrichment', 'qaPairsPerChunk', 'datasetSplit']) { for (const field of [
'semanticEnrichment',
'qaPairsPerChunk',
'datasetSplit',
'generationModelId',
'generationPrompt',
'qualityFilterEnabled',
'filterLowQuality',
'filterShortContent',
'minOutputLength',
]) {
assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`) assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`)
} }
assert.match( assert.match(
@@ -341,6 +387,12 @@ const baseUnstructuredOptions = {
semanticEnrichment: false, semanticEnrichment: false,
qaPairsPerChunk: 3, qaPairsPerChunk: 3,
datasetSplit: { train: 80, validation: 10, test: 10 }, datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: 1,
generationPrompt: '仅输出问答对',
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
} }
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) { for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
@@ -434,6 +486,31 @@ assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片'
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions) const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片') assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
const shortContentItems = [{
...samplePreviewItems[0],
editedContent: '问:示例\n短回答',
}]
const filteredShortResults = previewModelModule.createResults(shortContentItems, {
...baseUnstructuredOptions,
qualityFilterEnabled: true,
filterLowQuality: false,
filterShortContent: true,
minOutputLength: 20,
})
assert.equal(filteredShortResults.length, 0, '开启过短内容过滤后仍保留低于最少字数的结果')
const invalidContentItems = [{
...samplePreviewItems[0],
status: 'invalid',
}]
const filteredInvalidResults = previewModelModule.createResults(invalidContentItems, {
...baseUnstructuredOptions,
qualityFilterEnabled: true,
filterLowQuality: true,
filterShortContent: false,
})
assert.equal(filteredInvalidResults.length, 0, '开启低质量过滤后仍保留标记为无效的结果')
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check') const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏') assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')

View File

@@ -2,6 +2,7 @@
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue' import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router' import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, type UploadFile } from 'element-plus' import { ElMessage, type UploadFile } from 'element-plus'
import { storeToRefs } from 'pinia'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue' import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue' import TaskSetupStep from './create/TaskSetupStep.vue'
import SourceUploadStep from './create/SourceUploadStep.vue' import SourceUploadStep from './create/SourceUploadStep.vue'
@@ -9,6 +10,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue' import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue' import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel' import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import { useModelsStore } from '@/stores/models'
import type { import type {
ExternalDataSource, ExternalDataSource,
GenerationState, GenerationState,
@@ -21,11 +23,15 @@ import type {
} from './create/types' } from './create/types'
const router = useRouter() const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>() const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>() const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft' const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const DRAFT_SCHEMA_VERSION = 4 const DRAFT_SCHEMA_VERSION = 5
const PREVIEW_MODEL_VERSION = 'document-chunk-v2' const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
const WIZARD_STEPS = [ const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' }, { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
@@ -45,6 +51,15 @@ const structuredOptions = ref<StructuredProcessOptions>({
semanticEnrichment: false, semanticEnrichment: false,
qaPairsPerRow: 1, qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 }, datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
}) })
const unstructuredOptions = ref<UnstructuredProcessOptions>({ const unstructuredOptions = ref<UnstructuredProcessOptions>({
preprocessOptions: [ preprocessOptions: [
@@ -66,6 +81,15 @@ const unstructuredOptions = ref<UnstructuredProcessOptions>({
semanticEnrichment: false, semanticEnrichment: false,
qaPairsPerChunk: 1, qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 }, datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
}) })
interface UploadedDataFile { interface UploadedDataFile {
uid: number | string uid: number | string
@@ -219,19 +243,45 @@ function previewAffectingOptions() {
function generationAffectingOptions() { function generationAffectingOptions() {
if (processType.value === 'structured') { if (processType.value === 'structured') {
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value const {
return { semanticEnrichment, qaPairsPerRow, datasetSplit } semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
} = structuredOptions.value
return {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
}
} }
if (processType.value === 'unstructured') { if (processType.value === 'unstructured') {
const { const {
semanticEnrichment, semanticEnrichment,
qaPairsPerChunk, qaPairsPerChunk,
datasetSplit, datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
} = unstructuredOptions.value } = unstructuredOptions.value
return { return {
semanticEnrichment, semanticEnrichment,
qaPairsPerChunk, qaPairsPerChunk,
datasetSplit, datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
} }
} }
return null return null
@@ -337,6 +387,31 @@ function restoreDraft() {
qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk) qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk)
? restoredUnstructuredOptions.qaPairsPerChunk ? restoredUnstructuredOptions.qaPairsPerChunk
: unstructuredOptions.value.qaPairsPerChunk, : unstructuredOptions.value.qaPairsPerChunk,
generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId,
generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string'
? restoredUnstructuredOptions.generationPrompt
: unstructuredOptions.value.generationPrompt,
temperature: Number.isFinite(restoredUnstructuredOptions.temperature)
? restoredUnstructuredOptions.temperature
: unstructuredOptions.value.temperature,
maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens)
? restoredUnstructuredOptions.maxTokens
: unstructuredOptions.value.maxTokens,
jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean'
? restoredUnstructuredOptions.jsonMode
: unstructuredOptions.value.jsonMode,
qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean'
? restoredUnstructuredOptions.qualityFilterEnabled
: unstructuredOptions.value.qualityFilterEnabled,
filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean'
? restoredUnstructuredOptions.filterLowQuality
: unstructuredOptions.value.filterLowQuality,
filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean'
? restoredUnstructuredOptions.filterShortContent
: unstructuredOptions.value.filterShortContent,
minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength)
? restoredUnstructuredOptions.minOutputLength
: unstructuredOptions.value.minOutputLength,
datasetSplit: { datasetSplit: {
train: Number.isFinite(restoredDatasetSplit?.train) train: Number.isFinite(restoredDatasetSplit?.train)
? restoredDatasetSplit.train ? restoredDatasetSplit.train
@@ -819,7 +894,10 @@ onBeforeRouteLeave(async () => {
}) })
onBeforeUnmount(stopGenerationTimer) onBeforeUnmount(stopGenerationTimer)
onMounted(restoreDraft) onMounted(() => {
restoreDraft()
modelsStore.load()
})
</script> </script>
<template> <template>
@@ -861,6 +939,7 @@ onMounted(restoreDraft)
v-model:process-type="processType" v-model:process-type="processType"
v-model:structured-options="structuredOptions" v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions" v-model:unstructured-options="unstructuredOptions"
:generation-models="generationModels"
/> />
<SourceUploadStep <SourceUploadStep

View File

@@ -18,7 +18,6 @@ interface ProcessDetail {
status: string status: string
description: string description: string
processType: string processType: string
dataType: string
sourceDataset: string sourceDataset: string
outputDataset?: string outputDataset?: string
outputDatasetId?: number outputDatasetId?: number
@@ -75,8 +74,7 @@ const detailMap: Record<string, ProcessDetail> = {
name: '客服问答数据清洗', name: '客服问答数据清洗',
status: 'completed', status: 'completed',
description: '清洗客服对话中的无效记录、重复问答和格式异常内容,生成可用于模型训练的标准数据集。', description: '清洗客服对话中的无效记录、重复问答和格式异常内容,生成可用于模型训练的标准数据集。',
processType: '去重清洗', processType: '结构化数据',
dataType: '结构化数据',
sourceDataset: '客服对话原始集', sourceDataset: '客服对话原始集',
outputDataset: '客服对话清洗集', outputDataset: '客服对话清洗集',
outputDatasetId: 7, outputDatasetId: 7,
@@ -101,7 +99,7 @@ const detailMap: Record<string, ProcessDetail> = {
}, },
'492015': { '492015': {
id: '492015', name: '指令微调数据构造', status: 'running', id: '492015', name: '指令微调数据构造', status: 'running',
description: '从通用语料中构造指令微调训练样本。', processType: '指令构造', dataType: '非结构化数据', description: '从通用语料中构造指令微调训练样本。', processType: '非结构化数据',
sourceDataset: '通用语料库', outputDataset: 'SFT 指令集', outputDatasetId: 8, creator: '管理员', sourceDataset: '通用语料库', outputDataset: 'SFT 指令集', outputDatasetId: 8, creator: '管理员',
createTime: '2026-07-09 09:10:00', startTime: '2026-07-09 09:10:21', duration: '处理中', progress: 68, createTime: '2026-07-09 09:10:00', startTime: '2026-07-09 09:10:21', duration: '处理中', progress: 68,
inputCount: 18500, outputCount: 8568, filteredCount: 425, duplicateCount: 192, errorCount: 8, inputCount: 18500, outputCount: 8568, filteredCount: 425, duplicateCount: 192, errorCount: 8,
@@ -112,7 +110,7 @@ const detailMap: Record<string, ProcessDetail> = {
}, },
'731948': { '731948': {
id: '731948', name: '敏感信息脱敏处理', status: 'pending', description: '识别并脱敏用户反馈数据中的敏感字段。', id: '731948', name: '敏感信息脱敏处理', status: 'pending', description: '识别并脱敏用户反馈数据中的敏感字段。',
processType: '脱敏处理', dataType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集', processType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
outputDatasetId: 9, creator: '管理员', createTime: '2026-07-09 16:45:00', duration: '等待执行', progress: 0, outputDatasetId: 9, creator: '管理员', createTime: '2026-07-09 16:45:00', duration: '等待执行', progress: 0,
inputCount: 9340, outputCount: 0, filteredCount: 0, duplicateCount: 0, errorCount: 0, inputCount: 9340, outputCount: 0, filteredCount: 0, duplicateCount: 0, errorCount: 0,
config: [ config: [
@@ -122,7 +120,7 @@ const detailMap: Record<string, ProcessDetail> = {
}, },
'582012': { '582012': {
id: '582012', name: '多轮对话拼接', status: 'failed', description: '将单轮问答按会话标识拼接为多轮对话数据。', id: '582012', name: '多轮对话拼接', status: 'failed', description: '将单轮问答按会话标识拼接为多轮对话数据。',
processType: '格式转换', dataType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员', processType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
createTime: '2026-07-10 08:30:00', startTime: '2026-07-10 08:30:16', completeTime: '2026-07-10 08:31:04', createTime: '2026-07-10 08:30:00', startTime: '2026-07-10 08:30:16', completeTime: '2026-07-10 08:31:04',
duration: '48 秒', progress: 37, inputCount: 7520, outputCount: 2780, filteredCount: 24, duplicateCount: 0, errorCount: 1, duration: '48 秒', progress: 37, inputCount: 7520, outputCount: 2780, filteredCount: 24, duplicateCount: 0, errorCount: 1,
config: [ config: [
@@ -186,7 +184,6 @@ function resetPage() {
<dl class="heading-meta"> <dl class="heading-meta">
<div><dt>任务 ID</dt><dd>{{ detail.id }}</dd></div> <div><dt>任务 ID</dt><dd>{{ detail.id }}</dd></div>
<div><dt>处理类型</dt><dd>{{ detail.processType }}</dd></div> <div><dt>处理类型</dt><dd>{{ detail.processType }}</dd></div>
<div><dt>数据类型</dt><dd>{{ detail.dataType }}</dd></div>
<div><dt>创建人</dt><dd>{{ detail.creator }}</dd></div> <div><dt>创建人</dt><dd>{{ detail.creator }}</dd></div>
</dl> </dl>
</div> </div>

View File

@@ -5,17 +5,25 @@ import { ElMessage } from 'element-plus'
import DataTablePage from '@/components/DataTablePage.vue' import DataTablePage from '@/components/DataTablePage.vue'
import ModelStatusTag from '@/components/ModelStatusTag.vue' import ModelStatusTag from '@/components/ModelStatusTag.vue'
import type { ProcessType } from './create/types'
/** 数据处理任务类型 */ /** 数据处理任务类型 */
interface DataProcessTask { interface DataProcessTask {
id: number | string id: number | string
name: string name: string
status: string status: string
process_type: string process_type: ProcessType
source_dataset: string source_dataset: string
output_dataset?: string output_dataset?: string
create_time?: string create_time?: string
} }
const processTypeMap: Record<ProcessType, string> = {
structured: '结构化数据',
unstructured: '非结构化数据',
external: '外来数据源拉取',
}
// TODO: 接入真实接口前,先用本地 mock 数据 // TODO: 接入真实接口前,先用本地 mock 数据
const router = useRouter() const router = useRouter()
const dataList = ref<DataProcessTask[]>([ const dataList = ref<DataProcessTask[]>([
@@ -23,7 +31,7 @@ const dataList = ref<DataProcessTask[]>([
id: 183921, id: 183921,
name: '客服问答数据清洗', name: '客服问答数据清洗',
status: 'completed', status: 'completed',
process_type: '去重清洗', process_type: 'structured',
source_dataset: '客服对话原始集', source_dataset: '客服对话原始集',
output_dataset: '客服对话清洗集', output_dataset: '客服对话清洗集',
create_time: '2026-07-08 14:23:00', create_time: '2026-07-08 14:23:00',
@@ -32,7 +40,7 @@ const dataList = ref<DataProcessTask[]>([
id: 492015, id: 492015,
name: '指令微调数据构造', name: '指令微调数据构造',
status: 'running', status: 'running',
process_type: '指令构造', process_type: 'unstructured',
source_dataset: '通用语料库', source_dataset: '通用语料库',
output_dataset: 'SFT 指令集', output_dataset: 'SFT 指令集',
create_time: '2026-07-09 09:10:00', create_time: '2026-07-09 09:10:00',
@@ -41,7 +49,7 @@ const dataList = ref<DataProcessTask[]>([
id: 731948, id: 731948,
name: '敏感信息脱敏处理', name: '敏感信息脱敏处理',
status: 'pending', status: 'pending',
process_type: '脱敏处理', process_type: 'structured',
source_dataset: '用户反馈数据', source_dataset: '用户反馈数据',
create_time: '2026-07-09 16:45:00', create_time: '2026-07-09 16:45:00',
}, },
@@ -49,7 +57,7 @@ const dataList = ref<DataProcessTask[]>([
id: 582012, id: 582012,
name: '多轮对话拼接', name: '多轮对话拼接',
status: 'failed', status: 'failed',
process_type: '格式转换', process_type: 'structured',
source_dataset: '单轮问答集', source_dataset: '单轮问答集',
create_time: '2026-07-10 08:30:00', create_time: '2026-07-10 08:30:00',
}, },
@@ -98,7 +106,12 @@ function formatDateTime(value?: string) {
</template> </template>
</el-table-column> </el-table-column>
<el-table-column label="处理类型" align="center" width="140"> <el-table-column label="处理类型" align="center" width="140">
<template #default="{ row }">{{ row.process_type || '-' }}</template> <template #default="{ row }">
<el-tag v-if="row.process_type" size="small" type="info" effect="plain">
{{ processTypeMap[row.process_type as ProcessType] || row.process_type }}
</el-tag>
<span v-else>-</span>
</template>
</el-table-column> </el-table-column>
<el-table-column label="源数据集" align="center" show-overflow-tooltip> <el-table-column label="源数据集" align="center" show-overflow-tooltip>
<template #default="{ row }">{{ row.source_dataset || '-' }}</template> <template #default="{ row }">{{ row.source_dataset || '-' }}</template>

View File

@@ -0,0 +1,439 @@
<script setup lang="ts">
import { ref } from 'vue'
import type { ModelItem } from '@/types'
import type { GenerationControlOptions } from './types'
const props = defineProps<{
options: GenerationControlOptions
models: ModelItem[]
section: 'model' | 'quality'
validationMessage?: string
}>()
const advancedModelSettingsOpen = ref(false)
const emit = defineEmits<{
'update:options': [value: GenerationControlOptions]
}>()
function updateField<K extends keyof GenerationControlOptions>(
field: K,
value: GenerationControlOptions[K],
) {
emit('update:options', { ...props.options, [field]: value })
}
function updateQualityRules(value: Array<string | number>) {
const rules = value.filter((item): item is string => typeof item === 'string')
emit('update:options', {
...props.options,
filterLowQuality: rules.includes('low_quality'),
filterShortContent: rules.includes('short_content'),
})
}
const selectedQualityRules = () => [
props.options.filterLowQuality ? 'low_quality' : '',
props.options.filterShortContent ? 'short_content' : '',
].filter(Boolean)
function sectionValidationMessage() {
if (!props.validationMessage) return ''
const isModelMessage = props.validationMessage.includes('数据生成模型')
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
return isModelMessage ? '' : props.validationMessage
}
</script>
<template>
<div v-if="section === 'model'" class="generation-config-group model-config-group">
<div class="model-field">
<div class="field-copy">
<strong>数据生成模型 <em>*</em></strong>
<small>选择负责生成问答指令或结构化内容的大模型</small>
</div>
<el-select
class="model-select"
:model-value="options.generationModelId"
filterable
placeholder="请选择用于生成数据的模型"
aria-label="数据生成模型"
@update:model-value="updateField('generationModelId', $event)"
>
<el-option
v-for="model in models"
:key="model.id"
:label="model.name"
:value="model.id"
>
<div class="model-option">
<span>{{ model.name }}</span>
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
</div>
</el-option>
<template #empty>
<div class="model-empty">暂无可用的大模型请先在模型管理中添加</div>
</template>
</el-select>
</div>
<div class="model-field">
<div class="field-copy">
<strong>默认提示语</strong>
<small>用于约束生成内容的格式语气和完整性可按任务需要修改</small>
</div>
<el-input
class="prompt-input"
:model-value="options.generationPrompt"
type="textarea"
:rows="4"
maxlength="500"
show-word-limit
resize="vertical"
placeholder="请输入模型生成内容时需要遵循的要求"
aria-label="模型默认提示语"
@update:model-value="updateField('generationPrompt', $event)"
/>
<div class="prompt-variables-hint">
提示可在文本中通过 <code>{{ content }}</code> 引用当前正在处理的数据内容
</div>
</div>
<button
type="button"
class="advanced-settings-toggle"
:aria-expanded="advancedModelSettingsOpen"
@click="advancedModelSettingsOpen = !advancedModelSettingsOpen"
>
<span>
<strong>高级设置</strong>
<small>温度最大输出长度与格式约束</small>
</span>
<i class="fa" :class="advancedModelSettingsOpen ? 'fa-chevron-up' : 'fa-chevron-down'" />
</button>
<div v-if="advancedModelSettingsOpen" class="advanced-settings-panel">
<div class="advanced-settings-grid">
<label class="config-field">
<span class="config-field-label">生成温度 (Temperature)</span>
<el-slider
:model-value="options.temperature"
:min="0"
:max="1"
:step="0.1"
show-input
@update:model-value="updateField('temperature', Number($event) || 0)"
/>
<small>较低值输出更确定保守较高值输出更多样有创造性</small>
</label>
<label class="config-field">
<span class="config-field-label">最大输出长度 (Max Tokens)</span>
<span class="unit-input">
<el-input-number
:model-value="options.maxTokens"
:min="100"
:max="8192"
:step="256"
controls-position="right"
@update:model-value="updateField('maxTokens', Number($event) || 1024)"
/>
<span>Token</span>
</span>
<small>限制单次生成的最大内容长度防止过度发散</small>
</label>
</div>
<div class="json-mode-row">
<div class="field-copy">
<strong>强制 JSON 格式输出</strong>
<small>要求大模型返回纯 JSON 对象有助于提高下游结构化提取的稳定性</small>
</div>
<el-switch
:model-value="options.jsonMode"
inline-prompt
active-text=""
inactive-text=""
@update:model-value="updateField('jsonMode', Boolean($event))"
/>
</div>
</div>
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
{{ sectionValidationMessage() }}
</p>
</div>
<div v-else class="generation-config-group quality-config-group">
<div class="quality-switch-row">
<div class="config-group-heading">
<div>
<h4>质量筛选</h4>
<p>生成后自动过滤不符合要求的内容不影响源数据预处理</p>
</div>
</div>
<el-switch
:model-value="options.qualityFilterEnabled"
inline-prompt
active-text=""
inactive-text=""
aria-label="质量筛选"
@update:model-value="updateField('qualityFilterEnabled', Boolean($event))"
/>
</div>
<div v-if="options.qualityFilterEnabled" class="quality-rule-panel">
<el-checkbox-group
:model-value="selectedQualityRules()"
class="quality-rule-list"
@update:model-value="updateQualityRules"
>
<el-checkbox value="low_quality">
<span class="rule-copy">
<strong>过滤低质量内容</strong>
<small>按相关性完整性和可读性过滤明显不合格的生成结果</small>
</span>
</el-checkbox>
<el-checkbox value="short_content">
<span class="rule-copy">
<strong>过滤过短内容</strong>
<small>过滤正文长度低于指定字数的生成结果</small>
</span>
</el-checkbox>
</el-checkbox-group>
<label v-if="options.filterShortContent" class="minimum-length-field">
<span>最少字数</span>
<span class="minimum-length-input">
<el-input-number
:model-value="options.minOutputLength"
:min="1"
:max="1000"
:step="5"
:precision="0"
controls-position="right"
aria-label="生成内容最少字数"
@update:model-value="updateField('minOutputLength', Number($event) || 1)"
/>
<span></span>
</span>
<small>少于该字数的生成结果将被过滤</small>
</label>
<p v-if="!options.filterLowQuality && !options.filterShortContent" class="quality-rule-warning" role="alert">
请至少选择一项质量筛选规则
</p>
</div>
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
{{ sectionValidationMessage() }}
</p>
</div>
</template>
<style scoped lang="scss">
.generation-config-group {
padding: 14px;
border: 1px solid #e2e5ec;
border-radius: 8px;
background: #fff;
}
.config-group-heading {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
h4 { margin: 0; color: #344054; font-size: 13px; font-weight: 600; }
p { margin: 4px 0 0; color: #8a93a3; font-size: 12px; line-height: 1.5; }
}
.model-config-group {
display: flex;
flex-direction: column;
gap: 16px;
}
.model-field {
display: flex;
flex-direction: column;
gap: 8px;
}
.field-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
em { color: var(--el-color-danger); font-style: normal; }
}
.model-select { width: 100%; }
.prompt-input { width: 100%; }
.model-option { width: 100%; display: flex; align-items: center; justify-content: space-between; gap: 16px; }
.model-option small { color: #98a2b3; font-size: 11px; }
.model-empty { padding: 12px; color: #8a93a3; font-size: 12px; text-align: center; }
.prompt-variables-hint {
color: #8a93a3;
font-size: 12px;
code {
padding: 2px 4px;
color: #5b50f2;
background: #f0f0ff;
border-radius: 4px;
font-family: inherit;
}
}
.advanced-settings-toggle {
display: flex;
width: 100%;
min-height: 48px;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-top: 4px;
padding: 10px 14px;
color: #344054;
text-align: left;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
cursor: pointer;
transition: border-color 0.18s ease, background-color 0.18s ease;
&:hover {
background: #f8f7ff;
border-color: #b7b2f7;
}
> span {
display: grid;
gap: 3px;
}
strong {
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
i {
flex: 0 0 auto;
color: #7b8495;
font-size: 12px;
}
}
.advanced-settings-panel {
display: grid;
gap: 16px;
margin-top: -8px;
padding: 16px;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.advanced-settings-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
}
.config-field {
display: grid;
align-content: start;
gap: 8px;
min-width: 0;
padding: 14px;
color: #344054;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
> small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
:deep(.el-slider) {
margin: 0 8px;
}
}
.config-field-label {
font-size: 13px;
font-weight: 600;
}
.unit-input {
display: flex;
align-items: center;
gap: 8px;
color: #7b8495;
font-size: 12px;
:deep(.el-input-number) {
width: 100%;
}
}
.json-mode-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 20px;
padding: 14px;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.quality-switch-row { display: flex; align-items: center; justify-content: space-between; gap: 20px; }
.quality-rule-panel { margin-top: 12px; padding-top: 12px; border-top: 1px solid #e7eaf0; }
.quality-rule-list { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
.quality-rule-list :deep(.el-checkbox) {
min-height: 62px; height: auto; margin-right: 0; padding: 10px 12px;
align-items: flex-start; border: 1px solid #e2e5ec; border-radius: 7px; background: #fff;
}
.quality-rule-list :deep(.el-checkbox__input) { margin-top: 3px; }
.quality-rule-list :deep(.el-checkbox__label) { min-width: 0; padding-left: 9px; white-space: normal; }
.rule-copy { display: grid; gap: 3px; }
.rule-copy strong { color: #344054; font-size: 12px; font-weight: 600; }
.rule-copy small { color: #8a93a3; font-size: 11px; line-height: 1.5; }
.minimum-length-field {
margin-top: 12px; display: grid; grid-template-columns: 110px 220px minmax(0, 1fr);
align-items: center; gap: 12px; color: #475467; font-size: 12px;
}
.minimum-length-input { display: flex; align-items: center; gap: 8px; }
.minimum-length-input :deep(.el-input-number) { width: 180px; }
.minimum-length-field > small { color: #8a93a3; }
.quality-rule-warning { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
.generation-validation-message { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
@media (max-width: 900px) {
.advanced-settings-grid { grid-template-columns: 1fr; }
.quality-rule-list { grid-template-columns: 1fr; }
.minimum-length-field { grid-template-columns: 1fr; }
}
</style>

View File

@@ -97,7 +97,7 @@ const emit = defineEmits<{
align-items: center; align-items: center;
justify-content: space-between; justify-content: space-between;
padding: 15px 17px; padding: 15px 17px;
background: #fbfcfe; background: #fff;
border-bottom: 1px solid #e8ebf0; border-bottom: 1px solid #e8ebf0;
strong { strong {

View File

@@ -350,7 +350,7 @@ function lineRange(item: PreviewItem) {
min-height: 52px; min-height: 52px;
padding: 0 15px; padding: 0 15px;
color: #313949; color: #313949;
background: #fbfcfe; background: #fff;
border-bottom: 1px solid #e8ebf0; border-bottom: 1px solid #e8ebf0;
font-size: 13px; font-size: 13px;

View File

@@ -143,7 +143,7 @@ function selectRelative(offset: number) {
min-height: 52px; min-height: 52px;
padding: 0 15px; padding: 0 15px;
color: #344054; color: #344054;
background: #fbfcfe; background: #fff;
border-bottom: 1px solid #e8ebf0; border-bottom: 1px solid #e8ebf0;
font-size: 13px; font-size: 13px;

View File

@@ -397,7 +397,7 @@ function formatSize(size: number) {
width: 100%; width: 100%;
min-height: 154px; min-height: 154px;
padding: 32px 20px; padding: 32px 20px;
background: #fbfcfe; background: #fff;
border-color: #dfe3ea; border-color: #dfe3ea;
transition: border-color 0.18s ease, background-color 0.18s ease; transition: border-color 0.18s ease, background-color 0.18s ease;
@@ -435,7 +435,7 @@ function formatSize(size: number) {
padding: 10px 14px; padding: 10px 14px;
color: #5f6878; color: #5f6878;
font-size: 12px; font-size: 12px;
background: #fbfcfe; background: #fff;
border-bottom: 1px solid #edf0f5; border-bottom: 1px solid #edf0f5;
} }

View File

@@ -4,12 +4,15 @@ import type { FormInstance, FormRules } from 'element-plus'
import type { import type {
ChunkMethod, ChunkMethod,
DatasetSplitOptions, DatasetSplitOptions,
GenerationControlOptions,
PreprocessOption, PreprocessOption,
ProcessType, ProcessType,
StructuredProcessOptions, StructuredProcessOptions,
UnstructuredPreprocessOption, UnstructuredPreprocessOption,
UnstructuredProcessOptions, UnstructuredProcessOptions,
} from './types' } from './types'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
import type { ModelItem } from '@/types'
const props = defineProps<{ const props = defineProps<{
name: string name: string
@@ -17,6 +20,7 @@ const props = defineProps<{
processType: ProcessType processType: ProcessType
structuredOptions: StructuredProcessOptions structuredOptions: StructuredProcessOptions
unstructuredOptions: UnstructuredProcessOptions unstructuredOptions: UnstructuredProcessOptions
generationModels: ModelItem[]
}>() }>()
const emit = defineEmits<{ const emit = defineEmits<{
@@ -63,6 +67,10 @@ function updateStructuredField<K extends keyof StructuredProcessOptions>(
emit('update:structuredOptions', { ...props.structuredOptions, [field]: value }) emit('update:structuredOptions', { ...props.structuredOptions, [field]: value })
} }
function updateStructuredGenerationOptions(value: GenerationControlOptions) {
emit('update:structuredOptions', { ...props.structuredOptions, ...value })
}
function updatePreprocessOptions(value: Array<string | number | boolean>) { function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value)) const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter( const preprocessOptions = value.filter(
@@ -89,6 +97,10 @@ function updateUnstructuredField<K extends keyof UnstructuredProcessOptions>(
emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value }) emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value })
} }
function updateUnstructuredGenerationOptions(value: GenerationControlOptions) {
emit('update:unstructuredOptions', { ...props.unstructuredOptions, ...value })
}
function updateSmartPreprocess(value: string | number | boolean) { function updateSmartPreprocess(value: string | number | boolean) {
const enabled = Boolean(value) const enabled = Boolean(value)
const remainingOptions = props.unstructuredOptions.preprocessOptions.filter( const remainingOptions = props.unstructuredOptions.preprocessOptions.filter(
@@ -143,6 +155,7 @@ function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value:
const formRef = ref<FormInstance>() const formRef = ref<FormInstance>()
const advancedChunkSettingsOpen = ref(props.unstructuredOptions.chunkMethod === 'custom') const advancedChunkSettingsOpen = ref(props.unstructuredOptions.chunkMethod === 'custom')
const validationAttempted = ref(false)
const formModel = computed(() => ({ const formModel = computed(() => ({
name: props.name, name: props.name,
processType: props.processType, processType: props.processType,
@@ -172,6 +185,25 @@ const preserveSpecialContentEnabled = computed(() => (
&& props.unstructuredOptions.preserveLists && props.unstructuredOptions.preserveLists
)) ))
const activeGenerationOptions = computed<GenerationControlOptions | null>(() => {
if (props.processType === 'structured') return props.structuredOptions
if (props.processType === 'unstructured') return props.unstructuredOptions
return null
})
const generationValidationMessage = computed(() => {
const options = activeGenerationOptions.value
if (!options) return ''
if (options.generationModelId === '') return '请选择数据生成模型'
if (options.qualityFilterEnabled && !options.filterLowQuality && !options.filterShortContent) {
return '开启质量筛选后,请至少选择一项筛选规则'
}
if (options.qualityFilterEnabled && options.filterShortContent && options.minOutputLength < 1) {
return '最少字数必须大于 0'
}
return ''
})
const chunkValidationMessage = computed(() => { const chunkValidationMessage = computed(() => {
if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) { if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) {
return '重叠长度必须小于切片长度' return '重叠长度必须小于切片长度'
@@ -208,6 +240,7 @@ const rules: FormRules = {
} }
async function validate() { async function validate() {
validationAttempted.value = true
if (!formRef.value) return false if (!formRef.value) return false
try { try {
await formRef.value.validate() await formRef.value.validate()
@@ -219,6 +252,7 @@ async function validate() {
return false return false
} }
} }
if (generationValidationMessage.value) return false
return true return true
} catch { } catch {
return false return false
@@ -344,6 +378,13 @@ defineExpose({ validate })
</div> </div>
</div> </div>
<div class="generation-option-list"> <div class="generation-option-list">
<GenerationOptionsPanel
:options="structuredOptions"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateStructuredGenerationOptions"
/>
<div class="generation-option-row"> <div class="generation-option-row">
<div class="generation-option-copy"> <div class="generation-option-copy">
<strong>语义丰富表达</strong> <strong>语义丰富表达</strong>
@@ -435,6 +476,24 @@ defineExpose({ validate })
</div> </div>
</div> </div>
</div> </div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="structuredOptions"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateStructuredGenerationOptions"
/>
</div>
</div>
</template> </template>
<template v-if="processType === 'unstructured'"> <template v-if="processType === 'unstructured'">
@@ -625,6 +684,13 @@ defineExpose({ validate })
</div> </div>
</div> </div>
<div class="generation-option-list"> <div class="generation-option-list">
<GenerationOptionsPanel
:options="unstructuredOptions"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateUnstructuredGenerationOptions"
/>
<div class="generation-option-row"> <div class="generation-option-row">
<div class="generation-option-copy"> <div class="generation-option-copy">
<strong>语义丰富表达</strong> <strong>语义丰富表达</strong>
@@ -723,6 +789,24 @@ defineExpose({ validate })
</div> </div>
</div> </div>
</div> </div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="unstructuredOptions"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateUnstructuredGenerationOptions"
/>
</div>
</div>
</template> </template>
</el-form> </el-form>
@@ -843,7 +927,7 @@ defineExpose({ validate })
min-width: 0; min-width: 0;
padding: 14px; padding: 14px;
color: #344054; color: #344054;
background: #fbfcfe; background: #fff;
border: 1px solid #e2e5ec; border: 1px solid #e2e5ec;
border-radius: 8px; border-radius: 8px;
@@ -902,7 +986,7 @@ defineExpose({ validate })
padding: 10px 14px; padding: 10px 14px;
color: #344054; color: #344054;
text-align: left; text-align: left;
background: #fbfcfe; background: #fff;
border: 1px solid #e2e5ec; border: 1px solid #e2e5ec;
border-radius: 8px; border-radius: 8px;
cursor: pointer; cursor: pointer;
@@ -946,7 +1030,7 @@ defineExpose({ validate })
gap: 12px; gap: 12px;
margin-top: 8px; margin-top: 8px;
padding: 12px; padding: 12px;
background: #fbfcfe; background: #fff;
border: 1px solid #e2e5ec; border: 1px solid #e2e5ec;
border-radius: 8px; border-radius: 8px;
@@ -1154,3 +1238,4 @@ defineExpose({ validate })
} }
</style> </style>
GenerationControlOptions,

View File

@@ -501,7 +501,11 @@ export function createResults(
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1)) ? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1)) : Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
return items.flatMap((item, index) => { const generatedResults = items.flatMap((item, index) => {
if (options?.qualityFilterEnabled && options.filterLowQuality) {
if (item.status === 'invalid' || !item.editedContent.trim()) return []
}
const [firstLine = '', ...rest] = item.editedContent.split('\n') const [firstLine = '', ...rest] = item.editedContent.split('\n')
const output = rest.join('\n').trim() || item.editedContent.trim() const output = rest.join('\n').trim() || item.editedContent.trim()
const baseInstruction = firstLine.replace(/^问[:]\s*/, '').trim() || `数据条目 ${index + 1}` const baseInstruction = firstLine.replace(/^问[:]\s*/, '').trim() || `数据条目 ${index + 1}`
@@ -525,4 +529,12 @@ export function createResults(
} }
}) })
}) })
if (!options?.qualityFilterEnabled) return generatedResults
return generatedResults.filter((result) => {
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
return true
})
} }

View File

@@ -16,7 +16,19 @@ export interface DatasetSplitOptions {
test: number test: number
} }
export interface StructuredProcessOptions { export interface GenerationControlOptions {
generationModelId: string | number | ''
generationPrompt: string
temperature: number
maxTokens: number
jsonMode: boolean
qualityFilterEnabled: boolean
filterLowQuality: boolean
filterShortContent: boolean
minOutputLength: number
}
export interface StructuredProcessOptions extends GenerationControlOptions {
preprocessOptions: PreprocessOption[] preprocessOptions: PreprocessOption[]
semanticEnrichment: boolean semanticEnrichment: boolean
qaPairsPerRow: number qaPairsPerRow: number
@@ -34,7 +46,7 @@ export type UnstructuredPreprocessOption =
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom' export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
export interface UnstructuredProcessOptions { export interface UnstructuredProcessOptions extends GenerationControlOptions {
preprocessOptions: UnstructuredPreprocessOption[] preprocessOptions: UnstructuredPreprocessOption[]
chunkMethod: ChunkMethod chunkMethod: ChunkMethod
chunkSize: number chunkSize: number