feat: 数据处理向导新增生成控制选项
提取 GenerationOptionsPanel 组件统一管理生成模型、温度、最大长度、JSON 模式与质量过滤配置,StructuredProcessOptions 与 UnstructuredProcessOptions 继承 GenerationControlOptions,CreateView、TaskSetupStep 及各子步骤同步接入,详情/列表小幅调整,回归脚本扩充生成选项断言。
This commit is contained in:
@@ -126,6 +126,8 @@ assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄
|
||||
|
||||
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
||||
const taskSetupSource = await readFile(taskSetupPath, 'utf8')
|
||||
const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue')
|
||||
const generationControlSource = await readFile(generationOptionsPath, 'utf8')
|
||||
const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue')
|
||||
const sourceUploadSource = await readFile(sourceUploadPath, 'utf8')
|
||||
|
||||
@@ -135,7 +137,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
|
||||
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第二步没有挂载独立上传组件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:上传文件'/, '第一步主按钮没有指向上传文件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续:数据预览'/, '第二步主按钮没有指向数据预览')
|
||||
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = 4/, '五步索引变更后必须升级草稿版本')
|
||||
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = 5/, '生成配置扩展后必须升级草稿版本')
|
||||
|
||||
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
|
||||
const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromCreateStart)
|
||||
@@ -195,6 +197,40 @@ assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.st
|
||||
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
|
||||
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
|
||||
|
||||
for (const field of [
|
||||
'generationModelId',
|
||||
'generationPrompt',
|
||||
'qualityFilterEnabled',
|
||||
'filterLowQuality',
|
||||
'filterShortContent',
|
||||
'minOutputLength',
|
||||
]) {
|
||||
assert.ok(typesSource.includes(field), `生成控制配置缺少字段:${field}`)
|
||||
assert.ok(viewSource.includes(field), `父页面默认值或草稿状态缺少字段:${field}`)
|
||||
}
|
||||
assert.match(taskSetupSource, /GenerationOptionsPanel/, '生成选项没有复用统一的大模型与质量筛选组件')
|
||||
assert.match(taskSetupSource, /:options="structuredOptions"/, '结构化生成选项未接入统一配置组件')
|
||||
assert.match(taskSetupSource, /:options="unstructuredOptions"/, '非结构化生成选项未接入统一配置组件')
|
||||
assert.match(taskSetupSource, /<h3>大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类')
|
||||
assert.match(taskSetupSource, /section="model"/, '独立大模型分类没有挂载模型配置')
|
||||
assert.match(taskSetupSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
|
||||
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
|
||||
for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
|
||||
assert.ok(generationControlSource.includes(label), `生成控制界面缺少:${label}`)
|
||||
}
|
||||
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
|
||||
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
|
||||
assert.match(generationControlSource, /\.model-option-row\s*\{[\s\S]*?display:\s*flex[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有与上方生成选项使用一致的配置行样式')
|
||||
assert.match(viewSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语')
|
||||
assert.equal((viewSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语')
|
||||
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
|
||||
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
|
||||
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
|
||||
assert.match(taskSetupSource, /generationValidationMessage/, '生成模型与质量规则缺少继续前校验')
|
||||
assert.match(viewSource, /useModelsStore/, '创建页没有加载模型列表')
|
||||
assert.match(viewSource, /model\.type === 'LLM'/, '数据生成模型列表没有排除非大模型')
|
||||
assert.match(viewSource, /:generation-models="generationModels"/, '创建页没有向生成选项传递模型列表')
|
||||
|
||||
assert.match(typesSource, /export interface UnstructuredProcessOptions/, '缺少非结构化处理选项类型')
|
||||
for (const field of [
|
||||
'preprocessOptions',
|
||||
@@ -298,7 +334,17 @@ const generationOptionsStart = viewSource.indexOf('function generationAffectingO
|
||||
const generationOptionsEnd = viewSource.indexOf('const generationOptionsSignature', generationOptionsStart)
|
||||
assert.ok(generationOptionsStart >= 0 && generationOptionsEnd > generationOptionsStart, '缺少生成影响配置签名函数')
|
||||
const generationOptionsSource = viewSource.slice(generationOptionsStart, generationOptionsEnd)
|
||||
for (const field of ['semanticEnrichment', 'qaPairsPerChunk', 'datasetSplit']) {
|
||||
for (const field of [
|
||||
'semanticEnrichment',
|
||||
'qaPairsPerChunk',
|
||||
'datasetSplit',
|
||||
'generationModelId',
|
||||
'generationPrompt',
|
||||
'qualityFilterEnabled',
|
||||
'filterLowQuality',
|
||||
'filterShortContent',
|
||||
'minOutputLength',
|
||||
]) {
|
||||
assert.ok(generationOptionsSource.includes(field), `生成签名缺少字段:${field}`)
|
||||
}
|
||||
assert.match(
|
||||
@@ -341,6 +387,12 @@ const baseUnstructuredOptions = {
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 3,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: 1,
|
||||
generationPrompt: '仅输出问答对',
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
}
|
||||
|
||||
for (const chunkMethod of ['semantic', 'heading', 'fixed', 'custom']) {
|
||||
@@ -434,6 +486,31 @@ assert.ok(samplePreviewItems.length > 12, '测试文档未生成足够的切片'
|
||||
const generatedResults = previewModelModule.createResults(samplePreviewItems.slice(0, 13), baseUnstructuredOptions)
|
||||
assert.equal(generatedResults.length, 39, '每个切片生成 3 个问答对未完整应用到所有切片')
|
||||
|
||||
const shortContentItems = [{
|
||||
...samplePreviewItems[0],
|
||||
editedContent: '问:示例\n短回答',
|
||||
}]
|
||||
const filteredShortResults = previewModelModule.createResults(shortContentItems, {
|
||||
...baseUnstructuredOptions,
|
||||
qualityFilterEnabled: true,
|
||||
filterLowQuality: false,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
assert.equal(filteredShortResults.length, 0, '开启过短内容过滤后仍保留低于最少字数的结果')
|
||||
|
||||
const invalidContentItems = [{
|
||||
...samplePreviewItems[0],
|
||||
status: 'invalid',
|
||||
}]
|
||||
const filteredInvalidResults = previewModelModule.createResults(invalidContentItems, {
|
||||
...baseUnstructuredOptions,
|
||||
qualityFilterEnabled: true,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: false,
|
||||
})
|
||||
assert.equal(filteredInvalidResults.length, 0, '开启低质量过滤后仍保留标记为无效的结果')
|
||||
|
||||
const legacyExternalItems = previewModelModule.buildPreviewItems('a\nb\nc\nd', 'external', 'legacy-check')
|
||||
assert.equal(legacyExternalItems.length, 2, '外来数据原有的每 3 行分组行为被破坏')
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
|
||||
import { onBeforeRouteLeave, useRouter } from 'vue-router'
|
||||
import { ElMessage, type UploadFile } from 'element-plus'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
|
||||
import TaskSetupStep from './create/TaskSetupStep.vue'
|
||||
import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
@@ -9,6 +10,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
||||
import { useModelsStore } from '@/stores/models'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
GenerationState,
|
||||
@@ -21,11 +23,15 @@ import type {
|
||||
} from './create/types'
|
||||
|
||||
const router = useRouter()
|
||||
const modelsStore = useModelsStore()
|
||||
const { list: modelList } = storeToRefs(modelsStore)
|
||||
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
|
||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||
const DRAFT_SCHEMA_VERSION = 4
|
||||
const DRAFT_SCHEMA_VERSION = 5
|
||||
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
|
||||
const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容,答案应事实清晰、语言自然,不要添加分析过程、说明或无关内容。'
|
||||
|
||||
const WIZARD_STEPS = [
|
||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||
@@ -45,6 +51,15 @@ const structuredOptions = ref<StructuredProcessOptions>({
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
const unstructuredOptions = ref<UnstructuredProcessOptions>({
|
||||
preprocessOptions: [
|
||||
@@ -66,6 +81,15 @@ const unstructuredOptions = ref<UnstructuredProcessOptions>({
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
interface UploadedDataFile {
|
||||
uid: number | string
|
||||
@@ -219,19 +243,45 @@ function previewAffectingOptions() {
|
||||
|
||||
function generationAffectingOptions() {
|
||||
if (processType.value === 'structured') {
|
||||
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value
|
||||
return { semanticEnrichment, qaPairsPerRow, datasetSplit }
|
||||
const {
|
||||
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
|
||||
temperature, maxTokens, jsonMode,
|
||||
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
|
||||
} = structuredOptions.value
|
||||
return {
|
||||
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
|
||||
temperature, maxTokens, jsonMode,
|
||||
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
|
||||
}
|
||||
}
|
||||
if (processType.value === 'unstructured') {
|
||||
const {
|
||||
semanticEnrichment,
|
||||
qaPairsPerChunk,
|
||||
datasetSplit,
|
||||
generationModelId,
|
||||
generationPrompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
jsonMode,
|
||||
qualityFilterEnabled,
|
||||
filterLowQuality,
|
||||
filterShortContent,
|
||||
minOutputLength,
|
||||
} = unstructuredOptions.value
|
||||
return {
|
||||
semanticEnrichment,
|
||||
qaPairsPerChunk,
|
||||
datasetSplit,
|
||||
generationModelId,
|
||||
generationPrompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
jsonMode,
|
||||
qualityFilterEnabled,
|
||||
filterLowQuality,
|
||||
filterShortContent,
|
||||
minOutputLength,
|
||||
}
|
||||
}
|
||||
return null
|
||||
@@ -337,6 +387,31 @@ function restoreDraft() {
|
||||
qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk)
|
||||
? restoredUnstructuredOptions.qaPairsPerChunk
|
||||
: unstructuredOptions.value.qaPairsPerChunk,
|
||||
generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId,
|
||||
generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string'
|
||||
? restoredUnstructuredOptions.generationPrompt
|
||||
: unstructuredOptions.value.generationPrompt,
|
||||
temperature: Number.isFinite(restoredUnstructuredOptions.temperature)
|
||||
? restoredUnstructuredOptions.temperature
|
||||
: unstructuredOptions.value.temperature,
|
||||
maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens)
|
||||
? restoredUnstructuredOptions.maxTokens
|
||||
: unstructuredOptions.value.maxTokens,
|
||||
jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean'
|
||||
? restoredUnstructuredOptions.jsonMode
|
||||
: unstructuredOptions.value.jsonMode,
|
||||
qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean'
|
||||
? restoredUnstructuredOptions.qualityFilterEnabled
|
||||
: unstructuredOptions.value.qualityFilterEnabled,
|
||||
filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean'
|
||||
? restoredUnstructuredOptions.filterLowQuality
|
||||
: unstructuredOptions.value.filterLowQuality,
|
||||
filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean'
|
||||
? restoredUnstructuredOptions.filterShortContent
|
||||
: unstructuredOptions.value.filterShortContent,
|
||||
minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength)
|
||||
? restoredUnstructuredOptions.minOutputLength
|
||||
: unstructuredOptions.value.minOutputLength,
|
||||
datasetSplit: {
|
||||
train: Number.isFinite(restoredDatasetSplit?.train)
|
||||
? restoredDatasetSplit.train
|
||||
@@ -819,7 +894,10 @@ onBeforeRouteLeave(async () => {
|
||||
})
|
||||
|
||||
onBeforeUnmount(stopGenerationTimer)
|
||||
onMounted(restoreDraft)
|
||||
onMounted(() => {
|
||||
restoreDraft()
|
||||
modelsStore.load()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -861,6 +939,7 @@ onMounted(restoreDraft)
|
||||
v-model:process-type="processType"
|
||||
v-model:structured-options="structuredOptions"
|
||||
v-model:unstructured-options="unstructuredOptions"
|
||||
:generation-models="generationModels"
|
||||
/>
|
||||
|
||||
<SourceUploadStep
|
||||
|
||||
@@ -18,7 +18,6 @@ interface ProcessDetail {
|
||||
status: string
|
||||
description: string
|
||||
processType: string
|
||||
dataType: string
|
||||
sourceDataset: string
|
||||
outputDataset?: string
|
||||
outputDatasetId?: number
|
||||
@@ -75,8 +74,7 @@ const detailMap: Record<string, ProcessDetail> = {
|
||||
name: '客服问答数据清洗',
|
||||
status: 'completed',
|
||||
description: '清洗客服对话中的无效记录、重复问答和格式异常内容,生成可用于模型训练的标准数据集。',
|
||||
processType: '去重清洗',
|
||||
dataType: '结构化数据',
|
||||
processType: '结构化数据',
|
||||
sourceDataset: '客服对话原始集',
|
||||
outputDataset: '客服对话清洗集',
|
||||
outputDatasetId: 7,
|
||||
@@ -101,7 +99,7 @@ const detailMap: Record<string, ProcessDetail> = {
|
||||
},
|
||||
'492015': {
|
||||
id: '492015', name: '指令微调数据构造', status: 'running',
|
||||
description: '从通用语料中构造指令微调训练样本。', processType: '指令构造', dataType: '非结构化数据',
|
||||
description: '从通用语料中构造指令微调训练样本。', processType: '非结构化数据',
|
||||
sourceDataset: '通用语料库', outputDataset: 'SFT 指令集', outputDatasetId: 8, creator: '管理员',
|
||||
createTime: '2026-07-09 09:10:00', startTime: '2026-07-09 09:10:21', duration: '处理中', progress: 68,
|
||||
inputCount: 18500, outputCount: 8568, filteredCount: 425, duplicateCount: 192, errorCount: 8,
|
||||
@@ -112,7 +110,7 @@ const detailMap: Record<string, ProcessDetail> = {
|
||||
},
|
||||
'731948': {
|
||||
id: '731948', name: '敏感信息脱敏处理', status: 'pending', description: '识别并脱敏用户反馈数据中的敏感字段。',
|
||||
processType: '脱敏处理', dataType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
|
||||
processType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
|
||||
outputDatasetId: 9, creator: '管理员', createTime: '2026-07-09 16:45:00', duration: '等待执行', progress: 0,
|
||||
inputCount: 9340, outputCount: 0, filteredCount: 0, duplicateCount: 0, errorCount: 0,
|
||||
config: [
|
||||
@@ -122,7 +120,7 @@ const detailMap: Record<string, ProcessDetail> = {
|
||||
},
|
||||
'582012': {
|
||||
id: '582012', name: '多轮对话拼接', status: 'failed', description: '将单轮问答按会话标识拼接为多轮对话数据。',
|
||||
processType: '格式转换', dataType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
|
||||
processType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
|
||||
createTime: '2026-07-10 08:30:00', startTime: '2026-07-10 08:30:16', completeTime: '2026-07-10 08:31:04',
|
||||
duration: '48 秒', progress: 37, inputCount: 7520, outputCount: 2780, filteredCount: 24, duplicateCount: 0, errorCount: 1,
|
||||
config: [
|
||||
@@ -186,7 +184,6 @@ function resetPage() {
|
||||
<dl class="heading-meta">
|
||||
<div><dt>任务 ID</dt><dd>{{ detail.id }}</dd></div>
|
||||
<div><dt>处理类型</dt><dd>{{ detail.processType }}</dd></div>
|
||||
<div><dt>数据类型</dt><dd>{{ detail.dataType }}</dd></div>
|
||||
<div><dt>创建人</dt><dd>{{ detail.creator }}</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
@@ -5,17 +5,25 @@ import { ElMessage } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
|
||||
import type { ProcessType } from './create/types'
|
||||
|
||||
/** 数据处理任务类型 */
|
||||
interface DataProcessTask {
|
||||
id: number | string
|
||||
name: string
|
||||
status: string
|
||||
process_type: string
|
||||
process_type: ProcessType
|
||||
source_dataset: string
|
||||
output_dataset?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
const processTypeMap: Record<ProcessType, string> = {
|
||||
structured: '结构化数据',
|
||||
unstructured: '非结构化数据',
|
||||
external: '外来数据源拉取',
|
||||
}
|
||||
|
||||
// TODO: 接入真实接口前,先用本地 mock 数据
|
||||
const router = useRouter()
|
||||
const dataList = ref<DataProcessTask[]>([
|
||||
@@ -23,7 +31,7 @@ const dataList = ref<DataProcessTask[]>([
|
||||
id: 183921,
|
||||
name: '客服问答数据清洗',
|
||||
status: 'completed',
|
||||
process_type: '去重清洗',
|
||||
process_type: 'structured',
|
||||
source_dataset: '客服对话原始集',
|
||||
output_dataset: '客服对话清洗集',
|
||||
create_time: '2026-07-08 14:23:00',
|
||||
@@ -32,7 +40,7 @@ const dataList = ref<DataProcessTask[]>([
|
||||
id: 492015,
|
||||
name: '指令微调数据构造',
|
||||
status: 'running',
|
||||
process_type: '指令构造',
|
||||
process_type: 'unstructured',
|
||||
source_dataset: '通用语料库',
|
||||
output_dataset: 'SFT 指令集',
|
||||
create_time: '2026-07-09 09:10:00',
|
||||
@@ -41,7 +49,7 @@ const dataList = ref<DataProcessTask[]>([
|
||||
id: 731948,
|
||||
name: '敏感信息脱敏处理',
|
||||
status: 'pending',
|
||||
process_type: '脱敏处理',
|
||||
process_type: 'structured',
|
||||
source_dataset: '用户反馈数据',
|
||||
create_time: '2026-07-09 16:45:00',
|
||||
},
|
||||
@@ -49,7 +57,7 @@ const dataList = ref<DataProcessTask[]>([
|
||||
id: 582012,
|
||||
name: '多轮对话拼接',
|
||||
status: 'failed',
|
||||
process_type: '格式转换',
|
||||
process_type: 'structured',
|
||||
source_dataset: '单轮问答集',
|
||||
create_time: '2026-07-10 08:30:00',
|
||||
},
|
||||
@@ -98,7 +106,12 @@ function formatDateTime(value?: string) {
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="处理类型" align="center" width="140">
|
||||
<template #default="{ row }">{{ row.process_type || '-' }}</template>
|
||||
<template #default="{ row }">
|
||||
<el-tag v-if="row.process_type" size="small" type="info" effect="plain">
|
||||
{{ processTypeMap[row.process_type as ProcessType] || row.process_type }}
|
||||
</el-tag>
|
||||
<span v-else>-</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="源数据集" align="center" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.source_dataset || '-' }}</template>
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import type { ModelItem } from '@/types'
|
||||
import type { GenerationControlOptions } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
options: GenerationControlOptions
|
||||
models: ModelItem[]
|
||||
section: 'model' | 'quality'
|
||||
validationMessage?: string
|
||||
}>()
|
||||
|
||||
const advancedModelSettingsOpen = ref(false)
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:options': [value: GenerationControlOptions]
|
||||
}>()
|
||||
|
||||
function updateField<K extends keyof GenerationControlOptions>(
|
||||
field: K,
|
||||
value: GenerationControlOptions[K],
|
||||
) {
|
||||
emit('update:options', { ...props.options, [field]: value })
|
||||
}
|
||||
|
||||
function updateQualityRules(value: Array<string | number>) {
|
||||
const rules = value.filter((item): item is string => typeof item === 'string')
|
||||
emit('update:options', {
|
||||
...props.options,
|
||||
filterLowQuality: rules.includes('low_quality'),
|
||||
filterShortContent: rules.includes('short_content'),
|
||||
})
|
||||
}
|
||||
|
||||
const selectedQualityRules = () => [
|
||||
props.options.filterLowQuality ? 'low_quality' : '',
|
||||
props.options.filterShortContent ? 'short_content' : '',
|
||||
].filter(Boolean)
|
||||
|
||||
function sectionValidationMessage() {
|
||||
if (!props.validationMessage) return ''
|
||||
const isModelMessage = props.validationMessage.includes('数据生成模型')
|
||||
if (props.section === 'model') return isModelMessage ? props.validationMessage : ''
|
||||
return isModelMessage ? '' : props.validationMessage
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div v-if="section === 'model'" class="generation-config-group model-config-group">
|
||||
<div class="model-field">
|
||||
<div class="field-copy">
|
||||
<strong>数据生成模型 <em>*</em></strong>
|
||||
<small>选择负责生成问答、指令或结构化内容的大模型</small>
|
||||
</div>
|
||||
<el-select
|
||||
class="model-select"
|
||||
:model-value="options.generationModelId"
|
||||
filterable
|
||||
placeholder="请选择用于生成数据的模型"
|
||||
aria-label="数据生成模型"
|
||||
@update:model-value="updateField('generationModelId', $event)"
|
||||
>
|
||||
<el-option
|
||||
v-for="model in models"
|
||||
:key="model.id"
|
||||
:label="model.name"
|
||||
:value="model.id"
|
||||
>
|
||||
<div class="model-option">
|
||||
<span>{{ model.name }}</span>
|
||||
<small>{{ model.model_source === 'api' ? '在线模型' : '本地模型' }}</small>
|
||||
</div>
|
||||
</el-option>
|
||||
<template #empty>
|
||||
<div class="model-empty">暂无可用的大模型,请先在模型管理中添加</div>
|
||||
</template>
|
||||
</el-select>
|
||||
</div>
|
||||
|
||||
<div class="model-field">
|
||||
<div class="field-copy">
|
||||
<strong>默认提示语</strong>
|
||||
<small>用于约束生成内容的格式、语气和完整性,可按任务需要修改</small>
|
||||
</div>
|
||||
<el-input
|
||||
class="prompt-input"
|
||||
:model-value="options.generationPrompt"
|
||||
type="textarea"
|
||||
:rows="4"
|
||||
maxlength="500"
|
||||
show-word-limit
|
||||
resize="vertical"
|
||||
placeholder="请输入模型生成内容时需要遵循的要求"
|
||||
aria-label="模型默认提示语"
|
||||
@update:model-value="updateField('generationPrompt', $event)"
|
||||
/>
|
||||
<div class="prompt-variables-hint">
|
||||
提示:可在文本中通过 <code>{{ content }}</code> 引用当前正在处理的数据内容。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
class="advanced-settings-toggle"
|
||||
:aria-expanded="advancedModelSettingsOpen"
|
||||
@click="advancedModelSettingsOpen = !advancedModelSettingsOpen"
|
||||
>
|
||||
<span>
|
||||
<strong>高级设置</strong>
|
||||
<small>温度、最大输出长度与格式约束</small>
|
||||
</span>
|
||||
<i class="fa" :class="advancedModelSettingsOpen ? 'fa-chevron-up' : 'fa-chevron-down'" />
|
||||
</button>
|
||||
|
||||
<div v-if="advancedModelSettingsOpen" class="advanced-settings-panel">
|
||||
<div class="advanced-settings-grid">
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">生成温度 (Temperature)</span>
|
||||
<el-slider
|
||||
:model-value="options.temperature"
|
||||
:min="0"
|
||||
:max="1"
|
||||
:step="0.1"
|
||||
show-input
|
||||
@update:model-value="updateField('temperature', Number($event) || 0)"
|
||||
/>
|
||||
<small>较低值输出更确定、保守,较高值输出更多样、有创造性</small>
|
||||
</label>
|
||||
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">最大输出长度 (Max Tokens)</span>
|
||||
<span class="unit-input">
|
||||
<el-input-number
|
||||
:model-value="options.maxTokens"
|
||||
:min="100"
|
||||
:max="8192"
|
||||
:step="256"
|
||||
controls-position="right"
|
||||
@update:model-value="updateField('maxTokens', Number($event) || 1024)"
|
||||
/>
|
||||
<span>Token</span>
|
||||
</span>
|
||||
<small>限制单次生成的最大内容长度,防止过度发散</small>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="json-mode-row">
|
||||
<div class="field-copy">
|
||||
<strong>强制 JSON 格式输出</strong>
|
||||
<small>要求大模型返回纯 JSON 对象,有助于提高下游结构化提取的稳定性</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="options.jsonMode"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateField('jsonMode', Boolean($event))"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
|
||||
{{ sectionValidationMessage() }}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div v-else class="generation-config-group quality-config-group">
|
||||
<div class="quality-switch-row">
|
||||
<div class="config-group-heading">
|
||||
<div>
|
||||
<h4>质量筛选</h4>
|
||||
<p>生成后自动过滤不符合要求的内容,不影响源数据预处理</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="options.qualityFilterEnabled"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
aria-label="质量筛选"
|
||||
@update:model-value="updateField('qualityFilterEnabled', Boolean($event))"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div v-if="options.qualityFilterEnabled" class="quality-rule-panel">
|
||||
<el-checkbox-group
|
||||
:model-value="selectedQualityRules()"
|
||||
class="quality-rule-list"
|
||||
@update:model-value="updateQualityRules"
|
||||
>
|
||||
<el-checkbox value="low_quality">
|
||||
<span class="rule-copy">
|
||||
<strong>过滤低质量内容</strong>
|
||||
<small>按相关性、完整性和可读性过滤明显不合格的生成结果</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
<el-checkbox value="short_content">
|
||||
<span class="rule-copy">
|
||||
<strong>过滤过短内容</strong>
|
||||
<small>过滤正文长度低于指定字数的生成结果</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
|
||||
<label v-if="options.filterShortContent" class="minimum-length-field">
|
||||
<span>最少字数</span>
|
||||
<span class="minimum-length-input">
|
||||
<el-input-number
|
||||
:model-value="options.minOutputLength"
|
||||
:min="1"
|
||||
:max="1000"
|
||||
:step="5"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="生成内容最少字数"
|
||||
@update:model-value="updateField('minOutputLength', Number($event) || 1)"
|
||||
/>
|
||||
<span>字</span>
|
||||
</span>
|
||||
<small>少于该字数的生成结果将被过滤</small>
|
||||
</label>
|
||||
|
||||
<p v-if="!options.filterLowQuality && !options.filterShortContent" class="quality-rule-warning" role="alert">
|
||||
请至少选择一项质量筛选规则
|
||||
</p>
|
||||
</div>
|
||||
<p v-if="sectionValidationMessage()" class="generation-validation-message" role="alert">
|
||||
{{ sectionValidationMessage() }}
|
||||
</p>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.generation-config-group {
|
||||
padding: 14px;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.config-group-heading {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
|
||||
h4 { margin: 0; color: #344054; font-size: 13px; font-weight: 600; }
|
||||
p { margin: 4px 0 0; color: #8a93a3; font-size: 12px; line-height: 1.5; }
|
||||
}
|
||||
|
||||
.model-config-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.model-field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.field-copy {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
em { color: var(--el-color-danger); font-style: normal; }
|
||||
}
|
||||
|
||||
.model-select { width: 100%; }
|
||||
.prompt-input { width: 100%; }
|
||||
.model-option { width: 100%; display: flex; align-items: center; justify-content: space-between; gap: 16px; }
|
||||
.model-option small { color: #98a2b3; font-size: 11px; }
|
||||
.model-empty { padding: 12px; color: #8a93a3; font-size: 12px; text-align: center; }
|
||||
|
||||
.prompt-variables-hint {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
code {
|
||||
padding: 2px 4px;
|
||||
color: #5b50f2;
|
||||
background: #f0f0ff;
|
||||
border-radius: 4px;
|
||||
font-family: inherit;
|
||||
}
|
||||
}
|
||||
|
||||
.advanced-settings-toggle {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
min-height: 48px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-top: 4px;
|
||||
padding: 10px 14px;
|
||||
color: #344054;
|
||||
text-align: left;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||||
|
||||
&:hover {
|
||||
background: #f8f7ff;
|
||||
border-color: #b7b2f7;
|
||||
}
|
||||
|
||||
> span {
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
}
|
||||
|
||||
strong {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
i {
|
||||
flex: 0 0 auto;
|
||||
color: #7b8495;
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.advanced-settings-panel {
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
margin-top: -8px;
|
||||
padding: 16px;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.advanced-settings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.config-field {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 8px;
|
||||
min-width: 0;
|
||||
padding: 14px;
|
||||
color: #344054;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
|
||||
> small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
:deep(.el-slider) {
|
||||
margin: 0 8px;
|
||||
}
|
||||
}
|
||||
|
||||
.config-field-label {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.unit-input {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
color: #7b8495;
|
||||
font-size: 12px;
|
||||
|
||||
:deep(.el-input-number) {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.json-mode-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
padding: 14px;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.quality-switch-row { display: flex; align-items: center; justify-content: space-between; gap: 20px; }
|
||||
.quality-rule-panel { margin-top: 12px; padding-top: 12px; border-top: 1px solid #e7eaf0; }
|
||||
.quality-rule-list { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
|
||||
.quality-rule-list :deep(.el-checkbox) {
|
||||
min-height: 62px; height: auto; margin-right: 0; padding: 10px 12px;
|
||||
align-items: flex-start; border: 1px solid #e2e5ec; border-radius: 7px; background: #fff;
|
||||
}
|
||||
.quality-rule-list :deep(.el-checkbox__input) { margin-top: 3px; }
|
||||
.quality-rule-list :deep(.el-checkbox__label) { min-width: 0; padding-left: 9px; white-space: normal; }
|
||||
.rule-copy { display: grid; gap: 3px; }
|
||||
.rule-copy strong { color: #344054; font-size: 12px; font-weight: 600; }
|
||||
.rule-copy small { color: #8a93a3; font-size: 11px; line-height: 1.5; }
|
||||
|
||||
.minimum-length-field {
|
||||
margin-top: 12px; display: grid; grid-template-columns: 110px 220px minmax(0, 1fr);
|
||||
align-items: center; gap: 12px; color: #475467; font-size: 12px;
|
||||
}
|
||||
.minimum-length-input { display: flex; align-items: center; gap: 8px; }
|
||||
.minimum-length-input :deep(.el-input-number) { width: 180px; }
|
||||
.minimum-length-field > small { color: #8a93a3; }
|
||||
.quality-rule-warning { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
|
||||
.generation-validation-message { margin: 10px 0 0; color: #dc2626; font-size: 12px; }
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.advanced-settings-grid { grid-template-columns: 1fr; }
|
||||
.quality-rule-list { grid-template-columns: 1fr; }
|
||||
.minimum-length-field { grid-template-columns: 1fr; }
|
||||
}
|
||||
</style>
|
||||
@@ -97,7 +97,7 @@ const emit = defineEmits<{
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 15px 17px;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #e8ebf0;
|
||||
|
||||
strong {
|
||||
|
||||
@@ -350,7 +350,7 @@ function lineRange(item: PreviewItem) {
|
||||
min-height: 52px;
|
||||
padding: 0 15px;
|
||||
color: #313949;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #e8ebf0;
|
||||
font-size: 13px;
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ function selectRelative(offset: number) {
|
||||
min-height: 52px;
|
||||
padding: 0 15px;
|
||||
color: #344054;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #e8ebf0;
|
||||
font-size: 13px;
|
||||
|
||||
|
||||
@@ -397,7 +397,7 @@ function formatSize(size: number) {
|
||||
width: 100%;
|
||||
min-height: 154px;
|
||||
padding: 32px 20px;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border-color: #dfe3ea;
|
||||
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||||
|
||||
@@ -435,7 +435,7 @@ function formatSize(size: number) {
|
||||
padding: 10px 14px;
|
||||
color: #5f6878;
|
||||
font-size: 12px;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border-bottom: 1px solid #edf0f5;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,12 +4,15 @@ import type { FormInstance, FormRules } from 'element-plus'
|
||||
import type {
|
||||
ChunkMethod,
|
||||
DatasetSplitOptions,
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
ProcessType,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
import type { ModelItem } from '@/types'
|
||||
|
||||
const props = defineProps<{
|
||||
name: string
|
||||
@@ -17,6 +20,7 @@ const props = defineProps<{
|
||||
processType: ProcessType
|
||||
structuredOptions: StructuredProcessOptions
|
||||
unstructuredOptions: UnstructuredProcessOptions
|
||||
generationModels: ModelItem[]
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -63,6 +67,10 @@ function updateStructuredField<K extends keyof StructuredProcessOptions>(
|
||||
emit('update:structuredOptions', { ...props.structuredOptions, [field]: value })
|
||||
}
|
||||
|
||||
function updateStructuredGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:structuredOptions', { ...props.structuredOptions, ...value })
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = value.filter(
|
||||
@@ -89,6 +97,10 @@ function updateUnstructuredField<K extends keyof UnstructuredProcessOptions>(
|
||||
emit('update:unstructuredOptions', { ...props.unstructuredOptions, [field]: value })
|
||||
}
|
||||
|
||||
function updateUnstructuredGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:unstructuredOptions', { ...props.unstructuredOptions, ...value })
|
||||
}
|
||||
|
||||
function updateSmartPreprocess(value: string | number | boolean) {
|
||||
const enabled = Boolean(value)
|
||||
const remainingOptions = props.unstructuredOptions.preprocessOptions.filter(
|
||||
@@ -143,6 +155,7 @@ function updateUnstructuredDatasetSplit(field: keyof DatasetSplitOptions, value:
|
||||
|
||||
const formRef = ref<FormInstance>()
|
||||
const advancedChunkSettingsOpen = ref(props.unstructuredOptions.chunkMethod === 'custom')
|
||||
const validationAttempted = ref(false)
|
||||
const formModel = computed(() => ({
|
||||
name: props.name,
|
||||
processType: props.processType,
|
||||
@@ -172,6 +185,25 @@ const preserveSpecialContentEnabled = computed(() => (
|
||||
&& props.unstructuredOptions.preserveLists
|
||||
))
|
||||
|
||||
const activeGenerationOptions = computed<GenerationControlOptions | null>(() => {
|
||||
if (props.processType === 'structured') return props.structuredOptions
|
||||
if (props.processType === 'unstructured') return props.unstructuredOptions
|
||||
return null
|
||||
})
|
||||
|
||||
const generationValidationMessage = computed(() => {
|
||||
const options = activeGenerationOptions.value
|
||||
if (!options) return ''
|
||||
if (options.generationModelId === '') return '请选择数据生成模型'
|
||||
if (options.qualityFilterEnabled && !options.filterLowQuality && !options.filterShortContent) {
|
||||
return '开启质量筛选后,请至少选择一项筛选规则'
|
||||
}
|
||||
if (options.qualityFilterEnabled && options.filterShortContent && options.minOutputLength < 1) {
|
||||
return '最少字数必须大于 0'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const chunkValidationMessage = computed(() => {
|
||||
if (props.unstructuredOptions.chunkOverlap >= props.unstructuredOptions.chunkSize) {
|
||||
return '重叠长度必须小于切片长度'
|
||||
@@ -208,6 +240,7 @@ const rules: FormRules = {
|
||||
}
|
||||
|
||||
async function validate() {
|
||||
validationAttempted.value = true
|
||||
if (!formRef.value) return false
|
||||
try {
|
||||
await formRef.value.validate()
|
||||
@@ -219,6 +252,7 @@ async function validate() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (generationValidationMessage.value) return false
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
@@ -344,6 +378,13 @@ defineExpose({ validate })
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="structuredOptions"
|
||||
:models="generationModels"
|
||||
section="quality"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateStructuredGenerationOptions"
|
||||
/>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>语义丰富表达</strong>
|
||||
@@ -435,6 +476,24 @@ defineExpose({ validate })
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section model-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>大模型</h3>
|
||||
<p>选择数据生成模型,并设置模型输出内容的要求</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="structuredOptions"
|
||||
:models="generationModels"
|
||||
section="model"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateStructuredGenerationOptions"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<template v-if="processType === 'unstructured'">
|
||||
@@ -625,6 +684,13 @@ defineExpose({ validate })
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="unstructuredOptions"
|
||||
:models="generationModels"
|
||||
section="quality"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateUnstructuredGenerationOptions"
|
||||
/>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>语义丰富表达</strong>
|
||||
@@ -723,6 +789,24 @@ defineExpose({ validate })
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section model-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>大模型</h3>
|
||||
<p>选择数据生成模型,并设置模型输出内容的要求</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="unstructuredOptions"
|
||||
:models="generationModels"
|
||||
section="model"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateUnstructuredGenerationOptions"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
</el-form>
|
||||
@@ -843,7 +927,7 @@ defineExpose({ validate })
|
||||
min-width: 0;
|
||||
padding: 14px;
|
||||
color: #344054;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
|
||||
@@ -902,7 +986,7 @@ defineExpose({ validate })
|
||||
padding: 10px 14px;
|
||||
color: #344054;
|
||||
text-align: left;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
@@ -946,7 +1030,7 @@ defineExpose({ validate })
|
||||
gap: 12px;
|
||||
margin-top: 8px;
|
||||
padding: 12px;
|
||||
background: #fbfcfe;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
|
||||
@@ -1154,3 +1238,4 @@ defineExpose({ validate })
|
||||
|
||||
}
|
||||
</style>
|
||||
GenerationControlOptions,
|
||||
|
||||
@@ -501,7 +501,11 @@ export function createResults(
|
||||
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||||
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||||
|
||||
return items.flatMap((item, index) => {
|
||||
const generatedResults = items.flatMap((item, index) => {
|
||||
if (options?.qualityFilterEnabled && options.filterLowQuality) {
|
||||
if (item.status === 'invalid' || !item.editedContent.trim()) return []
|
||||
}
|
||||
|
||||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||||
const baseInstruction = firstLine.replace(/^问[::]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||
@@ -525,4 +529,12 @@ export function createResults(
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
if (!options?.qualityFilterEnabled) return generatedResults
|
||||
|
||||
return generatedResults.filter((result) => {
|
||||
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
|
||||
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,7 +16,19 @@ export interface DatasetSplitOptions {
|
||||
test: number
|
||||
}
|
||||
|
||||
export interface StructuredProcessOptions {
|
||||
export interface GenerationControlOptions {
|
||||
generationModelId: string | number | ''
|
||||
generationPrompt: string
|
||||
temperature: number
|
||||
maxTokens: number
|
||||
jsonMode: boolean
|
||||
qualityFilterEnabled: boolean
|
||||
filterLowQuality: boolean
|
||||
filterShortContent: boolean
|
||||
minOutputLength: number
|
||||
}
|
||||
|
||||
export interface StructuredProcessOptions extends GenerationControlOptions {
|
||||
preprocessOptions: PreprocessOption[]
|
||||
semanticEnrichment: boolean
|
||||
qaPairsPerRow: number
|
||||
@@ -34,7 +46,7 @@ export type UnstructuredPreprocessOption =
|
||||
|
||||
export type ChunkMethod = 'semantic' | 'heading' | 'fixed' | 'custom'
|
||||
|
||||
export interface UnstructuredProcessOptions {
|
||||
export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
preprocessOptions: UnstructuredPreprocessOption[]
|
||||
chunkMethod: ChunkMethod
|
||||
chunkSize: number
|
||||
|
||||
Reference in New Issue
Block a user