diff --git a/frontend/scripts/regression-eval-create-wizard.mjs b/frontend/scripts/regression-eval-create-wizard.mjs index 1130fba..51d5c24 100644 --- a/frontend/scripts/regression-eval-create-wizard.mjs +++ b/frontend/scripts/regression-eval-create-wizard.mjs @@ -11,6 +11,8 @@ const [ createSource, taskStepSource, ruleStepSource, + basicMetricStepSource, + startStepSource, dimensionFieldsSource, listSource, routerSource, @@ -19,6 +21,8 @@ const [ readFile(path.join(evalViewDir, 'EvalCreateView.vue'), 'utf8'), readFile(path.join(evalViewDir, 'create/EvalTaskSetupStep.vue'), 'utf8'), readFile(path.join(evalViewDir, 'create/EvalRuleSetupStep.vue'), 'utf8'), + readFile(path.join(evalViewDir, 'create/BasicMetricSetupStep.vue'), 'utf8'), + readFile(path.join(evalViewDir, 'create/StartEvalStep.vue'), 'utf8'), readFile(path.join(evalViewDir, 'create/DimensionFormFields.vue'), 'utf8'), readFile(path.join(evalViewDir, 'EvalView.vue'), 'utf8'), readFile(path.join(sourceRoot, 'router/index.ts'), 'utf8'), @@ -74,21 +78,43 @@ assert.match( assert.match(createSource, /EvalTaskSetupStep/, '向导缺少任务配置子组件') assert.match(createSource, /EvalRuleSetupStep/, '向导缺少评测规则子组件') +assert.match(createSource, /BasicMetricSetupStep/, '向导缺少基础评测指标子组件') +assert.match(createSource, /StartEvalStep/, '向导缺少开始评测确认子组件') assert.match( createSource, - /任务配置[\s\S]*?评测规则/, - '评测创建向导必须按“任务配置、评测规则”定义两个步骤', + /任务配置[\s\S]*?大模型评测指标[\s\S]*?基础评测指标[\s\S]*?开始评测/, + '评测创建向导必须按“任务配置、大模型评测指标、基础评测指标、开始评测”定义四个步骤', ) assert.match( ruleStepSource, /:show-description=["']false["']/, - '新建评测任务第二步不应显示重复的规则描述输入框', + '大模型评测指标步骤不应显示重复的规则描述输入框', +) +assert.match( + ruleStepSource, + /:show-status-settings=["']false["']/, + '大模型评测指标步骤不应显示维度管理状态字段', +) +assert.match( + ruleStepSource, + /LLM_METRIC_TYPES[^=]*=\s*\[['"]classification['"],\s*['"]metric['"]\][\s\S]*?:allowed-types=["']LLM_METRIC_TYPES["']/, + '大模型评测指标步骤只应提供依赖大模型的分类与评分指标', ) assert.match( dimensionFieldsSource, /v-if=["']showDescription["'][^>]*label=["']描述["']/, '共享维度字段必须支持按场景隐藏描述输入框', ) +assert.match( + dimensionFieldsSource, + /score_min[\s\S]*?score_max[\s\S]*?pass_threshold[\s\S]*?Math\.min[\s\S]*?Math\.max/, + '指标型评分区间变化时必须把通过阈值约束在合法范围内', +) +assert.match( + dimensionFieldsSource, + /prop=["']score_min["'][\s\S]*?:max=["'][^"']*score_max[^"']*["'][\s\S]*?prop=["']score_max["'][\s\S]*?:min=["'][^"']*score_min[^"']*["']/, + '评分最小值和最大值输入必须具备交叉边界约束', +) assert.match(createSource, /currentStep\s*=\s*ref\(0\)/, '评测创建向导缺少当前步骤状态') assert.match(createSource, /class=["']custom-wizard-steps["']/, '评测向导未复用数据处理的自定义步骤条结构') assert.match(createSource, /class=["']step-connector["']/, '评测向导步骤条缺少连接线') @@ -102,8 +128,18 @@ assert.match( ) assert.match( createSource, - /\s*开始评测\s* get('/model-eval') @@ -11,7 +11,7 @@ export const getEvalDetail = (id: string | number) => get(`/mode export const deleteEval = (id: string | number) => del(`/model-eval/${id}`) /** 启动评测 */ -export const startEval = (data: any) => post('/model-eval/start', data) +export const startEval = (data: StartEvalPayload) => post('/model-eval/start', data) /** 评测维度列表 */ export const getDimensionList = () => get('/dimension') diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index a4174df..f3dd501 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -224,6 +224,34 @@ export interface EvalTask { create_time?: string } +/** 启动评测时提交的可选基础指标配置。 */ +export interface BasicEvalMetricsConfig { + bleu: { + enabled: boolean + ngram: number + } + rouge: { + enabled: boolean + methods: string[] + } + cosine: { + enabled: boolean + } + output_precision: number +} + +export interface StartEvalPayload { + eval_task_name: string + eval_type: EvalType + model_id: string | number + gpu_id: string | number + dataset_id: string | number + dimension_id: string | number + data_source: 'dataset' | 'inference' + leaderboard: boolean + basic_metrics: BasicEvalMetricsConfig +} + /** 单个测试样本的评测结果 */ export interface EvalSampleResult { id: number | string diff --git a/frontend/src/views/eval/EvalCreateView.vue b/frontend/src/views/eval/EvalCreateView.vue index e9b2601..7da326e 100644 --- a/frontend/src/views/eval/EvalCreateView.vue +++ b/frontend/src/views/eval/EvalCreateView.vue @@ -5,7 +5,9 @@ import { ElMessage } from 'element-plus' import PageCard from '@/components/PageCard.vue' import EvalTaskSetupStep, { type EvalTaskSetupDraft } from './create/EvalTaskSetupStep.vue' import EvalRuleSetupStep, { type EvalRuleSetupDraft } from './create/EvalRuleSetupStep.vue' -import { createDimension, getDimensionList, startEval } from '@/api/modules/eval' +import BasicMetricSetupStep, { type BasicMetricSetupDraft } from './create/BasicMetricSetupStep.vue' +import StartEvalStep from './create/StartEvalStep.vue' +import { createDimension, startEval } from '@/api/modules/eval' import { getTrainedModels, getModelList } from '@/api/modules/model' import { getDatasetList } from '@/api/modules/dataset' import { getSystemInfo } from '@/api/modules/system' @@ -19,15 +21,17 @@ const submitting = ref(false) const currentStep = ref(0) const taskStepRef = ref() const ruleStepRef = ref() +const basicMetricStepRef = ref() const WIZARD_STEPS = [ { title: '任务配置', description: '选择模型、算力与评测数据' }, - { title: '评测规则', description: '选择或创建评测维度' }, + { title: '大模型评测指标', description: '配置评测模型、方式与标准' }, + { title: '基础评测指标', description: '选择 BLEU、ROUGE 等参考指标' }, + { title: '开始评测', description: '确认配置并启动评测任务' }, ] as const const trainedModels = ref([]) const evalDatasets = ref([]) -const dimensions = ref([]) const evalModels = ref([]) const gpus = ref([]) const createdDimensionId = ref('') @@ -42,26 +46,31 @@ const taskForm = ref({ }) const ruleForm = ref({ - ruleMode: 'new', - dimension_id: '', - newDimension: { - type: '', - description: '', - eval_model: '', - eval_method: '', - eval_prompt: '', - is_active: true, - is_default: false, - bleu_n: 1, - output_precision: 3, - score_min: 0, - score_max: 5, - pass_threshold: 3, - }, + type: '', + description: '', + eval_model: '', + eval_method: '', + eval_prompt: '', + is_active: true, + is_default: false, + bleu_n: 1, + output_precision: 3, + score_min: 0, + score_max: 5, + pass_threshold: 3, +}) + +const basicMetricForm = ref({ + bleu_enabled: false, + bleu_n: 4, + rouge_enabled: false, + rouge_methods: ['rouge_1', 'rouge_2'], + cosine_enabled: false, + output_precision: 3, }) watch( - () => ruleForm.value.newDimension, + ruleForm, () => { createdDimensionId.value = '' }, @@ -72,20 +81,18 @@ async function loadData() { loading.value = true const results = await Promise.allSettled([ getTrainedModels(), - getDimensionList(), getDatasetList(), getSystemInfo(), getModelList(), ]) if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || [] - if (results[1].status === 'fulfilled') dimensions.value = results[1].value || [] - if (results[2].status === 'fulfilled') { - evalDatasets.value = (results[2].value || []).filter((dataset) => dataset.type === 'eval') + if (results[1].status === 'fulfilled') { + evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval') } - if (results[3].status === 'fulfilled') gpus.value = results[3].value?.gpu || [] - if (results[4].status === 'fulfilled') { - evalModels.value = (results[4].value || []).filter((model) => model.purpose === 'evaluation') + if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || [] + if (results[3].status === 'fulfilled') { + evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation') } const failedCount = results.filter((result) => result.status === 'rejected').length @@ -95,25 +102,19 @@ async function loadData() { loading.value = false } - - function buildDimensionPayload() { - const draft = ruleForm.value.newDimension + const draft = ruleForm.value const payload: Partial = { name: `Custom_Dim_${Date.now()}`, type: draft.type, description: draft.description, - eval_model: draft.type === 'text_similarity' ? undefined : draft.eval_model, + eval_model: draft.eval_model, eval_method: draft.eval_method, - eval_prompt: draft.type === 'text_similarity' ? undefined : draft.eval_prompt, + eval_prompt: draft.eval_prompt, is_active: draft.is_active, is_default: draft.is_default, create_time: new Date().toISOString(), } - if (draft.type === 'text_similarity') { - payload.bleu_n = draft.bleu_n - payload.output_precision = draft.output_precision - } if (draft.type === 'metric') { payload.score_min = draft.score_min payload.score_max = draft.score_max @@ -123,8 +124,6 @@ function buildDimensionPayload() { } async function resolveDimensionId() { - if (ruleForm.value.ruleMode === 'baseline') return '' - if (ruleForm.value.ruleMode === 'existing') return ruleForm.value.dimension_id if (createdDimensionId.value !== '') return createdDimensionId.value const created = await createDimension(buildDimensionPayload()) @@ -137,24 +136,32 @@ async function resolveDimensionId() { async function handleSubmit() { if (loading.value || submitting.value) return - const ruleValid = await ruleStepRef.value?.validate() - if (!ruleValid) { - ElMessage.warning('请检查并完善评测规则') - return - } - submitting.value = true try { const dimensionId = await resolveDimensionId() await startEval({ eval_task_name: taskForm.value.eval_task_name, - eval_type: ruleForm.value.ruleMode === 'baseline' ? 'baseline' : 'custom', + eval_type: 'custom', model_id: taskForm.value.model_id, gpu_id: taskForm.value.gpu_id, dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '', dimension_id: dimensionId, data_source: taskForm.value.data_source, leaderboard: taskForm.value.leaderboard, + basic_metrics: { + bleu: { + enabled: basicMetricForm.value.bleu_enabled, + ngram: basicMetricForm.value.bleu_n, + }, + rouge: { + enabled: basicMetricForm.value.rouge_enabled, + methods: basicMetricForm.value.rouge_methods, + }, + cosine: { + enabled: basicMetricForm.value.cosine_enabled, + }, + output_precision: basicMetricForm.value.output_precision, + }, }) ElMessage.success('评测任务已创建并启动') router.push('/model-eval') @@ -168,17 +175,33 @@ async function handleSubmit() { async function handleNext() { if (loading.value || submitting.value) return - const taskValid = await taskStepRef.value?.validate() - if (!taskValid) { - ElMessage.warning('请检查并完善任务配置') - return + if (currentStep.value === 0) { + const taskValid = await taskStepRef.value?.validate() + if (!taskValid) { + ElMessage.warning('请检查并完善任务配置') + return + } } - currentStep.value = 1 + if (currentStep.value === 1) { + const ruleValid = await ruleStepRef.value?.validate() + if (!ruleValid) { + ElMessage.warning('请检查并完善大模型评测指标') + return + } + } + if (currentStep.value === 2) { + const basicMetricValid = await basicMetricStepRef.value?.validate() + if (!basicMetricValid) { + ElMessage.warning('请检查并完善基础评测指标') + return + } + } + currentStep.value = Math.min(currentStep.value + 1, WIZARD_STEPS.length - 1) } function handleBack() { if (submitting.value) return - currentStep.value = 0 + currentStep.value = Math.max(currentStep.value - 1, 0) } function handleCancel() { @@ -189,7 +212,10 @@ onMounted(loadData) @@ -305,7 +348,7 @@ onMounted(loadData) display: flex; align-items: center; justify-content: space-between; - max-width: 760px; + max-width: 1120px; margin: 0 auto; padding: 0 20px; } @@ -319,7 +362,7 @@ onMounted(loadData) .step-connector { flex: 1; height: 2px; - margin: 0 20px; + margin: 0 12px; background: #e2e8f0; transition: background-color 0.2s ease; } @@ -414,6 +457,12 @@ onMounted(loadData) margin-left: 6px; } +@media (max-width: 1100px) { + .step-description { + display: none; + } +} + @media (max-width: 760px) { .wizard-main { padding: 24px 20px; @@ -424,10 +473,10 @@ onMounted(loadData) } .step-connector { - margin: 0 10px; + margin: 0 8px; } - .step-description { + .step-text { display: none; } diff --git a/frontend/src/views/eval/create/DimensionFormFields.vue b/frontend/src/views/eval/create/DimensionFormFields.vue index 346f96b..eb06a23 100644 --- a/frontend/src/views/eval/create/DimensionFormFields.vue +++ b/frontend/src/views/eval/create/DimensionFormFields.vue @@ -21,19 +21,28 @@ export interface DimensionFormDraft { pass_threshold: number } -withDefaults( +const props = withDefaults( defineProps<{ evalModels: ModelItem[] showDescription?: boolean + showStatusSettings?: boolean + allowedTypes?: DimensionType[] }>(), { showDescription: true, + showStatusSettings: true, + allowedTypes: () => ['classification', 'metric', 'text_similarity'], }, ) const form = defineModel({ required: true }) +const SCORE_STEP = 0.5 -const typeOptions = Object.entries(DIMENSION_TYPE_MAP).map(([value, label]) => ({ value, label })) +const typeOptions = computed(() => + Object.entries(DIMENSION_TYPE_MAP) + .filter(([value]) => props.allowedTypes.includes(value as DimensionType)) + .map(([value, label]) => ({ value, label })), +) const currentMethods = computed(() => (form.value.type ? EVAL_METHODS[form.value.type] || [] : [])) /** @@ -65,6 +74,23 @@ watch( }, { flush: 'sync' }, ) + +/** 始终保持评分区间合法,并把通过阈值收敛到当前区间内。 */ +watch( + () => [form.value.type, form.value.score_min, form.value.score_max] as const, + ([type, scoreMin, scoreMax]) => { + if (type !== 'metric') return + if (scoreMax <= scoreMin) { + form.value.score_max = scoreMin + SCORE_STEP + return + } + form.value.pass_threshold = Math.min( + Math.max(form.value.pass_threshold, scoreMin), + scoreMax, + ) + }, + { flush: 'sync' }, +) - - - - - - +