dev #1
@@ -11,6 +11,8 @@ const [
|
|||||||
createSource,
|
createSource,
|
||||||
taskStepSource,
|
taskStepSource,
|
||||||
ruleStepSource,
|
ruleStepSource,
|
||||||
|
basicMetricStepSource,
|
||||||
|
startStepSource,
|
||||||
dimensionFieldsSource,
|
dimensionFieldsSource,
|
||||||
listSource,
|
listSource,
|
||||||
routerSource,
|
routerSource,
|
||||||
@@ -19,6 +21,8 @@ const [
|
|||||||
readFile(path.join(evalViewDir, 'EvalCreateView.vue'), 'utf8'),
|
readFile(path.join(evalViewDir, 'EvalCreateView.vue'), 'utf8'),
|
||||||
readFile(path.join(evalViewDir, 'create/EvalTaskSetupStep.vue'), 'utf8'),
|
readFile(path.join(evalViewDir, 'create/EvalTaskSetupStep.vue'), 'utf8'),
|
||||||
readFile(path.join(evalViewDir, 'create/EvalRuleSetupStep.vue'), 'utf8'),
|
readFile(path.join(evalViewDir, 'create/EvalRuleSetupStep.vue'), 'utf8'),
|
||||||
|
readFile(path.join(evalViewDir, 'create/BasicMetricSetupStep.vue'), 'utf8'),
|
||||||
|
readFile(path.join(evalViewDir, 'create/StartEvalStep.vue'), 'utf8'),
|
||||||
readFile(path.join(evalViewDir, 'create/DimensionFormFields.vue'), 'utf8'),
|
readFile(path.join(evalViewDir, 'create/DimensionFormFields.vue'), 'utf8'),
|
||||||
readFile(path.join(evalViewDir, 'EvalView.vue'), 'utf8'),
|
readFile(path.join(evalViewDir, 'EvalView.vue'), 'utf8'),
|
||||||
readFile(path.join(sourceRoot, 'router/index.ts'), 'utf8'),
|
readFile(path.join(sourceRoot, 'router/index.ts'), 'utf8'),
|
||||||
@@ -74,21 +78,43 @@ assert.match(
|
|||||||
|
|
||||||
assert.match(createSource, /EvalTaskSetupStep/, '向导缺少任务配置子组件')
|
assert.match(createSource, /EvalTaskSetupStep/, '向导缺少任务配置子组件')
|
||||||
assert.match(createSource, /EvalRuleSetupStep/, '向导缺少评测规则子组件')
|
assert.match(createSource, /EvalRuleSetupStep/, '向导缺少评测规则子组件')
|
||||||
|
assert.match(createSource, /BasicMetricSetupStep/, '向导缺少基础评测指标子组件')
|
||||||
|
assert.match(createSource, /StartEvalStep/, '向导缺少开始评测确认子组件')
|
||||||
assert.match(
|
assert.match(
|
||||||
createSource,
|
createSource,
|
||||||
/任务配置[\s\S]*?评测规则/,
|
/任务配置[\s\S]*?大模型评测指标[\s\S]*?基础评测指标[\s\S]*?开始评测/,
|
||||||
'评测创建向导必须按“任务配置、评测规则”定义两个步骤',
|
'评测创建向导必须按“任务配置、大模型评测指标、基础评测指标、开始评测”定义四个步骤',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
ruleStepSource,
|
ruleStepSource,
|
||||||
/:show-description=["']false["']/,
|
/:show-description=["']false["']/,
|
||||||
'新建评测任务第二步不应显示重复的规则描述输入框',
|
'大模型评测指标步骤不应显示重复的规则描述输入框',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
ruleStepSource,
|
||||||
|
/:show-status-settings=["']false["']/,
|
||||||
|
'大模型评测指标步骤不应显示维度管理状态字段',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
ruleStepSource,
|
||||||
|
/LLM_METRIC_TYPES[^=]*=\s*\[['"]classification['"],\s*['"]metric['"]\][\s\S]*?:allowed-types=["']LLM_METRIC_TYPES["']/,
|
||||||
|
'大模型评测指标步骤只应提供依赖大模型的分类与评分指标',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
dimensionFieldsSource,
|
dimensionFieldsSource,
|
||||||
/v-if=["']showDescription["'][^>]*label=["']描述["']/,
|
/v-if=["']showDescription["'][^>]*label=["']描述["']/,
|
||||||
'共享维度字段必须支持按场景隐藏描述输入框',
|
'共享维度字段必须支持按场景隐藏描述输入框',
|
||||||
)
|
)
|
||||||
|
assert.match(
|
||||||
|
dimensionFieldsSource,
|
||||||
|
/score_min[\s\S]*?score_max[\s\S]*?pass_threshold[\s\S]*?Math\.min[\s\S]*?Math\.max/,
|
||||||
|
'指标型评分区间变化时必须把通过阈值约束在合法范围内',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
dimensionFieldsSource,
|
||||||
|
/prop=["']score_min["'][\s\S]*?:max=["'][^"']*score_max[^"']*["'][\s\S]*?prop=["']score_max["'][\s\S]*?:min=["'][^"']*score_min[^"']*["']/,
|
||||||
|
'评分最小值和最大值输入必须具备交叉边界约束',
|
||||||
|
)
|
||||||
assert.match(createSource, /currentStep\s*=\s*ref\(0\)/, '评测创建向导缺少当前步骤状态')
|
assert.match(createSource, /currentStep\s*=\s*ref\(0\)/, '评测创建向导缺少当前步骤状态')
|
||||||
assert.match(createSource, /class=["']custom-wizard-steps["']/, '评测向导未复用数据处理的自定义步骤条结构')
|
assert.match(createSource, /class=["']custom-wizard-steps["']/, '评测向导未复用数据处理的自定义步骤条结构')
|
||||||
assert.match(createSource, /class=["']step-connector["']/, '评测向导步骤条缺少连接线')
|
assert.match(createSource, /class=["']step-connector["']/, '评测向导步骤条缺少连接线')
|
||||||
@@ -102,8 +128,18 @@ assert.match(
|
|||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
createSource,
|
createSource,
|
||||||
/<EvalRuleSetupStep[\s\S]*?v-else/,
|
/<EvalRuleSetupStep[\s\S]*?currentStep[^"']*1/,
|
||||||
'评测规则子组件没有绑定第二步',
|
'大模型评测指标子组件没有绑定第二步',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
createSource,
|
||||||
|
/<BasicMetricSetupStep[\s\S]*?currentStep[^"']*2/,
|
||||||
|
'基础评测指标子组件没有绑定第三步',
|
||||||
|
)
|
||||||
|
assert.match(
|
||||||
|
createSource,
|
||||||
|
/<StartEvalStep[\s\S]*?v-else/,
|
||||||
|
'开始评测确认子组件没有绑定第四步',
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.match(taskStepSource, /prop=["']dataset_id["']/, '数据集字段缺少表单校验标识')
|
assert.match(taskStepSource, /prop=["']dataset_id["']/, '数据集字段缺少表单校验标识')
|
||||||
@@ -113,16 +149,15 @@ assert.match(
|
|||||||
'选择评测数据集时必须校验 dataset_id',
|
'选择评测数据集时必须校验 dataset_id',
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.match(createSource, /['"]baseline['"]/, '向导缺少基线评测分支')
|
|
||||||
assert.match(
|
assert.match(
|
||||||
`${createSource}\n${ruleStepSource}`,
|
basicMetricStepSource,
|
||||||
/已有维度|existing/,
|
/BLEU[\s\S]*?ROUGE[\s\S]*?(?:Cosine|余弦)/,
|
||||||
'评测规则步骤缺少使用已有维度的分支',
|
'基础评测指标步骤必须提供 BLEU、ROUGE 与余弦相似度配置',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
`${createSource}\n${ruleStepSource}`,
|
basicMetricStepSource,
|
||||||
/新建(?:评测)?(?:维度|规则)|create/,
|
/bleu_enabled[\s\S]*?rouge_enabled[\s\S]*?cosine_enabled/,
|
||||||
'评测规则步骤缺少新建维度的分支',
|
'基础评测指标必须支持分别决定是否启用',
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.match(
|
assert.match(
|
||||||
@@ -137,17 +172,18 @@ assert.match(
|
|||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
createSource,
|
createSource,
|
||||||
/eval_type[\s\S]*?baseline[\s\S]*?(?:dimension_id|createDimension)/,
|
/basic_metrics[\s\S]*?bleu[\s\S]*?rouge[\s\S]*?cosine/,
|
||||||
'基线评测必须有明确的不创建自定义维度分支',
|
'启动评测时必须提交基础评测指标配置',
|
||||||
)
|
)
|
||||||
assert.match(
|
assert.match(
|
||||||
createSource,
|
startStepSource,
|
||||||
/ruleMode[\s\S]*?['"]existing['"]/,
|
/任务信息[\s\S]*?大模型评测指标[\s\S]*?基础评测指标/,
|
||||||
'已有维度模式必须有独立的提交分支',
|
'开始评测步骤必须展示前三步配置摘要',
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.match(createSource, /:loading=["']submitting["']/, '最终提交按钮缺少 loading 状态')
|
assert.match(createSource, /:loading=["']submitting["']/, '最终提交按钮缺少 loading 状态')
|
||||||
assert.match(createSource, /:disabled=["'][^"']*submitting/, '提交期间必须禁用重复操作')
|
assert.match(createSource, /:disabled=["'][^"']*submitting/, '提交期间必须禁用重复操作')
|
||||||
|
assert.match(createSource, />\s*开始评测\s*</, '最终提交按钮必须命名为“开始评测”')
|
||||||
assert.match(createSource, /class=["'][^"']*wizard-footer/, '评测向导缺少统一底部操作栏')
|
assert.match(createSource, /class=["'][^"']*wizard-footer/, '评测向导缺少统一底部操作栏')
|
||||||
assert.match(
|
assert.match(
|
||||||
createSource,
|
createSource,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { get, post, put, del } from '../request'
|
import { get, post, put, del } from '../request'
|
||||||
import type { EvalTask, EvalTaskDetail, Dimension } from '@/types'
|
import type { EvalTask, EvalTaskDetail, Dimension, StartEvalPayload } from '@/types'
|
||||||
|
|
||||||
/** 评测任务列表 */
|
/** 评测任务列表 */
|
||||||
export const getEvalList = () => get<EvalTask[]>('/model-eval')
|
export const getEvalList = () => get<EvalTask[]>('/model-eval')
|
||||||
@@ -11,7 +11,7 @@ export const getEvalDetail = (id: string | number) => get<EvalTaskDetail>(`/mode
|
|||||||
export const deleteEval = (id: string | number) => del(`/model-eval/${id}`)
|
export const deleteEval = (id: string | number) => del(`/model-eval/${id}`)
|
||||||
|
|
||||||
/** 启动评测 */
|
/** 启动评测 */
|
||||||
export const startEval = (data: any) => post('/model-eval/start', data)
|
export const startEval = (data: StartEvalPayload) => post('/model-eval/start', data)
|
||||||
|
|
||||||
/** 评测维度列表 */
|
/** 评测维度列表 */
|
||||||
export const getDimensionList = () => get<Dimension[]>('/dimension')
|
export const getDimensionList = () => get<Dimension[]>('/dimension')
|
||||||
|
|||||||
@@ -224,6 +224,34 @@ export interface EvalTask {
|
|||||||
create_time?: string
|
create_time?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 启动评测时提交的可选基础指标配置。 */
|
||||||
|
export interface BasicEvalMetricsConfig {
|
||||||
|
bleu: {
|
||||||
|
enabled: boolean
|
||||||
|
ngram: number
|
||||||
|
}
|
||||||
|
rouge: {
|
||||||
|
enabled: boolean
|
||||||
|
methods: string[]
|
||||||
|
}
|
||||||
|
cosine: {
|
||||||
|
enabled: boolean
|
||||||
|
}
|
||||||
|
output_precision: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StartEvalPayload {
|
||||||
|
eval_task_name: string
|
||||||
|
eval_type: EvalType
|
||||||
|
model_id: string | number
|
||||||
|
gpu_id: string | number
|
||||||
|
dataset_id: string | number
|
||||||
|
dimension_id: string | number
|
||||||
|
data_source: 'dataset' | 'inference'
|
||||||
|
leaderboard: boolean
|
||||||
|
basic_metrics: BasicEvalMetricsConfig
|
||||||
|
}
|
||||||
|
|
||||||
/** 单个测试样本的评测结果 */
|
/** 单个测试样本的评测结果 */
|
||||||
export interface EvalSampleResult {
|
export interface EvalSampleResult {
|
||||||
id: number | string
|
id: number | string
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import { ElMessage } from 'element-plus'
|
|||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import EvalTaskSetupStep, { type EvalTaskSetupDraft } from './create/EvalTaskSetupStep.vue'
|
import EvalTaskSetupStep, { type EvalTaskSetupDraft } from './create/EvalTaskSetupStep.vue'
|
||||||
import EvalRuleSetupStep, { type EvalRuleSetupDraft } from './create/EvalRuleSetupStep.vue'
|
import EvalRuleSetupStep, { type EvalRuleSetupDraft } from './create/EvalRuleSetupStep.vue'
|
||||||
import { createDimension, getDimensionList, startEval } from '@/api/modules/eval'
|
import BasicMetricSetupStep, { type BasicMetricSetupDraft } from './create/BasicMetricSetupStep.vue'
|
||||||
|
import StartEvalStep from './create/StartEvalStep.vue'
|
||||||
|
import { createDimension, startEval } from '@/api/modules/eval'
|
||||||
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
||||||
import { getDatasetList } from '@/api/modules/dataset'
|
import { getDatasetList } from '@/api/modules/dataset'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
import { getSystemInfo } from '@/api/modules/system'
|
||||||
@@ -19,15 +21,17 @@ const submitting = ref(false)
|
|||||||
const currentStep = ref(0)
|
const currentStep = ref(0)
|
||||||
const taskStepRef = ref<StepExposed>()
|
const taskStepRef = ref<StepExposed>()
|
||||||
const ruleStepRef = ref<StepExposed>()
|
const ruleStepRef = ref<StepExposed>()
|
||||||
|
const basicMetricStepRef = ref<StepExposed>()
|
||||||
|
|
||||||
const WIZARD_STEPS = [
|
const WIZARD_STEPS = [
|
||||||
{ title: '任务配置', description: '选择模型、算力与评测数据' },
|
{ title: '任务配置', description: '选择模型、算力与评测数据' },
|
||||||
{ title: '评测规则', description: '选择或创建评测维度' },
|
{ title: '大模型评测指标', description: '配置评测模型、方式与标准' },
|
||||||
|
{ title: '基础评测指标', description: '选择 BLEU、ROUGE 等参考指标' },
|
||||||
|
{ title: '开始评测', description: '确认配置并启动评测任务' },
|
||||||
] as const
|
] as const
|
||||||
|
|
||||||
const trainedModels = ref<TrainedModel[]>([])
|
const trainedModels = ref<TrainedModel[]>([])
|
||||||
const evalDatasets = ref<DatasetItem[]>([])
|
const evalDatasets = ref<DatasetItem[]>([])
|
||||||
const dimensions = ref<Dimension[]>([])
|
|
||||||
const evalModels = ref<ModelItem[]>([])
|
const evalModels = ref<ModelItem[]>([])
|
||||||
const gpus = ref<GpuInfo[]>([])
|
const gpus = ref<GpuInfo[]>([])
|
||||||
const createdDimensionId = ref<string | number>('')
|
const createdDimensionId = ref<string | number>('')
|
||||||
@@ -42,26 +46,31 @@ const taskForm = ref<EvalTaskSetupDraft>({
|
|||||||
})
|
})
|
||||||
|
|
||||||
const ruleForm = ref<EvalRuleSetupDraft>({
|
const ruleForm = ref<EvalRuleSetupDraft>({
|
||||||
ruleMode: 'new',
|
type: '',
|
||||||
dimension_id: '',
|
description: '',
|
||||||
newDimension: {
|
eval_model: '',
|
||||||
type: '',
|
eval_method: '',
|
||||||
description: '',
|
eval_prompt: '',
|
||||||
eval_model: '',
|
is_active: true,
|
||||||
eval_method: '',
|
is_default: false,
|
||||||
eval_prompt: '',
|
bleu_n: 1,
|
||||||
is_active: true,
|
output_precision: 3,
|
||||||
is_default: false,
|
score_min: 0,
|
||||||
bleu_n: 1,
|
score_max: 5,
|
||||||
output_precision: 3,
|
pass_threshold: 3,
|
||||||
score_min: 0,
|
})
|
||||||
score_max: 5,
|
|
||||||
pass_threshold: 3,
|
const basicMetricForm = ref<BasicMetricSetupDraft>({
|
||||||
},
|
bleu_enabled: false,
|
||||||
|
bleu_n: 4,
|
||||||
|
rouge_enabled: false,
|
||||||
|
rouge_methods: ['rouge_1', 'rouge_2'],
|
||||||
|
cosine_enabled: false,
|
||||||
|
output_precision: 3,
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
() => ruleForm.value.newDimension,
|
ruleForm,
|
||||||
() => {
|
() => {
|
||||||
createdDimensionId.value = ''
|
createdDimensionId.value = ''
|
||||||
},
|
},
|
||||||
@@ -72,20 +81,18 @@ async function loadData() {
|
|||||||
loading.value = true
|
loading.value = true
|
||||||
const results = await Promise.allSettled([
|
const results = await Promise.allSettled([
|
||||||
getTrainedModels(),
|
getTrainedModels(),
|
||||||
getDimensionList(),
|
|
||||||
getDatasetList(),
|
getDatasetList(),
|
||||||
getSystemInfo(),
|
getSystemInfo(),
|
||||||
getModelList(),
|
getModelList(),
|
||||||
])
|
])
|
||||||
|
|
||||||
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
||||||
if (results[1].status === 'fulfilled') dimensions.value = results[1].value || []
|
if (results[1].status === 'fulfilled') {
|
||||||
if (results[2].status === 'fulfilled') {
|
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
|
||||||
evalDatasets.value = (results[2].value || []).filter((dataset) => dataset.type === 'eval')
|
|
||||||
}
|
}
|
||||||
if (results[3].status === 'fulfilled') gpus.value = results[3].value?.gpu || []
|
if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || []
|
||||||
if (results[4].status === 'fulfilled') {
|
if (results[3].status === 'fulfilled') {
|
||||||
evalModels.value = (results[4].value || []).filter((model) => model.purpose === 'evaluation')
|
evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation')
|
||||||
}
|
}
|
||||||
|
|
||||||
const failedCount = results.filter((result) => result.status === 'rejected').length
|
const failedCount = results.filter((result) => result.status === 'rejected').length
|
||||||
@@ -95,25 +102,19 @@ async function loadData() {
|
|||||||
loading.value = false
|
loading.value = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function buildDimensionPayload() {
|
function buildDimensionPayload() {
|
||||||
const draft = ruleForm.value.newDimension
|
const draft = ruleForm.value
|
||||||
const payload: Partial<Dimension> = {
|
const payload: Partial<Dimension> = {
|
||||||
name: `Custom_Dim_${Date.now()}`,
|
name: `Custom_Dim_${Date.now()}`,
|
||||||
type: draft.type,
|
type: draft.type,
|
||||||
description: draft.description,
|
description: draft.description,
|
||||||
eval_model: draft.type === 'text_similarity' ? undefined : draft.eval_model,
|
eval_model: draft.eval_model,
|
||||||
eval_method: draft.eval_method,
|
eval_method: draft.eval_method,
|
||||||
eval_prompt: draft.type === 'text_similarity' ? undefined : draft.eval_prompt,
|
eval_prompt: draft.eval_prompt,
|
||||||
is_active: draft.is_active,
|
is_active: draft.is_active,
|
||||||
is_default: draft.is_default,
|
is_default: draft.is_default,
|
||||||
create_time: new Date().toISOString(),
|
create_time: new Date().toISOString(),
|
||||||
}
|
}
|
||||||
if (draft.type === 'text_similarity') {
|
|
||||||
payload.bleu_n = draft.bleu_n
|
|
||||||
payload.output_precision = draft.output_precision
|
|
||||||
}
|
|
||||||
if (draft.type === 'metric') {
|
if (draft.type === 'metric') {
|
||||||
payload.score_min = draft.score_min
|
payload.score_min = draft.score_min
|
||||||
payload.score_max = draft.score_max
|
payload.score_max = draft.score_max
|
||||||
@@ -123,8 +124,6 @@ function buildDimensionPayload() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function resolveDimensionId() {
|
async function resolveDimensionId() {
|
||||||
if (ruleForm.value.ruleMode === 'baseline') return ''
|
|
||||||
if (ruleForm.value.ruleMode === 'existing') return ruleForm.value.dimension_id
|
|
||||||
if (createdDimensionId.value !== '') return createdDimensionId.value
|
if (createdDimensionId.value !== '') return createdDimensionId.value
|
||||||
|
|
||||||
const created = await createDimension(buildDimensionPayload())
|
const created = await createDimension(buildDimensionPayload())
|
||||||
@@ -137,24 +136,32 @@ async function resolveDimensionId() {
|
|||||||
|
|
||||||
async function handleSubmit() {
|
async function handleSubmit() {
|
||||||
if (loading.value || submitting.value) return
|
if (loading.value || submitting.value) return
|
||||||
const ruleValid = await ruleStepRef.value?.validate()
|
|
||||||
if (!ruleValid) {
|
|
||||||
ElMessage.warning('请检查并完善评测规则')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
submitting.value = true
|
submitting.value = true
|
||||||
try {
|
try {
|
||||||
const dimensionId = await resolveDimensionId()
|
const dimensionId = await resolveDimensionId()
|
||||||
await startEval({
|
await startEval({
|
||||||
eval_task_name: taskForm.value.eval_task_name,
|
eval_task_name: taskForm.value.eval_task_name,
|
||||||
eval_type: ruleForm.value.ruleMode === 'baseline' ? 'baseline' : 'custom',
|
eval_type: 'custom',
|
||||||
model_id: taskForm.value.model_id,
|
model_id: taskForm.value.model_id,
|
||||||
gpu_id: taskForm.value.gpu_id,
|
gpu_id: taskForm.value.gpu_id,
|
||||||
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
||||||
dimension_id: dimensionId,
|
dimension_id: dimensionId,
|
||||||
data_source: taskForm.value.data_source,
|
data_source: taskForm.value.data_source,
|
||||||
leaderboard: taskForm.value.leaderboard,
|
leaderboard: taskForm.value.leaderboard,
|
||||||
|
basic_metrics: {
|
||||||
|
bleu: {
|
||||||
|
enabled: basicMetricForm.value.bleu_enabled,
|
||||||
|
ngram: basicMetricForm.value.bleu_n,
|
||||||
|
},
|
||||||
|
rouge: {
|
||||||
|
enabled: basicMetricForm.value.rouge_enabled,
|
||||||
|
methods: basicMetricForm.value.rouge_methods,
|
||||||
|
},
|
||||||
|
cosine: {
|
||||||
|
enabled: basicMetricForm.value.cosine_enabled,
|
||||||
|
},
|
||||||
|
output_precision: basicMetricForm.value.output_precision,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
ElMessage.success('评测任务已创建并启动')
|
ElMessage.success('评测任务已创建并启动')
|
||||||
router.push('/model-eval')
|
router.push('/model-eval')
|
||||||
@@ -168,17 +175,33 @@ async function handleSubmit() {
|
|||||||
|
|
||||||
async function handleNext() {
|
async function handleNext() {
|
||||||
if (loading.value || submitting.value) return
|
if (loading.value || submitting.value) return
|
||||||
const taskValid = await taskStepRef.value?.validate()
|
if (currentStep.value === 0) {
|
||||||
if (!taskValid) {
|
const taskValid = await taskStepRef.value?.validate()
|
||||||
ElMessage.warning('请检查并完善任务配置')
|
if (!taskValid) {
|
||||||
return
|
ElMessage.warning('请检查并完善任务配置')
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
currentStep.value = 1
|
if (currentStep.value === 1) {
|
||||||
|
const ruleValid = await ruleStepRef.value?.validate()
|
||||||
|
if (!ruleValid) {
|
||||||
|
ElMessage.warning('请检查并完善大模型评测指标')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (currentStep.value === 2) {
|
||||||
|
const basicMetricValid = await basicMetricStepRef.value?.validate()
|
||||||
|
if (!basicMetricValid) {
|
||||||
|
ElMessage.warning('请检查并完善基础评测指标')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
currentStep.value = Math.min(currentStep.value + 1, WIZARD_STEPS.length - 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleBack() {
|
function handleBack() {
|
||||||
if (submitting.value) return
|
if (submitting.value) return
|
||||||
currentStep.value = 0
|
currentStep.value = Math.max(currentStep.value - 1, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleCancel() {
|
function handleCancel() {
|
||||||
@@ -189,7 +212,10 @@ onMounted(loadData)
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
<PageCard title="新建评测任务" subtitle="完成任务配置与评测规则设置后启动评测">
|
<PageCard
|
||||||
|
title="新建评测任务"
|
||||||
|
subtitle="依次配置任务、大模型评测指标与基础评测指标,确认后开始评测"
|
||||||
|
>
|
||||||
<div class="create-wizard-layout">
|
<div class="create-wizard-layout">
|
||||||
<main v-loading="loading" class="wizard-main">
|
<main v-loading="loading" class="wizard-main">
|
||||||
<div class="wizard-steps-container" aria-label="评测任务创建步骤">
|
<div class="wizard-steps-container" aria-label="评测任务创建步骤">
|
||||||
@@ -207,6 +233,7 @@ onMounted(loadData)
|
|||||||
'is-completed': currentStep > index,
|
'is-completed': currentStep > index,
|
||||||
}"
|
}"
|
||||||
:aria-current="currentStep === index ? 'step' : undefined"
|
:aria-current="currentStep === index ? 'step' : undefined"
|
||||||
|
:aria-label="`${index + 1}. ${step.title}:${step.description}`"
|
||||||
>
|
>
|
||||||
<div class="step-node">
|
<div class="step-node">
|
||||||
<div class="step-icon" aria-hidden="true">
|
<div class="step-icon" aria-hidden="true">
|
||||||
@@ -224,48 +251,64 @@ onMounted(loadData)
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="wizard-content">
|
<div class="wizard-content">
|
||||||
<EvalTaskSetupStep
|
<EvalTaskSetupStep
|
||||||
v-if="currentStep === 0"
|
v-if="currentStep === 0"
|
||||||
ref="taskStepRef"
|
ref="taskStepRef"
|
||||||
v-model="taskForm"
|
v-model="taskForm"
|
||||||
:trained-models="trainedModels"
|
:trained-models="trainedModels"
|
||||||
:eval-datasets="evalDatasets"
|
:eval-datasets="evalDatasets"
|
||||||
:gpus="gpus"
|
:gpus="gpus"
|
||||||
:loading="loading"
|
:loading="loading"
|
||||||
:disabled="submitting"
|
:disabled="submitting"
|
||||||
/>
|
/>
|
||||||
<EvalRuleSetupStep
|
<EvalRuleSetupStep
|
||||||
v-else
|
v-else-if="currentStep === 1"
|
||||||
ref="ruleStepRef"
|
ref="ruleStepRef"
|
||||||
v-model="ruleForm"
|
v-model="ruleForm"
|
||||||
:dimensions="dimensions"
|
:eval-models="evalModels"
|
||||||
:eval-models="evalModels"
|
:disabled="submitting"
|
||||||
:loading="loading"
|
/>
|
||||||
:disabled="submitting"
|
<BasicMetricSetupStep
|
||||||
/>
|
v-else-if="currentStep === 2"
|
||||||
|
ref="basicMetricStepRef"
|
||||||
|
v-model="basicMetricForm"
|
||||||
|
:disabled="submitting"
|
||||||
|
/>
|
||||||
|
<StartEvalStep
|
||||||
|
v-else
|
||||||
|
:task="taskForm"
|
||||||
|
:llm-metric="ruleForm"
|
||||||
|
:basic-metrics="basicMetricForm"
|
||||||
|
:trained-models="trainedModels"
|
||||||
|
:eval-datasets="evalDatasets"
|
||||||
|
:eval-models="evalModels"
|
||||||
|
:gpus="gpus"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<footer class="wizard-footer">
|
<footer class="wizard-footer">
|
||||||
<el-button
|
<el-button
|
||||||
v-if="currentStep === 1"
|
v-if="currentStep > 0"
|
||||||
class="footer-back"
|
class="footer-back"
|
||||||
:disabled="submitting"
|
:disabled="submitting"
|
||||||
@click="handleBack"
|
@click="handleBack"
|
||||||
>
|
>
|
||||||
<i class="fa fa-arrow-left footer-button-icon" />返回:任务配置
|
<i class="fa fa-arrow-left footer-button-icon" />返回:{{ WIZARD_STEPS[currentStep - 1].title }}
|
||||||
</el-button>
|
</el-button>
|
||||||
<el-button v-else class="footer-back" :disabled="submitting" @click="handleCancel">
|
<el-button v-else class="footer-back" :disabled="submitting" @click="handleCancel">
|
||||||
取消
|
取消
|
||||||
</el-button>
|
</el-button>
|
||||||
|
|
||||||
<el-button
|
<el-button
|
||||||
v-if="currentStep === 0"
|
v-if="currentStep < WIZARD_STEPS.length - 1"
|
||||||
type="primary"
|
type="primary"
|
||||||
:disabled="loading || submitting"
|
:disabled="loading || submitting"
|
||||||
@click="handleNext"
|
@click="handleNext"
|
||||||
>
|
>
|
||||||
下一步:评测规则<i class="fa fa-arrow-right footer-button-icon is-right" />
|
下一步:{{ WIZARD_STEPS[currentStep + 1].title }}<i
|
||||||
|
class="fa fa-arrow-right footer-button-icon is-right"
|
||||||
|
/>
|
||||||
</el-button>
|
</el-button>
|
||||||
<el-button
|
<el-button
|
||||||
v-else
|
v-else
|
||||||
@@ -274,10 +317,10 @@ onMounted(loadData)
|
|||||||
:disabled="loading || submitting"
|
:disabled="loading || submitting"
|
||||||
@click="handleSubmit"
|
@click="handleSubmit"
|
||||||
>
|
>
|
||||||
创建并启动评测
|
开始评测
|
||||||
</el-button>
|
</el-button>
|
||||||
</footer>
|
</footer>
|
||||||
</div>
|
</div>
|
||||||
</PageCard>
|
</PageCard>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -305,7 +348,7 @@ onMounted(loadData)
|
|||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
max-width: 760px;
|
max-width: 1120px;
|
||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
padding: 0 20px;
|
padding: 0 20px;
|
||||||
}
|
}
|
||||||
@@ -319,7 +362,7 @@ onMounted(loadData)
|
|||||||
.step-connector {
|
.step-connector {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
height: 2px;
|
height: 2px;
|
||||||
margin: 0 20px;
|
margin: 0 12px;
|
||||||
background: #e2e8f0;
|
background: #e2e8f0;
|
||||||
transition: background-color 0.2s ease;
|
transition: background-color 0.2s ease;
|
||||||
}
|
}
|
||||||
@@ -414,6 +457,12 @@ onMounted(loadData)
|
|||||||
margin-left: 6px;
|
margin-left: 6px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@media (max-width: 1100px) {
|
||||||
|
.step-description {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 760px) {
|
@media (max-width: 760px) {
|
||||||
.wizard-main {
|
.wizard-main {
|
||||||
padding: 24px 20px;
|
padding: 24px 20px;
|
||||||
@@ -424,10 +473,10 @@ onMounted(loadData)
|
|||||||
}
|
}
|
||||||
|
|
||||||
.step-connector {
|
.step-connector {
|
||||||
margin: 0 10px;
|
margin: 0 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.step-description {
|
.step-text {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,19 +21,28 @@ export interface DimensionFormDraft {
|
|||||||
pass_threshold: number
|
pass_threshold: number
|
||||||
}
|
}
|
||||||
|
|
||||||
withDefaults(
|
const props = withDefaults(
|
||||||
defineProps<{
|
defineProps<{
|
||||||
evalModels: ModelItem[]
|
evalModels: ModelItem[]
|
||||||
showDescription?: boolean
|
showDescription?: boolean
|
||||||
|
showStatusSettings?: boolean
|
||||||
|
allowedTypes?: DimensionType[]
|
||||||
}>(),
|
}>(),
|
||||||
{
|
{
|
||||||
showDescription: true,
|
showDescription: true,
|
||||||
|
showStatusSettings: true,
|
||||||
|
allowedTypes: () => ['classification', 'metric', 'text_similarity'],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
const form = defineModel<DimensionFormDraft>({ required: true })
|
const form = defineModel<DimensionFormDraft>({ required: true })
|
||||||
|
const SCORE_STEP = 0.5
|
||||||
|
|
||||||
const typeOptions = Object.entries(DIMENSION_TYPE_MAP).map(([value, label]) => ({ value, label }))
|
const typeOptions = computed(() =>
|
||||||
|
Object.entries(DIMENSION_TYPE_MAP)
|
||||||
|
.filter(([value]) => props.allowedTypes.includes(value as DimensionType))
|
||||||
|
.map(([value, label]) => ({ value, label })),
|
||||||
|
)
|
||||||
const currentMethods = computed(() => (form.value.type ? EVAL_METHODS[form.value.type] || [] : []))
|
const currentMethods = computed(() => (form.value.type ? EVAL_METHODS[form.value.type] || [] : []))
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -65,6 +74,23 @@ watch(
|
|||||||
},
|
},
|
||||||
{ flush: 'sync' },
|
{ flush: 'sync' },
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/** 始终保持评分区间合法,并把通过阈值收敛到当前区间内。 */
|
||||||
|
watch(
|
||||||
|
() => [form.value.type, form.value.score_min, form.value.score_max] as const,
|
||||||
|
([type, scoreMin, scoreMax]) => {
|
||||||
|
if (type !== 'metric') return
|
||||||
|
if (scoreMax <= scoreMin) {
|
||||||
|
form.value.score_max = scoreMin + SCORE_STEP
|
||||||
|
return
|
||||||
|
}
|
||||||
|
form.value.pass_threshold = Math.min(
|
||||||
|
Math.max(form.value.pass_threshold, scoreMin),
|
||||||
|
scoreMax,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
{ flush: 'sync' },
|
||||||
|
)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -134,13 +160,24 @@ watch(
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template v-if="form.type === 'metric'">
|
<template v-if="form.type === 'metric'">
|
||||||
<el-form-item label="评分最小值">
|
<el-form-item label="评分最小值" prop="score_min">
|
||||||
<el-input-number v-model="form.score_min" :min="0" :step="1" style="width: 160px" />
|
<el-input-number
|
||||||
|
v-model="form.score_min"
|
||||||
|
:min="0"
|
||||||
|
:max="Math.max(form.score_max - SCORE_STEP, 0)"
|
||||||
|
:step="SCORE_STEP"
|
||||||
|
style="width: 160px"
|
||||||
|
/>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="评分最大值">
|
<el-form-item label="评分最大值" prop="score_max">
|
||||||
<el-input-number v-model="form.score_max" :min="0" :step="1" style="width: 160px" />
|
<el-input-number
|
||||||
|
v-model="form.score_max"
|
||||||
|
:min="form.score_min + SCORE_STEP"
|
||||||
|
:step="SCORE_STEP"
|
||||||
|
style="width: 160px"
|
||||||
|
/>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="通过阈值">
|
<el-form-item label="通过阈值" prop="pass_threshold">
|
||||||
<div style="width: 100%">
|
<div style="width: 100%">
|
||||||
<el-slider
|
<el-slider
|
||||||
v-model="form.pass_threshold"
|
v-model="form.pass_threshold"
|
||||||
@@ -154,12 +191,14 @@ watch(
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<el-form-item label="启用该维度">
|
<template v-if="showStatusSettings">
|
||||||
<el-switch v-model="form.is_active" />
|
<el-form-item label="启用该维度">
|
||||||
</el-form-item>
|
<el-switch v-model="form.is_active" />
|
||||||
<el-form-item label="设为默认">
|
</el-form-item>
|
||||||
<el-switch v-model="form.is_default" />
|
<el-form-item label="设为默认">
|
||||||
</el-form-item>
|
<el-switch v-model="form.is_default" />
|
||||||
|
</el-form-item>
|
||||||
|
</template>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -1,49 +1,31 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import type { FormInstance, FormRules } from 'element-plus'
|
import type { FormInstance, FormRules } from 'element-plus'
|
||||||
import DimensionFormFields, { type DimensionFormDraft } from './DimensionFormFields.vue'
|
import DimensionFormFields, { type DimensionFormDraft } from './DimensionFormFields.vue'
|
||||||
import type { Dimension, ModelItem } from '@/types'
|
import type { DimensionType, ModelItem } from '@/types'
|
||||||
|
|
||||||
export type EvalRuleMode = 'existing' | 'new' | 'baseline'
|
export type EvalRuleSetupDraft = DimensionFormDraft
|
||||||
|
|
||||||
export interface EvalRuleSetupDraft {
|
defineProps<{
|
||||||
ruleMode: EvalRuleMode
|
|
||||||
dimension_id: string | number
|
|
||||||
newDimension: DimensionFormDraft
|
|
||||||
}
|
|
||||||
|
|
||||||
const props = defineProps<{
|
|
||||||
dimensions: Dimension[]
|
|
||||||
evalModels: ModelItem[]
|
evalModels: ModelItem[]
|
||||||
loading?: boolean
|
|
||||||
disabled?: boolean
|
disabled?: boolean
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
const form = defineModel<EvalRuleSetupDraft>({ required: true })
|
const form = defineModel<EvalRuleSetupDraft>({ required: true })
|
||||||
const existingFormRef = ref<FormInstance>()
|
|
||||||
const newFormRef = ref<FormInstance>()
|
const newFormRef = ref<FormInstance>()
|
||||||
|
const LLM_METRIC_TYPES: DimensionType[] = ['classification', 'metric']
|
||||||
const existingRules: FormRules = {
|
|
||||||
dimension_id: [{ required: true, message: '请选择评测维度', trigger: 'change' }],
|
|
||||||
}
|
|
||||||
|
|
||||||
const dimensionRules: FormRules<DimensionFormDraft> = {
|
const dimensionRules: FormRules<DimensionFormDraft> = {
|
||||||
type: [{ required: true, message: '请选择指标类型', trigger: 'change' }],
|
type: [{ required: true, message: '请选择大模型评测指标类型', trigger: 'change' }],
|
||||||
eval_model: [{ required: true, message: '请选择大模型', trigger: 'change' }],
|
eval_model: [{ required: true, message: '请选择评测大模型', trigger: 'change' }],
|
||||||
eval_method: [{ required: true, message: '请选择评估方式', trigger: 'change' }],
|
eval_method: [{ required: true, message: '请选择评估方式', trigger: 'change' }],
|
||||||
eval_prompt: [{ required: true, message: '请填写评估 Prompt', trigger: 'blur' }],
|
eval_prompt: [{ required: true, message: '请填写评估 Prompt', trigger: 'blur' }],
|
||||||
}
|
}
|
||||||
|
|
||||||
const selectedDimension = computed(() =>
|
|
||||||
props.dimensions.find((dimension) => String(dimension.id) === String(form.value.dimension_id)),
|
|
||||||
)
|
|
||||||
|
|
||||||
async function validate() {
|
async function validate() {
|
||||||
const target = newFormRef.value
|
if (!newFormRef.value) return false
|
||||||
if (!target) return false
|
|
||||||
try {
|
try {
|
||||||
const valid = await target.validate()
|
return !!(await newFormRef.value.validate())
|
||||||
return !!valid
|
|
||||||
} catch {
|
} catch {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -56,7 +38,7 @@ defineExpose({ validate })
|
|||||||
<div class="rule-step">
|
<div class="rule-step">
|
||||||
<el-form
|
<el-form
|
||||||
ref="newFormRef"
|
ref="newFormRef"
|
||||||
:model="form.newDimension"
|
:model="form"
|
||||||
:rules="dimensionRules"
|
:rules="dimensionRules"
|
||||||
label-width="130px"
|
label-width="130px"
|
||||||
class="step-form step-form-wide"
|
class="step-form step-form-wide"
|
||||||
@@ -64,17 +46,17 @@ defineExpose({ validate })
|
|||||||
scroll-to-error
|
scroll-to-error
|
||||||
>
|
>
|
||||||
<DimensionFormFields
|
<DimensionFormFields
|
||||||
v-model="form.newDimension"
|
v-model="form"
|
||||||
:eval-models="evalModels"
|
:eval-models="evalModels"
|
||||||
:show-description="false"
|
:show-description="false"
|
||||||
|
:show-status-settings="false"
|
||||||
|
:allowed-types="LLM_METRIC_TYPES"
|
||||||
/>
|
/>
|
||||||
</el-form>
|
</el-form>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|
||||||
|
|
||||||
.step-form {
|
.step-form {
|
||||||
max-width: 760px;
|
max-width: 760px;
|
||||||
}
|
}
|
||||||
@@ -82,6 +64,4 @@ defineExpose({ validate })
|
|||||||
.step-form-wide {
|
.step-form-wide {
|
||||||
max-width: 920px;
|
max-width: 920px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
Reference in New Issue
Block a user