feat: 模型评测端到端闭环 — EvalRunner引擎 + 算力节点Job执行 + 结果回写

算力节点 (compute):
- 新建 eval_runner.py: 评测执行引擎,作为subprocess运行
  - 加载模型 + JSONL数据集 + 逐样本推理
  - BLEU/ROUGE/Cosine基础指标计算
  - LLM Judge评分(OpenAI兼容API调用)
  - 结果写入eval_results.json
- adapter.py: build_command新增engine=eval分支
- main.py: 新增/json模块导入,新增/compute/files/read端点,eval job校验

后端:
- platform.py: 重写startEval提交eval job到算力节点
  - 支持models表和trained_models表查找
  - 已合并模型不传adapter路径
- platform_store.py: 新增update_eval_task/running_eval_tasks/apply_eval_job_result
- sync.py: poller新增eval job同步,异步读取eval_results.json回写结果

前端:
- EvalCreateView/DimensionCreateView: eval模型过滤扩展(API类型+api_url)
- EvalCreateView: GPU过滤在线节点空闲GPU
- EvalTaskSetupStep: GPU value从数组index改为gpu.id
- BasicMetricSetupStep: ROUGE方法名修正(rouge_1→rouge1)
- EvalView: 新增5秒轮询刷新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-07-28 19:34:41 +08:00
parent c7c9ed925b
commit 0c39f2f5b9
12 changed files with 903 additions and 17 deletions

View File

@@ -79,7 +79,9 @@ async function loadEditData() {
async function loadModels() {
try {
const all = (await getModelList()) || []
evalModels.value = all.filter((m) => m.purpose === 'evaluation')
evalModels.value = all.filter(
(m) => m.purpose === 'evaluation' || (m.model_source === 'api' && !!m.api_url),
)
} catch {
evalModels.value = []
}

View File

@@ -11,6 +11,7 @@ import { createDimension, startEval } from '@/api/modules/eval'
import { getTrainedModels, getModelList } from '@/api/modules/model'
import { getDatasetList } from '@/api/modules/dataset'
import { getSystemInfo } from '@/api/modules/system'
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
type StepExposed = { validate: () => Promise<boolean> }
@@ -84,15 +85,26 @@ async function loadData() {
getDatasetList(),
getSystemInfo(),
getModelList(),
getComputeNodes(),
])
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
if (results[1].status === 'fulfilled') {
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
}
if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || []
if (results[2].status === 'fulfilled') {
const allGpus: GpuInfo[] = results[2].value?.gpu || []
const nodes: ComputeNode[] = (results[4].status === 'fulfilled' ? results[4].value : []) || []
const onlineIds = new Set(nodes.filter((n) => n.enabled && n.scheduler_status === 'online').map((n) => n.id))
// Only show idle GPUs from online compute nodes
gpus.value = allGpus.filter(
(g) => g.status === 'idle' && (!g.node_id || onlineIds.has(g.node_id)),
)
}
if (results[3].status === 'fulfilled') {
evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation')
evalModels.value = (results[3].value || []).filter(
(model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
)
}
const failedCount = results.filter((result) => result.status === 'rejected').length

View File

@@ -1,9 +1,10 @@
<script setup lang="ts">
import { ref, onMounted } from 'vue'
import { ref, onMounted, onUnmounted } from 'vue'
import { useRouter } from 'vue-router'
import { ElMessage, ElMessageBox } from 'element-plus'
import DataTablePage from '@/components/DataTablePage.vue'
import ModelStatusTag from '@/components/ModelStatusTag.vue'
import { usePolling } from '@/composables/usePolling'
import {
getEvalList,
deleteEval,
@@ -54,8 +55,19 @@ function handleViewDetail(row: any) {
router.push({ name: 'model-eval-detail', params: { id: row.id } })
}
onMounted(() => {
loadEvalList()
const { start: startPolling, stop: stopPolling } = usePolling(
() => loadEvalList(),
5000,
{ immediate: false },
)
onMounted(async () => {
await loadEvalList()
startPolling()
})
onUnmounted(() => {
stopPolling()
})
</script>

View File

@@ -56,9 +56,9 @@ defineExpose({ validate })
</el-form-item>
<el-form-item v-if="form.rouge_enabled" label="ROUGE methods">
<el-checkbox-group v-model="form.rouge_methods">
<el-checkbox value="rouge_1">ROUGE-1</el-checkbox>
<el-checkbox value="rouge_2">ROUGE-2</el-checkbox>
<el-checkbox value="rouge_l">ROUGE-L</el-checkbox>
<el-checkbox value="rouge1">ROUGE-1</el-checkbox>
<el-checkbox value="rouge2">ROUGE-2</el-checkbox>
<el-checkbox value="rougeL">ROUGE-L</el-checkbox>
</el-checkbox-group>
</el-form-item>

View File

@@ -103,10 +103,10 @@ defineExpose({ validate })
<el-form-item label="选择 GPU" prop="gpu_id">
<el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
<el-option
v-for="(gpu, index) in gpus"
:key="index"
:label="`${gpu.name} (GPU ${index})`"
:value="index"
v-for="gpu in gpus"
:key="gpu.id"
:label="`${gpu.name} (GPU ${gpu.id})`"
:value="gpu.id ?? 0"
/>
</el-select>
</el-form-item>