feat: 模型评测端到端闭环 — EvalRunner引擎 + 算力节点Job执行 + 结果回写
算力节点 (compute): - 新建 eval_runner.py: 评测执行引擎,作为subprocess运行 - 加载模型 + JSONL数据集 + 逐样本推理 - BLEU/ROUGE/Cosine基础指标计算 - LLM Judge评分(OpenAI兼容API调用) - 结果写入eval_results.json - adapter.py: build_command新增engine=eval分支 - main.py: 新增/json模块导入,新增/compute/files/read端点,eval job校验 后端: - platform.py: 重写startEval提交eval job到算力节点 - 支持models表和trained_models表查找 - 已合并模型不传adapter路径 - platform_store.py: 新增update_eval_task/running_eval_tasks/apply_eval_job_result - sync.py: poller新增eval job同步,异步读取eval_results.json回写结果 前端: - EvalCreateView/DimensionCreateView: eval模型过滤扩展(API类型+api_url) - EvalCreateView: GPU过滤在线节点空闲GPU - EvalTaskSetupStep: GPU value从数组index改为gpu.id - BasicMetricSetupStep: ROUGE方法名修正(rouge_1→rouge1) - EvalView: 新增5秒轮询刷新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -79,7 +79,9 @@ async function loadEditData() {
|
||||
async function loadModels() {
|
||||
try {
|
||||
const all = (await getModelList()) || []
|
||||
evalModels.value = all.filter((m) => m.purpose === 'evaluation')
|
||||
evalModels.value = all.filter(
|
||||
(m) => m.purpose === 'evaluation' || (m.model_source === 'api' && !!m.api_url),
|
||||
)
|
||||
} catch {
|
||||
evalModels.value = []
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import { createDimension, startEval } from '@/api/modules/eval'
|
||||
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
||||
import { getDatasetList } from '@/api/modules/dataset'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
||||
import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
|
||||
|
||||
type StepExposed = { validate: () => Promise<boolean> }
|
||||
@@ -84,15 +85,26 @@ async function loadData() {
|
||||
getDatasetList(),
|
||||
getSystemInfo(),
|
||||
getModelList(),
|
||||
getComputeNodes(),
|
||||
])
|
||||
|
||||
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
||||
if (results[1].status === 'fulfilled') {
|
||||
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
|
||||
}
|
||||
if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || []
|
||||
if (results[2].status === 'fulfilled') {
|
||||
const allGpus: GpuInfo[] = results[2].value?.gpu || []
|
||||
const nodes: ComputeNode[] = (results[4].status === 'fulfilled' ? results[4].value : []) || []
|
||||
const onlineIds = new Set(nodes.filter((n) => n.enabled && n.scheduler_status === 'online').map((n) => n.id))
|
||||
// Only show idle GPUs from online compute nodes
|
||||
gpus.value = allGpus.filter(
|
||||
(g) => g.status === 'idle' && (!g.node_id || onlineIds.has(g.node_id)),
|
||||
)
|
||||
}
|
||||
if (results[3].status === 'fulfilled') {
|
||||
evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation')
|
||||
evalModels.value = (results[3].value || []).filter(
|
||||
(model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
|
||||
)
|
||||
}
|
||||
|
||||
const failedCount = results.filter((result) => result.status === 'rejected').length
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted } from 'vue'
|
||||
import { ref, onMounted, onUnmounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import {
|
||||
getEvalList,
|
||||
deleteEval,
|
||||
@@ -54,8 +55,19 @@ function handleViewDetail(row: any) {
|
||||
router.push({ name: 'model-eval-detail', params: { id: row.id } })
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadEvalList()
|
||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||
() => loadEvalList(),
|
||||
5000,
|
||||
{ immediate: false },
|
||||
)
|
||||
|
||||
onMounted(async () => {
|
||||
await loadEvalList()
|
||||
startPolling()
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
stopPolling()
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
@@ -56,9 +56,9 @@ defineExpose({ validate })
|
||||
</el-form-item>
|
||||
<el-form-item v-if="form.rouge_enabled" label="ROUGE methods">
|
||||
<el-checkbox-group v-model="form.rouge_methods">
|
||||
<el-checkbox value="rouge_1">ROUGE-1</el-checkbox>
|
||||
<el-checkbox value="rouge_2">ROUGE-2</el-checkbox>
|
||||
<el-checkbox value="rouge_l">ROUGE-L</el-checkbox>
|
||||
<el-checkbox value="rouge1">ROUGE-1</el-checkbox>
|
||||
<el-checkbox value="rouge2">ROUGE-2</el-checkbox>
|
||||
<el-checkbox value="rougeL">ROUGE-L</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</el-form-item>
|
||||
|
||||
|
||||
@@ -103,10 +103,10 @@ defineExpose({ validate })
|
||||
<el-form-item label="选择 GPU" prop="gpu_id">
|
||||
<el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
|
||||
<el-option
|
||||
v-for="(gpu, index) in gpus"
|
||||
:key="index"
|
||||
:label="`${gpu.name} (GPU ${index})`"
|
||||
:value="index"
|
||||
v-for="gpu in gpus"
|
||||
:key="gpu.id"
|
||||
:label="`${gpu.name} (GPU ${gpu.id})`"
|
||||
:value="gpu.id ?? 0"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
|
||||
Reference in New Issue
Block a user