YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 推理/评测结果同步、数据集统计与算力节点管理增强

Browse Source 
后端:
- 抽取 fetch_eval_result_content 复用函数，model_eval_detail 直接应用评测任务结果
- health 接口移除数据库依赖，返回静态指标
- 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理
- 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout
- 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error

评测引擎:
- _load_dataset 支持 JSON/JSONL 文件
- 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制

前端:
- 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode
- 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
wuyongtao
2026-08-03 15:49:21 +08:00
parent cc08b164d0
commit 5cc306eb0a
15 changed files with 402 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/app/api/v1/endpoints/health.py
View File

@@ -1,7 +1,6 @@
from fastapi import APIRouter
from app.core.logging import get_logger
from app.db.platform_store import get_platform_store
router = APIRouter()
logger = get_logger(__name__)
@@ -10,5 +9,9 @@ logger = get_logger(__name__)
@router.get("/health")
async def health_check() -> dict[str, object]:
logger.info("health check requested")
return {"code": 0, "message": "ok", "data": get_platform_store().health_metrics()}
return {
"code": 0,
"message": "ok",
"data": {"cpu_percent": 0.0, "memory_percent": 0.0, "disk_percent": 0.0},
}

45
backend/app/api/v1/endpoints/platform.py
View File

@@ -15,7 +15,7 @@ from app.core.auth import filter_accessible_resource_ids, get_current_user, has_
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import poll_compute_jobs_once
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
router = APIRouter()
@@ -915,6 +915,7 @@ async def upload_dataset_files(
) -> dict[str, Any]:
created: list[dict[str, Any]] = []
compute_sync: list[dict[str, Any]] = []
pending_sync: list[tuple[str, str, bytes]] = []
store = get_platform_store()
try:
store.dataset(dataset_id)
@@ -926,16 +927,18 @@ async def upload_dataset_files(
content = raw.decode("utf-8", errors="replace")
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file)
if sync_to_compute:
compute_sync.extend(
await _sync_dataset_file_to_compute_nodes(
store,
dataset_id,
created_file["id"],
created_file["name"],
raw,
)
pending_sync.append((created_file["id"], created_file["name"], raw))
if sync_to_compute:
for file_id, file_name, raw in pending_sync:
compute_sync.extend(
await _sync_dataset_file_to_compute_nodes(
store,
dataset_id,
file_id,
file_name,
raw,
)
)
return ok({"files": created, "compute_sync": compute_sync})
@@ -1273,8 +1276,7 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
try:
store = get_platform_store()
task = store.eval_task(task_id)
# If the eval job completed on a compute node, try to load results
if task.get("result_artifact_path"):
if task.get("compute_job_id") and task.get("compute_node_id") and task.get("status") in {"queued", "running", "completed"}:
node = next(
(n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")),
None,
@@ -1283,11 +1285,10 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
try:
client = ComputeNodeClient(node["api_base_url"])
job = await client.get_job(task["compute_job_id"])
artifacts = job.get("artifacts") or []
for art in artifacts:
if art.get("name") == "eval_results.json":
task["_result_artifact"] = art
break
result_content = None
if job.get("status") == "completed" and not task.get("samples"):
result_content = await fetch_eval_result_content(client, node, job)
task = store.apply_eval_job_result(task_id, job, result_content)
except Exception:
pass
except KeyError:
@@ -1790,6 +1791,16 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
raise fail(400, str(exc))
@router.delete("/compute/nodes/{node_id}")
async def delete_compute_node(node_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().delete_compute_node(node_id))
except KeyError:
raise fail(404, "compute node not found")
except ValueError as exc:
raise fail(400, str(exc))
@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str) -> dict[str, Any]:
store = get_platform_store()

139
backend/app/db/platform_store.py
View File

@@ -88,6 +88,22 @@ def parse_size_bytes(value: Any) -> int:
return max(0, round(amount * _SIZE_UNIT_BYTES[unit]))
def count_dataset_records(content: str) -> int:
text = (content or "").strip()
if not text:
return 0
try:
value = json.loads(text)
if isinstance(value, list):
return len(value)
return 1
except (TypeError, ValueError, json.JSONDecodeError):
pass
return len([line for line in text.splitlines() if line.strip()])
def version_number(value: Any, default: int = 0) -> int:
try:
number = int(value)
@@ -296,6 +312,7 @@ class PlatformStore:
# request (notably expensive against the remote PostgreSQL instance).
# TCP keepalive 让操作系统持续保活连接，抵抗远程库空闲静默断连。
pool_kwargs = {
"connect_timeout": 5,
"keepalives": 1,
"keepalives_idle": 30,
"keepalives_interval": 10,
@@ -1311,6 +1328,7 @@ class PlatformStore:
file_size_bytes = int(file_row.get("size_bytes") or 0)
if file_size_bytes <= 0:
file_size_bytes = parse_size_bytes(file_row.get("size"))
file_record_count = int(file_row.get("record_count") or 0)
decoded_files.append(
{
"id": file_row["id"],
@@ -1319,7 +1337,7 @@ class PlatformStore:
"size_bytes": file_size_bytes,
**dataset_file_version_summary(file_row),
"create_time": file_row["create_time"],
"record_count": int(file_row.get("record_count") or 0),
"record_count": file_record_count,
"split": metadata.get("file_split"),
}
)
@@ -1337,6 +1355,9 @@ class PlatformStore:
total_size_bytes = int(row.get("size_bytes") or 0)
if total_size_bytes <= 0:
total_size_bytes = parse_size_bytes(row.get("size"))
total_record_count = sum(int(item.get("record_count") or 0) for item in decoded_files)
if not decoded_files:
total_record_count = int(row.get("record_count") or row.get("count") or 0)
current_version_nos = sorted(
{
int(item["current_version_no"])
@@ -1346,6 +1367,8 @@ class PlatformStore:
)
return {
**dict(row),
"count": total_record_count,
"record_count": total_record_count,
"size_bytes": total_size_bytes,
"current_version_no": (
current_version_nos[0] if len(current_version_nos) == 1 else None
@@ -1420,7 +1443,7 @@ class PlatformStore:
version_id = f"{file_id}_v1"
size_bytes = len(content.encode("utf-8"))
size = f"{size_bytes} B"
record_count = len([line for line in content.splitlines() if line.strip()])
record_count = count_dataset_records(content)
version = {
"id": version_id,
"version": 1,
@@ -1453,10 +1476,18 @@ class PlatformStore:
)
conn.execute(
"""UPDATE datasets
SET count=count+?, record_count=record_count+?,
size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B')
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=?
) stats
WHERE id=?""",
(record_count, record_count, size_bytes, size_bytes, dataset_id),
(dataset_id, dataset_id),
)
return {
"id": file_id,
@@ -1561,19 +1592,57 @@ class PlatformStore:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
content = payload.get("content", "")
size_bytes = len(content.encode("utf-8"))
record_count = count_dataset_records(content)
versions = json_loads(row["versions"], [])
version = {
"id": f"{file_id}_v{len(versions) + 1}",
"version": len(versions) + 1,
"version_no": len(versions) + 1,
"create_time": utcnow(),
"description": payload.get("description", "online edit"),
"size_bytes": size_bytes,
"record_count": record_count,
}
versions.append(version)
conn.execute(
"UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?",
(payload.get("content", ""), version["id"], json_dumps(versions), file_id),
"""
UPDATE dataset_files
SET content=?, active_version_id=?, current_version_id=?, versions=?,
size_bytes=?, size=?, record_count=?, version_no=?
WHERE id=?
""",
(
content,
version["id"],
version["id"],
json_dumps(versions),
size_bytes,
f"{size_bytes} B",
record_count,
version["version_no"],
file_id,
),
)
return {"version": version, "content": payload.get("content", "")}
conn.execute(
"""UPDATE datasets
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT dataset_id,
COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=(SELECT dataset_id FROM dataset_files WHERE id=?)
GROUP BY dataset_id
) stats
WHERE datasets.id=stats.dataset_id""",
(file_id,),
)
return {"version": version, "content": content}
def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
with self.connect() as conn:
@@ -2173,6 +2242,14 @@ class PlatformStore:
"sample_count": result_content.get("sample_count", 0),
"completed_count": result_content.get("completed_count", 0),
"passed_count": result_content.get("passed_count", 0),
"basic_metrics": result_content.get("basic_metrics", {}),
"score": result_content.get("overall_score", 0),
"completed_time": utcnow(),
})
elif new_status in {"failed", "stopped"}:
updates.update({
"error": job.get("error") or task.get("error") or "",
"completed_time": utcnow(),
})
return self.update_eval_task(task_id, updates)
@@ -2342,6 +2419,15 @@ class PlatformStore:
).fetchall()
return {int(row["gpu_index"]) for row in rows}
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
if rows:
return {int(row["gpu_index"]) for row in rows}
return set(range(max(0, int(node.get("gpu_count") or 0))))
def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []]
@@ -2349,13 +2435,15 @@ class PlatformStore:
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"]
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if not set(requested_gpus).intersection(self._active_gpu_indexes(conn, node["id"]))
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
@@ -2370,8 +2458,8 @@ class PlatformStore:
reason = "disabled"
elif node["scheduler_status"] != "online":
reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= node["max_parallel_jobs"]:
reason = f"capacity full {node['current_running_jobs']}/{node['max_parallel_jobs']}"
elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
else:
reason = "not selected"
reasons.append(f"{node['code']}({reason})")
@@ -2722,6 +2810,24 @@ class PlatformStore:
)
return next(node for node in self.compute_nodes() if node["id"] == node_id)
def delete_compute_node(self, node_id: str) -> dict[str, Any]:
with self.connect() as conn:
node = conn.execute("SELECT * FROM compute_nodes WHERE id=?", (node_id,)).fetchone()
if not node:
raise KeyError(node_id)
active = conn.execute(
"""
SELECT COUNT(*) AS cnt
FROM fine_tune_tasks
WHERE compute_node_id=? AND status IN ('syncing','queued','running')
""",
(node_id,),
).fetchone()
if active and int(active["cnt"] or 0) > 0:
raise ValueError("compute node has active training tasks")
conn.execute("DELETE FROM compute_nodes WHERE id=?", (node_id,))
return {"deleted": node_id}
def update_compute_node_health(self, node_id: str, health: dict[str, Any], success: bool, error: str | None = None) -> dict[str, Any]:
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if not current:
@@ -2890,11 +2996,12 @@ class PlatformStore:
}
def health_metrics(self) -> dict[str, float]:
info = self.system_info()
# Health checks must stay lightweight. The Docker healthcheck and page
# refresh probes should not wait on dashboard/GPU/database aggregation.
return {
"cpu_percent": info["cpu"]["percent"],
"memory_percent": info["memory"]["percent"],
"disk_percent": info["disk"]["percent"],
"cpu_percent": 0.0,
"memory_percent": 0.0,
"disk_percent": 0.0,
}
def queue(self) -> list[dict[str, Any]]:

3
backend/app/modules/compute_gateway/client.py
View File

@@ -207,7 +207,8 @@ class ComputeNodeClient:
"resource_id": resource_id or "",
}
files = {"file": (filename, content)}
async with httpx.AsyncClient(timeout=max(self.timeout, 60), headers=self.headers()) as client:
timeout = httpx.Timeout(max(self.timeout, 60), connect=self.timeout)
async with httpx.AsyncClient(timeout=timeout, headers=self.headers()) as client:
response = await client.post(
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/upload"),
data=data,

31
backend/app/modules/compute_gateway/sync.py
View File

@@ -10,6 +10,24 @@ def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
output_dir = job.get("output_dir")
if not output_dir:
return None
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
rel_path = full_path.lstrip("/")
import httpx
url = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
response = await http.get(url, params={"path": rel_path})
response.raise_for_status()
payload = response.json()
return payload if isinstance(payload, dict) else None
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
synced: list[dict[str, Any]] = []
@@ -66,18 +84,7 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
# Try to read eval_results.json from the job output directory
if job.get("status") == "completed" and job.get("output_dir"):
try:
full_path = f"{job['output_dir'].rstrip('/')}/eval_results.json"
# Convert absolute path to relative (strip YG_FT_DATA_ROOT prefix)
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
rel_path = full_path.lstrip("/")
import httpx
settings_path = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
read_resp = await http.get(settings_path, params={"path": rel_path})
if read_resp.status_code == 200:
result_content = read_resp.json()
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)