YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 推理/评测结果同步、数据集统计与算力节点管理增强

Browse Source 
后端:
- 抽取 fetch_eval_result_content 复用函数，model_eval_detail 直接应用评测任务结果
- health 接口移除数据库依赖，返回静态指标
- 数据集: count_dataset_records JSON 感知计数; 文件统计改为从 dataset_files 聚合重算; 在线编辑记录 size/record_count/version_no; 上传同步批处理
- 算力节点: 调度支持 requested GPU 子集校验与容量计算; 新增 delete_compute_node(含活动任务保护)及 DELETE 接口; 连接池 connect_timeout
- 评测任务落库 basic_metrics/score/completed_time, failed/stopped 记录 error

评测引擎:
- _load_dataset 支持 JSON/JSONL 文件
- 新增 exact match 与文本相似度指标, 余弦相似度去掉 2 样本限制

前端:
- 算力节点列表「维护」改为「删除」(带确认弹窗), compute.ts 新增 deleteComputeNode
- 数据集上传超时调整为 120s; FineTuneTask 增加 compute_node_id; GpuInfo 状态增加 reserved
This commit is contained in:
wuyongtao
2026-08-03 15:49:21 +08:00
parent cc08b164d0
commit 5cc306eb0a
15 changed files with 402 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
backend/app/db/platform_store.py
View File

@@ -88,6 +88,22 @@ def parse_size_bytes(value: Any) -> int:
return max(0, round(amount * _SIZE_UNIT_BYTES[unit]))
def count_dataset_records(content: str) -> int:
text = (content or "").strip()
if not text:
return 0
try:
value = json.loads(text)
if isinstance(value, list):
return len(value)
return 1
except (TypeError, ValueError, json.JSONDecodeError):
pass
return len([line for line in text.splitlines() if line.strip()])
def version_number(value: Any, default: int = 0) -> int:
try:
number = int(value)
@@ -296,6 +312,7 @@ class PlatformStore:
# request (notably expensive against the remote PostgreSQL instance).
# TCP keepalive 让操作系统持续保活连接，抵抗远程库空闲静默断连。
pool_kwargs = {
"connect_timeout": 5,
"keepalives": 1,
"keepalives_idle": 30,
"keepalives_interval": 10,
@@ -1311,6 +1328,7 @@ class PlatformStore:
file_size_bytes = int(file_row.get("size_bytes") or 0)
if file_size_bytes <= 0:
file_size_bytes = parse_size_bytes(file_row.get("size"))
file_record_count = int(file_row.get("record_count") or 0)
decoded_files.append(
{
"id": file_row["id"],
@@ -1319,7 +1337,7 @@ class PlatformStore:
"size_bytes": file_size_bytes,
**dataset_file_version_summary(file_row),
"create_time": file_row["create_time"],
"record_count": int(file_row.get("record_count") or 0),
"record_count": file_record_count,
"split": metadata.get("file_split"),
}
)
@@ -1337,6 +1355,9 @@ class PlatformStore:
total_size_bytes = int(row.get("size_bytes") or 0)
if total_size_bytes <= 0:
total_size_bytes = parse_size_bytes(row.get("size"))
total_record_count = sum(int(item.get("record_count") or 0) for item in decoded_files)
if not decoded_files:
total_record_count = int(row.get("record_count") or row.get("count") or 0)
current_version_nos = sorted(
{
int(item["current_version_no"])
@@ -1346,6 +1367,8 @@ class PlatformStore:
)
return {
**dict(row),
"count": total_record_count,
"record_count": total_record_count,
"size_bytes": total_size_bytes,
"current_version_no": (
current_version_nos[0] if len(current_version_nos) == 1 else None
@@ -1420,7 +1443,7 @@ class PlatformStore:
version_id = f"{file_id}_v1"
size_bytes = len(content.encode("utf-8"))
size = f"{size_bytes} B"
record_count = len([line for line in content.splitlines() if line.strip()])
record_count = count_dataset_records(content)
version = {
"id": version_id,
"version": 1,
@@ -1453,10 +1476,18 @@ class PlatformStore:
)
conn.execute(
"""UPDATE datasets
SET count=count+?, record_count=record_count+?,
size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B')
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=?
) stats
WHERE id=?""",
(record_count, record_count, size_bytes, size_bytes, dataset_id),
(dataset_id, dataset_id),
)
return {
"id": file_id,
@@ -1561,19 +1592,57 @@ class PlatformStore:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
content = payload.get("content", "")
size_bytes = len(content.encode("utf-8"))
record_count = count_dataset_records(content)
versions = json_loads(row["versions"], [])
version = {
"id": f"{file_id}_v{len(versions) + 1}",
"version": len(versions) + 1,
"version_no": len(versions) + 1,
"create_time": utcnow(),
"description": payload.get("description", "online edit"),
"size_bytes": size_bytes,
"record_count": record_count,
}
versions.append(version)
conn.execute(
"UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?",
(payload.get("content", ""), version["id"], json_dumps(versions), file_id),
"""
UPDATE dataset_files
SET content=?, active_version_id=?, current_version_id=?, versions=?,
size_bytes=?, size=?, record_count=?, version_no=?
WHERE id=?
""",
(
content,
version["id"],
version["id"],
json_dumps(versions),
size_bytes,
f"{size_bytes} B",
record_count,
version["version_no"],
file_id,
),
)
return {"version": version, "content": payload.get("content", "")}
conn.execute(
"""UPDATE datasets
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT dataset_id,
COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=(SELECT dataset_id FROM dataset_files WHERE id=?)
GROUP BY dataset_id
) stats
WHERE datasets.id=stats.dataset_id""",
(file_id,),
)
return {"version": version, "content": content}
def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
with self.connect() as conn:
@@ -2173,6 +2242,14 @@ class PlatformStore:
"sample_count": result_content.get("sample_count", 0),
"completed_count": result_content.get("completed_count", 0),
"passed_count": result_content.get("passed_count", 0),
"basic_metrics": result_content.get("basic_metrics", {}),
"score": result_content.get("overall_score", 0),
"completed_time": utcnow(),
})
elif new_status in {"failed", "stopped"}:
updates.update({
"error": job.get("error") or task.get("error") or "",
"completed_time": utcnow(),
})
return self.update_eval_task(task_id, updates)
@@ -2342,6 +2419,15 @@ class PlatformStore:
).fetchall()
return {int(row["gpu_index"]) for row in rows}
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
if rows:
return {int(row["gpu_index"]) for row in rows}
return set(range(max(0, int(node.get("gpu_count") or 0))))
def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []]
@@ -2349,13 +2435,15 @@ class PlatformStore:
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"]
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if not set(requested_gpus).intersection(self._active_gpu_indexes(conn, node["id"]))
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
@@ -2370,8 +2458,8 @@ class PlatformStore:
reason = "disabled"
elif node["scheduler_status"] != "online":
reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= node["max_parallel_jobs"]:
reason = f"capacity full {node['current_running_jobs']}/{node['max_parallel_jobs']}"
elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
else:
reason = "not selected"
reasons.append(f"{node['code']}({reason})")
@@ -2722,6 +2810,24 @@ class PlatformStore:
)
return next(node for node in self.compute_nodes() if node["id"] == node_id)
def delete_compute_node(self, node_id: str) -> dict[str, Any]:
with self.connect() as conn:
node = conn.execute("SELECT * FROM compute_nodes WHERE id=?", (node_id,)).fetchone()
if not node:
raise KeyError(node_id)
active = conn.execute(
"""
SELECT COUNT(*) AS cnt
FROM fine_tune_tasks
WHERE compute_node_id=? AND status IN ('syncing','queued','running')
""",
(node_id,),
).fetchone()
if active and int(active["cnt"] or 0) > 0:
raise ValueError("compute node has active training tasks")
conn.execute("DELETE FROM compute_nodes WHERE id=?", (node_id,))
return {"deleted": node_id}
def update_compute_node_health(self, node_id: str, health: dict[str, Any], success: bool, error: str | None = None) -> dict[str, Any]:
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if not current:
@@ -2890,11 +2996,12 @@ class PlatformStore:
}
def health_metrics(self) -> dict[str, float]:
info = self.system_info()
# Health checks must stay lightweight. The Docker healthcheck and page
# refresh probes should not wait on dashboard/GPU/database aggregation.
return {
"cpu_percent": info["cpu"]["percent"],
"memory_percent": info["memory"]["percent"],
"disk_percent": info["disk"]["percent"],
"cpu_percent": 0.0,
"memory_percent": 0.0,
"disk_percent": 0.0,
}
def queue(self) -> list[dict[str, Any]]: