YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新前端看板

Browse Source
This commit is contained in:
wangjiming
2026-08-03 16:33:08 +08:00
parent 0c601934a0 5cc306eb0a
commit 94230cad16
37 changed files with 1663 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
backend/app/db/platform_store.py
View File

@@ -88,6 +88,22 @@ def parse_size_bytes(value: Any) -> int:
return max(0, round(amount * _SIZE_UNIT_BYTES[unit]))
def count_dataset_records(content: str) -> int:
text = (content or "").strip()
if not text:
return 0
try:
value = json.loads(text)
if isinstance(value, list):
return len(value)
return 1
except (TypeError, ValueError, json.JSONDecodeError):
pass
return len([line for line in text.splitlines() if line.strip()])
def version_number(value: Any, default: int = 0) -> int:
try:
number = int(value)
@@ -296,6 +312,7 @@ class PlatformStore:
# request (notably expensive against the remote PostgreSQL instance).
# TCP keepalive 让操作系统持续保活连接，抵抗远程库空闲静默断连。
pool_kwargs = {
"connect_timeout": 5,
"keepalives": 1,
"keepalives_idle": 10,
"keepalives_interval": 5,
@@ -320,6 +337,19 @@ class PlatformStore:
self._pool.open()
self.ensure_schema()
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
self._inference_nodes: set[str] = set()
# ── inference node tracking ────────────────────────────────────
def mark_inference_loaded(self, node_id: str) -> None:
self._inference_nodes.add(node_id)
def mark_inference_unloaded(self, node_id: str) -> None:
self._inference_nodes.discard(node_id)
def is_inference_loaded(self, node_id: str) -> bool:
return node_id in self._inference_nodes
@contextmanager
def connect(self) -> Iterator["PgConnection"]:
@@ -1298,6 +1328,7 @@ class PlatformStore:
file_size_bytes = int(file_row.get("size_bytes") or 0)
if file_size_bytes <= 0:
file_size_bytes = parse_size_bytes(file_row.get("size"))
file_record_count = int(file_row.get("record_count") or 0)
decoded_files.append(
{
"id": file_row["id"],
@@ -1306,7 +1337,7 @@ class PlatformStore:
"size_bytes": file_size_bytes,
**dataset_file_version_summary(file_row),
"create_time": file_row["create_time"],
"record_count": int(file_row.get("record_count") or 0),
"record_count": file_record_count,
"split": metadata.get("file_split"),
}
)
@@ -1324,6 +1355,9 @@ class PlatformStore:
total_size_bytes = int(row.get("size_bytes") or 0)
if total_size_bytes <= 0:
total_size_bytes = parse_size_bytes(row.get("size"))
total_record_count = sum(int(item.get("record_count") or 0) for item in decoded_files)
if not decoded_files:
total_record_count = int(row.get("record_count") or row.get("count") or 0)
current_version_nos = sorted(
{
int(item["current_version_no"])
@@ -1333,6 +1367,8 @@ class PlatformStore:
)
return {
**dict(row),
"count": total_record_count,
"record_count": total_record_count,
"size_bytes": total_size_bytes,
"current_version_no": (
current_version_nos[0] if len(current_version_nos) == 1 else None
@@ -1407,7 +1443,7 @@ class PlatformStore:
version_id = f"{file_id}_v1"
size_bytes = len(content.encode("utf-8"))
size = f"{size_bytes} B"
record_count = len([line for line in content.splitlines() if line.strip()])
record_count = count_dataset_records(content)
version = {
"id": version_id,
"version": 1,
@@ -1440,10 +1476,18 @@ class PlatformStore:
)
conn.execute(
"""UPDATE datasets
SET count=count+?, record_count=record_count+?,
size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B')
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=?
) stats
WHERE id=?""",
(record_count, record_count, size_bytes, size_bytes, dataset_id),
(dataset_id, dataset_id),
)
return {
"id": file_id,
@@ -1548,19 +1592,57 @@ class PlatformStore:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
content = payload.get("content", "")
size_bytes = len(content.encode("utf-8"))
record_count = count_dataset_records(content)
versions = json_loads(row["versions"], [])
version = {
"id": f"{file_id}_v{len(versions) + 1}",
"version": len(versions) + 1,
"version_no": len(versions) + 1,
"create_time": utcnow(),
"description": payload.get("description", "online edit"),
"size_bytes": size_bytes,
"record_count": record_count,
}
versions.append(version)
conn.execute(
"UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?",
(payload.get("content", ""), version["id"], json_dumps(versions), file_id),
"""
UPDATE dataset_files
SET content=?, active_version_id=?, current_version_id=?, versions=?,
size_bytes=?, size=?, record_count=?, version_no=?
WHERE id=?
""",
(
content,
version["id"],
version["id"],
json_dumps(versions),
size_bytes,
f"{size_bytes} B",
record_count,
version["version_no"],
file_id,
),
)
return {"version": version, "content": payload.get("content", "")}
conn.execute(
"""UPDATE datasets
SET count=stats.record_count,
record_count=stats.record_count,
size_bytes=stats.size_bytes,
size=(stats.size_bytes::text || ' B')
FROM (
SELECT dataset_id,
COALESCE(SUM(record_count), 0) AS record_count,
COALESCE(SUM(size_bytes), 0) AS size_bytes
FROM dataset_files
WHERE dataset_id=(SELECT dataset_id FROM dataset_files WHERE id=?)
GROUP BY dataset_id
) stats
WHERE datasets.id=stats.dataset_id""",
(file_id,),
)
return {"version": version, "content": content}
def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
with self.connect() as conn:
@@ -2114,10 +2196,63 @@ class PlatformStore:
)
return self.eval_task(task_id)
def update_eval_task(self, task_id: str, updates: dict[str, Any]) -> dict[str, Any]:
"""Update fields in an eval task's payload without replacing the whole record."""
task = self.eval_task(task_id)
merged = {**task, **updates}
with self.connect() as conn:
conn.execute(
"UPDATE eval_tasks SET payload=?, status=? WHERE id=?",
(json_dumps(merged), merged.get("status", task.get("status", "pending")), task_id),
)
return self.eval_task(task_id)
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
return [
task for task in self.eval_tasks()
if task.get("compute_job_id") and task.get("status") in {"queued", "running"}
]
def apply_eval_job_result(self, task_id: str, job: dict[str, Any], result_content: dict[str, Any] | None = None) -> dict[str, Any]:
"""Sync a compute job status/result back to an eval task."""
task = self.eval_task(task_id)
job_status = str(job.get("status", ""))
status_map = {"queued": "running", "running": "running", "completed": "completed",
"failed": "failed", "stopped": "stopped"}
new_status = status_map.get(job_status, job_status or task.get("status", "pending"))
updates: dict[str, Any] = {
"status": new_status,
"progress": int(job.get("progress", 0)),
"output_dir": job.get("output_dir", task.get("output_dir", "")),
}
# On completion, populate results from eval_results.json content
if new_status == "completed" and result_content:
updates.update({
"overall_score": result_content.get("overall_score", 0),
"overall_score_max": result_content.get("overall_score_max", 100),
"overall_evaluation": result_content.get("overall_evaluation", ""),
"improvement_suggestions": result_content.get("improvement_suggestions", []),
"dimension_summary": result_content.get("dimension_summary", []),
"samples": result_content.get("samples", []),
"sample_count": result_content.get("sample_count", 0),
"completed_count": result_content.get("completed_count", 0),
"passed_count": result_content.get("passed_count", 0),
"basic_metrics": result_content.get("basic_metrics", {}),
"score": result_content.get("overall_score", 0),
"completed_time": utcnow(),
})
elif new_status in {"failed", "stopped"}:
updates.update({
"error": job.get("error") or task.get("error") or "",
"completed_time": utcnow(),
})
return self.update_eval_task(task_id, updates)
def dimensions(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
@@ -2284,6 +2419,15 @@ class PlatformStore:
).fetchall()
return {int(row["gpu_index"]) for row in rows}
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
if rows:
return {int(row["gpu_index"]) for row in rows}
return set(range(max(0, int(node.get("gpu_count") or 0))))
def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []]
@@ -2291,13 +2435,15 @@ class PlatformStore:
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"]
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if not set(requested_gpus).intersection(self._active_gpu_indexes(conn, node["id"]))
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
@@ -2312,8 +2458,8 @@ class PlatformStore:
reason = "disabled"
elif node["scheduler_status"] != "online":
reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= node["max_parallel_jobs"]:
reason = f"capacity full {node['current_running_jobs']}/{node['max_parallel_jobs']}"
elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
else:
reason = "not selected"
reasons.append(f"{node['code']}({reason})")
@@ -2664,6 +2810,24 @@ class PlatformStore:
)
return next(node for node in self.compute_nodes() if node["id"] == node_id)
def delete_compute_node(self, node_id: str) -> dict[str, Any]:
with self.connect() as conn:
node = conn.execute("SELECT * FROM compute_nodes WHERE id=?", (node_id,)).fetchone()
if not node:
raise KeyError(node_id)
active = conn.execute(
"""
SELECT COUNT(*) AS cnt
FROM fine_tune_tasks
WHERE compute_node_id=? AND status IN ('syncing','queued','running')
""",
(node_id,),
).fetchone()
if active and int(active["cnt"] or 0) > 0:
raise ValueError("compute node has active training tasks")
conn.execute("DELETE FROM compute_nodes WHERE id=?", (node_id,))
return {"deleted": node_id}
def update_compute_node_health(self, node_id: str, health: dict[str, Any], success: bool, error: str | None = None) -> dict[str, Any]:
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if not current:
@@ -2749,6 +2913,11 @@ class PlatformStore:
)
busy = task is not None and task.get("status") == "running"
reserved = task is not None and task.get("status") in {"syncing", "queued"}
# Also mark GPU as busy if an inference model is loaded on this node
inference_busy = self.is_inference_loaded(row["node_id"])
if inference_busy and not busy:
busy = True
reserved = False
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
gpu_percent = 86 if busy else 22 if reserved else 3
memory_total = float(row["memory_total_gb"] or 0)
@@ -2827,11 +2996,12 @@ class PlatformStore:
}
def health_metrics(self) -> dict[str, float]:
info = self.system_info()
# Health checks must stay lightweight. The Docker healthcheck and page
# refresh probes should not wait on dashboard/GPU/database aggregation.
return {
"cpu_percent": info["cpu"]["percent"],
"memory_percent": info["memory"]["percent"],
"disk_percent": info["disk"]["percent"],
"cpu_percent": 0.0,
"memory_percent": 0.0,
"disk_percent": 0.0,
}
def queue(self) -> list[dict[str, Any]]: