feat: 新增 compute_gateway、compute_poller、agent 模块，重构前端 dist
- 新增 backend/app/modules/compute_gateway（client/sync）计算网关模块 - 新增 backend/app/workers/compute_poller 计算轮询 worker - 新增 compute/agent/process_manager 进程管理器 - 新增 scripts/ 脚本目录 - 更新 Docker 部署配置（app/compute/nginx） - 更新后端平台 API、数据库 SQL、core 配置 - 更新前端多个视图组件及 API 模块 - 重构 frontend/dist 构建产物（新 hash） - 更新多项文档 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -53,6 +53,13 @@ def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def safe_float(value: Any, default: float = 0) -> float:
|
||||
try:
|
||||
return float(str(value).replace("[N/A]", "").strip() or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
@@ -178,12 +185,33 @@ class PlatformStore:
|
||||
schema_path = Path(__file__).with_name("sql") / "001_platform_runtime.sql"
|
||||
with self.connect() as conn:
|
||||
conn.executescript(schema_path.read_text(encoding="utf-8"))
|
||||
columns = conn.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='users'"
|
||||
).fetchall()
|
||||
column_names = {row["column_name"] for row in columns}
|
||||
if "password" in column_names and "password_hash" not in column_names:
|
||||
user_columns = self._column_names(conn, "users")
|
||||
if "password" in user_columns and "password_hash" not in user_columns:
|
||||
conn.execute("ALTER TABLE users RENAME COLUMN password TO password_hash")
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"compute_nodes",
|
||||
{
|
||||
"api_version": "TEXT NOT NULL DEFAULT 'v1'",
|
||||
"capabilities": "TEXT NOT NULL DEFAULT '[]'",
|
||||
"description": "TEXT",
|
||||
},
|
||||
)
|
||||
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
|
||||
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
|
||||
|
||||
def _column_names(self, conn: PgConnection, table_name: str) -> set[str]:
|
||||
columns = conn.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name=?",
|
||||
(table_name,),
|
||||
).fetchall()
|
||||
return {row["column_name"] for row in columns}
|
||||
|
||||
def _ensure_columns(self, conn: PgConnection, table_name: str, columns: dict[str, str]) -> None:
|
||||
existing = self._column_names(conn, table_name)
|
||||
for column, definition in columns.items():
|
||||
if column not in existing:
|
||||
conn.execute(f"ALTER TABLE {table_name} ADD COLUMN {column} {definition}")
|
||||
|
||||
def ensure_seed_data(self) -> None:
|
||||
with self.connect() as conn:
|
||||
@@ -436,7 +464,7 @@ class PlatformStore:
|
||||
utcnow(),
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone())
|
||||
|
||||
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.model(model_id)
|
||||
@@ -461,7 +489,7 @@ class PlatformStore:
|
||||
model_id,
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone())
|
||||
|
||||
def delete_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
@@ -481,6 +509,10 @@ class PlatformStore:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def delete_trained_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id))
|
||||
|
||||
def datasets(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM datasets ORDER BY create_time DESC").fetchall()
|
||||
@@ -641,6 +673,26 @@ class PlatformStore:
|
||||
conn.execute("UPDATE dataset_files SET active_version_id=? WHERE id=?", (version_id, file_id))
|
||||
return {"version": version, "content": row["content"]}
|
||||
|
||||
def delete_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(file_id)
|
||||
versions = json_loads(row["versions"], [])
|
||||
if row["active_version_id"] == version_id:
|
||||
raise ValueError("active dataset version cannot be deleted")
|
||||
if len(versions) <= 1:
|
||||
raise ValueError("last dataset version cannot be deleted")
|
||||
next_versions = [item for item in versions if item["id"] != version_id]
|
||||
if len(next_versions) == len(versions):
|
||||
raise KeyError(version_id)
|
||||
conn.execute("UPDATE dataset_files SET versions=? WHERE id=?", (json_dumps(next_versions), file_id))
|
||||
return {
|
||||
"versions": next_versions,
|
||||
"active_version_id": row["active_version_id"],
|
||||
"next_version_number": max(item.get("version", 0) for item in next_versions) + 1,
|
||||
}
|
||||
|
||||
def tasks(self) -> list[dict[str, Any]]:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
@@ -668,6 +720,8 @@ class PlatformStore:
|
||||
"train_duration": self._duration(row["start_time"], row["completed_at"]) if row["start_time"] else "",
|
||||
"compute_node_id": row["compute_node_id"],
|
||||
"sync_job_id": row["sync_job_id"],
|
||||
"compute_job_id": row.get("compute_job_id"),
|
||||
"completed_at": row.get("completed_at"),
|
||||
}
|
||||
)
|
||||
return payload
|
||||
@@ -689,6 +743,7 @@ class PlatformStore:
|
||||
"status": "pending",
|
||||
"train_type": payload.get("train_type", "SFT"),
|
||||
"train_method": payload.get("train_method", "lora"),
|
||||
"engine": payload.get("engine", payload.get("training_engine", "llama_factory")),
|
||||
"template": payload.get("template", "qwen"),
|
||||
"base_model": base_model,
|
||||
"train_dataset_id": train_dataset_id,
|
||||
@@ -751,7 +806,7 @@ class PlatformStore:
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status='syncing', progress=8, process_id=?, start_time=?,
|
||||
compute_node_id=?, gpus=?, sync_job_id=?
|
||||
compute_node_id=?, gpus=?, sync_job_id=?, compute_job_id=NULL
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
@@ -766,9 +821,140 @@ class PlatformStore:
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
def reset_task_for_retry(self, task_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
current = self.task(task_id)
|
||||
override = payload or {}
|
||||
merged = {
|
||||
**current,
|
||||
**override,
|
||||
"id": task_id,
|
||||
"status": "pending",
|
||||
"progress": 0,
|
||||
"process_id": None,
|
||||
"compute_job_id": None,
|
||||
}
|
||||
for runtime_key in ["failure_reason", "log_file", "artifacts"]:
|
||||
merged.pop(runtime_key, None)
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status='pending', progress=0, process_id=NULL, start_time=NULL,
|
||||
completed_at=NULL, compute_node_id=NULL, gpus=?, sync_job_id=NULL, compute_job_id=NULL
|
||||
WHERE id=?
|
||||
""",
|
||||
(json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def update_task_priority(self, task_id: str, priority: str) -> dict[str, Any]:
|
||||
current = self.task(task_id)
|
||||
priority = priority if priority in {"low", "normal", "high", "urgent"} else "normal"
|
||||
merged = {**current, "priority": priority}
|
||||
with self.connect() as conn:
|
||||
conn.execute("UPDATE fine_tune_tasks SET payload=? WHERE id=?", (json_dumps(merged), task_id))
|
||||
return self.task(task_id)
|
||||
|
||||
def prepare_compute_job_payload(self, task_id: str, payload: dict[str, Any] | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
|
||||
merged = {**task, **(payload or {}), "id": task_id}
|
||||
node = self.schedule_node(merged)
|
||||
selected_gpus = merged.get("gpus") or [0]
|
||||
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
|
||||
|
||||
def _compute_job_payload_from_task_node(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
node: dict[str, Any],
|
||||
selected_gpus: list[int] | list[Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT * FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
||||
dataset = conn.execute("SELECT * FROM datasets WHERE id=?", (task.get("train_dataset_id"),)).fetchone()
|
||||
model_path = (model and model.get("path")) or task.get("base_model")
|
||||
dataset_name = task.get("dataset") or (dataset and dataset.get("name")) or task.get("train_dataset_id")
|
||||
health_detail = node.get("health_detail") or {}
|
||||
dataset_root = str(health_detail.get("dataset_root") or f"{node['data_root'].rstrip('/')}/datasets")
|
||||
output_root = str(health_detail.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
|
||||
output_dir = task.get("output_dir") or f"{output_root.rstrip('/')}/{task['name']}"
|
||||
return {
|
||||
**task,
|
||||
"id": task["id"],
|
||||
"name": task["name"],
|
||||
"base_model": model_path,
|
||||
"model_name_or_path": model_path,
|
||||
"dataset": dataset_name,
|
||||
"dataset_dir": dataset_root,
|
||||
"output_dir": output_dir,
|
||||
"gpus": selected_gpus or task.get("gpus") or [0],
|
||||
"compute_node_id": node["id"],
|
||||
"compute_node_code": node["code"],
|
||||
}
|
||||
|
||||
def build_compute_job_payload(self, task_id: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
task = self.task(task_id)
|
||||
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
|
||||
if not node:
|
||||
raise RuntimeError("compute node not found")
|
||||
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0])
|
||||
|
||||
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
status_map = {
|
||||
"queued": "queued",
|
||||
"running": "running",
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
"stopped": "stopped",
|
||||
}
|
||||
current = self.task(task_id)
|
||||
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
|
||||
progress = int(job.get("progress", current.get("progress", 0)) or 0)
|
||||
payload = {
|
||||
**current,
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"process_id": job.get("pid") or current.get("process_id"),
|
||||
"compute_job_id": job.get("id") or current.get("compute_job_id"),
|
||||
"output_dir": job.get("output_dir") or current.get("output_dir"),
|
||||
"log_file": job.get("log_file") or current.get("log_file"),
|
||||
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
|
||||
}
|
||||
if status == "failed":
|
||||
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
|
||||
elif status in {"queued", "running", "completed"}:
|
||||
payload.pop("failure_reason", None)
|
||||
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
json_dumps(payload),
|
||||
status,
|
||||
progress,
|
||||
payload.get("process_id"),
|
||||
payload.get("compute_job_id"),
|
||||
completed_at,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
if status == "completed":
|
||||
self._ensure_trained_model(conn, payload)
|
||||
return self.task(task_id)
|
||||
|
||||
def running_compute_tasks(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
task
|
||||
for task in self.tasks()
|
||||
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
|
||||
]
|
||||
|
||||
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
||||
@@ -776,10 +962,179 @@ class PlatformStore:
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str, status: str = "stopped") -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": status, "progress": min(task.get("progress", 0), 99)})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status=?, payload=?, completed_at=? WHERE id=?",
|
||||
(status, json_dumps(task), utcnow(), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def delete_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||
payload = json_loads(row["payload"], {})
|
||||
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||
return payload
|
||||
|
||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
payload = self._json_payload_row(row)
|
||||
payload.setdefault("sample_count", 0)
|
||||
payload.setdefault("completed_count", 0)
|
||||
payload.setdefault("passed_count", 0)
|
||||
payload.setdefault("overall_score", payload.get("score") or 0)
|
||||
payload.setdefault("overall_score_max", 100)
|
||||
payload.setdefault("overall_evaluation", "")
|
||||
payload.setdefault("improvement_suggestions", [])
|
||||
payload.setdefault("dimension_summary", [])
|
||||
payload.setdefault("samples", [])
|
||||
return payload
|
||||
|
||||
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
|
||||
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {
|
||||
**payload,
|
||||
"id": task_id,
|
||||
"eval_task_name": name,
|
||||
"status": status,
|
||||
"create_time": now,
|
||||
"metric": payload.get("metric") or "custom",
|
||||
}
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||
if model:
|
||||
data.setdefault("model_name", model["name"])
|
||||
if dataset:
|
||||
data.setdefault("dataset", dataset["name"])
|
||||
conn.execute(
|
||||
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.eval_task(task_id)
|
||||
|
||||
def delete_eval_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def dimensions(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
|
||||
return [
|
||||
{
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def dimension(self, dimension_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(dimension_id)
|
||||
return {
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
|
||||
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
dimension_id = str(payload.get("id") or new_id("dim"))
|
||||
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.dimension(dimension_id)
|
||||
merged = {**current, **payload, "id": dimension_id}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
|
||||
(
|
||||
merged["name"],
|
||||
json_dumps(merged),
|
||||
1 if merged.get("is_active", True) else 0,
|
||||
1 if merged.get("is_default") else 0,
|
||||
dimension_id,
|
||||
),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def delete_dimension(self, dimension_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
|
||||
|
||||
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
return self._json_payload_row(row)
|
||||
|
||||
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or new_id("cmp"))
|
||||
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
|
||||
data.setdefault("load_status", json_dumps({"loaded_models": []}))
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.compare_task(task_id)
|
||||
merged = {**current, **payload, "id": task_id}
|
||||
status = str(merged.get("status") or current.get("status") or "pending")
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
|
||||
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def delete_compare_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def schedule_node(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
|
||||
nodes = self.compute_nodes()
|
||||
@@ -793,7 +1148,20 @@ class PlatformStore:
|
||||
if selected:
|
||||
return selected
|
||||
if not candidates:
|
||||
raise RuntimeError("no available compute node")
|
||||
if not nodes:
|
||||
raise RuntimeError("no available compute node: no compute node configured")
|
||||
reasons = []
|
||||
for node in nodes:
|
||||
if not node["enabled"]:
|
||||
reason = "disabled"
|
||||
elif node["scheduler_status"] != "online":
|
||||
reason = f"status={node['scheduler_status']}"
|
||||
elif node["current_running_jobs"] >= node["max_parallel_jobs"]:
|
||||
reason = f"capacity full {node['current_running_jobs']}/{node['max_parallel_jobs']}"
|
||||
else:
|
||||
reason = "not selected"
|
||||
reasons.append(f"{node['code']}({reason})")
|
||||
raise RuntimeError(f"no available compute node: {', '.join(reasons)}")
|
||||
return sorted(candidates, key=lambda n: (-n["scheduler_weight"], n["current_running_jobs"], n["code"]))[0]
|
||||
|
||||
def create_sync_job(self, node_id: str, task: dict[str, Any]) -> str:
|
||||
@@ -809,7 +1177,8 @@ class PlatformStore:
|
||||
sync_id,
|
||||
node_id,
|
||||
json_dumps(
|
||||
[
|
||||
task.get("resources")
|
||||
or [
|
||||
{"resource_type": "model", "resource_id": task.get("base_model")},
|
||||
{"resource_type": "dataset", "resource_id": task.get("train_dataset_id")},
|
||||
]
|
||||
@@ -819,6 +1188,46 @@ class PlatformStore:
|
||||
)
|
||||
return sync_id
|
||||
|
||||
def update_sync_job(self, sync_id: str, status: str, progress: int, completed: bool = False) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=COALESCE(?, completed_at) WHERE id=?",
|
||||
(status, progress, utcnow() if completed else None, sync_id),
|
||||
)
|
||||
return self.sync_job(sync_id)
|
||||
|
||||
def upsert_resource_replica(
|
||||
self,
|
||||
node_id: str,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
local_path: str,
|
||||
status: str = "available",
|
||||
sync_status: str = "synced",
|
||||
) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM resource_replicas WHERE node_id=? AND resource_type=? AND resource_id=?",
|
||||
(node_id, resource_type, resource_id),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE resource_replicas SET local_path=?, status=?, sync_status=? WHERE id=?",
|
||||
(local_path, status, sync_status, row["id"]),
|
||||
)
|
||||
replica_id = row["id"]
|
||||
else:
|
||||
replica_id = new_id("replica")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO resource_replicas
|
||||
(id, node_id, resource_type, resource_id, local_path, status, sync_status, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(replica_id, node_id, resource_type, resource_id, local_path, status, sync_status, utcnow()),
|
||||
)
|
||||
return dict(conn.execute("SELECT * FROM resource_replicas WHERE id=?", (replica_id,)).fetchone())
|
||||
|
||||
def progress(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
status = task.get("status", "pending")
|
||||
@@ -829,6 +1238,7 @@ class PlatformStore:
|
||||
"running": "training with LLaMA-Factory",
|
||||
"completed": "training completed",
|
||||
"failed": "training stopped",
|
||||
"stopped": "training stopped",
|
||||
}
|
||||
progress = int(task.get("progress", 0) or 0)
|
||||
eta = "--" if status in {"completed", "failed"} else f"{max(1, math.ceil((100 - progress) / 10))} min"
|
||||
@@ -853,23 +1263,62 @@ class PlatformStore:
|
||||
**dict(row),
|
||||
"enabled": bool(row["enabled"]),
|
||||
"tags": json_loads(row["tags"], []),
|
||||
"capabilities": json_loads(row.get("capabilities"), []),
|
||||
"health_detail": json_loads(row["health_detail"], {}),
|
||||
"current_running_jobs": running_map.get(row["id"], 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def _normalize_tags(self, value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
parts = value.replace("，", ",").split(",")
|
||||
return [item.strip() for item in parts if item.strip()]
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return []
|
||||
|
||||
def _normalize_compute_node_payload(self, payload: dict[str, Any], current: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
merged = {**(current or {}), **payload}
|
||||
api_base_url = str(merged.get("api_base_url") or "").rstrip("/")
|
||||
if not api_base_url:
|
||||
raise ValueError("api_base_url is required")
|
||||
file_gateway_url = str(merged.get("file_gateway_url") or api_base_url).rstrip("/")
|
||||
weight = max(0, min(1000, int(merged.get("scheduler_weight", 100))))
|
||||
max_jobs = max(1, int(merged.get("max_parallel_jobs", 1)))
|
||||
return {
|
||||
**merged,
|
||||
"code": str(merged.get("code") or "").strip(),
|
||||
"name": str(merged.get("name") or merged.get("code") or "").strip(),
|
||||
"api_base_url": api_base_url,
|
||||
"file_gateway_url": file_gateway_url,
|
||||
"enabled": bool(merged.get("enabled", True)),
|
||||
"scheduler_status": str(merged.get("scheduler_status") or "offline"),
|
||||
"scheduler_weight": weight,
|
||||
"tags": self._normalize_tags(merged.get("tags")),
|
||||
"gpu_count": max(0, int(merged.get("gpu_count", 0) or 0)),
|
||||
"max_parallel_jobs": max_jobs,
|
||||
"data_root": str(merged.get("data_root") or "/data/yg-ft"),
|
||||
"model_root": str(merged.get("model_root") or "/data/yg-ft/models"),
|
||||
"log_root": str(merged.get("log_root") or "/opt/yg-ft/logs/training"),
|
||||
"api_version": str(merged.get("api_version") or "v1"),
|
||||
"capabilities": merged.get("capabilities") or [],
|
||||
"description": merged.get("description") or "",
|
||||
"health_detail": merged.get("health_detail") or {"status": "registered"},
|
||||
}
|
||||
|
||||
def update_compute_node(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||
if not current:
|
||||
raise KeyError(node_id)
|
||||
merged = {**current, **payload}
|
||||
merged = self._normalize_compute_node_payload(payload, current)
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE compute_nodes
|
||||
SET name=?, api_base_url=?, file_gateway_url=?, enabled=?, scheduler_status=?,
|
||||
scheduler_weight=?, tags=?, max_parallel_jobs=?, last_health_check_at=?
|
||||
scheduler_weight=?, tags=?, max_parallel_jobs=?, data_root=?, model_root=?, log_root=?,
|
||||
api_version=?, capabilities=?, description=?, last_health_check_at=?, health_detail=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
@@ -881,46 +1330,116 @@ class PlatformStore:
|
||||
merged["scheduler_weight"],
|
||||
json_dumps(merged["tags"]),
|
||||
merged["max_parallel_jobs"],
|
||||
utcnow(),
|
||||
merged["data_root"],
|
||||
merged["model_root"],
|
||||
merged["log_root"],
|
||||
merged["api_version"],
|
||||
json_dumps(merged["capabilities"]),
|
||||
merged["description"],
|
||||
payload.get("last_health_check_at") or current.get("last_health_check_at"),
|
||||
json_dumps(merged["health_detail"]),
|
||||
node_id,
|
||||
),
|
||||
)
|
||||
return next(n for n in self.compute_nodes() if n["id"] == node_id)
|
||||
|
||||
def create_compute_node(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = self._normalize_compute_node_payload(payload)
|
||||
if not payload["code"]:
|
||||
raise ValueError("code is required")
|
||||
node_id = payload.get("id") or new_id("node")
|
||||
now = utcnow()
|
||||
tags = payload.get("tags") or []
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO compute_nodes
|
||||
(id, code, name, api_base_url, file_gateway_url, enabled, scheduler_status,
|
||||
scheduler_weight, tags, gpu_count, current_running_jobs, max_parallel_jobs,
|
||||
data_root, model_root, log_root, last_health_check_at, health_detail)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?)
|
||||
data_root, model_root, log_root, api_version, capabilities, description,
|
||||
last_health_check_at, health_detail)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
node_id,
|
||||
payload["code"],
|
||||
payload.get("name") or payload["code"],
|
||||
payload["name"] or payload["code"],
|
||||
payload["api_base_url"],
|
||||
payload.get("file_gateway_url") or payload["api_base_url"],
|
||||
1 if payload.get("enabled", True) else 0,
|
||||
payload.get("scheduler_status", "offline"),
|
||||
int(payload.get("scheduler_weight", 100)),
|
||||
json_dumps(tags),
|
||||
int(payload.get("gpu_count", 0)),
|
||||
int(payload.get("max_parallel_jobs", 1)),
|
||||
payload.get("data_root", "/data/yg-ft"),
|
||||
payload.get("model_root", "/models"),
|
||||
payload.get("log_root", "/data/yg-ft/training-logs"),
|
||||
payload["file_gateway_url"],
|
||||
1 if payload["enabled"] else 0,
|
||||
payload["scheduler_status"],
|
||||
payload["scheduler_weight"],
|
||||
json_dumps(payload["tags"]),
|
||||
payload["gpu_count"],
|
||||
payload["max_parallel_jobs"],
|
||||
payload["data_root"],
|
||||
payload["model_root"],
|
||||
payload["log_root"],
|
||||
payload["api_version"],
|
||||
json_dumps(payload["capabilities"]),
|
||||
payload["description"],
|
||||
now,
|
||||
json_dumps(payload.get("health_detail") or {"status": "registered"}),
|
||||
json_dumps(payload["health_detail"]),
|
||||
),
|
||||
)
|
||||
return next(node for node in self.compute_nodes() if node["id"] == node_id)
|
||||
|
||||
def update_compute_node_health(self, node_id: str, health: dict[str, Any], success: bool, error: str | None = None) -> dict[str, Any]:
|
||||
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||
if not current:
|
||||
raise KeyError(node_id)
|
||||
status = "online" if success and current.get("enabled") else "offline"
|
||||
if current.get("scheduler_status") == "draining" and success:
|
||||
status = "draining"
|
||||
detail = {
|
||||
**(current.get("health_detail") or {}),
|
||||
**health,
|
||||
"status": "ok" if success else "failed",
|
||||
"last_error": error or "",
|
||||
"checked_at": utcnow(),
|
||||
}
|
||||
return self.update_compute_node(
|
||||
node_id,
|
||||
{
|
||||
"scheduler_status": status,
|
||||
"last_health_check_at": detail["checked_at"],
|
||||
"health_detail": detail,
|
||||
"data_root": health.get("data_root") or current.get("data_root"),
|
||||
"api_version": str(health.get("api_version") or current.get("api_version") or "v1"),
|
||||
"capabilities": health.get("capabilities") or current.get("capabilities") or [],
|
||||
},
|
||||
)
|
||||
|
||||
def replace_node_gpus(self, node_id: str, gpus: list[dict[str, Any]]) -> None:
|
||||
now = utcnow()
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM gpus WHERE node_id=?", (node_id,))
|
||||
for index, item in enumerate(gpus):
|
||||
gpu_index = int(item.get("gpu_index", item.get("id", index)) or 0)
|
||||
memory_total = safe_float(item.get("memory_total_gb") or item.get("memory_total"))
|
||||
if not memory_total and item.get("memory_total_mb") is not None:
|
||||
memory_total = round(safe_float(item.get("memory_total_mb")) / 1024, 2)
|
||||
power_limit = safe_float(item.get("power_limit_w") or item.get("power_limit"))
|
||||
temperature = int(safe_float(item.get("temperature") or item.get("base_temperature"), 35))
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO gpus
|
||||
(id, node_id, gpu_index, uuid, name, memory_total_gb, power_limit_w, base_temperature, last_seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
f"{node_id}_gpu_{gpu_index}",
|
||||
node_id,
|
||||
gpu_index,
|
||||
str(item.get("uuid") or f"{node_id}-GPU-{gpu_index}"),
|
||||
str(item.get("name") or "Unknown GPU"),
|
||||
memory_total or 0,
|
||||
power_limit or 0,
|
||||
temperature,
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute("UPDATE compute_nodes SET gpu_count=? WHERE id=?", (len(gpus), node_id))
|
||||
|
||||
def gpus(self) -> list[dict[str, Any]]:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
@@ -951,6 +1470,7 @@ class PlatformStore:
|
||||
reserved = task is not None and task.get("status") in {"syncing", "queued"}
|
||||
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
|
||||
gpu_percent = 86 if busy else 22 if reserved else 3
|
||||
memory_total = float(row["memory_total_gb"] or 0)
|
||||
items.append(
|
||||
{
|
||||
"id": row["gpu_index"],
|
||||
@@ -961,8 +1481,8 @@ class PlatformStore:
|
||||
"uuid": row["uuid"],
|
||||
"gpu_percent": gpu_percent,
|
||||
"memory_used_gb": memory_used,
|
||||
"memory_total_gb": row["memory_total_gb"],
|
||||
"memory_percent": round(memory_used / row["memory_total_gb"] * 100, 1),
|
||||
"memory_total_gb": memory_total,
|
||||
"memory_percent": round(memory_used / memory_total * 100, 1) if memory_total else 0,
|
||||
"temperature": row["base_temperature"] + (21 if busy else 6 if reserved else 0),
|
||||
"power_w": round(row["power_limit_w"] * (0.7 if busy else 0.25 if reserved else 0.08), 1),
|
||||
"power_limit_w": row["power_limit_w"],
|
||||
@@ -1034,12 +1554,14 @@ class PlatformStore:
|
||||
}
|
||||
|
||||
def queue(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
priority_score = {"urgent": 3, "high": 2, "normal": 1, "low": 0}
|
||||
items = [
|
||||
{
|
||||
"id": task["id"],
|
||||
"name": task["name"],
|
||||
"status": task["status"],
|
||||
"progress": task.get("progress", 0),
|
||||
"priority": task.get("priority", "normal"),
|
||||
"compute_node_id": task.get("compute_node_id"),
|
||||
"gpus": task.get("gpus", []),
|
||||
"create_time": task.get("create_time"),
|
||||
@@ -1047,6 +1569,7 @@ class PlatformStore:
|
||||
for task in self.tasks()
|
||||
if task["status"] in {"pending", "syncing", "queued", "running"}
|
||||
]
|
||||
return sorted(items, key=lambda item: (-priority_score.get(item["priority"], 1), item["create_time"]), reverse=False)
|
||||
|
||||
def replicas(self, node_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
|
||||
Reference in New Issue
Block a user