YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
242407b676037e37ce2393c55055e926bea631b8
YG_FT/backend/app/db/platform_store.py

2613 lines
109 KiB
Python
Raw Normal View History

第一次提交
2026-07-27 09:12:47 +08:00
from __future__ import annotations
import json
import hashlib
import hmac
import math
import secrets
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterator
import psycopg
from app.core.config import get_settings
ALL_PERMISSIONS = [
"dashboard",
"fine-tune",
"model-eval",
"model-inference",
"model-manage",
"dataset",
"data-process",
"data-convert",
"compute",
"hardware",
"logs",
"user-settings",
]
def utcnow() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def parse_time(value: str | None) -> datetime | None:
if not value:
return None
return datetime.fromisoformat(value.replace("Z", "+00:00"))
def json_loads(value: str | None, default: Any) -> Any:
if not value:
return default
return json.loads(value)
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:12]}"
PASSWORD_HASH_ITERATIONS = 390_000
def hash_password(password: str) -> str:
salt = secrets.token_hex(16)
digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt.encode("utf-8"), PASSWORD_HASH_ITERATIONS)
return f"pbkdf2_sha256${PASSWORD_HASH_ITERATIONS}${salt}${digest.hex()}"
def verify_password(password: str, stored: str) -> tuple[bool, bool]:
if not stored.startswith("pbkdf2_sha256$"):
return hmac.compare_digest(password, stored), True
try:
_, iterations, salt, expected = stored.split("$", 3)
digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt.encode("utf-8"), int(iterations)).hex()
return hmac.compare_digest(digest, expected), False
except ValueError:
return False, False
def _psycopg_url(database_url: str) -> str:
return database_url.replace("postgresql+psycopg://", "postgresql://")
def _pg_sql(sql: str) -> str:
return sql.replace("?", "%s")
class PgRow(dict):
def __init__(self, columns: list[str], values: tuple[Any, ...]) -> None:
super().__init__(zip(columns, values))
self._values = values
def __getitem__(self, key: str | int) -> Any:
if isinstance(key, int):
return self._values[key]
return super().__getitem__(key)
class PgCursor:
def __init__(self, cursor: psycopg.Cursor[Any]) -> None:
self.cursor = cursor
def execute(self, sql: str, params: tuple[Any, ...] | list[Any] | None = None) -> "PgCursor":
self.cursor.execute(_pg_sql(sql), params)
return self
def fetchone(self) -> PgRow | None:
row = self.cursor.fetchone()
if row is None:
return None
return PgRow(self._columns(), tuple(row))
def fetchall(self) -> list[PgRow]:
columns = self._columns()
return [PgRow(columns, tuple(row)) for row in self.cursor.fetchall()]
def _columns(self) -> list[str]:
return [col.name for col in self.cursor.description or []]
class PgConnection:
def __init__(self, conn: psycopg.Connection[Any]) -> None:
self.conn = conn
def execute(self, sql: str, params: tuple[Any, ...] | list[Any] | None = None) -> PgCursor:
cursor = PgCursor(self.conn.cursor())
return cursor.execute(sql, params)
def executemany(self, sql: str, params_seq: list[tuple[Any, ...]] | list[list[Any]]) -> None:
with self.conn.cursor() as cursor:
cursor.executemany(_pg_sql(sql), params_seq)
def executescript(self, sql: str) -> None:
with self.conn.cursor() as cursor:
for statement in sql.split(";"):
statement = statement.strip()
if statement:
cursor.execute(statement)
def commit(self) -> None:
self.conn.commit()
def rollback(self) -> None:
self.conn.rollback()
def close(self) -> None:
self.conn.close()
class PlatformStore:
"""PostgreSQL-backed store for the first runnable platform version.
This store mirrors the API-facing subset needed by the first system
iteration while using the same PostgreSQL dependency as later production
development.
"""
def __init__(self, database_url: str | None = None) -> None:
settings = get_settings()
self.database_url = _psycopg_url(database_url or settings.database_url)
self.ensure_schema()
self.ensure_seed_data()
@contextmanager
def connect(self) -> Iterator["PgConnection"]:
raw_conn = psycopg.connect(self.database_url)
conn = PgConnection(raw_conn)
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def ensure_schema(self) -> None:
sql_dir = Path(__file__).with_name("sql")
with self.connect() as conn:
conn.executescript((sql_dir / "001_platform_runtime.sql").read_text(encoding="utf-8"))
conn.executescript((sql_dir / "002_governance.sql").read_text(encoding="utf-8"))
columns = conn.execute(
"SELECT column_name FROM information_schema.columns WHERE table_name='users'"
).fetchall()
column_names = {row["column_name"] for row in columns}
if "password" in column_names and "password_hash" not in column_names:
conn.execute("ALTER TABLE users RENAME COLUMN password TO password_hash")
def ensure_seed_data(self) -> None:
with self.connect() as conn:
if conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] > 0:
return
now = utcnow()
users = [
("u_admin", "admin", "admin123", "Platform Admin", "admin", "active", ALL_PERMISSIONS, 1),
(
"u_operator",
"operator",
"operator123",
"Platform Operator",
"operator",
"active",
[p for p in ALL_PERMISSIONS if p != "user-settings"],
0,
),
]
conn.executemany(
"""
INSERT INTO users
(id, username, password_hash, display_name, role, status, permissions, create_time, protected)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
[(u[0], u[1], hash_password(u[2]), u[3], u[4], u[5], json_dumps(u[6]), now, u[7]) for u in users],
)
def _duration(self, start_time: str | None, end_time: str | None = None) -> str:
start = parse_time(start_time)
if not start:
return ""
end = parse_time(end_time) or datetime.now(timezone.utc)
seconds = max(0, int((end - start).total_seconds()))
minutes, sec = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
if hours:
return f"{hours}h {minutes}m {sec}s"
if minutes:
return f"{minutes}m {sec}s"
return f"{sec}s"
def refresh_runtime_state(self) -> None:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
).fetchall()
now_dt = datetime.now(timezone.utc)
for row in rows:
update
2026-07-31 16:10:34 +08:00
payload = json_loads(row["payload"], {})
compute_job_id = payload.get("compute_job_id")
compute_node_api = payload.get("compute_node_api")
if compute_job_id and compute_node_api:
# 已派发到算力：状态/进度/日志回传来自算力进程（架构 §1.1）
self._sync_task_from_compute(conn, row, payload, compute_job_id, compute_node_api)
continue
if get_settings().compute_mode != "simulator":
continue
第一次提交
2026-07-27 09:12:47 +08:00
start = parse_time(row["start_time"])
if not start:
continue
age = max(0, int((now_dt - start).total_seconds()))
if age < 4:
status, progress = "syncing", 8 + age
elif age < 8:
status, progress = "queued", 18 + age
elif age < 70:
status = "running"
progress = min(96, 25 + int((age - 8) / 62 * 70))
else:
status, progress = "completed", 100
payload.update(
{
"status": status,
"progress": progress,
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
}
)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"""
UPDATE fine_tune_tasks
SET status=?, progress=?, payload=?, completed_at=?
WHERE id=?
""",
(status, progress, json_dumps(payload), completed_at, row["id"]),
)
if status == "completed":
self._ensure_trained_model(conn, payload)
update
2026-07-31 16:10:34 +08:00
if get_settings().compute_mode == "simulator":
sync_rows = conn.execute(
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
).fetchall()
for row in sync_rows:
created = parse_time(row["create_time"])
age = int((now_dt - created).total_seconds()) if created else 0
status = "completed" if age >= 6 else "running"
progress = 100 if status == "completed" else min(95, 15 + age * 12)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
(status, progress, completed_at, row["id"]),
)
def _sync_task_from_compute(
self,
conn,
row,
payload: dict[str, Any],
compute_job_id: str,
compute_node_api: str,
) -> None:
"""从算力节点拉回已派发任务的状态/进度/日志，写回本地任务记录。"""
try:
from app.modules.compute_gateway.client import ComputeNodeClient
job = ComputeNodeClient(compute_node_api).get_job(compute_job_id)
except Exception as exc: # noqa: BLE001
payload["compute_sync_error"] = str(exc)
conn.execute(
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
(json_dumps(payload), row["id"]),
)
return
status = job.get("status")
progress = int(job.get("progress", 0) or 0)
logs = job.get("logs") or ""
payload.update(
{
"status": status,
"progress": progress,
"compute_logs": logs,
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
}
)
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
conn.execute(
"""
UPDATE fine_tune_tasks
SET status=?, progress=?, payload=?, completed_at=?
WHERE id=?
""",
(status, progress, json_dumps(payload), completed_at, row["id"]),
)
if status == "completed":
self._ensure_trained_model(conn, payload)
第一次提交
2026-07-27 09:12:47 +08:00
def _ensure_trained_model(self, conn: PgConnection, task: dict[str, Any]) -> None:
name = task.get("output_model_name") or f"{task['name']}-lora"
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
if exists:
return
model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
conn.execute(
"""
INSERT INTO trained_models
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
new_id("tm"),
name,
json_dumps([{"name": task.get("train_method", "lora")}]),
model["path"] if model else "",
utcnow(),
0,
0,
update
2026-07-31 16:10:34 +08:00
task.get("output_dir") or f"/data/yg-ft/outputs/{task.get('name')}/adapter",
第一次提交
2026-07-27 09:12:47 +08:00
),
)
def users(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM users ORDER BY create_time").fetchall()
return [self._user(row) for row in rows]
def user_by_id(self, user_id: str) -> dict[str, Any] | None:
with self.connect() as conn:
row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
return self._user(row) if row else None
def roles(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM roles ORDER BY create_time").fetchall()
return [
{
"id": r["id"],
"name": r["name"],
"display_name": r["display_name"],
"permissions": json_loads(r["permissions"], []),
}
for r in rows
]
def audit_logs(
self,
*,
tenant_id: str | None = None,
project_id: str | None = None,
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
offset: int = 0,
) -> dict[str, Any]:
clauses: list[str] = []
params: list[Any] = []
if tenant_id:
clauses.append("tenant_id=?")
params.append(tenant_id)
if project_id:
clauses.append("project_id=?")
params.append(project_id)
if actor_id:
clauses.append("actor_id=?")
params.append(actor_id)
if action:
clauses.append("action=?")
params.append(action)
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if start_time:
clauses.append("time>=?")
params.append(start_time)
if end_time:
clauses.append("time<=?")
params.append(end_time)
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
with self.connect() as conn:
total = conn.execute(
f"SELECT COUNT(*) AS c FROM audit_logs{where}", tuple(params)
).fetchone()["c"]
rows = conn.execute(
f"SELECT * FROM audit_logs{where} ORDER BY time DESC LIMIT ? OFFSET ?",
tuple(params + [limit, offset]),
).fetchall()
return {"items": [dict(r) for r in rows], "total": total}
def record_audit(
self,
*,
action: str,
actor_id: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
tenant_id: str | None = None,
project_id: str | None = None,
detail: str | None = None,
client_ip: str | None = None,
) -> dict[str, Any]:
aid = new_id("log")
with self.connect() as conn:
conn.execute(
"""INSERT INTO audit_logs
(id, tenant_id, project_id, actor_id, action, target_type, target_id, detail, client_ip, time)
VALUES (?,?,?,?,?,?,?,?,?,?)""",
(aid, tenant_id, project_id, actor_id, action, target_type, target_id, detail, client_ip, utcnow()),
)
return {"id": aid}
update
2026-07-31 16:10:34 +08:00
# ---- 登录会话（采集在线时长） ----
def create_session(self, user: dict[str, Any]) -> str:
sid = new_id("sess")
with self.connect() as conn:
conn.execute(
"""INSERT INTO sessions
(id, user_id, username, display_name, role, login_at, logout_at, duration_seconds, create_time)
VALUES (?,?,?,?,?,?,?,?,?)""",
(
sid,
user.get("id"),
user.get("username"),
user.get("display_name"),
user.get("role"),
utcnow(),
None,
None,
utcnow(),
),
)
return sid
def close_session(self, session_id: str | None) -> None:
if not session_id:
return
with self.connect() as conn:
row = conn.execute(
"SELECT login_at FROM sessions WHERE id=? AND logout_at IS NULL", (session_id,)
).fetchone()
if not row:
return
now = datetime.now(timezone.utc)
start = parse_time(row["login_at"]) or now
seconds = max(0, int((now - start).total_seconds()))
conn.execute(
"UPDATE sessions SET logout_at=?, duration_seconds=? WHERE id=?",
(utcnow(), seconds, session_id),
)
def login_duration_rank(self, limit: int = 8) -> list[dict[str, Any]]:
"""本月登录时长排行：按用户聚合会话时长（小时）。"""
month_start = utcnow()[:7] + "01T00:00:00Z"
with self.connect() as conn:
rows = conn.execute(
"SELECT user_id, username, display_name, role, login_at, logout_at, duration_seconds "
"FROM sessions WHERE login_at >= ?",
(month_start,),
).fetchall()
agg: dict[str, dict[str, Any]] = {}
now = datetime.now(timezone.utc)
for r in rows:
uid = r["user_id"]
bucket = agg.setdefault(
uid,
{"user": r["display_name"] or r["username"], "role": r["role"] or "", "total": 0.0, "has": False},
)
dur = r["duration_seconds"]
if dur is None and r["logout_at"] is None:
start = parse_time(r["login_at"])
if start:
dur = max(0, int((now - start).total_seconds()))
if dur is None:
dur = 0
bucket["total"] += dur
bucket["has"] = True
result = [
{"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)}
for b in agg.values()
if b["has"]
]
result.sort(key=lambda x: x["duration"], reverse=True)
return result[:limit]
第一次提交
2026-07-27 09:12:47 +08:00
# ---- 审批模板 ----
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
tid = payload.get("id") or new_id("tpl")
steps = payload.get("steps") or []
if not isinstance(steps, list):
raise ValueError("steps 必须是列表")
with self.connect() as conn:
conn.execute(
"INSERT INTO approval_templates (id, name, steps, create_time) VALUES (?,?,?,?)",
(tid, payload["name"], json_dumps(steps), utcnow()),
)
return self.approval_template(tid)
def approval_templates(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM approval_templates ORDER BY create_time").fetchall()
return [self.approval_template(r["id"]) for r in rows]
def approval_template(self, tid: str) -> dict[str, Any]:
with self.connect() as conn:
r = conn.execute("SELECT * FROM approval_templates WHERE id=?", (tid,)).fetchone()
if not r:
raise KeyError(tid)
return {**dict(r), "steps": json_loads(r["steps"], [])}
# ---- 审批实例 ----
def create_approval_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
iid = new_id("apr")
template_id = payload.get("template_id")
resource_type = payload["resource_type"]
resource_id = payload["resource_id"]
applicant_id = payload["applicant_id"]
template = self.approval_template(template_id) if template_id else None
steps = template["steps"] if template else [{"approver_id": None}]
now = utcnow()
with self.connect() as conn:
conn.execute(
"""INSERT INTO approval_instances
(id, template_id, resource_type, resource_id, applicant_id, status, current_step, create_time)
VALUES (?,?,?,?,?,?,?,?)""",
(iid, template_id, resource_type, resource_id, applicant_id, "pending", 0, now),
)
for idx, step in enumerate(steps):
conn.execute(
"""INSERT INTO approval_steps (instance_id, step_index, approver_id, status, comment, time)
VALUES (?,?,?,?,?,?)""",
(iid, idx, step.get("approver_id"), "pending", None, None),
)
return self.approval_instance(iid)
def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]:
clauses = []
params: list[Any] = []
if status:
clauses.append("status=?")
params.append(status)
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
with self.connect() as conn:
rows = conn.execute(
f"SELECT * FROM approval_instances{where} ORDER BY create_time DESC", tuple(params)
).fetchall()
return [self.approval_instance(r["id"]) for r in rows]
def approval_instance(self, iid: str) -> dict[str, Any]:
with self.connect() as conn:
r = conn.execute("SELECT * FROM approval_instances WHERE id=?", (iid,)).fetchone()
if not r:
raise KeyError(iid)
steps = conn.execute(
"SELECT * FROM approval_steps WHERE instance_id=? ORDER BY step_index", (iid,)
).fetchall()
return {**dict(r), "steps": [dict(s) for s in steps]}
def decide_approval_step(self, iid: str, step_index: int, *, approver_id: str, approved: bool, comment: str | None = None) -> dict[str, Any]:
inst = self.approval_instance(iid)
if inst["status"] != "pending":
raise ValueError("审批已结束")
if step_index != inst["current_step"]:
raise ValueError("当前步骤不可审批")
new_status = "approved" if approved else "rejected"
with self.connect() as conn:
conn.execute(
"""UPDATE approval_steps SET status=?, approver_id=?, comment=?, time=?
WHERE instance_id=? AND step_index=?""",
(new_status, approver_id, comment, utcnow(), iid, step_index),
)
if not approved:
conn.execute("UPDATE approval_instances SET status='rejected' WHERE id=?", (iid,))
elif step_index + 1 >= len(inst["steps"]):
conn.execute("UPDATE approval_instances SET status='approved', current_step=? WHERE id=?", (step_index + 1, iid))
else:
conn.execute("UPDATE approval_instances SET current_step=? WHERE id=?", (step_index + 1, iid))
return self.approval_instance(iid)
# ---- 租户 ----
def tenants(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM tenants ORDER BY create_time"
).fetchall()
return [
{**dict(r), "quota": json_loads(r.get("quota"), {})}
for r in rows
]
def tenant(self, tenant_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute(
"SELECT * FROM tenants WHERE id=?", (tenant_id,)
).fetchone()
if not row:
raise KeyError(tenant_id)
return {**dict(row), "quota": json_loads(row.get("quota"), {})}
def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]:
tenant_id = payload.get("id") or new_id("tenant")
now = utcnow()
with self.connect() as conn:
conn.execute(
"""
INSERT INTO tenants
(id, name, code, status, owner_user_id, quota, retention_policy_id, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
tenant_id,
payload["name"],
payload.get("code") or payload["name"].lower().replace(" ", "-"),
payload.get("status", "active"),
payload.get("owner_user_id"),
json_dumps(payload.get("quota") or {}),
payload.get("retention_policy_id"),
now,
),
)
return self.tenant(tenant_id)
def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.tenant(tenant_id)
with self.connect() as conn:
conn.execute(
"""
UPDATE tenants
SET name=?, code=?, status=?, owner_user_id=?, quota=?, retention_policy_id=?
WHERE id=?
""",
(
payload.get("name", current["name"]),
payload.get("code", current["code"]),
payload.get("status", current["status"]),
payload.get("owner_user_id", current.get("owner_user_id")),
json_dumps(payload.get("quota", current.get("quota") or {})),
payload.get("retention_policy_id", current.get("retention_policy_id")),
tenant_id,
),
)
return self.tenant(tenant_id)
def set_tenant_quota(self, tenant_id: str, quota: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE tenants SET quota=? WHERE id=?",
(json_dumps(quota), tenant_id),
)
return self.tenant(tenant_id)
def set_tenant_retention(
self, tenant_id: str, retention_policy_id: str
) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE tenants SET retention_policy_id=? WHERE id=?",
(retention_policy_id, tenant_id),
)
return self.tenant(tenant_id)
# ---- 资源 ACL ----
def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT principal_type, principal_id, permission, granted
FROM resource_acl
WHERE resource_type=? AND resource_id=?
""",
(resource_type, resource_id),
).fetchall()
agg: dict[tuple[str, str], dict[str, Any]] = {}
for r in rows:
key = (r["principal_type"], r["principal_id"])
entry = agg.setdefault(
key,
{
"subject_type": r["principal_type"],
"subject_id": r["principal_id"],
"permissions": [],
},
)
if r["granted"]:
entry["permissions"].append(r["permission"])
return list(agg.values())
def set_acl(
self,
resource_type: str,
resource_id: str,
entries: list[dict[str, Any]],
) -> list[dict[str, Any]]:
flat: list[tuple[str, str, str, int]] = []
for e in entries:
for perm in e.get("permissions", []):
flat.append((e["subject_type"], e["subject_id"], perm, 1))
with self.connect() as conn:
conn.execute(
"DELETE FROM resource_acl WHERE resource_type=? AND resource_id=?",
(resource_type, resource_id),
)
for principal_type, principal_id, perm, granted in flat:
conn.execute(
"""
INSERT INTO resource_acl
(resource_type, resource_id, principal_type, principal_id, permission, granted)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, principal_type, principal_id, permission)
DO UPDATE SET granted=excluded.granted
""",
(resource_type, resource_id, principal_type, principal_id, perm, granted),
)
return self.get_acl(resource_type, resource_id)
def login(self, username: str, password: str) -> dict[str, Any] | None:
with self.connect() as conn:
row = conn.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
if not row or row["status"] != "active":
return None
matched, legacy_plaintext = verify_password(password, row["password_hash"])
if not matched:
return None
last_login = utcnow()
if legacy_plaintext:
conn.execute(
"UPDATE users SET password_hash=?, last_login=? WHERE id=?",
(hash_password(password), last_login, row["id"]),
)
else:
conn.execute("UPDATE users SET last_login=? WHERE id=?", (last_login, row["id"]))
data = self._user(row)
data["last_login"] = last_login
return data
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
user_id = new_id("u")
permissions = payload.get("permissions") or (ALL_PERMISSIONS if payload.get("role") == "admin" else ["dashboard"])
with self.connect() as conn:
conn.execute(
"""
INSERT INTO users
(id, username, password_hash, display_name, role, status, permissions, create_time, protected)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
""",
(
user_id,
payload["username"],
hash_password(payload.get("password", "platform123")),
payload.get("display_name") or payload["username"],
payload.get("role", "viewer"),
payload.get("status", "active"),
json_dumps(permissions),
utcnow(),
),
)
return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone())
def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
if not row:
raise KeyError(user_id)
values = {
"role": payload.get("role", row["role"]),
"status": payload.get("status", row["status"]),
"permissions": json_dumps(payload.get("permissions", json_loads(row["permissions"], []))),
}
conn.execute(
"UPDATE users SET role=?, status=?, permissions=? WHERE id=?",
(values["role"], values["status"], values["permissions"], user_id),
)
return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone())
def delete_user(self, user_id: str) -> None:
with self.connect() as conn:
row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone()
if not row:
raise KeyError(user_id)
if row["protected"]:
raise ValueError("protected user cannot be deleted")
conn.execute("DELETE FROM users WHERE id=?", (user_id,))
def reset_password(self, user_id: str, new_password: str) -> dict[str, Any]:
new_password = new_password or "platform123"
with self.connect() as conn:
row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone()
if not row:
raise KeyError(user_id)
if row["protected"]:
raise ValueError("protected user cannot be reset")
conn.execute(
"UPDATE users SET password_hash=? WHERE id=?",
(hash_password(new_password), user_id),
)
return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone())
def _user(self, row: PgRow) -> dict[str, Any]:
return {
"id": row["id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"status": row["status"],
"permissions": json_loads(row["permissions"], []),
"create_time": row["create_time"],
"last_login": row["last_login"],
"protected": bool(row["protected"]),
}
def models(self) -> list[dict[str, Any]]:
with self.connect() as conn:
return [dict(row) for row in conn.execute("SELECT * FROM models ORDER BY create_time DESC").fetchall()]
def model(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone()
if not row:
raise KeyError(model_id)
return dict(row)
def model_by_name(self, name: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM models WHERE name=?", (name,)).fetchone()
if not row:
raise KeyError(name)
return dict(row)
def create_model(self, payload: dict[str, Any]) -> dict[str, Any]:
model_id = payload.get("id") or new_id("m")
with self.connect() as conn:
conn.execute(
"""
INSERT INTO models
(id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
model_id,
payload["name"],
payload.get("type", "LLM"),
payload.get("purpose", "training"),
payload.get("model_source", "local"),
payload.get("description"),
payload.get("path"),
payload.get("api_url"),
payload.get("api_key"),
payload.get("online_model_name"),
utcnow(),
),
)
update
2026-07-31 16:10:34 +08:00
return self.model(model_id)
第一次提交
2026-07-27 09:12:47 +08:00
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.model(model_id)
merged = {**current, **payload}
with self.connect() as conn:
conn.execute(
"""
UPDATE models
SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?
WHERE id=?
""",
(
merged["name"],
merged.get("type", "LLM"),
merged.get("purpose", "training"),
merged.get("model_source", "local"),
merged.get("description"),
merged.get("path"),
merged.get("api_url"),
merged.get("api_key"),
merged.get("online_model_name"),
model_id,
),
)
update
2026-07-31 16:10:34 +08:00
return self.model(model_id)
第一次提交
2026-07-27 09:12:47 +08:00
def delete_model(self, model_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM models WHERE id=?", (model_id,))
def trained_models(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
return [
{
**dict(row),
"train_methods": json_loads(row["train_methods"], []),
"merged": bool(row["merged"]),
"merging": bool(row["merging"]),
}
for row in rows
]
def datasets(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM datasets ORDER BY create_time DESC").fetchall()
return [self._dataset(conn, row) for row in rows]
def dataset(self, dataset_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()
if not row:
raise KeyError(dataset_id)
return self._dataset(conn, row)
def _dataset(self, conn: PgConnection, row: PgRow) -> dict[str, Any]:
files = conn.execute(
"SELECT id, name, size, active_version_id, create_time FROM dataset_files WHERE dataset_id=? ORDER BY create_time",
(row["id"],),
).fetchall()
return {
**dict(row),
"files": [
{
"id": f["id"],
"name": f["name"],
"size": f["size"],
"active_version_id": f["active_version_id"],
"create_time": f["create_time"],
}
for f in files
],
}
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
dataset_id = payload.get("id") or new_id("ds")
with self.connect() as conn:
conn.execute(
"""
INSERT INTO datasets
(id, name, type, storage_type, source, task_id, size, count, description, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
dataset_id,
payload["name"],
payload.get("type", "train"),
payload.get("storage_type", "local"),
payload.get("source", "upload"),
payload.get("task_id"),
payload.get("size", "0 KB"),
payload.get("count", 0),
payload.get("description"),
utcnow(),
),
)
return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone())
def update_dataset(self, dataset_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.dataset(dataset_id)
merged = {**current, **payload}
with self.connect() as conn:
conn.execute(
"""
UPDATE datasets
SET name=?, type=?, storage_type=?, source=?, task_id=?, size=?, count=?, description=?
WHERE id=?
""",
(
merged["name"],
merged.get("type", "train"),
merged.get("storage_type", "local"),
merged.get("source", "upload"),
merged.get("task_id"),
merged.get("size", "0 KB"),
merged.get("count", 0),
merged.get("description"),
dataset_id,
),
)
return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone())
def delete_dataset(self, dataset_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,))
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
now = utcnow()
file_id = new_id("file")
version_id = f"{file_id}_v1"
size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB"
conn.execute(
"""
INSERT INTO dataset_files
(id, dataset_id, name, size, content, active_version_id, versions, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
file_id,
dataset_id,
name,
size,
content,
version_id,
json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]),
now,
),
)
count = len([line for line in content.splitlines() if line.strip()])
conn.execute(
"UPDATE datasets SET count=count+?, size=? WHERE id=?",
(count, size, dataset_id),
)
return {"id": file_id, "name": name, "size": size}
def dataset_file(self, file_id: str) -> PgRow:
with self.connect() as conn:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
return row
def file_versions(self, file_id: str) -> dict[str, Any]:
row = self.dataset_file(file_id)
versions = json_loads(row["versions"], [])
return {
"versions": versions,
"active_version_id": row["active_version_id"],
"next_version_number": len(versions) + 1,
}
def create_file_version(self, file_id: str, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
versions = json_loads(row["versions"], [])
version = {
"id": f"{file_id}_v{len(versions) + 1}",
"version": len(versions) + 1,
"create_time": utcnow(),
"description": payload.get("description", "online edit"),
}
versions.append(version)
conn.execute(
"UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?",
(payload.get("content", ""), version["id"], json_dumps(versions), file_id),
)
return {"version": version, "content": payload.get("content", "")}
def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
if not row:
raise KeyError(file_id)
versions = json_loads(row["versions"], [])
version = next((item for item in versions if item["id"] == version_id), None)
if not version:
raise KeyError(version_id)
conn.execute("UPDATE dataset_files SET active_version_id=? WHERE id=?", (version_id, file_id))
return {"version": version, "content": row["content"]}
def tasks(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
rows = conn.execute("SELECT * FROM fine_tune_tasks ORDER BY create_time DESC").fetchall()
return [self._task(row) for row in rows]
def task(self, task_id: str) -> dict[str, Any]:
self.refresh_runtime_state()
with self.connect() as conn:
row = conn.execute("SELECT * FROM fine_tune_tasks WHERE id=?", (task_id,)).fetchone()
if not row:
raise KeyError(task_id)
return self._task(row)
def _task(self, row: PgRow) -> dict[str, Any]:
payload = json_loads(row["payload"], {})
payload.update(
{
"id": row["id"],
"status": row["status"],
"progress": row["progress"],
"process_id": row["process_id"],
"create_time": row["create_time"],
"gpus": json_loads(row["gpus"], payload.get("gpus", [])),
"train_duration": self._duration(row["start_time"], row["completed_at"]) if row["start_time"] else "",
"compute_node_id": row["compute_node_id"],
"sync_job_id": row["sync_job_id"],
}
)
return payload
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("task_id") or payload.get("id") or new_id("ft"))
name = payload.get("name") or f"fine-tune-{task_id[-6:]}"
base_model = payload.get("base_model") or payload.get("base_model_id")
train_dataset_id = payload.get("train_dataset_id")
if not base_model:
raise ValueError("base_model or base_model_id is required")
if not train_dataset_id:
raise ValueError("train_dataset_id is required")
now = utcnow()
task = {
"id": task_id,
"name": name,
"description": payload.get("description", ""),
"status": "pending",
"train_type": payload.get("train_type", "SFT"),
"train_method": payload.get("train_method", "lora"),
"template": payload.get("template", "qwen"),
"base_model": base_model,
"train_dataset_id": train_dataset_id,
"auto_merge": bool(payload.get("auto_merge", False)),
"output_model_name": payload.get("output_model_name") or f"{name}-lora",
"gpus": payload.get("gpus") or [],
"batch_size": payload.get("batch_size", 2),
"learning_rate": payload.get("learning_rate", 0.0002),
"n_epochs": payload.get("n_epochs", 3),
"save_steps": payload.get("save_steps", 50),
"lr_scheduler_type": payload.get("lr_scheduler_type", "cosine"),
"max_length": payload.get("max_length", 2048),
"warmup_ratio": payload.get("warmup_ratio", 0.03),
"weight_decay": payload.get("weight_decay", 0.01),
"lora_alpha": payload.get("lora_alpha", 16),
"lora_dropout": payload.get("lora_dropout", 0.05),
"lora_rank": payload.get("lora_rank", 8),
"quantization_bit": payload.get("quantization_bit", 4),
"export_quantized": bool(payload.get("export_quantized", False)),
"quant_method": payload.get("quant_method", "bnb"),
"quant_bits": payload.get("quant_bits", 4),
"quant_group_size": payload.get("quant_group_size", 128),
"export_format": payload.get("export_format", "safetensors"),
"progress": 0,
"process_id": None,
"train_duration": "",
"create_time": now,
}
with self.connect() as conn:
conn.execute(
"""
INSERT INTO fine_tune_tasks
(id, name, payload, status, progress, process_id, create_time, gpus)
VALUES (?, ?, ?, 'pending', 0, NULL, ?, ?)
""",
(task_id, name, json_dumps(task), now, json_dumps(task["gpus"])),
)
return task
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.task(task_id)
merged = {**current, **payload, "id": task_id}
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET name=?, payload=?, gpus=? WHERE id=?",
(merged["name"], json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id),
)
return self.task(task_id)
def start_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("task_id") or payload.get("id"))
current = self.task(task_id)
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
node = self.schedule_node(payload)
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0]
process_id = int(43000 + (time.time() % 10000))
sync_job_id = self.create_sync_job(node["id"], current)
with self.connect() as conn:
conn.execute(
"""
UPDATE fine_tune_tasks
SET payload=?, status='syncing', progress=8, process_id=?, start_time=?,
compute_node_id=?, gpus=?, sync_job_id=?
WHERE id=?
""",
(
json_dumps({**merged, "process_id": process_id, "gpus": selected_gpus}),
process_id,
utcnow(),
node["id"],
json_dumps(selected_gpus),
sync_job_id,
task_id,
),
)
update
2026-07-31 16:10:34 +08:00
api_base = node.get("api_base_url")
if api_base:
# GPU 计算派发到算力节点进程执行；后端只做调度编排（架构 §1.1）
self._dispatch_to_compute(task_id, node, merged, selected_gpus)
elif get_settings().compute_mode != "simulator":
# 降级路径：未配置算力节点时后端本机执行（违反 §1.1，待移除）
第一次提交
2026-07-27 09:12:47 +08:00
from app.modules.fine_tune.service import launch_training
launch_training(task_id)
return self.task(task_id)
update
2026-07-31 16:10:34 +08:00
def _dispatch_to_compute(
self,
task_id: str,
node: dict[str, Any],
task: dict[str, Any],
gpus: list,
) -> None:
"""把训练作业派发到算力节点，GPU 计算在算力进程内执行；后端记录算力 job id。"""
from app.modules.compute_gateway.client import ComputeNodeClient
cfg = {
"id": f"ft_{task_id}",
"name": task.get("name") or task_id,
"type": "fine_tune",
"gpus": gpus,
"stage": str(task.get("train_type") or "SFT").lower(),
"base_model": task.get("base_model") or "placeholder-base-model",
"dataset": task.get("train_dataset_id") or task.get("dataset") or "placeholder-dataset",
"template": task.get("template") or "qwen",
"train_method": task.get("train_method") or "lora",
"output_dir": f"/data/yg-ft/outputs/{task.get('name') or task_id}/adapter",
"batch_size": int(task.get("batch_size", 2) or 2),
"learning_rate": float(task.get("learning_rate", 0.0002) or 0.0002),
"n_epochs": int(task.get("n_epochs", 3) or 3),
}
client = ComputeNodeClient(node["api_base_url"])
try:
job = client.create_job(cfg)
except Exception as exc: # noqa: BLE001
self.update_task_runtime(task_id, status="failed", extra={"dispatch_error": str(exc)})
raise RuntimeError(f"dispatch training job to compute node failed: {exc}") from exc
compute_job_id = job.get("id")
current = self.task(task_id)
updated = {**current, "compute_job_id": compute_job_id, "compute_node_api": node["api_base_url"]}
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET payload=? WHERE id=?",
(json_dumps(updated), task_id),
)
第一次提交
2026-07-27 09:12:47 +08:00
def stop_task(self, task_id: str) -> dict[str, Any]:
task = self.task(task_id)
update
2026-07-31 16:10:34 +08:00
# 如果任务已派发到算力节点，先通知算力停止
compute_job_id = task.get("compute_job_id")
node_id = task.get("compute_node_id")
if compute_job_id and node_id:
try:
from app.modules.compute_gateway.client import ComputeNodeClient
node = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if node and node.get("api_base_url"):
ComputeNodeClient(node["api_base_url"]).stop_job(compute_job_id)
except Exception: # noqa: BLE001 - best effort stop
pass
task.update({"status": "stopped", "progress": min(task.get("progress", 0), 99)})
第一次提交
2026-07-27 09:12:47 +08:00
with self.connect() as conn:
conn.execute(
update
2026-07-31 16:10:34 +08:00
"UPDATE fine_tune_tasks SET status='stopped', payload=?, completed_at=? WHERE id=?",
第一次提交
2026-07-27 09:12:47 +08:00
(json_dumps(task), utcnow(), task_id),
)
update
2026-07-31 16:10:34 +08:00
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task_id),
)
第一次提交
2026-07-27 09:12:47 +08:00
return self.task(task_id)
def update_task_runtime(
self,
task_id: str,
status: str | None = None,
progress: int | None = None,
process_id: int | None = None,
extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""训练执行层回写运行时状态（状态 / 进度 / 进程号 / 附加字段）。"""
current = self.task(task_id)
merged = {**current, **(extra or {})}
if status is not None:
merged["status"] = status
if progress is not None:
merged["progress"] = progress
if process_id is not None:
merged["process_id"] = process_id
with self.connect() as conn:
conn.execute(
"""
UPDATE fine_tune_tasks
SET payload=?, status=?, progress=?, process_id=?
WHERE id=?
""",
(
json_dumps(merged),
status or current["status"],
progress if progress is not None else current["progress"],
process_id if process_id is not None else current["process_id"],
task_id,
),
)
return self.task(task_id)
def ensure_trained_model_for_task(self, task_id: str, task: dict[str, Any], output_dir: str) -> None:
"""训练完成后登记训练产物，供模型管理与推理使用。"""
with self.connect() as conn:
self._ensure_trained_model(
conn,
{**task, "output_model_name": task.get("output_model_name") or f"{task['name']}-lora"},
output_dir,
)
def pause_task(self, task_id: str) -> bool:
from app.modules.fine_tune.service import pause
return pause(task_id)
def resume_task_engine(self, task_id: str) -> bool:
from app.modules.fine_tune.service import resume
return resume(task_id)
def cancel_task_engine(self, task_id: str) -> bool:
from app.modules.fine_tune.service import cancel
return cancel(task_id)
def delete_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,))
def schedule_node(self, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
nodes = self.compute_nodes()
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"]
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
if selected:
return selected
if not candidates:
raise RuntimeError("no available compute node")
return sorted(candidates, key=lambda n: (-n["scheduler_weight"], n["current_running_jobs"], n["code"]))[0]
def create_sync_job(self, node_id: str, task: dict[str, Any]) -> str:
sync_id = new_id("sync")
with self.connect() as conn:
conn.execute(
"""
INSERT INTO resource_sync_jobs
(id, target_node_id, resources, status, progress, create_time)
VALUES (?, ?, ?, 'pending', 0, ?)
""",
(
sync_id,
node_id,
json_dumps(
[
{"resource_type": "model", "resource_id": task.get("base_model")},
{"resource_type": "dataset", "resource_id": task.get("train_dataset_id")},
]
),
utcnow(),
),
)
return sync_id
def progress(self, task_id: str) -> dict[str, Any]:
task = self.task(task_id)
status = task.get("status", "pending")
labels = {
"pending": "waiting for start",
"syncing": "syncing model and dataset to compute node",
"queued": "waiting for GPU slot",
"running": "training with LLaMA-Factory",
"completed": "training completed",
"failed": "training stopped",
}
progress = int(task.get("progress", 0) or 0)
eta = "--" if status in {"completed", "failed"} else f"{max(1, math.ceil((100 - progress) / 10))} min"
return {
"status": status,
"progress": progress,
"step": labels.get(status, status),
"speed": task.get("train_speed") or "--",
"eta": eta,
}
def compute_nodes(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
running = conn.execute(
"SELECT compute_node_id, COUNT(*) AS cnt FROM fine_tune_tasks WHERE status IN ('syncing','queued','running') GROUP BY compute_node_id"
).fetchall()
running_map = {r["compute_node_id"]: r["cnt"] for r in running}
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
return [
{
**dict(row),
"enabled": bool(row["enabled"]),
"tags": json_loads(row["tags"], []),
"health_detail": json_loads(row["health_detail"], {}),
"current_running_jobs": running_map.get(row["id"], 0),
}
for row in rows
]
def update_compute_node(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
if not current:
raise KeyError(node_id)
merged = {**current, **payload}
with self.connect() as conn:
conn.execute(
"""
UPDATE compute_nodes
SET name=?, api_base_url=?, file_gateway_url=?, enabled=?, scheduler_status=?,
scheduler_weight=?, tags=?, max_parallel_jobs=?, last_health_check_at=?
WHERE id=?
""",
(
merged["name"],
merged["api_base_url"],
merged["file_gateway_url"],
1 if merged["enabled"] else 0,
merged["scheduler_status"],
merged["scheduler_weight"],
json_dumps(merged["tags"]),
merged["max_parallel_jobs"],
utcnow(),
node_id,
),
)
return next(n for n in self.compute_nodes() if n["id"] == node_id)
def create_compute_node(self, payload: dict[str, Any]) -> dict[str, Any]:
node_id = payload.get("id") or new_id("node")
now = utcnow()
tags = payload.get("tags") or []
with self.connect() as conn:
conn.execute(
"""
INSERT INTO compute_nodes
(id, code, name, api_base_url, file_gateway_url, enabled, scheduler_status,
scheduler_weight, tags, gpu_count, current_running_jobs, max_parallel_jobs,
data_root, model_root, log_root, last_health_check_at, health_detail)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?)
""",
(
node_id,
payload["code"],
payload.get("name") or payload["code"],
payload["api_base_url"],
payload.get("file_gateway_url") or payload["api_base_url"],
1 if payload.get("enabled", True) else 0,
payload.get("scheduler_status", "offline"),
int(payload.get("scheduler_weight", 100)),
json_dumps(tags),
int(payload.get("gpu_count", 0)),
int(payload.get("max_parallel_jobs", 1)),
payload.get("data_root", "/data/yg-ft"),
payload.get("model_root", "/models"),
payload.get("log_root", "/data/yg-ft/training-logs"),
now,
json_dumps(payload.get("health_detail") or {"status": "registered"}),
),
)
return next(node for node in self.compute_nodes() if node["id"] == node_id)
def gpus(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
rows = conn.execute(
"""
SELECT g.*, n.code AS node_code, n.name AS node_name
FROM gpus g JOIN compute_nodes n ON n.id = g.node_id
ORDER BY n.code, g.gpu_index
"""
).fetchall()
running_tasks = [
self._task(row)
for row in conn.execute(
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
).fetchall()
]
items = []
for row in rows:
task = next(
(
t
for t in running_tasks
if t.get("compute_node_id") == row["node_id"] and row["gpu_index"] in (t.get("gpus") or [])
),
None,
)
busy = task is not None and task.get("status") == "running"
reserved = task is not None and task.get("status") in {"syncing", "queued"}
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
gpu_percent = 86 if busy else 22 if reserved else 3
items.append(
{
"id": row["gpu_index"],
"node_id": row["node_id"],
"node_code": row["node_code"],
"node_name": row["node_name"],
"name": row["name"],
"uuid": row["uuid"],
"gpu_percent": gpu_percent,
"memory_used_gb": memory_used,
"memory_total_gb": row["memory_total_gb"],
"memory_percent": round(memory_used / row["memory_total_gb"] * 100, 1),
"temperature": row["base_temperature"] + (21 if busy else 6 if reserved else 0),
"power_w": round(row["power_limit_w"] * (0.7 if busy else 0.25 if reserved else 0.08), 1),
"power_limit_w": row["power_limit_w"],
"status": "busy" if busy else "reserved" if reserved else "idle",
"processes": [
{
"pid": task["process_id"],
"name": "llamafactory-cli",
"memory_used_gb": memory_used,
"task_name": task["name"],
"user": "admin",
}
]
if task
else [],
}
)
return items
def system_info(self) -> dict[str, Any]:
gpus = self.gpus()
busy = len([g for g in gpus if g["status"] in {"busy", "reserved"}])
cpu_percent = 18 + busy * 9
memory_percent = 37 + busy * 4
return {
"timestamp": utcnow(),
"cpu": {
"percent": min(cpu_percent, 95),
"cores": 32,
"percents": [min(cpu_percent + (i % 7) - 3, 99) for i in range(32)],
"model": "Platform x86_64 CPU",
"frequency_mhz": 2600,
"load_1m": round(cpu_percent / 10, 2),
},
"memory": {
"used_gb": round(256 * memory_percent / 100, 1),
"total_gb": 256,
"percent": min(memory_percent, 95),
"available_gb": round(256 * (100 - memory_percent) / 100, 1),
"cached_gb": 32,
},
"disk": {
"used_gb": 840,
"total_gb": 2048,
"percent": 41,
"read_mb_s": 120 if busy else 8,
"write_mb_s": 95 if busy else 5,
},
"gpu": gpus,
"network": {
"download_mb_s": 12 if busy else 1.2,
"upload_mb_s": 7 if busy else 0.8,
"download_mb": 8024,
"upload_mb": 1732,
},
"system": {
"uptime_seconds": int(time.time() % 100000),
"process_count": 248 + busy,
"os": "Linux platform compute image",
},
}
def health_metrics(self) -> dict[str, float]:
info = self.system_info()
return {
"cpu_percent": info["cpu"]["percent"],
"memory_percent": info["memory"]["percent"],
"disk_percent": info["disk"]["percent"],
}
def queue(self) -> list[dict[str, Any]]:
return [
{
"id": task["id"],
"name": task["name"],
"status": task["status"],
"progress": task.get("progress", 0),
"compute_node_id": task.get("compute_node_id"),
"gpus": task.get("gpus", []),
"create_time": task.get("create_time"),
}
for task in self.tasks()
if task["status"] in {"pending", "syncing", "queued", "running"}
]
def replicas(self, node_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM resource_replicas WHERE node_id=? ORDER BY create_time DESC",
(node_id,),
).fetchall()
return [dict(row) for row in rows]
def sync_job(self, sync_id: str) -> dict[str, Any]:
self.refresh_runtime_state()
with self.connect() as conn:
row = conn.execute("SELECT * FROM resource_sync_jobs WHERE id=?", (sync_id,)).fetchone()
if not row:
raise KeyError(sync_id)
return {**dict(row), "resources": json_loads(row["resources"], [])}
def training_log_files(self) -> list[dict[str, Any]]:
return [
{
"file": f"train_{task['id']}_pid{task['process_id'] or 0}.log",
"name": task["name"],
"pid": task.get("process_id") or 0,
"size": f"{max(1, int((task.get('progress', 0) or 0) * 1.5))} KB",
"date": task.get("create_time", "")[:10],
}
for task in self.tasks()
]
def training_log_content(self, file_name: str) -> dict[str, Any]:
task = next((t for t in self.tasks() if t["id"] in file_name), None)
if not task:
raise KeyError(file_name)
content = self.generate_training_log(task)
return {"file": file_name, "content": content, "size": f"{max(1, len(content.encode('utf-8')) // 1024)} KB"}
def generate_training_log(self, task: dict[str, Any]) -> str:
progress = int(task.get("progress", 0) or 0)
points = max(1, min(80, progress))
lines = [
f"[INFO] task={task['name']} engine=llama_factory status={task['status']}",
f"[INFO] base_model={task.get('base_model')} dataset={task.get('train_dataset_id')} gpus={task.get('gpus', [])}",
"[INFO] command=llamafactory-cli train --stage sft --finetuning_type lora --do_train true",
]
for step in range(1, points + 1):
if step % 3 != 0 and step != points:
continue
loss = max(0.12, 2.4 * math.exp(-step / 42))
grad_norm = 0.45 + (step % 8) * 0.03
lr = float(task.get("learning_rate") or 0.0002) * max(0.05, 1 - step / 120)
epoch = round(step / max(1, points) * float(task.get("n_epochs") or 3), 4)
lines.append(
"{"
f'"loss": {loss:.4f}, "grad_norm": {grad_norm:.4f}, '
f'"learning_rate": {lr:.8f}, "epoch": {epoch:.4f}'
"}"
)
if task.get("status") == "completed":
lines.extend(
[
"***** train metrics *****",
f"epoch = {task.get('n_epochs', 3)}",
"train_loss = 0.1248",
f"train_runtime = {task.get('train_duration') or '1m 10s'}",
"***** train metrics end *****",
]
)
return "\n".join(lines)
def log_files(self, date: str | None = None) -> list[dict[str, Any]]:
today = date or utcnow()[:10]
return [
{"file": f"backend-{today}.log", "name": f"backend-{today}.log", "size": "32 KB", "date": today},
{"file": f"error-{today}.log", "name": f"error-{today}.log", "size": "1 KB", "date": today},
]
def log_content(self, file_name: str) -> dict[str, Any]:
lines = [
json_dumps(
{
"timestamp": utcnow(),
"level": "INFO",
"logger": "platform",
"file": "backend/app/api/v1/endpoints/platform.py",
"line": 1,
"message": "Platform log stream is available.",
}
)
]
return {"file": file_name, "content": "\n".join(lines), "size": "1 KB"}
# ===================== Project Management (§13.2) =====================
def projects(self, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]:
with self.connect() as conn:
sql = "SELECT * FROM projects WHERE tenant_id=?"
params: list[Any] = [tenant_id]
if status:
sql += " AND status=?"
params.append(status)
if keyword:
sql += " AND (name LIKE ? OR code LIKE ?)"
kw = f"%{keyword}%"
params.extend([kw, kw])
sql += " ORDER BY create_time DESC"
rows = conn.execute(sql, tuple(params)).fetchall()
result: list[dict[str, Any]] = []
for row in rows:
project = dict(row)
project["member_count"] = conn.execute(
"SELECT COUNT(*) FROM project_members WHERE project_id=?", (row["id"],)
).fetchone()[0]
project["task_count"] = conn.execute(
"SELECT COUNT(*) FROM fine_tune_tasks WHERE project_id=?", (row["id"],)
).fetchone()[0]
project["quota"] = json_loads(row.get("quota"), {})
result.append(project)
return result
def project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM projects WHERE id=?", (project_id,)).fetchone()
if not row:
raise KeyError(project_id)
project = dict(row)
project["quota"] = json_loads(row.get("quota"), {})
project["member_count"] = conn.execute(
"SELECT COUNT(*) FROM project_members WHERE project_id=?", (project_id,)
).fetchone()[0]
project["task_count"] = conn.execute(
"SELECT COUNT(*) FROM fine_tune_tasks WHERE project_id=?", (project_id,)
).fetchone()[0]
return project
def create_project(self, payload: dict[str, Any]) -> dict[str, Any]:
project_id = payload.get("id") or new_id("proj")
now = utcnow()
with self.connect() as conn:
conn.execute(
"""
INSERT INTO projects
(id, tenant_id, name, code, description, quota, status, create_time, create_by, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?)
""",
(
project_id,
payload.get("tenant_id", "default"),
payload["name"],
payload.get("code", payload["name"].lower().replace(" ", "-")),
payload.get("description"),
json_dumps(payload.get("quota") or {}),
now,
payload.get("create_by"),
now,
),
)
# auto-add creator as owner
if payload.get("create_by"):
conn.execute(
"INSERT INTO project_members (project_id, user_id, role, create_time) VALUES (?, ?, 'owner', ?)",
(project_id, payload["create_by"], now),
)
return self.project(project_id)
def update_project(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.project(project_id)
with self.connect() as conn:
conn.execute(
"""
UPDATE projects
SET name=?, code=?, description=?, quota=?, updated_at=?
WHERE id=?
""",
(
payload.get("name", current["name"]),
payload.get("code", current["code"]),
payload.get("description", current.get("description")),
json_dumps(payload.get("quota", current.get("quota") or {})),
utcnow(),
project_id,
),
)
return self.project(project_id)
def archive_project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE projects SET status='archived', updated_at=? WHERE id=?",
(utcnow(), project_id),
)
return self.project(project_id)
def activate_project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE projects SET status='active', updated_at=? WHERE id=?",
(utcnow(), project_id),
)
return self.project(project_id)
def delete_project(self, project_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM project_members WHERE project_id=?", (project_id,))
conn.execute("DELETE FROM projects WHERE id=?", (project_id,))
def project_members(self, project_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT pm.project_id, pm.user_id, pm.role, pm.create_time,
u.username, u.display_name
FROM project_members pm
JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=?
ORDER BY pm.create_time
""",
(project_id,),
).fetchall()
return [
{
"project_id": row["project_id"],
"user_id": row["user_id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"create_time": row["create_time"],
}
for row in rows
]
def add_project_member(self, project_id: str, user_id: str, role: str = "member") -> dict[str, Any]:
with self.connect() as conn:
# verify user exists
user = conn.execute("SELECT id FROM users WHERE id=?", (user_id,)).fetchone()
if not user:
raise KeyError(f"user {user_id}")
existing = conn.execute(
"SELECT * FROM project_members WHERE project_id=? AND user_id=?",
(project_id, user_id),
).fetchone()
if existing:
conn.execute(
"UPDATE project_members SET role=? WHERE project_id=? AND user_id=?",
(role, project_id, user_id),
)
else:
conn.execute(
"INSERT INTO project_members (project_id, user_id, role, create_time) VALUES (?, ?, ?, ?)",
(project_id, user_id, role, utcnow()),
)
row = conn.execute(
"""
SELECT pm.*, u.username, u.display_name
FROM project_members pm JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=? AND pm.user_id=?
""",
(project_id, user_id),
).fetchone()
return {
"project_id": row["project_id"],
"user_id": row["user_id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"create_time": row["create_time"],
}
def update_project_member_role(self, project_id: str, user_id: str, role: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE project_members SET role=? WHERE project_id=? AND user_id=?",
(role, project_id, user_id),
)
row = conn.execute(
"""
SELECT pm.*, u.username, u.display_name
FROM project_members pm JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=? AND pm.user_id=?
""",
(project_id, user_id),
).fetchone()
if not row:
raise KeyError(user_id)
return {
"project_id": row["project_id"],
"user_id": row["user_id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"create_time": row["create_time"],
}
def remove_project_member(self, project_id: str, user_id: str) -> None:
with self.connect() as conn:
conn.execute(
"DELETE FROM project_members WHERE project_id=? AND user_id=?",
(project_id, user_id),
)
# ===================== Fine-tune Retry & Resume (§13.9) =====================
def retry_task(self, task_id: str) -> dict[str, Any]:
task = self.task(task_id)
if task.get("status") not in {"failed", "cancelled"}:
raise ValueError("only failed or cancelled tasks can be retried")
retry_count = 0
with self.connect() as conn:
row = conn.execute(
"SELECT retry_count FROM fine_tune_tasks WHERE id=?", (task_id,)
).fetchone()
if row:
retry_count = int(row[0] or 0) + 1
now = utcnow()
conn.execute(
"""
UPDATE fine_tune_tasks
SET status='pending', progress=0, completed_at=NULL,
error_message=NULL, payload=?, retry_count=?, last_retry_at=?
WHERE id=?
""",
(json_dumps({**task, "status": "pending", "progress": 0}), retry_count, now, task_id),
)
return self.task(task_id)
def resume_task(self, task_id: str, checkpoint_id: str) -> dict[str, Any]:
task = self.task(task_id)
if task.get("status") not in {"failed", "cancelled", "completed"}:
raise ValueError("only failed, cancelled or completed tasks can be resumed")
with self.connect() as conn:
ckpt = conn.execute(
"SELECT * FROM fine_tune_checkpoints WHERE id=? AND task_id=?",
(checkpoint_id, task_id),
).fetchone()
if not ckpt:
raise KeyError(checkpoint_id)
now = utcnow()
conn.execute(
"""
UPDATE fine_tune_tasks
SET status='pending', progress=0, completed_at=NULL,
error_message=NULL, payload=?, resumed_from_checkpoint_id=?
WHERE id=?
""",
(
json_dumps({**task, "status": "pending", "progress": 0, "resume_from": ckpt["path"]}),
checkpoint_id,
task_id,
),
)
return self.task(task_id)
# ===================== Checkpoint Management (§13.9) =====================
def checkpoints(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM fine_tune_checkpoints WHERE task_id=? ORDER BY step",
(task_id,),
).fetchall()
return [
{
"id": row["id"],
"task_id": row["task_id"],
"name": row["name"],
"path": row["path"],
"step": row["step"],
"loss": row["loss"],
"is_best": bool(row["is_best"]),
"size_bytes": row["size_bytes"],
"create_time": row["create_time"],
}
for row in rows
]
def delete_checkpoint(self, checkpoint_id: str) -> None:
with self.connect() as conn:
row = conn.execute(
"SELECT id FROM fine_tune_checkpoints WHERE id=?", (checkpoint_id,)
).fetchone()
if not row:
raise KeyError(checkpoint_id)
conn.execute("DELETE FROM fine_tune_checkpoints WHERE id=?", (checkpoint_id,))
def set_checkpoint_retention(self, task_id: str, policy: dict[str, Any]) -> dict[str, Any]:
"""policy: {"max_count": int, "keep_best": bool, "retention_hours": int}"""
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET checkpoint_retention_policy=? WHERE id=?",
(json_dumps(policy), task_id),
)
return {"task_id": task_id, "checkpoint_retention_policy": policy}
def get_checkpoint_retention(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute(
"SELECT checkpoint_retention_policy FROM fine_tune_tasks WHERE id=?",
(task_id,),
).fetchone()
if not row:
raise KeyError(task_id)
policy = json_loads(row["checkpoint_retention_policy"], {})
return {"task_id": task_id, "checkpoint_retention_policy": policy or {"max_count": 5, "keep_best": True, "retention_hours": 168}}
# ===================== Fine-tune SSE Events (§7.1) =====================
def task_events(self, task_id: str) -> list[dict[str, Any]]:
task = self.task(task_id)
progress = int(task.get("progress", 0) or 0)
events: list[dict[str, Any]] = [
{
"timestamp": task.get("create_time", utcnow()),
"type": "status",
"data": {"status": task.get("status", "pending"), "message": f"Task {task['name']} {task.get('status')}"},
}
]
train_logs = self.generate_training_log(task).split("\n")
for i, line in enumerate(train_logs):
events.append(
{
"timestamp": utcnow(),
"type": "log" if "loss" not in line else "metric",
"data": {"line": i + 1, "content": line},
}
)
events.append(
{
"timestamp": utcnow(),
"type": "progress",
"data": {"progress": progress, "status": task.get("status")},
}
)
return events
# ===================== Compute Jobs (§13.6) =====================
def create_compute_job(self, payload: dict[str, Any]) -> dict[str, Any]:
job_id = payload.get("id") or new_id("job")
now = utcnow()
with self.connect() as conn:
conn.execute(
"""
INSERT INTO compute_jobs
(id, task_id, node_id, name, type, status, command, gpu_count, priority,
timeout_seconds, progress, create_time)
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?, ?, ?, 0, ?)
""",
(
job_id,
payload["task_id"],
payload.get("node_id", ""),
payload.get("name", f"job-{job_id[-8:]}"),
payload.get("type", "train"),
payload.get("command"),
payload.get("gpu_count", 1),
payload.get("priority", 0),
payload.get("timeout_seconds"),
now,
),
)
return self.compute_job(job_id)
def compute_job(self, job_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
if not row:
raise KeyError(job_id)
return dict(row)
def compute_jobs_by_task(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM compute_jobs WHERE task_id=? ORDER BY create_time DESC",
(task_id,),
).fetchall()
return [dict(row) for row in rows]
def stop_compute_job(self, job_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
if not row:
raise KeyError(job_id)
conn.execute(
"UPDATE compute_jobs SET status='stopped', completed_at=? WHERE id=?",
(utcnow(), job_id),
)
return self.compute_job(job_id)
def compute_job_logs(self, job_id: str) -> dict[str, Any]:
job = self.compute_job(job_id)
task = self.task(job["task_id"])
content = self.generate_training_log(task)
return {"job_id": job_id, "content": content, "lines": len(content.splitlines())}
update
2026-07-31 16:10:34 +08:00
# ===================== Model Evaluation =====================
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
payload = json_loads(row["payload"], {})
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
return payload
def eval_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
return [self._json_payload_row(row) for row in rows]
def eval_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
if not row:
raise KeyError(task_id)
payload = self._json_payload_row(row)
payload.setdefault("sample_count", 0)
payload.setdefault("completed_count", 0)
payload.setdefault("passed_count", 0)
payload.setdefault("overall_score", payload.get("score") or 0)
payload.setdefault("overall_score_max", 100)
payload.setdefault("overall_evaluation", "")
payload.setdefault("improvement_suggestions", [])
payload.setdefault("dimension_summary", [])
payload.setdefault("samples", [])
return payload
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
status = str(payload.get("status") or "pending")
now = payload.get("create_time") or utcnow()
data = {
**payload,
"id": task_id,
"eval_task_name": name,
"status": status,
"create_time": now,
"metric": payload.get("metric") or "custom",
}
with self.connect() as conn:
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
if model:
data.setdefault("model_name", model["name"])
if dataset:
data.setdefault("dataset", dataset["name"])
conn.execute(
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
(task_id, name, json_dumps(data), status, now),
)
return self.eval_task(task_id)
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
def dimensions(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
return [
{
**json_loads(row["payload"], {}),
"id": row["id"],
"name": row["name"],
"is_active": bool(row["is_active"]),
"is_default": bool(row["is_default"]),
"create_time": row["create_time"],
}
for row in rows
]
def dimension(self, dimension_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
if not row:
raise KeyError(dimension_id)
return {
**json_loads(row["payload"], {}),
"id": row["id"],
"name": row["name"],
"is_active": bool(row["is_active"]),
"is_default": bool(row["is_default"]),
"create_time": row["create_time"],
}
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
dimension_id = str(payload.get("id") or new_id("dim"))
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
now = payload.get("create_time") or utcnow()
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
with self.connect() as conn:
conn.execute(
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
)
return self.dimension(dimension_id)
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.dimension(dimension_id)
merged = {**current, **payload, "id": dimension_id}
with self.connect() as conn:
conn.execute(
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
(
merged["name"],
json_dumps(merged),
1 if merged.get("is_active", True) else 0,
1 if merged.get("is_default") else 0,
dimension_id,
),
)
return self.dimension(dimension_id)
def delete_dimension(self, dimension_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
# ===================== Model Compare / Inference =====================
def compare_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
return [self._json_payload_row(row) for row in rows]
def compare_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
if not row:
raise KeyError(task_id)
return self._json_payload_row(row)
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
task_id = str(payload.get("id") or new_id("cmp"))
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
status = str(payload.get("status") or "pending")
now = payload.get("create_time") or utcnow()
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
data.setdefault("load_status", json_dumps({"loaded_models": []}))
with self.connect() as conn:
conn.execute(
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
(task_id, name, json_dumps(data), status, now),
)
return self.compare_task(task_id)
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
current = self.compare_task(task_id)
merged = {**current, **payload, "id": task_id}
status = str(merged.get("status") or current.get("status") or "pending")
with self.connect() as conn:
conn.execute(
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
)
return self.compare_task(task_id)
def delete_compare_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
# ===================== Compute Job Sync (派发回传) =====================
def _acquire_scheduler_lock(
self,
conn: PgConnection,
lock_key: str,
owner: str,
ttl_seconds: int = 30,
) -> bool:
now = utcnow()
expires_at = (datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds)).replace(microsecond=0).isoformat().replace("+00:00", "Z")
existing = conn.execute("SELECT owner, expires_at FROM scheduler_locks WHERE lock_key=?", (lock_key,)).fetchone()
if existing:
if existing["expires_at"] > now and existing["owner"] != owner:
return False
conn.execute(
"UPDATE scheduler_locks SET owner=?, expires_at=?, update_time=? WHERE lock_key=?",
(owner, expires_at, now, lock_key),
)
else:
conn.execute(
"INSERT INTO scheduler_locks (lock_key, owner, expires_at, create_time, update_time) VALUES (?, ?, ?, ?, ?)",
(lock_key, owner, expires_at, now, now),
)
return True
def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
now = utcnow()
command = job.get("command") or []
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
payload = json_dumps({**job, "task_id": task["id"]})
existing = conn.execute("SELECT id FROM compute_jobs WHERE id=?", (job_id,)).fetchone()
if existing:
conn.execute(
"""
UPDATE compute_jobs
SET task_id=?, node_id=?, engine=?, status=?, command=?, output_dir=?, log_file=?,
payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
task["id"],
task.get("compute_node_id"),
str(task.get("engine") or job.get("engine") or "llama_factory"),
status,
command_text,
job.get("output_dir") or task.get("output_dir"),
job.get("log_file") or task.get("log_file"),
payload,
now,
now if status in {"completed", "failed", "stopped"} else None,
job_id,
),
)
return
conn.execute(
"""
INSERT INTO compute_jobs
(id, task_id, node_id, engine, status, command, output_dir, log_file, payload, create_time, update_time, completed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
job_id,
task["id"],
task.get("compute_node_id"),
str(task.get("engine") or job.get("engine") or "llama_factory"),
status,
command_text,
job.get("output_dir") or task.get("output_dir"),
job.get("log_file") or task.get("log_file"),
payload,
now,
now,
now if status in {"completed", "failed", "stopped"} else None,
),
)
def _sync_gpu_allocations(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None:
terminal = status in {"completed", "failed", "stopped"}
if terminal:
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task["id"]),
)
return
job_id = str(job.get("id") or task.get("compute_job_id") or task["id"])
allocation_status = "running" if status == "running" else "allocated"
for gpu_index in [int(item) for item in task.get("gpus") or job.get("gpus") or []]:
existing = conn.execute(
"SELECT id FROM gpu_allocations WHERE task_id=? AND node_id=? AND gpu_index=? AND status IN ('allocated','running')",
(task["id"], task.get("compute_node_id"), gpu_index),
).fetchone()
if existing:
conn.execute("UPDATE gpu_allocations SET status=?, compute_job_id=? WHERE id=?", (allocation_status, job_id, existing["id"]))
continue
conn.execute(
"""
INSERT INTO gpu_allocations
(id, task_id, compute_job_id, node_id, gpu_index, status, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(new_id("gpu_alloc"), task["id"], job_id, task.get("compute_node_id"), gpu_index, allocation_status, utcnow()),
)
def _upsert_checkpoints(self, conn: PgConnection, task_id: str, checkpoints: list[dict[str, Any]]) -> None:
for item in checkpoints:
path = str(item.get("path") or "")
if not path:
continue
step = int(item.get("step") or 0)
name = str(item.get("name") or Path(path).name)
size_bytes = int(item.get("size_bytes") or item.get("size") or 0)
existing = conn.execute("SELECT id FROM fine_tune_checkpoints WHERE task_id=? AND path=?", (task_id, path)).fetchone()
if existing:
conn.execute(
"UPDATE fine_tune_checkpoints SET step=?, name=?, size_bytes=? WHERE id=?",
(step, name, size_bytes, existing["id"]),
)
continue
conn.execute(
"""
INSERT INTO fine_tune_checkpoints
(id, task_id, step, name, path, size_bytes, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(new_id("ckpt"), task_id, step, name, path, size_bytes, utcnow()),
)
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
status_map = {
"queued": "queued",
"running": "running",
"completed": "completed",
"failed": "failed",
"stopped": "stopped",
}
current = self.task(task_id)
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
progress = int(job.get("progress", current.get("progress", 0)) or 0)
payload = {
**current,
"status": status,
"progress": progress,
"process_id": job.get("pid") or current.get("process_id"),
"compute_job_id": job.get("id") or current.get("compute_job_id"),
"output_dir": job.get("output_dir") or current.get("output_dir"),
"log_file": job.get("log_file") or current.get("log_file"),
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
}
if status == "failed":
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
elif status in {"queued", "running", "completed"}:
payload.pop("failure_reason", None)
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
with self.connect() as conn:
conn.execute(
"""
UPDATE fine_tune_tasks
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
json_dumps(payload),
status,
progress,
payload.get("process_id"),
payload.get("compute_job_id"),
completed_at,
task_id,
),
)
self._upsert_compute_job(conn, payload, job, status)
self._sync_gpu_allocations(conn, payload, job, status)
self._upsert_checkpoints(conn, task_id, job.get("checkpoints") or [])
if status == "completed":
self._ensure_trained_model(conn, payload)
if status in {"failed", "stopped"}:
failure_reason = job.get("error") or job.get("message") or "compute job failed"
log_snippet = job.get("log_snippet") or ""
conn.execute(
"UPDATE fine_tune_tasks SET failure_reason = ? WHERE id = ?",
(failure_reason[:2000], task_id),
)
if log_snippet:
conn.execute(
"UPDATE fine_tune_tasks SET payload = ? WHERE id = ?",
(json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id),
)
return self.task(task_id)
def running_compute_tasks(self) -> list[dict[str, Any]]:
return [
task
for task in self.tasks()
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
]
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
task = self.task(task_id)
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
with self.connect() as conn:
conn.execute(
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
(json_dumps(task), utcnow(), task_id),
)
conn.execute(
"UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')",
(utcnow(), task_id),
)
return self.task(task_id)
def record_training_log_metrics(self, task_id: str, content: str) -> int:
rows: list[tuple[Any, ...]] = []
for line_number, line in enumerate(content.splitlines(), start=1):
metric = self._parse_training_metric(line)
if not metric:
continue
rows.append(
(
new_id("metric"),
task_id,
line_number,
metric.get("epoch"),
metric.get("loss"),
metric.get("grad_norm"),
metric.get("learning_rate"),
line[:2000],
utcnow(),
)
)
with self.connect() as conn:
conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,))
if rows:
conn.executemany(
"""
INSERT INTO fine_tune_metrics
(id, task_id, step, epoch, loss, grad_norm, learning_rate, raw, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
rows,
)
return len(rows)
def _parse_training_metric(self, line: str) -> dict[str, Any] | None:
if "loss" not in line or "learning_rate" not in line:
return None
import re
result: dict[str, Any] = {}
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
if match:
result[key] = float(match.group(1))
return result or None
def task_metrics(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT step, epoch, loss, grad_norm, learning_rate, raw, create_time
FROM fine_tune_metrics
WHERE task_id=?
ORDER BY step
""",
(task_id,),
).fetchall()
return [dict(row) for row in rows]
def task_checkpoints(self, task_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT id, step, name, path, size_bytes, create_time
FROM fine_tune_checkpoints
WHERE task_id=?
ORDER BY step, create_time
""",
(task_id,),
).fetchall()
return [dict(row) for row in rows]
def active_standalone_compute_jobs(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"""
SELECT * FROM compute_jobs
WHERE task_id IS NULL AND status IN ('queued','running')
ORDER BY create_time
""",
).fetchall()
return [json_loads(row["payload"], {}) if "payload" in row.keys() else dict(row) for row in rows]
def sync_model_merge_job(self, job_id: str, job: dict[str, Any]) -> dict[str, Any]:
current = self.compute_job(job_id)
payload = current.get("payload") or {}
job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {}
merged_payload = {**payload, "job": {**job_payload, **job}}
status = str(job.get("status") or current.get("status") or "queued")
command = job.get("command") or current.get("command") or []
command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "")
output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir")
with self.connect() as conn:
conn.execute(
"""
UPDATE compute_jobs
SET status=?, command=?, output_dir=?, log_file=?, payload=?, update_time=?, completed_at=COALESCE(?, completed_at)
WHERE id=?
""",
(
status,
command_text,
output_dir,
job.get("log_file") or current.get("log_file"),
json_dumps(merged_payload),
utcnow(),
utcnow() if status in {"completed", "failed", "stopped"} else None,
job_id,
),
)
return self.compute_job(job_id)
第一次提交
2026-07-27 09:12:47 +08:00
_store: PlatformStore | None = None
def get_platform_store() -> PlatformStore:
global _store
if _store is None:
_store = PlatformStore()
return _store
Reference in New Issue Copy Permalink