1487 lines
59 KiB
Python
1487 lines
59 KiB
Python
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import math
|
|||
|
|
import sqlite3
|
|||
|
|
import time
|
|||
|
|
import uuid
|
|||
|
|
from contextlib import contextmanager
|
|||
|
|
from datetime import datetime, timezone
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Any, Iterator
|
|||
|
|
|
|||
|
|
from app.core.config import get_settings
|
|||
|
|
|
|||
|
|
|
|||
|
|
ALL_PERMISSIONS = [
|
|||
|
|
"dashboard",
|
|||
|
|
"fine-tune",
|
|||
|
|
"model-eval",
|
|||
|
|
"model-inference",
|
|||
|
|
"model-manage",
|
|||
|
|
"dataset",
|
|||
|
|
"data-process",
|
|||
|
|
"data-convert",
|
|||
|
|
"compute",
|
|||
|
|
"hardware",
|
|||
|
|
"logs",
|
|||
|
|
"user-settings",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def utcnow() -> str:
|
|||
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_time(value: str | None) -> datetime | None:
|
|||
|
|
if not value:
|
|||
|
|
return None
|
|||
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def json_loads(value: str | None, default: Any) -> Any:
|
|||
|
|
if not value:
|
|||
|
|
return default
|
|||
|
|
return json.loads(value)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def json_dumps(value: Any) -> str:
|
|||
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def new_id(prefix: str) -> str:
|
|||
|
|
return f"{prefix}_{uuid.uuid4().hex[:12]}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
class PlatformStore:
|
|||
|
|
"""Small SQLite-backed store for the first runnable platform version.
|
|||
|
|
|
|||
|
|
The production model is PostgreSQL. This store mirrors the API-facing subset
|
|||
|
|
needed by the first system iteration so developers can run the app without
|
|||
|
|
provisioning enterprise infrastructure first.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, db_path: str | None = None) -> None:
|
|||
|
|
settings = get_settings()
|
|||
|
|
raw_path = db_path or settings.local_db_path
|
|||
|
|
self.db_path = Path(raw_path)
|
|||
|
|
if not self.db_path.is_absolute():
|
|||
|
|
self.db_path = Path.cwd() / self.db_path
|
|||
|
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
self.ensure_schema()
|
|||
|
|
self.ensure_seed_data()
|
|||
|
|
|
|||
|
|
@contextmanager
|
|||
|
|
def connect(self) -> Iterator[sqlite3.Connection]:
|
|||
|
|
conn = sqlite3.connect(self.db_path)
|
|||
|
|
conn.row_factory = sqlite3.Row
|
|||
|
|
try:
|
|||
|
|
yield conn
|
|||
|
|
conn.commit()
|
|||
|
|
finally:
|
|||
|
|
conn.close()
|
|||
|
|
|
|||
|
|
def ensure_schema(self) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.executescript(
|
|||
|
|
"""
|
|||
|
|
CREATE TABLE IF NOT EXISTS users (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
username TEXT NOT NULL UNIQUE,
|
|||
|
|
password TEXT NOT NULL,
|
|||
|
|
display_name TEXT NOT NULL,
|
|||
|
|
role TEXT NOT NULL,
|
|||
|
|
status TEXT NOT NULL,
|
|||
|
|
permissions TEXT NOT NULL,
|
|||
|
|
create_time TEXT NOT NULL,
|
|||
|
|
last_login TEXT,
|
|||
|
|
protected INTEGER NOT NULL DEFAULT 0
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS models (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
name TEXT NOT NULL UNIQUE,
|
|||
|
|
type TEXT NOT NULL,
|
|||
|
|
purpose TEXT NOT NULL,
|
|||
|
|
model_source TEXT NOT NULL,
|
|||
|
|
description TEXT,
|
|||
|
|
path TEXT,
|
|||
|
|
api_url TEXT,
|
|||
|
|
api_key TEXT,
|
|||
|
|
online_model_name TEXT,
|
|||
|
|
create_time TEXT NOT NULL
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS trained_models (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
name TEXT NOT NULL UNIQUE,
|
|||
|
|
train_methods TEXT NOT NULL,
|
|||
|
|
base_model_path TEXT,
|
|||
|
|
create_time TEXT NOT NULL,
|
|||
|
|
merged INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
merging INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
merged_path TEXT
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS datasets (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
name TEXT NOT NULL UNIQUE,
|
|||
|
|
type TEXT NOT NULL,
|
|||
|
|
storage_type TEXT NOT NULL,
|
|||
|
|
source TEXT NOT NULL,
|
|||
|
|
task_id TEXT,
|
|||
|
|
size TEXT,
|
|||
|
|
count INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
description TEXT,
|
|||
|
|
create_time TEXT NOT NULL
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS dataset_files (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
dataset_id TEXT NOT NULL,
|
|||
|
|
name TEXT NOT NULL,
|
|||
|
|
size TEXT,
|
|||
|
|
content TEXT NOT NULL,
|
|||
|
|
active_version_id TEXT NOT NULL,
|
|||
|
|
versions TEXT NOT NULL,
|
|||
|
|
create_time TEXT NOT NULL,
|
|||
|
|
FOREIGN KEY(dataset_id) REFERENCES datasets(id) ON DELETE CASCADE
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS compute_nodes (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
code TEXT NOT NULL UNIQUE,
|
|||
|
|
name TEXT NOT NULL,
|
|||
|
|
api_base_url TEXT NOT NULL,
|
|||
|
|
file_gateway_url TEXT NOT NULL,
|
|||
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|||
|
|
scheduler_status TEXT NOT NULL,
|
|||
|
|
scheduler_weight INTEGER NOT NULL DEFAULT 100,
|
|||
|
|
tags TEXT NOT NULL,
|
|||
|
|
gpu_count INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
current_running_jobs INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
max_parallel_jobs INTEGER NOT NULL DEFAULT 2,
|
|||
|
|
data_root TEXT NOT NULL,
|
|||
|
|
model_root TEXT NOT NULL,
|
|||
|
|
log_root TEXT NOT NULL,
|
|||
|
|
last_health_check_at TEXT,
|
|||
|
|
health_detail TEXT NOT NULL
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS gpus (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
node_id TEXT NOT NULL,
|
|||
|
|
gpu_index INTEGER NOT NULL,
|
|||
|
|
uuid TEXT NOT NULL,
|
|||
|
|
name TEXT NOT NULL,
|
|||
|
|
memory_total_gb REAL NOT NULL,
|
|||
|
|
power_limit_w REAL NOT NULL,
|
|||
|
|
base_temperature INTEGER NOT NULL,
|
|||
|
|
FOREIGN KEY(node_id) REFERENCES compute_nodes(id) ON DELETE CASCADE
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
name TEXT NOT NULL UNIQUE,
|
|||
|
|
payload TEXT NOT NULL,
|
|||
|
|
status TEXT NOT NULL,
|
|||
|
|
progress INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
process_id INTEGER,
|
|||
|
|
create_time TEXT NOT NULL,
|
|||
|
|
start_time TEXT,
|
|||
|
|
completed_at TEXT,
|
|||
|
|
compute_node_id TEXT,
|
|||
|
|
gpus TEXT NOT NULL,
|
|||
|
|
sync_job_id TEXT
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS resource_replicas (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
node_id TEXT NOT NULL,
|
|||
|
|
resource_type TEXT NOT NULL,
|
|||
|
|
resource_id TEXT NOT NULL,
|
|||
|
|
local_path TEXT NOT NULL,
|
|||
|
|
status TEXT NOT NULL,
|
|||
|
|
sync_status TEXT NOT NULL,
|
|||
|
|
create_time TEXT NOT NULL
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE TABLE IF NOT EXISTS resource_sync_jobs (
|
|||
|
|
id TEXT PRIMARY KEY,
|
|||
|
|
target_node_id TEXT NOT NULL,
|
|||
|
|
resources TEXT NOT NULL,
|
|||
|
|
status TEXT NOT NULL,
|
|||
|
|
progress INTEGER NOT NULL DEFAULT 0,
|
|||
|
|
create_time TEXT NOT NULL,
|
|||
|
|
completed_at TEXT
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id);
|
|||
|
|
"""
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def ensure_seed_data(self) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
if conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] > 0:
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
now = utcnow()
|
|||
|
|
users = [
|
|||
|
|
("u_admin", "admin", "admin123", "Platform Admin", "admin", "active", ALL_PERMISSIONS, 1),
|
|||
|
|
(
|
|||
|
|
"u_operator",
|
|||
|
|
"operator",
|
|||
|
|
"operator123",
|
|||
|
|
"Platform Operator",
|
|||
|
|
"operator",
|
|||
|
|
"active",
|
|||
|
|
[p for p in ALL_PERMISSIONS if p != "user-settings"],
|
|||
|
|
0,
|
|||
|
|
),
|
|||
|
|
]
|
|||
|
|
conn.executemany(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO users
|
|||
|
|
(id, username, password, display_name, role, status, permissions, create_time, protected)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
[(u[0], u[1], u[2], u[3], u[4], u[5], json_dumps(u[6]), now, u[7]) for u in users],
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
models = [
|
|||
|
|
(
|
|||
|
|
"m_qwen25_7b",
|
|||
|
|
"Qwen2.5-7B-Instruct",
|
|||
|
|
"LLM",
|
|||
|
|
"training",
|
|||
|
|
"local",
|
|||
|
|
"Sample base model for SFT and LoRA training.",
|
|||
|
|
"/models/Qwen2.5-7B-Instruct",
|
|||
|
|
),
|
|||
|
|
(
|
|||
|
|
"m_llama31_8b",
|
|||
|
|
"Llama-3.1-8B-Instruct",
|
|||
|
|
"LLM",
|
|||
|
|
"training",
|
|||
|
|
"local",
|
|||
|
|
"Reserved base model path for LLaMA-Factory dry-run.",
|
|||
|
|
"/models/Llama-3.1-8B-Instruct",
|
|||
|
|
),
|
|||
|
|
]
|
|||
|
|
conn.executemany(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO models
|
|||
|
|
(id, name, type, purpose, model_source, description, path, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
[(*m, now) for m in models],
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
self._insert_dataset(
|
|||
|
|
conn,
|
|||
|
|
"ds_finance_sft",
|
|||
|
|
"finance-sft-sample",
|
|||
|
|
"train",
|
|||
|
|
"Financial QA SFT samples for sample training.",
|
|||
|
|
"finance-sft-sample.jsonl",
|
|||
|
|
"\n".join(
|
|||
|
|
[
|
|||
|
|
'{"instruction":"Summarize revenue growth.","input":"Revenue grew from 10M to 13M.","output":"Revenue increased 30% year over year."}',
|
|||
|
|
'{"instruction":"Classify risk.","input":"Customer has overdue payment for 90 days.","output":"High credit risk."}',
|
|||
|
|
'{"instruction":"Draft an analyst note.","input":"Gross margin improved by 4 points.","output":"Margin expansion indicates stronger operating leverage."}',
|
|||
|
|
]
|
|||
|
|
),
|
|||
|
|
now,
|
|||
|
|
)
|
|||
|
|
self._insert_dataset(
|
|||
|
|
conn,
|
|||
|
|
"ds_customer_service",
|
|||
|
|
"customer-service-sample",
|
|||
|
|
"train",
|
|||
|
|
"Customer-service instruction tuning samples.",
|
|||
|
|
"customer-service-sample.jsonl",
|
|||
|
|
"\n".join(
|
|||
|
|
[
|
|||
|
|
'{"instruction":"Respond politely.","input":"My package is late.","output":"I am sorry for the delay. I can help check the latest shipment status."}',
|
|||
|
|
'{"instruction":"Escalate request.","input":"I need a refund for a defective item.","output":"I will create a refund case and share the next steps."}',
|
|||
|
|
]
|
|||
|
|
),
|
|||
|
|
now,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
node_rows = [
|
|||
|
|
(
|
|||
|
|
"node_01",
|
|||
|
|
"gpu-node-01",
|
|||
|
|
"GPU Node 01",
|
|||
|
|
"http://gpu-node-01:19100",
|
|||
|
|
"http://gpu-node-01:19101",
|
|||
|
|
1,
|
|||
|
|
"online",
|
|||
|
|
100,
|
|||
|
|
["A800", "80GB", "llama_factory"],
|
|||
|
|
4,
|
|||
|
|
0,
|
|||
|
|
2,
|
|||
|
|
"/data/yg-ft",
|
|||
|
|
"/models",
|
|||
|
|
"/data/yg-ft/training-logs",
|
|||
|
|
now,
|
|||
|
|
{"mode": "simulator", "heartbeat": "ok"},
|
|||
|
|
),
|
|||
|
|
(
|
|||
|
|
"node_02",
|
|||
|
|
"gpu-node-02",
|
|||
|
|
"GPU Node 02",
|
|||
|
|
"http://gpu-node-02:19100",
|
|||
|
|
"http://gpu-node-02:19101",
|
|||
|
|
1,
|
|||
|
|
"online",
|
|||
|
|
60,
|
|||
|
|
["4090", "24GB", "llama_factory"],
|
|||
|
|
4,
|
|||
|
|
0,
|
|||
|
|
1,
|
|||
|
|
"/data/yg-ft",
|
|||
|
|
"/models",
|
|||
|
|
"/data/yg-ft/training-logs",
|
|||
|
|
now,
|
|||
|
|
{"mode": "simulator", "heartbeat": "ok"},
|
|||
|
|
),
|
|||
|
|
]
|
|||
|
|
conn.executemany(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO compute_nodes
|
|||
|
|
(id, code, name, api_base_url, file_gateway_url, enabled, scheduler_status,
|
|||
|
|
scheduler_weight, tags, gpu_count, current_running_jobs, max_parallel_jobs,
|
|||
|
|
data_root, model_root, log_root, last_health_check_at, health_detail)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
[
|
|||
|
|
(
|
|||
|
|
r[0],
|
|||
|
|
r[1],
|
|||
|
|
r[2],
|
|||
|
|
r[3],
|
|||
|
|
r[4],
|
|||
|
|
r[5],
|
|||
|
|
r[6],
|
|||
|
|
r[7],
|
|||
|
|
json_dumps(r[8]),
|
|||
|
|
r[9],
|
|||
|
|
r[10],
|
|||
|
|
r[11],
|
|||
|
|
r[12],
|
|||
|
|
r[13],
|
|||
|
|
r[14],
|
|||
|
|
r[15],
|
|||
|
|
json_dumps(r[16]),
|
|||
|
|
)
|
|||
|
|
for r in node_rows
|
|||
|
|
],
|
|||
|
|
)
|
|||
|
|
for node_id, model_name, mem in [
|
|||
|
|
("node_01", "NVIDIA A800-SXM4-80GB", 80),
|
|||
|
|
("node_02", "NVIDIA GeForce RTX 4090", 24),
|
|||
|
|
]:
|
|||
|
|
for idx in range(4):
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO gpus
|
|||
|
|
(id, node_id, gpu_index, uuid, name, memory_total_gb, power_limit_w, base_temperature)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
f"{node_id}_gpu_{idx}",
|
|||
|
|
node_id,
|
|||
|
|
idx,
|
|||
|
|
f"GPU-{node_id.upper()}-{idx}",
|
|||
|
|
model_name,
|
|||
|
|
mem,
|
|||
|
|
300 if mem >= 80 else 450,
|
|||
|
|
35 + idx,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
self._insert_task(
|
|||
|
|
conn,
|
|||
|
|
task_id="ft_pending_sample",
|
|||
|
|
name="pending-sft-sample",
|
|||
|
|
status="pending",
|
|||
|
|
progress=0,
|
|||
|
|
create_time=now,
|
|||
|
|
start_time=None,
|
|||
|
|
completed_at=None,
|
|||
|
|
node_id=None,
|
|||
|
|
gpus=[],
|
|||
|
|
process_id=None,
|
|||
|
|
)
|
|||
|
|
started = datetime.now(timezone.utc).timestamp() - 22
|
|||
|
|
self._insert_task(
|
|||
|
|
conn,
|
|||
|
|
task_id="ft_running_sample",
|
|||
|
|
name="running-sft-sample",
|
|||
|
|
status="running",
|
|||
|
|
progress=45,
|
|||
|
|
create_time=now,
|
|||
|
|
start_time=datetime.fromtimestamp(started, timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
|||
|
|
completed_at=None,
|
|||
|
|
node_id="node_01",
|
|||
|
|
gpus=[0, 1],
|
|||
|
|
process_id=42001,
|
|||
|
|
)
|
|||
|
|
self._insert_task(
|
|||
|
|
conn,
|
|||
|
|
task_id="ft_completed_sample",
|
|||
|
|
name="completed-sft-sample",
|
|||
|
|
status="completed",
|
|||
|
|
progress=100,
|
|||
|
|
create_time=now,
|
|||
|
|
start_time=now,
|
|||
|
|
completed_at=now,
|
|||
|
|
node_id="node_01",
|
|||
|
|
gpus=[2],
|
|||
|
|
process_id=42002,
|
|||
|
|
)
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO trained_models
|
|||
|
|
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
"tm_completed_sample",
|
|||
|
|
"completed-sft-sample-lora",
|
|||
|
|
json_dumps([{"name": "lora"}]),
|
|||
|
|
"/models/Qwen2.5-7B-Instruct",
|
|||
|
|
now,
|
|||
|
|
0,
|
|||
|
|
0,
|
|||
|
|
"/data/yg-ft/outputs/completed-sft-sample/adapter",
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _insert_dataset(
|
|||
|
|
self,
|
|||
|
|
conn: sqlite3.Connection,
|
|||
|
|
dataset_id: str,
|
|||
|
|
name: str,
|
|||
|
|
dataset_type: str,
|
|||
|
|
description: str,
|
|||
|
|
file_name: str,
|
|||
|
|
content: str,
|
|||
|
|
now: str,
|
|||
|
|
) -> None:
|
|||
|
|
lines = [line for line in content.splitlines() if line.strip()]
|
|||
|
|
size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB"
|
|||
|
|
file_id = f"{dataset_id}_file_1"
|
|||
|
|
version_id = f"{file_id}_v1"
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO datasets
|
|||
|
|
(id, name, type, storage_type, source, size, count, description, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(dataset_id, name, dataset_type, "local", "upload", size, len(lines), description, now),
|
|||
|
|
)
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO dataset_files
|
|||
|
|
(id, dataset_id, name, size, content, active_version_id, versions, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
file_id,
|
|||
|
|
dataset_id,
|
|||
|
|
file_name,
|
|||
|
|
size,
|
|||
|
|
content,
|
|||
|
|
version_id,
|
|||
|
|
json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "initial seed"}]),
|
|||
|
|
now,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _insert_task(
|
|||
|
|
self,
|
|||
|
|
conn: sqlite3.Connection,
|
|||
|
|
task_id: str,
|
|||
|
|
name: str,
|
|||
|
|
status: str,
|
|||
|
|
progress: int,
|
|||
|
|
create_time: str,
|
|||
|
|
start_time: str | None,
|
|||
|
|
completed_at: str | None,
|
|||
|
|
node_id: str | None,
|
|||
|
|
gpus: list[int],
|
|||
|
|
process_id: int | None,
|
|||
|
|
) -> None:
|
|||
|
|
payload = {
|
|||
|
|
"id": task_id,
|
|||
|
|
"name": name,
|
|||
|
|
"description": "Seeded sample fine-tune task.",
|
|||
|
|
"status": status,
|
|||
|
|
"train_type": "SFT",
|
|||
|
|
"train_method": "lora",
|
|||
|
|
"template": "qwen",
|
|||
|
|
"base_model": "m_qwen25_7b",
|
|||
|
|
"train_dataset_id": "ds_finance_sft",
|
|||
|
|
"auto_merge": False,
|
|||
|
|
"output_model_name": f"{name}-lora",
|
|||
|
|
"gpus": gpus,
|
|||
|
|
"batch_size": 2,
|
|||
|
|
"learning_rate": 0.0002,
|
|||
|
|
"n_epochs": 3,
|
|||
|
|
"save_steps": 50,
|
|||
|
|
"lr_scheduler_type": "cosine",
|
|||
|
|
"max_length": 2048,
|
|||
|
|
"warmup_ratio": 0.03,
|
|||
|
|
"weight_decay": 0.01,
|
|||
|
|
"lora_alpha": 16,
|
|||
|
|
"lora_dropout": 0.05,
|
|||
|
|
"lora_rank": 8,
|
|||
|
|
"quantization_bit": 4,
|
|||
|
|
"export_quantized": False,
|
|||
|
|
"quant_method": "bnb",
|
|||
|
|
"quant_bits": 4,
|
|||
|
|
"quant_group_size": 128,
|
|||
|
|
"export_format": "safetensors",
|
|||
|
|
"progress": progress,
|
|||
|
|
"process_id": process_id,
|
|||
|
|
"train_duration": self._duration(start_time, completed_at),
|
|||
|
|
"create_time": create_time,
|
|||
|
|
}
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO fine_tune_tasks
|
|||
|
|
(id, name, payload, status, progress, process_id, create_time, start_time, completed_at, compute_node_id, gpus)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
task_id,
|
|||
|
|
name,
|
|||
|
|
json_dumps(payload),
|
|||
|
|
status,
|
|||
|
|
progress,
|
|||
|
|
process_id,
|
|||
|
|
create_time,
|
|||
|
|
start_time,
|
|||
|
|
completed_at,
|
|||
|
|
node_id,
|
|||
|
|
json_dumps(gpus),
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _duration(self, start_time: str | None, end_time: str | None = None) -> str:
|
|||
|
|
start = parse_time(start_time)
|
|||
|
|
if not start:
|
|||
|
|
return ""
|
|||
|
|
end = parse_time(end_time) or datetime.now(timezone.utc)
|
|||
|
|
seconds = max(0, int((end - start).total_seconds()))
|
|||
|
|
minutes, sec = divmod(seconds, 60)
|
|||
|
|
hours, minutes = divmod(minutes, 60)
|
|||
|
|
if hours:
|
|||
|
|
return f"{hours}h {minutes}m {sec}s"
|
|||
|
|
if minutes:
|
|||
|
|
return f"{minutes}m {sec}s"
|
|||
|
|
return f"{sec}s"
|
|||
|
|
|
|||
|
|
def refresh_runtime_state(self) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute(
|
|||
|
|
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
|
|||
|
|
).fetchall()
|
|||
|
|
now_dt = datetime.now(timezone.utc)
|
|||
|
|
for row in rows:
|
|||
|
|
start = parse_time(row["start_time"])
|
|||
|
|
if not start:
|
|||
|
|
continue
|
|||
|
|
age = max(0, int((now_dt - start).total_seconds()))
|
|||
|
|
if age < 4:
|
|||
|
|
status, progress = "syncing", 8 + age
|
|||
|
|
elif age < 8:
|
|||
|
|
status, progress = "queued", 18 + age
|
|||
|
|
elif age < 70:
|
|||
|
|
status = "running"
|
|||
|
|
progress = min(96, 25 + int((age - 8) / 62 * 70))
|
|||
|
|
else:
|
|||
|
|
status, progress = "completed", 100
|
|||
|
|
|
|||
|
|
payload = json_loads(row["payload"], {})
|
|||
|
|
payload.update(
|
|||
|
|
{
|
|||
|
|
"status": status,
|
|||
|
|
"progress": progress,
|
|||
|
|
"train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None),
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
UPDATE fine_tune_tasks
|
|||
|
|
SET status=?, progress=?, payload=?, completed_at=?
|
|||
|
|
WHERE id=?
|
|||
|
|
""",
|
|||
|
|
(status, progress, json_dumps(payload), completed_at, row["id"]),
|
|||
|
|
)
|
|||
|
|
if status == "completed":
|
|||
|
|
self._ensure_trained_model(conn, payload)
|
|||
|
|
|
|||
|
|
sync_rows = conn.execute(
|
|||
|
|
"SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')"
|
|||
|
|
).fetchall()
|
|||
|
|
for row in sync_rows:
|
|||
|
|
created = parse_time(row["create_time"])
|
|||
|
|
age = int((now_dt - created).total_seconds()) if created else 0
|
|||
|
|
status = "completed" if age >= 6 else "running"
|
|||
|
|
progress = 100 if status == "completed" else min(95, 15 + age * 12)
|
|||
|
|
completed_at = row["completed_at"] or (utcnow() if status == "completed" else None)
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?",
|
|||
|
|
(status, progress, completed_at, row["id"]),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _ensure_trained_model(self, conn: sqlite3.Connection, task: dict[str, Any]) -> None:
|
|||
|
|
name = task.get("output_model_name") or f"{task['name']}-lora"
|
|||
|
|
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
|||
|
|
if exists:
|
|||
|
|
return
|
|||
|
|
model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO trained_models
|
|||
|
|
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
new_id("tm"),
|
|||
|
|
name,
|
|||
|
|
json_dumps([{"name": task.get("train_method", "lora")}]),
|
|||
|
|
model["path"] if model else "",
|
|||
|
|
utcnow(),
|
|||
|
|
0,
|
|||
|
|
0,
|
|||
|
|
f"/data/yg-ft/outputs/{task['name']}/adapter",
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def users(self) -> list[dict[str, Any]]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute("SELECT * FROM users ORDER BY create_time").fetchall()
|
|||
|
|
return [self._user(row) for row in rows]
|
|||
|
|
|
|||
|
|
def login(self, username: str, password: str) -> dict[str, Any] | None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone()
|
|||
|
|
if not row or row["password"] != password or row["status"] != "active":
|
|||
|
|
return None
|
|||
|
|
last_login = utcnow()
|
|||
|
|
conn.execute("UPDATE users SET last_login=? WHERE id=?", (last_login, row["id"]))
|
|||
|
|
data = self._user(row)
|
|||
|
|
data["last_login"] = last_login
|
|||
|
|
return data
|
|||
|
|
|
|||
|
|
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
user_id = new_id("u")
|
|||
|
|
permissions = payload.get("permissions") or (ALL_PERMISSIONS if payload.get("role") == "admin" else ["dashboard"])
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO users
|
|||
|
|
(id, username, password, display_name, role, status, permissions, create_time, protected)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
user_id,
|
|||
|
|
payload["username"],
|
|||
|
|
payload.get("password", "platform123"),
|
|||
|
|
payload.get("display_name") or payload["username"],
|
|||
|
|
payload.get("role", "viewer"),
|
|||
|
|
payload.get("status", "active"),
|
|||
|
|
json_dumps(permissions),
|
|||
|
|
utcnow(),
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone())
|
|||
|
|
|
|||
|
|
def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(user_id)
|
|||
|
|
values = {
|
|||
|
|
"role": payload.get("role", row["role"]),
|
|||
|
|
"status": payload.get("status", row["status"]),
|
|||
|
|
"permissions": json_dumps(payload.get("permissions", json_loads(row["permissions"], []))),
|
|||
|
|
}
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE users SET role=?, status=?, permissions=? WHERE id=?",
|
|||
|
|
(values["role"], values["status"], values["permissions"], user_id),
|
|||
|
|
)
|
|||
|
|
return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone())
|
|||
|
|
|
|||
|
|
def delete_user(self, user_id: str) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(user_id)
|
|||
|
|
if row["protected"]:
|
|||
|
|
raise ValueError("protected user cannot be deleted")
|
|||
|
|
conn.execute("DELETE FROM users WHERE id=?", (user_id,))
|
|||
|
|
|
|||
|
|
def _user(self, row: sqlite3.Row) -> dict[str, Any]:
|
|||
|
|
return {
|
|||
|
|
"id": row["id"],
|
|||
|
|
"username": row["username"],
|
|||
|
|
"display_name": row["display_name"],
|
|||
|
|
"role": row["role"],
|
|||
|
|
"status": row["status"],
|
|||
|
|
"permissions": json_loads(row["permissions"], []),
|
|||
|
|
"create_time": row["create_time"],
|
|||
|
|
"last_login": row["last_login"],
|
|||
|
|
"protected": bool(row["protected"]),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def models(self) -> list[dict[str, Any]]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
return [dict(row) for row in conn.execute("SELECT * FROM models ORDER BY create_time DESC").fetchall()]
|
|||
|
|
|
|||
|
|
def model(self, model_id: str) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(model_id)
|
|||
|
|
return dict(row)
|
|||
|
|
|
|||
|
|
def model_by_name(self, name: str) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM models WHERE name=?", (name,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(name)
|
|||
|
|
return dict(row)
|
|||
|
|
|
|||
|
|
def create_model(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
model_id = payload.get("id") or new_id("m")
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO models
|
|||
|
|
(id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
model_id,
|
|||
|
|
payload["name"],
|
|||
|
|
payload.get("type", "LLM"),
|
|||
|
|
payload.get("purpose", "training"),
|
|||
|
|
payload.get("model_source", "local"),
|
|||
|
|
payload.get("description"),
|
|||
|
|
payload.get("path"),
|
|||
|
|
payload.get("api_url"),
|
|||
|
|
payload.get("api_key"),
|
|||
|
|
payload.get("online_model_name"),
|
|||
|
|
utcnow(),
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return self.model(model_id)
|
|||
|
|
|
|||
|
|
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
current = self.model(model_id)
|
|||
|
|
merged = {**current, **payload}
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
UPDATE models
|
|||
|
|
SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?
|
|||
|
|
WHERE id=?
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
merged["name"],
|
|||
|
|
merged.get("type", "LLM"),
|
|||
|
|
merged.get("purpose", "training"),
|
|||
|
|
merged.get("model_source", "local"),
|
|||
|
|
merged.get("description"),
|
|||
|
|
merged.get("path"),
|
|||
|
|
merged.get("api_url"),
|
|||
|
|
merged.get("api_key"),
|
|||
|
|
merged.get("online_model_name"),
|
|||
|
|
model_id,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return self.model(model_id)
|
|||
|
|
|
|||
|
|
def delete_model(self, model_id: str) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute("DELETE FROM models WHERE id=?", (model_id,))
|
|||
|
|
|
|||
|
|
def trained_models(self) -> list[dict[str, Any]]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
|
|||
|
|
return [
|
|||
|
|
{
|
|||
|
|
**dict(row),
|
|||
|
|
"train_methods": json_loads(row["train_methods"], []),
|
|||
|
|
"merged": bool(row["merged"]),
|
|||
|
|
"merging": bool(row["merging"]),
|
|||
|
|
}
|
|||
|
|
for row in rows
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def datasets(self) -> list[dict[str, Any]]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute("SELECT * FROM datasets ORDER BY create_time DESC").fetchall()
|
|||
|
|
return [self._dataset(conn, row) for row in rows]
|
|||
|
|
|
|||
|
|
def dataset(self, dataset_id: str) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(dataset_id)
|
|||
|
|
return self._dataset(conn, row)
|
|||
|
|
|
|||
|
|
def _dataset(self, conn: sqlite3.Connection, row: sqlite3.Row) -> dict[str, Any]:
|
|||
|
|
files = conn.execute(
|
|||
|
|
"SELECT id, name, size, active_version_id, create_time FROM dataset_files WHERE dataset_id=? ORDER BY create_time",
|
|||
|
|
(row["id"],),
|
|||
|
|
).fetchall()
|
|||
|
|
return {
|
|||
|
|
**dict(row),
|
|||
|
|
"files": [
|
|||
|
|
{
|
|||
|
|
"id": f["id"],
|
|||
|
|
"name": f["name"],
|
|||
|
|
"size": f["size"],
|
|||
|
|
"active_version_id": f["active_version_id"],
|
|||
|
|
"create_time": f["create_time"],
|
|||
|
|
}
|
|||
|
|
for f in files
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
dataset_id = payload.get("id") or new_id("ds")
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO datasets
|
|||
|
|
(id, name, type, storage_type, source, task_id, size, count, description, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
dataset_id,
|
|||
|
|
payload["name"],
|
|||
|
|
payload.get("type", "train"),
|
|||
|
|
payload.get("storage_type", "local"),
|
|||
|
|
payload.get("source", "upload"),
|
|||
|
|
payload.get("task_id"),
|
|||
|
|
payload.get("size", "0 KB"),
|
|||
|
|
payload.get("count", 0),
|
|||
|
|
payload.get("description"),
|
|||
|
|
utcnow(),
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
if not payload.get("files"):
|
|||
|
|
content = '{"instruction":"Sample instruction","input":"Sample input","output":"Sample output"}'
|
|||
|
|
self.add_dataset_file(conn, dataset_id, "sample.jsonl", content)
|
|||
|
|
return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone())
|
|||
|
|
|
|||
|
|
def update_dataset(self, dataset_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
current = self.dataset(dataset_id)
|
|||
|
|
merged = {**current, **payload}
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
UPDATE datasets
|
|||
|
|
SET name=?, type=?, storage_type=?, source=?, task_id=?, size=?, count=?, description=?
|
|||
|
|
WHERE id=?
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
merged["name"],
|
|||
|
|
merged.get("type", "train"),
|
|||
|
|
merged.get("storage_type", "local"),
|
|||
|
|
merged.get("source", "upload"),
|
|||
|
|
merged.get("task_id"),
|
|||
|
|
merged.get("size", "0 KB"),
|
|||
|
|
merged.get("count", 0),
|
|||
|
|
merged.get("description"),
|
|||
|
|
dataset_id,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone())
|
|||
|
|
|
|||
|
|
def delete_dataset(self, dataset_id: str) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,))
|
|||
|
|
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
|
|||
|
|
|
|||
|
|
def add_dataset_file(self, conn: sqlite3.Connection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
|
|||
|
|
now = utcnow()
|
|||
|
|
file_id = new_id("file")
|
|||
|
|
version_id = f"{file_id}_v1"
|
|||
|
|
size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB"
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO dataset_files
|
|||
|
|
(id, dataset_id, name, size, content, active_version_id, versions, create_time)
|
|||
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
file_id,
|
|||
|
|
dataset_id,
|
|||
|
|
name,
|
|||
|
|
size,
|
|||
|
|
content,
|
|||
|
|
version_id,
|
|||
|
|
json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]),
|
|||
|
|
now,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
count = len([line for line in content.splitlines() if line.strip()])
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE datasets SET count=count+?, size=? WHERE id=?",
|
|||
|
|
(count, size, dataset_id),
|
|||
|
|
)
|
|||
|
|
return {"id": file_id, "name": name, "size": size}
|
|||
|
|
|
|||
|
|
def dataset_file(self, file_id: str) -> sqlite3.Row:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(file_id)
|
|||
|
|
return row
|
|||
|
|
|
|||
|
|
def file_versions(self, file_id: str) -> dict[str, Any]:
|
|||
|
|
row = self.dataset_file(file_id)
|
|||
|
|
versions = json_loads(row["versions"], [])
|
|||
|
|
return {
|
|||
|
|
"versions": versions,
|
|||
|
|
"active_version_id": row["active_version_id"],
|
|||
|
|
"next_version_number": len(versions) + 1,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def create_file_version(self, file_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(file_id)
|
|||
|
|
versions = json_loads(row["versions"], [])
|
|||
|
|
version = {
|
|||
|
|
"id": f"{file_id}_v{len(versions) + 1}",
|
|||
|
|
"version": len(versions) + 1,
|
|||
|
|
"create_time": utcnow(),
|
|||
|
|
"description": payload.get("description", "online edit"),
|
|||
|
|
}
|
|||
|
|
versions.append(version)
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?",
|
|||
|
|
(payload.get("content", ""), version["id"], json_dumps(versions), file_id),
|
|||
|
|
)
|
|||
|
|
return {"version": version, "content": payload.get("content", "")}
|
|||
|
|
|
|||
|
|
def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(file_id)
|
|||
|
|
versions = json_loads(row["versions"], [])
|
|||
|
|
version = next((item for item in versions if item["id"] == version_id), None)
|
|||
|
|
if not version:
|
|||
|
|
raise KeyError(version_id)
|
|||
|
|
conn.execute("UPDATE dataset_files SET active_version_id=? WHERE id=?", (version_id, file_id))
|
|||
|
|
return {"version": version, "content": row["content"]}
|
|||
|
|
|
|||
|
|
def tasks(self) -> list[dict[str, Any]]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute("SELECT * FROM fine_tune_tasks ORDER BY create_time DESC").fetchall()
|
|||
|
|
return [self._task(row) for row in rows]
|
|||
|
|
|
|||
|
|
def task(self, task_id: str) -> dict[str, Any]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM fine_tune_tasks WHERE id=?", (task_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(task_id)
|
|||
|
|
return self._task(row)
|
|||
|
|
|
|||
|
|
def _task(self, row: sqlite3.Row) -> dict[str, Any]:
|
|||
|
|
payload = json_loads(row["payload"], {})
|
|||
|
|
payload.update(
|
|||
|
|
{
|
|||
|
|
"id": row["id"],
|
|||
|
|
"status": row["status"],
|
|||
|
|
"progress": row["progress"],
|
|||
|
|
"process_id": row["process_id"],
|
|||
|
|
"create_time": row["create_time"],
|
|||
|
|
"gpus": json_loads(row["gpus"], payload.get("gpus", [])),
|
|||
|
|
"train_duration": self._duration(row["start_time"], row["completed_at"]) if row["start_time"] else "",
|
|||
|
|
"compute_node_id": row["compute_node_id"],
|
|||
|
|
"sync_job_id": row["sync_job_id"],
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
return payload
|
|||
|
|
|
|||
|
|
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
task_id = str(payload.get("task_id") or payload.get("id") or new_id("ft"))
|
|||
|
|
name = payload.get("name") or f"fine-tune-{task_id[-6:]}"
|
|||
|
|
now = utcnow()
|
|||
|
|
task = {
|
|||
|
|
"id": task_id,
|
|||
|
|
"name": name,
|
|||
|
|
"description": payload.get("description", ""),
|
|||
|
|
"status": "pending",
|
|||
|
|
"train_type": payload.get("train_type", "SFT"),
|
|||
|
|
"train_method": payload.get("train_method", "lora"),
|
|||
|
|
"template": payload.get("template", "qwen"),
|
|||
|
|
"base_model": payload.get("base_model") or payload.get("base_model_id") or "m_qwen25_7b",
|
|||
|
|
"train_dataset_id": payload.get("train_dataset_id") or "ds_finance_sft",
|
|||
|
|
"auto_merge": bool(payload.get("auto_merge", False)),
|
|||
|
|
"output_model_name": payload.get("output_model_name") or f"{name}-lora",
|
|||
|
|
"gpus": payload.get("gpus") or [],
|
|||
|
|
"batch_size": payload.get("batch_size", 2),
|
|||
|
|
"learning_rate": payload.get("learning_rate", 0.0002),
|
|||
|
|
"n_epochs": payload.get("n_epochs", 3),
|
|||
|
|
"save_steps": payload.get("save_steps", 50),
|
|||
|
|
"lr_scheduler_type": payload.get("lr_scheduler_type", "cosine"),
|
|||
|
|
"max_length": payload.get("max_length", 2048),
|
|||
|
|
"warmup_ratio": payload.get("warmup_ratio", 0.03),
|
|||
|
|
"weight_decay": payload.get("weight_decay", 0.01),
|
|||
|
|
"lora_alpha": payload.get("lora_alpha", 16),
|
|||
|
|
"lora_dropout": payload.get("lora_dropout", 0.05),
|
|||
|
|
"lora_rank": payload.get("lora_rank", 8),
|
|||
|
|
"quantization_bit": payload.get("quantization_bit", 4),
|
|||
|
|
"export_quantized": bool(payload.get("export_quantized", False)),
|
|||
|
|
"quant_method": payload.get("quant_method", "bnb"),
|
|||
|
|
"quant_bits": payload.get("quant_bits", 4),
|
|||
|
|
"quant_group_size": payload.get("quant_group_size", 128),
|
|||
|
|
"export_format": payload.get("export_format", "safetensors"),
|
|||
|
|
"progress": 0,
|
|||
|
|
"process_id": None,
|
|||
|
|
"train_duration": "",
|
|||
|
|
"create_time": now,
|
|||
|
|
}
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO fine_tune_tasks
|
|||
|
|
(id, name, payload, status, progress, process_id, create_time, gpus)
|
|||
|
|
VALUES (?, ?, ?, 'pending', 0, NULL, ?, ?)
|
|||
|
|
""",
|
|||
|
|
(task_id, name, json_dumps(task), now, json_dumps(task["gpus"])),
|
|||
|
|
)
|
|||
|
|
return task
|
|||
|
|
|
|||
|
|
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
current = self.task(task_id)
|
|||
|
|
merged = {**current, **payload, "id": task_id}
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE fine_tune_tasks SET name=?, payload=?, gpus=? WHERE id=?",
|
|||
|
|
(merged["name"], json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id),
|
|||
|
|
)
|
|||
|
|
return self.task(task_id)
|
|||
|
|
|
|||
|
|
def start_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
task_id = str(payload.get("task_id") or payload.get("id"))
|
|||
|
|
current = self.task(task_id)
|
|||
|
|
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
|
|||
|
|
node = self.schedule_node(payload)
|
|||
|
|
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0]
|
|||
|
|
process_id = int(43000 + (time.time() % 10000))
|
|||
|
|
sync_job_id = self.create_sync_job(node["id"], current)
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
UPDATE fine_tune_tasks
|
|||
|
|
SET payload=?, status='syncing', progress=8, process_id=?, start_time=?,
|
|||
|
|
compute_node_id=?, gpus=?, sync_job_id=?
|
|||
|
|
WHERE id=?
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
json_dumps({**merged, "process_id": process_id, "gpus": selected_gpus}),
|
|||
|
|
process_id,
|
|||
|
|
utcnow(),
|
|||
|
|
node["id"],
|
|||
|
|
json_dumps(selected_gpus),
|
|||
|
|
sync_job_id,
|
|||
|
|
task_id,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return self.task(task_id)
|
|||
|
|
|
|||
|
|
def stop_task(self, task_id: str) -> dict[str, Any]:
|
|||
|
|
task = self.task(task_id)
|
|||
|
|
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
|||
|
|
(json_dumps(task), utcnow(), task_id),
|
|||
|
|
)
|
|||
|
|
return self.task(task_id)
|
|||
|
|
|
|||
|
|
def delete_task(self, task_id: str) -> None:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,))
|
|||
|
|
|
|||
|
|
def schedule_node(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
|
|||
|
|
nodes = self.compute_nodes()
|
|||
|
|
candidates = [
|
|||
|
|
n
|
|||
|
|
for n in nodes
|
|||
|
|
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"]
|
|||
|
|
]
|
|||
|
|
if requested:
|
|||
|
|
selected = next((n for n in candidates if n["id"] == requested), None)
|
|||
|
|
if selected:
|
|||
|
|
return selected
|
|||
|
|
if not candidates:
|
|||
|
|
raise RuntimeError("no available compute node")
|
|||
|
|
return sorted(candidates, key=lambda n: (-n["scheduler_weight"], n["current_running_jobs"], n["code"]))[0]
|
|||
|
|
|
|||
|
|
def create_sync_job(self, node_id: str, task: dict[str, Any]) -> str:
|
|||
|
|
sync_id = new_id("sync")
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
INSERT INTO resource_sync_jobs
|
|||
|
|
(id, target_node_id, resources, status, progress, create_time)
|
|||
|
|
VALUES (?, ?, ?, 'pending', 0, ?)
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
sync_id,
|
|||
|
|
node_id,
|
|||
|
|
json_dumps(
|
|||
|
|
[
|
|||
|
|
{"resource_type": "model", "resource_id": task.get("base_model")},
|
|||
|
|
{"resource_type": "dataset", "resource_id": task.get("train_dataset_id")},
|
|||
|
|
]
|
|||
|
|
),
|
|||
|
|
utcnow(),
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return sync_id
|
|||
|
|
|
|||
|
|
def progress(self, task_id: str) -> dict[str, Any]:
|
|||
|
|
task = self.task(task_id)
|
|||
|
|
status = task.get("status", "pending")
|
|||
|
|
labels = {
|
|||
|
|
"pending": "waiting for start",
|
|||
|
|
"syncing": "syncing model and dataset to compute node",
|
|||
|
|
"queued": "waiting for GPU slot",
|
|||
|
|
"running": "training with LLaMA-Factory simulator",
|
|||
|
|
"completed": "training completed",
|
|||
|
|
"failed": "training stopped",
|
|||
|
|
}
|
|||
|
|
progress = int(task.get("progress", 0) or 0)
|
|||
|
|
eta = "--" if status in {"completed", "failed"} else f"{max(1, math.ceil((100 - progress) / 10))} min"
|
|||
|
|
return {
|
|||
|
|
"status": status,
|
|||
|
|
"progress": progress,
|
|||
|
|
"step": labels.get(status, status),
|
|||
|
|
"speed": "42.5 samples/s" if status == "running" else "--",
|
|||
|
|
"eta": eta,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def compute_nodes(self) -> list[dict[str, Any]]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
running = conn.execute(
|
|||
|
|
"SELECT compute_node_id, COUNT(*) AS cnt FROM fine_tune_tasks WHERE status IN ('syncing','queued','running') GROUP BY compute_node_id"
|
|||
|
|
).fetchall()
|
|||
|
|
running_map = {r["compute_node_id"]: r["cnt"] for r in running}
|
|||
|
|
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
|
|||
|
|
return [
|
|||
|
|
{
|
|||
|
|
**dict(row),
|
|||
|
|
"enabled": bool(row["enabled"]),
|
|||
|
|
"tags": json_loads(row["tags"], []),
|
|||
|
|
"health_detail": json_loads(row["health_detail"], {}),
|
|||
|
|
"current_running_jobs": running_map.get(row["id"], 0),
|
|||
|
|
}
|
|||
|
|
for row in rows
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def update_compute_node(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
|||
|
|
if not current:
|
|||
|
|
raise KeyError(node_id)
|
|||
|
|
merged = {**current, **payload}
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
conn.execute(
|
|||
|
|
"""
|
|||
|
|
UPDATE compute_nodes
|
|||
|
|
SET name=?, api_base_url=?, file_gateway_url=?, enabled=?, scheduler_status=?,
|
|||
|
|
scheduler_weight=?, tags=?, max_parallel_jobs=?, last_health_check_at=?
|
|||
|
|
WHERE id=?
|
|||
|
|
""",
|
|||
|
|
(
|
|||
|
|
merged["name"],
|
|||
|
|
merged["api_base_url"],
|
|||
|
|
merged["file_gateway_url"],
|
|||
|
|
1 if merged["enabled"] else 0,
|
|||
|
|
merged["scheduler_status"],
|
|||
|
|
merged["scheduler_weight"],
|
|||
|
|
json_dumps(merged["tags"]),
|
|||
|
|
merged["max_parallel_jobs"],
|
|||
|
|
utcnow(),
|
|||
|
|
node_id,
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
return next(n for n in self.compute_nodes() if n["id"] == node_id)
|
|||
|
|
|
|||
|
|
def gpus(self) -> list[dict[str, Any]]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute(
|
|||
|
|
"""
|
|||
|
|
SELECT g.*, n.code AS node_code, n.name AS node_name
|
|||
|
|
FROM gpus g JOIN compute_nodes n ON n.id = g.node_id
|
|||
|
|
ORDER BY n.code, g.gpu_index
|
|||
|
|
"""
|
|||
|
|
).fetchall()
|
|||
|
|
running_tasks = [
|
|||
|
|
self._task(row)
|
|||
|
|
for row in conn.execute(
|
|||
|
|
"SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')"
|
|||
|
|
).fetchall()
|
|||
|
|
]
|
|||
|
|
items = []
|
|||
|
|
for row in rows:
|
|||
|
|
task = next(
|
|||
|
|
(
|
|||
|
|
t
|
|||
|
|
for t in running_tasks
|
|||
|
|
if t.get("compute_node_id") == row["node_id"] and row["gpu_index"] in (t.get("gpus") or [])
|
|||
|
|
),
|
|||
|
|
None,
|
|||
|
|
)
|
|||
|
|
busy = task is not None and task.get("status") == "running"
|
|||
|
|
reserved = task is not None and task.get("status") in {"syncing", "queued"}
|
|||
|
|
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
|
|||
|
|
gpu_percent = 86 if busy else 22 if reserved else 3
|
|||
|
|
items.append(
|
|||
|
|
{
|
|||
|
|
"id": row["gpu_index"],
|
|||
|
|
"node_id": row["node_id"],
|
|||
|
|
"node_code": row["node_code"],
|
|||
|
|
"node_name": row["node_name"],
|
|||
|
|
"name": row["name"],
|
|||
|
|
"uuid": row["uuid"],
|
|||
|
|
"gpu_percent": gpu_percent,
|
|||
|
|
"memory_used_gb": memory_used,
|
|||
|
|
"memory_total_gb": row["memory_total_gb"],
|
|||
|
|
"memory_percent": round(memory_used / row["memory_total_gb"] * 100, 1),
|
|||
|
|
"temperature": row["base_temperature"] + (21 if busy else 6 if reserved else 0),
|
|||
|
|
"power_w": round(row["power_limit_w"] * (0.7 if busy else 0.25 if reserved else 0.08), 1),
|
|||
|
|
"power_limit_w": row["power_limit_w"],
|
|||
|
|
"status": "busy" if busy else "reserved" if reserved else "idle",
|
|||
|
|
"processes": [
|
|||
|
|
{
|
|||
|
|
"pid": task["process_id"],
|
|||
|
|
"name": "llamafactory-cli",
|
|||
|
|
"memory_used_gb": memory_used,
|
|||
|
|
"task_name": task["name"],
|
|||
|
|
"user": "admin",
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
if task
|
|||
|
|
else [],
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
return items
|
|||
|
|
|
|||
|
|
def system_info(self) -> dict[str, Any]:
|
|||
|
|
gpus = self.gpus()
|
|||
|
|
busy = len([g for g in gpus if g["status"] in {"busy", "reserved"}])
|
|||
|
|
cpu_percent = 18 + busy * 9
|
|||
|
|
memory_percent = 37 + busy * 4
|
|||
|
|
return {
|
|||
|
|
"timestamp": utcnow(),
|
|||
|
|
"cpu": {
|
|||
|
|
"percent": min(cpu_percent, 95),
|
|||
|
|
"cores": 32,
|
|||
|
|
"percents": [min(cpu_percent + (i % 7) - 3, 99) for i in range(32)],
|
|||
|
|
"model": "Platform x86_64 CPU",
|
|||
|
|
"frequency_mhz": 2600,
|
|||
|
|
"load_1m": round(cpu_percent / 10, 2),
|
|||
|
|
},
|
|||
|
|
"memory": {
|
|||
|
|
"used_gb": round(256 * memory_percent / 100, 1),
|
|||
|
|
"total_gb": 256,
|
|||
|
|
"percent": min(memory_percent, 95),
|
|||
|
|
"available_gb": round(256 * (100 - memory_percent) / 100, 1),
|
|||
|
|
"cached_gb": 32,
|
|||
|
|
},
|
|||
|
|
"disk": {
|
|||
|
|
"used_gb": 840,
|
|||
|
|
"total_gb": 2048,
|
|||
|
|
"percent": 41,
|
|||
|
|
"read_mb_s": 120 if busy else 8,
|
|||
|
|
"write_mb_s": 95 if busy else 5,
|
|||
|
|
},
|
|||
|
|
"gpu": gpus,
|
|||
|
|
"network": {
|
|||
|
|
"download_mb_s": 12 if busy else 1.2,
|
|||
|
|
"upload_mb_s": 7 if busy else 0.8,
|
|||
|
|
"download_mb": 8024,
|
|||
|
|
"upload_mb": 1732,
|
|||
|
|
},
|
|||
|
|
"system": {
|
|||
|
|
"uptime_seconds": int(time.time() % 100000),
|
|||
|
|
"process_count": 248 + busy,
|
|||
|
|
"os": "Linux platform compute image",
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def health_metrics(self) -> dict[str, float]:
|
|||
|
|
info = self.system_info()
|
|||
|
|
return {
|
|||
|
|
"cpu_percent": info["cpu"]["percent"],
|
|||
|
|
"memory_percent": info["memory"]["percent"],
|
|||
|
|
"disk_percent": info["disk"]["percent"],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def queue(self) -> list[dict[str, Any]]:
|
|||
|
|
return [
|
|||
|
|
{
|
|||
|
|
"id": task["id"],
|
|||
|
|
"name": task["name"],
|
|||
|
|
"status": task["status"],
|
|||
|
|
"progress": task.get("progress", 0),
|
|||
|
|
"compute_node_id": task.get("compute_node_id"),
|
|||
|
|
"gpus": task.get("gpus", []),
|
|||
|
|
"create_time": task.get("create_time"),
|
|||
|
|
}
|
|||
|
|
for task in self.tasks()
|
|||
|
|
if task["status"] in {"pending", "syncing", "queued", "running"}
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def replicas(self, node_id: str) -> list[dict[str, Any]]:
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
rows = conn.execute(
|
|||
|
|
"SELECT * FROM resource_replicas WHERE node_id=? ORDER BY create_time DESC",
|
|||
|
|
(node_id,),
|
|||
|
|
).fetchall()
|
|||
|
|
if rows:
|
|||
|
|
return [dict(row) for row in rows]
|
|||
|
|
return [
|
|||
|
|
{
|
|||
|
|
"id": f"rep_{node_id}_model_qwen",
|
|||
|
|
"node_id": node_id,
|
|||
|
|
"resource_type": "model",
|
|||
|
|
"resource_id": "m_qwen25_7b",
|
|||
|
|
"local_path": "/models/Qwen2.5-7B-Instruct",
|
|||
|
|
"status": "available",
|
|||
|
|
"sync_status": "completed",
|
|||
|
|
"create_time": utcnow(),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": f"rep_{node_id}_dataset_finance",
|
|||
|
|
"node_id": node_id,
|
|||
|
|
"resource_type": "dataset",
|
|||
|
|
"resource_id": "ds_finance_sft",
|
|||
|
|
"local_path": "/data/yg-ft/datasets/finance-sft-sample.jsonl",
|
|||
|
|
"status": "available",
|
|||
|
|
"sync_status": "completed",
|
|||
|
|
"create_time": utcnow(),
|
|||
|
|
},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def sync_job(self, sync_id: str) -> dict[str, Any]:
|
|||
|
|
self.refresh_runtime_state()
|
|||
|
|
with self.connect() as conn:
|
|||
|
|
row = conn.execute("SELECT * FROM resource_sync_jobs WHERE id=?", (sync_id,)).fetchone()
|
|||
|
|
if not row:
|
|||
|
|
raise KeyError(sync_id)
|
|||
|
|
return {**dict(row), "resources": json_loads(row["resources"], [])}
|
|||
|
|
|
|||
|
|
def training_log_files(self) -> list[dict[str, Any]]:
|
|||
|
|
return [
|
|||
|
|
{
|
|||
|
|
"file": f"train_{task['id']}_pid{task['process_id'] or 0}.log",
|
|||
|
|
"name": task["name"],
|
|||
|
|
"pid": task.get("process_id") or 0,
|
|||
|
|
"size": f"{max(1, int((task.get('progress', 0) or 0) * 1.5))} KB",
|
|||
|
|
"date": task.get("create_time", "")[:10],
|
|||
|
|
}
|
|||
|
|
for task in self.tasks()
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def training_log_content(self, file_name: str) -> dict[str, Any]:
|
|||
|
|
task = next((t for t in self.tasks() if t["id"] in file_name), None)
|
|||
|
|
if not task:
|
|||
|
|
raise KeyError(file_name)
|
|||
|
|
content = self.generate_training_log(task)
|
|||
|
|
return {"file": file_name, "content": content, "size": f"{max(1, len(content.encode('utf-8')) // 1024)} KB"}
|
|||
|
|
|
|||
|
|
def generate_training_log(self, task: dict[str, Any]) -> str:
|
|||
|
|
progress = int(task.get("progress", 0) or 0)
|
|||
|
|
points = max(1, min(80, progress))
|
|||
|
|
lines = [
|
|||
|
|
f"[INFO] task={task['name']} engine=llama_factory mode=simulator status={task['status']}",
|
|||
|
|
f"[INFO] base_model={task.get('base_model')} dataset={task.get('train_dataset_id')} gpus={task.get('gpus', [])}",
|
|||
|
|
"[INFO] command=llamafactory-cli train --stage sft --finetuning_type lora --do_train true",
|
|||
|
|
]
|
|||
|
|
for step in range(1, points + 1):
|
|||
|
|
if step % 3 != 0 and step != points:
|
|||
|
|
continue
|
|||
|
|
loss = max(0.12, 2.4 * math.exp(-step / 42))
|
|||
|
|
grad_norm = 0.45 + (step % 8) * 0.03
|
|||
|
|
lr = float(task.get("learning_rate") or 0.0002) * max(0.05, 1 - step / 120)
|
|||
|
|
epoch = round(step / max(1, points) * float(task.get("n_epochs") or 3), 4)
|
|||
|
|
lines.append(
|
|||
|
|
"{"
|
|||
|
|
f"'loss': {loss:.4f}, 'grad_norm': {grad_norm:.4f}, "
|
|||
|
|
f"'learning_rate': {lr:.8f}, 'epoch': {epoch:.4f}"
|
|||
|
|
"}"
|
|||
|
|
)
|
|||
|
|
if task.get("status") == "completed":
|
|||
|
|
lines.extend(
|
|||
|
|
[
|
|||
|
|
"***** train metrics *****",
|
|||
|
|
f"epoch = {task.get('n_epochs', 3)}",
|
|||
|
|
"train_loss = 0.1248",
|
|||
|
|
f"train_runtime = {task.get('train_duration') or '1m 10s'}",
|
|||
|
|
"***** train metrics end *****",
|
|||
|
|
]
|
|||
|
|
)
|
|||
|
|
return "\n".join(lines)
|
|||
|
|
|
|||
|
|
def log_files(self, date: str | None = None) -> list[dict[str, Any]]:
|
|||
|
|
today = date or utcnow()[:10]
|
|||
|
|
return [
|
|||
|
|
{"file": f"backend-{today}.log", "name": f"backend-{today}.log", "size": "32 KB", "date": today},
|
|||
|
|
{"file": f"error-{today}.log", "name": f"error-{today}.log", "size": "1 KB", "date": today},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
def log_content(self, file_name: str) -> dict[str, Any]:
|
|||
|
|
lines = [
|
|||
|
|
json_dumps(
|
|||
|
|
{
|
|||
|
|
"timestamp": utcnow(),
|
|||
|
|
"level": "INFO",
|
|||
|
|
"logger": "platform",
|
|||
|
|
"file": "backend/app/api/v1/endpoints/platform.py",
|
|||
|
|
"line": 1,
|
|||
|
|
"message": "Platform log stream is available.",
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
]
|
|||
|
|
return {"file": file_name, "content": "\n".join(lines), "size": "1 KB"}
|
|||
|
|
|
|||
|
|
|
|||
|
|
_store: PlatformStore | None = None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_platform_store() -> PlatformStore:
|
|||
|
|
global _store
|
|||
|
|
if _store is None:
|
|||
|
|
_store = PlatformStore()
|
|||
|
|
return _store
|
|||
|
|
|