feat: 模型推理异步加载与对话链路修复,同步基线
模型推理全异步化改造: - 计算节点 InferenceSession 改为后台线程异步加载模型,load 立即返回, 加载期间事件循环保持响应(/inference/status 与 /health 不阻塞) - 后端模型加载改为异步派发 + 轮询对账器(reconcile_inference_loads), 任务状态由 starting 自动推进到 ready/error,解决多节点启动超时 (timeout of 120000ms exceeded) - 推理删除/卸载改为任务感知 + 短超时,删除先删记录再 best-effort 卸载, 不再被不可达节点阻塞;同节点新模型替换旧任务标记失效 - 流式对话透传 task_id/node_id 路由到真正加载模型的算力节点, useStreamChat 解析 SSE 错误帧以干净文案展示 - 对话历史按任务 id 本地持久化,退出重进可恢复;移除页脚提示文本 - 新增后端推理异步加载与计算节点异步状态机单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -208,8 +208,12 @@ def parse_training_metric_line(line: str) -> dict[str, float] | None:
|
||||
if "loss" not in line and "learning_rate" not in line:
|
||||
return None
|
||||
result: dict[str, float] = {}
|
||||
number_pattern = r"([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)"
|
||||
step_match = re.search(rf"(?:^|[\s,{{])['\"]?step['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I)
|
||||
if step_match:
|
||||
result["step"] = float(step_match.group(1))
|
||||
for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
|
||||
match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
|
||||
match = re.search(rf"['\"]?{key}['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I)
|
||||
if match:
|
||||
result[key] = float(match.group(1))
|
||||
return result or None
|
||||
@@ -452,6 +456,15 @@ class PlatformStore:
|
||||
)
|
||||
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
|
||||
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"trained_models",
|
||||
{
|
||||
"artifact_dir": "TEXT",
|
||||
"compute_node_id": "TEXT",
|
||||
"compute_node_name": "TEXT",
|
||||
},
|
||||
)
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"resource_replicas",
|
||||
@@ -589,6 +602,15 @@ class PlatformStore:
|
||||
name = task.get("output_model_name") or f"{task['name']}-lora"
|
||||
exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone()
|
||||
if exists:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE trained_models
|
||||
SET compute_node_id=COALESCE(compute_node_id, ?),
|
||||
compute_node_name=COALESCE(compute_node_name, ?)
|
||||
WHERE id=?
|
||||
""",
|
||||
(task.get("compute_node_id"), task.get("compute_node_code") or task.get("compute_node_name"), exists["id"]),
|
||||
)
|
||||
return
|
||||
model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
||||
output_dir = task.get("output_dir") or f"/data/yg-ft/outputs/{task['name']}"
|
||||
@@ -596,8 +618,8 @@ class PlatformStore:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO trained_models
|
||||
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
(id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir, compute_node_id, compute_node_name)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
trained_model_id,
|
||||
@@ -609,6 +631,8 @@ class PlatformStore:
|
||||
0,
|
||||
output_dir,
|
||||
output_dir,
|
||||
task.get("compute_node_id"),
|
||||
task.get("compute_node_code") or task.get("compute_node_name"),
|
||||
),
|
||||
)
|
||||
# Use real artifact data from compute node when available
|
||||
@@ -875,7 +899,7 @@ class PlatformStore:
|
||||
(
|
||||
new_id("metric"),
|
||||
task_id,
|
||||
line_number,
|
||||
int(metric.get("step") or line_number),
|
||||
metric.get("epoch"),
|
||||
metric.get("loss"),
|
||||
metric.get("grad_norm"),
|
||||
@@ -1342,15 +1366,25 @@ class PlatformStore:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
|
||||
return [
|
||||
{
|
||||
items = []
|
||||
for row in rows:
|
||||
item = {
|
||||
**dict(row),
|
||||
"train_methods": json_loads(row["train_methods"], []),
|
||||
"merged": bool(row["merged"]),
|
||||
"merging": bool(row["merging"]),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
if not item.get("compute_node_id"):
|
||||
task_rows = conn.execute("SELECT payload FROM fine_tune_tasks ORDER BY create_time DESC").fetchall()
|
||||
for task in task_rows:
|
||||
task_payload = json_loads(task["payload"], {})
|
||||
output_name = task_payload.get("output_model_name") or f"{task_payload.get('name')}-lora"
|
||||
if output_name == item["name"]:
|
||||
item["compute_node_id"] = task_payload.get("compute_node_id")
|
||||
item["compute_node_name"] = task_payload.get("compute_node_code") or task_payload.get("compute_node_name")
|
||||
break
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
def delete_trained_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
|
||||
@@ -33,7 +33,10 @@ CREATE TABLE IF NOT EXISTS trained_models (
|
||||
create_time TEXT NOT NULL,
|
||||
merged INTEGER NOT NULL DEFAULT 0,
|
||||
merging INTEGER NOT NULL DEFAULT 0,
|
||||
merged_path TEXT
|
||||
merged_path TEXT,
|
||||
artifact_dir TEXT,
|
||||
compute_node_id TEXT,
|
||||
compute_node_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||
|
||||
Reference in New Issue
Block a user