YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: 合并远程 ft_wyt 分支，解决冲突

Browse Source
This commit is contained in:
wangjiming
2026-08-19 17:39:18 +08:00
parent 78680a5c5c 080ef6ab00
commit 8455d99d49
64 changed files with 4616 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
backend/app/db/platform_store.py
View File

@@ -454,15 +454,21 @@ class PlatformStore:
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
self._inference_nodes: set[str] = set()
self._inference_gpu_indexes: dict[str, set[int]] = {}
self._last_runtime_refresh = 0.0
# ── inference node tracking ────────────────────────────────────
def mark_inference_loaded(self, node_id: str) -> None:
def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None:
self._inference_nodes.add(node_id)
if gpu_indexes is not None:
self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes}
else:
self._inference_gpu_indexes.pop(node_id, None)
def mark_inference_unloaded(self, node_id: str) -> None:
self._inference_nodes.discard(node_id)
self._inference_gpu_indexes.pop(node_id, None)
def is_inference_loaded(self, node_id: str) -> bool:
return node_id in self._inference_nodes
@@ -545,6 +551,16 @@ class PlatformStore:
"last_error": "TEXT",
},
)
self._ensure_columns(
conn,
"storage_objects",
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
)
self._ensure_columns(
conn,
"model_artifacts",
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
)
schema_dir = Path(__file__).with_name("sql")
for extra in (
"002_governance.sql",
@@ -556,7 +572,17 @@ class PlatformStore:
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
# data_convert_tasks 表补充 created_by 字段（用于数据隔离）
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
self._ensure_columns(
conn,
"data_convert_tasks",
{
"created_by": "TEXT",
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
"output_storage_object_id": "TEXT",
"output_content": "TEXT",
},
)
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
# 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录
try:
conn.execute("""
@@ -1254,6 +1280,13 @@ class PlatformStore:
raise KeyError(artifact_id)
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
with self.connect() as conn:
conn.execute(
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
(storage_object_id, artifact_id),
)
def model_lineage(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
parents = conn.execute(
@@ -2221,7 +2254,7 @@ class PlatformStore:
(str(validation_dataset_id),),
).fetchall(),
]
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
"dataset_split"
@@ -2482,12 +2515,18 @@ class PlatformStore:
def eval_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
# 评测任务采用软删除，普通列表不得再次返回已删除记录。
rows = conn.execute(
"SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC"
).fetchall()
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
def eval_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
row = conn.execute(
"SELECT * FROM eval_tasks WHERE id=? AND deleted_at IS NULL",
(task_id,),
).fetchone()
if not row:
raise KeyError(task_id)
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
@@ -2558,7 +2597,13 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
result = conn.execute(
"UPDATE eval_tasks SET deleted_at=?, deleted_by=? "
"WHERE id=? AND deleted_at IS NULL RETURNING id",
(utcnow(), "system", task_id),
)
if not result.fetchone():
raise KeyError(task_id)
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2766,7 +2811,35 @@ class PlatformStore:
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
(node_id,),
).fetchall()
return {int(row["gpu_index"]) for row in rows}
active = {int(row["gpu_index"]) for row in rows}
# Evaluation jobs use the same Compute ProcessManager GPU lock but do
# not have fine-tune allocation rows; derive their selected cards here
# so a training task cannot race onto an evaluation GPU.
for row in conn.execute(
"SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')"
).fetchall():
payload = json_loads(row["payload"], {})
if payload.get("compute_node_id") != node_id:
continue
selected = payload.get("gpu_indices") or payload.get("gpus")
if selected is None and payload.get("gpu_id") is not None:
selected = [payload.get("gpu_id")]
active.update(int(item) for item in selected or [])
# Loaded inference models also reserve only their selected cards.
active.update(self._inference_gpu_indexes.get(node_id, set()))
for row in conn.execute("SELECT payload FROM compare_tasks").fetchall():
payload = json_loads(row["payload"], {})
load_status = payload.get("load_status") or {}
if isinstance(load_status, str):
load_status = json_loads(load_status, {})
for item in load_status.get("loaded_models") or []:
if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}:
continue
selected = item.get("gpu_indices") or item.get("gpus")
if selected is None and item.get("gpu_id") is not None:
selected = [item.get("gpu_id")]
active.update(int(gpu) for gpu in selected or [])
return active
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
@@ -3000,17 +3073,18 @@ class PlatformStore:
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
json_dumps(payload.get("metadata") or {}),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
@@ -3049,6 +3123,13 @@ class PlatformStore:
).fetchall()
return [dict(row) for row in rows]
def storage_object(self, object_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
@@ -3195,6 +3276,9 @@ class PlatformStore:
# 推理模型占用算力节点同样计入：优先从 compare_tasks 持久化状态派生
# （重启后仍准确），并用内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3204,8 +3288,13 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
for nid in inference_node_ids:
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
for nid in set(inference_node_ids) | set(inference_gpu_indexes):
running_map[nid] = running_map.get(nid, 0) + 1
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
return [
@@ -3434,6 +3523,9 @@ class PlatformStore:
# 推理模型占用的节点：优先从 compare_tasks 持久化状态派生（重启后仍准确），
# 内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3443,7 +3535,12 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
items = []
for row in rows:
task = next(
@@ -3459,7 +3556,10 @@ class PlatformStore:
t
for t in eval_running
if t.get("compute_node_id") == row["node_id"]
and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1)
and row["gpu_index"] in {
int(item)
for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else []))
}
),
None,
)
@@ -3468,7 +3568,8 @@ class PlatformStore:
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
)
# Also mark GPU as busy if an inference model is loaded on this node
if row["node_id"] in inference_node_ids and not busy:
inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set())
if inference_on_gpu and not busy:
busy = True
reserved = False
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
@@ -3704,6 +3805,8 @@ class PlatformStore:
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
keyword: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
@@ -3726,6 +3829,13 @@ class PlatformStore:
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if target_id:
clauses.append("target_id=?")
params.append(target_id)
if keyword:
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
pattern = f"%{keyword}%"
params.extend([pattern, pattern])
if start_time:
clauses.append("time>=?")
params.append(start_time)

11
backend/app/db/sql/000_full_init.sql
View File

@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
path TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum_sha256 TEXT,
storage_object_id TEXT,
storage_backend TEXT NOT NULL DEFAULT 'minio',
metadata TEXT NOT NULL,
compute_job_id TEXT,
create_time TEXT NOT NULL
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
metadata TEXT NOT NULL DEFAULT '{}',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
input_count INTEGER NOT NULL DEFAULT 0,
output_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
output_content TEXT,
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员