feat: 平台治理与对象存储增强,审批中心与运行日志整合

- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚
- 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts
- 数据转换任务输入输出对象化,支持从 MinIO 读写
- 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions

View File

@@ -2,15 +2,75 @@ from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.storage.minio_store import get_object_storage
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
MAX_STARTING_ATTEMPTS = 40
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
node: dict[str, Any],
source_path: str,
resource_type: str,
resource_id: str,
version_id: str,
object_prefix: str,
) -> list[dict[str, Any]]:
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
for item in listing.get("items") or []:
item_relative = str(item.get("relative_path") or "")
if item.get("type") == "directory":
queue.append(item_relative)
continue
path = str(item.get("path") or "")
if not path:
continue
try:
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
archived.append(
store.create_storage_object(
{
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": relative_file,
"content_type": "application/octet-stream",
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
"checksum_sha256": result.get("checksum_sha256") or "",
"status": "available",
}
)
)
return archived
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -139,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
updated_task = store.apply_compute_job(task["id"], job)
if (
get_settings().minio_enabled
and job.get("status") == "completed"
and job.get("output_dir")
):
trained_model = next(
(
item
for item in store.trained_models()
if item.get("name")
== (task.get("output_model_name") or f"{task.get('name')}-lora")
),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
@@ -151,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
try:
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
payload = (store.compute_job(record["id"]).get("payload") or {})
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
if trained_model_id:
trained_model = next(
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
ComputeNodeClient(node["api_base_url"], timeout=900),
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or record["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
@@ -175,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"eval",
str(eval_task["id"]),
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001