feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -2,15 +2,75 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.core.config import get_settings
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.storage.minio_store import get_object_storage
|
||||
|
||||
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
||||
MAX_STARTING_ATTEMPTS = 40
|
||||
|
||||
|
||||
async def _archive_node_directory(
|
||||
store: Any,
|
||||
client: ComputeNodeClient,
|
||||
node: dict[str, Any],
|
||||
source_path: str,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
version_id: str,
|
||||
object_prefix: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Archive a completed node directory to MinIO, preserving subdirectories."""
|
||||
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
|
||||
source = Path(source_path).resolve()
|
||||
try:
|
||||
relative_root = source.relative_to(data_root).as_posix()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
|
||||
queue = [relative_root]
|
||||
archived: list[dict[str, Any]] = []
|
||||
while queue:
|
||||
relative = queue.pop(0)
|
||||
listing = await client.list_files(root="data", relative_path=relative)
|
||||
for item in listing.get("items") or []:
|
||||
item_relative = str(item.get("relative_path") or "")
|
||||
if item.get("type") == "directory":
|
||||
queue.append(item_relative)
|
||||
continue
|
||||
path = str(item.get("path") or "")
|
||||
if not path:
|
||||
continue
|
||||
try:
|
||||
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
|
||||
except ValueError:
|
||||
relative_file = Path(str(item.get("name") or Path(path).name)).name
|
||||
object_key = f"{object_prefix}/{version_id}/{relative_file}"
|
||||
upload_url = get_object_storage().presigned_put(object_key)
|
||||
result = await client.upload_file_to_url(path, upload_url, object_key)
|
||||
metadata = get_object_storage().stat(object_key)
|
||||
archived.append(
|
||||
store.create_storage_object(
|
||||
{
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"version_id": version_id,
|
||||
"bucket": get_object_storage().bucket,
|
||||
"object_key": object_key,
|
||||
"file_name": relative_file,
|
||||
"content_type": "application/octet-stream",
|
||||
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
|
||||
"checksum_sha256": result.get("checksum_sha256") or "",
|
||||
"status": "available",
|
||||
}
|
||||
)
|
||||
)
|
||||
return archived
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
@@ -139,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||
except Exception:
|
||||
pass
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
updated_task = store.apply_compute_job(task["id"], job)
|
||||
if (
|
||||
get_settings().minio_enabled
|
||||
and job.get("status") == "completed"
|
||||
and job.get("output_dir")
|
||||
):
|
||||
trained_model = next(
|
||||
(
|
||||
item
|
||||
for item in store.trained_models()
|
||||
if item.get("name")
|
||||
== (task.get("output_model_name") or f"{task.get('name')}-lora")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if trained_model:
|
||||
archived = await _archive_node_directory(
|
||||
store,
|
||||
client,
|
||||
node,
|
||||
str(job["output_dir"]),
|
||||
"trained_model",
|
||||
str(trained_model["id"]),
|
||||
str(job.get("id") or task.get("compute_job_id") or task["id"]),
|
||||
f"trained_models/{trained_model['id']}",
|
||||
)
|
||||
artifacts = store.model_artifacts(str(trained_model["id"]))
|
||||
if archived and artifacts:
|
||||
store.link_model_artifact_storage_object(
|
||||
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||
)
|
||||
synced.append(updated_task)
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
standalone_synced: list[dict[str, Any]] = []
|
||||
@@ -151,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
try:
|
||||
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
||||
payload = (store.compute_job(record["id"]).get("payload") or {})
|
||||
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
|
||||
if trained_model_id:
|
||||
trained_model = next(
|
||||
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
|
||||
None,
|
||||
)
|
||||
if trained_model:
|
||||
archived = await _archive_node_directory(
|
||||
store,
|
||||
ComputeNodeClient(node["api_base_url"], timeout=900),
|
||||
node,
|
||||
str(job["output_dir"]),
|
||||
"trained_model",
|
||||
str(trained_model["id"]),
|
||||
str(job.get("id") or record["id"]),
|
||||
f"trained_models/{trained_model['id']}",
|
||||
)
|
||||
artifacts = store.model_artifacts(str(trained_model["id"]))
|
||||
if archived and artifacts:
|
||||
store.link_model_artifact_storage_object(
|
||||
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||
|
||||
@@ -175,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
except Exception:
|
||||
pass
|
||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
||||
await _archive_node_directory(
|
||||
store,
|
||||
client,
|
||||
node,
|
||||
str(job["output_dir"]),
|
||||
"eval",
|
||||
str(eval_task["id"]),
|
||||
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
|
||||
f"evaluations/{eval_task['id']}",
|
||||
)
|
||||
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
||||
eval_synced += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
Reference in New Issue
Block a user