feat: 权限与日志治理完善,MinIO 独立部署与 tiktoken 离线打包适配

- 后端:强化平台/审批/资源/系统接口权限校验与操作日志,更新权限设计文档与测试用例
- 存储:新增 MinIO 独立部署适配(端口 19000/19001),外部端点与 host-gateway 互通
- 离线:打包 tiktoken cl100k_base 词表进镜像,避免无网环境联网下载
- 其他:算力节点接口微调,前端微调创建页小修,忽略 MinIO 运行时数据

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-12 15:21:23 +08:00
parent 2f64086177
commit 5ecca9f0bc
26 changed files with 101600 additions and 129 deletions

View File

@@ -2,6 +2,8 @@
from app.core.logging import get_logger
from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter()
logger = get_logger(__name__)
@@ -9,10 +11,30 @@ logger = get_logger(__name__)
@router.get("/health")
async def health_check() -> dict[str, object]:
logger.info("health check requested")
# Health endpoints are called frequently by Docker and the frontend.
# Keep failures visible without emitting one INFO line per probe.
logger.debug("health check requested")
storage_status: dict[str, object] = {"enabled": get_settings().minio_enabled, "status": "disabled"}
if get_settings().minio_enabled:
try:
get_object_storage().ensure_bucket()
storage_status = {
"enabled": True,
"status": "ready",
"endpoint": get_settings().minio_endpoint,
"bucket": get_settings().minio_bucket,
}
except (ObjectStorageError, OSError) as exc:
logger.warning("MinIO health check failed: %s", exc)
storage_status = {
"enabled": True,
"status": "unavailable",
"endpoint": get_settings().minio_endpoint,
"error": str(exc),
}
return {
"code": 0,
"message": "ok",
"data": get_platform_store().health_metrics(),
"data": {**get_platform_store().health_metrics(), "storage": storage_status},
}

View File

@@ -4,6 +4,7 @@ import json
import asyncio
import hashlib
import uuid
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
@@ -13,7 +14,7 @@ from fastapi.responses import PlainTextResponse, StreamingResponse
import httpx
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
@@ -21,6 +22,21 @@ from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_com
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
_DASHBOARD_CACHE_TTL = 5.0
_DASHBOARD_CACHE: dict[str, Any] = {}
def _cached_dashboard(key: str) -> dict[str, Any] | None:
item = _DASHBOARD_CACHE.get(key)
if not item or time.monotonic() - item["created_at"] >= _DASHBOARD_CACHE_TTL:
return None
return item["value"]
def _store_dashboard_cache(key: str, value: dict[str, Any]) -> dict[str, Any]:
_DASHBOARD_CACHE[key] = {"created_at": time.monotonic(), "value": value}
return value
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
@@ -77,6 +93,27 @@ def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) ->
return preferred + others
async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None:
"""Prepare MinIO resource files on a node and return the local directory."""
if not get_settings().minio_enabled or not resource_id:
return None
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
return None
client = ComputeNodeClient(node["api_base_url"], timeout=900)
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
for obj in objects:
await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": get_object_storage().presigned_get(obj["object_key"]),
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
})
return f"/data/yg-ft/{root_name}/{resource_id}"
def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
"""Convert frontend inference payload to compute API messages format.
@@ -396,13 +433,20 @@ async def _fine_tune_preflight_with_job_payload(
@router.post("/login")
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
ip = request.client.host if request and request.client else "unknown"
now = time.time()
recent = [stamp for stamp in _LOGIN_FAILURES.get(ip, []) if now - stamp < 300]
if len(recent) >= 5:
raise fail(429, "too many login attempts, retry later")
user = store.login(payload.get("username", ""), payload.get("password", ""))
if not user:
_LOGIN_FAILURES[ip] = [*recent, now]
raise fail(401, "invalid username or password")
sess = store.create_session(user["id"])
return ok({"token": f"platform-token-{user['id']}", "user": user, "session_id": sess["session_id"]})
_LOGIN_FAILURES.pop(ip, None)
sess = store.create_session(user["id"], ip=None)
return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]})
@router.post("/logout")
@@ -422,7 +466,7 @@ async def me(request: Request) -> dict[str, Any]:
token = auth.replace("Bearer ", "").strip()
# token 格式: platform-token-{user_id}
if token.startswith("platform-token-"):
user_id = token[len("platform-token-"):]
user_id = token[len("platform-token-"):].split(".", 1)[0]
for u in store.users():
if u.get("id") == user_id:
return ok(u)
@@ -431,9 +475,12 @@ async def me(request: Request) -> dict[str, Any]:
@router.get("/dashboard/overview")
async def dashboard_overview() -> dict[str, Any]:
cached = _cached_dashboard("overview")
if cached is not None:
return cached
store = get_platform_store()
tasks = store.tasks()
return ok(
return _store_dashboard_cache("overview", ok(
{
"models": len(store.models()),
"datasets": len(store.datasets()),
@@ -442,11 +489,14 @@ async def dashboard_overview() -> dict[str, Any]:
"compute_nodes": len(store.compute_nodes()),
"gpus": len(store.gpus()),
}
)
))
@router.get("/dashboard/stats")
async def dashboard_stats() -> dict[str, Any]:
cached = _cached_dashboard("stats")
if cached is not None:
return cached
"""看板聚合数据:基于平台真实数据;缺项做合理近似。"""
store = get_platform_store()
tasks = store.tasks()
@@ -603,7 +653,7 @@ async def dashboard_stats() -> dict[str, Any]:
except Exception:
pass
return ok(
return _store_dashboard_cache("stats", ok(
{
"online_services": sum(s["count"] for s in service_status),
"running_tasks": len(running_ft) + eval_running,
@@ -615,7 +665,7 @@ async def dashboard_stats() -> dict[str, Any]:
"login_duration_rank": login_duration_rank,
"recent_login_users": recent_login_users,
}
)
))
@router.get("/system-info")
@@ -731,7 +781,12 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
@router.delete("/model-manage/trained-models/{model_id}")
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("trained_model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this trained model")
pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}")
if pending:
return pending
get_platform_store().delete_trained_model(model_id)
return ok({"deleted": model_id, "type": type})
@@ -747,7 +802,9 @@ async def trained_model_lineage(model_id: str) -> dict[str, Any]:
@router.get("/model-manage/export-jobs")
async def model_export_jobs(trained_model_id: str | None = Query(default=None)) -> dict[str, Any]:
async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"):
raise fail(403, "no permission to access export jobs")
return ok(get_platform_store().model_export_jobs(trained_model_id))
@@ -878,7 +935,7 @@ async def delete_model(model_id: str, current_user: dict = Depends(get_current_u
@router.post("/model-manage/merge")
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
trained_model = next(
@@ -889,6 +946,12 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
),
None,
)
if not trained_model:
raise fail(404, "trained model not found")
if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"):
raise fail(403, "no permission to merge this trained model")
if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"):
raise fail(403, "no permission to use merge base model")
base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
adapter_path = (
payload.get("adapter_path")
@@ -906,6 +969,15 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
await _wait_for_object_storage()
except RuntimeError as exc:
raise fail(503, str(exc))
try:
prepared_base = await _prepare_resource_on_node(store, "model", str(payload.get("base_model_id") or base_model_path), node)
if prepared_base:
base_model_path = prepared_base
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", str(payload.get("adapter_model_id") or (trained_model and trained_model.get("id")) or ""), node)
if prepared_adapter:
adapter_path = prepared_adapter
except Exception as exc:
raise fail(502, f"merge resource preparation failed: {exc}")
health = node.get("health_detail") or {}
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
@@ -1075,6 +1147,7 @@ async def _sync_training_dataset_to_compute_node(
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"datasets/{dataset_id}/{target_name}",
})
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
@@ -1258,7 +1331,15 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/fine-tune")
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload.setdefault("created_by", current_user.get("id"))
if not is_admin(current_user):
model_id = str(payload.get("base_model") or payload.get("base_model_id") or "")
dataset_id = str(payload.get("train_dataset_id") or "")
if model_id and not has_resource_access("model", model_id, current_user, "execute"):
raise fail(403, "no permission to use this base model")
if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
raise fail(403, "no permission to use this dataset")
try:
task = get_platform_store().create_task(payload)
return ok({"id": task["id"]})
@@ -1275,11 +1356,16 @@ async def start_fine_tune(
# GPU 权限校验:普通用户只能使用被分配的 GPU
if not is_admin(current_user):
node_id = payload.get("compute_node_id") or payload.get("node_id")
gpu_indices = payload.get("gpus") or []
gpu_indices = payload.get("gpu_indices")
if gpu_indices is None:
gpu_indices = payload.get("gpus") or []
if node_id and gpu_indices:
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
raise fail(403, "无权使用所选 GPU请联系管理员分配")
# 记录创建者
if node_id and not gpu_indices:
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
payload["strict_node_selection"] = bool(node_id)
payload.setdefault("created_by", current_user.get("id"))
try:
return ok(await _submit_fine_tune_task(store, payload))
@@ -1411,7 +1497,9 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
@router.put("/fine-tune/{task_id}")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "write"):
raise fail(403, "no permission to update this task")
try:
return ok(get_platform_store().update_task(task_id, payload))
except KeyError:
@@ -1442,13 +1530,15 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
@router.post("/fine-tune/{task_id}/retry")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}
try:
task = store.task(task_id)
except KeyError:
raise fail(404, "fine tune task not found")
if not has_resource_access("fine-tune", task_id, current_user, "execute"):
raise fail(403, "no permission to retry this task")
if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
raise fail(409, "only failed or stopped tasks can be retried without force=true")
retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
@@ -1544,7 +1634,7 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
@router.post("/model-eval/start")
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Start an evaluation task: submit eval job to compute node."""
store = get_platform_store()
# 1. Create eval task record
@@ -1745,12 +1835,23 @@ async def dimension_delete(dimension_id: str) -> dict[str, Any]:
@router.get("/model-compare")
async def model_compare_list() -> dict[str, Any]:
return ok(get_platform_store().compare_tasks())
async def model_compare_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
tasks = get_platform_store().compare_tasks()
if is_admin(current_user):
return ok(tasks)
accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user)
return ok([item for item in tasks if item["id"] in accessible])
@router.post("/model-compare")
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload.setdefault("created_by", current_user.get("id"))
model_ids = payload.get("model_ids") or payload.get("models") or []
if not is_admin(current_user):
for model_id in model_ids:
if isinstance(model_id, dict): model_id = model_id.get("id") or model_id.get("model_id")
if model_id and not has_resource_access("model", str(model_id), current_user, "execute"):
raise fail(403, "no permission to use inference model")
task = get_platform_store().create_compare_task(payload)
return ok({"id": task["id"]})
@@ -1766,9 +1867,12 @@ async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict
@router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str) -> dict[str, Any]:
async def model_compare_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().compare_task(task_id))
task = get_platform_store().compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "read"):
raise fail(403, "no permission to access inference task")
return ok(task)
except KeyError:
raise fail(404, "compare task not found")
@@ -1805,12 +1909,17 @@ async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = No
@router.delete("/model-compare/{task_id}")
async def model_compare_delete(task_id: str) -> dict[str, Any]:
async def model_compare_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 先删记录(快),再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
try:
task = get_platform_store().compare_task(task_id)
except KeyError:
raise fail(404, "compare task not found")
if not has_resource_access("compare", task_id, current_user, "delete"):
raise fail(403, "no permission to delete inference task")
pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}")
if pending:
return pending
get_platform_store().delete_compare_task(task_id)
try:
await _unload_from_compute_node(get_platform_store(), task=task)
@@ -1873,7 +1982,7 @@ def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[
@router.post("/model-compare/{task_id}/load")
async def model_compare_load(task_id: str) -> dict[str, Any]:
async def model_compare_load(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""异步派发模型加载到算力节点,立即返回。
加载进度由轮询对账器compute_poller → reconcile_inference_loads推进
@@ -1883,6 +1992,8 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
try:
store = get_platform_store()
task = store.compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "execute"):
raise fail(403, "no permission to load inference task")
models = task.get("models") or []
if isinstance(models, str):
try:
@@ -1953,10 +2064,12 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
@router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str) -> dict[str, Any]:
async def model_compare_unload(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
store = get_platform_store()
task = store.compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "write"):
raise fail(403, "no permission to unload inference task")
# 任务感知卸载:只释放该任务实际加载到的节点,短超时快速返回
unload_result = await _unload_from_compute_node(store, task=task)
updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
@@ -2048,7 +2161,19 @@ async def model_chat_local_unload() -> dict[str, Any]:
if not store.is_inference_loaded(n["id"]):
continue
try:
result = await ComputeNodeClient(n["api_base_url"]).inference_unload()
client = ComputeNodeClient(n["api_base_url"])
last_error = ""
result = None
for attempt in range(3):
try:
result = await client.inference_unload()
break
except Exception as exc: # noqa: BLE001 - retry node cleanup
last_error = str(exc)
if attempt < 2:
await asyncio.sleep(2 ** attempt)
if result is None:
raise RuntimeError(last_error or "inference unload failed")
results.append({"node_id": n["id"], "success": True, "result": result})
except Exception as exc: # noqa: BLE001 - best-effort unload
results.append({"node_id": n["id"], "success": False, "error": str(exc)})
@@ -2073,19 +2198,31 @@ async def model_chat_local_status() -> dict[str, Any]:
@router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "")
if resource_id and not has_resource_access("trained_model", resource_id, current_user, "execute") and not has_resource_access("model", resource_id, current_user, "execute"):
raise fail(403, "no permission to load this model")
"""Load a trained model (base + adapter) on the compute node for inference."""
model_path = (payload.get("model_name_or_path") or "").strip()
if not model_path:
return ok({"loaded": False, "error": "model_name_or_path is required"})
store = get_platform_store()
node = _select_first_online_node(store)
requested_node_id = str(payload.get("compute_node_id") or payload.get("node_id") or "")
node = next((item for item in store.compute_nodes() if item.get("id") == requested_node_id and item.get("enabled") and item.get("scheduler_status") == "online"), None)
if not node:
node = _select_first_online_node(store)
if not node:
return ok({"loaded": False, "error": "no online compute node"})
try:
prepared_path = await _prepare_resource_on_node(store, "trained_model", str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or ""), node)
if prepared_path:
payload = {**payload, "model_name_or_path": prepared_path}
prepared_path = await _prepare_resource_on_node(store, "model", str(payload.get("model_id") or payload.get("resource_id") or ""), node)
if prepared_path:
payload = {**payload, "model_name_or_path": prepared_path}
client = ComputeNodeClient(node["api_base_url"])
# 计算节点现在异步加载HTTP 接受loading/ready即视为派发成功
result = await client.inference_load(payload)
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"])
return ok(result)
@@ -2094,8 +2231,13 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dic
@router.get("/compute/nodes")
async def compute_nodes() -> dict[str, Any]:
return ok(get_platform_store().compute_nodes())
async def compute_nodes(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
nodes = get_platform_store().compute_nodes()
if is_admin(current_user):
return ok(nodes)
# 普通用户只看到自己被分配 GPU 的节点,避免泄露节点拓扑和未授权资源。
assigned = {item["node_id"] for item in get_platform_store().gpu_assignments_for_user(current_user["id"])}
return ok([node for node in nodes if node["id"] in assigned])
@router.post("/storage/objects/presign")
@@ -2154,27 +2296,90 @@ async def prepare_storage_resource(
for obj in objects:
url = get_object_storage().presigned_get(obj["object_key"])
filename = Path(str(obj.get("file_name") or obj["object_key"])).name
result = await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
})
cache_job = store.create_storage_cache_job({"storage_object_id": obj["id"], "node_id": node_id, "direction": "download"})
try:
result = await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
})
store.update_storage_cache_job(cache_job["id"], {"status": "completed", "progress": 100, "local_path": result.get("local_path"), "completed_at": utcnow()})
except Exception as exc:
store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()})
raise
prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})
@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str) -> dict[str, Any]:
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
@router.get("/storage/cache/jobs/{node_id}")
async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().storage_cache_jobs_for_node(node_id, limit))
@router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}")
async def archive_node_files(
resource_type: str,
resource_id: str,
node_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""Archive completed node files to MinIO without proxying file bytes through Backend."""
if not has_resource_access(resource_type, resource_id, current_user, "execute"):
raise fail(403, "no permission to archive this resource")
store = get_platform_store()
if not is_admin(current_user):
model_id = str(payload.get("model_id") or "")
dataset_id = str(payload.get("dataset_id") or "")
if not model_id or not has_resource_access("model", model_id, current_user, "execute"):
raise fail(403, "no permission to evaluate this model")
if not dataset_id or not has_resource_access("dataset", dataset_id, current_user, "execute"):
raise fail(403, "no permission to evaluate this dataset")
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
files = payload.get("files") or []
if not isinstance(files, list) or not files:
raise fail(400, "files is required")
client = ComputeNodeClient(node["api_base_url"], timeout=900)
archived = []
for item in files:
path = str(item.get("path") or "")
name = Path(str(item.get("file_name") or Path(path).name)).name
version_id = str(item.get("version_id") or uuid.uuid4().hex)
object_key = str(item.get("object_key") or f"{resource_type}s/{resource_id}/versions/{version_id}/{name}")
url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, url, object_key, str(item.get("content_type") or "application/octet-stream"))
metadata = get_object_storage().stat(object_key)
record = store.create_storage_object({
"resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
"bucket": get_object_storage().bucket, "object_key": object_key, "file_name": name,
"content_type": item.get("content_type"), "checksum_sha256": result.get("checksum_sha256"),
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, "status": "available",
"created_by": current_user.get("id"),
})
archived.append({"object": record, "node_id": node_id})
return ok({"status": "available", "items": archived})
@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
if not is_admin(current_user) and not any(item["node_id"] == node_id for item in store.gpu_assignments_for_user(current_user["id"])):
raise fail(403, "no permission to access this compute node")
return ok(node)
@router.post("/compute/nodes")
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try:
return ok(get_platform_store().create_compute_node(payload))
except KeyError as exc:
@@ -2184,7 +2389,9 @@ async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str,
@router.put("/compute/nodes/{node_id}")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try:
return ok(get_platform_store().update_compute_node(node_id, payload))
except KeyError:
@@ -2194,7 +2401,9 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
@router.delete("/compute/nodes/{node_id}")
async def delete_compute_node(node_id: str) -> dict[str, Any]:
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try:
return ok(get_platform_store().delete_compute_node(node_id))
except KeyError:
@@ -2204,7 +2413,9 @@ async def delete_compute_node(node_id: str) -> dict[str, Any]:
@router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str) -> dict[str, Any]:
async def test_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
@@ -2238,22 +2449,28 @@ async def test_compute_node(node_id: str) -> dict[str, Any]:
@router.post("/compute/nodes/{node_id}/health-check")
async def health_check_compute_node(node_id: str) -> dict[str, Any]:
return await test_compute_node(node_id)
async def health_check_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return await test_compute_node(node_id, current_user)
@router.post("/compute/nodes/{node_id}/enable")
async def enable_compute_node(node_id: str) -> dict[str, Any]:
async def enable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))
@router.post("/compute/nodes/{node_id}/disable")
async def disable_compute_node(node_id: str) -> dict[str, Any]:
async def disable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))
@router.post("/compute/nodes/{node_id}/drain")
async def drain_compute_node(node_id: str) -> dict[str, Any]:
async def drain_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))
@@ -2460,8 +2677,13 @@ async def compute_node_engines(node_id: str) -> dict[str, Any]:
@router.get("/compute/gpus")
async def compute_gpus() -> dict[str, Any]:
return ok(get_platform_store().gpus())
async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
gpus = store.gpus()
if is_admin(current_user):
return ok(gpus)
assigned = {(item["node_id"], int(item["gpu_index"])) for item in store.gpu_assignments_for_user(current_user["id"])}
return ok([gpu for gpu in gpus if (gpu.get("node_id"), int(gpu.get("id", gpu.get("gpu_index", -1)))) in assigned])
@router.get("/compute/queue")
@@ -2485,8 +2707,10 @@ async def compute_job_detail(job_id: str) -> dict[str, Any]:
@router.post("/compute/jobs/{job_id}/stop")
async def compute_job_stop(job_id: str) -> dict[str, Any]:
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
task = _task_for_compute_job(job_id)
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
raise fail(403, "no permission to stop this compute job")
if not task:
node = _node_for_compute_job_record(job_id)
if not node:
@@ -2521,13 +2745,13 @@ async def compute_job_logs(
@router.post("/compute/jobs/{job_id}/retry")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}
task = _task_for_compute_job(job_id)
if not task:
raise fail(404, "compute job not found")
return await retry_fine_tune(task["id"], payload)
return await retry_fine_tune(task["id"], payload, current_user)
@router.post("/compute/jobs/{job_id}/priority")

View File

@@ -9,6 +9,13 @@ from app.db.platform_store import get_platform_store
# 无需鉴权的路径前缀(健康检查、登录等)
PUBLIC_PATHS = ("/health", "/login", "/system-info")
OWNER_TABLES = {
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
"fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"),
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
"project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"),
}
def _extract_token(request: Request) -> str | None:
@@ -19,6 +26,9 @@ def _extract_token(request: Request) -> str | None:
return token[len("platform-token-"):]
return None
def _session_token(user_id: str, session_id: str) -> str:
return f"platform-token-{user_id}.{session_id}"
def get_current_user(request: Request) -> dict[str, Any]:
"""
@@ -33,14 +43,30 @@ def get_current_user(request: Request) -> dict[str, Any]:
if path.endswith(prefix):
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
user_id = _extract_token(request)
if not user_id:
token_value = _extract_token(request)
if not token_value:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
store = get_platform_store()
for u in store.users():
if u.get("id") == user_id:
return u
user_id, _, session_id = token_value.partition(".")
if session_id:
with store.connect() as conn:
session = conn.execute(
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
).fetchone()
if not session or session["user_id"] != user_id or session["logout_at"]:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
if session["expires_at"]:
from datetime import datetime, timezone
try:
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
except ValueError:
pass
with store.connect() as conn:
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
if user_row:
return store._user(user_row)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
@@ -75,6 +101,23 @@ def has_resource_access(
user_id = user.get("id")
user_role = user.get("role")
owner_tables = OWNER_TABLES
table_info = owner_tables.get(resource_type)
if table_info and user_id:
table, column = table_info
with store.connect() as conn:
row = conn.execute(f"SELECT {column} FROM {table} WHERE id=?", (resource_id,)).fetchone()
if row:
owner = row[column]
if column == "payload":
try:
import json
owner = json.loads(owner or "{}").get("created_by")
except (TypeError, ValueError):
owner = None
if owner == user_id:
return True
for entry in acls:
# 按 user 授权
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
@@ -135,4 +178,40 @@ def filter_accessible_resource_ids(
).fetchall()
accessible = {r["resource_id"] for r in rows}
if resource_type in owner_tables:
table, column = owner_tables[resource_type]
with store.connect() as conn:
owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall()
accessible.update(row["id"] for row in owned)
return [rid for rid in all_ids if rid in accessible]
def filter_accessible_resource_ids_batch(
resource_type: str,
resource_ids: list[str],
user: dict[str, Any],
) -> set[str]:
"""Filter a list endpoint with one ACL query instead of one query per row."""
if is_admin(user):
return set(resource_ids)
if not resource_ids:
return set()
store = get_platform_store()
placeholders = ",".join("?" for _ in resource_ids)
with store.connect() as conn:
rows = conn.execute(
f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
"AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))",
(resource_type, *resource_ids, user.get("id"), user.get("role")),
).fetchall()
accessible = {row["resource_id"] for row in rows}
table_info = OWNER_TABLES.get(resource_type)
if table_info and user.get("id"):
table, column = table_info
with store.connect() as conn:
owned = conn.execute(
f"SELECT id FROM {table} WHERE id IN ({placeholders}) AND {column}=?",
(*resource_ids, user["id"]),
).fetchall()
accessible.update(row["id"] for row in owned)
return accessible

View File

@@ -61,7 +61,9 @@ class Settings:
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5)
minio_enabled: bool = _bool_env("MINIO_ENABLED", False)
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
# MinIO is an independent service and may run on another host. The
# endpoint must therefore be reachable from the Backend container.
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://host.docker.internal:19000")
minio_access_key: str = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")

View File

@@ -210,6 +210,10 @@ def configure_logging(settings: Settings | None = None) -> None:
logger.propagate = True
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
def get_logger(name: str) -> logging.Logger:
return logging.getLogger(name)
@@ -229,7 +233,15 @@ def setup_request_logging(app: FastAPI) -> None:
try:
response = await call_next(request)
elapsed_ms = (time.perf_counter() - started_at) * 1000
logger.info(
# Docker/frontend probes and polling endpoints are intentionally
# quiet at INFO; failures remain visible at WARNING/ERROR.
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
log_method = logger.debug
if response.status_code >= 400:
log_method = logger.warning
log_method(
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
request.method,
request.url.path,

View File

@@ -526,8 +526,15 @@ class PlatformStore:
"artifact_dir": "TEXT",
"compute_node_id": "TEXT",
"compute_node_name": "TEXT",
"created_by": "TEXT",
"tenant_id": "TEXT",
"project_id": "TEXT",
"deleted_at": "TEXT",
"deleted_by": "TEXT",
},
)
for table in ("models", "datasets", "eval_tasks"):
self._ensure_columns(conn, table, {"deleted_at": "TEXT", "deleted_by": "TEXT", "tenant_id": "TEXT", "project_id": "TEXT"})
self._ensure_columns(
conn,
"resource_replicas",
@@ -1470,12 +1477,12 @@ class PlatformStore:
def delete_model(self, model_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM models WHERE id=?", (model_id,))
conn.execute("UPDATE models SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", model_id))
def trained_models(self) -> list[dict[str, Any]]:
self.refresh_runtime_state()
with self.connect() as conn:
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
rows = conn.execute("SELECT * FROM trained_models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall()
items = []
for row in rows:
item = {
@@ -1498,7 +1505,7 @@ class PlatformStore:
def delete_trained_model(self, model_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id))
conn.execute("UPDATE trained_models SET deleted_at=?, deleted_by=? WHERE id=? OR name=?", (utcnow(), "system", model_id, model_id))
def datasets(self) -> list[dict[str, Any]]:
with self.connect() as conn:
@@ -1507,6 +1514,7 @@ class PlatformStore:
FROM datasets dataset
LEFT JOIN data_process_tasks task
ON task.id=COALESCE(dataset.source_task_id, dataset.task_id)
WHERE dataset.deleted_at IS NULL
ORDER BY dataset.create_time DESC"""
).fetchall()
return [self._dataset(conn, row) for row in rows]
@@ -1646,8 +1654,7 @@ class PlatformStore:
def delete_dataset(self, dataset_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,))
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
conn.execute("UPDATE datasets SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", dataset_id))
def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
now = utcnow()
@@ -2004,13 +2011,14 @@ class PlatformStore:
task_id = str(payload.get("task_id") or payload.get("id"))
current = self.task(task_id)
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0]
selected_gpus = payload.get("gpus") or merged.get("gpus") or []
process_id = int(43000 + (time.time() % 10000))
with self.connect() as conn:
owner = f"start:{task_id}:{uuid.uuid4().hex[:8]}"
if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner):
raise RuntimeError("compute scheduler is busy, please retry")
node = self._schedule_node_locked(conn, payload)
selected_gpus = list(node.get("selected_gpus") or selected_gpus)
sync_job_id = new_id("sync")
conn.execute(
"""
@@ -2094,7 +2102,7 @@ class PlatformStore:
task = self.task(task_id)
merged = {**task, **(payload or {}), "id": task_id}
node = self.select_compute_node(merged)
selected_gpus = merged.get("gpus") or [0]
selected_gpus = merged.get("gpus") or []
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -2108,7 +2116,7 @@ class PlatformStore:
"progress": int(payload.get("progress") or 0),
}
node = self.select_compute_node(transient_task)
selected_gpus = transient_task.get("gpus") or [0]
selected_gpus = transient_task.get("gpus") or []
return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus)
def _compute_job_payload_from_task_node(
@@ -2260,7 +2268,7 @@ class PlatformStore:
for item in runtime_files
],
"output_dir": output_dir,
"gpus": selected_gpus or task.get("gpus") or [0],
"gpus": selected_gpus if selected_gpus is not None else (task.get("gpus") or []),
"compute_node_id": node["id"],
"compute_node_code": node["code"],
}
@@ -2270,7 +2278,7 @@ class PlatformStore:
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
if not node:
raise RuntimeError("compute node not found")
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0])
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [])
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
status_map = {
@@ -2493,7 +2501,7 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2712,27 +2720,71 @@ class PlatformStore:
def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
@staticmethod
def _payload_gpu_indexes(payload: dict[str, Any]) -> list[int]:
raw = payload.get("gpu_indices")
if raw is None:
raw = payload.get("gpus")
if raw is None:
return []
try:
values = [int(item) for item in raw]
except (TypeError, ValueError) as exc:
raise RuntimeError("invalid GPU index") from exc
if any(item < 0 for item in values):
raise RuntimeError("GPU index must be non-negative")
return sorted(set(values))
def _select_node_gpus(
self,
conn: PgConnection,
node: dict[str, Any],
requested: list[int],
payload: dict[str, Any],
) -> list[int]:
available = self._node_gpu_indexes(conn, node)
active = self._active_gpu_indexes(conn, node["id"])
allowed = payload.get("allowed_gpu_indices")
if allowed is not None:
available &= {int(item) for item in allowed}
if requested:
selected = set(requested)
if not selected.issubset(available):
raise RuntimeError(f"requested GPU is not available on compute node {node['code']}")
if selected.intersection(active):
raise RuntimeError(f"requested GPU is busy on compute node {node['code']}")
return sorted(selected)
if payload.get("allow_cpu") or payload.get("device") == "cpu":
return []
count = max(1, int(payload.get("gpu_count") or 1))
free = sorted(available - active)
if len(free) < count:
raise RuntimeError(f"compute node {node['code']} has only {len(free)} available GPU(s)")
return free[:count]
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []]
requested_gpus = self._payload_gpu_indexes(payload)
nodes = self._compute_nodes_locked(conn)
candidates = [
n
for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested:
selected = next((n for n in candidates if n["id"] == requested), None)
if selected:
return selected
if not selected:
raise RuntimeError("selected compute node is unavailable")
selected["selected_gpus"] = self._select_node_gpus(conn, selected, requested_gpus, payload)
return selected
filtered = []
for node in candidates:
try:
node["selected_gpus"] = self._select_node_gpus(conn, node, requested_gpus, payload)
filtered.append(node)
except RuntimeError:
continue
candidates = filtered
if not candidates:
if not nodes:
raise RuntimeError("no available compute node: no compute node configured")
@@ -2744,6 +2796,8 @@ class PlatformStore:
reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
elif requested:
reason = "selected node unavailable"
else:
reason = "not selected"
reasons.append(f"{node['code']}({reason})")
@@ -2930,6 +2984,43 @@ class PlatformStore:
raise KeyError(object_id)
return dict(row)
def create_storage_cache_job(self, payload: dict[str, Any]) -> dict[str, Any]:
job_id = str(payload.get("id") or new_id("cache"))
with self.connect() as conn:
conn.execute(
"""
INSERT INTO storage_cache_jobs
(id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(job_id, payload["storage_object_id"], payload["node_id"], payload.get("direction", "download"),
payload.get("status", "running"), int(payload.get("progress", 0)), payload.get("local_path"),
payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at")),
)
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
return dict(row)
def update_storage_cache_job(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "progress", "local_path", "error", "completed_at"}
fields = {key: value for key, value in payload.items() if key in allowed}
if fields:
assignments = ", ".join(f"{key}=?" for key in fields)
with self.connect() as conn:
conn.execute(f"UPDATE storage_cache_jobs SET {assignments} WHERE id=?", (*fields.values(), job_id))
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
if not row:
raise KeyError(job_id)
return dict(row)
def storage_cache_jobs_for_node(self, node_id: str, limit: int = 100) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM storage_cache_jobs WHERE node_id=? ORDER BY create_time DESC LIMIT ?",
(node_id, max(1, min(limit, 500))),
).fetchall()
return [dict(row) for row in rows]
def update_resource_replica_sync_result(
self,
replica_id: str,
@@ -3613,9 +3704,9 @@ class PlatformStore:
login_at = utcnow()
with self.connect() as conn:
conn.execute(
"INSERT INTO sessions (id, user_id, username, login_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s)",
(sid, user_id, user_id, login_at, login_at),
"INSERT INTO sessions (id, user_id, username, login_at, issued_at, expires_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s, %s, %s)",
(sid, user_id, user_id, login_at, login_at, datetime.fromtimestamp(time.time() + 1800, timezone.utc).isoformat(), login_at),
)
return {"session_id": sid, "user_id": user_id, "login_at": login_at}
@@ -4263,6 +4354,14 @@ class PlatformStore:
assigned = {r["gpu_index"] for r in rows}
return all(idx in assigned for idx in gpu_indices)
def assigned_gpu_indexes(self, user_id: str, node_id: str) -> list[int]:
with self.connect() as conn:
rows = conn.execute(
"SELECT gpu_index FROM gpu_assignments WHERE user_id=? AND node_id=? ORDER BY gpu_index",
(user_id, node_id),
).fetchall()
return [int(row["gpu_index"]) for row in rows]
# ===================== 平台治理:资源可见性过滤 =====================
def _filter_accessible_ids(

View File

@@ -64,7 +64,12 @@ CREATE TABLE IF NOT EXISTS models (
api_key TEXT,
online_model_name TEXT,
can_train INTEGER NOT NULL DEFAULT 0,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS trained_models (
@@ -78,7 +83,12 @@ CREATE TABLE IF NOT EXISTS trained_models (
merged_path TEXT,
artifact_dir TEXT,
compute_node_id TEXT,
compute_node_name TEXT
compute_node_name TEXT,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS model_lineage (
@@ -130,7 +140,12 @@ CREATE TABLE IF NOT EXISTS datasets (
size TEXT,
count INTEGER NOT NULL DEFAULT 0,
description TEXT,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE TABLE IF NOT EXISTS dataset_files (
@@ -413,6 +428,18 @@ CREATE TABLE IF NOT EXISTS acls (
permission TEXT NOT NULL,
create_time TEXT
);
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE UNIQUE INDEX IF NOT EXISTS uq_acls_subject_permission
ON acls(resource_type, resource_id, principal_type, principal_id, permission);
CREATE INDEX IF NOT EXISTS idx_acls_resource ON acls(resource_type, resource_id);
CREATE INDEX IF NOT EXISTS idx_acls_principal ON acls(principal_type, principal_id);
-- ---- 权限扩展来源004_permissions.sql ----
@@ -475,6 +502,9 @@ CREATE TABLE IF NOT EXISTS approval_steps (
comment TEXT,
time TEXT
);
CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instances(applicant_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
CREATE TABLE IF NOT EXISTS audit_logs (
id TEXT PRIMARY KEY,

View File

@@ -1,21 +1,23 @@
from __future__ import annotations
from fastapi import APIRouter, Body
from fastapi import APIRouter, Body, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, is_admin
router = APIRouter(prefix="/approvals", tags=["approval"])
@router.get("/templates")
def list_templates() -> dict[str, Any]:
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().approval_templates())
@router.post("/templates")
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
def create_template(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
if not payload.get("name"):
raise fail(400, "name 必填")
return ok(get_platform_store().create_approval_template(payload))
@@ -30,7 +32,8 @@ def get_template(template_id: str) -> dict[str, Any]:
@router.put("/templates/{template_id}")
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try:
return ok(get_platform_store().update_approval_template(template_id, payload))
except KeyError:
@@ -38,7 +41,8 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> di
@router.delete("/templates/{template_id}")
def delete_template(template_id: str) -> dict[str, Any]:
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try:
return ok(get_platform_store().delete_approval_template(template_id))
except KeyError:
@@ -46,13 +50,15 @@ def delete_template(template_id: str) -> dict[str, Any]:
@router.get("")
def list_instances(status: str | None = None) -> dict[str, Any]:
return ok(get_platform_store().approval_instances(status=status))
def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
items = get_platform_store().approval_instances(status=status)
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
@router.post("")
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
for field in ("resource_type", "resource_id", "applicant_id"):
def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload["applicant_id"] = current_user.get("id")
for field in ("resource_type", "resource_id"):
if not payload.get(field):
raise fail(400, f"{field} 必填")
try:
@@ -62,9 +68,12 @@ def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/{instance_id}")
def get_instance(instance_id: str) -> dict[str, Any]:
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().approval_instance(instance_id))
item = get_platform_store().approval_instance(instance_id)
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
raise fail(403, "no permission to access approval")
return ok(item)
except KeyError:
raise fail(404, "instance not found")

View File

@@ -266,3 +266,8 @@ class ComputeNodeClient:
)
response.raise_for_status()
return _unwrap_dict(response.json())
async def upload_file_to_url(self, path: str, upload_url: str, object_key: str = "", content_type: str = "application/octet-stream") -> dict[str, Any]:
return await self._request("POST", "/compute/files/upload-to-url", json_data={
"path": path, "upload_url": upload_url, "object_key": object_key, "content_type": content_type,
}, timeout=900)

View File

@@ -1,10 +1,11 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from fastapi import APIRouter, Body, Request, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, has_resource_access, is_admin
router = APIRouter(prefix="/resources", tags=["resource"])
@@ -16,8 +17,10 @@ def _actor(request: Request) -> str | None:
@router.get("/{resource_type}/{resource_id}/acl")
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]:
def get_acl(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""查询资源 ACL返回按主体分组的权限列表。"""
if not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to access resource ACL")
return ok(get_platform_store().resource_acl(resource_type, resource_id))
@@ -27,9 +30,18 @@ def set_acl(
resource_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""设置资源 ACLbody: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "only resource owner or admin can update ACL")
entries = payload.get("entries") or []
allowed = {"read", "write", "execute", "download", "delete", "admin"}
for entry in entries:
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
raise fail(400, "invalid ACL principal")
if any(permission not in allowed for permission in entry.get("permissions") or []):
raise fail(400, "invalid ACL permission")
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
get_platform_store().record_audit(
action="resource.acl.set",

View File

@@ -1,9 +1,10 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Query, Request
from fastapi import APIRouter, Body, Query, Request, Depends
from fastapi.responses import StreamingResponse
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
from app.core.auth import get_current_user, is_admin
router = APIRouter(prefix="/system", tags=["system"])
@@ -32,13 +33,13 @@ def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
@router.get("/permissions/codes")
def permission_codes() -> dict:
def permission_codes(current_user: dict = Depends(get_current_user)) -> dict:
"""返回平台权限码清单(权限码接口)。"""
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
@router.get("/permissions")
def permissions_overview() -> dict:
def permissions_overview(current_user: dict = Depends(get_current_user)) -> dict:
"""返回权限码清单与角色定义。"""
store = get_platform_store()
return {
@@ -59,8 +60,12 @@ def audit_logs(
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store()
result = store.audit_logs(
tenant_id=tenant_id,
@@ -85,8 +90,12 @@ def audit_logs_export(
target_type: str | None = Query(default=None, description="目标类型"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
) -> StreamingResponse:
"""审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store()
result = store.audit_logs(
tenant_id=tenant_id,

View File

@@ -21,8 +21,10 @@ async def run_compute_poller() -> None:
while True:
try:
result = await poll_compute_jobs_once()
if result["synced"] or result["failed"]:
logger.info("compute jobs polled", extra={"result": result})
if result["failed"]:
logger.warning("compute polling reported failures", extra={"result": result})
elif result["synced"]:
logger.debug("compute jobs synchronized", extra={"result": result})
except asyncio.CancelledError:
logger.info("compute poller stopped")
raise