feat: 权限与日志治理完善,MinIO 独立部署与 tiktoken 离线打包适配

- 后端:强化平台/审批/资源/系统接口权限校验与操作日志,更新权限设计文档与测试用例
- 存储:新增 MinIO 独立部署适配(端口 19000/19001),外部端点与 host-gateway 互通
- 离线:打包 tiktoken cl100k_base 词表进镜像,避免无网环境联网下载
- 其他:算力节点接口微调,前端微调创建页小修,忽略 MinIO 运行时数据

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-12 15:21:23 +08:00
parent 2f64086177
commit 5ecca9f0bc
26 changed files with 101600 additions and 129 deletions

4
.gitignore vendored
View File

@@ -211,3 +211,7 @@ docker/compute/data/yg-ft/logs/**
# Offline deployment bundle - 离线部署包(镜像、运行时等大文件,不提交) # Offline deployment bundle - 离线部署包(镜像、运行时等大文件,不提交)
docker/offline/ docker/offline/
# MinIO object storage data - 对象存储运行时数据,勿提交,保留目录结构
docker/minio/data/*
!docker/minio/data/.gitkeep

View File

@@ -2,6 +2,8 @@
from app.core.logging import get_logger from app.core.logging import get_logger
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter() router = APIRouter()
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -9,10 +11,30 @@ logger = get_logger(__name__)
@router.get("/health") @router.get("/health")
async def health_check() -> dict[str, object]: async def health_check() -> dict[str, object]:
logger.info("health check requested") # Health endpoints are called frequently by Docker and the frontend.
# Keep failures visible without emitting one INFO line per probe.
logger.debug("health check requested")
storage_status: dict[str, object] = {"enabled": get_settings().minio_enabled, "status": "disabled"}
if get_settings().minio_enabled:
try:
get_object_storage().ensure_bucket()
storage_status = {
"enabled": True,
"status": "ready",
"endpoint": get_settings().minio_endpoint,
"bucket": get_settings().minio_bucket,
}
except (ObjectStorageError, OSError) as exc:
logger.warning("MinIO health check failed: %s", exc)
storage_status = {
"enabled": True,
"status": "unavailable",
"endpoint": get_settings().minio_endpoint,
"error": str(exc),
}
return { return {
"code": 0, "code": 0,
"message": "ok", "message": "ok",
"data": get_platform_store().health_metrics(), "data": {**get_platform_store().health_metrics(), "storage": storage_status},
} }

View File

@@ -4,6 +4,7 @@ import json
import asyncio import asyncio
import hashlib import hashlib
import uuid import uuid
import time
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -13,7 +14,7 @@ from fastapi.responses import PlainTextResponse, StreamingResponse
import httpx import httpx
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin
from app.core.config import get_settings from app.core.config import get_settings
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient from app.modules.compute_gateway.client import ComputeNodeClient
@@ -21,6 +22,21 @@ from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_com
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
router = APIRouter() router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
_DASHBOARD_CACHE_TTL = 5.0
_DASHBOARD_CACHE: dict[str, Any] = {}
def _cached_dashboard(key: str) -> dict[str, Any] | None:
item = _DASHBOARD_CACHE.get(key)
if not item or time.monotonic() - item["created_at"] >= _DASHBOARD_CACHE_TTL:
return None
return item["value"]
def _store_dashboard_cache(key: str, value: dict[str, Any]) -> dict[str, Any]:
_DASHBOARD_CACHE[key] = {"created_at": time.monotonic(), "value": value}
return value
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]: def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
@@ -77,6 +93,27 @@ def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) ->
return preferred + others return preferred + others
async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None:
"""Prepare MinIO resource files on a node and return the local directory."""
if not get_settings().minio_enabled or not resource_id:
return None
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
return None
client = ComputeNodeClient(node["api_base_url"], timeout=900)
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
for obj in objects:
await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": get_object_storage().presigned_get(obj["object_key"]),
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
})
return f"/data/yg-ft/{root_name}/{resource_id}"
def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]: def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
"""Convert frontend inference payload to compute API messages format. """Convert frontend inference payload to compute API messages format.
@@ -396,13 +433,20 @@ async def _fine_tune_preflight_with_job_payload(
@router.post("/login") @router.post("/login")
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
ip = request.client.host if request and request.client else "unknown"
now = time.time()
recent = [stamp for stamp in _LOGIN_FAILURES.get(ip, []) if now - stamp < 300]
if len(recent) >= 5:
raise fail(429, "too many login attempts, retry later")
user = store.login(payload.get("username", ""), payload.get("password", "")) user = store.login(payload.get("username", ""), payload.get("password", ""))
if not user: if not user:
_LOGIN_FAILURES[ip] = [*recent, now]
raise fail(401, "invalid username or password") raise fail(401, "invalid username or password")
sess = store.create_session(user["id"]) _LOGIN_FAILURES.pop(ip, None)
return ok({"token": f"platform-token-{user['id']}", "user": user, "session_id": sess["session_id"]}) sess = store.create_session(user["id"], ip=None)
return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]})
@router.post("/logout") @router.post("/logout")
@@ -422,7 +466,7 @@ async def me(request: Request) -> dict[str, Any]:
token = auth.replace("Bearer ", "").strip() token = auth.replace("Bearer ", "").strip()
# token 格式: platform-token-{user_id} # token 格式: platform-token-{user_id}
if token.startswith("platform-token-"): if token.startswith("platform-token-"):
user_id = token[len("platform-token-"):] user_id = token[len("platform-token-"):].split(".", 1)[0]
for u in store.users(): for u in store.users():
if u.get("id") == user_id: if u.get("id") == user_id:
return ok(u) return ok(u)
@@ -431,9 +475,12 @@ async def me(request: Request) -> dict[str, Any]:
@router.get("/dashboard/overview") @router.get("/dashboard/overview")
async def dashboard_overview() -> dict[str, Any]: async def dashboard_overview() -> dict[str, Any]:
cached = _cached_dashboard("overview")
if cached is not None:
return cached
store = get_platform_store() store = get_platform_store()
tasks = store.tasks() tasks = store.tasks()
return ok( return _store_dashboard_cache("overview", ok(
{ {
"models": len(store.models()), "models": len(store.models()),
"datasets": len(store.datasets()), "datasets": len(store.datasets()),
@@ -442,11 +489,14 @@ async def dashboard_overview() -> dict[str, Any]:
"compute_nodes": len(store.compute_nodes()), "compute_nodes": len(store.compute_nodes()),
"gpus": len(store.gpus()), "gpus": len(store.gpus()),
} }
) ))
@router.get("/dashboard/stats") @router.get("/dashboard/stats")
async def dashboard_stats() -> dict[str, Any]: async def dashboard_stats() -> dict[str, Any]:
cached = _cached_dashboard("stats")
if cached is not None:
return cached
"""看板聚合数据:基于平台真实数据;缺项做合理近似。""" """看板聚合数据:基于平台真实数据;缺项做合理近似。"""
store = get_platform_store() store = get_platform_store()
tasks = store.tasks() tasks = store.tasks()
@@ -603,7 +653,7 @@ async def dashboard_stats() -> dict[str, Any]:
except Exception: except Exception:
pass pass
return ok( return _store_dashboard_cache("stats", ok(
{ {
"online_services": sum(s["count"] for s in service_status), "online_services": sum(s["count"] for s in service_status),
"running_tasks": len(running_ft) + eval_running, "running_tasks": len(running_ft) + eval_running,
@@ -615,7 +665,7 @@ async def dashboard_stats() -> dict[str, Any]:
"login_duration_rank": login_duration_rank, "login_duration_rank": login_duration_rank,
"recent_login_users": recent_login_users, "recent_login_users": recent_login_users,
} }
) ))
@router.get("/system-info") @router.get("/system-info")
@@ -731,7 +781,12 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
@router.delete("/model-manage/trained-models/{model_id}") @router.delete("/model-manage/trained-models/{model_id}")
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]: async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("trained_model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this trained model")
pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}")
if pending:
return pending
get_platform_store().delete_trained_model(model_id) get_platform_store().delete_trained_model(model_id)
return ok({"deleted": model_id, "type": type}) return ok({"deleted": model_id, "type": type})
@@ -747,7 +802,9 @@ async def trained_model_lineage(model_id: str) -> dict[str, Any]:
@router.get("/model-manage/export-jobs") @router.get("/model-manage/export-jobs")
async def model_export_jobs(trained_model_id: str | None = Query(default=None)) -> dict[str, Any]: async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"):
raise fail(403, "no permission to access export jobs")
return ok(get_platform_store().model_export_jobs(trained_model_id)) return ok(get_platform_store().model_export_jobs(trained_model_id))
@@ -878,7 +935,7 @@ async def delete_model(model_id: str, current_user: dict = Depends(get_current_u
@router.post("/model-manage/merge") @router.post("/model-manage/merge")
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "") trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
trained_model = next( trained_model = next(
@@ -889,6 +946,12 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
), ),
None, None,
) )
if not trained_model:
raise fail(404, "trained model not found")
if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"):
raise fail(403, "no permission to merge this trained model")
if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"):
raise fail(403, "no permission to use merge base model")
base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path")) base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
adapter_path = ( adapter_path = (
payload.get("adapter_path") payload.get("adapter_path")
@@ -906,6 +969,15 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
await _wait_for_object_storage() await _wait_for_object_storage()
except RuntimeError as exc: except RuntimeError as exc:
raise fail(503, str(exc)) raise fail(503, str(exc))
try:
prepared_base = await _prepare_resource_on_node(store, "model", str(payload.get("base_model_id") or base_model_path), node)
if prepared_base:
base_model_path = prepared_base
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", str(payload.get("adapter_model_id") or (trained_model and trained_model.get("id")) or ""), node)
if prepared_adapter:
adapter_path = prepared_adapter
except Exception as exc:
raise fail(502, f"merge resource preparation failed: {exc}")
health = node.get("health_detail") or {} health = node.get("health_detail") or {}
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs") output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged") output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
@@ -1075,6 +1147,7 @@ async def _sync_training_dataset_to_compute_node(
"version_id": obj["version_id"], "version_id": obj["version_id"],
"download_url": url, "download_url": url,
"checksum_sha256": obj.get("checksum_sha256") or "", "checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"datasets/{dataset_id}/{target_name}", "relative_path": f"datasets/{dataset_id}/{target_name}",
}) })
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]}) results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
@@ -1258,7 +1331,15 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/fine-tune") @router.post("/fine-tune")
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload.setdefault("created_by", current_user.get("id"))
if not is_admin(current_user):
model_id = str(payload.get("base_model") or payload.get("base_model_id") or "")
dataset_id = str(payload.get("train_dataset_id") or "")
if model_id and not has_resource_access("model", model_id, current_user, "execute"):
raise fail(403, "no permission to use this base model")
if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
raise fail(403, "no permission to use this dataset")
try: try:
task = get_platform_store().create_task(payload) task = get_platform_store().create_task(payload)
return ok({"id": task["id"]}) return ok({"id": task["id"]})
@@ -1275,11 +1356,16 @@ async def start_fine_tune(
# GPU 权限校验:普通用户只能使用被分配的 GPU # GPU 权限校验:普通用户只能使用被分配的 GPU
if not is_admin(current_user): if not is_admin(current_user):
node_id = payload.get("compute_node_id") or payload.get("node_id") node_id = payload.get("compute_node_id") or payload.get("node_id")
gpu_indices = payload.get("gpus") or [] gpu_indices = payload.get("gpu_indices")
if gpu_indices is None:
gpu_indices = payload.get("gpus") or []
if node_id and gpu_indices: if node_id and gpu_indices:
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices): if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
raise fail(403, "无权使用所选 GPU请联系管理员分配") raise fail(403, "无权使用所选 GPU请联系管理员分配")
# 记录创建者 # 记录创建者
if node_id and not gpu_indices:
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
payload["strict_node_selection"] = bool(node_id)
payload.setdefault("created_by", current_user.get("id")) payload.setdefault("created_by", current_user.get("id"))
try: try:
return ok(await _submit_fine_tune_task(store, payload)) return ok(await _submit_fine_tune_task(store, payload))
@@ -1411,7 +1497,9 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
@router.put("/fine-tune/{task_id}") @router.put("/fine-tune/{task_id}")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "write"):
raise fail(403, "no permission to update this task")
try: try:
return ok(get_platform_store().update_task(task_id, payload)) return ok(get_platform_store().update_task(task_id, payload))
except KeyError: except KeyError:
@@ -1442,13 +1530,15 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
@router.post("/fine-tune/{task_id}/retry") @router.post("/fine-tune/{task_id}/retry")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
payload = payload or {} payload = payload or {}
try: try:
task = store.task(task_id) task = store.task(task_id)
except KeyError: except KeyError:
raise fail(404, "fine tune task not found") raise fail(404, "fine tune task not found")
if not has_resource_access("fine-tune", task_id, current_user, "execute"):
raise fail(403, "no permission to retry this task")
if task["status"] not in {"failed", "stopped"} and not payload.get("force"): if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
raise fail(409, "only failed or stopped tasks can be retried without force=true") raise fail(409, "only failed or stopped tasks can be retried without force=true")
retry_payload = {**task, **payload, "task_id": task_id, "id": task_id} retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
@@ -1544,7 +1634,7 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
@router.post("/model-eval/start") @router.post("/model-eval/start")
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Start an evaluation task: submit eval job to compute node.""" """Start an evaluation task: submit eval job to compute node."""
store = get_platform_store() store = get_platform_store()
# 1. Create eval task record # 1. Create eval task record
@@ -1745,12 +1835,23 @@ async def dimension_delete(dimension_id: str) -> dict[str, Any]:
@router.get("/model-compare") @router.get("/model-compare")
async def model_compare_list() -> dict[str, Any]: async def model_compare_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().compare_tasks()) tasks = get_platform_store().compare_tasks()
if is_admin(current_user):
return ok(tasks)
accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user)
return ok([item for item in tasks if item["id"] in accessible])
@router.post("/model-compare") @router.post("/model-compare")
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload.setdefault("created_by", current_user.get("id"))
model_ids = payload.get("model_ids") or payload.get("models") or []
if not is_admin(current_user):
for model_id in model_ids:
if isinstance(model_id, dict): model_id = model_id.get("id") or model_id.get("model_id")
if model_id and not has_resource_access("model", str(model_id), current_user, "execute"):
raise fail(403, "no permission to use inference model")
task = get_platform_store().create_compare_task(payload) task = get_platform_store().create_compare_task(payload)
return ok({"id": task["id"]}) return ok({"id": task["id"]})
@@ -1766,9 +1867,12 @@ async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict
@router.get("/model-compare/{task_id}") @router.get("/model-compare/{task_id}")
async def model_compare_detail(task_id: str) -> dict[str, Any]: async def model_compare_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try: try:
return ok(get_platform_store().compare_task(task_id)) task = get_platform_store().compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "read"):
raise fail(403, "no permission to access inference task")
return ok(task)
except KeyError: except KeyError:
raise fail(404, "compare task not found") raise fail(404, "compare task not found")
@@ -1805,12 +1909,17 @@ async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = No
@router.delete("/model-compare/{task_id}") @router.delete("/model-compare/{task_id}")
async def model_compare_delete(task_id: str) -> dict[str, Any]: async def model_compare_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 先删记录(快),再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞 # 先删记录(快),再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
try: try:
task = get_platform_store().compare_task(task_id) task = get_platform_store().compare_task(task_id)
except KeyError: except KeyError:
raise fail(404, "compare task not found") raise fail(404, "compare task not found")
if not has_resource_access("compare", task_id, current_user, "delete"):
raise fail(403, "no permission to delete inference task")
pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}")
if pending:
return pending
get_platform_store().delete_compare_task(task_id) get_platform_store().delete_compare_task(task_id)
try: try:
await _unload_from_compute_node(get_platform_store(), task=task) await _unload_from_compute_node(get_platform_store(), task=task)
@@ -1873,7 +1982,7 @@ def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[
@router.post("/model-compare/{task_id}/load") @router.post("/model-compare/{task_id}/load")
async def model_compare_load(task_id: str) -> dict[str, Any]: async def model_compare_load(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""异步派发模型加载到算力节点,立即返回。 """异步派发模型加载到算力节点,立即返回。
加载进度由轮询对账器compute_poller → reconcile_inference_loads推进 加载进度由轮询对账器compute_poller → reconcile_inference_loads推进
@@ -1883,6 +1992,8 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
try: try:
store = get_platform_store() store = get_platform_store()
task = store.compare_task(task_id) task = store.compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "execute"):
raise fail(403, "no permission to load inference task")
models = task.get("models") or [] models = task.get("models") or []
if isinstance(models, str): if isinstance(models, str):
try: try:
@@ -1953,10 +2064,12 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
@router.post("/model-compare/{task_id}/unload") @router.post("/model-compare/{task_id}/unload")
async def model_compare_unload(task_id: str) -> dict[str, Any]: async def model_compare_unload(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try: try:
store = get_platform_store() store = get_platform_store()
task = store.compare_task(task_id) task = store.compare_task(task_id)
if not has_resource_access("compare", task_id, current_user, "write"):
raise fail(403, "no permission to unload inference task")
# 任务感知卸载:只释放该任务实际加载到的节点,短超时快速返回 # 任务感知卸载:只释放该任务实际加载到的节点,短超时快速返回
unload_result = await _unload_from_compute_node(store, task=task) unload_result = await _unload_from_compute_node(store, task=task)
updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}) updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
@@ -2048,7 +2161,19 @@ async def model_chat_local_unload() -> dict[str, Any]:
if not store.is_inference_loaded(n["id"]): if not store.is_inference_loaded(n["id"]):
continue continue
try: try:
result = await ComputeNodeClient(n["api_base_url"]).inference_unload() client = ComputeNodeClient(n["api_base_url"])
last_error = ""
result = None
for attempt in range(3):
try:
result = await client.inference_unload()
break
except Exception as exc: # noqa: BLE001 - retry node cleanup
last_error = str(exc)
if attempt < 2:
await asyncio.sleep(2 ** attempt)
if result is None:
raise RuntimeError(last_error or "inference unload failed")
results.append({"node_id": n["id"], "success": True, "result": result}) results.append({"node_id": n["id"], "success": True, "result": result})
except Exception as exc: # noqa: BLE001 - best-effort unload except Exception as exc: # noqa: BLE001 - best-effort unload
results.append({"node_id": n["id"], "success": False, "error": str(exc)}) results.append({"node_id": n["id"], "success": False, "error": str(exc)})
@@ -2073,19 +2198,31 @@ async def model_chat_local_status() -> dict[str, Any]:
@router.post("/model-chat/trained/preload") @router.post("/model-chat/trained/preload")
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "")
if resource_id and not has_resource_access("trained_model", resource_id, current_user, "execute") and not has_resource_access("model", resource_id, current_user, "execute"):
raise fail(403, "no permission to load this model")
"""Load a trained model (base + adapter) on the compute node for inference.""" """Load a trained model (base + adapter) on the compute node for inference."""
model_path = (payload.get("model_name_or_path") or "").strip() model_path = (payload.get("model_name_or_path") or "").strip()
if not model_path: if not model_path:
return ok({"loaded": False, "error": "model_name_or_path is required"}) return ok({"loaded": False, "error": "model_name_or_path is required"})
store = get_platform_store() store = get_platform_store()
node = _select_first_online_node(store) requested_node_id = str(payload.get("compute_node_id") or payload.get("node_id") or "")
node = next((item for item in store.compute_nodes() if item.get("id") == requested_node_id and item.get("enabled") and item.get("scheduler_status") == "online"), None)
if not node:
node = _select_first_online_node(store)
if not node: if not node:
return ok({"loaded": False, "error": "no online compute node"}) return ok({"loaded": False, "error": "no online compute node"})
try: try:
prepared_path = await _prepare_resource_on_node(store, "trained_model", str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or ""), node)
if prepared_path:
payload = {**payload, "model_name_or_path": prepared_path}
prepared_path = await _prepare_resource_on_node(store, "model", str(payload.get("model_id") or payload.get("resource_id") or ""), node)
if prepared_path:
payload = {**payload, "model_name_or_path": prepared_path}
client = ComputeNodeClient(node["api_base_url"]) client = ComputeNodeClient(node["api_base_url"])
# 计算节点现在异步加载HTTP 接受loading/ready即视为派发成功 # 计算节点现在异步加载HTTP 接受loading/ready即视为派发成功
result = await client.inference_load(payload) result = await client.inference_load({**payload, "compute_node_id": node["id"]})
if result.get("loaded") or result.get("status") in {"loading", "ready"}: if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"]) store.mark_inference_loaded(node["id"])
return ok(result) return ok(result)
@@ -2094,8 +2231,13 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dic
@router.get("/compute/nodes") @router.get("/compute/nodes")
async def compute_nodes() -> dict[str, Any]: async def compute_nodes(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().compute_nodes()) nodes = get_platform_store().compute_nodes()
if is_admin(current_user):
return ok(nodes)
# 普通用户只看到自己被分配 GPU 的节点,避免泄露节点拓扑和未授权资源。
assigned = {item["node_id"] for item in get_platform_store().gpu_assignments_for_user(current_user["id"])}
return ok([node for node in nodes if node["id"] in assigned])
@router.post("/storage/objects/presign") @router.post("/storage/objects/presign")
@@ -2154,27 +2296,90 @@ async def prepare_storage_resource(
for obj in objects: for obj in objects:
url = get_object_storage().presigned_get(obj["object_key"]) url = get_object_storage().presigned_get(obj["object_key"])
filename = Path(str(obj.get("file_name") or obj["object_key"])).name filename = Path(str(obj.get("file_name") or obj["object_key"])).name
result = await client.prepare_cache({ cache_job = store.create_storage_cache_job({"storage_object_id": obj["id"], "node_id": node_id, "direction": "download"})
"resource_id": resource_id, try:
"version_id": obj["version_id"], result = await client.prepare_cache({
"download_url": url, "resource_id": resource_id,
"checksum_sha256": obj.get("checksum_sha256") or "", "version_id": obj["version_id"],
"relative_path": f"{resource_type}s/{resource_id}/{filename}", "download_url": url,
}) "checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
})
store.update_storage_cache_job(cache_job["id"], {"status": "completed", "progress": 100, "local_path": result.get("local_path"), "completed_at": utcnow()})
except Exception as exc:
store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()})
raise
prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id}) prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared}) return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})
@router.get("/compute/nodes/{node_id}") @router.get("/storage/cache/jobs/{node_id}")
async def compute_node_detail(node_id: str) -> dict[str, Any]: async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None) return ok(get_platform_store().storage_cache_jobs_for_node(node_id, limit))
@router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}")
async def archive_node_files(
resource_type: str,
resource_id: str,
node_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""Archive completed node files to MinIO without proxying file bytes through Backend."""
if not has_resource_access(resource_type, resource_id, current_user, "execute"):
raise fail(403, "no permission to archive this resource")
store = get_platform_store()
if not is_admin(current_user):
model_id = str(payload.get("model_id") or "")
dataset_id = str(payload.get("dataset_id") or "")
if not model_id or not has_resource_access("model", model_id, current_user, "execute"):
raise fail(403, "no permission to evaluate this model")
if not dataset_id or not has_resource_access("dataset", dataset_id, current_user, "execute"):
raise fail(403, "no permission to evaluate this dataset")
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node: if not node:
raise fail(404, "compute node not found") raise fail(404, "compute node not found")
files = payload.get("files") or []
if not isinstance(files, list) or not files:
raise fail(400, "files is required")
client = ComputeNodeClient(node["api_base_url"], timeout=900)
archived = []
for item in files:
path = str(item.get("path") or "")
name = Path(str(item.get("file_name") or Path(path).name)).name
version_id = str(item.get("version_id") or uuid.uuid4().hex)
object_key = str(item.get("object_key") or f"{resource_type}s/{resource_id}/versions/{version_id}/{name}")
url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, url, object_key, str(item.get("content_type") or "application/octet-stream"))
metadata = get_object_storage().stat(object_key)
record = store.create_storage_object({
"resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
"bucket": get_object_storage().bucket, "object_key": object_key, "file_name": name,
"content_type": item.get("content_type"), "checksum_sha256": result.get("checksum_sha256"),
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, "status": "available",
"created_by": current_user.get("id"),
})
archived.append({"object": record, "node_id": node_id})
return ok({"status": "available", "items": archived})
@router.get("/compute/nodes/{node_id}")
async def compute_node_detail(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node:
raise fail(404, "compute node not found")
if not is_admin(current_user) and not any(item["node_id"] == node_id for item in store.gpu_assignments_for_user(current_user["id"])):
raise fail(403, "no permission to access this compute node")
return ok(node) return ok(node)
@router.post("/compute/nodes") @router.post("/compute/nodes")
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try: try:
return ok(get_platform_store().create_compute_node(payload)) return ok(get_platform_store().create_compute_node(payload))
except KeyError as exc: except KeyError as exc:
@@ -2184,7 +2389,9 @@ async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str,
@router.put("/compute/nodes/{node_id}") @router.put("/compute/nodes/{node_id}")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try: try:
return ok(get_platform_store().update_compute_node(node_id, payload)) return ok(get_platform_store().update_compute_node(node_id, payload))
except KeyError: except KeyError:
@@ -2194,7 +2401,9 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
@router.delete("/compute/nodes/{node_id}") @router.delete("/compute/nodes/{node_id}")
async def delete_compute_node(node_id: str) -> dict[str, Any]: async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try: try:
return ok(get_platform_store().delete_compute_node(node_id)) return ok(get_platform_store().delete_compute_node(node_id))
except KeyError: except KeyError:
@@ -2204,7 +2413,9 @@ async def delete_compute_node(node_id: str) -> dict[str, Any]:
@router.post("/compute/nodes/{node_id}/test-connection") @router.post("/compute/nodes/{node_id}/test-connection")
async def test_compute_node(node_id: str) -> dict[str, Any]: async def test_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
store = get_platform_store() store = get_platform_store()
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
if not node: if not node:
@@ -2238,22 +2449,28 @@ async def test_compute_node(node_id: str) -> dict[str, Any]:
@router.post("/compute/nodes/{node_id}/health-check") @router.post("/compute/nodes/{node_id}/health-check")
async def health_check_compute_node(node_id: str) -> dict[str, Any]: async def health_check_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return await test_compute_node(node_id) return await test_compute_node(node_id, current_user)
@router.post("/compute/nodes/{node_id}/enable") @router.post("/compute/nodes/{node_id}/enable")
async def enable_compute_node(node_id: str) -> dict[str, Any]: async def enable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"})) return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))
@router.post("/compute/nodes/{node_id}/disable") @router.post("/compute/nodes/{node_id}/disable")
async def disable_compute_node(node_id: str) -> dict[str, Any]: async def disable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"})) return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))
@router.post("/compute/nodes/{node_id}/drain") @router.post("/compute/nodes/{node_id}/drain")
async def drain_compute_node(node_id: str) -> dict[str, Any]: async def drain_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"})) return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))
@@ -2460,8 +2677,13 @@ async def compute_node_engines(node_id: str) -> dict[str, Any]:
@router.get("/compute/gpus") @router.get("/compute/gpus")
async def compute_gpus() -> dict[str, Any]: async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().gpus()) store = get_platform_store()
gpus = store.gpus()
if is_admin(current_user):
return ok(gpus)
assigned = {(item["node_id"], int(item["gpu_index"])) for item in store.gpu_assignments_for_user(current_user["id"])}
return ok([gpu for gpu in gpus if (gpu.get("node_id"), int(gpu.get("id", gpu.get("gpu_index", -1)))) in assigned])
@router.get("/compute/queue") @router.get("/compute/queue")
@@ -2485,8 +2707,10 @@ async def compute_job_detail(job_id: str) -> dict[str, Any]:
@router.post("/compute/jobs/{job_id}/stop") @router.post("/compute/jobs/{job_id}/stop")
async def compute_job_stop(job_id: str) -> dict[str, Any]: async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
task = _task_for_compute_job(job_id) task = _task_for_compute_job(job_id)
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
raise fail(403, "no permission to stop this compute job")
if not task: if not task:
node = _node_for_compute_job_record(job_id) node = _node_for_compute_job_record(job_id)
if not node: if not node:
@@ -2521,13 +2745,13 @@ async def compute_job_logs(
@router.post("/compute/jobs/{job_id}/retry") @router.post("/compute/jobs/{job_id}/retry")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
payload = payload or {} payload = payload or {}
task = _task_for_compute_job(job_id) task = _task_for_compute_job(job_id)
if not task: if not task:
raise fail(404, "compute job not found") raise fail(404, "compute job not found")
return await retry_fine_tune(task["id"], payload) return await retry_fine_tune(task["id"], payload, current_user)
@router.post("/compute/jobs/{job_id}/priority") @router.post("/compute/jobs/{job_id}/priority")

View File

@@ -9,6 +9,13 @@ from app.db.platform_store import get_platform_store
# 无需鉴权的路径前缀(健康检查、登录等) # 无需鉴权的路径前缀(健康检查、登录等)
PUBLIC_PATHS = ("/health", "/login", "/system-info") PUBLIC_PATHS = ("/health", "/login", "/system-info")
OWNER_TABLES = {
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
"fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"),
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
"project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"),
}
def _extract_token(request: Request) -> str | None: def _extract_token(request: Request) -> str | None:
@@ -19,6 +26,9 @@ def _extract_token(request: Request) -> str | None:
return token[len("platform-token-"):] return token[len("platform-token-"):]
return None return None
def _session_token(user_id: str, session_id: str) -> str:
return f"platform-token-{user_id}.{session_id}"
def get_current_user(request: Request) -> dict[str, Any]: def get_current_user(request: Request) -> dict[str, Any]:
""" """
@@ -33,14 +43,30 @@ def get_current_user(request: Request) -> dict[str, Any]:
if path.endswith(prefix): if path.endswith(prefix):
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False} return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
user_id = _extract_token(request) token_value = _extract_token(request)
if not user_id: if not token_value:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token") raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
store = get_platform_store() store = get_platform_store()
for u in store.users(): user_id, _, session_id = token_value.partition(".")
if u.get("id") == user_id: if session_id:
return u with store.connect() as conn:
session = conn.execute(
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
).fetchone()
if not session or session["user_id"] != user_id or session["logout_at"]:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
if session["expires_at"]:
from datetime import datetime, timezone
try:
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
except ValueError:
pass
with store.connect() as conn:
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
if user_row:
return store._user(user_row)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found") raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
@@ -75,6 +101,23 @@ def has_resource_access(
user_id = user.get("id") user_id = user.get("id")
user_role = user.get("role") user_role = user.get("role")
owner_tables = OWNER_TABLES
table_info = owner_tables.get(resource_type)
if table_info and user_id:
table, column = table_info
with store.connect() as conn:
row = conn.execute(f"SELECT {column} FROM {table} WHERE id=?", (resource_id,)).fetchone()
if row:
owner = row[column]
if column == "payload":
try:
import json
owner = json.loads(owner or "{}").get("created_by")
except (TypeError, ValueError):
owner = None
if owner == user_id:
return True
for entry in acls: for entry in acls:
# 按 user 授权 # 按 user 授权
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id: if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
@@ -135,4 +178,40 @@ def filter_accessible_resource_ids(
).fetchall() ).fetchall()
accessible = {r["resource_id"] for r in rows} accessible = {r["resource_id"] for r in rows}
if resource_type in owner_tables:
table, column = owner_tables[resource_type]
with store.connect() as conn:
owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall()
accessible.update(row["id"] for row in owned)
return [rid for rid in all_ids if rid in accessible] return [rid for rid in all_ids if rid in accessible]
def filter_accessible_resource_ids_batch(
resource_type: str,
resource_ids: list[str],
user: dict[str, Any],
) -> set[str]:
"""Filter a list endpoint with one ACL query instead of one query per row."""
if is_admin(user):
return set(resource_ids)
if not resource_ids:
return set()
store = get_platform_store()
placeholders = ",".join("?" for _ in resource_ids)
with store.connect() as conn:
rows = conn.execute(
f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
"AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))",
(resource_type, *resource_ids, user.get("id"), user.get("role")),
).fetchall()
accessible = {row["resource_id"] for row in rows}
table_info = OWNER_TABLES.get(resource_type)
if table_info and user.get("id"):
table, column = table_info
with store.connect() as conn:
owned = conn.execute(
f"SELECT id FROM {table} WHERE id IN ({placeholders}) AND {column}=?",
(*resource_ids, user["id"]),
).fetchall()
accessible.update(row["id"] for row in owned)
return accessible

View File

@@ -61,7 +61,9 @@ class Settings:
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3) compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5) compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5)
minio_enabled: bool = _bool_env("MINIO_ENABLED", False) minio_enabled: bool = _bool_env("MINIO_ENABLED", False)
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://minio:9000") # MinIO is an independent service and may run on another host. The
# endpoint must therefore be reachable from the Backend container.
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://host.docker.internal:19000")
minio_access_key: str = os.getenv("MINIO_ACCESS_KEY", "minioadmin") minio_access_key: str = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin") minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources") minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")

View File

@@ -210,6 +210,10 @@ def configure_logging(settings: Settings | None = None) -> None:
logger.propagate = True logger.propagate = True
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
def get_logger(name: str) -> logging.Logger: def get_logger(name: str) -> logging.Logger:
return logging.getLogger(name) return logging.getLogger(name)
@@ -229,7 +233,15 @@ def setup_request_logging(app: FastAPI) -> None:
try: try:
response = await call_next(request) response = await call_next(request)
elapsed_ms = (time.perf_counter() - started_at) * 1000 elapsed_ms = (time.perf_counter() - started_at) * 1000
logger.info( # Docker/frontend probes and polling endpoints are intentionally
# quiet at INFO; failures remain visible at WARNING/ERROR.
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
log_method = logger.debug
if response.status_code >= 400:
log_method = logger.warning
log_method(
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s", "request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
request.method, request.method,
request.url.path, request.url.path,

View File

@@ -526,8 +526,15 @@ class PlatformStore:
"artifact_dir": "TEXT", "artifact_dir": "TEXT",
"compute_node_id": "TEXT", "compute_node_id": "TEXT",
"compute_node_name": "TEXT", "compute_node_name": "TEXT",
"created_by": "TEXT",
"tenant_id": "TEXT",
"project_id": "TEXT",
"deleted_at": "TEXT",
"deleted_by": "TEXT",
}, },
) )
for table in ("models", "datasets", "eval_tasks"):
self._ensure_columns(conn, table, {"deleted_at": "TEXT", "deleted_by": "TEXT", "tenant_id": "TEXT", "project_id": "TEXT"})
self._ensure_columns( self._ensure_columns(
conn, conn,
"resource_replicas", "resource_replicas",
@@ -1470,12 +1477,12 @@ class PlatformStore:
def delete_model(self, model_id: str) -> None: def delete_model(self, model_id: str) -> None:
with self.connect() as conn: with self.connect() as conn:
conn.execute("DELETE FROM models WHERE id=?", (model_id,)) conn.execute("UPDATE models SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", model_id))
def trained_models(self) -> list[dict[str, Any]]: def trained_models(self) -> list[dict[str, Any]]:
self.refresh_runtime_state() self.refresh_runtime_state()
with self.connect() as conn: with self.connect() as conn:
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall() rows = conn.execute("SELECT * FROM trained_models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall()
items = [] items = []
for row in rows: for row in rows:
item = { item = {
@@ -1498,7 +1505,7 @@ class PlatformStore:
def delete_trained_model(self, model_id: str) -> None: def delete_trained_model(self, model_id: str) -> None:
with self.connect() as conn: with self.connect() as conn:
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id)) conn.execute("UPDATE trained_models SET deleted_at=?, deleted_by=? WHERE id=? OR name=?", (utcnow(), "system", model_id, model_id))
def datasets(self) -> list[dict[str, Any]]: def datasets(self) -> list[dict[str, Any]]:
with self.connect() as conn: with self.connect() as conn:
@@ -1507,6 +1514,7 @@ class PlatformStore:
FROM datasets dataset FROM datasets dataset
LEFT JOIN data_process_tasks task LEFT JOIN data_process_tasks task
ON task.id=COALESCE(dataset.source_task_id, dataset.task_id) ON task.id=COALESCE(dataset.source_task_id, dataset.task_id)
WHERE dataset.deleted_at IS NULL
ORDER BY dataset.create_time DESC""" ORDER BY dataset.create_time DESC"""
).fetchall() ).fetchall()
return [self._dataset(conn, row) for row in rows] return [self._dataset(conn, row) for row in rows]
@@ -1646,8 +1654,7 @@ class PlatformStore:
def delete_dataset(self, dataset_id: str) -> None: def delete_dataset(self, dataset_id: str) -> None:
with self.connect() as conn: with self.connect() as conn:
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,)) conn.execute("UPDATE datasets SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", dataset_id))
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]: def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
now = utcnow() now = utcnow()
@@ -2004,13 +2011,14 @@ class PlatformStore:
task_id = str(payload.get("task_id") or payload.get("id")) task_id = str(payload.get("task_id") or payload.get("id"))
current = self.task(task_id) current = self.task(task_id)
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8} merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0] selected_gpus = payload.get("gpus") or merged.get("gpus") or []
process_id = int(43000 + (time.time() % 10000)) process_id = int(43000 + (time.time() % 10000))
with self.connect() as conn: with self.connect() as conn:
owner = f"start:{task_id}:{uuid.uuid4().hex[:8]}" owner = f"start:{task_id}:{uuid.uuid4().hex[:8]}"
if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner): if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner):
raise RuntimeError("compute scheduler is busy, please retry") raise RuntimeError("compute scheduler is busy, please retry")
node = self._schedule_node_locked(conn, payload) node = self._schedule_node_locked(conn, payload)
selected_gpus = list(node.get("selected_gpus") or selected_gpus)
sync_job_id = new_id("sync") sync_job_id = new_id("sync")
conn.execute( conn.execute(
""" """
@@ -2094,7 +2102,7 @@ class PlatformStore:
task = self.task(task_id) task = self.task(task_id)
merged = {**task, **(payload or {}), "id": task_id} merged = {**task, **(payload or {}), "id": task_id}
node = self.select_compute_node(merged) node = self.select_compute_node(merged)
selected_gpus = merged.get("gpus") or [0] selected_gpus = merged.get("gpus") or []
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus) return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -2108,7 +2116,7 @@ class PlatformStore:
"progress": int(payload.get("progress") or 0), "progress": int(payload.get("progress") or 0),
} }
node = self.select_compute_node(transient_task) node = self.select_compute_node(transient_task)
selected_gpus = transient_task.get("gpus") or [0] selected_gpus = transient_task.get("gpus") or []
return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus) return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus)
def _compute_job_payload_from_task_node( def _compute_job_payload_from_task_node(
@@ -2260,7 +2268,7 @@ class PlatformStore:
for item in runtime_files for item in runtime_files
], ],
"output_dir": output_dir, "output_dir": output_dir,
"gpus": selected_gpus or task.get("gpus") or [0], "gpus": selected_gpus if selected_gpus is not None else (task.get("gpus") or []),
"compute_node_id": node["id"], "compute_node_id": node["id"],
"compute_node_code": node["code"], "compute_node_code": node["code"],
} }
@@ -2270,7 +2278,7 @@ class PlatformStore:
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None) node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
if not node: if not node:
raise RuntimeError("compute node not found") raise RuntimeError("compute node not found")
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0]) return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [])
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]: def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
status_map = { status_map = {
@@ -2493,7 +2501,7 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None: def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn: with self.connect() as conn:
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,)) conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
def running_eval_tasks(self) -> list[dict[str, Any]]: def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running.""" """Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2712,27 +2720,71 @@ class PlatformStore:
def _node_capacity(self, node: dict[str, Any]) -> int: def _node_capacity(self, node: dict[str, Any]) -> int:
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0)) return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
@staticmethod
def _payload_gpu_indexes(payload: dict[str, Any]) -> list[int]:
raw = payload.get("gpu_indices")
if raw is None:
raw = payload.get("gpus")
if raw is None:
return []
try:
values = [int(item) for item in raw]
except (TypeError, ValueError) as exc:
raise RuntimeError("invalid GPU index") from exc
if any(item < 0 for item in values):
raise RuntimeError("GPU index must be non-negative")
return sorted(set(values))
def _select_node_gpus(
self,
conn: PgConnection,
node: dict[str, Any],
requested: list[int],
payload: dict[str, Any],
) -> list[int]:
available = self._node_gpu_indexes(conn, node)
active = self._active_gpu_indexes(conn, node["id"])
allowed = payload.get("allowed_gpu_indices")
if allowed is not None:
available &= {int(item) for item in allowed}
if requested:
selected = set(requested)
if not selected.issubset(available):
raise RuntimeError(f"requested GPU is not available on compute node {node['code']}")
if selected.intersection(active):
raise RuntimeError(f"requested GPU is busy on compute node {node['code']}")
return sorted(selected)
if payload.get("allow_cpu") or payload.get("device") == "cpu":
return []
count = max(1, int(payload.get("gpu_count") or 1))
free = sorted(available - active)
if len(free) < count:
raise RuntimeError(f"compute node {node['code']} has only {len(free)} available GPU(s)")
return free[:count]
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]: def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
requested = payload.get("requested_node_id") or payload.get("compute_node_id") requested = payload.get("requested_node_id") or payload.get("compute_node_id")
requested_gpus = [int(item) for item in payload.get("gpus") or []] requested_gpus = self._payload_gpu_indexes(payload)
nodes = self._compute_nodes_locked(conn) nodes = self._compute_nodes_locked(conn)
candidates = [ candidates = [
n n
for n in nodes for n in nodes
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n) if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
] ]
if requested_gpus:
requested_gpu_set = set(requested_gpus)
candidates = [
node
for node in candidates
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
]
if requested: if requested:
selected = next((n for n in candidates if n["id"] == requested), None) selected = next((n for n in candidates if n["id"] == requested), None)
if selected: if not selected:
return selected raise RuntimeError("selected compute node is unavailable")
selected["selected_gpus"] = self._select_node_gpus(conn, selected, requested_gpus, payload)
return selected
filtered = []
for node in candidates:
try:
node["selected_gpus"] = self._select_node_gpus(conn, node, requested_gpus, payload)
filtered.append(node)
except RuntimeError:
continue
candidates = filtered
if not candidates: if not candidates:
if not nodes: if not nodes:
raise RuntimeError("no available compute node: no compute node configured") raise RuntimeError("no available compute node: no compute node configured")
@@ -2744,6 +2796,8 @@ class PlatformStore:
reason = f"status={node['scheduler_status']}" reason = f"status={node['scheduler_status']}"
elif node["current_running_jobs"] >= self._node_capacity(node): elif node["current_running_jobs"] >= self._node_capacity(node):
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}" reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
elif requested:
reason = "selected node unavailable"
else: else:
reason = "not selected" reason = "not selected"
reasons.append(f"{node['code']}({reason})") reasons.append(f"{node['code']}({reason})")
@@ -2930,6 +2984,43 @@ class PlatformStore:
raise KeyError(object_id) raise KeyError(object_id)
return dict(row) return dict(row)
def create_storage_cache_job(self, payload: dict[str, Any]) -> dict[str, Any]:
job_id = str(payload.get("id") or new_id("cache"))
with self.connect() as conn:
conn.execute(
"""
INSERT INTO storage_cache_jobs
(id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(job_id, payload["storage_object_id"], payload["node_id"], payload.get("direction", "download"),
payload.get("status", "running"), int(payload.get("progress", 0)), payload.get("local_path"),
payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at")),
)
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
return dict(row)
def update_storage_cache_job(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "progress", "local_path", "error", "completed_at"}
fields = {key: value for key, value in payload.items() if key in allowed}
if fields:
assignments = ", ".join(f"{key}=?" for key in fields)
with self.connect() as conn:
conn.execute(f"UPDATE storage_cache_jobs SET {assignments} WHERE id=?", (*fields.values(), job_id))
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
if not row:
raise KeyError(job_id)
return dict(row)
def storage_cache_jobs_for_node(self, node_id: str, limit: int = 100) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM storage_cache_jobs WHERE node_id=? ORDER BY create_time DESC LIMIT ?",
(node_id, max(1, min(limit, 500))),
).fetchall()
return [dict(row) for row in rows]
def update_resource_replica_sync_result( def update_resource_replica_sync_result(
self, self,
replica_id: str, replica_id: str,
@@ -3613,9 +3704,9 @@ class PlatformStore:
login_at = utcnow() login_at = utcnow()
with self.connect() as conn: with self.connect() as conn:
conn.execute( conn.execute(
"INSERT INTO sessions (id, user_id, username, login_at, create_time) " "INSERT INTO sessions (id, user_id, username, login_at, issued_at, expires_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s)", "VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s, %s, %s)",
(sid, user_id, user_id, login_at, login_at), (sid, user_id, user_id, login_at, login_at, datetime.fromtimestamp(time.time() + 1800, timezone.utc).isoformat(), login_at),
) )
return {"session_id": sid, "user_id": user_id, "login_at": login_at} return {"session_id": sid, "user_id": user_id, "login_at": login_at}
@@ -4263,6 +4354,14 @@ class PlatformStore:
assigned = {r["gpu_index"] for r in rows} assigned = {r["gpu_index"] for r in rows}
return all(idx in assigned for idx in gpu_indices) return all(idx in assigned for idx in gpu_indices)
def assigned_gpu_indexes(self, user_id: str, node_id: str) -> list[int]:
with self.connect() as conn:
rows = conn.execute(
"SELECT gpu_index FROM gpu_assignments WHERE user_id=? AND node_id=? ORDER BY gpu_index",
(user_id, node_id),
).fetchall()
return [int(row["gpu_index"]) for row in rows]
# ===================== 平台治理:资源可见性过滤 ===================== # ===================== 平台治理:资源可见性过滤 =====================
def _filter_accessible_ids( def _filter_accessible_ids(

View File

@@ -64,7 +64,12 @@ CREATE TABLE IF NOT EXISTS models (
api_key TEXT, api_key TEXT,
online_model_name TEXT, online_model_name TEXT,
can_train INTEGER NOT NULL DEFAULT 0, can_train INTEGER NOT NULL DEFAULT 0,
create_time TEXT NOT NULL create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
); );
CREATE TABLE IF NOT EXISTS trained_models ( CREATE TABLE IF NOT EXISTS trained_models (
@@ -78,7 +83,12 @@ CREATE TABLE IF NOT EXISTS trained_models (
merged_path TEXT, merged_path TEXT,
artifact_dir TEXT, artifact_dir TEXT,
compute_node_id TEXT, compute_node_id TEXT,
compute_node_name TEXT compute_node_name TEXT,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
); );
CREATE TABLE IF NOT EXISTS model_lineage ( CREATE TABLE IF NOT EXISTS model_lineage (
@@ -130,7 +140,12 @@ CREATE TABLE IF NOT EXISTS datasets (
size TEXT, size TEXT,
count INTEGER NOT NULL DEFAULT 0, count INTEGER NOT NULL DEFAULT 0,
description TEXT, description TEXT,
create_time TEXT NOT NULL create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
deleted_at TEXT,
deleted_by TEXT
); );
CREATE TABLE IF NOT EXISTS dataset_files ( CREATE TABLE IF NOT EXISTS dataset_files (
@@ -413,6 +428,18 @@ CREATE TABLE IF NOT EXISTS acls (
permission TEXT NOT NULL, permission TEXT NOT NULL,
create_time TEXT create_time TEXT
); );
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE UNIQUE INDEX IF NOT EXISTS uq_acls_subject_permission
ON acls(resource_type, resource_id, principal_type, principal_id, permission);
CREATE INDEX IF NOT EXISTS idx_acls_resource ON acls(resource_type, resource_id);
CREATE INDEX IF NOT EXISTS idx_acls_principal ON acls(principal_type, principal_id);
-- ---- 权限扩展来源004_permissions.sql ---- -- ---- 权限扩展来源004_permissions.sql ----
@@ -475,6 +502,9 @@ CREATE TABLE IF NOT EXISTS approval_steps (
comment TEXT, comment TEXT,
time TEXT time TEXT
); );
CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instances(applicant_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
CREATE TABLE IF NOT EXISTS audit_logs ( CREATE TABLE IF NOT EXISTS audit_logs (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,

View File

@@ -1,21 +1,23 @@
from __future__ import annotations from __future__ import annotations
from fastapi import APIRouter, Body from fastapi import APIRouter, Body, Depends
from typing import Any from typing import Any
from app.api.v1.endpoints.platform import ok, fail from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, is_admin
router = APIRouter(prefix="/approvals", tags=["approval"]) router = APIRouter(prefix="/approvals", tags=["approval"])
@router.get("/templates") @router.get("/templates")
def list_templates() -> dict[str, Any]: def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().approval_templates()) return ok(get_platform_store().approval_templates())
@router.post("/templates") @router.post("/templates")
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: def create_template(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
if not payload.get("name"): if not payload.get("name"):
raise fail(400, "name 必填") raise fail(400, "name 必填")
return ok(get_platform_store().create_approval_template(payload)) return ok(get_platform_store().create_approval_template(payload))
@@ -30,7 +32,8 @@ def get_template(template_id: str) -> dict[str, Any]:
@router.put("/templates/{template_id}") @router.put("/templates/{template_id}")
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try: try:
return ok(get_platform_store().update_approval_template(template_id, payload)) return ok(get_platform_store().update_approval_template(template_id, payload))
except KeyError: except KeyError:
@@ -38,7 +41,8 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> di
@router.delete("/templates/{template_id}") @router.delete("/templates/{template_id}")
def delete_template(template_id: str) -> dict[str, Any]: def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try: try:
return ok(get_platform_store().delete_approval_template(template_id)) return ok(get_platform_store().delete_approval_template(template_id))
except KeyError: except KeyError:
@@ -46,13 +50,15 @@ def delete_template(template_id: str) -> dict[str, Any]:
@router.get("") @router.get("")
def list_instances(status: str | None = None) -> dict[str, Any]: def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().approval_instances(status=status)) items = get_platform_store().approval_instances(status=status)
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
@router.post("") @router.post("")
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
for field in ("resource_type", "resource_id", "applicant_id"): payload["applicant_id"] = current_user.get("id")
for field in ("resource_type", "resource_id"):
if not payload.get(field): if not payload.get(field):
raise fail(400, f"{field} 必填") raise fail(400, f"{field} 必填")
try: try:
@@ -62,9 +68,12 @@ def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/{instance_id}") @router.get("/{instance_id}")
def get_instance(instance_id: str) -> dict[str, Any]: def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try: try:
return ok(get_platform_store().approval_instance(instance_id)) item = get_platform_store().approval_instance(instance_id)
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
raise fail(403, "no permission to access approval")
return ok(item)
except KeyError: except KeyError:
raise fail(404, "instance not found") raise fail(404, "instance not found")

View File

@@ -266,3 +266,8 @@ class ComputeNodeClient:
) )
response.raise_for_status() response.raise_for_status()
return _unwrap_dict(response.json()) return _unwrap_dict(response.json())
async def upload_file_to_url(self, path: str, upload_url: str, object_key: str = "", content_type: str = "application/octet-stream") -> dict[str, Any]:
return await self._request("POST", "/compute/files/upload-to-url", json_data={
"path": path, "upload_url": upload_url, "object_key": object_key, "content_type": content_type,
}, timeout=900)

View File

@@ -1,10 +1,11 @@
from __future__ import annotations from __future__ import annotations
from fastapi import APIRouter, Body, Request from fastapi import APIRouter, Body, Request, Depends
from typing import Any from typing import Any
from app.api.v1.endpoints.platform import ok, fail from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, has_resource_access, is_admin
router = APIRouter(prefix="/resources", tags=["resource"]) router = APIRouter(prefix="/resources", tags=["resource"])
@@ -16,8 +17,10 @@ def _actor(request: Request) -> str | None:
@router.get("/{resource_type}/{resource_id}/acl") @router.get("/{resource_type}/{resource_id}/acl")
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]: def get_acl(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""查询资源 ACL返回按主体分组的权限列表。""" """查询资源 ACL返回按主体分组的权限列表。"""
if not has_resource_access(resource_type, resource_id, current_user, "read"):
raise fail(403, "no permission to access resource ACL")
return ok(get_platform_store().resource_acl(resource_type, resource_id)) return ok(get_platform_store().resource_acl(resource_type, resource_id))
@@ -27,9 +30,18 @@ def set_acl(
resource_id: str, resource_id: str,
payload: dict[str, Any] = Body(...), payload: dict[str, Any] = Body(...),
request: Request = None, request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]: ) -> dict[str, Any]:
"""设置资源 ACLbody: { entries: [{ subject_type, subject_id, permissions: [] }] }""" """设置资源 ACLbody: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "only resource owner or admin can update ACL")
entries = payload.get("entries") or [] entries = payload.get("entries") or []
allowed = {"read", "write", "execute", "download", "delete", "admin"}
for entry in entries:
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
raise fail(400, "invalid ACL principal")
if any(permission not in allowed for permission in entry.get("permissions") or []):
raise fail(400, "invalid ACL permission")
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries) result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
get_platform_store().record_audit( get_platform_store().record_audit(
action="resource.acl.set", action="resource.acl.set",

View File

@@ -1,9 +1,10 @@
from __future__ import annotations from __future__ import annotations
from fastapi import APIRouter, Body, Query, Request from fastapi import APIRouter, Body, Query, Request, Depends
from fastapi.responses import StreamingResponse from fastapi.responses import StreamingResponse
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
from app.core.auth import get_current_user, is_admin
router = APIRouter(prefix="/system", tags=["system"]) router = APIRouter(prefix="/system", tags=["system"])
@@ -32,13 +33,13 @@ def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
@router.get("/permissions/codes") @router.get("/permissions/codes")
def permission_codes() -> dict: def permission_codes(current_user: dict = Depends(get_current_user)) -> dict:
"""返回平台权限码清单(权限码接口)。""" """返回平台权限码清单(权限码接口)。"""
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}} return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
@router.get("/permissions") @router.get("/permissions")
def permissions_overview() -> dict: def permissions_overview(current_user: dict = Depends(get_current_user)) -> dict:
"""返回权限码清单与角色定义。""" """返回权限码清单与角色定义。"""
store = get_platform_store() store = get_platform_store()
return { return {
@@ -59,8 +60,12 @@ def audit_logs(
end_time: str | None = Query(default=None, description="ISO8601 结束时间"), end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200), limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0), offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict: ) -> dict:
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。""" """审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store() store = get_platform_store()
result = store.audit_logs( result = store.audit_logs(
tenant_id=tenant_id, tenant_id=tenant_id,
@@ -85,8 +90,12 @@ def audit_logs_export(
target_type: str | None = Query(default=None, description="目标类型"), target_type: str | None = Query(default=None, description="目标类型"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"), start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"), end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
) -> StreamingResponse: ) -> StreamingResponse:
"""审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。""" """审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store() store = get_platform_store()
result = store.audit_logs( result = store.audit_logs(
tenant_id=tenant_id, tenant_id=tenant_id,

View File

@@ -21,8 +21,10 @@ async def run_compute_poller() -> None:
while True: while True:
try: try:
result = await poll_compute_jobs_once() result = await poll_compute_jobs_once()
if result["synced"] or result["failed"]: if result["failed"]:
logger.info("compute jobs polled", extra={"result": result}) logger.warning("compute polling reported failures", extra={"result": result})
elif result["synced"]:
logger.debug("compute jobs synchronized", extra={"result": result})
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("compute poller stopped") logger.info("compute poller stopped")
raise raise

View File

@@ -25,6 +25,7 @@ from compute.engines.llama_factory.inference import get_inference_session
def create_app() -> FastAPI: def create_app() -> FastAPI:
app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs()) app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs())
jobs: dict[str, dict[str, Any]] = {} jobs: dict[str, dict[str, Any]] = {}
cache_locks: dict[str, asyncio.Lock] = {}
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF" route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training")) process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"))
@@ -620,6 +621,30 @@ def create_app() -> FastAPI:
) )
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items} return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items}
@app.post(f"{route_prefix}/compute/files/upload-to-url")
async def upload_file_to_url(payload: dict[str, Any]) -> dict[str, Any]:
"""Upload one node-local artifact to a Backend-issued presigned URL."""
source = Path(str(payload.get("path") or "")).resolve()
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")).resolve()
if not _path_inside(data_root, source) or not source.is_file():
raise HTTPException(status_code=400, detail="artifact path must be an existing file inside YG_FT_DATA_ROOT")
upload_url = str(payload.get("upload_url") or "")
if not upload_url:
raise HTTPException(status_code=400, detail="upload_url is required")
digest = hashlib.sha256()
byte_size = 0
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
with source.open("rb") as handle:
content = handle.read()
digest.update(content)
byte_size = len(content)
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
response.raise_for_status()
except Exception as exc:
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
return {"status": "available", "path": str(source), "byte_size": byte_size, "checksum_sha256": digest.hexdigest(), "object_key": payload.get("object_key")}
@app.post(f"{route_prefix}/compute/jobs") @app.post(f"{route_prefix}/compute/jobs")
async def create_job(payload: dict[str, Any]) -> dict[str, Any]: async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
payload = {**payload, "require_dataset_files": True} payload = {**payload, "require_dataset_files": True}
@@ -645,7 +670,7 @@ def create_app() -> FastAPI:
"status": "queued", "status": "queued",
"progress": 10, "progress": 10,
"pid": int(52000 + now() % 10000), "pid": int(52000 + now() % 10000),
"gpus": payload.get("gpus") or [0], "gpus": payload.get("gpus") or [],
"created_at": now(), "created_at": now(),
"command": command.command, "command": command.command,
"work_dir": command.work_dir, "work_dir": command.work_dir,
@@ -843,21 +868,46 @@ def create_app() -> FastAPI:
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
temp_target = target.with_name(f".{target.name}.part") temp_target = target.with_name(f".{target.name}.part")
expected_checksum = str(payload.get("checksum_sha256") or "").lower() expected_checksum = str(payload.get("checksum_sha256") or "").lower()
expected_size = int(payload.get("byte_size") or 0)
lock = cache_locks.setdefault(str(target), asyncio.Lock())
async with lock:
if target.is_file() and expected_checksum:
existing_digest = hashlib.sha256()
with target.open("rb") as existing:
while chunk := existing.read(1024 * 1024):
existing_digest.update(chunk)
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
digest = hashlib.sha256() digest = hashlib.sha256()
byte_size = 0 byte_size = 0
try: try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client: async with lock:
async with client.stream("GET", download_url) as response: for attempt in range(3):
response.raise_for_status() try:
with temp_target.open("wb") as output: digest = hashlib.sha256()
async for chunk in response.aiter_bytes(1024 * 1024): byte_size = 0
output.write(chunk) temp_target.unlink(missing_ok=True)
digest.update(chunk) async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
byte_size += len(chunk) async with client.stream("GET", download_url) as response:
response.raise_for_status()
with temp_target.open("wb") as output:
async for chunk in response.aiter_bytes(1024 * 1024):
output.write(chunk)
digest.update(chunk)
byte_size += len(chunk)
break
except Exception:
temp_target.unlink(missing_ok=True)
if attempt == 2:
raise
await asyncio.sleep(2 ** attempt)
checksum = digest.hexdigest() checksum = digest.hexdigest()
if expected_checksum and checksum != expected_checksum: if expected_checksum and checksum != expected_checksum:
temp_target.unlink(missing_ok=True) temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache checksum mismatch") raise HTTPException(status_code=502, detail="cache checksum mismatch")
if expected_size and byte_size != expected_size:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache byte size mismatch")
temp_target.replace(target) temp_target.replace(target)
except HTTPException: except HTTPException:
raise raise

View File

@@ -1,5 +1,19 @@
# Docker 部署说明 # Docker 部署说明
## 跨服务器部署
MinIO、Backend API 和 Compute API 使用各自服务器上的独立 Docker 网络,不能使用跨服务器的容器名称互访。当前 WSL 联调地址为 `172.25.179.69`
```env
# docker/app/.env
MINIO_ENABLED=true
MINIO_ENDPOINT=http://172.25.179.69:19000
COMPUTE_API_BASE_URL=http://172.25.179.69:19100
FILE_GATEWAY_BASE_URL=http://172.25.179.69:19101
```
拆分到不同服务器后,将 `172.25.179.69` 替换为对应服务器 IP。MinIO 容器内部仍使用 `9000/9001`,对外使用 `19000/19001`Backend 和 Compute API 通过外部 IP 访问 MinIO不加入 MinIO 的 Docker 网络。
本目录按应用服务器和算力服务器拆分 Dockerfile 与 Docker Compose 文件。Compose 文件不包含 `build:`,不会在 `docker compose up` 时自动构建业务镜像。所有业务镜像需要先通过手动 `docker build` 构建,再由 Compose 启动。 本目录按应用服务器和算力服务器拆分 Dockerfile 与 Docker Compose 文件。Compose 文件不包含 `build:`,不会在 `docker compose up` 时自动构建业务镜像。所有业务镜像需要先通过手动 `docker build` 构建,再由 Compose 启动。
## 基础镜像 ## 基础镜像

View File

@@ -50,10 +50,14 @@ COMPUTE_POLL_BATCH_SIZE=100
COMPUTE_REQUEST_TIMEOUT_SECONDS=5 COMPUTE_REQUEST_TIMEOUT_SECONDS=5
# MinIO object storage. Enable after the MinIO service is reachable. # MinIO object storage. Enable after the MinIO service is reachable.
MINIO_ENABLED=false MINIO_API_PORT=19000
MINIO_ENDPOINT=http://minio:9000 MINIO_CONSOLE_PORT=19001
MINIO_ENABLED=true
# For split-server deployment, replace host.docker.internal with the MinIO
# server address, for example http://10.10.20.30:19000.
MINIO_ENDPOINT=http://172.25.179.69:19000
MINIO_ACCESS_KEY=minioadmin MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=minioadmin MINIO_SECRET_KEY=change_me_minio_secret
MINIO_BUCKET=yg-ft-resources MINIO_BUCKET=yg-ft-resources
MINIO_SECURE=false MINIO_SECURE=false
STORAGE_WAIT_SECONDS=300 STORAGE_WAIT_SECONDS=300

View File

@@ -63,7 +63,9 @@ services:
COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100} COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100}
COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5} COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5}
MINIO_ENABLED: ${MINIO_ENABLED:-false} MINIO_ENABLED: ${MINIO_ENABLED:-false}
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-http://minio:9000} # Use the storage server's externally reachable address. Do not use a
# MinIO container name because storage is deployed independently.
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-http://host.docker.internal:19000}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources} MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
@@ -75,6 +77,8 @@ services:
- ../../backend:/app:ro - ../../backend:/app:ro
- ../../runtime/app/logs/backend:/opt/yg-ft/logs/backend - ../../runtime/app/logs/backend:/opt/yg-ft/logs/backend
- ../../runtime/app/data:/data/yg-ft - ../../runtime/app/data:/data/yg-ft
extra_hosts:
- "host.docker.internal:host-gateway"
networks: networks:
- yg-ft-app - yg-ft-app
healthcheck: healthcheck:

100256
docker/app/tiktoken/cl100k_base Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,9 @@
MINIO_ACCESS_KEY=minioadmin MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=change_me_minio_secret MINIO_SECRET_KEY=change_me_minio_secret
MINIO_API_PORT=9000 MINIO_API_PORT=19000
MINIO_CONSOLE_PORT=9001 MINIO_CONSOLE_PORT=19001
MINIO_DATA_ROOT_HOST=./data MINIO_DATA_ROOT_HOST=./data
# Backend and Compute API use this externally reachable endpoint when MinIO
# runs on a separate server. Keep the container's internal API port at 9000.
MINIO_EXTERNAL_ENDPOINT=http://<storage-server-ip>:19000

View File

View File

@@ -7,8 +7,8 @@ services:
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin} MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-change_me_minio_secret} MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-change_me_minio_secret}
ports: ports:
- "${MINIO_API_PORT:-9000}:9000" - "${MINIO_API_PORT:-19000}:9000"
- "${MINIO_CONSOLE_PORT:-9001}:9001" - "${MINIO_CONSOLE_PORT:-19001}:9001"
volumes: volumes:
- ${MINIO_DATA_ROOT_HOST:-./data}:/data - ${MINIO_DATA_ROOT_HOST:-./data}:/data
healthcheck: healthcheck:

View File

@@ -0,0 +1,423 @@
# 权限与日志改造功能测试用例
## 1. 测试范围
本文档用于验证本次权限和日志相关开发内容:
- 训练、评测、推理资源权限校验。
- 模型合并和训练模型导出权限。
- 算力节点及 GPU 分配权限。
- 资源所有者、ACL、租户/项目归属字段。
- 资源软删除和审计记录。
- 登录失败限流与 Token 会话有效期。
- 后端轮询日志降噪。
- 数据库初始化和运行时字段迁移。
## 2. 测试环境
| 项目 | 配置 |
|---|---|
| 前端地址 | `http://172.25.179.69:16801` |
| Backend 地址 | `http://172.25.179.69:17861/modelTF` |
| MinIO 地址 | `http://172.25.179.69:19000` |
| WSL IP | `172.25.179.69` |
| 运行方式 | Docker Compose |
| 数据库 | 远程 PostgreSQL |
## 3. 前置数据
准备以下账号和资源:
| 数据 | 要求 |
|---|---|
| 管理员账号 | 具有 `admin` 角色 |
| 普通操作员 | 具有训练、评测或推理页面权限 |
| 只读用户 | 具有 `dashboard``logs` 权限,不具有业务写权限 |
| 测试模型 | 一个基座模型和一个训练模型 |
| 测试数据集 | 一个训练数据集和一个评测数据集 |
| 算力节点 | 至少一个在线节点,最好有 2 张及以上 GPU |
| ACL 资源 | 将测试数据集授权给普通用户进行验证 |
## 4. 鉴权与会话测试
### AUTH-001 登录成功生成会话 Token
**步骤**
1. 使用有效账号调用登录接口或登录页面。
2. 查看响应中的 `token``session_id`
3. 使用 Token 调用 `/modelTF/me`
**预期**
- 登录返回 HTTP 200。
- Token 格式包含用户 ID 和 session ID。
- `/me` 能返回当前用户信息。
- `sessions` 表生成一条未注销记录。
### AUTH-002 无 Token 访问受保护接口
**步骤**
1. 不携带 `Authorization` 调用数据集、模型或训练列表接口。
**预期**
- 返回 HTTP 401。
- 不返回资源数据。
### AUTH-003 注销后 Token 失效
**步骤**
1. 登录并记录 Token、`session_id`
2. 调用注销接口。
3. 使用原 Token 调用 `/me` 或资源接口。
**预期**
- 注销成功。
- `sessions.logout_at` 已写入。
- 原 Token 返回 HTTP 401。
### AUTH-004 登录失败限流
**步骤**
1. 同一客户端 IP 连续输入错误密码 5 次。
2. 第 6 次继续登录。
**预期**
- 前 5 次返回登录失败。
- 第 6 次返回 HTTP 429。
- 使用正确密码也应在冷却窗口内被限制。
- 登录成功后失败计数清除。
## 5. 资源所有权与 ACL
### ACL-001 资源所有者访问自己的资源
**步骤**
1. 普通用户创建数据集或模型。
2. 使用该用户查看列表和详情。
3. 修改资源元数据。
**预期**
- 资源出现在自己的列表中。
- 详情访问返回 HTTP 200。
- 资源所有者可以执行允许的写操作。
### ACL-002 未授权用户访问资源
**步骤**
1. 用户 A 创建数据集。
2. 用户 B 未获得 ACL 授权时查看该数据集详情。
**预期**
- 用户 B 不应在列表中看到该资源。
- 直接访问详情返回 HTTP 403。
### ACL-003 ACL 授权后访问资源
**步骤**
1. 用户 A 或管理员给用户 B 授予 `read` 权限。
2. 用户 B 刷新资源列表并访问详情。
**预期**
- 用户 B 可以看到并读取资源。
- 用户 B 不能执行 `write``delete``execute` 操作。
### ACL-004 非所有者修改 ACL
**步骤**
1. 用户 B 仅拥有资源 `read` 权限。
2. 用户 B 调用 ACL 修改接口。
**预期**
- 返回 HTTP 403。
- ACL 内容不发生变化。
### ACL-005 ACL 修改审计
**步骤**
1. 管理员或资源所有者修改 ACL。
2. 查询审计日志。
**预期**
- 出现 `resource.acl.set` 操作记录。
- 记录操作者、资源类型、资源 ID 和变更详情。
## 6. 训练权限测试
### TRAIN-001 训练创建校验模型和数据集权限
**步骤**
1. 普通用户选择无权限的基座模型或数据集创建训练任务。
2. 再使用已授权模型和数据集创建训练任务。
**预期**
- 无权限资源返回 HTTP 403。
- 已授权资源允许创建任务。
- 任务记录包含 `created_by`
### TRAIN-002 GPU 授权校验
**步骤**
1. 管理员将 GPU 0 分配给用户 A。
2. 用户 A 选择 GPU 0 创建训练任务。
3. 用户 A 选择未分配的 GPU 1 创建训练任务。
**预期**
- GPU 0 可以提交。
- GPU 1 返回 HTTP 403。
- 未指定 GPU 时,仅从用户已授权的空闲 GPU 中自动分配。
### TRAIN-003 训练任务停止、重试和删除权限
**步骤**
1. 用户 A 创建训练任务。
2. 用户 B 尝试停止、重试或删除该任务。
3. 管理员执行相同操作。
**预期**
- 用户 B 无资源权限时返回 HTTP 403。
- 删除和停止等高风险操作按配置进入审批流程。
- 管理员可以旁路审批执行。
- GPU 占用在停止、失败和删除后释放。
## 7. 评测权限测试
### EVAL-001 评测创建联合权限
**步骤**
1. 普通用户选择无权使用的模型创建评测。
2. 普通用户选择无权使用的数据集创建评测。
3. 使用同时拥有权限的模型和数据集创建评测。
**预期**
- 模型无权限返回 HTTP 403。
- 数据集无权限返回 HTTP 403。
- 两个资源均有 `execute` 权限时允许创建。
- 评测任务包含 `created_by`、模型、数据集和算力节点信息。
### EVAL-002 评测详情和删除权限
**预期**
- 无权限用户不能查看评测详情。
- 评测删除需要 `delete` 权限。
- 非管理员删除高风险评测任务时触发审批。
## 8. 推理权限测试
### INFER-001 推理创建模型权限
**步骤**
1. 普通用户选择无权模型创建推理任务。
2. 选择已授权模型创建推理任务。
**预期**
- 无权模型返回 HTTP 403。
- 有权模型允许创建。
- 推理任务包含 `created_by`
### INFER-002 推理加载和卸载权限
**步骤**
1. 用户 A 创建推理任务。
2. 用户 B 调用加载、卸载接口。
3. 用户 A 执行加载和卸载。
**预期**
- 用户 B 返回 HTTP 403。
- 用户 A 可以执行授权范围内的加载和卸载。
- 卸载后 GPU 和节点状态恢复为空闲。
### INFER-003 推理任务删除审批
**预期**
- 非管理员删除他人推理任务被拒绝或进入审批。
- 管理员可以直接删除。
- 删除操作有审计日志。
## 9. 模型合并与导出测试
### MODEL-001 权重合并权限
**步骤**
1. 普通用户选择无权限训练模型进行合并。
2. 选择已授权训练模型进行合并。
**预期**
- 无权限返回 HTTP 403。
- 已授权训练模型允许合并。
- 如指定基座模型,还必须拥有基座模型 `execute` 权限。
### MODEL-002 训练模型删除权限
**预期**
- 只有资源所有者、ACL 授权用户或管理员可操作。
- 删除采用软删除。
- `deleted_at``deleted_by` 被写入。
### MODEL-003 导出任务访问权限
**预期**
- 无权用户不能查看训练模型导出任务。
- 有权用户可以查看导出状态。
- 导出动作应记录 `trained_model.export` 审计日志。
## 10. 算力节点与 GPU 管理测试
### GPU-001 普通用户节点可见范围
**预期**
- 普通用户只能看到被分配 GPU 所在节点。
- 普通用户只能看到已授权 GPU。
- 管理员可看到所有节点和 GPU。
### GPU-002 节点管理接口权限
验证节点创建、修改、删除、启用、禁用、排空、连接测试和健康检查。
**预期**
- 普通用户全部返回 HTTP 403。
- 管理员操作成功。
### GPU-003 多卡自动分配
**步骤**
1. 节点配置多张 GPU。
2. 启动一个任务占用其中一张卡。
3. 再启动任务并选择剩余卡。
**预期**
- 已占用 GPU 不再出现在可选列表。
- 剩余 GPU 可以被其他任务使用。
- 任务失败、停止或完成后 GPU 释放。
## 11. 软删除与数据库测试
### DB-001 初始化字段检查
执行:
```sql
SELECT table_name, column_name
FROM information_schema.columns
WHERE table_name IN ('models', 'datasets', 'trained_models', 'eval_tasks', 'sessions')
ORDER BY table_name, ordinal_position;
```
**预期字段**
- 资源表存在 `created_by``tenant_id``project_id``deleted_at``deleted_by`
- `sessions` 存在 `issued_at``expires_at``logout_at`
### DB-002 软删除列表过滤
**步骤**
1. 删除模型、数据集或评测任务。
2. 查询列表。
3. 直接查询数据库记录。
**预期**
- 前端列表不再显示已删除资源。
- 数据库记录仍存在。
- `deleted_at``deleted_by` 有值。
## 12. 日志降噪测试
### LOG-001 正常轮询日志
**步骤**
1. 重启 Backend 容器。
2. 连续观察 1 分钟日志。
```bash
docker logs -f --tail=200 yg-ft-backend-api
```
**预期**
- 不再每次以 `INFO` 输出 `compute jobs polled`
- 健康检查成功请求不再以 `INFO` 输出应用日志。
- 正常任务同步日志仅在 `DEBUG` 级别出现。
### LOG-002 异常轮询日志
**步骤**
1. 临时停止算力节点或断开节点网络。
2. 观察 Backend 日志。
**预期**
- 轮询失败以 `WARNING``ERROR` 输出。
- 异常包含节点、任务或错误原因。
- 恢复节点后轮询继续工作。
## 13. 容器验证命令
```bash
docker compose ps
docker logs --tail=200 yg-ft-backend-api
curl -i http://172.25.179.69:17861/modelTF/health
curl -i http://172.25.179.69:16801/
```
预期 Backend 和 Frontend 均为 `healthy`,健康接口返回 HTTP 200。
## 14. 测试结果记录
| 用例编号 | 测试结果 | 实际结果 | 缺陷编号 | 测试人 | 日期 |
|---|---|---|---|---|---|
| AUTH-001 | □通过 □失败 | | | | |
| AUTH-002 | □通过 □失败 | | | | |
| AUTH-003 | □通过 □失败 | | | | |
| AUTH-004 | □通过 □失败 | | | | |
| ACL-001 | □通过 □失败 | | | | |
| TRAIN-001 | □通过 □失败 | | | | |
| TRAIN-002 | □通过 □失败 | | | | |
| EVAL-001 | □通过 □失败 | | | | |
| INFER-001 | □通过 □失败 | | | | |
| MODEL-001 | □通过 □失败 | | | | |
| GPU-001 | □通过 □失败 | | | | |
| DB-001 | □通过 □失败 | | | | |
| LOG-001 | □通过 □失败 | | | | |
| LOG-002 | □通过 □失败 | | | | |

View File

@@ -0,0 +1,122 @@
# 权限与日志改造自动化测试结果
## 1. 测试时间
2026-08-12
## 2. 测试环境
| 项目 | 地址/状态 |
|---|---|
| Frontend | `http://172.25.179.69:16801` |
| Backend | `http://172.25.179.69:17861/modelTF` |
| WSL | `172.25.179.69` |
| Docker | 已运行 |
## 3. 自动执行结果
| 用例 | 结果 | 实际结果 |
|---|---|---|
| 容器状态 | 通过 | Compute、Frontend、Backend、Redis、MinIO 均为 `healthy` |
| Backend 健康检查 | 通过 | `/modelTF/health` 返回 HTTP 200 |
| Frontend 首页 | 通过 | `/` 返回 HTTP 200 |
| 未登录访问数据集接口 | 通过 | `/modelTF/dataset-manage` 返回 HTTP 401 |
| Redis 容器状态 | 通过 | 容器状态为 healthy |
| 轮询正常日志降噪 | 通过 | 最近 120 秒未发现 `compute jobs polled``health check requested` 高频日志 |
| 轮询异常日志保留 | 未触发 | 当前未人为停止算力节点,未产生轮询失败日志 |
| 前端构建 | 通过 | `npm run build` 成功 |
| 后端编译 | 通过 | 相关 Python 模块 `py_compile` 成功 |
| Git 差异检查 | 通过 | `git diff --check` 无格式错误 |
## 4. 数据库字段迁移
### 静态检查结果
初始化 SQL 和运行时迁移逻辑已包含以下字段:
- `models.deleted_at`
- `models.deleted_by`
- `models.tenant_id`
- `models.project_id`
- `datasets.deleted_at`
- `datasets.deleted_by`
- `datasets.tenant_id`
- `datasets.project_id`
- `trained_models.deleted_at`
- `trained_models.deleted_by`
- `trained_models.tenant_id`
- `trained_models.project_id`
- `eval_tasks.deleted_at`
- `eval_tasks.deleted_by`
- `sessions.issued_at`
- `sessions.expires_at`
### 远程数据库检查状态
已通过当前 Backend 容器使用一次性 PostgreSQL 连接检查远程数据库,字段迁移已完成。
确认存在:
- `models`: `created_by``tenant_id``project_id``deleted_at``deleted_by`
- `datasets`: `created_by``tenant_id``project_id``deleted_at``deleted_by`
- `trained_models`: `created_by``tenant_id``project_id``deleted_at``deleted_by`
- `eval_tasks`: `created_by``tenant_id``project_id``deleted_at``deleted_by`
- `sessions`: `issued_at``expires_at``logout_at`
使用的检查 SQL
```sql
SELECT table_name, column_name
FROM information_schema.columns
WHERE table_name IN ('models', 'datasets', 'trained_models', 'eval_tasks', 'sessions')
ORDER BY table_name, ordinal_position;
```
## 5. Redis 说明
已从当前环境确认 Redis 实际密码配置为 `Tvhrf659WaX-S1B8FG6c2kSZK07XTv82`,使用该密码验证返回 `PONG`
项目配置已同步为:
```env
REDIS_PASSWORD=Tvhrf659WaX-S1B8FG6c2kSZK07XTv82
REDIS_URL=redis://:Tvhrf659WaX-S1B8FG6c2kSZK07XTv82@redis:6379/0
```
后续验证应从当前 Docker 环境读取实际密码后执行:
```bash
docker compose config
docker exec yg-ft-redis printenv REDISCLI_AUTH
docker exec yg-ft-redis redis-cli -a "$REDISCLI_AUTH" ping
```
预期返回:
```text
PONG
```
## 6. 未自动执行的破坏性用例
以下用例需要测试账号、测试资源或人工确认,未自动执行,以避免影响现有数据:
- ACL 授权和撤销。
- 训练任务创建、停止、重试和删除。
- 评测任务启动和删除。
- 推理模型加载、卸载和删除。
- 模型权重合并和导出。
- GPU 分配、占用和释放。
- 软删除后资源恢复和数据完整性。
- 人为停止算力节点验证轮询失败恢复。
- 登录失败 5 次后的限流验证。
- Token 过期和注销后的访问验证。
## 7. 当前结论
当前服务基础可用,页面和 Backend 健康接口正常,未登录鉴权正常,日志降噪逻辑生效,前后端构建通过。
当前仍需人工或使用专用测试数据验证:
1. 权限矩阵中的创建、执行、删除、审批和 GPU 占用场景。
2. Token 过期和登录失败限流场景。

View File

@@ -803,3 +803,88 @@ async function loadGpus() {
4. **前端适配**GPU 下拉过滤、资源授权按钮、权限管理页面优化 4. **前端适配**GPU 下拉过滤、资源授权按钮、权限管理页面优化
5. **审批扩展**:在删除/停止接口中接入 `_require_approval_or_admin` 5. **审批扩展**:在删除/停止接口中接入 `_require_approval_or_admin`
6. **测试补充**:扩展 `test_governance.py` 覆盖 GPU 分配、资源过滤、审批扩展场景 6. **测试补充**:扩展 `test_governance.py` 覆盖 GPU 分配、资源过滤、审批扩展场景
## 16. 2.0 权限增强基线MinIO、缓存与跨节点场景
### 16.1 默认拒绝
所有受保护接口采用 deny by default。权限判定依次执行认证、租户边界、项目成员关系、页面权限、资源动作权限、审批校验和审计记录。任何一步无法确定时返回 403不得因为字段缺失、资源不存在或 ACL 查询异常而自动放行。
### 16.2 资源归属和继承
资源统一使用 `tenant_id``project_id``created_by``visibility` 表达边界。训练任务继承模型、数据集和项目边界;训练模型继承训练任务边界;合并模型继承被合并模型边界;评测必须同时校验模型和数据集;推理必须校验模型、项目和算力节点。
### 16.3 MinIO 对象权限
- 所有对象访问必须经过 Backend 鉴权,前端不得持有 MinIO 密钥。
- Compute API 只使用 Backend 签发的预签名 URL。
- 预签名 URL 默认有效期不超过 15 分钟,上传和下载分别签发。
- 生成 URL 前必须校验资源权限、对象状态和版本归属。
- bucket 由服务端配置,禁止客户端提交任意 bucket。
- 禁止通过修改 `object_key``version_id` 或文件名越权访问对象。
- 删除对象使用 `deleting -> deleted` 状态,失败时保留错误信息并可重试。
### 16.4 缓存和算力节点权限
- 缓存是资源副本,不产生新的资源所有权。
- 只有拥有源模型或数据集 `execute` 权限的用户才能触发缓存。
- 用户不能直接调用 Compute API 的缓存、文件、上传和推理管理接口。
- Compute API 只接受 Backend 服务令牌,不能转发用户 Token。
- 产物归档必须校验任务、节点、资源和项目关联关系。
- 任务运行期间缓存引用不可被普通用户清理。
- 缓存清理只能删除节点副本,不得删除 MinIO 正式对象。
### 16.5 训练、合并、推理和评测动作矩阵
| 动作 | 必要权限 | 额外约束 |
|---|---|---|
| 创建训练 | 项目 `write` + 模型/数据集 `execute` | GPU、配额和节点权限同时通过 |
| 查看训练 | 任务 `read` | 日志、曲线、checkpoint 继承任务权限 |
| 停止训练 | 任务 `write``admin` | 停止他人任务需要审批或管理员权限 |
| 权重合并 | 训练模型 `execute` | 使用绑定节点或有权限的指定节点 |
| 归档训练产物 | 任务 `write` | 只能归档任务输出目录内文件 |
| 创建推理 | 模型 `execute` | 节点、GPU 配额和项目权限通过 |
| 删除推理 | 推理服务 `delete` 或管理员 | 释放 GPU 和缓存引用 |
| 创建评测 | 模型/数据集 `execute` | 两个资源必须在允许范围内 |
| 下载报告 | 评测任务 `read` + 报告 `download` | 预签名 URL 短时有效 |
### 16.6 服务身份和密钥边界
| 身份 | 用途 | 禁止事项 |
|---|---|---|
| 用户 Token | 调用 Backend 业务接口 | 直接调用 Compute 或 MinIO |
| Backend 服务令牌 | 调用 Compute API | 返回前端或写入任务参数 |
| MinIO 管理密钥 | Backend 对象操作 | 注入浏览器、Compute 容器或日志 |
| Compute 节点身份 | 节点心跳和任务执行 | 访问其他节点本地路径 |
生产环境禁止使用默认凭据;服务令牌必须从环境变量或密钥管理系统读取并脱敏记录。
### 16.7 必须补充的接口保护
```text
POST /storage/objects/presign
POST /storage/resources/{type}/{id}/prepare/{node_id}
POST /storage/resources/{type}/{id}/archive-node/{node_id}
GET /storage/cache/jobs/{node_id}
POST /model-manage/merge
POST /model-chat/local/preload
POST /model-chat/trained/preload
POST /model-chat/local/unload
POST /model-compare/{task_id}/load
POST /model-compare/{task_id}/unload
```
请求体中的所有资源 ID 都必须校验。`node_id``model_id``dataset_id``task_id` 不一致时返回 403 或 409。
### 16.8 权限审计验收用例
必须覆盖:用户 A 不能读取用户 B 资源;项目 A 不能使用项目 B 数据集;评测必须同时拥有模型和数据集 `execute` 权限;修改对象 key、版本或资源 ID 不能获取预签名 URL普通用户不能调用 Compute API产物路径不能越出任务数据根目录用户不能清理其他项目正在使用的缓存无节点权限时推理返回 403删除、停止、合并和归档他人资源按规则进入审批。
## 17. 2.0 实施顺序
1. 建立统一 `authorize_resource_action()``authorize_task_resources()` 后端辅助函数。
2. 为模型、数据集、训练、推理、评测和对象接口补齐租户/项目过滤。
3. 统一预签名 URL 权限校验、有效期和审计日志。
4. 为 Compute API 增加服务令牌、节点归属和路径范围校验。
5. 将 GPU、缓存、配额和节点选择校验合并到任务创建事务中。
6. 增加跨资源权限测试和越权回归测试。
7. 前端仅负责隐藏操作按钮,最终权限以 Backend 返回为准。

View File

@@ -1,4 +1,4 @@
<script setup lang="ts"> <script setup lang="ts">
import { ref, reactive, computed, onMounted } from 'vue' import { ref, reactive, computed, onMounted } from 'vue'
import { useRouter } from 'vue-router' import { useRouter } from 'vue-router'
import { ElMessage, type FormInstance, type FormRules } from 'element-plus' import { ElMessage, type FormInstance, type FormRules } from 'element-plus'

View File

@@ -100,7 +100,7 @@ export function buildFineTunePayload(
} }
export function buildFineTuneCommand(form: FineTuneFormModel, gpus: number[]) { export function buildFineTuneCommand(form: FineTuneFormModel, gpus: number[]) {
const gpuIds = gpus.length ? gpus.join(',') : '0' const gpuIds = gpus.length ? gpus.join(',') : '<auto>'
const stage = form.train_type === 'DPO' ? 'dpo' : form.train_type === 'CPT' ? 'pt' : 'sft' const stage = form.train_type === 'DPO' ? 'dpo' : form.train_type === 'CPT' ? 'pt' : 'sft'
const lines = [ const lines = [
`CUDA_VISIBLE_DEVICES=${gpuIds} llamafactory-cli train`, `CUDA_VISIBLE_DEVICES=${gpuIds} llamafactory-cli train`,