diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..021580f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: yg-ft-ci + +on: + push: + branches: [main, master, develop] + pull_request: + +jobs: + frontend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - run: npm run build + + backend: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r backend/requirements.txt + - run: python -m compileall -q backend/app compute + - run: python -m pytest -q backend/tests/test_storage_security.py + env: + PYTHONPATH: backend diff --git a/.gitignore b/.gitignore index 44aa534..7f3bdbb 100644 --- a/.gitignore +++ b/.gitignore @@ -203,6 +203,7 @@ docker/llamafactory-latest.tar.gz docker/compute/data/yg-ft/datasets/* docker/compute/data/yg-ft/models/* docker/compute/data/yg-ft/outputs/* +docker/compute/data/yg-ft/trained_models/* docker/compute/data/yg-ft/logs/** !docker/compute/data/yg-ft/logs/compute/ !docker/compute/data/yg-ft/logs/training/ @@ -215,3 +216,7 @@ docker/offline/ # MinIO object storage data - 对象存储运行时数据，勿提交，保留目录结构 docker/minio/data/* !docker/minio/data/.gitkeep + +# nlp-eval-demo - 独立演示项目，不进版本库 +nlp-eval-demo/ +nlp-eval-demo.zip diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index 8c68518..0b4c052 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -29,12 +29,18 @@ from fastapi import ( Header, HTTPException, Query, + Request, UploadFile, ) from fastapi.responses import StreamingResponse from psycopg.rows import dict_row -from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin +from app.core.auth import ( + filter_accessible_resource_ids, + get_current_user, + has_resource_access, + is_admin, +) from app.core.config import get_settings from app.db.platform_store import get_platform_store from app.modules.data_process.algorithms import ( @@ -110,7 +116,6 @@ from app.schemas.data_process import ( ResultUpdate, ) -router = APIRouter(prefix="/data-process") logger = logging.getLogger(__name__) MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024 MAX_SOURCE_FILE_COUNT = 20 @@ -163,6 +168,27 @@ def fail(status_code: int, message: str) -> HTTPException: ) +def _authorize_data_process_request( + request: Request, + task_id: str | None = None, + current_user: dict[str, Any] = Depends(get_current_user), +) -> None: + """Apply resource ACL and tenant checks to every task-scoped endpoint.""" + if not task_id or is_admin(current_user): + return + permission = "read" if request.method in {"GET", "HEAD"} else "write" + if any(marker in request.url.path for marker in ("/start", "/generate", "/regenerate", "/repeat", "/publish", "/external/")): + permission = "execute" + if not has_resource_access("data_process", task_id, current_user, permission): + raise fail(403, "no permission to access this data process task") + + +router = APIRouter( + prefix="/data-process", + dependencies=[Depends(_authorize_data_process_request)], +) + + @contextmanager def api_errors() -> Iterator[None]: try: @@ -887,7 +913,7 @@ def list_tasks( keyword=keyword, status=status, process_type=process_type, - tenant_id=tenant_id, + tenant_id=tenant_id if is_admin(current_user) else (current_user.get("tenant_id") or "default"), project_id=project_id, ) # #4 资源 ACL 过滤：admin 放行，普通用户只看到自己被授权的数据处理任务 @@ -908,9 +934,15 @@ def list_tasks( def create_task( payload: DataProcessTaskCreate, store: DataProcessStore = Depends(get_data_process_store), + current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: with api_errors(): - task = store.create_task(payload.model_dump(mode="json")) + values = payload.model_dump(mode="json") + values["created_by"] = current_user.get("id") + values["owner_id"] = current_user.get("id") + values["tenant_id"] = current_user.get("tenant_id") or "default" + get_platform_store().assert_active_tenant(values["tenant_id"]) + task = store.create_task(values) return ok(task, "data process task created") diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py index 34a77ba..d7dc726 100644 --- a/backend/app/api/v1/endpoints/platform.py +++ b/backend/app/api/v1/endpoints/platform.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import asyncio import hashlib +import re import uuid import time from io import BytesIO @@ -13,17 +14,22 @@ from urllib.parse import quote from zipfile import ZIP_DEFLATED, ZipFile from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile -from fastapi.responses import PlainTextResponse, Response, StreamingResponse +from fastapi.responses import Response, StreamingResponse import httpx -from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin +from app.core.auth import bind_active_tenant, filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin, require_admin, resource_record from app.core.config import get_settings from app.core.audit import audit_log, AuditActions from app.core.op_log import op_log, OpModule, OpAction -from app.db.platform_store import get_platform_store +from app.db.platform_store import get_platform_store, utcnow from app.modules.compute_gateway.client import ComputeNodeClient -from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once +from app.modules.compute_gateway.sync import ( + _archive_node_directory, + fetch_eval_progress_content, + fetch_eval_result_content, + poll_compute_jobs_once, +) from app.modules.storage.minio_store import ObjectStorageError, get_object_storage from app.modules.storage.policy import should_store_in_minio @@ -32,6 +38,150 @@ _LOGIN_FAILURES: dict[str, list[float]] = {} _DASHBOARD_CACHE_TTL = 5.0 _DASHBOARD_CACHE: dict[str, Any] = {} +_STORAGE_RESOURCE_ROOTS = { + "dataset": "datasets", + "model": "models", + "trained_model": "trained_models", + "eval": "evaluations", + "fine-tune": "fine-tunes", + "fine_tune_task": "fine-tunes", + "compare": "comparisons", + "inference": "comparisons", + "data_process": "data-process", + "data_convert": "data-convert", +} +_STORAGE_RESOURCE_TABLES = { + "dataset": "datasets", + "model": "models", + "trained_model": "trained_models", + "eval": "eval_tasks", + "fine-tune": "fine_tune_tasks", + "fine_tune_task": "fine_tune_tasks", + "compare": "compare_tasks", + "inference": "compare_tasks", + "data_process": "data_process_tasks", + "data_convert": "data_convert_tasks", +} +_STORAGE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") +_STORAGE_CONTENT_TYPES = { + "application/octet-stream", + "application/json", + "application/jsonl", + "text/plain", + "text/csv", + "application/pdf", + "application/zip", + "application/gzip", + "application/x-gzip", +} +_MODEL_SECRET_FIELDS = {"api_key", "secret_key", "password", "token", "access_token"} + + +def _public_model(model: dict[str, Any]) -> dict[str, Any]: + """Return a model DTO that never exposes provider credentials.""" + result = { + key: value for key, value in dict(model).items() if key not in _MODEL_SECRET_FIELDS + } + result["api_key_configured"] = bool(model.get("api_key")) + return result + + +def _public_models(models: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [_public_model(model) for model in models] + + +def _storage_resource_row(store: Any, resource_type: str, resource_id: str) -> dict[str, Any] | None: + table = _STORAGE_RESOURCE_TABLES.get(resource_type) + if not table: + return None + with store.connect() as conn: + row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (resource_id,)).fetchone() + if not row: + return None + item = dict(row) + if item.get("deleted_at"): + return None + return item + + +def _validate_storage_key( + resource_type: str, + resource_id: str, + version_id: str, + object_key: str | None, + file_name: str | None, +) -> str: + root = _STORAGE_RESOURCE_ROOTS[resource_type] + prefix = f"{root}/{resource_id}/versions/{version_id}/" + requested = str(object_key or "").replace("\\", "/").strip() + if not requested: + safe_name = Path(str(file_name or "resource")).name or "resource" + requested = f"{prefix}{safe_name}" + if ( + not requested + or requested.startswith("/") + or "\x00" in requested + or any(part in {"", ".", ".."} for part in requested.split("/")) + or not requested.startswith(prefix) + or len(requested) > 1024 + ): + raise ValueError("object_key must stay inside the resource version prefix") + return requested + + +def _capture_resource_versions(store: Any, payload: dict[str, Any]) -> dict[str, Any]: + """Freeze the resource versions used by a task at task-creation time.""" + snapshot: dict[str, Any] = {} + dataset_id = str(payload.get("dataset_id") or payload.get("train_dataset_id") or "") + if dataset_id: + try: + dataset = store.dataset(dataset_id) + snapshot["dataset"] = { + "id": dataset_id, + "files": [ + { + "id": item.get("id"), + "active_version_id": item.get("active_version_id") or item.get("current_version_id") or item.get("id"), + "name": item.get("name"), + } + for item in dataset.get("files") or [] + ], + } + except KeyError: + pass + + raw_model_ids = payload.get("model_ids") or payload.get("models") or [ + payload.get("model_id") or payload.get("base_model_id") or payload.get("base_model"), + payload.get("trained_model_id"), + ] + model_ids = [ + str(item.get("model_id") or item.get("id") or item.get("name")) if isinstance(item, dict) else str(item) + for item in raw_model_ids + if item + ] + model_snapshots: list[dict[str, Any]] = [] + for model_id in dict.fromkeys(model_ids): + try: + model = store.model(model_id) + except KeyError: + model = next((item for item in store.trained_models() if str(item.get("id")) == model_id or str(item.get("name")) == model_id), None) + if not model: + continue + objects = store.storage_objects_for_resource( + "trained_model" if str(model.get("id") or "").startswith("tm_") else "model", + str(model.get("id") or model_id), + ) + model_snapshots.append( + { + "id": str(model.get("id") or model_id), + "name": model.get("name"), + "storage_versions": sorted({str(item.get("version_id")) for item in objects if item.get("version_id")}), + } + ) + if model_snapshots: + snapshot["models"] = model_snapshots + return snapshot + def _cached_dashboard(key: str) -> dict[str, Any] | None: item = _DASHBOARD_CACHE.get(key) @@ -226,6 +376,73 @@ def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) -> return preferred + others +async def _archive_model_from_source_node( + store: Any, + model: dict[str, Any], + preferred_node_id: str | None = None, +) -> list[dict[str, Any]]: + """Archive a registered base model from any reachable node into MinIO. + + A model path is a legacy compute-host path, not ownership metadata. The + source node is discovered once, then every consumer uses the MinIO object + manifest and its own local cache. + """ + model_id = str(model.get("id") or "") + source_path = str(model.get("path") or "") + if not model_id or not source_path: + return [] + existing = store.storage_objects_for_resource("model", model_id) + if existing: + return existing + if not get_settings().minio_enabled: + return [] + nodes = _candidate_online_nodes(store, preferred_node_id) + last_error = "no reachable source compute node" + for source_node in nodes: + client = ComputeNodeClient(source_node["api_base_url"], timeout=30) + try: + check = await client.check_paths([{"path": source_path, "type": "dir", "required": True}]) + if not check.get("valid") or not all(item.get("ok") for item in check.get("items") or []): + continue + version_id = str(model.get("storage_version_id") or "initial") + store.update_model_storage(model_id, "archiving", "", version_id) + archived = await _archive_node_directory( + store, + ComputeNodeClient(source_node["api_base_url"], timeout=900), + source_node, + source_path, + "model", + model_id, + version_id, + f"models/{model_id}", + ) + if archived: + store.update_model_storage(model_id, "available", "", version_id) + return store.storage_objects_for_resource("model", model_id) + last_error = "model directory contains no files" + except Exception as exc: # noqa: BLE001 - persist status and try another node + last_error = str(exc) + continue + try: + store.update_model_storage(model_id, "failed", last_error) + except Exception: + pass + return [] + + +async def _archive_model_in_background(model_id: str) -> None: + """Best-effort asynchronous import for newly registered local models.""" + store = get_platform_store() + try: + model = store.model(model_id) + await _archive_model_from_source_node(store, model) + except Exception as exc: # noqa: BLE001 - status is persisted by helper when possible + try: + store.update_model_storage(model_id, "failed", str(exc)) + except Exception: + pass + + async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None: """Prepare MinIO resource files on a node and return the local directory.""" if not get_settings().minio_enabled or not resource_id: @@ -253,7 +470,9 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: or "" ) resolved_id = str((resource or {}).get("id") or resource_id) - if source_path and resolved_id: + if source_path and resolved_id and resource_type == "model": + objects = await _archive_model_from_source_node(store, resource or {"id": resolved_id, "path": source_path}, node.get("id")) + elif source_path and resolved_id: client = ComputeNodeClient(node["api_base_url"], timeout=900) await _archive_node_directory( store, @@ -281,7 +500,7 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: suffix = suffix.split("/", 1)[1] if suffix: relative_name = suffix - await client.prepare_cache({ + result = await client.prepare_cache({ "resource_id": resource_id, "version_id": obj["version_id"], "download_url": get_object_storage().presigned_get(object_key), @@ -289,6 +508,16 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: "byte_size": obj.get("byte_size") or 0, "relative_path": f"{root_name}/{resource_id}/{relative_name}", }) + store.upsert_resource_replica( + node["id"], + resource_type, + resource_id, + str(result.get("local_path") or f"/data/yg-ft/{root_name}/{resource_id}"), + version_id=str(obj.get("version_id") or ""), + storage_object_id=str(obj.get("id") or ""), + status="available", + sync_status="synced", + ) return f"/data/yg-ft/{root_name}/{resource_id}" @@ -394,6 +623,7 @@ def _require_approval_or_admin( resource_id: str, current_user: dict[str, Any], action_desc: str = "", + action: str = "resource.change", ) -> dict[str, Any] | None: """ 高风险操作审批旁路： @@ -409,11 +639,21 @@ def _require_approval_or_admin( if _check_owner(resource_type, resource_id, current_user.get("id")): return None store = get_platform_store() + if store.consume_approved_approval( + resource_type, + resource_id, + str(current_user.get("id") or ""), + action, + ): + return None instance = store.create_approval_instance({ "resource_type": resource_type, "resource_id": resource_id, "applicant_id": current_user.get("id"), "template_id": None, + "action": action, + "tenant_id": current_user.get("tenant_id") or "default", + "reason": action_desc, }) return { "code": 202, @@ -685,56 +925,64 @@ async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> _LOGIN_FAILURES[ip] = [*recent, now] raise fail(401, "invalid username or password") _LOGIN_FAILURES.pop(ip, None) - sess = store.create_session(user["id"], ip=None) + sess = store.create_session(user["id"], ip=ip) return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]}) @router.post("/logout") -async def logout(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def logout(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: store = get_platform_store() session_id = payload.get("session_id", "") if session_id: - store.finish_session(session_id) + try: + store.finish_session(session_id, current_user.get("id")) + except TypeError: + store.finish_session(session_id) return ok(None) @router.get("/me") -async def me(request: Request) -> dict[str, Any]: - """根据 Authorization header 中的 token 返回当前登录用户信息""" - store = get_platform_store() - auth = request.headers.get("Authorization", "") - token = auth.replace("Bearer ", "").strip() - # token 格式: platform-token-{user_id} - if token.startswith("platform-token-"): - user_id = token[len("platform-token-"):].split(".", 1)[0] - for u in store.users(): - if u.get("id") == user_id: - return ok(u) - raise fail(401, "invalid or missing token") +async def me(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + """Return the current session user after the normal token/session checks.""" + return ok(current_user) @router.get("/dashboard/overview") -async def dashboard_overview() -> dict[str, Any]: - cached = _cached_dashboard("overview") +async def dashboard_overview(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + cache_key = f"overview:{current_user.get('id') or current_user.get('role')}" + cached = _cached_dashboard(cache_key) if cached is not None: return cached store = get_platform_store() tasks = store.tasks() - return _store_dashboard_cache("overview", ok( + datasets = store.datasets(None if is_admin(current_user) else current_user.get("tenant_id") or "default") + compares = store.compare_tasks() + if not is_admin(current_user): + task_ids = filter_accessible_resource_ids_batch("fine-tune", [str(item.get("id") or "") for item in tasks], current_user) + dataset_ids = filter_accessible_resource_ids_batch("dataset", [str(item.get("id") or "") for item in datasets], current_user) + compare_ids = filter_accessible_resource_ids_batch("compare", [str(item.get("id") or "") for item in compares], current_user) + tasks = [item for item in tasks if str(item.get("id") or "") in task_ids] + datasets = [item for item in datasets if str(item.get("id") or "") in dataset_ids] + compares = [item for item in compares if str(item.get("id") or "") in compare_ids] + nodes = [item for item in store.compute_nodes() if any(gpu.get("node_id") == item.get("id") for gpu in store.gpu_assignments_for_user(current_user.get("id")))] + else: + nodes = store.compute_nodes() + return _store_dashboard_cache(cache_key, ok( { "models": len(store.models()), - "datasets": len(store.datasets()), + "datasets": len(datasets), "fine_tune_tasks": len(tasks), "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]), - "compute_nodes": len(store.compute_nodes()), + "compute_nodes": len(nodes), "gpus": len(store.gpus()), } )) @router.get("/dashboard/stats") -async def dashboard_stats() -> dict[str, Any]: - cached = _cached_dashboard("stats") +async def dashboard_stats(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + cache_key = f"stats:{current_user.get('id') or current_user.get('role')}" + cached = _cached_dashboard(cache_key) if cached is not None: return cached """看板聚合数据：基于平台真实数据；缺项做合理近似。""" @@ -743,7 +991,19 @@ async def dashboard_stats() -> dict[str, Any]: users = store.users() nodes = store.compute_nodes() datasets = store.datasets() + compares = store.compare_tasks() eval_tasks = store.eval_tasks() + if not is_admin(current_user): + task_ids = filter_accessible_resource_ids_batch("fine-tune", [str(item.get("id") or "") for item in tasks], current_user) + dataset_ids = filter_accessible_resource_ids_batch("dataset", [str(item.get("id") or "") for item in datasets], current_user) + eval_ids = filter_accessible_resource_ids_batch("eval", [str(item.get("id") or "") for item in eval_tasks], current_user) + compare_ids = filter_accessible_resource_ids_batch("compare", [str(item.get("id") or "") for item in compares], current_user) + tasks = [item for item in tasks if str(item.get("id") or "") in task_ids] + datasets = [item for item in datasets if str(item.get("id") or "") in dataset_ids] + eval_tasks = [item for item in eval_tasks if str(item.get("id") or "") in eval_ids] + compares = [item for item in compares if str(item.get("id") or "") in compare_ids] + users = [user for user in users if user.get("id") == current_user.get("id")] + nodes = [item for item in nodes if any(gpu.get("node_id") == item.get("id") for gpu in store.gpu_assignments_for_user(current_user.get("id")))] # 数据处理任务总数（来自 data_process 模块） try: from app.modules.data_process.store import get_data_process_store @@ -760,7 +1020,7 @@ async def dashboard_stats() -> dict[str, Any]: # 评测中运行的任务数 eval_running = 0 try: - eval_tasks = store.eval_tasks() + eval_tasks = eval_tasks if not is_admin(current_user) else store.eval_tasks() eval_running = len([e for e in eval_tasks if e.get("status") in running_statuses]) except Exception: eval_running = 0 @@ -795,7 +1055,7 @@ async def dashboard_stats() -> dict[str, Any]: service_checks = [ ("模型训练", "fine-tune", len(tasks)), ("模型评测", "model-eval", len(eval_tasks)), - ("模型推理", "model-inference", len(store.compare_tasks())), + ("模型推理", "model-inference", len(compares)), ("模型管理", "model-manage", len(store.models())), ("数据集管理", "dataset-manage", len(datasets)), ("数据处理", "data-process", dp_count), @@ -863,7 +1123,7 @@ async def dashboard_stats() -> dict[str, Any]: return label return None - audit = store.audit_logs(limit=1000) + audit = store.audit_logs(limit=1000, actor_id=None if is_admin(current_user) else current_user.get("id")) op_counter: dict[str, int] = {label: 0 for label in OP_ORDER} for log in audit.get("items", []): label = _op_module(log.get("action") or "") @@ -893,7 +1153,7 @@ async def dashboard_stats() -> dict[str, Any]: except Exception: pass - return _store_dashboard_cache("stats", ok( + return _store_dashboard_cache(cache_key, ok( { "online_services": sum(s["count"] for s in service_status), "running_tasks": len(running_ft) + eval_running, @@ -914,17 +1174,20 @@ async def system_info() -> dict[str, Any]: @router.get("/users") -async def users() -> dict[str, Any]: +async def users(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().users()) @router.post("/users") -async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def create_user(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: + payload = dict(payload) + payload.setdefault("password", "123456") + payload.setdefault("tenant_id", current_user.get("tenant_id") or "admin") return ok(get_platform_store().create_user(payload)) @router.put("/users/{user_id}") -async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().update_user(user_id, payload)) except KeyError: @@ -932,9 +1195,15 @@ async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict @router.delete("/users/{user_id}") -async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]: +async def delete_user(user_id: str, current_username: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: - get_platform_store().delete_user(user_id) + get_platform_store().delete_user(user_id, deleted_by=str(current_user.get("id") or "system")) + get_platform_store().record_audit( + action="user.delete", actor_id=current_user.get("id"), + target_type="user", target_id=user_id, + tenant_id=current_user.get("tenant_id"), + detail="user tombstoned; sessions, memberships and grants revoked", + ) return ok({"deleted": user_id, "current_username": current_username}) except KeyError: raise fail(404, "user not found") @@ -946,6 +1215,7 @@ async def delete_user(user_id: str, current_username: str | None = Query(default async def reset_user_password( user_id: str, payload: dict[str, Any] = Body(default={}), + current_user: dict = Depends(require_admin), ) -> dict[str, Any]: new_password = payload.get("password") or "Platform@123" try: @@ -981,30 +1251,49 @@ async def change_my_password( @router.get("/model-manage/local-models") -async def local_models() -> dict[str, Any]: +async def local_models(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: store = get_platform_store() - models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()] - seen = {item["path"] for item in models if item.get("path")} - if get_settings().compute_mode != "simulator": + all_models = [ + item for item in store.models() + if str(item.get("model_source") or "").lower() not in {"api", "online"} + ] + accessible = filter_accessible_resource_ids_batch("model", [str(item["id"]) for item in all_models], current_user) + models = [ + { + "path": item.get("path") or "", + "name": item["name"], + "storage_status": item.get("storage_status") or ("available" if store.storage_objects_for_resource("model", str(item["id"])) else "pending"), + } + for item in all_models + if str(item.get("id") or "") in accessible + ] + # Admins may discover legacy model directories that have not been + # registered yet. The directory is only an import source: do not return + # the node id/code, and never persist it as model ownership. Once selected + # and created, the model is archived to MinIO and future consumers use the + # shared object manifest plus a local cache on their target node. + if is_admin(current_user) and get_settings().compute_mode != "simulator": + seen_paths = {str(item.get("path") or "") for item in models if item.get("path")} for node in store.compute_nodes(): if not node.get("enabled"): continue try: - result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True) + result = await ComputeNodeClient(node["api_base_url"], timeout=10).list_files( + root="models", + directories_only=True, + ) except Exception: continue for item in result.get("items") or []: path = str(item.get("path") or "") - if not path or path in seen: + if not path or path in seen_paths: continue - seen.add(path) - models.append( - { - "path": path, - "name": item.get("name") or path.rsplit("/", 1)[-1], - "source": f"compute:{node.get('code')}", - } - ) + seen_paths.add(path) + models.append({ + "path": path, + "name": item.get("name") or path.rsplit("/", 1)[-1], + "storage_status": "pending", + }) return ok({"models": models}) @@ -1015,7 +1304,7 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict return ok({"models": all_models}) # 普通用户只能看到自己创建的 + ACL 授权的 user_id = current_user.get("id") - accessible = set(filter_accessible_resource_ids("trained_model", [m["id"] for m in all_models], current_user)) + accessible = filter_accessible_resource_ids_batch("trained_model", [m["id"] for m in all_models], current_user) result = [m for m in all_models if m.get("created_by") == user_id or m["id"] in accessible] return ok({"models": result}) @@ -1024,7 +1313,7 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not has_resource_access("trained_model", model_id, current_user, "delete"): raise fail(403, "no permission to delete this trained model") - pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}") + pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}", "trained_model.delete") if pending: return pending get_platform_store().delete_trained_model(model_id) @@ -1032,12 +1321,16 @@ async def delete_trained_model(model_id: str, type: str = Query(default="merged" @router.get("/model-manage/trained-models/{model_id}/artifacts") -async def trained_model_artifacts(model_id: str) -> dict[str, Any]: +async def trained_model_artifacts(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("trained_model", model_id, current_user, "read"): + raise fail(403, "no permission to access model artifacts") return ok(get_platform_store().model_artifacts(model_id)) @router.get("/model-manage/trained-models/{model_id}/lineage") -async def trained_model_lineage(model_id: str) -> dict[str, Any]: +async def trained_model_lineage(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("trained_model", model_id, current_user, "read"): + raise fail(403, "no permission to access model lineage") return ok(get_platform_store().model_lineage(model_id)) @@ -1045,25 +1338,32 @@ async def trained_model_lineage(model_id: str) -> dict[str, Any]: async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"): raise fail(403, "no permission to access export jobs") - return ok(get_platform_store().model_export_jobs(trained_model_id)) + jobs = get_platform_store().model_export_jobs(trained_model_id) + if not is_admin(current_user): + jobs = [item for item in jobs if has_resource_access("trained_model", str(item.get("trained_model_id") or ""), current_user, "read")] + return ok(jobs) @router.get("/model-manage/name/{name}") -async def model_by_name(name: str) -> dict[str, Any]: +async def model_by_name(name: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok(get_platform_store().model_by_name(name)) + model = get_platform_store().model_by_name(name) + if not has_resource_access("model", str(model["id"]), current_user, "read"): + raise fail(403, "no permission to read this model") + return ok(_public_model(model)) except KeyError: raise fail(404, "model not found") @router.get("/model-manage") async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - # 基座模型是平台共享资源，所有登录用户均可查看 - return ok(get_platform_store().models()) + models = get_platform_store().models() + accessible = filter_accessible_resource_ids_batch("model", [str(item["id"]) for item in models], current_user) + return ok(_public_models([item for item in models if str(item.get("id") or "") in accessible])) @router.post("/model-manage/test-online") -async def test_online_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def test_online_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """测试在线模型 API 是否可用：发送一个简单的 chat/completions 请求验证连通性。""" api_url = (payload.get("api_url") or "").rstrip("/") api_key = payload.get("api_key") or "" @@ -1129,10 +1429,22 @@ async def test_online_model(payload: dict[str, Any] = Body(...)) -> dict[str, An target_type="model", detail_template="创建模型: {name}", ) -async def create_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - payload.setdefault("created_by", current_user.get("id")) +async def create_model( + background_tasks: BackgroundTasks, + payload: dict[str, Any] = Body(...), + current_user: dict = Depends(get_current_user), +) -> dict[str, Any]: + model_source = str(payload.get("model_source") or "local").lower() + if model_source not in {"api", "online"} and not is_admin(current_user): + raise fail(403, "only administrators can register local base models") + payload["created_by"] = current_user.get("id") + if model_source in {"api", "online"}: + payload = bind_active_tenant(payload, current_user) try: - return ok(get_platform_store().create_model(payload)) + model = get_platform_store().create_model(payload) + if model_source not in {"api", "online"} and model.get("path") and get_settings().minio_enabled: + background_tasks.add_task(_archive_model_in_background, str(model["id"])) + return ok(_public_model(model)) except KeyError as exc: raise fail(400, f"missing field: {exc}") except ValueError as exc: @@ -1143,12 +1455,13 @@ async def create_model(payload: dict[str, Any] = Body(...), current_user: dict = @router.get("/model-manage/{model_id}") async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - # 基座模型（配置模型）是平台共享资源，所有登录用户均可查看 try: model = get_platform_store().model(model_id) except KeyError: raise fail(404, "model not found") - return ok(model) + if not has_resource_access("model", model_id, current_user, "read"): + raise fail(403, "no permission to read this model") + return ok(_public_model(model)) @router.put("/model-manage/{model_id}") @@ -1162,7 +1475,7 @@ async def update_model(model_id: str, payload: dict[str, Any] = Body(...), curre if not is_admin(current_user): raise fail(403, "只有管理员可以修改模型配置") try: - return ok(get_platform_store().update_model(model_id, payload)) + return ok(_public_model(get_platform_store().update_model(model_id, payload))) except KeyError: raise fail(404, "model not found") @@ -1173,7 +1486,7 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(... if not is_admin(current_user): raise fail(403, "只有管理员可以修改模型用途") try: - return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")})) + return ok(_public_model(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")}))) except KeyError: raise fail(404, "model not found") @@ -1191,6 +1504,9 @@ async def delete_model(model_id: str, current_user: dict = Depends(get_current_u @op_log(module=OpModule.MODEL_MANAGE, action=OpAction.MERGE, target_type="trained_model", target_name_param="trained_model_id") async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: store = get_platform_store() + engine = str(payload.get("engine") or "merge").strip().lower() + if engine not in {"merge", "export", "llama_factory_export"}: + raise fail(400, "engine must be merge, export or llama_factory_export") trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "") trained_model = next( ( @@ -1204,16 +1520,29 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = raise fail(404, "trained model not found") if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"): raise fail(403, "no permission to merge this trained model") + if engine != "merge" and not has_resource_access("trained_model", trained_model["id"], current_user, "download"): + raise fail(403, "no permission to export this trained model") + store.record_audit( + action="trained_model.export" if engine != "merge" else "trained_model.merge", + actor_id=current_user.get("id"), target_type="trained_model", target_id=trained_model["id"], + tenant_id=trained_model.get("tenant_id"), detail=f"engine={engine}", + ) # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权即可使用 - base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path")) + base_model_path = payload.get("base_model_path") or ( + trained_model.get("merged_path") if engine != "merge" and trained_model and trained_model.get("merged") else None + ) or (trained_model and trained_model.get("base_model_path")) adapter_path = ( payload.get("adapter_path") or payload.get("adapter_name_or_path") - or (trained_model and (trained_model.get("artifact_dir") or trained_model.get("adapter_path") or trained_model.get("merged_path"))) + or ( + trained_model + and not (engine != "merge" and trained_model.get("merged")) + and (trained_model.get("artifact_dir") or trained_model.get("adapter_path") or trained_model.get("merged_path")) + ) ) if not base_model_path: raise fail(400, "base_model_path is required") - if not adapter_path: + if not adapter_path and engine == "merge": raise fail(400, "adapter_path is required") requested_node_id = payload.get("requested_node_id") or payload.get("compute_node_id") or (trained_model and trained_model.get("compute_node_id")) node = store.schedule_node({**payload, "requested_node_id": requested_node_id, "gpus": payload.get("gpus") or []}) @@ -1237,9 +1566,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = output_dir = str(payload.get("output_dir") or f"{output_root.rstrip('/')}/{output_name}") job_payload = { **payload, - "id": str(payload.get("job_id") or f"merge_{uuid.uuid4().hex[:12]}"), + "id": str(payload.get("job_id") or f"{('export' if engine != 'merge' else 'merge')}_{uuid.uuid4().hex[:12]}"), "name": output_name, - "engine": "merge", + "engine": engine, "base_model": base_model_path, "model_name_or_path": base_model_path, "adapter_name_or_path": adapter_path, @@ -1251,6 +1580,8 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = "model_name": trained_model["name"] if trained_model else payload.get("model_name"), "compute_node_id": node["id"], "compute_node_code": node.get("code"), + "resource_versions": payload.get("resource_versions") or _capture_resource_versions(store, {**payload, "model_id": trained_model["id"]}), + "tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default", } if get_settings().compute_mode == "simulator": job = {"id": job_payload["id"], "status": "queued", "progress": 10, "command": [], "output_dir": output_dir} @@ -1263,10 +1594,55 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = return ok(store.record_model_merge_job(node, job_payload, job, trained_model["id"] if trained_model else trained_model_id)) +def _authorize_inference_payload(payload: dict[str, Any], current_user: dict[str, Any]) -> None: + """Require execute permission for an explicitly selected inference task/model.""" + task_id = str(payload.get("task_id") or payload.get("inference_task_id") or "") + if task_id and not has_resource_access("compare", task_id, current_user, "execute"): + raise fail(403, "no permission to use this inference task") + trained_id = str(payload.get("trained_model_id") or "") + if not trained_id and payload.get("model_id") and resource_record("trained_model", str(payload["model_id"])): + trained_id = str(payload["model_id"]) + if trained_id and not has_resource_access("trained_model", trained_id, current_user, "execute"): + raise fail(403, "no permission to use this trained model") + model_id = str(payload.get("model_id") or "") + if model_id and not trained_id and resource_record("model", model_id): + if not has_resource_access("model", model_id, current_user, "execute"): + raise fail(403, "no permission to use this base model") + if not task_id and not trained_id and not model_id and not is_admin(current_user): + raise fail(403, "inference must reference an authorized model or inference task") + + +def _authorize_compute_node(node_id: str, current_user: dict[str, Any]) -> None: + if is_admin(current_user): + return + assigned = get_platform_store().gpu_assignments_for_user(str(current_user.get("id") or "")) + if not any(str(item.get("node_id")) == str(node_id) for item in assigned): + raise fail(403, "no permission to access this compute node") + + +def _authorize_training_payload(payload: dict[str, Any], current_user: dict[str, Any]) -> None: + dataset_id = str(payload.get("train_dataset_id") or payload.get("dataset_id") or "") + if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"): + raise fail(403, "no permission to use this training dataset") + base_model_id = str(payload.get("base_model_id") or "") + if base_model_id and not has_resource_access("model", base_model_id, current_user, "execute"): + raise fail(403, "no permission to use this base model") + + +@router.post("/model-manage/export") +@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.MERGE, target_type="trained_model", target_name_param="trained_model_id") +async def export_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + """Export a trained model through the same node, cache and archive path as merge.""" + return await merge_model({**payload, "engine": "export"}, current_user) + + @router.get("/dataset-manage/preview/{file_id}") -async def dataset_preview(file_id: str) -> dict[str, Any]: +async def dataset_preview(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"): + raise fail(403, "no permission to preview this dataset") content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace") return ok({"content": content}) except KeyError: @@ -1274,55 +1650,78 @@ async def dataset_preview(file_id: str) -> dict[str, Any]: @router.get("/dataset-manage/records/{file_id}/sources") -async def dataset_record_sources(file_id: str) -> dict[str, Any]: +async def dataset_record_sources(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok({"items": get_platform_store().dataset_file_record_sources(file_id)}) + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"): + raise fail(403, "no permission to access dataset records") + return ok({"items": store.dataset_file_record_sources(file_id)}) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}") -async def dataset_versions(file_id: str) -> dict[str, Any]: +async def dataset_versions(file_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok(get_platform_store().file_versions(file_id)) + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"): + raise fail(403, "no permission to access dataset versions") + return ok(store.file_versions(file_id)) except KeyError: raise fail(404, "dataset file not found") @router.get("/dataset-manage/versions/{file_id}/{version_id}") -async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]: +async def dataset_version_content(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - row = get_platform_store().dataset_file(file_id) - versions = get_platform_store().file_versions(file_id)["versions"] + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "read"): + raise fail(403, "no permission to access dataset version") + versions = store.file_versions(file_id)["versions"] version = next((item for item in versions if item["id"] == version_id), None) if not version: raise KeyError(version_id) - content = _dataset_version_bytes(get_platform_store(), file_id, version_id) + content = _dataset_version_bytes(store, file_id, version_id) return ok({"version": version, "content": content.decode("utf-8", errors="replace")}) except KeyError: raise fail(404, "dataset version not found") @router.post("/dataset-manage/versions/{file_id}") -async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok(get_platform_store().create_file_version(file_id, payload)) + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"): + raise fail(403, "no permission to create dataset version") + return ok(store.create_file_version(file_id, payload)) except KeyError: raise fail(404, "dataset file not found") @router.put("/dataset-manage/versions/{file_id}/active") -async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok(get_platform_store().activate_file_version(file_id, payload["version_id"])) + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"): + raise fail(403, "no permission to activate dataset version") + return ok(store.activate_file_version(file_id, payload["version_id"])) except KeyError: raise fail(404, "dataset version not found") @router.delete("/dataset-manage/versions/{file_id}/{version_id}") -async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]: +async def delete_dataset_version(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: - return ok(get_platform_store().delete_file_version(file_id, version_id)) + store = get_platform_store() + row = store.dataset_file(file_id) + if not has_resource_access("dataset", str(row["dataset_id"]), current_user, "write"): + raise fail(403, "no permission to delete dataset version") + return ok(store.delete_file_version(file_id, version_id)) except KeyError: raise fail(404, "dataset version not found") except ValueError as exc: @@ -1518,6 +1917,7 @@ async def upload_dataset_files( dataset_id: str, files: list[UploadFile] = File(default=[]), sync_to_compute: bool = Query(default=True), + current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: created: list[dict[str, Any]] = [] compute_sync: list[dict[str, Any]] = [] @@ -1527,6 +1927,8 @@ async def upload_dataset_files( store.dataset(dataset_id) except KeyError: raise fail(404, "dataset not found") + if not has_resource_access("dataset", dataset_id, current_user, "write"): + raise fail(403, "no permission to upload files to this dataset") with store.connect() as conn: for file in files: raw = await file.read() @@ -1571,8 +1973,9 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur dataset = store.dataset(dataset_id) except KeyError: raise fail(404, "dataset not found") - if not has_resource_access("dataset", dataset_id, current_user, "read"): - raise fail(403, "no permission to access this dataset") + if not has_resource_access("dataset", dataset_id, current_user, "download"): + raise fail(403, "no permission to download this dataset") + store.record_audit(action="dataset.download", actor_id=current_user.get("id"), target_type="dataset", target_id=dataset_id, tenant_id=dataset.get("tenant_id"), detail="dataset bundle download") files = [] for item in dataset.get("files", []): @@ -1625,23 +2028,53 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur @router.get("/dataset-manage/download/{dataset_id}/{file_id}") -async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse: +async def download_dataset_file( + dataset_id: str, + file_id: str, + version_id: str | None = Query(default=None), + current_user: dict = Depends(get_current_user), +) -> Response: store = get_platform_store() row = store.dataset_file(file_id) if str(row.get("dataset_id")) != str(dataset_id): raise fail(404, "dataset file not found") + if not has_resource_access("dataset", dataset_id, current_user, "download"): + raise fail(403, "no permission to download this dataset") + store.record_audit(action="dataset.file.download", actor_id=current_user.get("id"), target_type="dataset_file", target_id=file_id, tenant_id=row.get("tenant_id"), detail=f"dataset_id={dataset_id}") + object_id = str(row.get("storage_object_id") or "") + if version_id: + version = next((item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id), None) + if not version: + raise fail(404, "dataset version not found") + object_id = str(version.get("storage_object_id") or "") + if get_settings().minio_enabled and object_id: + try: + obj = store.storage_object(object_id) + filename = Path(str(obj.get("file_name") or row.get("name") or file_id)).name + return StreamingResponse( + get_object_storage().iter_bytes(str(obj["object_key"])), + media_type=str(obj.get("content_type") or "application/octet-stream"), + headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename, safe='')}"}, + ) + except KeyError: + pass content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id) - return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain") + filename = Path(str(row.get("name") or file_id)).name + return Response( + content=content, + media_type="application/octet-stream", + headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename, safe='')}"}, + ) @router.get("/dataset-manage") async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - datasets = get_platform_store().datasets() + datasets = get_platform_store().datasets(None if is_admin(current_user) else current_user.get("tenant_id") or "default") if is_admin(current_user): return ok(datasets) # 普通用户可见：自己创建的 + ACL 授权的 user_id = current_user.get("id") - accessible = set(filter_accessible_resource_ids("dataset", [d["id"] for d in datasets], current_user)) + accessible = filter_accessible_resource_ids_batch("dataset", [d["id"] for d in datasets], current_user) result = [d for d in datasets if d.get("created_by") == user_id or d["id"] in accessible] return ok(result) @@ -1654,8 +2087,11 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s ) @op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name") async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + bind_active_tenant(payload, current_user) payload.setdefault("created_by", current_user.get("id")) + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") try: + get_platform_store().assert_active_tenant(payload["tenant_id"]) dataset = get_platform_store().create_dataset(payload) return ok({"id": dataset["id"]}) except ValueError as exc: @@ -1679,7 +2115,9 @@ async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_curre target_type="dataset", detail_template="更新数据集: {dataset_id}", ) -async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("dataset", dataset_id, current_user, "write"): + raise fail(403, "no permission to update this dataset") try: return ok(get_platform_store().update_dataset(dataset_id, payload)) except KeyError: @@ -1696,7 +2134,7 @@ async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) - async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not has_resource_access("dataset", dataset_id, current_user, "delete"): raise fail(403, "no permission to delete this dataset") - pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}") + pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}", "dataset.delete") if pending: return pending get_platform_store().delete_dataset(dataset_id) @@ -1704,13 +2142,15 @@ async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_curre @router.get("/fine-tune/check-name") -async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]: +async def check_fine_tune_name(name: str = Query(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: exists = any(task["name"] == name for task in get_platform_store().tasks()) return ok({"exists": exists}) @router.get("/fine-tune/progress/{task_id}") -async def fine_tune_progress(task_id: str) -> dict[str, Any]: +async def fine_tune_progress(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to access training progress") try: return ok(get_platform_store().progress(task_id)) except KeyError: @@ -1718,18 +2158,18 @@ async def fine_tune_progress(task_id: str) -> dict[str, Any]: @router.post("/fine-tune/tensorboard/start") -async def tensorboard_start() -> dict[str, Any]: +async def tensorboard_start(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: return ok({"status": "running", "url": "http://localhost:6006"}) @router.get("/fine-tune") async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: tasks = get_platform_store().tasks() - if current_user.get("role") == "admin" or current_user.get("protected"): + if is_admin(current_user): return ok(tasks) # 普通用户可见：自己创建的 + ACL 授权的 user_id = current_user.get("id") - accessible = set(filter_accessible_resource_ids("fine-tune", [t["id"] for t in tasks], current_user)) + accessible = filter_accessible_resource_ids_batch("fine-tune", [t["id"] for t in tasks], current_user) result = [t for t in tasks if t.get("created_by") == user_id or t["id"] in accessible] return ok(result) @@ -1741,14 +2181,25 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict detail_template="创建微调任务: {name}", ) async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + bind_active_tenant(payload, current_user) + _authorize_training_payload(payload, current_user) payload.setdefault("created_by", current_user.get("id")) + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") + try: + get_platform_store().assert_active_tenant(payload["tenant_id"]) + except ValueError as exc: + raise fail(400, str(exc)) if not is_admin(current_user): model_id = str(payload.get("base_model") or payload.get("base_model_id") or "") dataset_id = str(payload.get("train_dataset_id") or "") + if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"): + raise fail(403, "no permission to use this base model") # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权 if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"): raise fail(403, "no permission to use this dataset") try: + if not payload.get("resource_versions"): + payload["resource_versions"] = _capture_resource_versions(get_platform_store(), payload) task = get_platform_store().create_task(payload) return ok({"id": task["id"]}) except ValueError as exc: @@ -1762,6 +2213,8 @@ async def start_fine_tune( current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: store = get_platform_store() + bind_active_tenant(payload, current_user) + _authorize_training_payload(payload, current_user) # GPU 权限校验：普通用户只能使用被分配的 GPU if not is_admin(current_user): node_id = payload.get("compute_node_id") or payload.get("node_id") @@ -1776,6 +2229,20 @@ async def start_fine_tune( # 页面明确选择节点时，调度器必须保持节点约束；否则可能落到其它节点。 payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id")) payload.setdefault("created_by", current_user.get("id")) + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") + try: + store.assert_active_tenant(payload["tenant_id"]) + except ValueError as exc: + raise fail(400, str(exc)) + if not payload.get("resource_versions"): + snapshot_payload = dict(payload) + task_id = str(payload.get("task_id") or payload.get("id") or "") + if task_id: + try: + snapshot_payload = {**store.task(task_id), **snapshot_payload} + except KeyError: + pass + payload["resource_versions"] = _capture_resource_versions(store, snapshot_payload) try: return ok(await _submit_fine_tune_task(store, payload)) except KeyError: @@ -1793,7 +2260,8 @@ async def start_fine_tune( @router.post("/fine-tune/preflight") -async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_training_payload(payload, current_user) try: return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True, sync_resources=True)) except RuntimeError as exc: @@ -1803,7 +2271,8 @@ async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dic @router.post("/fine-tune/command-preview") -async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_training_payload(payload, current_user) try: return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False, sync_resources=False)) except RuntimeError as exc: @@ -1813,7 +2282,9 @@ async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) @router.post("/fine-tune/{task_id}/preflight") -async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: +async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to preflight this task") try: return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True)) except KeyError: @@ -1825,7 +2296,9 @@ async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Bod @router.post("/fine-tune/{task_id}/command-preview") -async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]: +async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to preview this task") try: return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False)) except KeyError: @@ -1837,7 +2310,9 @@ async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None @router.get("/fine-tune/{task_id}") -async def fine_tune_detail(task_id: str) -> dict[str, Any]: +async def fine_tune_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to access this task") try: return ok(get_platform_store().task(task_id)) except KeyError: @@ -1850,6 +2325,7 @@ async def fine_tune_logs( tail_lines: int | None = Query(default=500, ge=1, le=5000), offset: int | None = Query(default=None, ge=0), limit: int | None = Query(default=None, ge=1, le=5000), + current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: store = get_platform_store() try: @@ -1880,7 +2356,7 @@ async def fine_tune_logs( @router.get("/fine-tune/{task_id}/diagnostics") -async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]: +async def fine_tune_diagnostics(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: store = get_platform_store() try: task = store.task(task_id) @@ -1958,7 +2434,7 @@ async def stop_fine_tune(task_id: str, current_user: dict = Depends(get_current_ try: task = store.task(task_id) # 审批拦截：非 admin 停止他人任务需审批 - pending = _require_approval_or_admin("fine_tune_task", task_id, current_user, f"停止训练任务 {task_id}") + pending = _require_approval_or_admin("fine_tune_task", task_id, current_user, f"停止训练任务 {task_id}", "fine_tune.stop") if pending: return pending node = _node_for_task(task) @@ -2003,7 +2479,7 @@ async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(de async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not has_resource_access("fine-tune", task_id, current_user, "delete"): raise fail(403, "no permission to delete this task") - pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}") + pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}", "fine_tune.delete") if pending: return pending get_platform_store().delete_task(task_id) @@ -2035,32 +2511,75 @@ async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]: @router.get("/fine-tune/{task_id}/metrics") -async def fine_tune_metrics(task_id: str) -> dict[str, Any]: +async def fine_tune_metrics(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: store = get_platform_store() try: store.task(task_id) except KeyError: raise fail(404, "fine tune task not found") + if not has_resource_access("fine-tune", task_id, current_user, "read"): + raise fail(403, "no permission to access training metrics") return ok(store.task_metrics(task_id)) @router.get("/model-eval") async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - tasks = get_platform_store().eval_tasks() - if current_user.get("role") == "admin" or current_user.get("protected"): + tasks = get_platform_store().eval_tasks(None if is_admin(current_user) else current_user.get("tenant_id") or "default") + if is_admin(current_user): return ok(tasks) # 普通用户可见：自己创建的 + ACL 授权的 user_id = current_user.get("id") - accessible = set(filter_accessible_resource_ids("eval", [t["id"] for t in tasks], current_user)) + accessible = filter_accessible_resource_ids_batch("eval", [t["id"] for t in tasks], current_user) result = [t for t in tasks if t.get("created_by") == user_id or t["id"] in accessible] return ok(result) +@router.get("/model-eval/{task_id}/report") +async def model_eval_report(task_id: str, current_user: dict = Depends(get_current_user)) -> Response: + """Stream the archived evaluation report, with DB JSON as compatibility fallback.""" + store = get_platform_store() + try: + task = store.eval_task(task_id) + except KeyError: + raise fail(404, "eval task not found") + if not has_resource_access("eval", task_id, current_user, "download"): + raise fail(403, "no permission to access this eval report") + store.record_audit(action="eval.report.download", actor_id=current_user.get("id"), target_type="eval", target_id=task_id, tenant_id=task.get("tenant_id"), detail="evaluation report download") + report_object_id = str(task.get("report_storage_object_id") or "") + if get_settings().minio_enabled and report_object_id: + try: + obj = store.storage_object(report_object_id) + return StreamingResponse( + get_object_storage().iter_bytes(str(obj["object_key"])), + media_type=str(obj.get("content_type") or "application/json"), + headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(f'{task_id}-report.json', safe='')}"}, + ) + except (KeyError, ObjectStorageError): + pass + payload = { + "task_id": task_id, + "status": task.get("status"), + "overall_score": task.get("overall_score"), + "overall_score_max": task.get("overall_score_max"), + "dimension_summary": task.get("dimension_summary") or [], + "samples": task.get("samples") or [], + "overall_evaluation": task.get("overall_evaluation") or "", + "improvement_suggestions": task.get("improvement_suggestions") or [], + } + return Response( + content=json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"), + media_type="application/json", + headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(f'{task_id}-report.json', safe='')}"}, + ) + + @router.get("/model-eval/{task_id}") async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: store = get_platform_store() task = store.eval_task(task_id) + if not has_resource_access("eval", task_id, current_user, "read"): + raise fail(403, "no permission to access this eval task") if task.get("compute_job_id") and task.get("compute_node_id") and task.get("status") in {"queued", "running", "completed"}: node = next( (n for n in store.compute_nodes() if n["id"] == task.get("compute_node_id")), @@ -2071,15 +2590,21 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre client = ComputeNodeClient(node["api_base_url"]) job = await client.get_job(task["compute_job_id"]) result_content = None - if job.get("status") == "completed" and not task.get("samples"): + if job.get("status") == "completed": + result_content = await fetch_eval_result_content(client, node, job) + elif job.get("status") in {"queued", "running"}: + progress_content = await fetch_eval_progress_content(client, node, job) + if progress_content: + task = store.update_eval_task( + task_id, + {"progress_detail": progress_content, "progress": progress_content.get("percentage", task.get("progress", 0))}, + ) result_content = await fetch_eval_result_content(client, node, job) task = store.apply_eval_job_result(task_id, job, result_content) except Exception: pass except KeyError: raise fail(404, "eval task not found") - if not has_resource_access("eval", task_id, current_user, "read"): - raise fail(403, "no permission to access this eval task") return ok(task) @@ -2088,6 +2613,29 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Start an evaluation task: submit eval job to compute node.""" store = get_platform_store() + bind_active_tenant(payload, current_user) + model_id = str(payload.get("model_id") or "") + dataset_id = str(payload.get("dataset_id") or "") + if not dataset_id: + raise fail(400, "dataset_id is required") + try: + store.dataset(dataset_id) + except KeyError: + raise fail(404, "dataset not found") + # Base models are shared read-only platform resources. Trained models must + # pass resource-level execute permission before a task record is created. + trained_model = next((item for item in store.trained_models() if str(item.get("id")) == model_id or str(item.get("name")) == model_id), None) + if trained_model and not has_resource_access("trained_model", str(trained_model["id"]), current_user, "execute"): + raise fail(403, "no permission to evaluate this trained model") + if not is_admin(current_user) and not has_resource_access("dataset", dataset_id, current_user, "execute"): + raise fail(403, "no permission to evaluate this dataset") + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") + try: + store.assert_active_tenant(payload["tenant_id"]) + except ValueError as exc: + raise fail(400, str(exc)) + if not payload.get("resource_versions"): + payload["resource_versions"] = _capture_resource_versions(store, payload) try: gpu_indices = _normalize_gpu_indices(payload) except ValueError as exc: @@ -2171,7 +2719,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di store.update_eval_task(task["id"], {"status": "failed", "error": "dataset not found or no files"}) return ok({"task_id": task["id"], "status": "failed", "error": "dataset not found or no files"}) - # 4. Resolve dimension config + # 4. Resolve dimension config. Admin-maintained dimensions are referenced + # by ID; user-created evaluation rules are carried inline with the task + # and must not be persisted into the global dimension catalog. dimension_id = str(payload.get("dimension_id", "")) dimension_cfg: dict[str, Any] = {} if dimension_id: @@ -2190,7 +2740,14 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di api_key = eval_model.get("api_key", "") # 模型记录里的 model_name 是真实 API 模型名（如 deepseek-chat）， # 优先传给评测器，避免用平台内部名称调用 LLM API - api_model_name = eval_model.get("model_name") or "" + # 在线模型的真实服务名保存在 online_model_name； + # model_name 是历史字段，不能回退到平台内部名称。 + api_model_name = ( + eval_model.get("model_name") + or eval_model.get("online_model_name") + or eval_model.get("name") + or "" + ) except (KeyError, Exception): pass dimension_cfg = { @@ -2207,6 +2764,23 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di } except KeyError: pass + elif isinstance(payload.get("dimension"), dict): + dimension_cfg = dict(payload["dimension"]) + eval_model_name = str(dimension_cfg.get("eval_model") or "") + if eval_model_name: + try: + eval_model = store.model(eval_model_name) if eval_model_name.startswith("m_") else store.model_by_name(eval_model_name) + if isinstance(eval_model, dict): + dimension_cfg["api_url"] = eval_model.get("api_url", "") + dimension_cfg["api_key"] = eval_model.get("api_key", "") + dimension_cfg["api_model"] = ( + eval_model.get("model_name") + or eval_model.get("online_model_name") + or eval_model.get("name") + or eval_model_name + ) + except KeyError: + pass # 5. Select compute node: 优先页面选择的节点 / 模型所在节点，避免多节点时选错 preferred_node_id = payload.get("compute_node_id") or payload.get("node_id") or model_node_id @@ -2216,6 +2790,11 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di store.update_eval_task(task["id"], {"status": "failed", "error": message}) return ok({"task_id": task["id"], "status": "failed", "error": message}) + if not is_admin(current_user) and not store.check_gpu_access(current_user["id"], node["id"], gpu_indices): + message = "selected GPU is not assigned to the current user" + store.update_eval_task(task["id"], {"status": "failed", "error": message}) + return ok({"task_id": task["id"], "status": "failed", "error": message}) + if get_settings().minio_enabled and get_settings().compute_mode != "simulator": try: prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node) @@ -2232,14 +2811,16 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"}) return ok({"task_id": task["id"], "status": "failed", "error": str(exc)}) - node_gpus = { - int(item.get("id", item.get("gpu_index", -1))): item - for item in store.gpus() - if item.get("node_id") == node["id"] - } - unavailable = [index for index in gpu_indices if node_gpus.get(index, {}).get("status") != "idle"] - if unavailable: - message = f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}" + try: + store.reserve_external_gpus( + "eval", + str(task["id"]), + node["id"], + gpu_indices, + tenant_id=str(task.get("tenant_id") or current_user.get("tenant_id") or "default"), + ) + except (RuntimeError, ValueError) as exc: + message = str(exc) store.update_eval_task(task["id"], {"status": "failed", "error": message}) return ok({"task_id": task["id"], "status": "failed", "error": message}) @@ -2288,12 +2869,15 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di "status": "running", "compute_job_id": job.get("id"), "compute_node_id": node["id"], + "gpu_indices": gpu_indices, + "gpus": gpu_indices, "output_dir": output_dir, }) # 评测占用 GPU 由 eval_tasks 派生（gpus()/compute_nodes() 直接统计）， # 不再复用 mark_inference_loaded 内存标记，避免删除评测后 GPU 状态残留 busy return ok({"task_id": task["id"], "status": "running", "job": job}) except Exception as exc: + store.release_external_gpus("eval", str(task["id"]), node["id"]) store.update_eval_task(task["id"], {"status": "failed", "error": str(exc)}) return ok({"task_id": task["id"], "status": "failed", "error": str(exc)}) @@ -2302,7 +2886,7 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not has_resource_access("eval", task_id, current_user, "delete"): raise fail(403, "no permission to delete this eval task") - pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}") + pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}", "eval.delete") if pending: return pending try: @@ -2313,17 +2897,17 @@ async def model_eval_delete(task_id: str, current_user: dict = Depends(get_curre @router.get("/dimension") -async def dimension_list() -> dict[str, Any]: +async def dimension_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: return ok(get_platform_store().dimensions()) @router.post("/dimension") -async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def dimension_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().create_dimension(payload)) @router.get("/dimension/{dimension_id}") -async def dimension_detail(dimension_id: str) -> dict[str, Any]: +async def dimension_detail(dimension_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: return ok(get_platform_store().dimension(dimension_id)) except KeyError: @@ -2331,7 +2915,7 @@ async def dimension_detail(dimension_id: str) -> dict[str, Any]: @router.put("/dimension/{dimension_id}") -async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().update_dimension(dimension_id, payload)) except KeyError: @@ -2339,7 +2923,7 @@ async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(... @router.delete("/dimension/{dimension_id}") -async def dimension_delete(dimension_id: str) -> dict[str, Any]: +async def dimension_delete(dimension_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: get_platform_store().delete_dimension(dimension_id) return ok({"deleted": dimension_id}) @@ -2349,6 +2933,8 @@ async def model_compare_list(current_user: dict = Depends(get_current_user)) -> tasks = get_platform_store().compare_tasks() if is_admin(current_user): return ok(tasks) + tenant_id = current_user.get("tenant_id") or "default" + tasks = [task for task in tasks if task.get("tenant_id") in {None, "", tenant_id}] # 普通用户可见：自己创建的 + ACL 授权的 user_id = current_user.get("id") accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user) @@ -2358,20 +2944,37 @@ async def model_compare_list(current_user: dict = Depends(get_current_user)) -> @router.post("/model-compare") async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + bind_active_tenant(payload, current_user) payload.setdefault("created_by", current_user.get("id")) + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") + try: + get_platform_store().assert_active_tenant(payload["tenant_id"]) + except ValueError as exc: + raise fail(400, str(exc)) model_ids = payload.get("model_ids") or payload.get("models") or [] # 基座模型（配置模型）是平台共享资源，不需要 ACL 授权即可用于推理 + if not is_admin(current_user): + for item in model_ids: + model_id = str(item.get("model_id") if isinstance(item, dict) else item) + trained_model = next( + (model for model in get_platform_store().trained_models() if str(model.get("id")) == model_id or str(model.get("name")) == model_id), + None, + ) + if trained_model and not has_resource_access("trained_model", str(trained_model["id"]), current_user, "execute"): + raise fail(403, f"no permission to use trained model: {model_id}") + if not payload.get("resource_versions"): + payload["resource_versions"] = _capture_resource_versions(get_platform_store(), payload) task = get_platform_store().create_compare_task(payload) return ok({"id": task["id"]}) @router.post("/model-compare/all/stop-all") -async def model_compare_stop_all() -> dict[str, Any]: +async def model_compare_stop_all(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok({"stopped": True}) @router.post("/model-compare/stop-by-pid") -async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok({"stopped": True, "pid": payload.get("pid")}) @@ -2414,6 +3017,8 @@ async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = No results.append({"node_id": node["id"], "node_code": node.get("code"), "success": False, "error": str(exc)}) finally: store.mark_inference_unloaded(node["id"]) + if task: + store.release_external_gpus("inference", str(task.get("id") or ""), node["id"]) return {"unloaded": bool(results), "nodes": results} @@ -2444,7 +3049,7 @@ async def model_compare_delete(task_id: str, current_user: dict = Depends(get_cu if not is_admin(current_user) and not is_owner: if not has_resource_access("compare", task_id, current_user, "delete"): raise fail(403, "no permission to delete inference task") - pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}") + pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}", "inference.delete") if pending: return pending get_platform_store().delete_compare_task(task_id) @@ -2456,11 +3061,13 @@ async def model_compare_delete(task_id: str, current_user: dict = Depends(get_cu @router.get("/model-compare/{task_id}/load-status") -async def model_compare_load_status(task_id: str) -> dict[str, Any]: +async def model_compare_load_status(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: task = get_platform_store().compare_task(task_id) except KeyError: raise fail(404, "compare task not found") + if not has_resource_access("compare", task_id, current_user, "read"): + raise fail(403, "no permission to access inference status") load_status = task.get("load_status") or {"loaded_models": []} if isinstance(load_status, str): try: @@ -2471,7 +3078,9 @@ async def model_compare_load_status(task_id: str) -> dict[str, Any]: @router.post("/model-compare/{task_id}/load-status") -async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("compare", task_id, current_user, "write"): + raise fail(403, "no permission to update inference status") try: return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}})) except KeyError: @@ -2502,6 +3111,7 @@ def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[ if item.get("node_id") in taken_node_ids and item.get("status") in {"ready", "running"}: item["status"] = "error" item["error"] = "模型已被其他推理任务替换" + store.release_external_gpus("inference", str(other["id"]), item.get("node_id")) changed = True if changed: new_status = "loaded" if any(i.get("status") in {"ready", "running"} for i in items) else "failed" @@ -2582,17 +3192,15 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr candidate_nodes = candidate_nodes[:1] for node in candidate_nodes: try: - node_gpu_map = { - int(gpu.get("id", gpu.get("gpu_index", -1))): gpu - for gpu in store.gpus() - if gpu.get("node_id") == node["id"] - } - unavailable = [ - index for index in item_gpu_indices - if node_gpu_map.get(index, {}).get("status") != "idle" - ] - if unavailable: - raise RuntimeError(f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}") + if not is_admin(current_user) and not store.check_gpu_access(current_user["id"], node["id"], item_gpu_indices): + raise PermissionError("selected GPU is not assigned to the current user") + store.reserve_external_gpus( + "inference", + str(task_id), + node["id"], + item_gpu_indices, + tenant_id=str(current_user.get("tenant_id") or "default"), + ) if get_settings().minio_enabled: await _wait_for_object_storage() client = ComputeNodeClient(node["api_base_url"]) @@ -2602,6 +3210,7 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr item_dispatched = True break except Exception as exc: # noqa: BLE001 - try next candidate node + store.release_external_gpus("inference", str(task_id), node["id"]) errors.append(f"{node.get('name') or node.get('code')}: {exc}") if not item_dispatched: loaded_models.append({**item, "status": "error", "error": "; ".join(errors) or "load dispatch failed"}) @@ -2635,13 +3244,16 @@ async def model_compare_unload(task_id: str, current_user: dict = Depends(get_cu @router.post("/model-compare/{task_id}/start-model") -async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not has_resource_access("compare", task_id, current_user, "execute"): + raise fail(403, "no permission to start inference task") return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id}) @router.post("/model-compare/chat-with-port") -async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Proxy non-streaming chat to the compute node running the inference model.""" + _authorize_inference_payload(payload, current_user) store = get_platform_store() node = _node_for_inference_payload(store, payload) if not node: @@ -2655,21 +3267,24 @@ async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> d @router.post("/model-compare/stream-chat") -async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> StreamingResponse: +async def model_compare_stream_chat(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> StreamingResponse: """Stream chat from the compute node (SSE proxy).""" + _authorize_inference_payload(payload, current_user) return await _stream_chat_proxy(payload) @router.post("/model-chat/batch") -async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_chat_batch(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_inference_payload(payload, current_user) return ok({"responses": [], "request": payload}) @router.post("/model-chat/local/chat") -async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_chat_local(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Proxy chat to the compute node running the inference model.""" + _authorize_inference_payload(payload, current_user) store = get_platform_store() - node = _select_first_online_node(store) + node = _node_for_inference_payload(store, payload) if not node: return ok({"response": "no online compute node available for inference", "request": payload}) try: @@ -2681,19 +3296,21 @@ async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any @router.post("/model-chat/local/chat/stream") -async def model_chat_local_stream(payload: dict[str, Any] = Body(...)) -> StreamingResponse: +async def model_chat_local_stream(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> StreamingResponse: """Stream chat from the compute node.""" + _authorize_inference_payload(payload, current_user) return await _stream_chat_proxy(payload) @router.post("/model-chat/local/preload") -async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def model_chat_local_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Load a model on the compute node for inference.""" + _authorize_inference_payload(payload, current_user) model_path = (payload.get("model_name_or_path") or "").strip() if not model_path: return ok({"loaded": False, "error": "model_name_or_path is required"}) store = get_platform_store() - node = _select_first_online_node(store) + node = _node_for_inference_payload(store, payload) if not node: return ok({"loaded": False, "error": "no online compute node"}) try: @@ -2708,9 +3325,14 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[ @router.post("/model-chat/local/unload") -async def model_chat_local_unload() -> dict[str, Any]: +async def model_chat_local_unload(payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """Unload the inference model from the compute node.""" store = get_platform_store() + task_id = str((payload or {}).get("task_id") or (payload or {}).get("compare_task_id") or "") + if task_id and not has_resource_access("compare", task_id, current_user, "execute"): + raise fail(403, "no permission to unload inference task") + if not task_id and not is_admin(current_user): + raise fail(403, "task_id is required for non-administrator unload") # 释放所有已加载推理的节点（短超时，best-effort） results: list[dict[str, Any]] = [] for n in store.compute_nodes(): @@ -2739,8 +3361,16 @@ async def model_chat_local_unload() -> dict[str, Any]: @router.get("/model-chat/local/status") -async def model_chat_local_status() -> dict[str, Any]: +async def model_chat_local_status( + task_id: str | None = Query(default=None), + current_user: dict = Depends(get_current_user), +) -> dict[str, Any]: """Get inference session status from compute node.""" + if not is_admin(current_user): + if not task_id: + raise fail(403, "task_id is required for non-administrator status queries") + if not has_resource_access("compare", task_id, current_user, "read"): + raise fail(403, "no permission to inspect inference status") store = get_platform_store() node = _select_first_online_node(store) if not node: @@ -2756,8 +3386,14 @@ async def model_chat_local_status() -> dict[str, Any]: @router.post("/model-chat/trained/preload") async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "") - # 训练模型需要 ACL 授权；基座模型（配置模型）是平台共享资源，不需要 ACL - if resource_id and not has_resource_access("trained_model", resource_id, current_user, "execute"): + # 训练模型必须对应一个仍然存在的资源。管理员可以兼容历史的 + # 直接路径调用，普通用户不能绕过资源 ACL 直接加载任意节点路径。 + trained_resource = resource_record("trained_model", resource_id) if resource_id else None + if resource_id and not trained_resource: + raise fail(404, "trained model not found") + if not resource_id and not is_admin(current_user): + raise fail(403, "trained_model_id is required for non-administrator preload") + if trained_resource and not has_resource_access("trained_model", resource_id, current_user, "execute"): raise fail(403, "no permission to load this trained model") """Load a trained model (base + adapter) on the compute node for inference.""" model_path = (payload.get("model_name_or_path") or "").strip() @@ -2804,23 +3440,127 @@ async def presign_storage_object(payload: dict[str, Any] = Body(...), current_us raise fail(503, "MinIO object storage is disabled") resource_type = str(payload.get("resource_type") or "") resource_id = str(payload.get("resource_id") or "") + method = str(payload.get("method") or "put").lower() + if resource_type not in _STORAGE_RESOURCE_ROOTS or not resource_id: + raise fail(400, "unsupported resource_type or missing resource_id") + if not _STORAGE_ID_PATTERN.fullmatch(resource_id): + raise fail(400, "invalid resource_id") + if not _storage_resource_row(get_platform_store(), resource_type, resource_id): + raise fail(404, "storage resource not found") + if method not in {"put", "get"}: + raise fail(400, "method must be put or get") + if not has_resource_access(resource_type, resource_id, current_user, "download" if method == "get" else "write"): + raise fail(403, f"no permission to {method} this resource") + if resource_type == "model" and method == "put" and not is_admin(current_user): + raise fail(403, "only administrators can upload base model objects") version_id = str(payload.get("version_id") or uuid.uuid4().hex) - object_key = str(payload.get("object_key") or f"{resource_type}/{resource_id}/versions/{version_id}/resource") - if not resource_type or not resource_id: - raise fail(400, "resource_type and resource_id are required") - if payload.get("method", "put").lower() == "get" and not has_resource_access(resource_type, resource_id, current_user, "read"): - raise fail(403, "no permission to read this resource") + if not _STORAGE_ID_PATTERN.fullmatch(version_id): + raise fail(400, "invalid version_id") try: storage = get_object_storage() - url = storage.presigned_get(object_key) if payload.get("method", "put").lower() == "get" else storage.presigned_put(object_key) - record = get_platform_store().create_storage_object({ - "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id, - "bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"), - "content_type": payload.get("content_type"), "created_by": current_user.get("id"), - }) - return ok({"url": url, "method": payload.get("method", "put").lower(), "expires_seconds": 3600, "object": record}) + store = get_platform_store() + if method == "get": + object_key = _validate_storage_key(resource_type, resource_id, version_id, payload.get("object_key"), payload.get("file_name")) + record = next( + (item for item in store.storage_objects_for_resource(resource_type, resource_id) if item.get("object_key") == object_key), + None, + ) + if not record: + raise fail(404, "storage object not found") + url = storage.presigned_get(object_key) + else: + object_key = _validate_storage_key(resource_type, resource_id, version_id, payload.get("object_key"), payload.get("file_name")) + expected_size = int(payload.get("byte_size") or payload.get("size_bytes") or 0) + if expected_size < 0 or expected_size > get_settings().minio_presign_max_bytes: + raise fail(400, "invalid byte_size") + content_type = str(payload.get("content_type") or "application/octet-stream").split(";", 1)[0].strip().lower() + if content_type not in _STORAGE_CONTENT_TYPES and not content_type.startswith(("application/vnd.", "text/", "image/")): + raise fail(400, "unsupported content_type") + url = storage.presigned_put(object_key) + record = store.create_storage_object({ + "resource_type": resource_type, "resource_id": resource_id, "version_id": version_id, + "bucket": storage.bucket, "object_key": object_key, "file_name": payload.get("file_name"), + "content_type": content_type, "byte_size": expected_size, + "metadata": { + "expected_size": expected_size, + "expected_checksum_sha256": payload.get("checksum_sha256") or "", + "upload_created_at": utcnow(), + "upload_expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + }, + "created_by": current_user.get("id"), + }) + return ok({"url": url, "method": method, "expires_seconds": 3600, "object": record}) except ObjectStorageError as exc: raise fail(503, str(exc)) + except ValueError as exc: + raise fail(400, str(exc)) + + +@router.post("/storage/objects/{object_id}/complete") +async def complete_storage_object( + object_id: str, + payload: dict[str, Any] = Body(default={}), + current_user: dict[str, Any] = Depends(get_current_user), +) -> dict[str, Any]: + """Verify a presigned upload and make it visible to resource consumers.""" + if not get_settings().minio_enabled: + raise fail(503, "MinIO object storage is disabled") + store = get_platform_store() + try: + record = store.storage_object(object_id) + except KeyError: + raise fail(404, "storage object not found") + if not has_resource_access(str(record["resource_type"]), str(record["resource_id"]), current_user, "write"): + raise fail(403, "no permission to complete this upload") + if record.get("status") == "available": + return ok(record) + try: + metadata = record.get("metadata") or {} + if isinstance(metadata, str): + metadata = json.loads(metadata or "{}") + expires_at = str(metadata.get("upload_expires_at") or "") + if expires_at: + try: + if datetime.fromisoformat(expires_at.replace("Z", "+00:00")) <= datetime.now(timezone.utc): + raise fail(409, "presigned upload has expired") + except ValueError: + pass + stat = get_object_storage().stat(str(record["object_key"])) + except (ObjectStorageError, ValueError, json.JSONDecodeError) as exc: + raise fail(409, f"uploaded object is not available: {exc}") + expected_size = int(metadata.get("expected_size") or record.get("byte_size") or 0) + actual_size = int(stat.get("byte_size") or 0) + expected_checksum = str(metadata.get("expected_checksum_sha256") or "").lower() + supplied_checksum = str(payload.get("checksum_sha256") or "").lower() + if expected_checksum and supplied_checksum and supplied_checksum != expected_checksum: + raise fail(409, "uploaded object checksum does not match presign request") + if expected_checksum: + digest = hashlib.sha256() + try: + for chunk in get_object_storage().iter_bytes(str(record["object_key"])): + digest.update(chunk) + except ObjectStorageError as exc: + raise fail(409, f"uploaded object checksum could not be verified: {exc}") + if digest.hexdigest().lower() != expected_checksum: + raise fail(409, "uploaded object content checksum does not match presign request") + if actual_size > get_settings().minio_presign_max_bytes: + raise fail(409, "uploaded object exceeds configured size limit") + if expected_size and actual_size != expected_size: + raise fail(409, f"uploaded object size mismatch: expected {expected_size}, got {actual_size}") + supplied_size = payload.get("byte_size") + if supplied_size is not None and int(supplied_size) != actual_size: + raise fail(409, "uploaded object size does not match completion payload") + completed_metadata = {**metadata, "etag": stat.get("etag") or "", "completed_at": utcnow()} + completed = store.update_storage_object( + object_id, + { + "status": "available", + "byte_size": actual_size, + "checksum_sha256": payload.get("checksum_sha256") or record.get("checksum_sha256") or "", + "metadata": completed_metadata, + }, + ) + return ok(completed) @router.get("/storage/resources/{resource_type}/{resource_id}") @@ -2830,6 +3570,32 @@ async def storage_resource_objects(resource_type: str, resource_id: str, current return ok(get_platform_store().storage_objects_for_resource(resource_type, resource_id)) +@router.get("/storage/resources/{resource_type}/{resource_id}/manifest") +async def storage_resource_manifest(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + """Return the immutable object/version manifest used by training and inference.""" + if resource_type not in _STORAGE_RESOURCE_TABLES or not _storage_resource_row(get_platform_store(), resource_type, resource_id): + raise fail(404, "storage resource not found") + if not has_resource_access(resource_type, resource_id, current_user, "read"): + raise fail(403, "no permission to read this resource") + objects = get_platform_store().storage_objects_for_resource(resource_type, resource_id) + versions: dict[str, dict[str, Any]] = {} + for item in objects: + version_id = str(item.get("version_id") or "legacy") + entry = versions.setdefault(version_id, {"version_id": version_id, "objects": [], "total_bytes": 0}) + manifest_item = { + "id": item.get("id"), + "object_key": item.get("object_key"), + "file_name": item.get("file_name"), + "content_type": item.get("content_type"), + "byte_size": int(item.get("byte_size") or 0), + "checksum_sha256": item.get("checksum_sha256") or "", + "status": item.get("status"), + } + entry["objects"].append(manifest_item) + entry["total_bytes"] += manifest_item["byte_size"] + return ok({"resource_type": resource_type, "resource_id": resource_id, "versions": list(versions.values())}) + + @router.post("/storage/resources/{resource_type}/{resource_id}/prepare/{node_id}") async def prepare_storage_resource( resource_type: str, @@ -2853,7 +3619,15 @@ async def prepare_storage_resource( for obj in objects: url = get_object_storage().presigned_get(obj["object_key"]) filename = Path(str(obj.get("file_name") or obj["object_key"])).name - cache_job = store.create_storage_cache_job({"storage_object_id": obj["id"], "node_id": node_id, "direction": "download"}) + cache_job = store.create_storage_cache_job({ + "storage_object_id": obj["id"], + "node_id": node_id, + "direction": "download", + "version_id": obj.get("version_id"), + "checksum_sha256": obj.get("checksum_sha256") or "", + "byte_size": obj.get("byte_size") or 0, + "protected_until": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + }) try: result = await client.prepare_cache({ "resource_id": resource_id, @@ -2863,7 +3637,10 @@ async def prepare_storage_resource( "byte_size": obj.get("byte_size") or 0, "relative_path": f"{resource_type}s/{resource_id}/{filename}", }) - store.update_storage_cache_job(cache_job["id"], {"status": "completed", "progress": 100, "local_path": result.get("local_path"), "completed_at": utcnow()}) + store.update_storage_cache_job(cache_job["id"], { + "status": "completed", "progress": 100, "local_path": result.get("local_path"), + "last_accessed_at": utcnow(), "completed_at": utcnow(), + }) except Exception as exc: store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()}) raise @@ -2873,7 +3650,80 @@ async def prepare_storage_resource( @router.get("/storage/cache/jobs/{node_id}") async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - return ok(get_platform_store().storage_cache_jobs_for_node(node_id, limit)) + store = get_platform_store() + if not is_admin(current_user): + assigned_nodes = {item["node_id"] for item in store.gpu_assignments_for_user(str(current_user.get("id") or ""))} + if node_id not in assigned_nodes: + raise fail(403, "no permission to inspect this compute node cache") + jobs = store.storage_cache_jobs_for_node(node_id, limit) + if not is_admin(current_user): + visible = [] + for job in jobs: + try: + obj = store.storage_object(str(job.get("storage_object_id") or "")) + except KeyError: + continue + if has_resource_access(str(obj.get("resource_type") or ""), str(obj.get("resource_id") or ""), current_user, "read"): + visible.append(job) + jobs = visible + return ok(jobs) + + +@router.post("/storage/cleanup") +async def cleanup_storage_objects( + payload: dict[str, Any] = Body(default={}), + current_user: dict = Depends(require_admin), +) -> dict[str, Any]: + """Purge MinIO objects whose business resources were soft-deleted.""" + store = get_platform_store() + limit = max(1, min(int(payload.get("limit") or 100), 1000)) + dry_run = bool(payload.get("dry_run", False)) + candidates = store.storage_objects_pending_cleanup(limit) + if dry_run: + return ok({"dry_run": True, "total": len(candidates), "items": candidates}) + results: list[dict[str, Any]] = [] + for item in candidates: + job = store.create_storage_cleanup_job(str(item["id"])) + try: + get_object_storage().delete(str(item["object_key"])) + store.finish_storage_cleanup_job(job["id"], True) + results.append({"storage_object_id": item["id"], "status": "purged"}) + except Exception as exc: # noqa: BLE001 - retain failed cleanup for retry + store.finish_storage_cleanup_job(job["id"], False, str(exc)) + results.append({"storage_object_id": item["id"], "status": "failed", "error": str(exc)}) + return ok({"dry_run": False, "total": len(results), "items": results}) + + +@router.get("/storage/orphans") +async def storage_orphans( + cleanup: bool = Query(default=False), + limit: int = Query(default=500, ge=1, le=5000), + current_user: dict[str, Any] = Depends(require_admin), +) -> dict[str, Any]: + """Find MinIO objects that have no database storage_objects reference.""" + store = get_platform_store() + with store.connect() as conn: + known_rows = conn.execute("SELECT object_key FROM storage_objects WHERE status <> 'purged'").fetchall() + known = {str(row["object_key"]) for row in known_rows} + all_objects: list[dict[str, Any]] = [] + try: + for root in sorted(set(_STORAGE_RESOURCE_ROOTS.values())): + all_objects.extend(get_object_storage().list_objects(f"{root}/")) + except ObjectStorageError as exc: + raise fail(503, str(exc)) + orphans = [item for item in all_objects if str(item.get("object_key") or "") not in known][:limit] + deleted: list[str] = [] + if cleanup: + for item in orphans: + key = str(item.get("object_key") or "") + if not key: + continue + try: + get_object_storage().delete(key) + deleted.append(key) + except Exception: + continue + return ok({"total": len(orphans), "items": orphans, "cleanup": cleanup, "deleted": deleted}) @router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}") @@ -3033,12 +3883,13 @@ async def drain_compute_node(node_id: str, current_user: dict = Depends(get_curr @router.get("/compute/nodes/{node_id}/replicas") -async def compute_node_replicas(node_id: str) -> dict[str, Any]: +async def compute_node_replicas(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_compute_node(node_id, current_user) return ok(get_platform_store().replicas(node_id)) @router.get("/compute/sync-jobs/{sync_id}") -async def compute_sync_job_detail(sync_id: str) -> dict[str, Any]: +async def compute_sync_job_detail(sync_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().sync_job(sync_id)) except KeyError: @@ -3046,7 +3897,8 @@ async def compute_sync_job_detail(sync_id: str) -> dict[str, Any]: @router.get("/compute/nodes/{node_id}/replicas/drift") -async def compute_node_replica_drift(node_id: str) -> dict[str, Any]: +async def compute_node_replica_drift(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_compute_node(node_id, current_user) store = get_platform_store() node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) if not node: @@ -3170,6 +4022,7 @@ async def compute_node_replica_repair( node_id: str, background_tasks: BackgroundTasks, payload: dict[str, Any] | None = Body(default=None), + current_user: dict = Depends(require_admin), ) -> dict[str, Any]: store = get_platform_store() node = next((item for item in store.compute_nodes() if item["id"] == node_id), None) @@ -3203,7 +4056,8 @@ async def compute_node_replica_repair( @router.get("/compute/nodes/{node_id}/engines") -async def compute_node_engines(node_id: str) -> dict[str, Any]: +async def compute_node_engines(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + _authorize_compute_node(node_id, current_user) node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None) if not node: raise fail(404, "compute node not found") @@ -3245,13 +4099,17 @@ async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[s @router.get("/compute/queue") -async def compute_queue() -> dict[str, Any]: +async def compute_queue(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().queue()) @router.get("/compute/jobs/{job_id}") -async def compute_job_detail(job_id: str) -> dict[str, Any]: +async def compute_job_detail(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: task = _task_for_compute_job(job_id) + if task and not has_resource_access("fine-tune", str(task["id"]), current_user, "read"): + raise fail(403, "no permission to access this compute job") + if not task and not is_admin(current_user): + raise fail(403, "only administrators can access this compute job") if not task: node = _node_for_compute_job_record(job_id) if not node: @@ -3289,8 +4147,13 @@ async def compute_job_logs( tail_lines: int | None = Query(default=200, ge=1, le=5000), offset: int | None = Query(default=None, ge=0), limit: int | None = Query(default=None, ge=1, le=5000), + current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: task = _task_for_compute_job(job_id) + if task and not has_resource_access("fine-tune", str(task["id"]), current_user, "read"): + raise fail(403, "no permission to access compute job logs") + if not task and not is_admin(current_user): + raise fail(403, "only administrators can access compute job logs") if not task: node = _node_for_compute_job_record(job_id) if not node: @@ -3313,7 +4176,7 @@ async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(d @router.post("/compute/jobs/{job_id}/priority") -async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: task = _task_for_compute_job(job_id) if not task: raise fail(404, "compute job not found") @@ -3322,12 +4185,12 @@ async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) @router.post("/internal/compute-sync/jobs/poll") -async def poll_compute_jobs() -> dict[str, Any]: +async def poll_compute_jobs(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(await poll_compute_jobs_once()) @router.post("/internal/compute-sync/resources") -async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def create_compute_sync(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() node_id = payload.get("target_node_id") or payload.get("target_compute_node_id") if not node_id: @@ -3366,7 +4229,7 @@ async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, @router.get("/internal/compute-sync/resources/{sync_id}") -async def compute_sync_detail(sync_id: str) -> dict[str, Any]: +async def compute_sync_detail(sync_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().sync_job(sync_id)) except KeyError: @@ -3374,12 +4237,12 @@ async def compute_sync_detail(sync_id: str) -> dict[str, Any]: @router.get("/training-log-files") -async def training_log_files() -> dict[str, Any]: +async def training_log_files(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().training_log_files()) @router.get("/training-log-content") -async def training_log_content(file: str = Query(...)) -> dict[str, Any]: +async def training_log_content(file: str = Query(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().training_log_content(file)) except KeyError: @@ -3387,15 +4250,15 @@ async def training_log_content(file: str = Query(...)) -> dict[str, Any]: @router.get("/log-files") -async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]: +async def log_files(date: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().log_files(date)) @router.get("/log-content") -async def log_content(file: str = Query(...)) -> dict[str, Any]: +async def log_content(file: str = Query(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().log_content(file)) @router.post("/web-log") -async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: +async def web_log(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: return ok({"received": True, **payload}) diff --git a/backend/app/core/audit.py b/backend/app/core/audit.py index 6601c52..8cafb34 100644 --- a/backend/app/core/audit.py +++ b/backend/app/core/audit.py @@ -21,7 +21,7 @@ from typing import Any, Callable, Optional, TypeVar from fastapi import Request -from app.core.logging import get_logger, request_id_var +from app.core.logging import get_client_ip, get_logger, mask_sensitive_string, request_id_var logger = get_logger("app.audit") @@ -67,12 +67,25 @@ def audit_log( detail=detail, trace_id=trace_id, duration_ms=elapsed_ms, + kwargs=kwargs, + args=args, ) return result - except Exception: - logger.error( - "审计日志记录失败 action=%s", action, exc_info=True + except Exception as exc: + _record_audit( + action=action, + actor_id=_extract_actor_id(kwargs), + target_type=target_type, + target_id=_extract_target_id(None, kwargs, extract_target_id), + detail=_build_detail(detail_template, kwargs), + trace_id=trace_id, + duration_ms=(time.perf_counter() - started_at) * 1000, + result="failure", + reason=_safe_exception_reason(exc), + kwargs=kwargs, + args=args, ) + logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc)) raise return async_wrapper # type: ignore @@ -94,12 +107,25 @@ def audit_log( detail=detail, trace_id=trace_id, duration_ms=elapsed_ms, + kwargs=kwargs, + args=args, ) return result - except Exception: - logger.error( - "审计日志记录失败 action=%s", action, exc_info=True + except Exception as exc: + _record_audit( + action=action, + actor_id=_extract_actor_id(kwargs), + target_type=target_type, + target_id=_extract_target_id(None, kwargs, extract_target_id), + detail=_build_detail(detail_template, kwargs), + trace_id=trace_id, + duration_ms=(time.perf_counter() - started_at) * 1000, + result="failure", + reason=_safe_exception_reason(exc), + kwargs=kwargs, + args=args, ) + logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc)) raise return sync_wrapper # type: ignore @@ -144,18 +170,36 @@ def _record_audit( detail: str, trace_id: str, duration_ms: float, + *, + kwargs: dict[str, Any] | None = None, + args: tuple[Any, ...] = (), + result: str = "success", + reason: str | None = None, ) -> None: """通过已有的 record_audit 方法写入审计日志""" try: from app.db.platform_store import get_platform_store store = get_platform_store() + kwargs = kwargs or {} + request = _extract_request(args, kwargs) + current_user = kwargs.get("current_user") or kwargs.get("user") or {} + request_id = request.headers.get("X-Request-ID") if request else None + request_id = request_id or trace_id + client_ip = get_client_ip(request) or None + detail_text = f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}" store.record_audit( action=action, actor_id=actor_id, target_type=target_type or None, target_id=target_id, - detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}", + tenant_id=str(current_user.get("tenant_id") or "") or None, + detail=mask_sensitive_string(detail_text), + result=result, + reason=mask_sensitive_string(reason or "") or None, + request_id=request_id, + session_id=str(current_user.get("session_id") or "") or None, + ip=client_ip, ) except Exception: logger.error("写入审计日志失败 action=%s", action, exc_info=True) @@ -170,6 +214,19 @@ def _extract_actor_id(kwargs: dict) -> Optional[str]: return None +def _extract_request(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Request | None: + for value in tuple(kwargs.values()) + tuple(args): + if isinstance(value, Request): + return value + return None + + +def _safe_exception_reason(exc: Exception) -> str: + """Keep audit failures useful without recording credentials or tokens.""" + value = getattr(exc, "detail", None) or str(exc) or exc.__class__.__name__ + return mask_sensitive_string(str(value))[:500] + + # ==================== 预定义的审计操作常量 ==================== class AuditActions: diff --git a/backend/app/core/auth.py b/backend/app/core/auth.py index 403e75e..e00c92b 100644 --- a/backend/app/core/auth.py +++ b/backend/app/core/auth.py @@ -2,20 +2,76 @@ from __future__ import annotations from typing import Any +import json from fastapi import Depends, HTTPException, Query, Request, status from app.db.platform_store import get_platform_store +from app.core.logging import get_client_ip # 无需鉴权的路径前缀（健康检查、登录等） PUBLIC_PATHS = ("/health", "/login", "/system-info") OWNER_TABLES = { "dataset": ("datasets", "created_by"), "model": ("models", "created_by"), "trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"), - "fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"), + "fine-tune": ("fine_tune_tasks", "created_by"), "fine_tune_task": ("fine_tune_tasks", "created_by"), "compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"), - "project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"), + "project": ("projects", "create_by"), "data_process": ("data_process_tasks", "created_by"), + "data_convert": ("data_convert_tasks", "created_by"), } +RESOURCE_TABLES = { + "dataset": "datasets", + "model": "models", + "trained_model": "trained_models", + "eval": "eval_tasks", + "fine-tune": "fine_tune_tasks", + "fine_tune_task": "fine_tune_tasks", + "compare": "compare_tasks", + "inference": "compare_tasks", + "project": "projects", + "data_process": "data_process_tasks", + "data_convert": "data_convert_tasks", +} +MODULE_PERMISSIONS = { + "dashboard": "dashboard", + "fine-tune": "fine-tune", + "model-eval": "model-eval", + "model-compare": "model-inference", + "model-inference": "model-inference", + "model-chat": "model-inference", + "model-manage": "model-manage", + "dataset-manage": "dataset", + "data-process": "data-process", + "data-convert": "data-convert", + "compute": "compute", + "hardware": "hardware", + "users": "user-settings", +} + +# These endpoints are the user-facing compute view used by training, +# inference, and evaluation forms. They only return the current user's +# assigned nodes/GPUs in the endpoint implementation, so they must remain +# available after an approval without granting access to the admin compute +# management page. +SELF_SERVICE_COMPUTE_PATHS = { + "/compute/nodes", + "/compute/gpus", + "/compute/my-gpus", +} + +# Resource actions are deliberately kept separate from module permissions. +# A user may be allowed to open a module while still lacking the action on a +# specific resource (for example, download or delete). +RESOURCE_ACTIONS = frozenset({ + "read", + "write", + "execute", + "download", + "export", + "delete", + "admin", +}) +RESOURCE_ACTION_ALIASES = {"export": "download"} def _extract_token(request: Request) -> str | None: @@ -30,6 +86,26 @@ def _session_token(user_id: str, session_id: str) -> str: return f"platform-token-{user_id}.{session_id}" +def _record_auth_event(actor_id: str | None, action: str, reason: str, request: Request) -> None: + """Best-effort security audit for authentication and permission denials.""" + try: + get_platform_store().record_audit( + action=action, + actor_id=actor_id, + target_type="auth", + target_id=request.url.path, + detail=reason, + result="denied", + reason=reason, + request_id=request.headers.get("X-Request-ID"), + ip=get_client_ip(request) or None, + ) + except Exception: + # An audit failure must never turn an authentication decision into an + # accidental allow or an unrelated 500 response. + pass + + def get_current_user(request: Request) -> dict[str, Any]: """ FastAPI 依赖：解析当前登录用户。 @@ -55,31 +131,219 @@ def get_current_user(request: Request) -> dict[str, Any]: "SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,) ).fetchone() if not session or session["user_id"] != user_id or session["logout_at"]: + _record_auth_event(user_id, "auth.session.denied", "session expired or logged out", request) raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired") if session["expires_at"]: from datetime import datetime, timezone try: if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc): + _record_auth_event(user_id, "auth.session.denied", "session expired", request) raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired") except ValueError: pass with store.connect() as conn: user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone() if user_row: - return store._user(user_row) + user = store._user(user_row) + if user.get("status") != "active": + _record_auth_event(user_id, "auth.user.disabled", "user is not active", request) + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user disabled") + requested_tenant = request.headers.get("X-Tenant-ID", "").strip() + if requested_tenant and (is_admin(user) or requested_tenant in user_tenant_ids(user)): + user["tenant_id"] = requested_tenant + # Keep the session identifier in request context so business audit + # records can be traced back to the exact login session. + if session_id: + user["session_id"] = session_id + path_parts = path.strip("/").split("/") + # The API may be mounted directly at /modelTF or behind /api/v1/modelTF. + # Locate the first known module segment instead of relying on a fixed index. + segment = next((part for part in path_parts if part in MODULE_PERMISSIONS), "") + required = MODULE_PERMISSIONS.get(segment) + relative_path = "/" + "/".join(path_parts[path_parts.index(segment):]) if segment else path + self_service_compute = relative_path.rstrip("/") in SELF_SERVICE_COMPUTE_PATHS + if ( + required + and not is_admin(user) + and required not in (user.get("permissions") or []) + and not self_service_compute + ): + _record_auth_event(user_id, "auth.permission.denied", f"missing permission: {required}", request) + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"missing permission: {required}") + return user raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found") def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]: - """FastAPI 依赖：要求当前用户是管理员（role=admin 或 protected）。""" - if current_user.get("role") == "admin" or current_user.get("protected"): + """FastAPI 依赖：要求当前用户是平台管理员。""" + if is_admin(current_user): return current_user raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required") +def require_tenant_admin( + tenant_id: str, + current_user: dict[str, Any] = Depends(get_current_user), +) -> dict[str, Any]: + """Allow platform admins and active owner/admin tenant members.""" + if is_admin(current_user) or is_tenant_admin(current_user, tenant_id): + return current_user + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant admin permission required") + + def is_admin(user: dict[str, Any]) -> bool: - """判断用户是否为管理员（admin 角色或 protected 标记）。""" - return user.get("role") == "admin" or user.get("protected", False) + """判断用户是否为平台管理员；兼容历史 role=admin/protected 数据。""" + return ( + user.get("platform_role") == "platform_admin" + or user.get("role") == "admin" + or user.get("protected", False) + ) + + +def user_tenant_ids(user: dict[str, Any]) -> set[str]: + """Return the tenant scope of a user. + + Existing installations keep the primary tenant on ``users.tenant_id``. + ``tenant_members`` is optional during the migration and adds memberships + when the permission v2 schema is available. + """ + if is_admin(user): + return {"*"} + primary_tenant = str(user.get("tenant_id") or "default") + result: set[str] = set() + user_id = user.get("id") + if not user_id: + return result + try: + store = get_platform_store() + with store.connect() as conn: + rows = conn.execute( + "SELECT tenant_id FROM tenant_members " + "WHERE user_id=? AND status='active' " + "AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)", + (user_id,), + ).fetchall() + result.update(str(row["tenant_id"]) for row in rows if row.get("tenant_id")) + if not result: + active_tenant = conn.execute( + "SELECT id FROM tenants WHERE id=? " + "AND COALESCE(status, 'active')='active' " + "AND COALESCE(deleted_at, '')=''", + (primary_tenant,), + ).fetchone() + if active_tenant: + result.add(primary_tenant) + except Exception: + # Older databases are upgraded lazily. The primary users.tenant_id + # remains a valid fallback until the additive table is available. + result.add(primary_tenant) + return result + + +def tenant_membership(user: dict[str, Any], tenant_id: str | None = None) -> dict[str, Any] | None: + """Return the active membership for the selected tenant, if any.""" + if is_admin(user): + return {"tenant_id": tenant_id or user.get("tenant_id") or "default", "role": "owner", "status": "active"} + target = str(tenant_id or user.get("tenant_id") or "default") + try: + with get_platform_store().connect() as conn: + row = conn.execute( + "SELECT tenant_id, role, status, expires_at FROM tenant_members " + "WHERE tenant_id=? AND user_id=? AND status='active'", + (target, user.get("id")), + ).fetchone() + if not row: + return None + if row.get("expires_at"): + from datetime import datetime, timezone + try: + if datetime.fromisoformat(str(row["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc): + return None + except ValueError: + return None + return dict(row) + except Exception: + return None + + +def is_tenant_admin(user: dict[str, Any], tenant_id: str | None = None) -> bool: + membership = tenant_membership(user, tenant_id) + return bool(membership and membership.get("role") in {"owner", "admin"}) + + +def is_tenant_admin_for_resource(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool: + if is_admin(user) or resource_type == "model": + return False + tenant_id = resource_tenant_id(resource_type, resource) + return bool(tenant_id and is_tenant_admin(user, tenant_id)) + + +def bind_active_tenant(payload: dict[str, Any], user: dict[str, Any]) -> dict[str, Any]: + """Bind a new resource to the authenticated tenant context. + + Platform administrators may explicitly create a resource in another + active tenant. Ordinary users can only use the tenant selected by the + authenticated session/X-Tenant-ID header, never a client-supplied tenant + id alone. + """ + requested = str(payload.get("tenant_id") or user.get("tenant_id") or "default") + if not is_admin(user) and requested not in user_tenant_ids(user): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant access denied") + payload["tenant_id"] = requested if is_admin(user) else str(user.get("tenant_id") or requested) + return payload + + +def resource_record(resource_type: str, resource_id: str) -> dict[str, Any] | None: + """Load a resource row for authorization without exposing storage details.""" + table = RESOURCE_TABLES.get(resource_type) + if not table or not resource_id: + return None + store = get_platform_store() + try: + with store.connect() as conn: + row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (resource_id,)).fetchone() + except Exception: + return None + if not row: + return None + result = dict(row) + if result.get("deleted_at"): + return None + return result + + +def resource_tenant_id(resource_type: str, resource: dict[str, Any]) -> str | None: + """Resolve tenant ownership, including legacy JSON-backed task rows.""" + # Administrator-created base models are platform shared. Online models + # created by ordinary users retain tenant scope, so their visibility can + # be limited to the owner and members of that tenant. + if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}: + return None + tenant_id = resource.get("tenant_id") + if tenant_id: + return str(tenant_id) + payload = resource.get("payload") + if payload: + try: + data = json.loads(payload) if isinstance(payload, str) else payload + if isinstance(data, dict) and data.get("tenant_id"): + return str(data["tenant_id"]) + except (TypeError, ValueError, json.JSONDecodeError): + pass + return None + + +def resource_in_user_tenant(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool: + if is_admin(user): + return True + if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}: + return True + tenant_id = resource_tenant_id(resource_type, resource) + if not tenant_id: + # A missing tenant is not an implicit shared scope. The migration must + # assign historical rows before ordinary users can access them. + return False + return "*" in user_tenant_ids(user) or tenant_id in user_tenant_ids(user) def has_resource_access( @@ -93,16 +357,57 @@ def has_resource_access( - admin/protected 用户直接放行（旁路）。 - 其他用户检查 acls 表中是否有对应授权。 """ - if user.get("role") == "admin" or user.get("protected"): + if permission not in RESOURCE_ACTIONS: + return False + permission = RESOURCE_ACTION_ALIASES.get(permission, permission) + if is_admin(user): return True store = get_platform_store() + resource = resource_record(resource_type, resource_id) + if not resource: + return False + # Some historical datasets were created before tenant_id was introduced. + # They are not shared by default, but an explicit user ACL granted by an + # administrator is still a valid compatibility path for that legacy row. + legacy_unscoped = resource_type != "model" and not resource_tenant_id(resource_type, resource) + if not resource_in_user_tenant(resource_type, resource, user) and not legacy_unscoped: + return False + if resource_type == "model": + model_source = str(resource.get("model_source") or "").lower() + if permission in {"read", "execute"}: + # Local/registered base models are platform resources. For online + # models, only administrator-created records are global; a normal + # user's online model is shared with the active tenant below. + if model_source not in {"api", "online"}: + return True + creator_id = str(resource.get("created_by") or "") + if creator_id: + with store.connect() as conn: + creator = conn.execute( + "SELECT platform_role, role, protected FROM users WHERE id=?", + (creator_id,), + ).fetchone() + if creator and ( + creator.get("platform_role") == "platform_admin" + or creator.get("role") == "admin" + or bool(creator.get("protected")) + ): + return True + # The tenant check above has already rejected models outside the + # user's active tenant. Same-tenant online models are usable. + if resource_tenant_id(resource_type, resource) in user_tenant_ids(user): + return True + if is_tenant_admin_for_resource(resource_type, resource, user): + return True acls = store.get_acl(resource_type, resource_id) user_id = user.get("id") - user_role = user.get("role") + # ACL role principals are tenant roles, not platform roles. Falling back + # to the platform role keeps compatibility with old ACL records. + membership = tenant_membership(user, resource_tenant_id(resource_type, resource)) + user_role = (membership or {}).get("role") or user.get("role") - owner_tables = OWNER_TABLES - table_info = owner_tables.get(resource_type) + table_info = OWNER_TABLES.get(resource_type) if table_info and user_id: table, column = table_info with store.connect() as conn: @@ -111,7 +416,6 @@ def has_resource_access( owner = row[column] if column == "payload": try: - import json owner = json.loads(owner or "{}").get("created_by") except (TypeError, ValueError): owner = None @@ -119,6 +423,15 @@ def has_resource_access( return True for entry in acls: + if entry.get("revoked_at"): + continue + if entry.get("expires_at"): + from datetime import datetime, timezone + try: + if datetime.fromisoformat(str(entry["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc): + continue + except ValueError: + pass # 按 user 授权 if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id: if _permission_covers(entry.get("permission"), permission): @@ -154,48 +467,13 @@ def filter_accessible_resource_ids( - admin 直接返回全部。 - 普通用户查 acls 表取交集。 """ - if user.get("role") == "admin" or user.get("protected"): + if is_admin(user): return all_ids if not all_ids: return [] - store = get_platform_store() - user_id = user.get("id") - user_role = user.get("role") - - # 查询该用户在该资源类型下有 read 权限的所有 resource_id - with store.connect() as conn: - rows = conn.execute( - """ - SELECT DISTINCT resource_id FROM acls - WHERE resource_type=? AND ( - (principal_type='user' AND principal_id=?) - OR (principal_type='role' AND principal_id=?) - ) - """, - (resource_type, user_id, user_role), - ).fetchall() - - accessible = {r["resource_id"] for r in rows} - if resource_type in OWNER_TABLES: - table, column = OWNER_TABLES[resource_type] - if column == "payload": - # payload 是 JSON 字符串，需要查出后解析 created_by - with store.connect() as conn: - owned = conn.execute(f"SELECT id, {column} FROM {table}").fetchall() - for row in owned: - try: - import json - payload = json.loads(row[column] or "{}") - if payload.get("created_by") == user_id: - accessible.add(row["id"]) - except (TypeError, ValueError): - pass - else: - with store.connect() as conn: - owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall() - accessible.update(row["id"] for row in owned) + accessible = filter_accessible_resource_ids_batch(resource_type, all_ids, user) return [rid for rid in all_ids if rid in accessible] @@ -204,37 +482,134 @@ def filter_accessible_resource_ids_batch( resource_ids: list[str], user: dict[str, Any], ) -> set[str]: - """Filter a list endpoint with one ACL query instead of one query per row.""" + """Filter a list endpoint with a bounded set of SQL queries. + + The previous implementation called ``has_resource_access`` once per + resource. Each call loaded the resource, tenant membership and ACL again, + which made ordinary-user list pages slow on a remote PostgreSQL server. + """ if is_admin(user): return set(resource_ids) - if not resource_ids: + ids = list(dict.fromkeys(str(item) for item in resource_ids if item)) + if not ids: return set() + + table = RESOURCE_TABLES.get(resource_type) + if not table: + return set() + placeholders = ",".join("?" for _ in ids) + primary_tenant = str(user.get("tenant_id") or "default") + tenant_ids = {primary_tenant} + tenant_roles: dict[str, str] = {primary_tenant: str(user.get("role") or "")} + user_id = str(user.get("id") or "") store = get_platform_store() - placeholders = ",".join("?" for _ in resource_ids) with store.connect() as conn: - rows = conn.execute( - f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) " - "AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))", - (resource_type, *resource_ids, user.get("id"), user.get("role")), + memberships = conn.execute( + "SELECT tenant_id, role FROM tenant_members " + "WHERE user_id=? AND status='active' " + "AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)", + (user_id,), ).fetchall() - accessible = {row["resource_id"] for row in rows} - table_info = OWNER_TABLES.get(resource_type) - if table_info and user.get("id"): - table, column = table_info - with store.connect() as conn: - owned = conn.execute( - f"SELECT id, {column} FROM {table} WHERE id IN ({placeholders})", - (*resource_ids,), - ).fetchall() - for row in owned: - owner = row[column] - # 如果列是 payload（JSON），需要解析后提取 created_by - if column == "payload": + for membership in memberships: + tenant_id = str(membership["tenant_id"] or "") + if tenant_id: + tenant_ids.add(tenant_id) + tenant_roles[tenant_id] = str(membership["role"] or "") + + rows = conn.execute( + f"SELECT * FROM {table} WHERE id IN ({placeholders})", tuple(ids) + ).fetchall() + row_by_id = {str(row["id"]): dict(row) for row in rows} + acl_rows = conn.execute( + "SELECT resource_id, principal_type, principal_id, permission, expires_at " + f"FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) " + "AND (revoked_at IS NULL OR revoked_at='')", + (resource_type, *ids), + ).fetchall() + + acl_by_resource: dict[str, list[dict[str, Any]]] = {} + for row in acl_rows: + acl_by_resource.setdefault(str(row["resource_id"]), []).append(dict(row)) + + allowed: set[str] = set() + for resource_id in ids: + resource = row_by_id.get(resource_id) + if not resource or resource.get("deleted_at"): + continue + if resource_type == "model": + model_source = str(resource.get("model_source") or "").lower() + if model_source not in {"api", "online"}: + allowed.add(resource_id) + continue + creator_id = str(resource.get("created_by") or "") + if creator_id: + with store.connect() as conn: + creator = conn.execute( + "SELECT platform_role, role, protected FROM users WHERE id=?", + (creator_id,), + ).fetchone() + if creator and ( + creator.get("platform_role") == "platform_admin" + or creator.get("role") == "admin" + or bool(creator.get("protected")) + ): + allowed.add(resource_id) + continue + tenant_id = resource_tenant_id(resource_type, resource) + if tenant_id and tenant_id in tenant_ids: + allowed.add(resource_id) + continue + tenant_id = resource_tenant_id(resource_type, resource) + if not tenant_id: + # Legacy rows without a tenant are never implicitly visible. Only + # a direct user ACL can expose one to its explicitly named user; + # role ACLs remain blocked until the row is tenant-migrated. + for entry in acl_by_resource.get(resource_id, []): + expires_at = entry.get("expires_at") + if expires_at: + try: + from datetime import datetime, timezone + if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc): + continue + except ValueError: + continue + if ( + entry.get("principal_type") == "user" + and entry.get("principal_id") == user_id + and _permission_covers(entry.get("permission"), "read") + ): + allowed.add(resource_id) + break + continue + if tenant_id not in tenant_ids: + continue + if tenant_roles.get(tenant_id) in {"owner", "admin"}: + allowed.add(resource_id) + continue + owner = resource.get("created_by") + if resource_type in {"eval", "fine-tune", "fine_tune_task", "compare", "inference"} and resource.get("payload"): + try: + payload = json.loads(resource["payload"]) if isinstance(resource["payload"], str) else resource["payload"] + if isinstance(payload, dict) and payload.get("created_by"): + owner = payload["created_by"] + except (TypeError, ValueError, json.JSONDecodeError): + pass + if owner == user_id: + allowed.add(resource_id) + continue + role = tenant_roles.get(tenant_id) or str(user.get("role") or "") + for entry in acl_by_resource.get(resource_id, []): + expires_at = entry.get("expires_at") + if expires_at: try: - import json - owner = json.loads(owner or "{}").get("created_by") - except (TypeError, ValueError): - owner = None - if owner == user["id"]: - accessible.add(row["id"]) - return accessible + from datetime import datetime, timezone + if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc): + continue + except ValueError: + continue + user_match = entry.get("principal_type") == "user" and entry.get("principal_id") == user_id + role_match = entry.get("principal_type") == "role" and entry.get("principal_id") == role + if (user_match or role_match) and _permission_covers(entry.get("permission"), "read"): + allowed.add(resource_id) + break + return allowed diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 99d60e7..8b75042 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -71,6 +71,7 @@ class Settings: # Small text/data files stay inline in PostgreSQL to avoid unnecessary # MinIO round trips. Larger files remain the shared canonical objects. minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024) + minio_presign_max_bytes: int = _int_env("MINIO_PRESIGN_MAX_BYTES", 1024 * 1024 * 1024 * 1024) storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300) storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10) compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "") diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py index 51453f3..bf28341 100644 --- a/backend/app/core/logging.py +++ b/backend/app/core/logging.py @@ -17,6 +17,18 @@ from fastapi import FastAPI, Request from app.core.config import Settings, get_settings request_id_var: ContextVar[str] = ContextVar("request_id", default="-") +client_ip_var: ContextVar[str] = ContextVar("client_ip", default="") + + +def get_client_ip(request: Request | None) -> str: + """获取客户端地址，兼容前置反向代理传递的真实地址。""" + if request is None: + return "" + for header in ("X-Real-IP", "X-Forwarded-For"): + value = request.headers.get(header, "") + if value: + return value.split(",", 1)[0].strip() + return request.client.host if request.client else "" # ==================== 敏感数据脱敏规则 ==================== @@ -65,14 +77,21 @@ def mask_sensitive_string(text: str) -> str: """从文本中脱敏常见敏感信息""" if not text: return text - + + # Mask the value as well as the key. Replacing only ``api_key=`` would + # still leak the credential in audit messages and exception text. + assignment_pattern = ( + r"(Bearer\s+|(?:api[-_]?key|access[_-]?token|refresh[_-]?token|" + r"secret[_-]?key|password|private[_-]?key|token)\s*[:=]\s*)" + r"(\"[^\"]*\"|'[^']*'|[^\s,;]+)" + ) + try: + text = re.sub(assignment_pattern, r"\1***", text, flags=re.IGNORECASE) + except re.error: + pass + patterns = [ - (r'Bearer\s+[A-Za-z0-9\-._]+', '***'), - (r'token\s*[:=]\s*', '***'), - (r'password\s*[:=]\s*', '***'), - (r'secret[_-]?key\s*[:=]', '***'), - (r'api[-_]?key\s*[:=]', '***'), - (r'private[_-]?key\s*[:=]', '***'), + (r'Bearer\s+[A-Za-z0-9\-._]+', 'Bearer ***'), (r'\d{11}', r'\d{3}\*\d{4}'), # 手机号/身份证 (r'1[3-9]\d{9}', r'1\*{3}\*{4}'), # 手机号 ] @@ -335,6 +354,7 @@ def setup_request_logging(app: FastAPI) -> None: async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def] request_id = request.headers.get("X-Request-ID") or str(uuid4()) token = request_id_var.set(request_id) + ip_token = client_ip_var.set(get_client_ip(request)) started_at = time.perf_counter() try: response = await call_next(request) @@ -411,6 +431,7 @@ def setup_request_logging(app: FastAPI) -> None: raise finally: request_id_var.reset(token) + client_ip_var.reset(ip_token) # ==================== 配置函数 ==================== diff --git a/backend/app/core/op_log.py b/backend/app/core/op_log.py index 2020356..970b2d1 100644 --- a/backend/app/core/op_log.py +++ b/backend/app/core/op_log.py @@ -31,7 +31,7 @@ from typing import Any, Callable, Optional, TypeVar from fastapi import Request -from app.core.logging import get_logger, request_id_var +from app.core.logging import get_client_ip, get_logger, request_id_var from app.db.platform_store import get_platform_store, new_id, utcnow logger = get_logger("app.op_log") @@ -351,7 +351,7 @@ def _write_log( req_method = None req_path = None if request: - client_ip = request.client.host if request.client else None + client_ip = get_client_ip(request) or None req_method = request.method req_path = request.url.path diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index ce35551..762302b 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -17,6 +17,7 @@ import psycopg from psycopg_pool import ConnectionPool from app.core.config import get_settings +from app.core.logging import client_ip_var ALL_PERMISSIONS = [ "dashboard", @@ -209,11 +210,12 @@ def parse_training_metric_line(line: str) -> dict[str, float] | None: return None result: dict[str, float] = {} number_pattern = r"([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)" - step_match = re.search(rf"(?:^|[\s,{{])['\"]?step['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I) + quoted_number = rf"['\"]?\s*{number_pattern}\s*['\"]?" + step_match = re.search(rf"(?:^|[\s,{{])['\"]?step['\"]?\s*(?:=|:)\s*{quoted_number}", line, re.I) if step_match: result["step"] = float(step_match.group(1)) for key in ["loss", "grad_norm", "learning_rate", "epoch"]: - match = re.search(rf"['\"]?{key}['\"]?\s*(?:=|:)\s*{number_pattern}", line, re.I) + match = re.search(rf"['\"]?{key}['\"]?\s*(?:=|:)\s*{quoted_number}", line, re.I) if match: result[key] = float(match.group(1)) return result or None @@ -470,6 +472,218 @@ class PlatformStore: self._inference_nodes.discard(node_id) self._inference_gpu_indexes.pop(node_id, None) + def reserve_external_gpus( + self, + owner_type: str, + owner_id: str, + node_id: str, + gpu_indices: list[int] | list[Any], + tenant_id: str | None = None, + ) -> list[int]: + """Atomically reserve GPUs for evaluation or inference. + + Training uses ``gpu_allocations`` because it owns a fine-tune task. + Evaluation and inference have different lifecycle tables, so they use + this common reservation table to close the scheduling race between + resource selection and remote job/model dispatch. + """ + if owner_type not in {"eval", "inference"}: + raise ValueError("unsupported external GPU reservation owner") + requested = sorted({int(item) for item in gpu_indices or []}) + if any(item < 0 for item in requested): + raise ValueError("GPU index must be non-negative") + if not requested: + return [] + with self.connect() as conn: + lock_owner = f"gpu:{owner_type}:{owner_id}:{uuid.uuid4().hex[:8]}" + if not self._acquire_scheduler_lock(conn, "compute-scheduler", lock_owner): + raise RuntimeError("compute scheduler is busy, please retry") + node = conn.execute( + "SELECT * FROM compute_nodes WHERE id=?", (node_id,) + ).fetchone() + if not node or not node["enabled"] or node["scheduler_status"] not in {"online", "draining"}: + raise RuntimeError("selected compute node is unavailable") + node_dict = {**dict(node), "gpu_count": int(node.get("gpu_count") or 0)} + available = self._node_gpu_indexes(conn, node_dict) + if not set(requested).issubset(available): + raise RuntimeError(f"requested GPU is not available on compute node {node.get('code')}") + own_rows = conn.execute( + "SELECT gpu_index FROM gpu_reservations WHERE owner_type=? AND owner_id=? AND status='reserved'", + (owner_type, owner_id), + ).fetchall() + own = {int(row["gpu_index"]) for row in own_rows} + active = self._active_gpu_indexes(conn, node_id) - own + if set(requested).intersection(active): + raise RuntimeError(f"selected GPU is busy on compute node {node.get('code')}") + self._reserve_tenant_quota_locked( + conn, + tenant_id or "default", + owner_type, + owner_id, + len(requested), + ) + now = utcnow() + for gpu_index in requested: + if gpu_index in own: + continue + conn.execute( + """ + INSERT INTO gpu_reservations + (id, node_id, gpu_index, owner_type, owner_id, status, create_time) + VALUES (?, ?, ?, ?, ?, 'reserved', ?) + """, + (new_id("gpu_res"), node_id, gpu_index, owner_type, owner_id, now), + ) + conn.execute("DELETE FROM scheduler_locks WHERE lock_key=? AND owner=?", ("compute-scheduler", lock_owner)) + return requested + + def release_external_gpus( + self, + owner_type: str, + owner_id: str, + node_id: str | None = None, + ) -> None: + clauses = ["owner_type=?", "owner_id=?", "status='reserved'"] + params: list[Any] = [owner_type, owner_id] + if node_id: + clauses.append("node_id=?") + params.append(node_id) + params.append(utcnow()) + with self.connect() as conn: + conn.execute( + f"UPDATE gpu_reservations SET status='released', released_at=? WHERE {' AND '.join(clauses)}", + (params[-1], *params[:-1]), + ) + self._release_tenant_quota_locked(conn, owner_type, owner_id) + + def release_external_gpus_for_node(self, node_id: str) -> None: + with self.connect() as conn: + owners = conn.execute( + "SELECT DISTINCT owner_type, owner_id FROM gpu_reservations " + "WHERE node_id=? AND status='reserved'", + (node_id,), + ).fetchall() + conn.execute( + "UPDATE gpu_reservations SET status='released', released_at=? WHERE node_id=? AND status='reserved'", + (utcnow(), node_id), + ) + for owner in owners: + self._release_tenant_quota_locked(conn, owner["owner_type"], owner["owner_id"]) + + @staticmethod + def _quota_limit(quota: Any, *keys: str) -> int: + value = quota + if isinstance(value, str): + try: + value = json_loads(value, {}) + except Exception: + value = {} + if not isinstance(value, dict): + return 0 + for key in keys: + if key in value: + try: + return max(0, int(value[key] or 0)) + except (TypeError, ValueError): + return 0 + return 0 + + def _reserve_tenant_quota_locked( + self, + conn: PgConnection, + tenant_id: str, + owner_type: str, + owner_id: str, + gpu_count: int = 0, + ) -> dict[str, Any]: + """Reserve tenant GPU quota in the same transaction as GPU binding.""" + tenant_id = str(tenant_id or "default") + gpu_count = max(0, int(gpu_count or 0)) + existing = conn.execute( + "SELECT * FROM tenant_quota_reservations " + "WHERE owner_type=? AND owner_id=? AND status='reserved' FOR UPDATE", + (owner_type, owner_id), + ).fetchone() + if existing: + return dict(existing) + tenant = conn.execute( + "SELECT id, quota, status, deleted_at FROM tenants WHERE id=? FOR UPDATE", + (tenant_id,), + ).fetchone() + if tenant and (tenant.get("status") not in {None, "active"} or tenant.get("deleted_at")): + raise RuntimeError("tenant is inactive") + quota = json_loads(tenant.get("quota"), {}) if tenant else {} + gpu_limit = self._quota_limit(quota, "gpu", "gpu_quota", "max_gpu") + used = conn.execute( + "SELECT COALESCE(SUM(gpu_count), 0) AS value FROM tenant_quota_reservations " + "WHERE tenant_id=? AND status='reserved'", + (tenant_id,), + ).fetchone()["value"] + if gpu_limit > 0 and int(used or 0) + gpu_count > gpu_limit: + raise RuntimeError( + f"tenant GPU quota exceeded: requested={gpu_count}, " + f"used={int(used or 0)}, limit={gpu_limit}" + ) + reservation = { + "id": new_id("quota"), + "tenant_id": tenant_id, + "owner_type": owner_type, + "owner_id": owner_id, + "gpu_count": gpu_count, + "storage_bytes": 0, + "status": "reserved", + "create_time": utcnow(), + } + conn.execute( + "INSERT INTO tenant_quota_reservations " + "(id, tenant_id, owner_type, owner_id, gpu_count, storage_bytes, status, create_time) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + tuple(reservation.values()), + ) + return reservation + + def _release_tenant_quota_locked(self, conn: PgConnection, owner_type: str, owner_id: str) -> None: + conn.execute( + "UPDATE tenant_quota_reservations SET status='released', released_at=? " + "WHERE owner_type=? AND owner_id=? AND status='reserved'", + (utcnow(), owner_type, owner_id), + ) + + def reserve_tenant_quota( + self, + tenant_id: str, + owner_type: str, + owner_id: str, + gpu_count: int = 0, + ) -> dict[str, Any]: + with self.connect() as conn: + return self._reserve_tenant_quota_locked( + conn, tenant_id, owner_type, owner_id, gpu_count + ) + + def release_tenant_quota(self, owner_type: str, owner_id: str) -> None: + with self.connect() as conn: + self._release_tenant_quota_locked(conn, owner_type, owner_id) + + def tenant_quota_usage(self, tenant_id: str) -> dict[str, Any]: + with self.connect() as conn: + tenant = conn.execute("SELECT quota FROM tenants WHERE id=?", (tenant_id,)).fetchone() + row = conn.execute( + "SELECT COALESCE(SUM(gpu_count), 0) AS gpu_reserved, " + "COALESCE(SUM(storage_bytes), 0) AS storage_reserved, COUNT(*) AS reservations " + "FROM tenant_quota_reservations WHERE tenant_id=? AND status='reserved'", + (tenant_id,), + ).fetchone() + quota = json_loads(tenant.get("quota"), {}) if tenant else {} + return { + "tenant_id": tenant_id, + "quota": quota, + "gpu_limit": self._quota_limit(quota, "gpu", "gpu_quota", "max_gpu"), + "gpu_reserved": int(row.get("gpu_reserved") or 0), + "storage_reserved": int(row.get("storage_reserved") or 0), + "reservations": int(row.get("reservations") or 0), + } + def is_inference_loaded(self, node_id: str) -> bool: return node_id in self._inference_nodes @@ -496,10 +710,19 @@ class PlatformStore: def ensure_schema(self) -> None: schema_path = Path(__file__).with_name("sql") / "001_platform_runtime.sql" with self.connect() as conn: - conn.executescript(schema_path.read_text(encoding="utf-8")) + # Multiple workers/health checks can initialize PlatformStore at + # the same time. Serialize the additive DDL on PostgreSQL so the + # first request cannot deadlock with the background poller. + conn.execute("SELECT pg_advisory_xact_lock(73582001)") + # Offline packages keep only the complete 000_full_init.sql. The + # runtime migration is optional because a fresh offline database + # already contains the complete schema. + if schema_path.exists(): + conn.executescript(schema_path.read_text(encoding="utf-8")) user_columns = self._column_names(conn, "users") if "password" in user_columns and "password_hash" not in user_columns: conn.execute("ALTER TABLE users RENAME COLUMN password TO password_hash") + self._ensure_columns(conn, "users", {"tenant_id": "TEXT NOT NULL DEFAULT 'default'"}) self._ensure_columns( conn, "compute_nodes", @@ -510,7 +733,17 @@ class PlatformStore: }, ) self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"}) - self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"}) + self._ensure_columns( + conn, + "fine_tune_tasks", + { + "compute_job_id": "TEXT", + "tenant_id": "TEXT NOT NULL DEFAULT 'default'", + "created_by": "TEXT", + "deleted_at": "TEXT", + "deleted_by": "TEXT", + }, + ) self._ensure_columns( conn, "sessions", @@ -541,10 +774,40 @@ class PlatformStore: ) for table in ("models", "datasets", "eval_tasks"): self._ensure_columns(conn, table, {"deleted_at": "TEXT", "deleted_by": "TEXT", "tenant_id": "TEXT", "project_id": "TEXT", "created_by": "TEXT"}) + self._ensure_columns( + conn, + "models", + { + "storage_status": "TEXT NOT NULL DEFAULT 'pending'", + "storage_error": "TEXT", + "storage_version_id": "TEXT", + }, + ) + conn.execute( + "UPDATE models SET storage_status=CASE " + "WHEN model_source IN ('api', 'online') THEN 'not_applicable' " + "WHEN storage_status IS NULL OR storage_status='' THEN 'pending' " + "ELSE storage_status END " + "WHERE storage_status IS NULL OR storage_status='' OR model_source IN ('api', 'online')" + ) + self._ensure_columns( + conn, + "users", + { + "deleted_at": "TEXT", + "deleted_by": "TEXT", + "platform_role": "TEXT NOT NULL DEFAULT 'platform_user'", + }, + ) + self._ensure_columns(conn, "compare_tasks", {"deleted_at": "TEXT", "deleted_by": "TEXT"}) + self._ensure_columns(conn, "eval_tasks", {"created_by": "TEXT", "tenant_id": "TEXT"}) + self._ensure_columns(conn, "compare_tasks", {"created_by": "TEXT", "tenant_id": "TEXT"}) self._ensure_columns( conn, "resource_replicas", { + "version_id": "TEXT", + "storage_object_id": "TEXT", "checksum_sha256": "TEXT", "byte_size": "BIGINT NOT NULL DEFAULT 0", "last_checked_at": "TEXT", @@ -554,7 +817,14 @@ class PlatformStore: self._ensure_columns( conn, "storage_objects", - {"metadata": "TEXT NOT NULL DEFAULT '{}'"}, + { + "metadata": "TEXT NOT NULL DEFAULT '{}'", + "deleted_at": "TEXT", + "deleted_by": "TEXT", + "cleanup_attempts": "INTEGER NOT NULL DEFAULT 0", + "last_cleanup_error": "TEXT", + "last_verified_at": "TEXT", + }, ) self._ensure_columns( conn, @@ -567,6 +837,13 @@ class PlatformStore: "003_model_path_governance.sql", "003_tenant_quota.sql", "004_permissions.sql", + "006_gpu_reservations.sql", + "007_platform_completion.sql", + "008_permission_v2.sql", + "009_permission_completion.sql", + "010_permission_quota_membership.sql", + "011_permission_lifecycle.sql", + "012_tenant_user_hierarchy.sql", ): extra_path = schema_dir / extra if extra_path.exists(): @@ -577,6 +854,9 @@ class PlatformStore: "data_convert_tasks", { "created_by": "TEXT", + "processed_by": "TEXT", + "processed_at": "TEXT", + "deleted_by": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'", "output_storage_object_id": "TEXT", "output_content": "TEXT", @@ -628,12 +908,46 @@ class PlatformStore: def ensure_seed_data(self) -> None: with self.connect() as conn: + # Platform administration has its own stable tenant. Keep the + # legacy default tenant for historical users and resources. + conn.execute( + """ + INSERT INTO tenants (id, name, code, status, quota, create_time) + VALUES ('admin', '管理员租户', 'admin', 'active', '{}', ?) + ON CONFLICT (id) DO UPDATE SET + name = '管理员租户', code = 'admin', status = 'active', + deleted_at = NULL, deleted_by = NULL + """, + (utcnow(),), + ) + conn.execute( + "UPDATE users SET display_name='Admin', tenant_id='admin' " + "WHERE username='admin' AND COALESCE(deleted_at, '')=''" + ) + admin_row = conn.execute( + "SELECT id FROM users WHERE username='admin' AND COALESCE(deleted_at, '')=''" + ).fetchone() + if admin_row: + conn.execute( + """ + INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) + VALUES ('admin', ?, 'owner', 'active', ?) + ON CONFLICT (tenant_id, user_id) DO UPDATE + SET role='owner', status='active' + """, + (admin_row["id"], utcnow()), + ) + conn.execute( + "UPDATE tenant_members SET status='disabled' " + "WHERE tenant_id='default' AND user_id=?", + (admin_row["id"],), + ) if conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] > 0: return now = utcnow() users = [ - ("u_admin", "admin", "admin123", "Platform Admin", "admin", "active", ALL_PERMISSIONS, 1), + ("u_admin", "admin", "admin123", "Admin", "admin", "active", ALL_PERMISSIONS, 1, "admin"), ( "u_operator", "operator", @@ -641,17 +955,18 @@ class PlatformStore: "Platform Operator", "operator", "active", - [p for p in ALL_PERMISSIONS if p != "user-settings"], - 0, - ), + [p for p in ALL_PERMISSIONS if p != "user-settings"], + 0, + "default", + ), ] conn.executemany( """ INSERT INTO users - (id, username, password_hash, display_name, role, status, permissions, create_time, protected) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, username, password_hash, display_name, role, status, permissions, create_time, protected, tenant_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, - [(u[0], u[1], hash_password(u[2]), u[3], u[4], u[5], json_dumps(u[6]), now, u[7]) for u in users], + [(u[0], u[1], hash_password(u[2]), u[3], u[4], u[5], json_dumps(u[6]), now, u[7], u[8]) for u in users], ) def _duration(self, start_time: str | None, end_time: str | None = None) -> str: @@ -738,20 +1053,32 @@ class PlatformStore: """ UPDATE trained_models SET compute_node_id=COALESCE(compute_node_id, ?), - compute_node_name=COALESCE(compute_node_name, ?) + compute_node_name=COALESCE(compute_node_name, ?), + created_by=COALESCE(created_by, ?), + tenant_id=COALESCE(tenant_id, ?) WHERE id=? """, - (task.get("compute_node_id"), task.get("compute_node_code") or task.get("compute_node_name"), exists["id"]), + ( + task.get("compute_node_id"), + task.get("compute_node_code") or task.get("compute_node_name"), + task.get("created_by"), + task.get("tenant_id") or "default", + exists["id"], + ), ) return - model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone() + base_model_lookup = str(task.get("base_model_id") or task.get("base_model") or "") + model = conn.execute( + "SELECT path FROM models WHERE id=? OR name=? OR path=?", + (base_model_lookup, base_model_lookup, base_model_lookup), + ).fetchone() output_dir = task.get("output_dir") or f"/data/yg-ft/outputs/{task['name']}" trained_model_id = new_id("tm") conn.execute( """ INSERT INTO trained_models - (id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir, compute_node_id, compute_node_name, created_by) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, name, train_methods, base_model_path, create_time, merged, merging, merged_path, artifact_dir, compute_node_id, compute_node_name, created_by, tenant_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( trained_model_id, @@ -766,6 +1093,7 @@ class PlatformStore: task.get("compute_node_id"), task.get("compute_node_code") or task.get("compute_node_name"), task.get("created_by"), + task.get("tenant_id") or "default", ), ) # Use real artifact data from compute node when available @@ -819,11 +1147,23 @@ class PlatformStore: "trained_model", trained_model_id, "base_model", - str(task.get("base_model") or ""), + str(task.get("base_model_id") or task.get("base_model") or ""), "fine_tuned_from", task.get("compute_job_id"), {"task_id": task.get("id"), "output_dir": output_dir}, ) + dataset_id = task.get("train_dataset_id") or task.get("dataset_id") + if dataset_id: + self._insert_model_lineage( + conn, + "trained_model", + trained_model_id, + "dataset", + str(dataset_id), + "trained_on", + task.get("compute_job_id"), + {"task_id": task.get("id"), "dataset_id": str(dataset_id)}, + ) def _upsert_compute_job(self, conn: PgConnection, task: dict[str, Any], job: dict[str, Any], status: str) -> None: job_id = str(job.get("id") or task.get("compute_job_id") or task["id"]) @@ -1024,10 +1364,21 @@ class PlatformStore: def record_training_log_metrics(self, task_id: str, content: str) -> int: rows: list[tuple[Any, ...]] = [] + fallback_step = 0 + summary_metrics: list[dict[str, float]] = [] for line_number, line in enumerate(content.splitlines(), start=1): metric = parse_training_metric_line(line) if not metric: continue + # LLaMA-Factory prints a final ``train_loss`` summary in addition + # to the per-step history. Keep the history points for the curve + # and use the summary only when no history was emitted. + if "train_loss" in line and not re.search(r"['\"]loss['\"]", line): + summary_metrics.append(metric) + continue + if metric.get("step") is None: + fallback_step += 1 + metric["step"] = float(fallback_step) rows.append( ( new_id("metric"), @@ -1041,6 +1392,21 @@ class PlatformStore: utcnow(), ) ) + if not rows and summary_metrics: + metric = summary_metrics[-1] + rows.append( + ( + new_id("metric"), + task_id, + 1, + metric.get("epoch"), + metric.get("loss"), + metric.get("grad_norm"), + metric.get("learning_rate"), + "train_loss summary", + utcnow(), + ) + ) with self.connect() as conn: conn.execute("DELETE FROM fine_tune_metrics WHERE task_id=?", (task_id,)) if rows: @@ -1099,6 +1465,40 @@ class PlatformStore: ).fetchall() return [{**dict(row), "payload": json_loads(row["payload"], {})} for row in rows] + def standalone_compute_jobs_pending_archive(self) -> list[dict[str, Any]]: + """Return completed merge/export jobs whose node output is not archived yet.""" + with self.connect() as conn: + rows = conn.execute( + "SELECT * FROM compute_jobs WHERE task_id IS NULL AND engine IN ('merge','export','llama_factory_export') AND status='completed' ORDER BY update_time" + ).fetchall() + result: list[dict[str, Any]] = [] + for row in rows: + item = {**dict(row), "payload": json_loads(row["payload"], {})} + if str(item["payload"].get("archive_status") or "") != "completed": + result.append(item) + return result + + def update_compute_job_archive( + self, + job_id: str, + status: str, + object_ids: list[str], + error: str = "", + ) -> dict[str, Any]: + current = self.compute_job(job_id) + payload = { + **(current.get("payload") or {}), + "archive_status": status, + "archive_object_ids": object_ids, + "archive_error": error, + } + with self.connect() as conn: + conn.execute( + "UPDATE compute_jobs SET payload=?, update_time=? WHERE id=?", + (json_dumps(payload), utcnow(), job_id), + ) + return self.compute_job(job_id) + def record_model_merge_job( self, node: dict[str, Any], @@ -1107,6 +1507,7 @@ class PlatformStore: trained_model_id: str | None = None, ) -> dict[str, Any]: job_id = str(job.get("id") or payload.get("id") or new_id("merge")) + engine = str(payload.get("engine") or "merge") now = utcnow() command = job.get("command") or [] command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "") @@ -1141,18 +1542,22 @@ class PlatformStore: now if str(job.get("status")) in {"completed", "failed", "stopped"} else None, ), ) - if trained_model_id: + if trained_model_id and engine == "merge": conn.execute("UPDATE trained_models SET merging=1 WHERE id=? OR name=?", (trained_model_id, trained_model_id)) + if trained_model_id: conn.execute( """ INSERT INTO model_export_jobs - (id, trained_model_id, compute_job_id, node_id, export_type, quantization_bit, status, output_dir, payload, create_time, completed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, trained_model_id, compute_job_id, node_id, export_type, quantization_bit, status, output_dir, payload, create_time, completed_at, + created_by, tenant_id, archive_status, archive_error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET status=EXCLUDED.status, output_dir=EXCLUDED.output_dir, payload=EXCLUDED.payload, - completed_at=EXCLUDED.completed_at + completed_at=EXCLUDED.completed_at, + created_by=COALESCE(EXCLUDED.created_by, model_export_jobs.created_by), + tenant_id=COALESCE(EXCLUDED.tenant_id, model_export_jobs.tenant_id) """, ( job_id, @@ -1166,6 +1571,10 @@ class PlatformStore: json_dumps(payload), now, now if str(job.get("status")) in {"completed", "failed", "stopped"} else None, + payload.get("created_by"), + payload.get("tenant_id") or "default", + str(payload.get("archive_status") or "pending"), + str(payload.get("archive_error") or ""), ), ) return self.compute_job(job_id) @@ -1176,6 +1585,7 @@ class PlatformStore: job_payload = payload.get("job") if isinstance(payload.get("job"), dict) else {} merged_payload = {**payload, "job": {**job_payload, **job}} status = str(job.get("status") or current.get("status") or "queued") + engine = str(payload.get("engine") or current.get("engine") or "merge") command = job.get("command") or current.get("command") or [] command_text = " ".join(str(part) for part in command) if isinstance(command, list) else str(command or "") output_dir = job.get("output_dir") or payload.get("output_dir") or current.get("output_dir") @@ -1198,10 +1608,22 @@ class PlatformStore: ), ) trained_model_id = payload.get("trained_model_id") or payload.get("model_name") - if trained_model_id and status == "completed": + if trained_model_id and status == "completed" and engine == "merge": conn.execute( - "UPDATE trained_models SET merged=1, merging=0, merged_path=? WHERE id=? OR name=?", - (output_dir or "", trained_model_id, trained_model_id), + """ + UPDATE trained_models + SET merged=1, + merging=0, + merged_path=?, + base_model_path=COALESCE(NULLIF(?, ''), base_model_path) + WHERE id=? OR name=? + """, + ( + output_dir or "", + payload.get("base_model_path") or payload.get("base_model") or "", + trained_model_id, + trained_model_id, + ), ) model_row = conn.execute( "SELECT id, name FROM trained_models WHERE id=? OR name=?", @@ -1242,19 +1664,58 @@ class PlatformStore: job_id, {"adapter_path": payload.get("adapter_name_or_path"), "output_dir": output_dir}, ) - elif trained_model_id and status in {"failed", "stopped"}: + elif trained_model_id and status in {"failed", "stopped"} and engine == "merge": conn.execute("UPDATE trained_models SET merging=0 WHERE id=? OR name=?", (trained_model_id, trained_model_id)) + if trained_model_id and status == "completed" and engine != "merge": + model_row = conn.execute( + "SELECT id FROM trained_models WHERE id=? OR name=?", + (trained_model_id, trained_model_id), + ).fetchone() + artifact_model_id = model_row["id"] if model_row else str(trained_model_id) + artifact_items = job.get("artifacts") or [] + artifact_size = sum(int(item.get("size") or item.get("size_bytes") or 0) for item in artifact_items) + checksums = [str(item.get("checksum_sha256") or "") for item in artifact_items if item.get("checksum_sha256")] + artifact_id = self._upsert_model_artifact( + conn, + artifact_model_id, + "trained_model", + "exported_model", + str(output_dir or ""), + artifact_size, + checksums[0] if len(checksums) == 1 else "", + { + "export_type": engine, + "quantization_bit": payload.get("export_quantization_bit", payload.get("quantization_bit", 0)) or 0, + "artifacts": artifact_items, + "checksums": checksums, + }, + job_id, + ) + self._insert_model_lineage( + conn, + "model_artifact", + artifact_id, + "trained_model", + artifact_model_id, + "exported_from_model", + job_id, + {"source_model": payload.get("model_name_or_path"), "output_dir": output_dir}, + ) if trained_model_id: conn.execute( """ UPDATE model_export_jobs - SET status=?, output_dir=?, payload=?, completed_at=COALESCE(?, completed_at) + SET status=?, output_dir=?, payload=?, archive_status=CASE WHEN ?='completed' THEN 'pending' ELSE archive_status END, + archive_error=CASE WHEN ?='completed' THEN '' ELSE archive_error END, + completed_at=COALESCE(?, completed_at) WHERE compute_job_id=? """, ( status, output_dir, json_dumps(merged_payload), + status, + status, utcnow() if status in {"completed", "failed", "stopped"} else None, job_id, ), @@ -1325,13 +1786,43 @@ class PlatformStore: def users(self) -> list[dict[str, Any]]: with self.connect() as conn: rows = conn.execute("SELECT * FROM users ORDER BY create_time").fetchall() - return [self._user(row) for row in rows] + memberships = conn.execute( + "SELECT tenant_id, user_id, role, status, expires_at FROM tenant_members ORDER BY tenant_id" + ).fetchall() + by_user: dict[str, list[dict[str, Any]]] = {} + for membership in memberships: + by_user.setdefault(str(membership["user_id"]), []).append(dict(membership)) + result = [] + for row in rows: + user = self._user(row) + user["tenant_memberships"] = by_user.get(str(user["id"]), []) + user["tenant_count"] = len(user["tenant_memberships"]) + result.append(user) + return result def login(self, username: str, password: str) -> dict[str, Any] | None: with self.connect() as conn: row = conn.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone() if not row or row["status"] != "active": return None + is_platform_admin = ( + row.get("platform_role") == "platform_admin" + or row.get("role") == "admin" + or bool(row.get("protected")) + ) + if not is_platform_admin: + active_membership = conn.execute( + """SELECT 1 + FROM tenant_members tm + JOIN tenants t ON t.id=tm.tenant_id + WHERE tm.user_id=? AND tm.status='active' + AND (tm.expires_at IS NULL OR tm.expires_at='' OR tm.expires_at > NOW()::text) + AND COALESCE(t.status, 'active')='active' + AND COALESCE(t.deleted_at, '')='' LIMIT 1""", + (row["id"],), + ).fetchone() + if not active_membership: + return None matched, legacy_plaintext = verify_password(password, row["password_hash"]) if not matched: return None @@ -1349,31 +1840,48 @@ class PlatformStore: def create_user(self, payload: dict[str, Any]) -> dict[str, Any]: user_id = new_id("u") - role = payload.get("role", "user") + requested_role = str(payload.get("role", "operator")) + role = requested_role if requested_role in {"admin", "operator", "viewer", "user"} else "viewer" + platform_role = "platform_admin" if role == "admin" else "platform_user" if role == "admin": permissions = ALL_PERMISSIONS else: - # 普通用户：默认拥有所有业务权限，仅排除 user-settings 和 compute - role = "user" + # 普通用户不能获得平台治理和节点管理权限。 permissions = [p for p in ALL_PERMISSIONS if p not in ("user-settings", "compute")] + tenant_id = str(payload.get("tenant_id") or "admin").strip() or "admin" with self.connect() as conn: + if tenant_id != "default": + tenant = conn.execute("SELECT id, status FROM tenants WHERE id=?", (tenant_id,)).fetchone() + if not tenant or tenant.get("status") not in {None, "active"}: + raise ValueError("tenant not found or inactive") conn.execute( """ INSERT INTO users - (id, username, password_hash, display_name, role, status, permissions, create_time, protected) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + (id, username, password_hash, display_name, role, platform_role, status, permissions, create_time, protected, tenant_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?) """, ( user_id, payload["username"], - hash_password(payload.get("password", "platform123")), + hash_password(payload.get("password", "123456")), payload.get("display_name") or payload["username"], role, + platform_role, payload.get("status", "active"), json_dumps(permissions), utcnow(), + tenant_id, ), ) + tenant_role = str(payload.get("tenant_role") or "member") + if tenant_role not in {"owner", "admin", "member", "viewer"}: + raise ValueError("invalid tenant member role") + conn.execute( + """INSERT INTO tenant_members (tenant_id, user_id, role, status, invited_by, joined_at) + VALUES (?, ?, ?, 'active', ?, ?) ON CONFLICT (tenant_id, user_id) + DO UPDATE SET role=EXCLUDED.role, status='active'""", + (tenant_id, user_id, tenant_role, None, utcnow()), + ) return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()) def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]: @@ -1381,7 +1889,15 @@ class PlatformStore: row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone() if not row: raise KeyError(user_id) - is_admin = row["role"] == "admin" or bool(row["protected"]) + if row.get("deleted_at") and payload.get("status") == "active": + raise ValueError("deleted user cannot be reactivated") + requested_status = payload.get("status", row["status"]) + if requested_status not in {"active", "disabled", "pending"}: + raise ValueError("invalid user status; use delete endpoint for tombstoning") + requested_role = payload.get("role", row["role"]) + if requested_role not in {"admin", "operator", "viewer", "user"}: + raise ValueError("invalid user role") + is_admin = row.get("platform_role") == "platform_admin" or row["role"] == "admin" or bool(row["protected"]) if "permissions" in payload: perms = payload["permissions"] if is_admin: @@ -1392,56 +1908,113 @@ class PlatformStore: perms = [p for p in (perms or []) if p not in ("user-settings", "compute")] payload = {**payload, "permissions": perms} values = { - "role": payload.get("role", row["role"]), - "status": payload.get("status", row["status"]), + "role": requested_role, + "platform_role": "platform_admin" if requested_role == "admin" else "platform_user", + "status": requested_status, "permissions": json_dumps( payload.get("permissions", json_loads(row["permissions"], [])) ), } conn.execute( - "UPDATE users SET role=?, status=?, permissions=? WHERE id=?", - (values["role"], values["status"], values["permissions"], user_id), + "UPDATE users SET role=?, platform_role=?, status=?, permissions=? WHERE id=?", + (values["role"], values["platform_role"], values["status"], values["permissions"], user_id), ) + if values["status"] != "active": + conn.execute( + "UPDATE sessions SET logout_at=COALESCE(logout_at, ?) WHERE user_id=? AND logout_at IS NULL", + (utcnow(), user_id), + ) return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()) - def delete_user(self, user_id: str) -> None: + def delete_user(self, user_id: str, deleted_by: str | None = None) -> None: + tombstoned_resources: list[tuple[str, str]] = [] with self.connect() as conn: - row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone() + row = conn.execute("SELECT * FROM users WHERE id=? FOR UPDATE", (user_id,)).fetchone() if not row: raise KeyError(user_id) if row["protected"]: raise ValueError("protected user cannot be deleted") - # 级联删除该用户关联的数据 - tables_to_clean = [ - # 登录会话（避免残留 session 导致统计显示 user_id） - ("sessions", "user_id=?", [user_id]), - # ACL 授权 - ("acls", "principal_type='user' AND principal_id=?", [user_id]), - # 审批实例（申请人） - ("approval_instances", "applicant_id=?", [user_id]), - # 审计日志 - ("audit_logs", "actor_id=?", [user_id]), - # 项目成员 - ("project_members", "user_id=?", [user_id]), - # GPU 分配 - ("gpu_assignments", "user_id=?", [user_id]), - # 数据集 - ("datasets", "created_by=?", [user_id]), - # 基座模型 - ("models", "created_by=?", [user_id]), - # 微调产物 - ("trained_models", "created_by=?", [user_id]), - # 评测任务 - ("eval_tasks", "created_by=?", [user_id]), - # 对比/推理任务（payload 中 creator） - # 训练任务：仅标记为已删除或保留（有 compute_job_id 关联），不清物理数据 - ] - for table_name, where_clause, params in tables_to_clean: + owner_rows = conn.execute( + "SELECT tenant_id FROM tenant_members WHERE user_id=? AND role='owner' AND status='active'", + (user_id,), + ).fetchall() + for owner in owner_rows: + count = conn.execute( + "SELECT COUNT(*) AS count FROM tenant_members WHERE tenant_id=? AND role='owner' AND status='active' AND user_id<>?", + (owner["tenant_id"], user_id), + ).fetchone()["count"] + if int(count or 0) == 0: + raise ValueError("cannot delete the last active tenant owner") + + now = utcnow() + actor = deleted_by or "system" + # Keep the user, audit trail and resource lineage. Tombstoning is + # reversible only through an explicit future recovery workflow. + conn.execute( + "UPDATE users SET status='deleted', deleted_at=?, deleted_by=? WHERE id=?", + (now, actor, user_id), + ) + conn.execute( + "UPDATE sessions SET logout_at=COALESCE(logout_at, ?) WHERE user_id=? AND logout_at IS NULL", + (now, user_id), + ) + conn.execute( + "UPDATE tenant_members SET status='disabled' WHERE user_id=? AND status='active'", + (user_id,), + ) + conn.execute( + "UPDATE acls SET revoked_at=COALESCE(revoked_at, ?) WHERE principal_type='user' AND principal_id=? AND revoked_at IS NULL", + (now, user_id), + ) + conn.execute( + "UPDATE approval_instances SET status='cancelled', execution_status='cancelled', executed_by=?, executed_at=? " + "WHERE applicant_id=? AND status='pending'", + (actor, now, user_id), + ) + conn.execute( + "UPDATE resource_access_requests SET status='cancelled', cancelled_at=COALESCE(cancelled_at, ?), cancelled_by=? " + "WHERE applicant_id=? AND status='pending'", + (now, actor, user_id), + ) + conn.execute("DELETE FROM gpu_assignments WHERE user_id=?", (user_id,)) + resource_tables = ( + ("dataset", "datasets"), + ("model", "models"), + ("trained_model", "trained_models"), + ("eval", "eval_tasks"), + ("data_process", "data_process_tasks"), + ("data_convert", "data_convert_tasks"), + ("compare", "compare_tasks"), + ) + for resource_type, table in resource_tables: try: - conn.execute(f"DELETE FROM {table_name} WHERE {where_clause}", params) + owned = conn.execute( + f"SELECT id FROM {table} WHERE created_by=? AND deleted_at IS NULL", + (user_id,), + ).fetchall() + conn.execute( + f"UPDATE {table} SET deleted_at=COALESCE(deleted_at, ?), deleted_by=COALESCE(deleted_by, ?) " + "WHERE created_by=? AND deleted_at IS NULL", + (now, actor, user_id), + ) + for item in owned: + resource_id = str(item["id"]) + tombstoned_resources.append((resource_type, resource_id)) + # Keep MinIO as the source of truth, but make object + # cleanup explicit and retryable after a user delete. + conn.execute( + "UPDATE storage_objects SET status='deleted', deleted_at=COALESCE(deleted_at, ?), " + "deleted_by=COALESCE(deleted_by, ?) " + "WHERE resource_type=? AND resource_id=? AND status <> 'purged'", + (now, actor, resource_type, resource_id), + ) except Exception: - pass # 表可能不存在或字段不存在，跳过 - conn.execute("DELETE FROM users WHERE id=?", (user_id,)) + # Older installations may not yet contain every optional + # resource table. The user tombstone itself must still + # succeed; available resources are revoked below. + continue + for resource_type, resource_id in tombstoned_resources: + self.revoke_resource_security_state(resource_type, resource_id) def reset_password(self, user_id: str, new_password: str) -> None: with self.connect() as conn: @@ -1456,32 +2029,39 @@ class PlatformStore: ) def _user(self, row: PgRow) -> dict[str, Any]: + platform_role = row.get("platform_role") + if platform_role not in {"platform_admin", "platform_user"}: + platform_role = "platform_admin" if row.get("role") == "admin" or row.get("protected") else "platform_user" return { "id": row["id"], "username": row["username"], "display_name": row["display_name"], "role": row["role"], + "platform_role": platform_role, "status": row["status"], "permissions": json_loads(row["permissions"], []), "create_time": row["create_time"], "last_login": row["last_login"], "protected": bool(row["protected"]), + "tenant_id": row.get("tenant_id") or "default", + "deleted_at": row.get("deleted_at"), + "deleted_by": row.get("deleted_by"), } def models(self) -> list[dict[str, Any]]: with self.connect() as conn: - return [dict(row) for row in conn.execute("SELECT * FROM models ORDER BY create_time DESC").fetchall()] + return [dict(row) for row in conn.execute("SELECT * FROM models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall()] def model(self, model_id: str) -> dict[str, Any]: with self.connect() as conn: - row = conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone() + row = conn.execute("SELECT * FROM models WHERE id=? AND deleted_at IS NULL", (model_id,)).fetchone() if not row: raise KeyError(model_id) return dict(row) def model_by_name(self, name: str) -> dict[str, Any]: with self.connect() as conn: - row = conn.execute("SELECT * FROM models WHERE name=?", (name,)).fetchone() + row = conn.execute("SELECT * FROM models WHERE name=? AND deleted_at IS NULL", (name,)).fetchone() if not row: raise KeyError(name) return dict(row) @@ -1492,12 +2072,13 @@ class PlatformStore: path = payload.get("path", "") # Automatically determine can_train: local models with a path can be trained can_train = 1 if (model_source != "api" and path and str(path).strip()) else 0 + storage_status = payload.get("storage_status") or ("pending" if can_train else "not_applicable") with self.connect() as conn: conn.execute( """ INSERT INTO models - (id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, can_train, create_time, created_by) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, can_train, create_time, created_by, tenant_id, storage_status, storage_error, storage_version_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( model_id, @@ -1513,10 +2094,26 @@ class PlatformStore: can_train, utcnow(), payload.get("created_by"), + payload.get("tenant_id") or "default", + storage_status, + payload.get("storage_error"), + payload.get("storage_version_id"), ), ) return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone()) + def update_model_storage(self, model_id: str, status: str, error: str = "", version_id: str = "") -> dict[str, Any]: + """Persist the MinIO archive state for a registered base model.""" + with self.connect() as conn: + conn.execute( + "UPDATE models SET storage_status=?, storage_error=?, storage_version_id=? WHERE id=?", + (status, error[:2000], version_id or None, model_id), + ) + row = conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone() + if not row: + raise KeyError(model_id) + return dict(row) + def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]: current = self.model(model_id) merged = {**current, **payload} @@ -1524,11 +2121,18 @@ class PlatformStore: model_source = merged.get("model_source", "local") path = merged.get("path", "") can_train = 1 if (model_source != "api" and path and str(path).strip()) else 0 + storage_status = merged.get("storage_status") or ("pending" if can_train else "not_applicable") + storage_error = merged.get("storage_error") or "" + storage_version_id = merged.get("storage_version_id") or "" + if model_source != current.get("model_source") or str(path or "") != str(current.get("path") or ""): + storage_status = "pending" if can_train else "not_applicable" + storage_error = "" + storage_version_id = "" with self.connect() as conn: conn.execute( """ UPDATE models - SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?, can_train=? + SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=?, can_train=?, storage_status=?, storage_error=?, storage_version_id=? WHERE id=? """, ( @@ -1542,19 +2146,47 @@ class PlatformStore: merged.get("api_key"), merged.get("online_model_name"), can_train, + storage_status, + storage_error, + storage_version_id, model_id, ), ) return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone()) + def revoke_resource_security_state(self, resource_type: str, resource_id: str) -> None: + """Revoke permissions and pending workflows when a resource is tombstoned.""" + with self.connect() as conn: + conn.execute( + "UPDATE acls SET revoked_at=COALESCE(revoked_at, ?) WHERE resource_type=? AND resource_id=? AND revoked_at IS NULL", + (utcnow(), resource_type, resource_id), + ) + conn.execute( + "UPDATE resource_access_requests SET status='cancelled', cancelled_at=COALESCE(cancelled_at, ?), cancelled_by='system' " + "WHERE resource_type=? AND resource_id=? AND status='pending'", + (utcnow(), resource_type, resource_id), + ) + conn.execute( + "UPDATE approval_instances SET status='cancelled', execution_status='cancelled' " + "WHERE resource_type=? AND resource_id=? AND status='pending'", + (resource_type, resource_id), + ) + def delete_model(self, model_id: str) -> None: with self.connect() as conn: conn.execute("UPDATE models SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", model_id)) + self.revoke_resource_security_state("model", model_id) def trained_models(self) -> list[dict[str, Any]]: self.refresh_runtime_state() with self.connect() as conn: rows = conn.execute("SELECT * FROM trained_models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall() + task_output_names: dict[str, dict[str, Any]] = {} + for task_row in conn.execute("SELECT payload FROM fine_tune_tasks ORDER BY create_time DESC").fetchall(): + task_payload = json_loads(task_row["payload"], {}) + output_name = task_payload.get("output_model_name") or f"{task_payload.get('name')}-lora" + if output_name and output_name not in task_output_names: + task_output_names[output_name] = task_payload items = [] for row in rows: item = { @@ -1564,41 +2196,50 @@ class PlatformStore: "merging": bool(row["merging"]), } if not item.get("compute_node_id"): - task_rows = conn.execute("SELECT payload FROM fine_tune_tasks ORDER BY create_time DESC").fetchall() - for task in task_rows: - task_payload = json_loads(task["payload"], {}) - output_name = task_payload.get("output_model_name") or f"{task_payload.get('name')}-lora" - if output_name == item["name"]: - item["compute_node_id"] = task_payload.get("compute_node_id") - item["compute_node_name"] = task_payload.get("compute_node_code") or task_payload.get("compute_node_name") - break + task_payload = task_output_names.get(item["name"]) + if task_payload: + item["compute_node_id"] = task_payload.get("compute_node_id") + item["compute_node_name"] = task_payload.get("compute_node_code") or task_payload.get("compute_node_name") items.append(item) return items def delete_trained_model(self, model_id: str) -> None: with self.connect() as conn: + rows = conn.execute("SELECT id FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchall() conn.execute("UPDATE trained_models SET deleted_at=?, deleted_by=? WHERE id=? OR name=?", (utcnow(), "system", model_id, model_id)) + for row in rows: + self.revoke_resource_security_state("trained_model", row["id"]) - def datasets(self) -> list[dict[str, Any]]: + def datasets(self, tenant_id: str | None = None) -> list[dict[str, Any]]: with self.connect() as conn: + clauses = ["dataset.deleted_at IS NULL"] + params: list[Any] = [] + if tenant_id: + clauses.append("(dataset.tenant_id=? OR dataset.tenant_id IS NULL)") + params.append(tenant_id) rows = conn.execute( - """SELECT dataset.*, task.name AS task_name + f"""SELECT dataset.*, task.name AS task_name, + creator.display_name AS creator_name FROM datasets dataset LEFT JOIN data_process_tasks task ON task.id=COALESCE(dataset.source_task_id, dataset.task_id) - WHERE dataset.deleted_at IS NULL - ORDER BY dataset.create_time DESC""" + LEFT JOIN users creator ON creator.id=dataset.created_by + WHERE {' AND '.join(clauses)} + ORDER BY dataset.create_time DESC""", + tuple(params), ).fetchall() return [self._dataset(conn, row) for row in rows] def dataset(self, dataset_id: str) -> dict[str, Any]: with self.connect() as conn: row = conn.execute( - """SELECT dataset.*, task.name AS task_name + """SELECT dataset.*, task.name AS task_name, + creator.display_name AS creator_name FROM datasets dataset LEFT JOIN data_process_tasks task ON task.id=COALESCE(dataset.source_task_id, dataset.task_id) - WHERE dataset.id=?""", + LEFT JOIN users creator ON creator.id=dataset.created_by + WHERE dataset.id=? AND dataset.deleted_at IS NULL""", (dataset_id,), ).fetchone() if not row: @@ -1698,8 +2339,8 @@ class PlatformStore: conn.execute( """ INSERT INTO datasets - (id, name, type, storage_type, source, task_id, size, count, description, create_time, created_by) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, name, type, storage_type, source, task_id, size, count, description, create_time, created_by, tenant_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( dataset_id, @@ -1713,6 +2354,7 @@ class PlatformStore: payload.get("description"), utcnow(), payload.get("created_by"), + payload.get("tenant_id") or "default", ), ) return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()) @@ -1744,6 +2386,7 @@ class PlatformStore: def delete_dataset(self, dataset_id: str) -> None: with self.connect() as conn: conn.execute("UPDATE datasets SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", dataset_id)) + self.revoke_resource_security_state("dataset", dataset_id) def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]: now = utcnow() @@ -1987,19 +2630,23 @@ class PlatformStore: def tasks(self) -> list[dict[str, Any]]: self.refresh_runtime_state() with self.connect() as conn: - rows = conn.execute("SELECT * FROM fine_tune_tasks ORDER BY create_time DESC").fetchall() + rows = conn.execute("SELECT * FROM fine_tune_tasks WHERE COALESCE(deleted_at, '')='' ORDER BY create_time DESC").fetchall() return [self._task(row) for row in rows] def task(self, task_id: str) -> dict[str, Any]: self.refresh_runtime_state() with self.connect() as conn: - row = conn.execute("SELECT * FROM fine_tune_tasks WHERE id=?", (task_id,)).fetchone() + row = conn.execute("SELECT * FROM fine_tune_tasks WHERE id=? AND COALESCE(deleted_at, '')=''", (task_id,)).fetchone() if not row: raise KeyError(task_id) return self._task(row) def _task(self, row: PgRow) -> dict[str, Any]: payload = json_loads(row["payload"], {}) + # Structured ownership fields are authoritative; payload values are + # retained only as a fallback for historical task rows. + payload["tenant_id"] = row.get("tenant_id") or payload.get("tenant_id") or "default" + payload["created_by"] = row.get("created_by") or payload.get("created_by") payload.update( { "id": row["id"], @@ -2053,6 +2700,7 @@ class PlatformStore: "auto_merge": bool(payload.get("auto_merge", False)), "output_model_name": payload.get("output_model_name") or f"{name}-lora", "gpus": payload.get("gpus") or [], + "logging_steps": max(1, int(payload.get("logging_steps", 1) or 1)), "batch_size": payload.get("batch_size", 2), "learning_rate": payload.get("learning_rate", 0.0002), "n_epochs": payload.get("n_epochs", 3), @@ -2070,6 +2718,8 @@ class PlatformStore: "quant_bits": payload.get("quant_bits", 4), "quant_group_size": payload.get("quant_group_size", 128), "export_format": payload.get("export_format", "safetensors"), + "resource_versions": payload.get("resource_versions") or {}, + "tenant_id": payload.get("tenant_id") or "default", "progress": 0, "process_id": None, "train_duration": "", @@ -2080,10 +2730,18 @@ class PlatformStore: conn.execute( """ INSERT INTO fine_tune_tasks - (id, name, payload, status, progress, process_id, create_time, gpus) - VALUES (?, ?, ?, 'pending', 0, NULL, ?, ?) + (id, name, payload, status, progress, process_id, create_time, gpus, tenant_id, created_by) + VALUES (?, ?, ?, 'pending', 0, NULL, ?, ?, ?, ?) """, - (task_id, name, json_dumps(task), now, json_dumps(task["gpus"])), + ( + task_id, + name, + json_dumps(task), + now, + json_dumps(task["gpus"]), + task["tenant_id"], + task.get("created_by"), + ), ) return task @@ -2092,8 +2750,15 @@ class PlatformStore: merged = {**current, **payload, "id": task_id} with self.connect() as conn: conn.execute( - "UPDATE fine_tune_tasks SET name=?, payload=?, gpus=? WHERE id=?", - (merged["name"], json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id), + "UPDATE fine_tune_tasks SET name=?, payload=?, gpus=?, tenant_id=?, created_by=? WHERE id=?", + ( + merged["name"], + json_dumps(merged), + json_dumps(merged.get("gpus", [])), + merged.get("tenant_id") or "default", + merged.get("created_by"), + task_id, + ), ) return self.task(task_id) @@ -2152,6 +2817,14 @@ class PlatformStore: node["id"], selected_gpus, ) + self._reserve_tenant_quota_locked( + conn, + str(merged.get("tenant_id") or "default"), + "training", + task_id, + len(selected_gpus), + ) + conn.execute("DELETE FROM scheduler_locks WHERE lock_key=? AND owner=?", ("compute-scheduler", owner)) return self.task(task_id) def reset_task_for_retry(self, task_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: @@ -2216,11 +2889,15 @@ class PlatformStore: selected_gpus: list[int] | list[Any] | None = None, ) -> dict[str, Any]: base_model_id = task.get("base_model") or task.get("model_id") + model_lookup = str(base_model_id or "") dataset_id = str(task.get("train_dataset_id") or task.get("dataset_id") or "") with self.connect() as conn: - model = conn.execute("SELECT * FROM models WHERE id=?", (base_model_id,)).fetchone() + model = conn.execute( + "SELECT * FROM models WHERE id=? OR name=? OR path=?", + (model_lookup, model_lookup, model_lookup), + ).fetchone() if not model: - raise RuntimeError(f"base model not found: {base_model_id}") + raise RuntimeError(f"base model not found: {model_lookup}") # P0-1: Reject non-trainable models (API models or models without local path) if not model.get("can_train"): model_source = model.get("model_source") or "unknown" @@ -2254,7 +2931,7 @@ class PlatformStore: (str(validation_dataset_id),), ).fetchall(), ] - model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id + model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or model_lookup dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {} if not dataset or dataset.get("type") != "train" or dataset_metadata.get( "dataset_split" @@ -2432,13 +3109,22 @@ class PlatformStore: "UPDATE fine_tune_tasks SET payload = ? WHERE id = ?", (json_dumps({**payload, "last_log_snippet": log_snippet[:8192]}), task_id), ) + if status in {"completed", "failed", "stopped"}: + self._release_tenant_quota_locked(conn, "training", task_id) return self.task(task_id) def running_compute_tasks(self) -> list[dict[str, Any]]: + nodes = { + str(node.get("id")): node + for node in self.compute_nodes() + if node.get("enabled") and node.get("scheduler_status") in {"online", "draining"} + } return [ task for task in self.tasks() - if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"} + if task.get("compute_job_id") + and task.get("compute_node_id") in nodes + and task["status"] in {"syncing", "queued", "running"} ] def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]: @@ -2453,6 +3139,7 @@ class PlatformStore: "UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')", (utcnow(), task_id), ) + self._release_tenant_quota_locked(conn, "training", task_id) return self.task(task_id) def stop_task(self, task_id: str, status: str = "stopped") -> dict[str, Any]: @@ -2467,10 +3154,12 @@ class PlatformStore: "UPDATE gpu_allocations SET status='released', released_at=COALESCE(released_at, ?) WHERE task_id=? AND status IN ('allocated','running')", (utcnow(), task_id), ) + self._release_tenant_quota_locked(conn, "training", task_id) return self.task(task_id) def delete_task(self, task_id: str) -> None: with self.connect() as conn: + self._release_tenant_quota_locked(conn, "training", task_id) conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,)) def _json_payload_row(self, row: PgRow) -> dict[str, Any]: @@ -2498,9 +3187,9 @@ class PlatformStore: if dataset: data["dataset"] = dataset["name"] - dimension = None + dimension = data.get("dimension") if isinstance(data.get("dimension"), dict) else None dimension_id = str(data.get("dimension_id") or "") - if dimension_id: + if dimension is None and dimension_id: dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() if dimension_row: dimension = json_loads(dimension_row["payload"], {}) @@ -2513,11 +3202,17 @@ class PlatformStore: data["metric"] = data["metric_label"] return data - def eval_tasks(self) -> list[dict[str, Any]]: + def eval_tasks(self, tenant_id: str | None = None) -> list[dict[str, Any]]: with self.connect() as conn: # 评测任务采用软删除，普通列表不得再次返回已删除记录。 + clause = "WHERE deleted_at IS NULL" + params: tuple[Any, ...] = () + if tenant_id: + clause += " AND (tenant_id=? OR tenant_id IS NULL)" + params = (tenant_id,) rows = conn.execute( - "SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC" + f"SELECT * FROM eval_tasks {clause} ORDER BY create_time DESC", + params, ).fetchall() return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows] @@ -2559,9 +3254,9 @@ class PlatformStore: model = conn.execute("SELECT name FROM models WHERE id=?", (model_id,)).fetchone() trained_model = conn.execute("SELECT name FROM trained_models WHERE id=? OR name=?", (model_id, model_id)).fetchone() dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone() - dimension = None + dimension = payload.get("dimension") if isinstance(payload.get("dimension"), dict) else None dimension_id = str(payload.get("dimension_id") or "") - if dimension_id: + if dimension is None and dimension_id: dimension_row = conn.execute("SELECT payload FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone() if dimension_row: dimension = json_loads(dimension_row["payload"], {}) @@ -2579,8 +3274,8 @@ class PlatformStore: if data.get("metric") == "custom": data["metric"] = data["metric_label"] conn.execute( - "INSERT INTO eval_tasks (id, name, payload, status, create_time, created_by) VALUES (?, ?, ?, ?, ?, ?)", - (task_id, name, json_dumps(data), status, now, data.get("created_by")), + "INSERT INTO eval_tasks (id, name, payload, status, create_time, created_by, tenant_id) VALUES (?, ?, ?, ?, ?, ?, ?)", + (task_id, name, json_dumps(data), status, now, data.get("created_by"), data.get("tenant_id") or "default"), ) return self.eval_task(task_id) @@ -2593,10 +3288,28 @@ class PlatformStore: "UPDATE eval_tasks SET payload=?, status=? WHERE id=?", (json_dumps(merged), merged.get("status", task.get("status", "pending")), task_id), ) + if "report_storage_object_id" in updates: + conn.execute( + "UPDATE eval_tasks SET report_storage_object_id=? WHERE id=?", + (updates.get("report_storage_object_id") or None, task_id), + ) + if merged.get("status") in {"completed", "failed", "stopped"}: + conn.execute( + "UPDATE gpu_reservations SET status='released', released_at=COALESCE(released_at, ?) " + "WHERE owner_type='eval' AND owner_id=? AND status='reserved'", + (utcnow(), task_id), + ) + self._release_tenant_quota_locked(conn, "eval", task_id) return self.eval_task(task_id) def delete_eval_task(self, task_id: str) -> None: with self.connect() as conn: + conn.execute( + "UPDATE gpu_reservations SET status='released', released_at=COALESCE(released_at, ?) " + "WHERE owner_type='eval' AND owner_id=? AND status='reserved'", + (utcnow(), task_id), + ) + self._release_tenant_quota_locked(conn, "eval", task_id) result = conn.execute( "UPDATE eval_tasks SET deleted_at=?, deleted_by=? " "WHERE id=? AND deleted_at IS NULL RETURNING id", @@ -2604,6 +3317,7 @@ class PlatformStore: ) if not result.fetchone(): raise KeyError(task_id) + self.revoke_resource_security_state("eval", task_id) def running_eval_tasks(self) -> list[dict[str, Any]]: """Return eval tasks that have been submitted to a compute node and are still running.""" @@ -2619,13 +3333,18 @@ class PlatformStore: status_map = {"queued": "running", "running": "running", "completed": "completed", "failed": "failed", "stopped": "stopped"} new_status = status_map.get(job_status, job_status or task.get("status", "pending")) + progress = int(job.get("progress", task.get("progress", 0)) or 0) + progress_detail = task.get("progress_detail") or {} + if progress_detail.get("percentage") is not None: + progress = int(float(progress_detail.get("percentage") or 0)) updates: dict[str, Any] = { "status": new_status, - "progress": int(job.get("progress", 0)), + "progress": progress, "output_dir": job.get("output_dir", task.get("output_dir", "")), } - # On completion, populate results from eval_results.json content - if new_status == "completed" and result_content: + # Persist both live partial results and the final report. The payload + # JSON keeps this compatible with existing installations. + if result_content: updates.update({ "overall_score": result_content.get("overall_score", 0), "overall_score_max": result_content.get("overall_score_max", 100), @@ -2638,8 +3357,10 @@ class PlatformStore: "passed_count": result_content.get("passed_count", 0), "basic_metrics": result_content.get("basic_metrics", {}), "score": result_content.get("overall_score", 0), - "completed_time": utcnow(), + "metric_summary_version": result_content.get("metric_summary_version", 2), }) + if new_status == "completed": + updates["completed_time"] = utcnow() elif new_status in {"failed", "stopped"}: updates.update({ "error": job.get("error") or task.get("error") or "", @@ -2710,12 +3431,12 @@ class PlatformStore: def compare_tasks(self) -> list[dict[str, Any]]: with self.connect() as conn: - rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall() + rows = conn.execute("SELECT * FROM compare_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall() return [self._json_payload_row(row) for row in rows] def compare_task(self, task_id: str) -> dict[str, Any]: with self.connect() as conn: - row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone() + row = conn.execute("SELECT * FROM compare_tasks WHERE id=? AND deleted_at IS NULL", (task_id,)).fetchone() if not row: raise KeyError(task_id) return self._json_payload_row(row) @@ -2729,8 +3450,8 @@ class PlatformStore: data.setdefault("load_status", json_dumps({"loaded_models": []})) with self.connect() as conn: conn.execute( - "INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)", - (task_id, name, json_dumps(data), status, now), + "INSERT INTO compare_tasks (id, name, payload, status, create_time, created_by, tenant_id) VALUES (?, ?, ?, ?, ?, ?, ?)", + (task_id, name, json_dumps(data), status, now, data.get("created_by"), data.get("tenant_id") or "default"), ) return self.compare_task(task_id) @@ -2743,11 +3464,28 @@ class PlatformStore: "UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?", (merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id), ) + if status in {"completed", "failed", "stopped"}: + conn.execute( + "UPDATE gpu_reservations SET status='released', released_at=COALESCE(released_at, ?) " + "WHERE owner_type='inference' AND owner_id=? AND status='reserved'", + (utcnow(), task_id), + ) + self._release_tenant_quota_locked(conn, "inference", task_id) return self.compare_task(task_id) def delete_compare_task(self, task_id: str) -> None: with self.connect() as conn: - conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,)) + conn.execute( + "UPDATE gpu_reservations SET status='released', released_at=COALESCE(released_at, ?) " + "WHERE owner_type='inference' AND owner_id=? AND status='reserved'", + (utcnow(), task_id), + ) + self._release_tenant_quota_locked(conn, "inference", task_id) + conn.execute( + "UPDATE compare_tasks SET deleted_at=COALESCE(deleted_at, ?), deleted_by=COALESCE(deleted_by, ?) WHERE id=? AND deleted_at IS NULL", + (utcnow(), "system", task_id), + ) + self.revoke_resource_security_state("compare", task_id) def _acquire_scheduler_lock( self, @@ -2812,6 +3550,11 @@ class PlatformStore: (node_id,), ).fetchall() active = {int(row["gpu_index"]) for row in rows} + external = conn.execute( + "SELECT gpu_index FROM gpu_reservations WHERE node_id=? AND status='reserved'", + (node_id,), + ).fetchall() + active.update(int(row["gpu_index"]) for row in external) # Evaluation jobs use the same Compute ProcessManager GPU lock but do # not have fine-tune allocation rows; derive their selected cards here # so a training task cannot race onto an evaluation GPU. @@ -2939,7 +3682,9 @@ class PlatformStore: owner = f"schedule:{uuid.uuid4().hex[:8]}" if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner): raise RuntimeError("compute scheduler is busy, please retry") - return self._schedule_node_locked(conn, payload) + selected = self._schedule_node_locked(conn, payload) + conn.execute("DELETE FROM scheduler_locks WHERE lock_key=? AND owner=?", ("compute-scheduler", owner)) + return selected def select_compute_node(self, payload: dict[str, Any]) -> dict[str, Any]: with self.connect() as conn: @@ -3008,6 +3753,8 @@ class PlatformStore: local_path: str, status: str = "available", sync_status: str = "synced", + version_id: str | None = None, + storage_object_id: str | None = None, ) -> dict[str, Any]: with self.connect() as conn: row = conn.execute( @@ -3016,8 +3763,8 @@ class PlatformStore: ).fetchone() if row: conn.execute( - "UPDATE resource_replicas SET local_path=?, status=?, sync_status=? WHERE id=?", - (local_path, status, sync_status, row["id"]), + "UPDATE resource_replicas SET local_path=?, status=?, sync_status=?, version_id=?, storage_object_id=? WHERE id=?", + (local_path, status, sync_status, version_id or row.get("version_id"), storage_object_id or row.get("storage_object_id"), row["id"]), ) replica_id = row["id"] else: @@ -3025,10 +3772,10 @@ class PlatformStore: conn.execute( """ INSERT INTO resource_replicas - (id, node_id, resource_type, resource_id, local_path, status, sync_status, create_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + (id, node_id, resource_type, resource_id, version_id, storage_object_id, local_path, status, sync_status, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, - (replica_id, node_id, resource_type, resource_id, local_path, status, sync_status, utcnow()), + (replica_id, node_id, resource_type, resource_id, version_id, storage_object_id, local_path, status, sync_status, utcnow()), ) return dict(conn.execute("SELECT * FROM resource_replicas WHERE id=?", (replica_id,)).fetchone()) @@ -3131,8 +3878,15 @@ class PlatformStore: return dict(row) def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]: - allowed = {"status", "checksum_sha256", "byte_size", "content_type"} + allowed = { + "status", "checksum_sha256", "byte_size", "content_type", "metadata", + "deleted_at", "deleted_by", "cleanup_attempts", "last_cleanup_error", "last_verified_at", + } fields = {key: value for key, value in payload.items() if key in allowed} + if fields.get("status") == "deleted": + fields.setdefault("deleted_at", utcnow()) + if "metadata" in fields and not isinstance(fields["metadata"], str): + fields["metadata"] = json_dumps(fields["metadata"]) if fields: assignments = ", ".join(f"{key}=?" for key in fields) with self.connect() as conn: @@ -3143,24 +3897,85 @@ class PlatformStore: raise KeyError(object_id) return dict(row) + def storage_objects_pending_cleanup(self, limit: int = 100) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT * FROM storage_objects + WHERE status <> 'purged' AND (status='deleted' OR deleted_at IS NOT NULL) + ORDER BY COALESCE(deleted_at, create_time), object_key + LIMIT ? + """, + (max(1, min(int(limit), 1000)),), + ).fetchall() + return [dict(row) for row in rows] + + def create_storage_cleanup_job(self, storage_object_id: str, action: str = "delete") -> dict[str, Any]: + job_id = new_id("cleanup") + with self.connect() as conn: + conn.execute( + """ + INSERT INTO storage_cleanup_jobs + (id, storage_object_id, action, status, attempts, create_time) + VALUES (?, ?, ?, 'running', 1, ?) + """, + (job_id, storage_object_id, action, utcnow()), + ) + row = conn.execute("SELECT * FROM storage_cleanup_jobs WHERE id=?", (job_id,)).fetchone() + return dict(row) + + def finish_storage_cleanup_job(self, job_id: str, success: bool, error: str = "") -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM storage_cleanup_jobs WHERE id=?", (job_id,)).fetchone() + if not row: + raise KeyError(job_id) + status = "completed" if success else "failed" + conn.execute( + "UPDATE storage_cleanup_jobs SET status=?, error=?, completed_at=? WHERE id=?", + (status, error[:2000], utcnow(), job_id), + ) + conn.execute( + """ + UPDATE storage_objects + SET cleanup_attempts=cleanup_attempts+1, + last_cleanup_error=?, + last_verified_at=? + WHERE id=? + """, + (error[:2000], utcnow(), row["storage_object_id"]), + ) + if success: + conn.execute( + "UPDATE storage_objects SET status='purged', last_cleanup_error='' WHERE id=?", + (row["storage_object_id"],), + ) + result = conn.execute("SELECT * FROM storage_cleanup_jobs WHERE id=?", (job_id,)).fetchone() + return dict(result) + def create_storage_cache_job(self, payload: dict[str, Any]) -> dict[str, Any]: job_id = str(payload.get("id") or new_id("cache")) with self.connect() as conn: conn.execute( """ INSERT INTO storage_cache_jobs - (id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at, + version_id, checksum_sha256, byte_size, last_accessed_at, protected_until) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, (job_id, payload["storage_object_id"], payload["node_id"], payload.get("direction", "download"), payload.get("status", "running"), int(payload.get("progress", 0)), payload.get("local_path"), - payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at")), + payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at"), + payload.get("version_id"), payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), + payload.get("last_accessed_at") or utcnow(), payload.get("protected_until")), ) row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone() return dict(row) def update_storage_cache_job(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]: - allowed = {"status", "progress", "local_path", "error", "completed_at"} + allowed = { + "status", "progress", "local_path", "error", "completed_at", "version_id", + "checksum_sha256", "byte_size", "last_accessed_at", "protected_until", + } fields = {key: value for key, value in payload.items() if key in allowed} if fields: assignments = ", ".join(f"{key}=?" for key in fields) @@ -3507,6 +4322,12 @@ class PlatformStore: ORDER BY n.code, g.gpu_index """ ).fetchall() + external_reservations = { + (str(row["node_id"]), int(row["gpu_index"])): dict(row) + for row in conn.execute( + "SELECT node_id, gpu_index, owner_type, owner_id FROM gpu_reservations WHERE status='reserved'" + ).fetchall() + } running_tasks = [ self._task(row) for row in conn.execute( @@ -3569,6 +4390,9 @@ class PlatformStore: ) # Also mark GPU as busy if an inference model is loaded on this node inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set()) + externally_reserved = external_reservations.get((str(row["node_id"]), int(row["gpu_index"]))) + if externally_reserved and not busy and not inference_on_gpu: + reserved = True if inference_on_gpu and not busy: busy = True reserved = False @@ -3611,6 +4435,16 @@ class PlatformStore: } ] if eval_task + else [ + { + "pid": 0, + "name": externally_reserved.get("owner_type") if externally_reserved else "", + "memory_used_gb": memory_used, + "task_name": externally_reserved.get("owner_id") if externally_reserved else "", + "user": "reserved", + } + ] + if externally_reserved else [], } ) @@ -3863,13 +4697,20 @@ class PlatformStore: project_id: str | None = None, detail: str | None = None, ip: str | None = None, + result: str = "success", + reason: str | None = None, + request_id: str | None = None, + session_id: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: + ip = ip or client_ip_var.get("") with self.connect() as conn: conn.execute( """ INSERT INTO audit_logs - (id, tenant_id, project_id, actor_id, action, target_type, target_id, detail, client_ip, time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, tenant_id, project_id, actor_id, action, target_type, target_id, detail, + client_ip, time, result, reason, request_id, session_id, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( new_id("log"), @@ -3882,6 +4723,11 @@ class PlatformStore: detail, ip, utcnow(), + result, + reason, + request_id, + session_id, + json_dumps(metadata or {}), ), ) @@ -3898,15 +4744,19 @@ class PlatformStore: ) return {"session_id": sid, "user_id": user_id, "login_at": login_at} - def finish_session(self, session_id: str) -> None: + def finish_session(self, session_id: str, user_id: str | None = None) -> None: """登出时记录 logout_at 与时长（秒）。""" logout_at = utcnow() with self.connect() as conn: + owner_clause = " AND user_id=%s" if user_id else "" + params: list[Any] = [logout_at, logout_at, session_id] + if user_id: + params.append(user_id) conn.execute( "UPDATE sessions SET logout_at=%s, " "duration_seconds=EXTRACT(EPOCH FROM (%s::timestamptz - login_at::timestamptz))::int " - "WHERE id=%s AND logout_at IS NULL", - (logout_at, logout_at, session_id), + f"WHERE id=%s AND logout_at IS NULL{owner_clause}", + tuple(params), ) def active_sessions(self, user_id: str) -> list[dict[str, Any]]: @@ -3996,8 +4846,23 @@ class PlatformStore: with self.connect() as conn: tid = new_id("tpl") conn.execute( - "INSERT INTO approval_templates (id, name, steps, create_time) VALUES (?, ?, ?, ?)", - (tid, payload["name"], json_dumps(payload.get("steps", [])), utcnow()), + """INSERT INTO approval_templates + (id, name, steps, create_time, tenant_id, action, resource_type, + scope, status, created_by, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + tid, + payload["name"], + json_dumps(payload.get("steps", [])), + utcnow(), + payload.get("tenant_id"), + payload.get("action"), + payload.get("resource_type"), + payload.get("scope") or "tenant", + payload.get("status") or "active", + payload.get("created_by"), + utcnow(), + ), ) return self.approval_template(tid) @@ -4014,7 +4879,9 @@ class PlatformStore: return dict(row) def update_approval_template(self, template_id: str, payload: dict[str, Any]) -> dict[str, Any]: - fields = {k: v for k, v in payload.items() if k in ("name", "steps")} + fields = {k: v for k, v in payload.items() if k in ( + "name", "steps", "tenant_id", "action", "resource_type", "scope", "status", "updated_at" + )} if "steps" in fields: fields["steps"] = json_dumps(fields["steps"]) if not fields: @@ -4038,45 +4905,288 @@ class PlatformStore: steps = [] if template_id: tpl = self.approval_template(template_id) - steps = json_loads(tpl["steps"]) if tpl.get("steps") else [] + steps = json_loads(tpl["steps"], []) if tpl.get("steps") else [] + elif payload.get("action"): + with self.connect() as conn: + tpl = conn.execute( + """SELECT * FROM approval_templates + WHERE status='active' + AND (tenant_id IS NULL OR tenant_id=? OR scope='platform') + AND (action IS NULL OR action=? ) + AND (resource_type IS NULL OR resource_type=?) + ORDER BY CASE WHEN tenant_id=? THEN 0 ELSE 1 END, + CASE WHEN action=? THEN 0 ELSE 1 END, + create_time DESC LIMIT 1""", + ( + payload.get("tenant_id") or "default", + payload.get("action"), + payload.get("resource_type"), + payload.get("tenant_id") or "default", + payload.get("action"), + ), + ).fetchone() + if tpl: + template_id = tpl["id"] + steps = json_loads(tpl["steps"], []) if tpl.get("steps") else [] + if not steps: + # Untemplated requests are still actionable: a platform admin is + # the implicit approver instead of creating an undecidable task. + steps = [{"approver_id": None, "approver_type": "admin"}] + with self.connect() as conn: + existing = conn.execute( + """SELECT id FROM approval_instances + WHERE resource_type=? AND resource_id=? AND applicant_id=? + AND action=? AND status='pending' + ORDER BY create_time DESC LIMIT 1""", + ( + payload["resource_type"], payload["resource_id"], + payload["applicant_id"], payload.get("action"), + ), + ).fetchone() + if existing: + iid = existing["id"] + else: + iid = new_id("appr") + conn.execute( + """ + INSERT INTO approval_instances + (id, template_id, resource_type, resource_id, applicant_id, status, current_step, create_time, + action, requested_permissions, reason, tenant_id, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + iid, template_id, payload["resource_type"], payload["resource_id"], + payload["applicant_id"], "pending", 0, utcnow(), payload.get("action"), + json_dumps(payload.get("requested_permissions") or []), payload.get("reason"), + payload.get("tenant_id") or "default", payload.get("expires_at"), + ), + ) + for idx, step in enumerate(steps): + conn.execute( + "INSERT INTO approval_steps (id, instance_id, step_index, approver_id, status, time, approver_type) VALUES (?, ?, ?, ?, ?, ?, ?)", + (new_id("step"), iid, idx, step.get("approver_id"), "pending", None, step.get("approver_type") or "user"), + ) + return self.approval_instance(iid) + + def consume_approved_approval( + self, + resource_type: str, + resource_id: str, + applicant_id: str, + action: str, + ) -> bool: + """Consume one approved high-risk action so retry executes exactly once.""" + with self.connect() as conn: + row = conn.execute( + """SELECT id FROM approval_instances + WHERE resource_type=? AND resource_id=? AND applicant_id=? + AND action=? AND status='approved' AND execution_status='pending' + AND (expires_at IS NULL OR expires_at='' + OR expires_at > NOW()::text) + ORDER BY decided_at DESC NULLS LAST, create_time DESC LIMIT 1 + FOR UPDATE""", + (resource_type, resource_id, applicant_id, action), + ).fetchone() + if not row: + return False + conn.execute( + "UPDATE approval_instances SET execution_status='consumed', executed_by=?, executed_at=? WHERE id=?", + (applicant_id, utcnow(), row["id"]), + ) + return True + + def apply_approval_effect(self, instance: dict[str, Any], approver_id: str) -> dict[str, Any] | None: + """Apply approval actions that have a deterministic transactional effect.""" + action = str(instance.get("action") or "") + reason = instance.get("reason") or "{}" + try: + payload = json_loads(reason, {}) if isinstance(reason, str) else reason + except Exception: + payload = {} + result: dict[str, Any] | None = None + if action == "gpu.assign": + assignments = payload.get("assignments") if isinstance(payload, dict) else None + if not assignments: + raise ValueError("GPU approval has no assignments") + result = self.assign_gpus(assignments, assigned_by=approver_id) + elif action == "tenant.quota.update": + tenant_id = str(instance.get("resource_id") or "") + quota = payload.get("quota") if isinstance(payload, dict) else None + if not tenant_id or not isinstance(quota, dict): + raise ValueError("quota approval payload is invalid") + result = self.set_tenant_quota(tenant_id, quota) + elif action == "resource.access": + result = {"status": "acl-granted"} + if result is not None: + with self.connect() as conn: + conn.execute( + "UPDATE approval_instances SET execution_status='completed', executed_by=?, executed_at=? WHERE id=?", + (approver_id, utcnow(), instance["id"]), + ) + return result + + def create_resource_access_request(self, payload: dict[str, Any]) -> dict[str, Any]: + """Create an access request and its approval instance together logically.""" + with self.connect() as conn: + existing = conn.execute( + """SELECT * FROM resource_access_requests + WHERE resource_type=? AND resource_id=? AND applicant_id=? + AND principal_type=? AND principal_id=? AND status='pending' + ORDER BY created_at DESC LIMIT 1""", + ( + payload["resource_type"], payload["resource_id"], payload["applicant_id"], + payload.get("principal_type") or "user", + payload.get("principal_id") or payload["applicant_id"], + ), + ).fetchone() + if existing: + return { + "request": self.resource_access_request(existing["id"]), + "approval": self.approval_instance(existing["approval_id"]), + } + request_id = new_id("access") + requested = payload.get("requested_permissions") or [] + approval = self.create_approval_instance({ + "resource_type": payload["resource_type"], + "resource_id": payload["resource_id"], + "applicant_id": payload["applicant_id"], + "template_id": payload.get("template_id"), + "action": "resource.access", + "requested_permissions": requested, + "reason": payload.get("reason"), + "tenant_id": payload.get("tenant_id") or "default", + "expires_at": payload.get("expires_at"), + }) with self.connect() as conn: - iid = new_id("appr") conn.execute( """ - INSERT INTO approval_instances - (id, template_id, resource_type, resource_id, applicant_id, status, current_step, create_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO resource_access_requests + (id, tenant_id, resource_type, resource_id, applicant_id, principal_type, principal_id, + requested_permissions, reason, approval_id, status, expires_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?) """, ( - iid, - template_id, + request_id, + payload.get("tenant_id") or "default", payload["resource_type"], payload["resource_id"], payload["applicant_id"], - "pending", - 0, + payload.get("principal_type") or "user", + payload.get("principal_id") or payload["applicant_id"], + json_dumps(requested), + payload.get("reason"), + approval["id"], + payload.get("expires_at"), utcnow(), ), ) - for idx, step in enumerate(steps): - conn.execute( - "INSERT INTO approval_steps (id, instance_id, step_index, approver_id, status, time) VALUES (?, ?, ?, ?, ?, ?)", - (new_id("step"), iid, idx, step.get("approver_id"), "pending", None), - ) - return self.approval_instance(iid) + return {"request": self.resource_access_request(request_id), "approval": approval} - def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]: + def resource_access_request(self, request_id: str) -> dict[str, Any]: with self.connect() as conn: + row = conn.execute("SELECT * FROM resource_access_requests WHERE id=?", (request_id,)).fetchone() + if not row: + raise KeyError(request_id) + result = dict(row) + result["requested_permissions"] = json_loads(result.get("requested_permissions"), []) + return result + + def resource_access_requests(self, *, user_id: str | None = None, status: str | None = None) -> list[dict[str, Any]]: + clauses: list[str] = [] + params: list[Any] = [] + if user_id: + clauses.append("applicant_id=?") + params.append(user_id) + if status: + clauses.append("status=?") + params.append(status) + where = f" WHERE {' AND '.join(clauses)}" if clauses else "" + with self.connect() as conn: + conn.execute( + "UPDATE resource_access_requests SET status='expired' " + "WHERE status='pending' AND expires_at IS NOT NULL AND expires_at <> '' AND expires_at <= NOW()::text" + ) + conn.execute( + "UPDATE approval_instances ai SET status='expired', execution_status='expired' " + "WHERE ai.status='pending' AND EXISTS " + "(SELECT 1 FROM resource_access_requests rr WHERE rr.approval_id=ai.id AND rr.status='expired')" + ) + rows = conn.execute( + f"SELECT * FROM resource_access_requests{where} ORDER BY created_at DESC", + tuple(params), + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["requested_permissions"] = json_loads(item.get("requested_permissions"), []) + result.append(item) + return result + + def cancel_resource_access_request(self, request_id: str, actor_id: str, *, is_admin_actor: bool = False) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM resource_access_requests WHERE id=?", (request_id,)).fetchone() + if not row: + raise KeyError(request_id) + if not is_admin_actor and row["applicant_id"] != actor_id: + raise PermissionError("only applicant or administrator can cancel request") + if row["status"] != "pending": + raise ValueError("request is not pending") + now = utcnow() + conn.execute( + "UPDATE resource_access_requests SET status='cancelled', cancelled_at=?, cancelled_by=? WHERE id=?", + (now, actor_id, request_id), + ) + conn.execute( + "UPDATE approval_instances SET status='cancelled', execution_status='cancelled', executed_by=?, executed_at=? WHERE id=? AND status='pending'", + (actor_id, now, row["approval_id"]), + ) + return self.resource_access_request(request_id) + + def approval_instances( + self, + *, + status: str | None = None, + applicant_id: str | None = None, + ) -> list[dict[str, Any]]: + with self.connect() as conn: + conn.execute( + """UPDATE approval_instances SET status='expired', execution_status='expired' + WHERE status='pending' AND expires_at IS NOT NULL AND expires_at <> '' + AND expires_at <= NOW()::text""" + ) + clauses: list[str] = [] + params: list[Any] = [] if status: - rows = conn.execute( - "SELECT * FROM approval_instances WHERE status=? ORDER BY create_time DESC", (status,) - ).fetchall() - else: - rows = conn.execute("SELECT * FROM approval_instances ORDER BY create_time DESC").fetchall() - return [dict(r) for r in rows] + clauses.append("status=?") + params.append(status) + if applicant_id: + clauses.append("applicant_id=?") + params.append(applicant_id) + where = f" WHERE {' AND '.join(clauses)}" if clauses else "" + rows = conn.execute( + f"SELECT * FROM approval_instances{where} ORDER BY create_time DESC", tuple(params) + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["steps"] = [ + dict(step) + for step in conn.execute( + "SELECT * FROM approval_steps WHERE instance_id=? ORDER BY step_index", + (row["id"],), + ).fetchall() + ] + result.append(item) + return result def approval_instance(self, instance_id: str) -> dict[str, Any]: with self.connect() as conn: + conn.execute( + """UPDATE approval_instances SET status='expired', execution_status='expired' + WHERE id=? AND status='pending' AND expires_at IS NOT NULL AND expires_at <> '' + AND expires_at <= NOW()::text""", + (instance_id,), + ) row = conn.execute("SELECT * FROM approval_instances WHERE id=?", (instance_id,)).fetchone() if not row: raise KeyError(instance_id) @@ -4119,6 +5229,56 @@ class PlatformStore: conn.execute("UPDATE approval_instances SET status='approved' WHERE id=?", (instance_id,)) else: conn.execute("UPDATE approval_instances SET status='rejected' WHERE id=?", (instance_id,)) + conn.execute( + "UPDATE approval_instances SET decided_by=?, decided_at=? WHERE id=?", + (approver_id, utcnow(), instance_id), + ) + final_status = conn.execute( + "SELECT status FROM approval_instances WHERE id=?", (instance_id,) + ).fetchone()["status"] + if final_status == "approved": + conn.execute( + "UPDATE approval_instances SET execution_status='ready' WHERE id=?", + (instance_id,), + ) + elif final_status == "rejected": + conn.execute( + "UPDATE approval_instances SET execution_status='rejected' WHERE id=?", + (instance_id,), + ) + request_row = conn.execute( + "SELECT * FROM resource_access_requests WHERE approval_id=?", (instance_id,) + ).fetchone() + if request_row and final_status in {"approved", "rejected"}: + request_status = "approved" if final_status == "approved" else "rejected" + if request_status == "approved": + permissions = json_loads(request_row.get("requested_permissions"), []) + for permission in permissions: + exists = conn.execute( + """SELECT 1 FROM acls WHERE resource_type=? AND resource_id=? + AND principal_type=? AND principal_id=? AND permission=? + AND revoked_at IS NULL""", + ( + request_row["resource_type"], request_row["resource_id"], + request_row["principal_type"], request_row["principal_id"], permission, + ), + ).fetchone() + if not exists: + conn.execute( + """INSERT INTO acls + (id, resource_type, resource_id, principal_type, principal_id, permission, + create_time, tenant_id, granted_by, source_request_id, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + new_id("acl"), request_row["resource_type"], request_row["resource_id"], + request_row["principal_type"], request_row["principal_id"], permission, + utcnow(), request_row["tenant_id"], approver_id, request_row["id"], request_row["expires_at"], + ), + ) + conn.execute( + "UPDATE resource_access_requests SET status=?, decided_at=?, decided_by=? WHERE id=?", + (request_status, utcnow(), approver_id, request_row["id"]), + ) return self.approval_instance(instance_id) # ===================== 平台治理：租户 ===================== @@ -4137,7 +5297,20 @@ class PlatformStore: def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]: with self.connect() as conn: - tid = new_id("tnt") + requested_id = str(payload.get("id") or payload.get("code") or "").strip() + tid = requested_id or new_id("tnt") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", tid): + raise ValueError("tenant id must be 1-64 characters: letters, digits, '_' or '-'") + if conn.execute("SELECT 1 FROM tenants WHERE id=?", (tid,)).fetchone(): + raise ValueError("tenant id already exists") + owner_user_id = payload.get("owner_user_id") + if owner_user_id: + owner = conn.execute( + "SELECT id, status FROM users WHERE id=? AND COALESCE(deleted_at, '')=''", + (owner_user_id,), + ).fetchone() + if not owner or owner.get("status") != "active": + raise KeyError(owner_user_id) conn.execute( """ INSERT INTO tenants (id, name, code, status, owner_user_id, quota, retention_policy_id, create_time) @@ -4146,14 +5319,22 @@ class PlatformStore: ( tid, payload["name"], - payload.get("code"), + # Tenant ID and tenant code are one business identifier. + tid, "active", - payload.get("owner_user_id"), + owner_user_id, json_dumps(payload.get("quota", {})), payload.get("retention_policy_id"), utcnow(), ), ) + if owner_user_id: + conn.execute( + """INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) + VALUES (?, ?, 'owner', 'active', ?) + ON CONFLICT (tenant_id, user_id) DO UPDATE SET role='owner', status='active'""", + (tid, owner_user_id, utcnow()), + ) return self.tenant(tid) def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]: @@ -4178,33 +5359,275 @@ class PlatformStore: conn.execute("UPDATE tenants SET retention_policy_id=? WHERE id=?", (retention_policy_id, tenant_id)) return self.tenant(tenant_id) - def delete_tenant(self, tenant_id: str) -> dict[str, Any]: + def delete_tenant(self, tenant_id: str, deleted_by: str | None = None) -> dict[str, Any]: with self.connect() as conn: row = conn.execute("SELECT * FROM tenants WHERE id=?", (tenant_id,)).fetchone() if not row: raise KeyError(tenant_id) - conn.execute("DELETE FROM tenants WHERE id=?", (tenant_id,)) + if tenant_id in {"default", "admin"}: + raise ValueError(f"{tenant_id} tenant cannot be deleted") + now = utcnow() + conn.execute( + "UPDATE tenants SET status='deleted', deleted_at=COALESCE(deleted_at, ?), deleted_by=COALESCE(deleted_by, ?) WHERE id=?", + (now, deleted_by or "system", tenant_id), + ) + conn.execute("UPDATE tenant_members SET status='disabled' WHERE tenant_id=? AND status='active'", (tenant_id,)) return dict(row) + def restore_tenant(self, tenant_id: str, restored_by: str | None = None) -> dict[str, Any]: + """Restore a soft-deleted tenant and its suspended memberships.""" + with self.connect() as conn: + row = conn.execute("SELECT * FROM tenants WHERE id=?", (tenant_id,)).fetchone() + if not row: + raise KeyError(tenant_id) + if tenant_id in {"default", "admin"}: + return dict(row) + if row.get("status") != "deleted" and not row.get("deleted_at"): + raise ValueError("tenant is already active") + conn.execute( + "UPDATE tenants SET status='active', deleted_at=NULL, deleted_by=NULL WHERE id=?", + (tenant_id,), + ) + conn.execute( + "UPDATE tenant_members SET status='active' " + "WHERE tenant_id=? AND status='disabled'", + (tenant_id,), + ) + restored = conn.execute("SELECT * FROM tenants WHERE id=?", (tenant_id,)).fetchone() + return dict(restored) + + def user_tenants(self, user_id: str, include_all: bool = False) -> list[dict[str, Any]]: + """Return active tenants visible to a user and their member role.""" + with self.connect() as conn: + if include_all: + rows = conn.execute( + """SELECT t.*, tm.role AS member_role, tm.status AS member_status + FROM tenants t + LEFT JOIN tenant_members tm ON tm.tenant_id=t.id AND tm.user_id=? + WHERE COALESCE(t.status, 'active')='active' AND COALESCE(t.deleted_at, '')='' + ORDER BY t.create_time DESC""", + (user_id,), + ).fetchall() + else: + rows = conn.execute( + """SELECT t.*, tm.role AS member_role, tm.status AS member_status + FROM tenants t + JOIN tenant_members tm ON tm.tenant_id=t.id + WHERE tm.user_id=? AND tm.status='active' + AND (tm.expires_at IS NULL OR tm.expires_at='' OR tm.expires_at > NOW()::text) + AND COALESCE(t.status, 'active')='active' AND COALESCE(t.deleted_at, '')='' + ORDER BY t.create_time DESC""", + (user_id,), + ).fetchall() + return [dict(row) for row in rows] + + def assert_active_tenant(self, tenant_id: str | None) -> str: + resolved = str(tenant_id or "default") + with self.connect() as conn: + row = conn.execute( + "SELECT id, status, deleted_at FROM tenants WHERE id=?", + (resolved,), + ).fetchone() + if not row or row.get("status") not in {None, "active"} or row.get("deleted_at"): + raise ValueError("tenant not found or inactive") + return resolved + + def tenant_members(self, tenant_id: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """SELECT tm.*, u.username, u.display_name, u.role AS user_role + FROM tenant_members tm JOIN users u ON u.id=tm.user_id + WHERE tm.tenant_id=? ORDER BY tm.joined_at, u.username""", + (tenant_id,), + ).fetchall() + return [dict(row) for row in rows] + + def add_tenant_member(self, tenant_id: str, user_id: str, role: str = "member", invited_by: str | None = None) -> dict[str, Any]: + if role not in {"owner", "admin", "member", "viewer"}: + raise ValueError("invalid tenant member role") + with self.connect() as conn: + if not conn.execute("SELECT 1 FROM tenants WHERE id=?", (tenant_id,)).fetchone(): + raise KeyError(tenant_id) + if not conn.execute("SELECT 1 FROM users WHERE id=?", (user_id,)).fetchone(): + raise KeyError(user_id) + conn.execute( + """INSERT INTO tenant_members (tenant_id, user_id, role, status, invited_by, joined_at) + VALUES (?, ?, ?, 'active', ?, ?) ON CONFLICT (tenant_id, user_id) + DO UPDATE SET role=EXCLUDED.role, status='active', invited_by=EXCLUDED.invited_by""", + (tenant_id, user_id, role, invited_by, utcnow()), + ) + row = conn.execute( + """SELECT tm.*, u.username, u.display_name, u.role AS user_role + FROM tenant_members tm JOIN users u ON u.id=tm.user_id + WHERE tm.tenant_id=? AND tm.user_id=?""", + (tenant_id, user_id), + ).fetchone() + return dict(row) + + def invite_tenant_member( + self, + tenant_id: str, + user_id: str, + role: str = "member", + invited_by: str | None = None, + expires_at: str | None = None, + ) -> dict[str, Any]: + """Create a pending tenant invitation using the existing membership table.""" + if role not in {"admin", "member", "viewer"}: + raise ValueError("tenant invitations cannot grant owner role") + invitation_expiry = expires_at or ( + datetime.now(timezone.utc) + timedelta(days=7) + ).replace(microsecond=0).isoformat().replace("+00:00", "Z") + with self.connect() as conn: + tenant = conn.execute( + "SELECT id, status, deleted_at FROM tenants WHERE id=?", (tenant_id,) + ).fetchone() + user = conn.execute( + "SELECT id, status FROM users WHERE id=?", (user_id,) + ).fetchone() + if not tenant or tenant.get("status") not in {None, "active"} or tenant.get("deleted_at"): + raise KeyError(tenant_id) + if not user or user.get("status") != "active": + raise KeyError(user_id) + conn.execute( + """INSERT INTO tenant_members + (tenant_id, user_id, role, status, invited_by, joined_at, expires_at) + VALUES (?, ?, ?, 'pending', ?, NULL, ?) + ON CONFLICT (tenant_id, user_id) DO UPDATE SET + role=EXCLUDED.role, status='pending', invited_by=EXCLUDED.invited_by, + joined_at=NULL, expires_at=EXCLUDED.expires_at""", + (tenant_id, user_id, role, invited_by, invitation_expiry), + ) + row = conn.execute( + """SELECT tm.*, u.username, u.display_name, u.role AS user_role + FROM tenant_members tm JOIN users u ON u.id=tm.user_id + WHERE tm.tenant_id=? AND tm.user_id=?""", + (tenant_id, user_id), + ).fetchone() + return dict(row) + + def tenant_invitations(self, user_id: str) -> list[dict[str, Any]]: + now = utcnow() + with self.connect() as conn: + conn.execute( + "UPDATE tenant_members SET status='expired' " + "WHERE user_id=? AND status='pending' AND expires_at IS NOT NULL AND expires_at <> '' AND expires_at <= ?", + (user_id, now), + ) + rows = conn.execute( + """SELECT tm.*, t.name AS tenant_name, t.code AS tenant_code + FROM tenant_members tm JOIN tenants t ON t.id=tm.tenant_id + WHERE tm.user_id=? AND tm.status='pending' + ORDER BY tm.expires_at, t.name""", + (user_id,), + ).fetchall() + return [dict(row) for row in rows] + + def accept_tenant_invitation(self, tenant_id: str, user_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute( + "SELECT * FROM tenant_members WHERE tenant_id=? AND user_id=? FOR UPDATE", + (tenant_id, user_id), + ).fetchone() + if not row: + raise KeyError(user_id) + if row.get("status") != "pending": + raise ValueError("tenant invitation is not pending") + if row.get("expires_at") and row["expires_at"] <= utcnow(): + conn.execute( + "UPDATE tenant_members SET status='expired' WHERE tenant_id=? AND user_id=?", + (tenant_id, user_id), + ) + raise ValueError("tenant invitation has expired") + conn.execute( + "UPDATE tenant_members SET status='active', joined_at=? WHERE tenant_id=? AND user_id=?", + (utcnow(), tenant_id, user_id), + ) + updated = conn.execute( + """SELECT tm.*, u.username, u.display_name, u.role AS user_role + FROM tenant_members tm JOIN users u ON u.id=tm.user_id + WHERE tm.tenant_id=? AND tm.user_id=?""", + (tenant_id, user_id), + ).fetchone() + return dict(updated) + + def update_tenant_member(self, tenant_id: str, user_id: str, payload: dict[str, Any]) -> dict[str, Any]: + fields: dict[str, Any] = {} + if "role" in payload: + if payload["role"] not in {"owner", "admin", "member", "viewer"}: + raise ValueError("invalid tenant member role") + fields["role"] = payload["role"] + if "status" in payload: + if payload["status"] not in {"active", "disabled", "pending"}: + raise ValueError("invalid tenant member status") + fields["status"] = payload["status"] + if not fields: + members = [item for item in self.tenant_members(tenant_id) if item.get("user_id") == user_id] + if not members: + raise KeyError(user_id) + return members[0] + with self.connect() as conn: + current = conn.execute("SELECT * FROM tenant_members WHERE tenant_id=? AND user_id=?", (tenant_id, user_id)).fetchone() + if not current: + raise KeyError(user_id) + if fields.get("role") and current["role"] == "owner" and fields["role"] != "owner": + owners = conn.execute( + "SELECT COUNT(*) AS count FROM tenant_members WHERE tenant_id=? AND role='owner' AND status='active'", + (tenant_id,), + ).fetchone()["count"] + if int(owners or 0) <= 1: + raise ValueError("tenant must retain at least one active owner") + set_clause = ", ".join(f"{key}=?" for key in fields) + conn.execute( + f"UPDATE tenant_members SET {set_clause} WHERE tenant_id=? AND user_id=?", + (*fields.values(), tenant_id, user_id), + ) + return next(item for item in self.tenant_members(tenant_id) if item.get("user_id") == user_id) + + def remove_tenant_member(self, tenant_id: str, user_id: str) -> None: + with self.connect() as conn: + current = conn.execute("SELECT * FROM tenant_members WHERE tenant_id=? AND user_id=?", (tenant_id, user_id)).fetchone() + if not current: + raise KeyError(user_id) + if current["role"] == "owner": + owners = conn.execute( + "SELECT COUNT(*) AS count FROM tenant_members WHERE tenant_id=? AND role='owner' AND status='active'", + (tenant_id,), + ).fetchone()["count"] + if int(owners or 0) <= 1: + raise ValueError("tenant must retain at least one active owner") + conn.execute("DELETE FROM tenant_members WHERE tenant_id=? AND user_id=?", (tenant_id, user_id)) + def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]: with self.connect() as conn: rows = conn.execute( - "SELECT * FROM acls WHERE resource_type=? AND resource_id=?", + "SELECT * FROM acls WHERE resource_type=? AND resource_id=? " + "AND (revoked_at IS NULL OR revoked_at='')", (resource_type, resource_id), ).fetchall() return [dict(r) for r in rows] - def set_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]: + def set_acl( + self, + resource_type: str, + resource_id: str, + entries: list[dict[str, Any]], + *, + granted_by: str | None = None, + ) -> list[dict[str, Any]]: with self.connect() as conn: conn.execute( - "DELETE FROM acls WHERE resource_type=? AND resource_id=?", - (resource_type, resource_id), + "UPDATE acls SET revoked_at=? WHERE resource_type=? AND resource_id=? " + "AND (revoked_at IS NULL OR revoked_at='')", + (utcnow(), resource_type, resource_id), ) for e in entries: conn.execute( """ - INSERT INTO acls (id, resource_type, resource_id, principal_type, principal_id, permission, create_time) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO acls + (id, resource_type, resource_id, principal_type, principal_id, + permission, create_time, granted_by, revoked_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL) """, ( new_id("acl"), @@ -4214,6 +5637,7 @@ class PlatformStore: e.get("principal_id"), e.get("permission"), utcnow(), + granted_by, ), ) rows = conn.execute( @@ -4225,7 +5649,7 @@ class PlatformStore: # ===================== 平台治理：项目空间 ===================== def projects(self, *, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]: - clauses = ["tenant_id=?"] + clauses = ["tenant_id=?", "COALESCE(deleted_at, '')=''"] params: list[Any] = [tenant_id] if status: clauses.append("status=?") @@ -4294,7 +5718,13 @@ class PlatformStore: def delete_project(self, project_id: str) -> None: with self.connect() as conn: - conn.execute("DELETE FROM projects WHERE id=?", (project_id,)) + result = conn.execute( + "UPDATE projects SET status='deleted', deleted_at=COALESCE(deleted_at, ?), updated_at=? WHERE id=? AND COALESCE(deleted_at, '')='' RETURNING id", + (utcnow(), utcnow(), project_id), + ) + if not result.fetchone(): + raise KeyError(project_id) + self.revoke_resource_security_state("project", project_id) def project_members(self, project_id: str) -> list[dict[str, Any]]: with self.connect() as conn: @@ -4390,7 +5820,12 @@ class PlatformStore: return list(grouped.values()) def set_resource_acl( - self, resource_type: str, resource_id: str, entries: list[dict[str, Any]] + self, + resource_type: str, + resource_id: str, + entries: list[dict[str, Any]], + *, + granted_by: str | None = None, ) -> list[dict[str, Any]]: """按前端格式设置资源 ACL：entries 为 [{subject_type, subject_id, permissions: []}]。""" flat: list[dict[str, Any]] = [] @@ -4403,7 +5838,7 @@ class PlatformStore: "permission": perm, } ) - self.set_acl(resource_type, resource_id, flat) + self.set_acl(resource_type, resource_id, flat, granted_by=granted_by) return self.resource_acl(resource_type, resource_id) # ===================== 平台治理：留存策略 ===================== @@ -4509,19 +5944,53 @@ class PlatformStore: return [dict(r) for r in rows] def assign_gpus(self, assignments: list[dict[str, Any]], assigned_by: str | None = None) -> list[dict[str, Any]]: - """批量分配 GPU（幂等：已存在的分配跳过）。""" + """批量分配 GPU（幂等，并校验节点、卡存在性和占用冲突）。""" now = utcnow() with self.connect() as conn: for a in assignments: - node_id = a["node_id"] - gpu_index = a["gpu_index"] - user_id = a["user_id"] + if not isinstance(a, dict) or not a.get("node_id") or not a.get("user_id"): + raise ValueError("GPU assignment requires node_id and user_id") + node_id = str(a["node_id"]) + user_id = str(a["user_id"]) + try: + gpu_index = int(a["gpu_index"]) + except (KeyError, TypeError, ValueError): + raise ValueError("gpu_index must be an integer") + if gpu_index < 0: + raise ValueError("gpu_index must be non-negative") + node = conn.execute( + "SELECT id, enabled, gpu_count FROM compute_nodes WHERE id=? FOR UPDATE", + (node_id,), + ).fetchone() + if not node or not node.get("enabled"): + raise ValueError("compute node is not available") + gpu = conn.execute( + "SELECT 1 FROM gpus WHERE node_id=? AND gpu_index=?", + (node_id, gpu_index), + ).fetchone() + if not gpu and gpu_index >= int(node.get("gpu_count") or 0): + raise ValueError("GPU index does not exist on compute node") + user = conn.execute( + "SELECT id, status FROM users WHERE id=?", + (user_id,), + ).fetchone() + if not user or user.get("status") != "active": + raise ValueError("target user is not active") existing = conn.execute( "SELECT id FROM gpu_assignments WHERE node_id=? AND gpu_index=? AND user_id=?", (node_id, gpu_index, user_id), ).fetchone() if existing: continue + occupied = conn.execute( + """SELECT user_id FROM gpu_assignments + WHERE node_id=? AND gpu_index=? AND user_id<>?""", + (node_id, gpu_index, user_id), + ).fetchone() + if occupied: + raise ValueError( + f"GPU {node_id}:{gpu_index} is already assigned to another user" + ) aid = new_id("ga") conn.execute( """ @@ -4569,7 +6038,7 @@ class PlatformStore: - 资源所有者可见（需调用方在 all_ids 中提供 owned ids）。 - ACL 授权的用户/角色可见。 """ - if user.get("role") == "admin" or user.get("protected"): + if user.get("platform_role") == "platform_admin" or user.get("role") == "admin" or user.get("protected"): return all_ids if not all_ids: return [] diff --git a/backend/app/db/sql/000_full_init.sql b/backend/app/db/sql/000_full_init.sql index 7ec5813..441139f 100644 --- a/backend/app/db/sql/000_full_init.sql +++ b/backend/app/db/sql/000_full_init.sql @@ -4,7 +4,7 @@ -- 用途：切换到新的 PG 数据集时，一次性创建平台运行所需的全部数据库对象与 -- 基础种子数据（幂等，可重复执行）。 -- --- 覆盖范围（与运行时代码实际使用的表一致）： +-- 覆盖范围（与运行时代码实际使用的表一致，离线新库只需执行本文件）： -- 001_platform_runtime.sql 平台核心表 -- 002_governance.sql 治理表（租户 / 审批 / 审计 / 留存） -- 003_tenant_quota.sql 租户配额列 @@ -16,10 +16,9 @@ -- 说明： -- * 本脚本通过 psql 执行，包含 DO $$ ... $$ 块与事务，不能用应用的 -- executescript()（按分号切分）执行。 --- * 应用启动时 PlatformStore.ensure_schema() 只会自动执行 --- 001 / 002_governance / 003_tenant_quota；数据处理表需另跑 --- 002_data_process.sql（本脚本已包含）。应用首次启动还会自动补充 --- admin/operator 种子用户（本脚本已包含，二选一即可）。 +-- * 本文件已经合并平台运行、治理、数据处理、权限、MinIO、GPU 预留、 +-- 缓存治理和数据转换等全部初始化对象；全新数据库无需再执行其他 SQL。 +-- 应用启动时的 ensure_schema() 仅作为兼容兜底，不是离线初始化前置条件。 -- * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS， -- 可在已初始化的库上安全重复执行。 -- @@ -45,12 +44,20 @@ CREATE TABLE IF NOT EXISTS users ( password_hash TEXT NOT NULL, display_name TEXT NOT NULL, role TEXT NOT NULL, + platform_role TEXT NOT NULL DEFAULT 'platform_user', status TEXT NOT NULL, permissions TEXT NOT NULL, create_time TEXT NOT NULL, last_login TEXT, - protected INTEGER NOT NULL DEFAULT 0 + protected INTEGER NOT NULL DEFAULT 0, + tenant_id TEXT NOT NULL DEFAULT 'default', + deleted_at TEXT, + deleted_by TEXT ); +ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at); +CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at); CREATE TABLE IF NOT EXISTS models ( id TEXT PRIMARY KEY, @@ -68,6 +75,9 @@ CREATE TABLE IF NOT EXISTS models ( created_by TEXT, tenant_id TEXT, project_id TEXT, + storage_status TEXT NOT NULL DEFAULT 'pending', + storage_error TEXT, + storage_version_id TEXT, deleted_at TEXT, deleted_by TEXT ); @@ -131,6 +141,10 @@ CREATE TABLE IF NOT EXISTS model_export_jobs ( create_time TEXT NOT NULL, completed_at TEXT ); +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT; CREATE TABLE IF NOT EXISTS datasets ( id TEXT PRIMARY KEY, @@ -209,8 +223,16 @@ CREATE TABLE IF NOT EXISTS fine_tune_tasks ( compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL, gpus TEXT NOT NULL, sync_job_id TEXT, - compute_job_id TEXT + compute_job_id TEXT, + tenant_id TEXT NOT NULL DEFAULT 'default', + created_by TEXT, + deleted_at TEXT, + deleted_by TEXT ); +CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active + ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator + ON fine_tune_tasks(created_by, deleted_at, create_time DESC); CREATE TABLE IF NOT EXISTS fine_tune_metrics ( id TEXT PRIMARY KEY, @@ -273,6 +295,8 @@ CREATE TABLE IF NOT EXISTS resource_replicas ( node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE, resource_type TEXT NOT NULL, resource_id TEXT NOT NULL, + version_id TEXT, + storage_object_id TEXT, local_path TEXT NOT NULL, status TEXT NOT NULL, sync_status TEXT NOT NULL, @@ -310,6 +334,11 @@ CREATE TABLE IF NOT EXISTS storage_objects ( create_time TEXT NOT NULL, UNIQUE (resource_type, resource_id, version_id, object_key) ); +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT; CREATE TABLE IF NOT EXISTS storage_cache_jobs ( id TEXT PRIMARY KEY, @@ -323,6 +352,23 @@ CREATE TABLE IF NOT EXISTS storage_cache_jobs ( create_time TEXT NOT NULL, completed_at TEXT ); +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT; + +CREATE TABLE IF NOT EXISTS storage_cleanup_jobs ( + id TEXT PRIMARY KEY, + storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE, + action TEXT NOT NULL DEFAULT 'delete', + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT, + create_time TEXT NOT NULL, + completed_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time); CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id); CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status); @@ -332,8 +378,16 @@ CREATE TABLE IF NOT EXISTS eval_tasks ( name TEXT NOT NULL, payload TEXT NOT NULL, status TEXT NOT NULL, - create_time TEXT NOT NULL + create_time TEXT NOT NULL, + created_by TEXT, + tenant_id TEXT, + deleted_at TEXT, + deleted_by TEXT ); +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; CREATE TABLE IF NOT EXISTS eval_dimensions ( id TEXT PRIMARY KEY, @@ -349,8 +403,17 @@ CREATE TABLE IF NOT EXISTS compare_tasks ( name TEXT NOT NULL, payload TEXT NOT NULL, status TEXT NOT NULL, - create_time TEXT NOT NULL + create_time TEXT NOT NULL, + created_by TEXT, + tenant_id TEXT, + deleted_at TEXT, + deleted_by TEXT ); +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at); CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status); CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id); @@ -368,6 +431,24 @@ CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id); CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status); CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status); CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running'); + +-- 评测/推理等非训练任务的统一 GPU 原子预留。训练任务继续使用 gpu_allocations。 +CREATE TABLE IF NOT EXISTS gpu_reservations ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE, + gpu_index INTEGER NOT NULL, + owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')), + owner_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')), + create_time TEXT NOT NULL, + released_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner + ON gpu_reservations(owner_type, owner_id, status); +CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status + ON gpu_reservations(node_id, status); +CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active + ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved'; CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at); CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id); CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id); @@ -393,6 +474,9 @@ CREATE TABLE IF NOT EXISTS projects ( create_by TEXT, updated_at TEXT ); +ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at); CREATE TABLE IF NOT EXISTS project_members ( project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, @@ -433,6 +517,15 @@ CREATE TABLE IF NOT EXISTS acls ( ); ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT; ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_error TEXT; +ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_version_id TEXT; +UPDATE models SET storage_status = CASE + WHEN model_source IN ('api', 'online') THEN 'not_applicable' + WHEN storage_status IS NULL OR storage_status = '' THEN 'pending' + ELSE storage_status +END +WHERE storage_status IS NULL OR storage_status = '' OR model_source IN ('api', 'online'); ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT; ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT; ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT; @@ -462,6 +555,7 @@ ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT; ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT; ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT; ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default'; -- ============================================================================ -- 二、治理表（来源：002_governance.sql） @@ -477,6 +571,15 @@ CREATE TABLE IF NOT EXISTS tenants ( retention_policy_id TEXT, create_time TEXT ); +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at); +INSERT INTO tenants (id, name, code, status, quota, create_time) +VALUES ('default', '默认租户', 'default', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) +ON CONFLICT (id) DO NOTHING; +INSERT INTO tenants (id, name, code, status, quota, create_time) +VALUES ('admin', '管理员租户', 'admin', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) +ON CONFLICT (id) DO UPDATE SET name='管理员租户', code='admin', status='active', deleted_at=NULL, deleted_by=NULL; CREATE TABLE IF NOT EXISTS approval_templates ( id TEXT PRIMARY KEY, @@ -534,6 +637,34 @@ CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id); CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action); CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time); +-- ---- 操作日志（接口操作审计） ---- +CREATE TABLE IF NOT EXISTS operation_logs ( + id TEXT PRIMARY KEY, + user_id TEXT, + username TEXT, + module TEXT, + action TEXT, + target_type TEXT, + target_id TEXT, + target_name TEXT, + status TEXT NOT NULL, + error_message TEXT, + error_type TEXT, + error_traceback TEXT, + func_name TEXT, + detail TEXT, + client_ip TEXT, + request_method TEXT, + request_path TEXT, + trace_id TEXT, + duration_ms REAL, + create_time TEXT +); +CREATE INDEX IF NOT EXISTS idx_operation_logs_user_time ON operation_logs(user_id, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_operation_logs_module_time ON operation_logs(module, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_operation_logs_status ON operation_logs(status, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_operation_logs_create_time ON operation_logs(create_time DESC); + CREATE TABLE IF NOT EXISTS retention_policies ( id TEXT PRIMARY KEY, name TEXT NOT NULL, @@ -876,29 +1007,201 @@ CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_ta ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio'; ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT; ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_by TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_at TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default'; +CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at); ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT; +ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT; +ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT; +CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_002 ON resource_replicas(storage_object_id, node_id); -- ============================================================================ --- 七、种子数据：初始管理员 / 操作员 +-- 六、权限 2.0：租户成员、资源申请、ACL 生命周期和审批动作 +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS tenant_members ( + tenant_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'member', + status TEXT NOT NULL DEFAULT 'active', + invited_by TEXT, + joined_at TEXT, + expires_at TEXT, + PRIMARY KEY (tenant_id, user_id) +); +CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status); +CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status); +INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) +SELECT COALESCE(u.tenant_id, 'default'), u.id, + CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END, + 'active', COALESCE(u.create_time, NOW()::text) +FROM users u +ON CONFLICT (tenant_id, user_id) DO NOTHING; + +-- GPU/存储任务使用的租户配额原子预留账本。任务提交时预留，终态或故障回收。 +CREATE TABLE IF NOT EXISTS tenant_quota_reservations ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + owner_type TEXT NOT NULL, + owner_id TEXT NOT NULL, + gpu_count INTEGER NOT NULL DEFAULT 0, + storage_bytes BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'reserved', + create_time TEXT NOT NULL, + released_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant + ON tenant_quota_reservations(tenant_id, status); +CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner + ON tenant_quota_reservations(owner_type, owner_id, status); +CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner + ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved'; + +CREATE TABLE IF NOT EXISTS resource_access_requests ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + applicant_id TEXT NOT NULL, + principal_type TEXT NOT NULL DEFAULT 'user', + principal_id TEXT NOT NULL, + requested_permissions TEXT NOT NULL DEFAULT '[]', + reason TEXT, + approval_id TEXT, + status TEXT NOT NULL DEFAULT 'pending', + expires_at TEXT, + created_at TEXT NOT NULL, + decided_at TEXT, + decided_by TEXT +); +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource + ON resource_access_requests(resource_type, resource_id, status); +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant + ON resource_access_requests(applicant_id, status); + +ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT; +CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at); + +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT; +ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user'; + +-- ============================================================================ +-- 七、权限 2.0 完整闭环：审批策略、执行状态、结构化审计与历史归属 +-- ============================================================================ + +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant'; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active'; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT; +CREATE INDEX IF NOT EXISTS idx_approval_templates_match + ON approval_templates(tenant_id, action, resource_type, status); + +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT; +CREATE INDEX IF NOT EXISTS idx_approval_instances_execution + ON approval_instances(status, execution_status, action, resource_type, resource_id); + +ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT; +ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT; +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry + ON resource_access_requests(status, expires_at); + +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success'; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT; +CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id); +CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time); + +UPDATE datasets d +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = ''); +UPDATE models m +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = ''); +UPDATE trained_models m +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = ''); +UPDATE eval_tasks e +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = ''); + +-- ============================================================================ +-- 八、种子数据：初始管理员 / 操作员 -- 应用首次启动（ensure_seed_data）也会自动创建；此处提供以便脱离应用直接初始化。 -- 密码：admin / admin123，operator / operator123（上线前请改密）。 -- ============================================================================ INSERT INTO users - (id, username, password_hash, display_name, role, status, permissions, create_time, protected) + (id, username, password_hash, display_name, role, platform_role, status, permissions, create_time, protected, tenant_id) VALUES ( 'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd', - 'Platform Admin', 'admin', 'active', + 'Admin', 'admin', 'platform_admin', 'active', '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]', - to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1 + to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1, 'admin' ), ( 'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125', - 'Platform Operator', 'operator', 'active', - '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]', - to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0 + 'Platform Operator', 'operator', 'platform_user', 'active', + '["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","hardware","logs"]', + to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0, 'default' ) ON CONFLICT (username) DO NOTHING; +UPDATE users SET display_name='Admin', tenant_id='admin' WHERE username='admin'; + +INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) +VALUES + ('admin', 'u_admin', 'owner', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), + ('default', 'u_operator', 'member', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')) +ON CONFLICT (tenant_id, user_id) DO NOTHING; +UPDATE tenant_members SET status='disabled' WHERE tenant_id='default' AND user_id='u_admin'; + +-- Canonical hierarchy fields for new and historical rows. Project columns are +-- intentionally retained only as compatibility data and are not used here. +UPDATE users +SET platform_role = CASE WHEN role = 'admin' OR COALESCE(protected, 0) = 1 + THEN 'platform_admin' ELSE 'platform_user' END +WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user'); +INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) +SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text) +FROM tenants t +JOIN users u ON u.id = t.owner_user_id +WHERE t.owner_user_id IS NOT NULL AND COALESCE(t.status, 'active') = 'active' +ON CONFLICT (tenant_id, user_id) DO UPDATE SET role='owner', status='active'; +UPDATE tenant_members tm +SET role='member' +FROM users u +WHERE tm.user_id=u.id AND tm.role='owner' + AND (u.role <> 'admin' AND COALESCE(u.protected, 0)=0) + AND NOT EXISTS (SELECT 1 FROM tenants t WHERE t.id=tm.tenant_id AND t.owner_user_id=tm.user_id); +CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC); COMMIT; diff --git a/backend/app/db/sql/005_storage_progress_migration.sql b/backend/app/db/sql/005_storage_progress_migration.sql new file mode 100644 index 0000000..b7481f3 --- /dev/null +++ b/backend/app/db/sql/005_storage_progress_migration.sql @@ -0,0 +1,32 @@ +-- YG Fine-Tune Platform additive migration: MinIO completion and cache manifest. +-- Execute with psql against an existing database after taking a schema backup. +-- The statements are idempotent and are also included in 000_full_init.sql. + +BEGIN; + +ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT; +ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default'; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}'; +ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT; +ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio'; +ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio'; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS project_id TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT; + +CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_005 + ON resource_replicas(storage_object_id, node_id); +CREATE INDEX IF NOT EXISTS idx_storage_objects_resource_005 + ON storage_objects(resource_type, resource_id, version_id, status); +CREATE INDEX IF NOT EXISTS idx_eval_tasks_active_005 + ON eval_tasks(create_time DESC) WHERE deleted_at IS NULL; + +COMMIT; diff --git a/backend/app/db/sql/006_gpu_reservations.sql b/backend/app/db/sql/006_gpu_reservations.sql new file mode 100644 index 0000000..999a8d0 --- /dev/null +++ b/backend/app/db/sql/006_gpu_reservations.sql @@ -0,0 +1,21 @@ +-- 原子 GPU 预留：评测和推理与训练统一纳入调度占用模型。 +BEGIN; + +CREATE TABLE IF NOT EXISTS gpu_reservations ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE, + gpu_index INTEGER NOT NULL, + owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')), + owner_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')), + create_time TEXT NOT NULL, + released_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner + ON gpu_reservations(owner_type, owner_id, status); +CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status + ON gpu_reservations(node_id, status); +CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active + ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved'; + +COMMIT; diff --git a/backend/app/db/sql/007_platform_completion.sql b/backend/app/db/sql/007_platform_completion.sql new file mode 100644 index 0000000..0006f31 --- /dev/null +++ b/backend/app/db/sql/007_platform_completion.sql @@ -0,0 +1,33 @@ +-- 平台可靠性闭环：导出权限、对象清理、缓存 manifest 元数据。 +BEGIN; + +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT; + +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT; + +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT; +ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT; + +CREATE TABLE IF NOT EXISTS storage_cleanup_jobs ( + id TEXT PRIMARY KEY, + storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE, + action TEXT NOT NULL DEFAULT 'delete', + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT, + create_time TEXT NOT NULL, + completed_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time); + +COMMIT; diff --git a/backend/app/db/sql/008_permission_v2.sql b/backend/app/db/sql/008_permission_v2.sql new file mode 100644 index 0000000..7c703dc --- /dev/null +++ b/backend/app/db/sql/008_permission_v2.sql @@ -0,0 +1,53 @@ +-- 权限 2.0：租户成员、资源申请、ACL 生命周期和审批动作 + +CREATE TABLE IF NOT EXISTS tenant_members ( + tenant_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'member', + status TEXT NOT NULL DEFAULT 'active', + invited_by TEXT, + joined_at TEXT, + expires_at TEXT, + PRIMARY KEY (tenant_id, user_id) +); +CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status); +CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status); + +CREATE TABLE IF NOT EXISTS resource_access_requests ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + applicant_id TEXT NOT NULL, + principal_type TEXT NOT NULL DEFAULT 'user', + principal_id TEXT NOT NULL, + requested_permissions TEXT NOT NULL DEFAULT '[]', + reason TEXT, + approval_id TEXT, + status TEXT NOT NULL DEFAULT 'pending', + expires_at TEXT, + created_at TEXT NOT NULL, + decided_at TEXT, + decided_by TEXT +); +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource + ON resource_access_requests(resource_type, resource_id, status); +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant + ON resource_access_requests(applicant_id, status); + +ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT; +CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at); + +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT; +ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user'; + diff --git a/backend/app/db/sql/009_permission_completion.sql b/backend/app/db/sql/009_permission_completion.sql new file mode 100644 index 0000000..8bad8f5 --- /dev/null +++ b/backend/app/db/sql/009_permission_completion.sql @@ -0,0 +1,116 @@ +-- 权限 2.0 完整闭环：审批策略、执行状态、结构化审计与历史租户归属 + +-- 该迁移可独立执行：兼容仅执行过 000_full_init.sql、或未执行 008 的旧数据库。 +CREATE TABLE IF NOT EXISTS tenant_members ( + tenant_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'member', + status TEXT NOT NULL DEFAULT 'active', + invited_by TEXT, + joined_at TEXT, + expires_at TEXT, + PRIMARY KEY (tenant_id, user_id) +); +CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status); +CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status); +INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) +SELECT COALESCE(u.tenant_id, 'default'), u.id, + CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END, + 'active', COALESCE(u.create_time, NOW()::text) +FROM users u +ON CONFLICT (tenant_id, user_id) DO NOTHING; + +CREATE TABLE IF NOT EXISTS resource_access_requests ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT 'default', + resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + applicant_id TEXT NOT NULL, + principal_type TEXT NOT NULL DEFAULT 'user', + principal_id TEXT NOT NULL, + requested_permissions TEXT NOT NULL DEFAULT '[]', + reason TEXT, + approval_id TEXT, + status TEXT NOT NULL DEFAULT 'pending', + expires_at TEXT, + created_at TEXT NOT NULL DEFAULT to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), + decided_at TEXT, + decided_by TEXT +); +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT; +ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user'; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT; +ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT; +ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE models ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; + +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant'; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active'; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT; +CREATE INDEX IF NOT EXISTS idx_approval_templates_match + ON approval_templates(tenant_id, action, resource_type, status); + +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending'; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT; +ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT; +CREATE INDEX IF NOT EXISTS idx_approval_instances_execution + ON approval_instances(status, execution_status, action, resource_type, resource_id); + +ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT; +ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT; +CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry + ON resource_access_requests(status, expires_at); + +ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at); + +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT; +CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at); + +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default'; +CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at); + +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success'; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT; +ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT; +CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id); +CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time); + +-- 历史资源按创建者租户补齐归属。无法识别的资源保留 default，后续由管理员复核。 +UPDATE datasets d +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = ''); +UPDATE models m +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = ''); +UPDATE trained_models m +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = ''); +UPDATE eval_tasks e +SET tenant_id = COALESCE(u.tenant_id, 'default') +FROM users u +WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = ''); diff --git a/backend/app/db/sql/010_permission_quota_membership.sql b/backend/app/db/sql/010_permission_quota_membership.sql new file mode 100644 index 0000000..7e0c352 --- /dev/null +++ b/backend/app/db/sql/010_permission_quota_membership.sql @@ -0,0 +1,21 @@ +-- Permission 2.0: atomic tenant quota reservations for GPU-backed tasks. +-- This migration is additive and safe to run repeatedly. + +CREATE TABLE IF NOT EXISTS tenant_quota_reservations ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + owner_type TEXT NOT NULL, + owner_id TEXT NOT NULL, + gpu_count INTEGER NOT NULL DEFAULT 0, + storage_bytes BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'reserved', + create_time TEXT NOT NULL, + released_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant + ON tenant_quota_reservations(tenant_id, status); +CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner + ON tenant_quota_reservations(owner_type, owner_id, status); +CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner + ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved'; diff --git a/backend/app/db/sql/011_permission_lifecycle.sql b/backend/app/db/sql/011_permission_lifecycle.sql new file mode 100644 index 0000000..6477943 --- /dev/null +++ b/backend/app/db/sql/011_permission_lifecycle.sql @@ -0,0 +1,20 @@ +-- Permission 2.0: user and inference task lifecycle tombstones. +-- Additive migration, safe to execute repeatedly. + +ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT; +ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT; + +CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at); +CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at); diff --git a/backend/app/db/sql/012_tenant_user_hierarchy.sql b/backend/app/db/sql/012_tenant_user_hierarchy.sql new file mode 100644 index 0000000..172ea02 --- /dev/null +++ b/backend/app/db/sql/012_tenant_user_hierarchy.sql @@ -0,0 +1,63 @@ +-- Tenant/user hierarchy alignment. Additive and safe to run repeatedly. +-- New business flows use tenant_members, platform_role, tenant_id and +-- created_by. Legacy role/project fields remain for compatibility only. + +ALTER TABLE users ADD COLUMN IF NOT EXISTS platform_role TEXT NOT NULL DEFAULT 'platform_user'; +UPDATE users +SET platform_role = CASE + WHEN role = 'admin' OR COALESCE(protected, 0) = 1 THEN 'platform_admin' + ELSE 'platform_user' +END +WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user'); +CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at); + +ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default'; +ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS created_by TEXT; +ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT; +ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT; +UPDATE fine_tune_tasks +SET tenant_id = COALESCE(NULLIF(tenant_id, ''), payload::json->>'tenant_id', 'default'), + created_by = COALESCE(NULLIF(created_by, ''), payload::json->>'created_by') +WHERE tenant_id IS NULL OR tenant_id = '' OR created_by IS NULL OR created_by = ''; +CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active + ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator + ON fine_tune_tasks(created_by, deleted_at, create_time DESC); + +INSERT INTO tenants (id, name, code, status, quota, create_time) +VALUES ('default', '默认租户', 'default', 'active', '{}', NOW()::text) +ON CONFLICT (id) DO UPDATE +SET status = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN 'active' ELSE tenants.status END, + deleted_at = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_at END, + deleted_by = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_by END; + +-- Make the tenant owner relationship explicit and repair older tenant rows. +INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at) +SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text) +FROM tenants t +JOIN users u ON u.id = t.owner_user_id +WHERE t.owner_user_id IS NOT NULL + AND COALESCE(t.status, 'active') = 'active' +ON CONFLICT (tenant_id, user_id) DO UPDATE +SET role = 'owner', status = 'active'; + +-- Existing account creation used admin as a tenant owner. Platform role and +-- tenant role are separate, so keep only explicit tenant owners as owners. +UPDATE tenant_members tm +SET role = 'member' +FROM users u +WHERE tm.user_id = u.id + AND tm.role = 'owner' + AND (u.role <> 'admin' AND COALESCE(u.protected, 0) = 0) + AND NOT EXISTS ( + SELECT 1 FROM tenants t + WHERE t.id = tm.tenant_id AND t.owner_user_id = tm.user_id + ); + +-- Project is no longer a business isolation boundary. Keep historical columns +-- readable, but make tenant-scoped queries the only supported new path. +CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC); +CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC); diff --git a/backend/app/modules/approval/router.py b/backend/app/modules/approval/router.py index 4ed3d80..3832c0c 100644 --- a/backend/app/modules/approval/router.py +++ b/backend/app/modules/approval/router.py @@ -1,17 +1,105 @@ from __future__ import annotations +import json from fastapi import APIRouter, Body, Depends from typing import Any from app.api.v1.endpoints.platform import ok, fail from app.db.platform_store import get_platform_store -from app.core.auth import get_current_user, is_admin +from app.core.auth import ( + get_current_user, + has_resource_access, + is_tenant_admin, + is_admin, + resource_in_user_tenant, + resource_record, +) router = APIRouter(prefix="/approvals", tags=["approval"]) +# Approval actions are deliberately finite. A caller must not be able to +# create an arbitrary approval that no executor or audit policy understands. +ALLOWED_APPROVAL_ACTIONS = { + "resource.access", + "gpu.assign", + "tenant.quota.update", + "tenant.member.add", + "tenant.member.remove", + "model.use", + "dataset.use", + "dataset.delete", + "trained_model.merge", + "trained_model.export", + "trained_model.delete", + "fine_tune.stop", + "fine_tune.delete", + "eval.delete", + "inference.delete", + "project.archive", + "project.delete", +} + + +def _available_gpu_options() -> list[dict[str, Any]]: + """Return only online nodes and currently unassigned, idle GPUs.""" + store = get_platform_store() + nodes = store.compute_nodes() + gpus = store.gpus() + assigned = {(str(item.get("node_id")), int(item.get("gpu_index"))) for item in store.gpu_assignments()} + by_node: dict[str, list[dict[str, Any]]] = {} + for gpu in gpus: + node_id = str(gpu.get("node_id") or "") + index = int(gpu.get("id") or 0) + if gpu.get("status") != "idle" or (node_id, index) in assigned: + continue + by_node.setdefault(node_id, []).append({ + "index": index, + "name": gpu.get("name") or "GPU", + "memory_total_gb": float(gpu.get("memory_total_gb") or 0), + }) + result = [] + for node in nodes: + node_id = str(node.get("id") or "") + if not node.get("enabled") or node.get("scheduler_status") != "online" or not by_node.get(node_id): + continue + result.append({ + "id": node_id, + "code": node.get("code") or node_id, + "name": node.get("name") or node.get("code") or node_id, + "gpus": sorted(by_node[node_id], key=lambda item: item["index"]), + }) + return result + + +def _validate_gpu_request(assignments: Any) -> list[dict[str, Any]]: + if not isinstance(assignments, list) or not assignments: + raise fail(400, "assignments 不能为空") + available = { + (node["id"], gpu["index"]) + for node in _available_gpu_options() + for gpu in node["gpus"] + } + normalized = [] + seen: set[tuple[str, int]] = set() + for item in assignments: + if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None: + raise fail(400, "每项必须包含 node_id 和 gpu_index") + try: + key = (str(item["node_id"]), int(item["gpu_index"])) + except (TypeError, ValueError): + raise fail(400, "gpu_index 必须是整数") + if key not in available: + raise fail(409, f"GPU {key[0]}:{key[1]} 当前不可申请") + if key not in seen: + seen.add(key) + normalized.append({"node_id": key[0], "gpu_index": key[1]}) + return normalized + @router.get("/templates") def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not is_admin(current_user): + raise fail(403, "admin permission required") return ok(get_platform_store().approval_templates()) @@ -20,11 +108,31 @@ def create_template(payload: dict[str, Any] = Body(...), current_user: dict = De if not is_admin(current_user): raise fail(403, "admin permission required") if not payload.get("name"): raise fail(400, "name 必填") - return ok(get_platform_store().create_approval_template(payload)) + steps = payload.get("steps") or [] + if not isinstance(steps, list) or any(not isinstance(step, dict) for step in steps): + raise fail(400, "steps 格式无效") + if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS: + raise fail(400, "不支持的审批动作") + payload = { + **payload, + "created_by": current_user.get("id"), + "tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default", + "scope": payload.get("scope") or "tenant", + "status": payload.get("status") or "active", + } + template = get_platform_store().create_approval_template(payload) + get_platform_store().record_audit( + action="approval.template.create", actor_id=current_user.get("id"), + target_type="approval_template", target_id=template["id"], + tenant_id=template.get("tenant_id"), + ) + return ok(template) @router.get("/templates/{template_id}") -def get_template(template_id: str) -> dict[str, Any]: +def get_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not is_admin(current_user): + raise fail(403, "admin permission required") try: return ok(get_platform_store().approval_template(template_id)) except KeyError: @@ -34,8 +142,16 @@ def get_template(template_id: str) -> dict[str, Any]: @router.put("/templates/{template_id}") def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not is_admin(current_user): raise fail(403, "admin permission required") + if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS: + raise fail(400, "不支持的审批动作") try: - return ok(get_platform_store().update_approval_template(template_id, payload)) + template = get_platform_store().update_approval_template(template_id, payload) + get_platform_store().record_audit( + action="approval.template.update", actor_id=current_user.get("id"), + target_type="approval_template", target_id=template_id, + tenant_id=template.get("tenant_id"), detail=f"fields={','.join(payload.keys())}", + ) + return ok(template) except KeyError: raise fail(404, "template not found") @@ -44,15 +160,77 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...), curre def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: if not is_admin(current_user): raise fail(403, "admin permission required") try: - return ok(get_platform_store().delete_approval_template(template_id)) + template = get_platform_store().delete_approval_template(template_id) + get_platform_store().record_audit( + action="approval.template.delete", actor_id=current_user.get("id"), + target_type="approval_template", target_id=template_id, + tenant_id=template.get("tenant_id"), + ) + return ok(template) except KeyError: raise fail(404, "template not found") @router.get("") -def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: - items = get_platform_store().approval_instances(status=status) - return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")]) +def list_instances( + status: str | None = None, + mine: bool = False, + current_user: dict = Depends(get_current_user), +) -> dict[str, Any]: + user_id = current_user.get("id") + items = get_platform_store().approval_instances( + status=status, + applicant_id=user_id if mine and not is_admin(current_user) else None, + ) + if is_admin(current_user): + return ok(items) + if mine: + return ok(items) + visible = [] + for item in items: + if item.get("applicant_id") == user_id: + visible.append(item) + continue + if any(step.get("approver_id") == user_id and step.get("status") == "pending" for step in item.get("steps", [])): + visible.append(item) + continue + if is_tenant_admin(current_user, item.get("tenant_id")) and item.get("status") == "pending": + visible.append(item) + return ok(visible) + + +@router.get("/gpu-options") +def gpu_request_options(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + """Self-service GPU options; does not expose the admin compute page.""" + return ok({"nodes": _available_gpu_options()}) + + +@router.post("/gpu-requests") +def create_gpu_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + assignments = _validate_gpu_request(payload.get("assignments")) + user_id = str(current_user.get("id") or "") + if is_admin(current_user): + try: + return ok({"approval_required": False, "assignments": get_platform_store().assign_gpus( + [{**item, "user_id": user_id} for item in assignments], assigned_by=user_id, + )}) + except ValueError as exc: + raise fail(409, str(exc)) + normalized = [{**item, "user_id": user_id} for item in assignments] + instance = get_platform_store().create_approval_instance({ + "resource_type": "gpu", + "resource_id": f"batch:{user_id}", + "applicant_id": user_id, + "action": "gpu.assign", + "tenant_id": current_user.get("tenant_id") or "default", + "reason": json.dumps({"assignments": normalized, "reason": payload.get("reason")}, ensure_ascii=False), + }) + get_platform_store().record_audit( + action="gpu.assign.request", actor_id=user_id, target_type="gpu", + target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default", + detail=f"count={len(normalized)}", + ) + return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance}) @router.post("") @@ -61,6 +239,24 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De for field in ("resource_type", "resource_id"): if not payload.get(field): raise fail(400, f"{field} 必填") + resource = resource_record(str(payload["resource_type"]), str(payload["resource_id"])) + if not resource and not is_admin(current_user): + raise fail(404, "resource not found") + action = str(payload.get("action") or "") + if not action or action not in ALLOWED_APPROVAL_ACTIONS: + raise fail(400, "不支持的审批动作") + if action != "resource.access" and not is_admin(current_user) and not has_resource_access( + str(payload["resource_type"]), str(payload["resource_id"]), current_user, "read" + ): + raise fail(403, "no permission to request approval for this resource") + if resource and not is_admin(current_user) and not resource_in_user_tenant(str(payload["resource_type"]), resource, current_user): + raise fail(403, "resource belongs to another tenant") + requested = payload.get("requested_permissions") or [] + allowed = {"read", "write", "execute", "download"} + if any(permission not in allowed for permission in requested): + raise fail(400, "invalid requested permission") + payload["tenant_id"] = current_user.get("tenant_id") or "default" + payload["requested_permissions"] = requested try: return ok(get_platform_store().create_approval_instance(payload)) except KeyError: @@ -71,7 +267,7 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: try: item = get_platform_store().approval_instance(instance_id) - if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"): + if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id") and not is_tenant_admin(current_user, item.get("tenant_id")): raise fail(403, "no permission to access approval") return ok(item) except KeyError: @@ -83,18 +279,118 @@ def decide( instance_id: str, step_index: int, payload: dict[str, Any] = Body(...), + current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: - if not payload.get("approver_id"): - raise fail(400, "approver_id 必填") try: - return ok( - get_platform_store().decide_approval_step( + instance = get_platform_store().approval_instance(instance_id) + if instance.get("applicant_id") == current_user.get("id"): + raise fail(403, "applicant cannot approve own request") + step = next((item for item in instance.get("steps", []) if int(item.get("step_index", -1)) == step_index), None) + if not step and is_admin(current_user) and not instance.get("steps"): + step = {"approver_id": None, "status": "pending"} + if not step: + raise fail(404, "approval step not found") + designated = step.get("approver_id") + if not is_admin(current_user) and designated != current_user.get("id") and not ( + step.get("approver_type") == "admin" and is_tenant_admin(current_user, instance.get("tenant_id")) + ): + raise fail(403, "current user is not the designated approver") + if instance.get("action") == "tenant.quota.update" and not is_admin(current_user): + raise fail(403, "only platform administrator can approve tenant quota changes") + submitted_approver = payload.get("approver_id") + if submitted_approver and submitted_approver != current_user.get("id"): + raise fail(403, "approver_id must match current session") + store = get_platform_store() + result = store.decide_approval_step( instance_id, step_index, - approver_id=payload["approver_id"], + approver_id=str(current_user.get("id")), approved=bool(payload.get("approved", False)), comment=payload.get("comment"), ) + if result.get("status") == "approved" and result.get("execution_status") == "ready": + try: + effect = store.apply_approval_effect(result, str(current_user.get("id") or "")) + if effect is not None: + result = store.approval_instance(instance_id) + except Exception as exc: + with store.connect() as conn: + conn.execute( + "UPDATE approval_instances SET execution_status='failed', execution_error=? WHERE id=?", + (str(exc), instance_id), + ) + raise fail(409, f"审批已通过，但执行失败：{exc}") + store.record_audit( + action="approval.decision", actor_id=current_user.get("id"), + target_type="approval_instance", target_id=instance_id, + tenant_id=result.get("tenant_id"), result="success" if payload.get("approved") else "rejected", + detail=f"step={step_index}", reason=payload.get("comment"), ) + return ok(result) except (KeyError, ValueError) as e: raise fail(400, str(e)) + + +@router.post("/resource-access/requests") +def create_resource_access_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + resource_type = str(payload.get("resource_type") or "") + resource_id = str(payload.get("resource_id") or "") + resource = resource_record(resource_type, resource_id) + if not resource: + raise fail(404, "resource not found") + if not is_admin(current_user) and not resource_in_user_tenant(resource_type, resource, current_user): + raise fail(403, "resource belongs to another tenant") + permissions = payload.get("requested_permissions") or ["read"] + allowed = {"read", "write", "execute", "download"} + if not permissions or any(permission not in allowed for permission in permissions): + raise fail(400, "invalid requested permissions") + result = get_platform_store().create_resource_access_request({ + "resource_type": resource_type, + "resource_id": resource_id, + "applicant_id": current_user.get("id"), + "principal_type": "user", + "principal_id": current_user.get("id"), + "requested_permissions": permissions, + "reason": payload.get("reason"), + "template_id": payload.get("template_id"), + "tenant_id": current_user.get("tenant_id") or "default", + "expires_at": payload.get("expires_at"), + }) + get_platform_store().record_audit( + action="resource.access.request", actor_id=current_user.get("id"), + target_type=resource_type, target_id=resource_id, + tenant_id=current_user.get("tenant_id") or "default", + detail=f"permissions={','.join(permissions)}", + ) + return ok(result) + + +@router.get("/resource-access/requests") +def list_resource_access_requests(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + items = get_platform_store().resource_access_requests( + user_id=None if is_admin(current_user) else current_user.get("id"), + status=status, + ) + return ok(items) + + +@router.post("/resource-access/requests/{request_id}/cancel") +def cancel_resource_access_request(request_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + try: + item = get_platform_store().cancel_resource_access_request( + request_id, + str(current_user.get("id") or ""), + is_admin_actor=is_admin(current_user), + ) + except KeyError: + raise fail(404, "access request not found") + except PermissionError as exc: + raise fail(403, str(exc)) + except ValueError as exc: + raise fail(409, str(exc)) + get_platform_store().record_audit( + action="resource.access.cancel", actor_id=current_user.get("id"), + target_type="resource_access_request", target_id=request_id, + tenant_id=item.get("tenant_id"), + ) + return ok(item) diff --git a/backend/app/modules/compute_gateway/sync.py b/backend/app/modules/compute_gateway/sync.py index ce6183d..7c33aa1 100644 --- a/backend/app/modules/compute_gateway/sync.py +++ b/backend/app/modules/compute_gateway/sync.py @@ -14,6 +14,18 @@ from app.modules.storage.minio_store import get_object_storage MAX_STARTING_ATTEMPTS = 40 +def _extract_job_failure_reason(log_text: str, limit: int = 2000) -> str: + """Return a concise actionable reason from a failed Compute job log.""" + lines = [line.strip() for line in str(log_text or "").splitlines() if line.strip()] + if not lines: + return "" + markers = ("[eval] FAILED", "Traceback", "RuntimeError", "Error:", "ERROR") + for index in range(len(lines) - 1, -1, -1): + if any(marker in lines[index] for marker in markers): + return "\n".join(lines[index : index + 8])[-limit:] + return "\n".join(lines[-8:])[-limit:] + + async def _archive_node_directory( store: Any, client: ComputeNodeClient, @@ -27,12 +39,15 @@ async def _archive_node_directory( """Archive a completed node directory to MinIO, preserving subdirectories.""" data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve() source = Path(source_path).resolve() + if source == data_root: + raise RuntimeError("refuse to archive compute data root; output_dir must be a task subdirectory") try: relative_root = source.relative_to(data_root).as_posix() except ValueError as exc: raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc queue = [relative_root] archived: list[dict[str, Any]] = [] + max_files = 10000 while queue: relative = queue.pop(0) listing = await client.list_files(root="data", relative_path=relative) @@ -49,6 +64,8 @@ async def _archive_node_directory( except ValueError: relative_file = Path(str(item.get("name") or Path(path).name)).name object_key = f"{object_prefix}/{version_id}/{relative_file}" + if len(archived) >= max_files: + raise RuntimeError(f"archive file count exceeds limit {max_files}") upload_url = get_object_storage().presigned_put(object_key) result = await client.upload_file_to_url(path, upload_url, object_key) metadata = get_object_storage().stat(object_key) @@ -115,11 +132,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]: item["status"] = "error" item["error"] = "compute node deleted" store.mark_inference_unloaded(item.get("node_id") or "") + store.release_external_gpus("inference", str(task["id"]), item.get("node_id")) continue if not node.get("enabled") or node.get("scheduler_status") != "online": item["status"] = "error" item["error"] = "compute node offline" store.mark_inference_unloaded(node["id"]) + store.release_external_gpus("inference", str(task["id"]), node["id"]) continue try: status = await ComputeNodeClient(node["api_base_url"]).inference_status() @@ -128,6 +147,7 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]: item["status"] = "error" item["error"] = f"compute node unreachable: {exc}" store.mark_inference_unloaded(node["id"]) + store.release_external_gpus("inference", str(task["id"]), node["id"]) continue node_status = status.get("status") if node_status == "ready": @@ -139,11 +159,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]: item["status"] = "error" item["error"] = status.get("error") or "model load failed on compute node" store.mark_inference_unloaded(node["id"]) + store.release_external_gpus("inference", str(task["id"]), node["id"]) elif node_status == "idle": # 节点重启导致已加载模型丢失 item["status"] = "error" item["error"] = "model disappeared from compute node (node may have restarted)" store.mark_inference_unloaded(node["id"]) + store.release_external_gpus("inference", str(task["id"]), node["id"]) # node_status == "loading" -> 保持 starting，下轮再查 if dirty: if any(i.get("status") in {"ready", "running"} for i in items): @@ -157,11 +179,16 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]: return reconciled -async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None: +async def fetch_eval_result_content( + client: ComputeNodeClient, + node: dict[str, Any], + job: dict[str, Any], + file_name: str = "eval_results.json", +) -> dict[str, Any] | None: output_dir = job.get("output_dir") if not output_dir: return None - full_path = f"{str(output_dir).rstrip('/')}/eval_results.json" + full_path = f"{str(output_dir).rstrip('/')}/{file_name}" data_root = "/data/yg-ft/" if full_path.startswith(data_root): full_path = full_path[len(data_root):] @@ -175,11 +202,36 @@ async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, A return payload if isinstance(payload, dict) else None -async def poll_compute_jobs_once() -> dict[str, Any]: - store = get_platform_store() +async def fetch_eval_progress_content( + client: ComputeNodeClient, + node: dict[str, Any], + job: dict[str, Any], +) -> dict[str, Any] | None: + return await fetch_eval_result_content(client, node, job, "eval_progress.json") + + +async def poll_compute_jobs_once(store: Any | None = None) -> dict[str, Any]: + store = store or get_platform_store() synced: list[dict[str, Any]] = [] failed: list[dict[str, str]] = [] - for task in store.running_compute_tasks(): + online_nodes = { + str(node.get("id")) + for node in store.compute_nodes() + if node.get("enabled") and node.get("scheduler_status") in {"online", "draining"} + } + training_tasks = {str(task["id"]): task for task in store.running_compute_tasks()} + # Completed tasks whose MinIO archive was interrupted remain eligible for + # reconciliation after a Backend restart or a transient node failure. + if get_settings().minio_enabled: + for task in store.tasks(): + if task.get("status") != "completed" or not task.get("compute_job_id"): + continue + if ( + str(task.get("archive_status") or "") != "completed" + and str(task.get("compute_node_id")) in online_nodes + ): + training_tasks.setdefault(str(task["id"]), task) + for task in training_tasks.values(): node = _node_for_task(task) if not node: failed.append({"task_id": task["id"], "error": "compute node not found"}) @@ -215,26 +267,47 @@ async def poll_compute_jobs_once() -> dict[str, Any]: None, ) if trained_model: - archived = await _archive_node_directory( - store, - client, - node, - str(job["output_dir"]), - "trained_model", - str(trained_model["id"]), - str(job.get("id") or task.get("compute_job_id") or task["id"]), - f"trained_models/{trained_model['id']}", - ) - artifacts = store.model_artifacts(str(trained_model["id"])) - if archived and artifacts: - store.link_model_artifact_storage_object( - str(artifacts[0]["id"]), str(archived[0]["id"]) + try: + archived = await _archive_node_directory( + store, + client, + node, + str(job["output_dir"]), + "trained_model", + str(trained_model["id"]), + str(job.get("id") or task.get("compute_job_id") or task["id"]), + f"trained_models/{trained_model['id']}", ) + artifacts = store.model_artifacts(str(trained_model["id"])) + if archived and artifacts: + store.link_model_artifact_storage_object( + str(artifacts[0]["id"]), str(archived[0]["id"]) + ) + store.update_task(task["id"], { + "archive_status": "completed", + "archive_object_ids": [str(item["id"]) for item in archived], + "archive_error": "", + }) + except Exception as archive_exc: + store.update_task(task["id"], { + "archive_status": "pending", + "archive_error": str(archive_exc)[:2000], + }) + raise synced.append(updated_task) except Exception as exc: # noqa: BLE001 - keep polling other jobs failed.append({"task_id": task["id"], "error": str(exc)}) standalone_synced: list[dict[str, Any]] = [] - for record in store.active_standalone_compute_jobs(): + standalone_jobs = { + str(record["id"]): record + for record in store.active_standalone_compute_jobs() + if str(record.get("node_id")) in online_nodes + } + if get_settings().minio_enabled: + for record in store.standalone_compute_jobs_pending_archive(): + if str(record.get("node_id")) in online_nodes: + standalone_jobs.setdefault(str(record["id"]), record) + for record in standalone_jobs.values(): node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None) if not node: failed.append({"job_id": record["id"], "error": "compute node not found"}) @@ -266,12 +339,32 @@ async def poll_compute_jobs_once() -> dict[str, Any]: store.link_model_artifact_storage_object( str(artifacts[0]["id"]), str(archived[0]["id"]) ) + store.update_compute_job_archive(record["id"], "completed", [str(item["id"]) for item in archived]) except Exception as exc: # noqa: BLE001 - keep polling other jobs + try: + store.update_compute_job_archive(record["id"], "pending", [], str(exc)[:2000]) + except Exception: + pass failed.append({"job_id": record["id"], "error": str(exc)}) # ── Eval job sync ──────────────────────────────────────────────── eval_synced = 0 - for eval_task in store.running_eval_tasks(): + eval_tasks = {str(task["id"]): task for task in store.running_eval_tasks()} + if get_settings().minio_enabled: + # A completed evaluation can win the race with the poller: its status + # is persisted before the report archive finishes. Keep such tasks in + # the reconciliation set until the report object is available. + for task in store.eval_tasks(): + if ( + task.get("status") == "completed" + and task.get("compute_job_id") + and str(task.get("archive_status") or "") != "completed" + and str(task.get("compute_node_id")) in online_nodes + ): + eval_tasks.setdefault(str(task["id"]), task) + for eval_task in eval_tasks.values(): + if str(eval_task.get("compute_node_id")) not in online_nodes: + continue node = next( (item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")), None, @@ -283,15 +376,39 @@ async def poll_compute_jobs_once() -> dict[str, Any]: client = ComputeNodeClient(node["api_base_url"]) job = await client.get_job(eval_task["compute_job_id"]) result_content = None - # Try to read eval_results.json from the job output directory + # Read live progress and partial results while the evaluator is running. + if job.get("status") in {"queued", "running"} and job.get("output_dir"): + try: + progress_content = await fetch_eval_progress_content(client, node, job) + if progress_content: + store.update_eval_task( + eval_task["id"], + { + "progress_detail": progress_content, + "progress": progress_content.get("percentage", eval_task.get("progress", 0)), + }, + ) + except Exception: + pass + try: + result_content = await fetch_eval_result_content(client, node, job) + except Exception: + result_content = None + # Try to read eval_results.json from the job output directory on completion. if job.get("status") == "completed" and job.get("output_dir"): try: result_content = await fetch_eval_result_content(client, node, job) except Exception: pass + if job.get("status") in {"failed", "stopped"} and not job.get("error"): + try: + failure_logs = await client.job_logs(eval_task["compute_job_id"], tail_lines=120) + job["error"] = _extract_job_failure_reason(str(failure_logs.get("content") or "")) + except Exception: + pass store.apply_eval_job_result(eval_task["id"], job, result_content) if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"): - await _archive_node_directory( + archived = await _archive_node_directory( store, client, node, @@ -301,9 +418,28 @@ async def poll_compute_jobs_once() -> dict[str, Any]: str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]), f"evaluations/{eval_task['id']}", ) + report_object = next( + (item for item in archived if Path(str(item.get("file_name") or "")).name == "eval_results.json"), + archived[0] if archived else None, + ) + store.update_eval_task(eval_task["id"], { + "report_storage_object_id": str(report_object["id"]) if report_object else "", + "archive_status": "completed", + "archive_object_ids": [str(item["id"]) for item in archived], + "archive_error": "", + }) # 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记 eval_synced += 1 except Exception as exc: # noqa: BLE001 + try: + current_eval = store.eval_task(eval_task["id"]) + except Exception: + current_eval = eval_task + if current_eval.get("status") == "completed": + try: + store.update_eval_task(eval_task["id"], {"archive_status": "pending", "archive_error": str(exc)[:2000]}) + except Exception: + pass failed.append({"eval_task_id": eval_task["id"], "error": str(exc)}) # ── Inference load reconciliation ───────────────────────────────────── diff --git a/backend/app/modules/data_convert/router.py b/backend/app/modules/data_convert/router.py index 2a364b6..0b2453d 100644 --- a/backend/app/modules/data_convert/router.py +++ b/backend/app/modules/data_convert/router.py @@ -6,11 +6,11 @@ import os from pathlib import Path from typing import Any -from fastapi import APIRouter, Body, Depends, File, UploadFile +from fastapi import APIRouter, Body, Depends, File, Request, UploadFile from fastapi.responses import FileResponse, Response from app.api.v1.endpoints.platform import ok, fail -from app.core.auth import get_current_user, is_admin +from app.core.auth import get_current_user, has_resource_access, is_admin from app.core.config import get_settings from app.core.op_log import op_log, OpModule, OpAction from app.db.platform_store import get_platform_store, new_id @@ -18,7 +18,26 @@ from app.modules.storage.minio_store import get_object_storage from app.modules.storage.policy import should_store_in_minio -router = APIRouter(prefix="/data-convert", tags=["data-convert"]) +def _authorize_data_convert_request( + request: Request, + task_id: str | None = None, + current_user: dict[str, Any] = Depends(get_current_user), +) -> None: + """Protect every task-scoped conversion endpoint with resource ACL.""" + if not task_id or is_admin(current_user): + return + permission = "read" if request.method in {"GET", "HEAD"} else "write" + if request.url.path.endswith("/run") or request.url.path.endswith("/import-as-dataset"): + permission = "execute" + if not has_resource_access("data_convert", task_id, current_user, permission): + raise fail(403, "no permission to access this data convert task") + + +router = APIRouter( + prefix="/data-convert", + tags=["data-convert"], + dependencies=[Depends(_authorize_data_convert_request)], +) # 存储根目录 STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert" @@ -207,24 +226,32 @@ def list_tasks( if is_admin(current_user): # 管理员可见全部 rows = conn.execute( - "SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL " - "ORDER BY create_time DESC LIMIT %s OFFSET %s", + "SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name " + "FROM data_convert_tasks task " + "LEFT JOIN users creator ON creator.id=task.created_by " + "LEFT JOIN users processor ON processor.id=task.processed_by " + "WHERE task.deleted_at IS NULL " + "ORDER BY task.create_time DESC LIMIT %s OFFSET %s", (page_size, (page - 1) * page_size), ).fetchall() total = conn.execute( "SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL" ).fetchone()[0] else: - # 普通用户只能看到自己创建的 + # 普通用户只能看到本租户且由自己创建的任务；跨租户 ACL 通过任务级依赖访问。 user_id = current_user.get("id") rows = conn.execute( - "SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s " - "ORDER BY create_time DESC LIMIT %s OFFSET %s", - (user_id, page_size, (page - 1) * page_size), + "SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name " + "FROM data_convert_tasks task " + "LEFT JOIN users creator ON creator.id=task.created_by " + "LEFT JOIN users processor ON processor.id=task.processed_by " + "WHERE task.deleted_at IS NULL AND task.tenant_id=%s AND task.created_by=%s " + "ORDER BY task.create_time DESC LIMIT %s OFFSET %s", + (current_user.get("tenant_id") or "default", user_id, page_size, (page - 1) * page_size), ).fetchall() total = conn.execute( - "SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s", - (user_id,) + "SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND tenant_id=%s AND created_by=%s", + (current_user.get("tenant_id") or "default", user_id,) ).fetchone()[0] return ok({"items": [dict(r) for r in rows], "total": total}) @@ -243,11 +270,16 @@ def create_task( description = str(payload.get("description") or "").strip() user_id = current_user.get("id") store = get_platform_store() + tenant_id = current_user.get("tenant_id") or "default" + try: + store.assert_active_tenant(tenant_id) + except ValueError as exc: + raise fail(400, str(exc)) with store.connect() as conn: conn.execute( - "INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) " - "VALUES (%s, %s, %s, %s, %s)", - (task_id, name, description, output_filename, user_id), + "INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by, tenant_id) " + "VALUES (%s, %s, %s, %s, %s, %s)", + (task_id, name, description, output_filename, user_id, tenant_id), ) # MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。 if not _minio_enabled(): @@ -318,8 +350,8 @@ async def upload_source_files( # 标记上传完成 with store.connect() as conn: conn.execute( - "UPDATE data_convert_tasks SET status='uploaded', update_time=NOW() WHERE id=%s", - (task_id,), + "UPDATE data_convert_tasks SET status='uploaded', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (current_user.get("id"), task_id), ) # 自动转换并导入数据集 try: @@ -332,8 +364,8 @@ async def upload_source_files( with store.connect() as conn: conn.execute( "UPDATE data_convert_tasks SET status='completed', " - "input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s", - (input_count, output_count, task_id), + "input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (input_count, output_count, current_user.get("id"), task_id), ) # 自动导入数据集 content = output.decode("utf-8") @@ -348,6 +380,7 @@ async def upload_source_files( "count": output_count, "description": f"由数据类型转换任务 {task_id} 自动导入", "created_by": task.get("created_by") or current_user.get("id"), + "tenant_id": task.get("tenant_id") or current_user.get("tenant_id") or "default", }) dataset_id = dataset["id"] with store.connect() as conn: @@ -375,8 +408,8 @@ async def upload_source_files( except Exception as exc: with store.connect() as conn: conn.execute( - "UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s", - (str(exc)[:500], task_id), + "UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (str(exc)[:500], current_user.get("id"), task_id), ) return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]}) @@ -396,8 +429,8 @@ def run_convert( store = get_platform_store() with store.connect() as conn: conn.execute( - "UPDATE data_convert_tasks SET status='running', error_message='', update_time=NOW() WHERE id=%s", - (task_id,), + "UPDATE data_convert_tasks SET status='running', error_message='', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (current_user.get("id"), task_id), ) try: if _minio_enabled(): @@ -408,14 +441,14 @@ def run_convert( with store.connect() as conn: conn.execute( "UPDATE data_convert_tasks SET status='completed', " - "input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s", - (input_count, output_count, task_id), + "input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (input_count, output_count, current_user.get("id"), task_id), ) except Exception as exc: with store.connect() as conn: conn.execute( - "UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s", - (str(exc)[:500], task_id), + "UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s", + (str(exc)[:500], current_user.get("id"), task_id), ) raise fail(500, f"convert failed: {exc}") return ok(_get_task(task_id)) diff --git a/backend/app/modules/data_process/store/tasks.py b/backend/app/modules/data_process/store/tasks.py index c103614..7f858c4 100644 --- a/backend/app/modules/data_process/store/tasks.py +++ b/backend/app/modules/data_process/store/tasks.py @@ -84,10 +84,14 @@ class TasksMixin: rows = conn.execute( f""" SELECT task.*, + creator.display_name AS creator_name, + processor.display_name AS processor_name, (SELECT COUNT(*) FROM data_process_source_files source_file WHERE source_file.task_id=task.id AND source_file.deleted_at IS NULL) AS source_file_count FROM data_process_tasks task + LEFT JOIN users creator ON creator.id=task.created_by + LEFT JOIN users processor ON processor.id=task.updated_by WHERE {where} ORDER BY task.created_at DESC, task.id DESC LIMIT %s OFFSET %s @@ -410,6 +414,8 @@ class TasksMixin: row = conn.execute( """ SELECT task.*, + creator.display_name AS creator_name, + processor.display_name AS processor_name, (SELECT COUNT(*) FROM data_process_source_files source WHERE source.task_id=task.id AND source.deleted_at IS NULL) AS source_file_count, @@ -442,6 +448,8 @@ class TasksMixin: ELSE NULL END AS duration_seconds FROM data_process_tasks task + LEFT JOIN users creator ON creator.id=task.created_by + LEFT JOIN users processor ON processor.id=task.updated_by WHERE task.id=%s AND task.deleted_at IS NULL """, (task_id,), diff --git a/backend/app/modules/gpu/router.py b/backend/app/modules/gpu/router.py index 524ee13..83acf6e 100644 --- a/backend/app/modules/gpu/router.py +++ b/backend/app/modules/gpu/router.py @@ -1,25 +1,18 @@ """GPU 算力分配管理路由。""" from __future__ import annotations +import json from typing import Any from fastapi import APIRouter, Body, Depends, Request from app.api.v1.endpoints.platform import ok, fail -from app.core.auth import get_current_user, is_admin +from app.core.auth import get_current_user, is_admin, user_tenant_ids from app.db.platform_store import get_platform_store router = APIRouter(prefix="/compute", tags=["gpu-assignment"]) -def _actor_id(request: Request) -> str | None: - auth = request.headers.get("Authorization", "") - token = auth.replace("Bearer ", "").strip() - if token.startswith("platform-token-"): - return token[len("platform-token-"):] - return None - - @router.get("/gpu-assignments") def list_assignments(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: """查看全部分配关系（仅 admin）。""" @@ -40,8 +33,11 @@ def assign_gpus( assignments = payload.get("assignments") or [] if not assignments: raise fail(400, "assignments 不能为空") - actor = _actor_id(request) if request else None - result = get_platform_store().assign_gpus(assignments, assigned_by=actor) + actor = current_user.get("id") + try: + result = get_platform_store().assign_gpus(assignments, assigned_by=actor) + except ValueError as exc: + raise fail(409, str(exc)) get_platform_store().record_audit( action="gpu.assign", actor_id=actor, @@ -51,6 +47,41 @@ def assign_gpus( return ok(result) +@router.post("/gpu-assignments/request") +def request_gpu_assignment( + payload: dict[str, Any] = Body(...), + current_user: dict = Depends(get_current_user), +) -> dict[str, Any]: + assignments = payload.get("assignments") or [] + if not assignments: + raise fail(400, "assignments 不能为空") + if is_admin(current_user): + try: + return ok(get_platform_store().assign_gpus(assignments, assigned_by=current_user.get("id"))) + except ValueError as exc: + raise fail(409, str(exc)) + user_id = str(current_user.get("id") or "") + normalized = [] + for item in assignments: + if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None: + raise fail(400, "每项必须包含 node_id 和 gpu_index") + normalized.append({**item, "user_id": user_id}) + instance = get_platform_store().create_approval_instance({ + "resource_type": "gpu", + "resource_id": f"batch:{user_id}", + "applicant_id": user_id, + "action": "gpu.assign", + "tenant_id": current_user.get("tenant_id") or "default", + "reason": json.dumps({"assignments": normalized}, ensure_ascii=False), + }) + get_platform_store().record_audit( + action="gpu.assign.request", actor_id=user_id, target_type="gpu", + target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default", + detail=f"count={len(normalized)}", + ) + return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance}) + + @router.delete("/gpu-assignments/{assignment_id}") def unassign_gpu( assignment_id: str, @@ -61,7 +92,7 @@ def unassign_gpu( if not is_admin(current_user): raise fail(403, "admin permission required") get_platform_store().unassign_gpu(assignment_id) - actor = _actor_id(request) if request else None + actor = current_user.get("id") get_platform_store().record_audit( action="gpu.unassign", actor_id=actor, diff --git a/backend/app/modules/project/router.py b/backend/app/modules/project/router.py index 7e7de8a..e5b54d5 100644 --- a/backend/app/modules/project/router.py +++ b/backend/app/modules/project/router.py @@ -32,16 +32,24 @@ def _require_approval_or_admin( resource_id: str, current_user: dict[str, Any], action_desc: str = "", + action: str = "project.change", ) -> dict[str, Any] | None: """高风险操作审批旁路：admin 直接放行，普通用户创建审批实例（code=202）。""" if is_admin(current_user): return None + if not has_resource_access(resource_type, resource_id, current_user, "write"): + raise fail(403, "no permission to request this project change") store = get_platform_store() + if store.consume_approved_approval(resource_type, resource_id, str(current_user.get("id") or ""), action): + return None instance = store.create_approval_instance({ "resource_type": resource_type, "resource_id": resource_id, "applicant_id": current_user.get("id"), "template_id": None, + "action": action, + "tenant_id": current_user.get("tenant_id") or "default", + "reason": action_desc, }) return { "code": 202, @@ -68,12 +76,20 @@ def list_projects( @router.post("") -def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def create_project(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not is_admin(current_user) and str(payload.get("tenant_id") or current_user.get("tenant_id") or "default") != str(current_user.get("tenant_id") or "default"): + raise fail(403, "cannot create project in another tenant") + payload.setdefault("tenant_id", current_user.get("tenant_id") or "default") + payload.setdefault("create_by", current_user.get("id")) store = get_platform_store() + try: + store.assert_active_tenant(payload["tenant_id"]) + except ValueError as exc: + raise fail(400, str(exc)) proj = store.create_project(payload) store.record_audit( action="project.create", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project", target_id=proj["id"], tenant_id=proj.get("tenant_id"), @@ -109,7 +125,7 @@ def update_project( raise fail(404, "project not found") store.record_audit( action="project.update", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project", target_id=project_id, tenant_id=proj.get("tenant_id"), @@ -125,7 +141,7 @@ def archive_project( current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: _require_no_pending_approval("project", project_id) - pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}") + pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}", "project.archive") if pending: return pending store = get_platform_store() @@ -135,7 +151,7 @@ def archive_project( raise fail(404, "project not found") store.record_audit( action="project.archive", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project", target_id=project_id, tenant_id=proj.get("tenant_id"), @@ -150,14 +166,14 @@ def delete_project( current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: _require_no_pending_approval("project", project_id) - pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}") + pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}", "project.delete") if pending: return pending store = get_platform_store() store.delete_project(project_id) store.record_audit( action="project.delete", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project", target_id=project_id, ) @@ -190,7 +206,7 @@ def add_member( raise fail(404, "project not found") store.record_audit( action="project.member.add", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project.member", target_id=project_id, detail=f"user_id={payload.get('user_id')},role={payload.get('role')}", @@ -215,7 +231,7 @@ def update_member( raise fail(404, "project or member not found") store.record_audit( action="project.member.update", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="project.member", target_id=project_id, detail=f"user_id={user_id},role={payload.get('role')}", diff --git a/backend/app/modules/resource/router.py b/backend/app/modules/resource/router.py index 4bb6413..1d5bec4 100644 --- a/backend/app/modules/resource/router.py +++ b/backend/app/modules/resource/router.py @@ -5,16 +5,21 @@ from typing import Any from app.api.v1.endpoints.platform import ok, fail from app.db.platform_store import get_platform_store -from app.core.auth import get_current_user, has_resource_access, is_admin +from app.core.auth import ( + get_current_user, + has_resource_access, + is_admin, + resource_record, + resource_tenant_id, + user_tenant_ids, +) from app.core.audit import audit_log, AuditActions router = APIRouter(prefix="/resources", tags=["resource"]) -def _actor(request: Request) -> str | None: - auth = request.headers.get("Authorization", "") - token = auth.replace("Bearer ", "").strip() - return token or None +def _actor(request: Request, current_user: dict[str, Any]) -> str | None: + return str(current_user.get("id") or "") or None @router.get("/{resource_type}/{resource_id}/acl") @@ -39,19 +44,48 @@ def set_acl( current_user: dict = Depends(get_current_user), ) -> dict[str, Any]: """设置资源 ACL，body: { entries: [{ subject_type, subject_id, permissions: [] }] }""" + resource = resource_record(resource_type, resource_id) + if not resource and not is_admin(current_user): + raise fail(404, "resource not found") if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"): raise fail(403, "only resource owner or admin can update ACL") entries = payload.get("entries") or [] allowed = {"read", "write", "execute", "download", "delete", "admin"} + owner_allowed = {"read", "write", "execute", "download"} + tenant_id = resource_tenant_id(resource_type, resource) if resource else None + tenant_ids = user_tenant_ids(current_user) for entry in entries: if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"): raise fail(400, "invalid ACL principal") - if any(permission not in allowed for permission in entry.get("permissions") or []): + permissions = set(entry.get("permissions") or []) + if any(permission not in allowed for permission in permissions): raise fail(400, "invalid ACL permission") - result = get_platform_store().set_resource_acl(resource_type, resource_id, entries) + if not is_admin(current_user) and permissions - owner_allowed: + raise fail(403, "resource owners cannot grant delete or admin permission") + if entry.get("principal_type") == "user": + with get_platform_store().connect() as conn: + principal = conn.execute( + "SELECT id, tenant_id, status FROM users WHERE id=?", + (entry["principal_id"],), + ).fetchone() + if not principal or principal.get("status") != "active": + raise fail(400, "ACL user does not exist or is inactive") + principal_tenant = str(principal.get("tenant_id") or "default") + if not is_admin(current_user) and tenant_id and principal_tenant not in tenant_ids: + raise fail(403, "cannot grant resource access across tenants") + elif not is_admin(current_user): + # Role ACLs are global in the legacy schema and therefore cannot + # be safely scoped to one tenant by a normal resource owner. + raise fail(403, "only administrators can grant role-based ACLs") + result = get_platform_store().set_resource_acl( + resource_type, + resource_id, + entries, + granted_by=str(current_user.get("id") or "") or None, + ) get_platform_store().record_audit( action="resource.acl.set", - actor_id=_actor(request) if request else None, + actor_id=_actor(request, current_user) if request else current_user.get("id"), target_type=resource_type, target_id=resource_id, detail=f"entries={len(entries)}", diff --git a/backend/app/modules/retention/router.py b/backend/app/modules/retention/router.py index 0031743..6c398b1 100644 --- a/backend/app/modules/retention/router.py +++ b/backend/app/modules/retention/router.py @@ -1,10 +1,11 @@ from __future__ import annotations -from fastapi import APIRouter, Body, Request +from fastapi import APIRouter, Body, Request, Depends from typing import Any from app.api.v1.endpoints.platform import ok, fail from app.db.platform_store import get_platform_store +from app.core.auth import require_admin router = APIRouter(prefix="/retention-policies", tags=["retention"]) @@ -16,18 +17,18 @@ def _actor(request: Request) -> str | None: @router.get("") -def list_policies() -> dict[str, Any]: +def list_policies(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().retention_policies()) @router.post("") -def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def create_policy(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: if not payload.get("name"): raise fail(400, "name 必填") policy = get_platform_store().create_retention_policy(payload) get_platform_store().record_audit( action="retention.create", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="retention_policy", target_id=policy["id"], detail=f"name={policy.get('name')}", @@ -36,7 +37,7 @@ def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) @router.get("/{policy_id}") -def get_policy(policy_id: str) -> dict[str, Any]: +def get_policy(policy_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().retention_policy(policy_id)) except KeyError: @@ -45,7 +46,8 @@ def get_policy(policy_id: str) -> dict[str, Any]: @router.put("/{policy_id}") def update_policy( - policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None + policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None, + current_user: dict = Depends(require_admin), ) -> dict[str, Any]: store = get_platform_store() try: @@ -54,7 +56,7 @@ def update_policy( raise fail(404, "retention policy not found") store.record_audit( action="retention.update", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="retention_policy", target_id=policy_id, detail=f"fields={','.join(payload.keys())}", @@ -63,12 +65,12 @@ def update_policy( @router.delete("/{policy_id}") -def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]: +def delete_policy(policy_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() store.delete_retention_policy(policy_id) store.record_audit( action="retention.delete", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="retention_policy", target_id=policy_id, ) diff --git a/backend/app/modules/storage/minio_store.py b/backend/app/modules/storage/minio_store.py index 25de9ba..3fc089b 100644 --- a/backend/app/modules/storage/minio_store.py +++ b/backend/app/modules/storage/minio_store.py @@ -8,6 +8,7 @@ from typing import Any from minio import Minio from minio.error import S3Error +import urllib3 from app.core.config import get_settings @@ -20,7 +21,20 @@ class MinioObjectStorage: def __init__(self) -> None: settings = get_settings() endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/") - self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure) + # MinIO outages must fail fast; higher-level workflows own the retry + # policy and should not wait through urllib3's default retry chain. + http_client = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED" if settings.minio_secure else "CERT_NONE", + timeout=urllib3.Timeout(connect=2.0, read=10.0), + retries=False, + ) + self.client = Minio( + endpoint, + access_key=settings.minio_access_key, + secret_key=settings.minio_secret_key, + secure=settings.minio_secure, + http_client=http_client, + ) self.bucket = settings.minio_bucket def _ensure_enabled(self) -> None: @@ -32,7 +46,7 @@ class MinioObjectStorage: try: if not self.client.bucket_exists(self.bucket): self.client.make_bucket(self.bucket) - except S3Error as exc: + except Exception as exc: # noqa: BLE001 - normalize network/client failures raise ObjectStorageError(str(exc)) from exc def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str: diff --git a/backend/app/modules/system/router.py b/backend/app/modules/system/router.py index d6d57f4..df9c4d8 100644 --- a/backend/app/modules/system/router.py +++ b/backend/app/modules/system/router.py @@ -1,33 +1,56 @@ from __future__ import annotations -from fastapi import APIRouter, Body, Query, Request, Depends +import csv +import io + +from fastapi import APIRouter, Body, HTTPException, Query, Request, Depends from fastapi.responses import StreamingResponse from app.db.platform_store import ALL_PERMISSIONS, get_platform_store from app.core.auth import get_current_user, is_admin +from app.core.logging import get_client_ip router = APIRouter(prefix="/system", tags=["system"]) +_VISIT_MODULES = { + "dashboard", + "fine-tune", + "model-eval", + "model-inference", + "model-manage", + "dataset", + "data-process", + "data-convert", +} + @router.post("/audit/visit") -def record_visit(payload: dict = Body(...), request: Request = None) -> dict: +def record_visit( + payload: dict = Body(...), + request: Request = None, + current_user: dict = Depends(get_current_user), +) -> dict: """记录用户访问业务模块的行为，用于看板用户操作分布统计。""" action = str(payload.get("action") or payload.get("module") or "").strip() if not action: return {"code": 0, "message": "ok", "data": {"recorded": False}} - actor_id = "" - if request is not None: - auth = request.headers.get("Authorization", "") - token = auth.replace("Bearer ", "").strip() - if token.startswith("platform-token-"): - actor_id = token[len("platform-token-"):] + # Visit statistics are intentionally limited to known module names. This + # endpoint must not become a free-form audit-log injection point. + if action not in _VISIT_MODULES: + return {"code": 0, "message": "ok", "data": {"recorded": False}} + actor_id = current_user.get("id") get_platform_store().record_audit( action=action, actor_id=actor_id or None, target_type="module", target_id=action, - detail=str(payload.get("detail") or ""), + tenant_id=str(current_user.get("tenant_id") or "") or None, + session_id=str(current_user.get("session_id") or "") or None, + request_id=(request.headers.get("X-Request-ID") if request else None), + detail="module visit", + metadata={"source": "frontend", "detail_length": len(str(payload.get("detail") or ""))}, + ip=get_client_ip(request) or None, ) return {"code": 0, "message": "ok", "data": {"recorded": True}} @@ -117,13 +140,22 @@ def audit_logs_export( offset=0, ) items = result["items"] - columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"] - header = ",".join(columns) + "\n" + columns = [ + "time", "tenant_id", "project_id", "actor_id", "action", "target_type", + "target_id", "detail", "client_ip", "result", "reason", "request_id", + "session_id", "metadata", + ] def iter_rows(): - yield header + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow(columns) + yield buffer.getvalue() for row in items: - yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n" + buffer.seek(0) + buffer.truncate(0) + writer.writerow([row.get(c, "") or "" for c in columns]) + yield buffer.getvalue() return StreamingResponse( iter_rows(), @@ -134,6 +166,16 @@ def audit_logs_export( # ===================== 操作日志 ===================== +def _operation_log_scope(current_user: dict, conditions: list[str], params: list) -> None: + """校验操作日志权限，并为普通用户追加本人范围。""" + if is_admin(current_user): + return + if "logs" not in (current_user.get("permissions") or []): + raise HTTPException(status_code=403, detail="missing permission: logs") + conditions.append("user_id = %s") + params.append(str(current_user.get("id") or "")) + + @router.get("/operation-logs") def operation_logs( user_id: str | None = Query(default=None, description="按用户 ID 筛选"), @@ -147,14 +189,12 @@ def operation_logs( offset: int = Query(default=0, ge=0), current_user: dict = Depends(get_current_user), ) -> dict: - """操作日志查询：按用户/模块/动作/状态/关键字/时间范围分页过滤。""" - if not is_admin(current_user): - from app.api.v1.endpoints.platform import fail - raise fail(403, "admin permission required") + """操作日志查询：管理员查全量，普通用户只能查本人记录。""" store = get_platform_store() conditions = [] params: list = [] - if user_id: + _operation_log_scope(current_user, conditions, params) + if user_id and is_admin(current_user): conditions.append("user_id = %s") params.append(user_id) if module: @@ -192,13 +232,11 @@ def operation_logs_stats( end_time: str | None = Query(default=None, description="ISO8601 结束时间"), current_user: dict = Depends(get_current_user), ) -> dict: - """操作日志统计：总操作数、成功数、失败数、失败率、各模块失败分布、最近错误列表。""" - if not is_admin(current_user): - from app.api.v1.endpoints.platform import fail - raise fail(403, "admin permission required") + """操作日志统计：管理员统计全量，普通用户统计本人记录。""" store = get_platform_store() conditions = [] params: list = [] + _operation_log_scope(current_user, conditions, params) if start_time: conditions.append("create_time >= %s") params.append(start_time) @@ -260,13 +298,17 @@ def operation_logs_stats( @router.get("/operation-logs/modules") def operation_log_modules(current_user: dict = Depends(get_current_user)) -> dict: """返回操作日志中出现的模块列表（用于筛选下拉框）。""" - if not is_admin(current_user): - from app.api.v1.endpoints.platform import fail - raise fail(403, "admin permission required") store = get_platform_store() + conditions: list[str] = [] + params: list = [] + _operation_log_scope(current_user, conditions, params) + where = " WHERE " + " AND ".join(conditions) if conditions else "" with store.connect() as conn: rows = conn.execute( - "SELECT DISTINCT module FROM operation_logs WHERE module IS NOT NULL ORDER BY module" + f"SELECT DISTINCT module FROM operation_logs{where}" + + (" AND" if where else " WHERE") + + " module IS NOT NULL ORDER BY module", + tuple(params), ).fetchall() modules = [{"value": r["module"], "label": r["module"]} for r in rows] return {"code": 0, "message": "ok", "data": modules} diff --git a/backend/app/modules/tenant/router.py b/backend/app/modules/tenant/router.py index da60cac..5098338 100644 --- a/backend/app/modules/tenant/router.py +++ b/backend/app/modules/tenant/router.py @@ -1,10 +1,12 @@ from __future__ import annotations -from fastapi import APIRouter, Body, Request +import json +from fastapi import APIRouter, Body, Depends, Request from typing import Any from app.api.v1.endpoints.platform import ok, fail from app.db.platform_store import get_platform_store +from app.core.auth import is_admin, require_admin, require_tenant_admin, get_current_user, user_tenant_ids router = APIRouter(prefix="/tenants", tags=["tenant"]) @@ -16,20 +18,33 @@ def _actor(request: Request) -> str | None: @router.get("") -def list_tenants() -> dict[str, Any]: +def list_tenants(current_user: dict = Depends(require_admin)) -> dict[str, Any]: return ok(get_platform_store().tenants()) +@router.get("/invitations") +def my_invitations(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + return ok(get_platform_store().tenant_invitations(str(current_user.get("id") or ""))) + + +@router.get("/mine") +def my_tenants(current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + return ok(get_platform_store().user_tenants(str(current_user.get("id") or ""), include_all=is_admin(current_user))) + + @router.post("") -def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() + payload = {**payload, "owner_user_id": payload.get("owner_user_id") or current_user.get("id")} try: tenant = store.create_tenant(payload) except KeyError as e: raise fail(400, f"missing field: {e}") + except ValueError as exc: + raise fail(400, str(exc)) store.record_audit( action="tenant.create", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="tenant", target_id=tenant["id"], tenant_id=tenant["id"], @@ -39,7 +54,7 @@ def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) @router.get("/{tenant_id}") -def get_tenant(tenant_id: str) -> dict[str, Any]: +def get_tenant(tenant_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: try: return ok(get_platform_store().tenant(tenant_id)) except KeyError: @@ -47,7 +62,7 @@ def get_tenant(tenant_id: str) -> dict[str, Any]: @router.put("/{tenant_id}") -def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() try: tenant = store.update_tenant(tenant_id, payload) @@ -55,7 +70,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: raise fail(404, "tenant not found") store.record_audit( action="tenant.update", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="tenant", target_id=tenant_id, tenant_id=tenant_id, @@ -65,7 +80,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: @router.put("/{tenant_id}/quota") -def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() try: tenant = store.set_tenant_quota(tenant_id, payload) @@ -73,7 +88,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ raise fail(404, "tenant not found") store.record_audit( action="tenant.quota.set", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="tenant", target_id=tenant_id, tenant_id=tenant_id, @@ -82,7 +97,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ @router.put("/{tenant_id}/retention-policy") -def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: +def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() try: tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id")) @@ -90,7 +105,7 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: raise fail(404, "tenant not found") store.record_audit( action="tenant.retention.set", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="tenant", target_id=tenant_id, tenant_id=tenant_id, @@ -99,18 +114,213 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: @router.delete("/{tenant_id}") -def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]: +def delete_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: store = get_platform_store() try: - tenant = store.delete_tenant(tenant_id) + tenant = store.delete_tenant(tenant_id, str(current_user.get("id") or "system")) except KeyError: raise fail(404, "tenant not found") + except ValueError as exc: + raise fail(400, str(exc)) store.record_audit( action="tenant.delete", - actor_id=_actor(request) if request else None, + actor_id=current_user.get("id"), target_type="tenant", target_id=tenant_id, tenant_id=tenant_id, detail=f"name={tenant.get('name')}", ) return ok(tenant) + + +@router.post("/{tenant_id}/restore") +def restore_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]: + store = get_platform_store() + try: + tenant = store.restore_tenant(tenant_id, str(current_user.get("id") or "system")) + except KeyError: + raise fail(404, "tenant not found") + except ValueError as exc: + raise fail(400, str(exc)) + store.record_audit( + action="tenant.restore", + actor_id=current_user.get("id"), + target_type="tenant", + target_id=tenant_id, + tenant_id=tenant_id, + detail=f"name={tenant.get('name')}", + ) + return ok(tenant) + + +@router.get("/{tenant_id}/quota/usage") +def quota_usage(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]: + return ok(get_platform_store().tenant_quota_usage(tenant_id)) + + +@router.post("/{tenant_id}/quota/request") +def request_quota_change(tenant_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: + if not is_admin(current_user) and tenant_id not in user_tenant_ids(current_user): + raise fail(403, "tenant access denied") + try: + get_platform_store().assert_active_tenant(tenant_id) + except ValueError as exc: + raise fail(400, str(exc)) + quota = payload.get("quota") + if not isinstance(quota, dict): + raise fail(400, "quota must be an object") + instance = get_platform_store().create_approval_instance({ + "resource_type": "tenant", + "resource_id": tenant_id, + "applicant_id": current_user.get("id"), + "action": "tenant.quota.update", + "tenant_id": tenant_id, + "reason": json.dumps({"quota": quota}, ensure_ascii=False), + }) + get_platform_store().record_audit( + action="tenant.quota.request", actor_id=current_user.get("id"), + target_type="tenant", target_id=tenant_id, tenant_id=tenant_id, + ) + return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance}) + + +@router.get("/{tenant_id}/members") +def list_members(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]: + try: + return ok(get_platform_store().tenant_members(tenant_id)) + except KeyError: + raise fail(404, "tenant not found") + + +@router.post("/{tenant_id}/members") +def add_member( + tenant_id: str, + payload: dict[str, Any] = Body(...), + request: Request = None, + current_user: dict = Depends(require_tenant_admin), +) -> dict[str, Any]: + if not payload.get("user_id"): + raise fail(400, "user_id 必填") + if not is_admin(current_user) and payload.get("role") == "owner": + raise fail(403, "only platform administrator can grant owner role") + try: + member = get_platform_store().add_tenant_member( + tenant_id, + str(payload["user_id"]), + str(payload.get("role") or "member"), + current_user.get("id"), + ) + except KeyError: + raise fail(404, "tenant or user not found") + except ValueError as exc: + raise fail(400, str(exc)) + get_platform_store().record_audit( + action="tenant.member.add", + actor_id=current_user.get("id"), + target_type="tenant_member", + target_id=f"{tenant_id}:{payload['user_id']}", + tenant_id=tenant_id, + ) + return ok(member) + + +@router.post("/{tenant_id}/members/invite") +def invite_member( + tenant_id: str, + payload: dict[str, Any] = Body(...), + current_user: dict = Depends(require_tenant_admin), +) -> dict[str, Any]: + user_id = str(payload.get("user_id") or "") + if not user_id: + raise fail(400, "user_id 必填") + if payload.get("role") == "owner": + raise fail(403, "tenant invitations cannot grant owner role") + try: + member = get_platform_store().invite_tenant_member( + tenant_id, + user_id, + str(payload.get("role") or "member"), + current_user.get("id"), + payload.get("expires_at"), + ) + except KeyError: + raise fail(404, "tenant or active user not found") + except ValueError as exc: + raise fail(400, str(exc)) + get_platform_store().record_audit( + action="tenant.member.invite", + actor_id=current_user.get("id"), + target_type="tenant_member", + target_id=f"{tenant_id}:{user_id}", + tenant_id=tenant_id, + detail=f"role={member.get('role')};expires_at={member.get('expires_at')}", + ) + return ok(member) + + +@router.put("/{tenant_id}/members/{user_id}") +def update_member( + tenant_id: str, + user_id: str, + payload: dict[str, Any] = Body(...), + current_user: dict = Depends(require_tenant_admin), +) -> dict[str, Any]: + try: + if not is_admin(current_user) and payload.get("role") == "owner": + raise fail(403, "only platform administrator can grant owner role") + member = get_platform_store().update_tenant_member(tenant_id, user_id, payload) + get_platform_store().record_audit( + action="tenant.member.update", + actor_id=current_user.get("id"), + target_type="tenant_member", + target_id=f"{tenant_id}:{user_id}", + tenant_id=tenant_id, + detail=f"fields={','.join(payload.keys())}", + ) + return ok(member) + except KeyError: + raise fail(404, "tenant member not found") + except ValueError as exc: + raise fail(400, str(exc)) + + +@router.delete("/{tenant_id}/members/{user_id}") +def remove_member(tenant_id: str, user_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]: + try: + get_platform_store().remove_tenant_member(tenant_id, user_id) + except KeyError: + raise fail(404, "tenant member not found") + except ValueError as exc: + raise fail(400, str(exc)) + get_platform_store().record_audit( + action="tenant.member.remove", + actor_id=current_user.get("id"), + target_type="tenant_member", + target_id=f"{tenant_id}:{user_id}", + tenant_id=tenant_id, + ) + return ok({"tenant_id": tenant_id, "user_id": user_id, "removed": True}) + + +@router.post("/{tenant_id}/members/{user_id}/accept") +def accept_invitation( + tenant_id: str, + user_id: str, + current_user: dict = Depends(get_current_user), +) -> dict[str, Any]: + if not is_admin(current_user) and str(current_user.get("id") or "") != user_id: + raise fail(403, "only the invited user can accept this invitation") + try: + member = get_platform_store().accept_tenant_invitation(tenant_id, user_id) + except KeyError: + raise fail(404, "tenant invitation not found") + except ValueError as exc: + raise fail(409, str(exc)) + get_platform_store().record_audit( + action="tenant.member.accept", + actor_id=current_user.get("id"), + target_type="tenant_member", + target_id=f"{tenant_id}:{user_id}", + tenant_id=tenant_id, + ) + return ok(member) diff --git a/backend/app/workers/compute_poller.py b/backend/app/workers/compute_poller.py index dd2ac7b..6b68f8a 100644 --- a/backend/app/workers/compute_poller.py +++ b/backend/app/workers/compute_poller.py @@ -1,9 +1,11 @@ from __future__ import annotations import asyncio +import time from app.core.config import get_settings from app.core.logging import get_logger +from app.db.platform_store import get_platform_store from app.modules.compute_gateway.sync import poll_compute_jobs_once @@ -18,11 +20,33 @@ async def run_compute_poller() -> None: interval = max(3, settings.compute_poll_interval_seconds) logger.info("compute poller started", extra={"interval_seconds": interval}) + # PlatformStore may run additive schema checks against a remote PostgreSQL + # server on first use. Keep that startup work off the Uvicorn event loop so + # health checks and normal API requests can still respond while the DB is + # unavailable or slow. + store = None + last_failure_signature = "" + last_failure_logged_at = 0.0 + await asyncio.sleep(1) while True: try: - result = await poll_compute_jobs_once() + if store is None: + store = await asyncio.to_thread(get_platform_store) + result = await poll_compute_jobs_once(store) if result["failed"]: - logger.warning("compute polling reported failures", extra={"result": result}) + signature = "|".join(sorted({ + str(item.get("error") or "")[:120] + for item in result["failed"] + })) + now = time.monotonic() + if signature != last_failure_signature or now - last_failure_logged_at >= 300: + logger.warning( + "compute polling reported failures count=%d first_error=%s", + len(result["failed"]), + signature[:500], + ) + last_failure_signature = signature + last_failure_logged_at = now elif result["synced"]: logger.debug("compute jobs synchronized", extra={"result": result}) except asyncio.CancelledError: @@ -30,4 +54,6 @@ async def run_compute_poller() -> None: raise except Exception as exc: # noqa: BLE001 - keep background polling alive logger.exception("compute poller failed", extra={"error": str(exc)}) + if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)): + store = None await asyncio.sleep(interval) diff --git a/backend/requirements.txt b/backend/requirements.txt index cfb38b7..66657c1 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -9,6 +9,7 @@ alembic>=1.13.1 redis>=5.0.4 httpx>=0.27.0 minio>=7.2.7 +urllib3>=2.0.7 PyJWT>=2.8.0 passlib[bcrypt]>=1.7.4 python-dotenv>=1.0.1 diff --git a/backend/tests/test_permission_security.py b/backend/tests/test_permission_security.py new file mode 100644 index 0000000..52aa7eb --- /dev/null +++ b/backend/tests/test_permission_security.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from app.api.v1.endpoints.platform import _public_model +from app.core.audit import _safe_exception_reason +from app.core.auth import RESOURCE_ACTIONS, RESOURCE_ACTION_ALIASES + + +def test_public_model_never_exposes_provider_credentials() -> None: + result = _public_model({ + "id": "m_1", + "name": "online", + "api_url": "https://example.invalid/v1", + "api_key": "secret-value", + }) + + assert "api_key" not in result + assert result["api_key_configured"] is True + assert result["name"] == "online" + + +def test_resource_action_registry_keeps_export_separate_from_module_permissions() -> None: + assert "download" in RESOURCE_ACTIONS + assert "execute" in RESOURCE_ACTIONS + assert RESOURCE_ACTION_ALIASES["export"] == "download" + + +def test_audit_exception_reason_masks_credentials() -> None: + reason = _safe_exception_reason(ValueError("api_key=secret-value")) + assert "secret-value" not in reason + assert "***" in reason diff --git a/backend/tests/test_storage_security.py b/backend/tests/test_storage_security.py new file mode 100644 index 0000000..fda2986 --- /dev/null +++ b/backend/tests/test_storage_security.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from app.api.v1.endpoints.platform import _validate_storage_key + + +def test_storage_key_is_scoped_to_resource_version() -> None: + assert ( + _validate_storage_key("dataset", "ds_123", "v1", None, "train.jsonl") + == "datasets/ds_123/versions/v1/train.jsonl" + ) + + +@pytest.mark.parametrize( + "object_key", + [ + "models/other/versions/v1/model.bin", + "datasets/ds_123/versions/v1/../secret.bin", + "datasets/ds_123/versions/v1/../../secret.bin", + "/datasets/ds_123/versions/v1/model.bin", + ], +) +def test_storage_key_rejects_escape_or_cross_resource_paths(object_key: str) -> None: + with pytest.raises(ValueError): + _validate_storage_key("dataset", "ds_123", "v1", object_key, None) diff --git a/compute/api/main.py b/compute/api/main.py index 2f71703..2bbd91d 100644 --- a/compute/api/main.py +++ b/compute/api/main.py @@ -9,6 +9,7 @@ import shutil import subprocess import time from pathlib import Path +from datetime import datetime from typing import Any import httpx @@ -70,6 +71,76 @@ def create_app() -> FastAPI: except ValueError: return False + def cache_max_bytes() -> int: + return max(0, _int_env("COMPUTE_CACHE_MAX_BYTES", 0)) + + def cache_ttl_seconds() -> int: + return max(0, _int_env("COMPUTE_CACHE_TTL_SECONDS", 0)) + + def cache_meta_path(target: Path) -> Path: + return target.with_name(f".{target.name}.cache-meta.json") + + def cache_protected_until(target: Path) -> float: + try: + value = json.loads(cache_meta_path(target).read_text(encoding="utf-8")).get("protected_until") + return float(value or 0) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return 0.0 + + def write_cache_meta(target: Path, resource_id: str, version_id: str, protected_until: float) -> None: + meta = cache_meta_path(target) + meta.write_text(json.dumps({ + "resource_id": resource_id, + "version_id": version_id, + "protected_until": protected_until, + "last_accessed_at": now(), + }), encoding="utf-8") + + def cache_usage(cache_root: Path) -> int: + total = 0 + if not cache_root.exists(): + return 0 + for item in cache_root.rglob("*"): + try: + if item.is_file() and not item.name.endswith(".part"): + total += item.stat().st_size + except OSError: + continue + return total + + def ensure_cache_capacity(cache_root: Path, required_bytes: int, protected: Path) -> None: + limit = cache_max_bytes() + if not limit or required_bytes <= 0: + return + usage = cache_usage(cache_root) + if usage + required_bytes <= limit: + return + candidates: list[tuple[float, int, Path]] = [] + for item in cache_root.rglob("*"): + try: + if ( + item.is_file() + and not item.name.endswith(".part") + and not item.name.endswith(".cache-meta.json") + and item.resolve() != protected.resolve() + and cache_protected_until(item) <= now() + ): + stat = item.stat() + candidates.append((stat.st_atime, stat.st_size, item)) + except OSError: + continue + candidates.sort(key=lambda value: value[0]) + for _, size, item in candidates: + try: + item.unlink(missing_ok=True) + usage -= size + except OSError: + continue + if usage + required_bytes <= limit: + break + if usage + required_bytes > limit: + raise HTTPException(status_code=507, detail="compute cache capacity is insufficient") + def _llama_factory_version() -> str: for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]): try: @@ -662,13 +733,26 @@ def create_app() -> FastAPI: raise HTTPException(status_code=400, detail="upload_url is required") digest = hashlib.sha256() byte_size = 0 + content_length = source.stat().st_size + + async def file_chunks(): + nonlocal byte_size + with source.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + byte_size += len(chunk) + yield chunk + try: async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client: - with source.open("rb") as handle: - content = handle.read() - digest.update(content) - byte_size = len(content) - response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")}) + response = await client.put( + upload_url, + content=file_chunks(), + headers={ + "Content-Type": str(payload.get("content_type") or "application/octet-stream"), + "Content-Length": str(content_length), + }, + ) response.raise_for_status() except Exception as exc: raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc @@ -916,6 +1000,14 @@ def create_app() -> FastAPI: temp_target = target.with_name(f".{target.name}.part") expected_checksum = str(payload.get("checksum_sha256") or "").lower() expected_size = int(payload.get("byte_size") or 0) + requested_protected_until = str(payload.get("protected_until") or "") + try: + protected_until = float(requested_protected_until) + except ValueError: + try: + protected_until = datetime.fromisoformat(requested_protected_until.replace("Z", "+00:00")).timestamp() + except (ValueError, TypeError): + protected_until = now() + cache_ttl_seconds() if cache_ttl_seconds() else 0.0 lock = cache_locks.setdefault(str(target), asyncio.Lock()) async with lock: if target.is_file() and expected_checksum: @@ -924,7 +1016,10 @@ def create_app() -> FastAPI: while chunk := existing.read(1024 * 1024): existing_digest.update(chunk) if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size): + os.utime(target, None) + write_cache_meta(target, resource_id, version_id, protected_until) return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True} + ensure_cache_capacity(cache_root / "resources", expected_size, target) digest = hashlib.sha256() byte_size = 0 try: @@ -956,6 +1051,7 @@ def create_app() -> FastAPI: temp_target.unlink(missing_ok=True) raise HTTPException(status_code=502, detail="cache byte size mismatch") temp_target.replace(target) + write_cache_meta(target, resource_id, version_id, protected_until) except HTTPException: raise except Exception as exc: @@ -975,12 +1071,20 @@ def create_app() -> FastAPI: data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root))) target = cache_root / "resources" / resource_id / version_id / "resource" + ttl = cache_ttl_seconds() + if target.is_file() and ttl and target.stat().st_atime + ttl < now() and cache_protected_until(target) <= now(): + target.unlink(missing_ok=True) + cache_meta_path(target).unlink(missing_ok=True) return { "resource_id": resource_id, "version_id": version_id, "status": "ready" if target.is_file() else "missing", "local_path": str(target), "byte_size": target.stat().st_size if target.is_file() else 0, + "cache_usage_bytes": cache_usage(cache_root / "resources"), + "cache_max_bytes": cache_max_bytes(), + "cache_ttl_seconds": ttl, + "protected_until": cache_protected_until(target) if target.is_file() else 0, } @app.delete(f"{route_prefix}/compute/cache") diff --git a/compute/engines/llama_factory/adapter.py b/compute/engines/llama_factory/adapter.py index 324691a..c568cd3 100644 --- a/compute/engines/llama_factory/adapter.py +++ b/compute/engines/llama_factory/adapter.py @@ -313,7 +313,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA- "--save_steps", str(config.get("save_steps", 50)), "--logging_steps", - str(config.get("logging_steps", 10)), + str(max(1, int(config.get("logging_steps", 1) or 1))), "--overwrite_output_dir", "true", "--plot_loss", diff --git a/compute/engines/llama_factory/eval_runner.py b/compute/engines/llama_factory/eval_runner.py index 332e17a..1923190 100644 --- a/compute/engines/llama_factory/eval_runner.py +++ b/compute/engines/llama_factory/eval_runner.py @@ -16,6 +16,7 @@ import math import re import sys import time +from datetime import datetime, timezone from difflib import SequenceMatcher from pathlib import Path from typing import Any @@ -94,15 +95,13 @@ def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4) try: from sacrebleu.metrics import BLEU except ImportError: - return {"enabled": False, "error": "sacrebleu not installed", "score": 0} + return _metric_record(None, len(predictions), "sacrebleu 未安装", available=False) + if not predictions: + return _metric_record(0, 0) bleu = BLEU(max_ngram_order=ngram) # sacrebleu expects list-of-strings; we have one reference per prediction score = bleu.corpus_score(predictions, [references]) - return { - "enabled": True, - "score": round(score.score, 2), - "bleu": round(score.score, 2), - } + return _metric_record(score.score, len(predictions), bleu=round(score.score, 2)) def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]: @@ -110,20 +109,27 @@ def _compute_rouge(references: list[str], predictions: list[str], methods: list[ try: from rouge_score import rouge_scorer except ImportError: - return {"enabled": False, "error": "rouge-score not installed", "score": 0} + rouge_scorer = None methods = methods or ["rouge1", "rouge2", "rougeL"] # Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL" _rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"} methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods] - scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True) - totals: dict[str, float] = {} - n = max(len(predictions), 1) - for ref, pred in zip(references, predictions): - result = scorer.score(ref, pred) - for key in methods: - totals[key] = totals.get(key, 0) + result[key].fmeasure - avg = {k: round(v / n, 4) for k, v in totals.items()} - return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg} + if rouge_scorer is None: + values = [_pair_rouge(ref, pred) for ref, pred in zip(references, predictions)] + avg = {key: round(sum(item.get(key, 0.0) for item in values) / max(len(values), 1), 4) for key in methods} + else: + class _Tokenizer: + def tokenize(self, value: str) -> list[str]: + return value.split() + scorer = rouge_scorer.RougeScorer(methods, use_stemmer=False, tokenizer=_Tokenizer()) + totals: dict[str, float] = {} + n = max(len(predictions), 1) + for ref, pred in zip(references, predictions): + result = scorer.score(_rouge_tokens(ref), _rouge_tokens(pred)) + for key in methods: + totals[key] = totals.get(key, 0) + result[key].fmeasure + avg = {key: round(value / n, 4) for key, value in totals.items()} + return _metric_record(avg.get("rougeL", avg.get("rouge1", 0)) * 100, len(predictions), **avg) def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]: @@ -132,7 +138,9 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity except ImportError: - return {"enabled": False, "error": "scikit-learn not installed", "score": 0} + return _metric_record(None, len(predictions), "scikit-learn 未安装", available=False) + if not predictions: + return _metric_record(0, 0) try: vectorizer = TfidfVectorizer() tfidf = vectorizer.fit_transform(references + predictions) @@ -140,41 +148,145 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, ref_vec = tfidf[:n] pred_vec = tfidf[n:] sims = cosine_similarity(ref_vec, pred_vec).diagonal() - return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)} - except ValueError: - return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"} + return _metric_record(float(sims.mean()) * 100, n) + except ValueError as exc: + return _metric_record(0, len(predictions), str(exc)) def _normalize_text(value: str) -> str: return re.sub(r"\s+", " ", str(value or "").strip().lower()) +def _metric_record(score: float | None, sample_count: int, error: str = "", available: bool = True, **extra: Any) -> dict[str, Any]: + return { + "enabled": True, + "available": available, + "score": None if score is None else round(max(0.0, min(100.0, float(score))), 2), + "max_score": 100, + "unit": "percent", + "sample_count": sample_count, + "error": error, + **extra, + } + + +def _rouge_tokens(text: str) -> str: + text = str(text or "").strip().lower() + tokens: list[str] = [] + for segment in re.findall(r"[一-鿿]+|[^\s一-鿿]+", text): + tokens.extend(segment if "一" <= segment[0] <= "鿿" else [segment]) + return " ".join(tokens) + + +def _pair_rouge(reference: str, prediction: str) -> dict[str, float]: + try: + from rouge_score import rouge_scorer + except ImportError: + reference_tokens = _rouge_tokens(reference).split() + prediction_tokens = _rouge_tokens(prediction).split() + if not reference_tokens or not prediction_tokens: + return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0} + + def f1(overlap: int, reference_count: int, prediction_count: int) -> float: + if not reference_count or not prediction_count or not overlap: + return 0.0 + precision = overlap / prediction_count + recall = overlap / reference_count + return 2 * precision * recall / (precision + recall) + + from collections import Counter + reference_unigrams = Counter(reference_tokens) + prediction_unigrams = Counter(prediction_tokens) + unigram_overlap = sum((reference_unigrams & prediction_unigrams).values()) + reference_bigrams = Counter(zip(reference_tokens, reference_tokens[1:])) + prediction_bigrams = Counter(zip(prediction_tokens, prediction_tokens[1:])) + bigram_overlap = sum((reference_bigrams & prediction_bigrams).values()) + matrix = [[0] * (len(prediction_tokens) + 1) for _ in range(len(reference_tokens) + 1)] + for row, reference_token in enumerate(reference_tokens, start=1): + for column, prediction_token in enumerate(prediction_tokens, start=1): + matrix[row][column] = matrix[row - 1][column - 1] + 1 if reference_token == prediction_token else max(matrix[row - 1][column], matrix[row][column - 1]) + return { + "rouge1": f1(unigram_overlap, len(reference_tokens), len(prediction_tokens)), + "rouge2": f1(bigram_overlap, max(len(reference_tokens) - 1, 0), max(len(prediction_tokens) - 1, 0)), + "rougeL": f1(matrix[-1][-1], len(reference_tokens), len(prediction_tokens)), + } + class _Tokenizer: + def tokenize(self, value: str) -> list[str]: + return value.split() + if not reference.strip() or not prediction.strip(): + return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0} + scorer = rouge_scorer.RougeScorer(("rouge1", "rouge2", "rougeL"), use_stemmer=False, tokenizer=_Tokenizer()) + scores = scorer.score(_rouge_tokens(reference), _rouge_tokens(prediction)) + return {key: float(value.fmeasure) for key, value in scores.items()} + + def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]: total = len(predictions) if not total: - return {"enabled": True, "score": 0, "matched": 0, "total": 0} + return _metric_record(0, 0, matched=0, total=0) matched = sum( 1 for ref, pred in zip(references, predictions) if _normalize_text(ref) == _normalize_text(pred) ) - return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total} + return _metric_record(matched / total * 100, total, matched=matched, total=total) def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]: if not predictions: - return {"enabled": True, "score": 0} + return _metric_record(0, 0) scores = [ SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio() for ref, pred in zip(references, predictions) ] - return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)} + return _metric_record(sum(scores) / max(len(scores), 1) * 100, len(predictions)) # --------------------------------------------------------------------------- # LLM Judge # --------------------------------------------------------------------------- +def _normalise_api_url(value: str) -> str: + url = str(value or "").strip().rstrip("/") + return url + "/chat/completions" if url.endswith("/v1") else url + "/v1/chat/completions" + + +def _parse_judge_reply(reply: str, score_min: float, score_max: float) -> tuple[float | None, dict[str, Any], str]: + payload: dict[str, Any] = {} + match = re.search(r"\{[\s\S]*\}", str(reply or "")) + if match: + try: + value = json.loads(match.group(0)) + if isinstance(value, dict): + payload = value + except json.JSONDecodeError: + pass + reason = str(payload.get("综合评价") or payload.get("reason") or reply or "") + raw = payload.get("score", payload.get("overall_score")) + if raw is None: + matches = re.findall(r"(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)", reason, re.IGNORECASE) + raw = matches[-1] if matches else None + try: + numeric = float(raw) + except (TypeError, ValueError): + dimensions = payload.get("dimensions") if isinstance(payload.get("dimensions"), dict) else {} + if not dimensions: + dimensions = { + key: value + for key, value in payload.items() + if key not in {"score", "overall_score", "综合评价", "reason"} + } + values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))] + numeric = sum(values) / len(values) if values else None + if numeric is None: + return None, payload, reason + # The judge prompt uses the configured score range. Do not treat a + # decimal such as 0.5/5 as a 0.5/1 score, otherwise low scores are + # incorrectly inflated (0.5/5 would become 50 instead of 10). + normalized = (numeric - score_min) / max(score_max - score_min, 1e-9) * 100 + return max(0.0, min(100.0, normalized)), payload, reason + + def _judge_sample( question: str, reference: str, @@ -198,7 +310,7 @@ def _judge_sample( pass_threshold = float(config.get("pass_threshold", 3)) if not api_url or not eval_model: - return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置", + return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "未配置", "evaluation_reason": "未配置评测模型", "error_type": "其他"} system_msg = ( @@ -227,7 +339,7 @@ def _judge_sample( }).encode("utf-8") req = urllib.request.Request( - f"{api_url}/v1/chat/completions", + _normalise_api_url(api_url), data=body, headers={ "Content-Type": "application/json", @@ -238,39 +350,54 @@ def _judge_sample( data = json.loads(resp.read().decode("utf-8")) reply = data["choices"][0]["message"]["content"] except Exception as exc: - return {"score": 0, "max_score": score_max, "passed": False, + return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}", "error_type": "其他"} - # Parse score from reply — look for patterns like "4分" or "Score: 4" - score = 0 - import re - score_patterns = [ - r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)', - r'(\d+(?:\.\d+)?)\s*分', - r'(\d+(?:\.\d+)?)\s*/\s*\d+', - ] - for pat in score_patterns: - m = re.search(pat, reply, re.IGNORECASE) - if m: - try: - score = float(m.group(1)) - except ValueError: - continue - break - score = max(score_min, min(score_max, score)) - passed = score >= pass_threshold + parsed: dict[str, Any] = {} + match = re.search(r"\{[\s\S]*\}", reply) + if match: + try: + value = json.loads(match.group(0)) + if isinstance(value, dict): + parsed = value + except json.JSONDecodeError: + pass + raw_score = parsed.get("score", parsed.get("overall_score")) + reason = str(parsed.get("综合评价") or parsed.get("reason") or reply) + if raw_score is None: + matches = re.findall(r'(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)', reason, re.IGNORECASE) + raw_score = matches[-1] if matches else None + try: + numeric_score = float(raw_score) + except (TypeError, ValueError): + dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {} + if not dimensions: + dimensions = { + key: value + for key, value in parsed.items() + if key not in {"score", "overall_score", "综合评价", "reason"} + } + values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))] + numeric_score = sum(values) / len(values) if values else None + if numeric_score is None: + return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "错误", + "evaluation_reason": "评测模型未返回可解析分数：" + reason[:1800], "error_type": "其他"} + normalized_score = (numeric_score - score_min) / max(score_max - score_min, 1e-9) * 100 + normalized_score = max(0, min(100, normalized_score)) + normalized_threshold = (pass_threshold - score_min) / max(score_max - score_min, 1e-9) * 100 + passed = normalized_score >= normalized_threshold # Determine judgement label - if score >= pass_threshold + 1: + if normalized_score >= min(100, normalized_threshold + 20): judgement = "正确" - elif score >= pass_threshold: + elif passed: judgement = "部分正确" else: judgement = "错误" # Guess error type from reply - reply_lower = reply.lower() + reply_lower = (reply + " " + reason).lower() if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]): error_type = "幻觉" elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]): @@ -282,16 +409,84 @@ def _judge_sample( else: error_type = "其他" + parsed_dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else { + key: value + for key, value in parsed.items() + if key not in {"score", "overall_score", "综合评价", "reason"} + } return { - "score": score, - "max_score": score_max, + "available": True, + "score": round(normalized_score, 2), + "max_score": 100, + "raw_score": numeric_score, + "raw_max_score": score_max, "passed": passed, "judgement": judgement, - "evaluation_reason": reply[:2000], + "evaluation_reason": reason[:2000], "error_type": error_type, + "dimension_scores": [ + { + "name": str(name), + "score": round( + max( + 0, + min( + 100, + (float(value) - score_min) + / max(score_max - score_min, 1e-9) + * 100, + ), + ), + 2, + ), + "max_score": 100, + } + for name, value in parsed_dimensions.items() + if isinstance(value, (int, float)) + ], } +def _write_json(path: Path, payload: dict[str, Any]) -> None: + temporary = path.with_suffix(path.suffix + ".part") + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + temporary.replace(path) + + +def _write_eval_progress( + output_dir: Path, + status: str, + stage: str, + total: int, + completed: int, + message: str = "", + current_index: int = 0, +) -> None: + _write_json( + output_dir / "eval_progress.json", + { + "status": status, + "stage": stage, + "total": total, + "completed": completed, + "percentage": round(completed / total * 100, 1) if total else 100, + "current_index": current_index, + "message": message, + "updated_at": datetime.now(timezone.utc).isoformat(), + }, + ) + + +def _deterministic_sample_score(reference: str, prediction: str) -> float: + values = [SequenceMatcher(None, _normalize_text(reference), _normalize_text(prediction)).ratio()] + if _normalize_text(reference) == _normalize_text(prediction): + values.append(1.0) + rouge = _pair_rouge(reference, prediction) + if rouge.get("rougeL") is not None: + values.append(float(rouge["rougeL"])) + return round(sum(values) / len(values) * 100, 2) + + # --------------------------------------------------------------------------- # Main entry point # --------------------------------------------------------------------------- @@ -312,11 +507,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: print(f"[eval] loading dataset: {dataset_path}") raw_samples = _load_dataset(dataset_path) print(f"[eval] loaded {len(raw_samples)} samples") + _write_eval_progress(output_dir, "running", "dataset", len(raw_samples), 0, f"已加载 {len(raw_samples)} 条样本") # ---- 2. Load model ---- print(f"[eval] loading model: {model_path}") from compute.engines.llama_factory.inference import InferenceSession session = InferenceSession() + _write_eval_progress(output_dir, "running", "model_loading", len(raw_samples), 0, "正在加载评测模型") session.load( model_name_or_path=model_path, adapter_name_or_path=adapter_path, @@ -329,6 +526,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: if not load_result.get("loaded"): raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}") print(f"[eval] model loaded OK") + _write_eval_progress(output_dir, "running", "inference", len(raw_samples), 0, "开始生成模型回答") # ---- 3. Run inference on each sample ---- samples: list[dict[str, Any]] = [] @@ -384,12 +582,45 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: ] if judge_result else [], "status": "completed", }) + if not judge_enabled: + deterministic_score = _deterministic_sample_score(reference, prediction) + samples[-1].update({ + "score": deterministic_score, + "max_score": 100, + "raw_score": deterministic_score, + "raw_max_score": 100, + "passed": deterministic_score >= 60, + "judgement": "正确" if deterministic_score >= 80 else "部分正确" if deterministic_score >= 60 else "错误", + "evaluation_reason": "基于精确匹配、文本相似度和 ROUGE-L 的确定性评分", + }) + _write_json( + output_dir / "eval_results.json", + { + "status": "running", + "overall_score": round( + sum(float(item["score"]) for item in samples if item.get("score") is not None) + / max(len([item for item in samples if item.get("score") is not None]), 1), + output_precision, + ), + "overall_score_max": 100, + "overall_evaluation": f"已完成 {len(samples)}/{total} 条样本", + "dimension_summary": [], + "samples": samples, + "sample_count": total, + "completed_count": len(samples), + "passed_count": sum(bool(item.get("passed")) for item in samples), + "basic_metrics": {}, + "metric_summary_version": 2, + }, + ) progress_pct = int(idx / max(total, 1) * 100) + _write_eval_progress(output_dir, "running", "inference", total, len(samples), f"已完成第 {idx} 条样本", idx) print(f"[eval] sample {idx}/{total} ({progress_pct}%) done") # ---- 4. Compute basic metrics ---- print(f"[eval] computing basic metrics on {len(predictions)} predictions") + _write_eval_progress(output_dir, "running", "metrics", total, len(samples), "正在计算评测指标", total) metrics_result: dict[str, Any] = {} bleu_cfg = basic_cfg.get("bleu", {}) @@ -397,11 +628,11 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4))) rouge_cfg = basic_cfg.get("rouge", {}) - if rouge_cfg.get("enabled"): + if rouge_cfg.get("enabled") or not judge_enabled: metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods")) cosine_cfg = basic_cfg.get("cosine", {}) - if cosine_cfg.get("enabled"): + if cosine_cfg.get("enabled") or not judge_enabled: metrics_result["cosine"] = _compute_cosine(references, predictions) metrics_result["exact_match"] = _compute_exact_match(references, predictions) metrics_result["text_similarity"] = _compute_text_similarity(references, predictions) @@ -412,16 +643,15 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: scored = [s for s in samples if s.get("score") is not None] passed_count = len([s for s in scored if s.get("passed")]) avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision) - max_score = dimension_cfg.get("score_max", 5) - overall_score = round(avg_score / max_score * 100, output_precision) + overall_score = avg_score overall_score_max = 100 dimension_summary = [{ - "name": "综合评分", + "name": "LLM Judge", "score": overall_score, "max_score": 100, "pass_rate": round(passed_count / max(completed, 1) * 100, 1), }] - overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/{max_score} 分" + overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/100 分" else: passed_count = 0 enabled_scores = [ @@ -444,6 +674,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: overall_evaluation = f"评测完成：{completed} 样本（未配置 LLM 评委）" result = { + "status": "completed", "overall_score": overall_score, "overall_score_max": overall_score_max, "overall_evaluation": overall_evaluation, @@ -454,11 +685,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]: "completed_count": completed, "passed_count": passed_count, "basic_metrics": metrics_result, + "metric_summary_version": 2, } # ---- 6. Write results ---- result_path = output_dir / "eval_results.json" result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + _write_eval_progress(output_dir, "completed", "completed", total, completed, "评测完成", total) print(f"[eval] results written to {result_path}") return result diff --git a/compute/engines/llama_factory/inference.py b/compute/engines/llama_factory/inference.py index 018ec0b..e2ba822 100644 --- a/compute/engines/llama_factory/inference.py +++ b/compute/engines/llama_factory/inference.py @@ -7,6 +7,28 @@ import uuid from typing import Any, Iterator +def _ensure_peft_transformers_compat() -> None: + """Bridge a removed PEFT helper used by the bundled Transformers build. + + The offline Compute image currently contains Transformers 5.8.0 and PEFT + 0.18.1. Transformers imports this private helper when a model directory + contains PEFT metadata, but PEFT 0.18.1 does not expose it. Evaluation + runs in single-process HuggingFace mode, so tensor-parallel sharding is + not applicable and a no-op compatibility hook is the correct behavior. + """ + try: + from peft.utils import save_and_load + except Exception: + return + if hasattr(save_and_load, "_maybe_shard_state_dict_for_tp"): + return + + def _maybe_shard_state_dict_for_tp(_model: Any, _state_dict: dict[str, Any], _adapter_name: str) -> None: + return None + + save_and_load._maybe_shard_state_dict_for_tp = _maybe_shard_state_dict_for_tp + + class InferenceSession: """Manages a loaded model for inference with LLaMA-Factory ChatModel. @@ -143,6 +165,7 @@ class InferenceSession: args = dict(self._load_args) infer_result = get_infer_args(args) + _ensure_peft_transformers_compat() model = ChatModel(args) tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer generating_args = infer_result[-1] @@ -182,6 +205,7 @@ class InferenceSession: self._loaded_at = time.time() self._status = "ready" + def _release_model(self) -> None: with self._chat_lock: with self._state_lock: diff --git a/compute/tests/test_eval_runner.py b/compute/tests/test_eval_runner.py index 3a96ed5..0c8d6eb 100644 --- a/compute/tests/test_eval_runner.py +++ b/compute/tests/test_eval_runner.py @@ -2,7 +2,14 @@ from __future__ import annotations import json -from compute.engines.llama_factory.eval_runner import _load_dataset +from compute.engines.llama_factory.eval_runner import ( + _compute_exact_match, + _compute_rouge, + _compute_text_similarity, + _normalise_api_url, + _parse_judge_reply, + _load_dataset, +) def _write(tmp_path, name: str, text: str) -> str: @@ -40,3 +47,48 @@ def test_load_jsonl_with_bom_and_embedded_array(tmp_path) -> None: "" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]), ) assert len(_load_dataset(path)) == 2 + + +def test_deterministic_metrics_use_percent_scale() -> None: + references = ["北京是中国的首都"] + predictions = ["北京是中国的首都"] + assert _compute_exact_match(references, predictions)["score"] == 100 + assert _compute_text_similarity(references, predictions)["score"] == 100 + + +def test_rouge_supports_chinese_character_tokenization() -> None: + import pytest + pytest.importorskip("rouge_score") + result = _compute_rouge(["北京是中国的首都"], ["北京是中国的首都"]) + assert result["available"] is True + assert result["score"] == 100 + + +def test_judge_reply_accepts_json_and_normalises_score() -> None: + score, payload, reason = _parse_judge_reply( + '{"score": 4, "dimensions": {"正确性": 4}, "reason": "内容正确"}', + 0, + 5, + ) + assert score == 80 + assert payload["dimensions"]["正确性"] == 4 + assert reason == "内容正确" + + +def test_judge_reply_accepts_nlp_demo_dimension_format() -> None: + score, _, _ = _parse_judge_reply( + '{"语义一致性": 4, "信息完整性": 3, "事实准确性": 5, "语言流畅性": 4, "综合评价": "整体良好，0.8"}', + 0, + 5, + ) + assert score == 80 + + +def test_judge_reply_keeps_decimal_scores_in_configured_range() -> None: + score, _, _ = _parse_judge_reply('{"score": 0.5}', 0, 5) + assert score == 10 + + +def test_openai_url_does_not_duplicate_v1() -> None: + assert _normalise_api_url("https://example.test/v1") == "https://example.test/v1/chat/completions" + assert _normalise_api_url("https://example.test") == "https://example.test/v1/chat/completions" diff --git a/docker/app/.env b/docker/app/.env index f2cf2c4..6e5a215 100644 --- a/docker/app/.env +++ b/docker/app/.env @@ -61,5 +61,6 @@ MINIO_SECRET_KEY=change_me_minio_secret MINIO_BUCKET=yg-ft-resources MINIO_SECURE=false MINIO_INLINE_MAX_BYTES=262144 +MINIO_PRESIGN_MAX_BYTES=1099511627776 STORAGE_WAIT_SECONDS=300 STORAGE_CHECK_INTERVAL_SECONDS=10 diff --git a/docker/app/docker-compose.yml b/docker/app/docker-compose.yml index c1fe42d..2711833 100644 --- a/docker/app/docker-compose.yml +++ b/docker/app/docker-compose.yml @@ -71,6 +71,7 @@ services: MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources} MINIO_SECURE: ${MINIO_SECURE:-false} MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144} + MINIO_PRESIGN_MAX_BYTES: ${MINIO_PRESIGN_MAX_BYTES:-1099511627776} STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300} STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10} DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process} diff --git a/docker/compute/.env b/docker/compute/.env index 4819556..a313ddb 100644 --- a/docker/compute/.env +++ b/docker/compute/.env @@ -25,6 +25,8 @@ YG_FT_DATASET_ROOT=/data/yg-ft/datasets YG_FT_DATASET_ROOT_HOST=./data/yg-ft/datasets YG_FT_OUTPUT_ROOT=/data/yg-ft/outputs YG_FT_OUTPUT_ROOT_HOST=./data/yg-ft/outputs +COMPUTE_CACHE_MAX_BYTES=0 +COMPUTE_CACHE_TTL_SECONDS=0 TRAINING_LOG_ROOT=/opt/yg-ft/logs/training TRAINING_LOG_ROOT_HOST=./data/yg-ft/logs/training COMPUTE_LOG_ROOT_HOST=./data/yg-ft/logs/compute diff --git a/docker/compute/docker-compose.yml b/docker/compute/docker-compose.yml index 4e13372..3f05c4c 100644 --- a/docker/compute/docker-compose.yml +++ b/docker/compute/docker-compose.yml @@ -31,6 +31,8 @@ services: NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility} YG_FT_CACHE_ROOT: ${YG_FT_CACHE_ROOT:-/data/yg-ft} + COMPUTE_CACHE_MAX_BYTES: ${COMPUTE_CACHE_MAX_BYTES:-0} + COMPUTE_CACHE_TTL_SECONDS: ${COMPUTE_CACHE_TTL_SECONDS:-0} PYTHONPATH: /app volumes: - ../../compute:/app/compute:ro diff --git a/docs/20260812/database-migration.md b/docs/20260812/database-migration.md new file mode 100644 index 0000000..21badd0 --- /dev/null +++ b/docs/20260812/database-migration.md @@ -0,0 +1,38 @@ +# 数据库迁移说明 + +## 当前迁移文件 + +- 新库初始化：`backend/app/db/sql/000_full_init.sql` +- 已有数据库增量迁移：`backend/app/db/sql/005_storage_progress_migration.sql` +- GPU 预留迁移：`backend/app/db/sql/006_gpu_reservations.sql` +- 平台闭环迁移：`backend/app/db/sql/007_platform_completion.sql` +- 离线部署使用：`docker/offline/src/backend/app/db/sql/000_full_init.sql` + +## 执行方式 + +```bash +for migration in 005_storage_progress_migration.sql 006_gpu_reservations.sql 007_platform_completion.sql; do + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "backend/app/db/sql/$migration" +done +``` + +迁移前应完成数据库备份，并保存以下结构快照： + +```sql +SELECT table_name, column_name, data_type +FROM information_schema.columns +WHERE table_schema = 'public' +ORDER BY table_name, ordinal_position; +``` + +## 本次迁移内容 + +- 为算力节点资源副本增加 `version_id` 和 `storage_object_id`，用于记录 MinIO 版本与本地缓存对应关系。 +- 补齐 `storage_objects.metadata`、模型产物、数据集文件和数据转换任务的 MinIO 字段。 +- 补齐评测任务软删除、创建者、租户和报告对象字段。 +- 增加活动评测任务、资源副本和 MinIO 对象索引。 +- 增加 GPU 预留表，统一训练、推理、评测的卡级占用约束。 +- 增加模型导出创建者/租户/归档状态字段。 +- 增加 MinIO 对象软删除清理、缓存版本校验和访问保护字段及清理任务表。 + +`000_full_init.sql` 已包含相同的幂等变更，新增数据库直接执行初始化脚本即可；已有数据库不要依赖容器重启自动完成迁移。 diff --git a/docs/20260812/当前项目开发进度.md b/docs/20260812/当前项目开发进度.md new file mode 100644 index 0000000..4757fda --- /dev/null +++ b/docs/20260812/当前项目开发进度.md @@ -0,0 +1,383 @@ +# 当前项目开发进度 + +> 评估基线：2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。 +> +> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容，统一标记为“部分完成”或“未完成”。 + +## 一、项目定位与总体结论 + +当前项目是一个面向多用户、多算力节点的模型训练与推理平台，主要链路为： + +```text +Vue 前端 + | +FastAPI Backend API + |-- PostgreSQL：业务元数据、权限、任务状态、小型内容和预览数据 + |-- Redis：会话、限流、短期缓存和任务辅助状态 + |-- MinIO：模型、数据集、报告和大文件的统一对象存储 + |-- Compute API / Agent：训练、推理、评测、模型合并和 GPU 执行 + | +多台算力节点 +``` + +整体判断： + +| 范围 | 当前状态 | 结论 | +|---|---|---| +| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 | +| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 | +| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入，跨节点一致性和失败恢复仍需加强 | +| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现，但完整的租户/项目隔离尚未闭环 | +| MinIO 统一存储 | 部分完成 | 大文件和模型已接入，仍存在兼容性的本地路径和部分数据双写/回退路径 | +| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 | +| 前端体验 | 基本可用，需优化 | 构建问题已持续修复，但页面响应等待、首屏体积和部分错误提示仍需优化 | + +## 二、已完成的功能 + +### 2.1 平台基础与部署 + +- 已建立 Vue 3 + TypeScript + Vite 前端工程。 +- 已建立 FastAPI 后端服务，提供登录、平台管理和模型业务接口。 +- 已建立 Compute API / Agent，用于连接算力节点并执行训练、推理、评测和模型处理任务。 +- 已使用 PostgreSQL 保存核心业务数据，Redis 提供会话、限流和缓存能力。 +- 已增加 MinIO 服务及 Backend 的 MinIO 配置，支持和 Compute Agent 分离部署。 +- 已提供 `docker/app`、`docker/compute`、`docker/minio` 和 `docker/offline` 部署目录。 +- 已考虑后端、算力服务、MinIO 分布在不同服务器时使用独立网络；Compute 节点访问 MinIO 需要配置所有节点都能访问的固定 IP 或 DNS。 +- 离线部署目录已经同步后端、算力相关源码和初始化 SQL 的主要改造内容。 + +### 2.2 登录、用户和权限基础 + +- 用户登录、退出、当前用户信息和密码修改接口已经存在。 +- 已有 Token 会话、Redis 会话记录和登录限流逻辑。 +- 已建立用户、角色、权限码和角色权限关系。 +- 已实现管理员、普通用户等基础角色分层。 +- 已实现页面路由守卫、菜单过滤和前端按钮级权限的基础能力。 +- 已建立资源 ACL 管理页面和相关接口，可对用户或角色授予资源级权限。 +- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。 +- 已建立运行日志、审计日志查询页面及审计记录写入机制。 +- 已加入软删除相关字段和部分删除逻辑，避免直接物理删除业务资源。 + +### 2.3 算力节点与 GPU 资源 + +- 已实现算力节点的新增、编辑、启用、禁用、维护/删除、连通性测试和健康检查。 +- 已实现节点列表、节点详情、节点副本/同步状态和 Compute Agent 连接。 +- 已实现 GPU 信息发现、GPU 状态查询和队列查询。 +- 已建立 `gpu_allocations`、`gpu_assignments`、调度锁等资源分配表。 +- 训练任务已经支持选择调度节点和一张或多张 GPU，并在预检阶段校验资源可用性。 +- Compute Agent 已支持训练、评测、推理和缓存准备等任务接口。 +- 已存在资源副本和同步任务模型，用于记录节点侧资源同步状态。 + +### 2.4 数据集管理 + +- 已实现数据集创建、列表、详情、编辑、删除和文件上传。 +- 已实现数据集文件下载、预览、记录列表和数据记录编辑入口。 +- 已处理 JSON 与 JSONL 的记录数差异：JSON 数组按元素计数，JSONL 按有效行计数，避免把整个 JSON 文件误按行数统计。 +- 已提供数据集版本列表、版本详情、创建版本、切换当前激活版本和删除版本接口。 +- 已增加数据集文件、版本、数据记录等初始化表结构。 +- 已支持数据集文件在数据库小内容和 MinIO 大文件之间按策略存储。 +- 已在训练预检中检查数据集文件是否存在、是否可从 MinIO 获取以及是否能准备到目标算力节点。 + +### 2.5 数据处理与数据转换 + +- 已提供结构化数据、非结构化数据、外部数据源的处理创建流程。 +- 已实现源文件上传、预览、分片/切分、生成、质量检查、去重和结果管理等数据处理流程。 +- 已支持处理结果生成数据集或导入数据集版本。 +- 已建立数据处理任务、源文件、预览项、结果等数据表。 +- 已提供 JSON、JSONL 等数据格式转换页面和后端任务接口。 +- 已将数据转换输出接入 MinIO/数据库分层存储：小型文本结果可存数据库，大文件存 MinIO。 +- 已处理输出文件下载和转换结果元数据保存问题。 + +### 2.6 模型训练 + +- 已实现训练任务创建、配置预检、命令预览、启动、停止、重试和删除。 +- 已接入 LLaMA-Factory 等训练适配逻辑。 +- 已支持选择训练数据、基座模型、算力节点和 GPU。 +- 已实现训练日志获取、训练任务概览、诊断信息、检查点和训练指标查询。 +- 前端训练详情已经具备训练曲线解析和展示逻辑，日志轮询间隔已调整为 3 秒。 +- 已增加 GPU 详情展示入口，包括显存和利用率等 Compute Agent 上报信息。 +- 已支持训练任务的 MinIO 数据准备和目标算力节点缓存准备。 + +### 2.7 模型管理与权重合并 + +- 已实现在线模型/基座模型和训练模型的列表、创建、详情、用途修改和删除。 +- 已建立模型、训练模型、模型血缘、模型产物和导出任务相关表。 +- 已提供权重合并入口，能够根据训练任务准备基座模型和 Adapter，并提交 Compute Agent 执行合并。 +- 已增加模型产物和 MinIO 对象关联字段。 +- 合并结果能够在任务完成后归档到 MinIO 的设计和主要代码路径已经建立。 + +### 2.8 模型推理与模型对比 + +- 已实现推理模型列表、创建、详情和删除入口。 +- 已实现模型加载、卸载、服务启动、服务状态查询和对话调用。 +- 已实现模型对比任务及多模型聊天相关接口。 +- 已增加推理失败重试、停止和资源释放的处理路径。 +- 已支持根据页面选择的算力节点准备模型缓存，兼容训练时所选节点优先的业务要求。 +- Compute Agent 已提供本地推理会话和模型缓存状态接口。 + +### 2.9 模型评测 + +- 已实现评测任务列表、创建、详情和删除。 +- 已实现评测维度管理和评测规则配置页面。 +- 已建立评测任务、评测维度、对比任务等数据库表。 +- 已支持选择模型、数据集、评测维度、算力节点和 GPU 的基础流程。 +- 已接入 Compute Agent 执行评测任务，并保存评测结果和报告相关元数据。 + +### 2.10 数据存储策略 + +- 已建立 `storage_objects`、`storage_cache_jobs` 等 MinIO 元数据和缓存任务表。 +- 已建立 MinIO 对象上传、下载、预签名 URL 和节点缓存准备的主要接口。 +- 已采用分层策略： + - 小型 JSON、JSONL、CSV、任务参数快照、预览数据保留在数据库，降低频繁预览的 MinIO 延迟。 + - 模型权重、训练产物、评测报告和大文件使用 MinIO。 + - 小型 PDF、DOCX、XLSX 仍优先存 MinIO，以保留原始二进制文件内容。 +- 已增加 `data_convert_tasks.output_content`，用于保存小型转换结果，避免所有小结果都依赖 MinIO。 +- Backend 和离线包中的 `000_full_init.sql` 已同步，当前两份初始化脚本内容一致。 + +## 三、部分完成、仍需完善的功能 + +### 3.1 MinIO 统一数据源尚未完全闭环 + +当前 MinIO 已成为模型、大文件和跨节点资源的主存储方向，但仍保留以下兼容路径： + +- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。 +- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段，这是当前已确认的小文件性能策略，不是错误，但必须统一记录来源、大小、校验值和版本。 +- Compute Agent 节点侧的大文件上传已改为流式读取；Backend 端部分历史上传/下载路径仍未完成统一的分片传输。 +- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联，数据库没有直接外键，删除和数据一致性需要应用层保证。 +- 删除业务资源后，对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。 + +### 3.2 MinIO 预签名接口的权限边界 + +资源写权限、服务端对象 Key、上传完成确认和对象大小校验已经完成；仍需补充 Content-Type 白名单、对象过期回收和完整审计事件。 + +### 3.3 激活版本和跨节点资源版本仍需加强 + +数据集已经有 `active_version_id` 和版本表，但以下场景仍需补充： + +- 训练、评测、模型对比和权重合并已在任务创建时固化资源版本 ID；推理服务启动和导出任务仍需统一补齐。 +- 同一文件名的不同版本不能只依靠文件名同步，应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。 +- 已创建任务在后续切换激活版本后，不能被意外切换到新版本。 +- 已为资源副本保存版本、对象 ID 和本地路径；对象 ETag/校验值及多文件 manifest 仍需补齐。 +- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。 + +### 3.4 权限 2.0 尚未完全落地 + +已有用户、角色、权限码、ACL、审批和审计基础，但仍存在以下差距： + +- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。 +- 项目空间设计已经讨论过取消，但数据库中仍保留 `projects`、`project_members` 等历史结构，需要明确兼容策略和最终迁移方式。 +- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。 +- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。 +- 前端按钮权限已经有基础实现，但不能替代后端鉴权；仍需要对所有关键动作进行后端默认拒绝校验。 +- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。 + +### 3.5 模型合并、导出和评测报告闭环不足 + +- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立，但失败时的清理、重试和幂等性仍需加强。 +- 合并结果归档已增加失败状态和服务重启后的补偿轮询；仍需加入独立归档队列和幂等对象清单，降低重复扫描成本。 +- 模型导出任务目前有查询模型和表结构，但完整的创建、执行、进度、失败重试和下载闭环尚未完成。 +- 评测结果和报告字段已经存在，但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。 +- 评测指标配置和执行器返回指标之间仍需要强类型映射，避免前端显示为通用的 `custom`。 + +### 3.6 GPU 资源分配需要统一到所有任务类型 + +- 训练已经有较完整的节点/GPU 选择和预检流程。 +- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码，但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。 +- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。 +- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。 +- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报，仍需要校验采样时间、单位、空值和任务对应关系。 + +## 四、尚未完成的功能 + +以下功能在当前代码中没有形成可验收的完整闭环，或仍处于设计/基础代码阶段： + +1. **完整的租户隔离和资源继承模型**：核心列表、详情、下载、缓存、训练、推理、评测和导出接口已补齐基础租户/ACL校验；历史 NULL 租户归属仍需专项迁移。 +2. **项目取消后的正式数据迁移方案**：当前保留项目表作为兼容结构，正式删除项目设计前仍需为历史数据生成归属迁移和回滚脚本。 +3. **预签名上传策略的完整约束**：已完成 Content-Type、大小、过期时间、Key 白名单、资源写权限、对象存在性和真实 SHA-256 校验；定时清理过期未完成对象仍需接入运维调度。 +4. **MinIO 对象生命周期管理**：已提供软删除清理、失败记录/重试、孤儿扫描和管理员清理入口；自动定时执行策略需由部署环境接入。 +5. **Compute Agent 缓存治理**：已完成容量上限、LRU、TTL、保护窗口、版本/校验清单和状态查询；运行中任务动态保护和磁盘告警仍需结合生产指标系统。 +6. **统一的资源归档编排器**：训练、合并、导出、评测已纳入重启补偿轮询和 MinIO 归档；独立消息队列和大规模断点分片仍属于生产增强项。 +7. **模型导出完整流程**：已完成后端创建、节点准备、CPU 导出、量化参数、任务记录、制品血缘、归档轮询、权限和前端按钮；真实大模型多格式压力测试待专用环境执行。 +8. **流式和分片文件传输**：数据集单文件下载、Compute Agent 上传/下载已采用流式处理；多文件 ZIP 和超大对象的分片上传仍需继续增强。 +9. **生产级 MinIO 安全和高可用**：开发环境接入和故障快速失败已完成；默认密钥替换、TLS、网络隔离、Console 隔离、容量监控、备份恢复需在生产部署阶段实施。 +10. **完整的端到端测试和持续集成**：已新增前端构建、后端编译和存储安全测试 CI 基线；跨租户、多节点、多 GPU 并行压力和故障注入仍需扩展执行矩阵。 +11. **统一数据库迁移体系**：已形成 005/006/007 幂等增量迁移并加入运行时兼容执行；后续可再替换为 Alembic 等正式迁移工具以满足生产审计要求。 + +## 五、需要优化的功能 + +### 5.1 后端响应性能 + +- 页面列表接口需要避免每条记录重复查询用户、资源、MinIO 元数据和 Compute 节点状态。 +- MinIO 的 Bucket 检查、对象 Head 和预签名生成应使用连接复用、短期缓存和批量查询。 +- 训练、推理、评测页面不应通过过短间隔轮询大量详情接口，应按任务状态动态退避，并在完成后停止轮询。 +- 对 dashboard、节点健康、GPU 状态等高频数据应区分实时数据和缓存数据。 +- 后端日志轮询和健康检查日志需要继续降噪，仅在状态变化、失败或达到较长周期时输出。 + +### 5.2 前端加载和交互 + +- 列表页面应区分首屏 loading、刷新 loading、操作 loading，避免整页长时间无反馈。 +- 推理、评测、训练详情应使用统一的任务状态刷新策略和超时提示。 +- 前端仍有 FontAwesome 在线资源解析警告，应清理对外部网络文件的依赖，保证离线环境打开速度。 +- 应继续拆分首屏大体积 chunk，并减少一次性加载不相关页面组件。 +- GPU 选择组件需要明确显示空闲、占用、不可达、预留和已分配状态。 +- 错误提示应携带资源名称、节点名称、版本和下一步处理建议，减少只显示 500/404 的情况。 + +### 5.3 训练、推理和评测可靠性 + +- 所有任务创建前应执行同一套资源权限、版本存在性、MinIO 可用性和 GPU 原子分配校验。 +- 任务创建接口应支持幂等键，避免前端重复点击造成重复任务。 +- 节点不可达时应快速失败或进入可见的等待状态，不能让页面长时间无反馈。 +- 失败重试应区分网络瞬时失败、资源不足、模型文件缺失、参数错误和执行器失败。 +- 任务停止后必须释放 GPU 分配、推理端口、缓存锁和临时目录。 + +### 5.4 数据和模型一致性 + +- 每个对象都应保存大小、校验值、版本 ID、来源、创建者、租户和引用状态。 +- 数据库中的小文件内容和 MinIO 对象不能同时被当作可独立修改的主副本；需要明确唯一写入入口。 +- 数据集激活版本变更需要留下审计记录，并影响后续任务创建但不改变已创建任务。 +- 模型权重、Adapter、合并结果和导出结果需要形成完整血缘关系。 + +## 六、数据库和初始化脚本状态 + +当前 `backend/app/db/sql/000_full_init.sql` 已包含以下主要类别： + +- 用户、模型、训练模型、模型血缘、模型产物、模型导出任务。 +- 数据集、数据集文件、数据集版本、数据集记录。 +- 算力节点、GPU、GPU 分配、调度锁、Compute Job。 +- 资源副本、资源同步任务、MinIO 对象、缓存任务。 +- 评测任务、评测维度、模型对比任务。 +- 租户、项目兼容表、项目成员、角色、会话、ACL。 +- 审批模板、审批实例、审批步骤、审计日志、留存策略。 +- 数据处理任务、源文件、预览项、处理结果、数据转换任务。 + +已确认的近期字段包括： + +- `model_artifacts.storage_object_id` +- `model_artifacts.storage_backend` +- `dataset_files.storage_object_id` +- `data_convert_tasks.output_content` +- `data_convert_tasks.output_storage_object_id` +- `data_convert_tasks.storage_backend` +- `eval_tasks.report_storage_object_id` + +离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意： + +- 初始化 SQL 主要用于新数据库或新数据卷；已有数据库不能仅靠重启容器自动获得全部新字段。 +- 生产/测试数据库需要执行可追踪的迁移脚本，并在迁移前备份或生成结构快照。 +- `ensure_schema` 类运行时补字段逻辑只能作为兼容兜底，不能替代正式迁移。 +- 后续如果正式移除项目设计，需要先完成数据归属迁移，再决定是否删除历史表，不能直接从初始化 SQL 中删除表。 + +## 七、当前验证结果 + +### 7.1 2026-08-19 本轮按计划落地内容 + +- P0 MinIO 预签名上传已增加资源存在性、资源写权限、管理员基座模型写权限、资源版本和对象 Key 前缀校验；新增上传完成确认接口，会校验 MinIO 对象存在、大小和状态后再标记为可用。 +- 训练、评测、模型对比和权重合并任务会保存创建时的租户信息与资源版本快照，后续切换数据集激活版本不会改变已创建任务的输入版本。 +- 数据集、评测任务、训练任务和模型对比列表增加租户范围过滤；用户表、模型/数据集创建入口补齐租户归属；数据转换资源补充所有者权限映射。 +- Compute Agent 准备缓存后会回写资源副本的版本、MinIO 对象和本地路径；缓存支持可选容量上限、按访问时间淘汰非临时文件，并提供当前占用量和上限状态。 +- 训练产物、权重合并结果和评测报告增加 MinIO 归档、失败状态记录以及 Backend 重启后的补偿轮询；离线部署源码已同步对应逻辑。 +- Compute Agent 大文件上传已改为流式读取，避免将整个文件一次性读入内存；离线 Compute Agent 同步完成。 +- 增加 `005_storage_progress_migration.sql` 增量迁移脚本，并修复 `000_full_init.sql` 中旧数据库执行时索引早于字段补齐的问题；主工程和离线初始化脚本已同步。 +- 增加 `MINIO_PRESIGN_MAX_BYTES` 配置和上传 Content-Type 白名单；首次数据库 schema 检查移出 Compute Poller 的 Uvicorn 事件循环，避免远程 PostgreSQL 慢连接拖垮健康检查；轮询相同失败改为 300 秒限频记录，并跳过已标记 offline 节点。 +- 前端 `npm run build` 已通过；容器内 MinIO Key 越权校验专项测试为 `5 passed`，Backend 和 Compute Agent 重启后均为 healthy。 + +已完成的静态和局部验证： + +- Backend 和离线 Backend 源码 `compileall` 检查通过。 +- MinIO 分层策略冒烟验证通过：小型 JSON/JSONL 可落数据库，小型二进制和超过阈值的内容进入 MinIO。 +- 主工程和离线包初始化 SQL 已做同步检查，内容一致。 +- 前端此前已完成 `npm run build` 类型错误修复，构建剩余问题主要是非阻断的资源/分包警告。 +- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。 +- 当前 WSL 运行验证中 Backend、Compute Agent、MinIO 均为 healthy；`gpu-node-02`（`172.25.179.69:19100`）连接、GPU 0 分配和任务执行均正常。`gpu-node-01` 的历史地址不可达，已通过健康检查标记为 offline，轮询器不再重复轮询其历史任务。 + +### 7.2 2026-08-19 gpu-node-02 真实流程验证 + +- 训练：使用 `qwen3.5-0.8B` + `test_data_0817`，任务 `ft_172a2da65140` 完成，GPU 0 使用期间可看到显存、利用率、温度和进程，结束后恢复 idle；24 个训练产物已归档 MinIO。 +- 资源快照：任务 `ft_d93b0fdae1c9` 创建时已保存数据集 `ds_8f6b8c5714c7` 的活动版本 `file_d550c2128722_v1`。 +- 训练曲线：任务 `ft_d93b0fdae1c9` 通过 `logging_steps=1` 生成 3 个 loss/epoch/learning-rate/grad-norm 数据点，并生成 `training_loss.png`；后端解析器已兼容带引号数字格式。 +- 权重合并：任务 `merge_1dc15a290810` 完成，基座模型路径、合并路径已回写，8 个合并模型文件归档 MinIO。 +- 推理：任务 `cmp_abdf0762869d` 在 GPU 0 加载 `qwen3.5-0.8B` 成功，对话接口返回正常；卸载后 GPU 恢复 idle。 +- 评测：任务 `eval_3c3f84263e23` 完成 2 条样本评测，报告、样本明细和基础指标已落库；GPU 恢复 idle。评审模型 HTTP 400 导致评审分数为 0，属于评审模型接口配置问题，算力评测链路本身已跑通。 + +### 7.3 多 GPU、MinIO 故障和跨用户权限专项验证 + +- 多 GPU 调度开发：新增 `gpu_reservations` 表，评测和推理在远程提交/加载前与训练统一纳入数据库原子预留；失败、停止、删除、节点不可达和卸载路径自动释放预留。当前 `gpu-node-02` 只有 GPU 0，已通过同卡“推理预留后评测抢占”测试，评测被明确拒绝，释放后 GPU 恢复 idle。多节点、多卡的真实并行测试待增加第二个可达节点和多卡节点后执行。 +- 调度锁优化：调度锁改为事务内持有并在提交前释放，避免一次调度成功后后续请求等待 30 秒 TTL；schema 初始化增加 PostgreSQL advisory lock，避免轮询器与首个请求并发初始化造成死锁。 +- MinIO 故障注入：停止 `yg-ft-minio` 后，`GET /modelTF/health` 返回 HTTP 200 且 `storage.status=unavailable`；恢复容器后返回 `storage.status=ready`。MinIO 客户端关闭默认重试链，故障健康探测耗时从约 6 秒降至约 0.3 秒。 +- 跨用户权限：创建临时用户 `qa_user_0819`，未授权访问管理员数据集返回 403；授予资源 `read/download` ACL 后列表和详情可访问；撤销 ACL 后再次返回 403；非管理员创建用户返回 403。测试用户已删除，未遗留测试 ACL 或 GPU 预留。 +- 用户管理接口已补齐管理员依赖，创建、列表、修改、删除和重置密码不再允许普通用户调用。 + +当前不能据此宣称“全量功能测试通过”： + +- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设，需要按当前分层存储策略更新。 +- 本轮已完成当前 WSL Docker 容器健康检查、`gpu-node-02` 单节点真实流程、单卡冲突、MinIO 故障恢复和跨用户 ACL 专项验证；多节点、多卡的真实并行测试仍需要第二个可达节点和多卡节点。 + +- 本轮专项安全测试为 `5 passed`；全量 pytest 仍未执行完毕，需要按测试类别拆分并设置超时。 + +### 7.4 本轮 11 项未验证能力的补齐结果 + +- 租户管理接口已统一要求管理员身份；模型制品、模型血缘、导出任务、评测报告和 MinIO 资源清单均增加资源级访问校验。 +- 新增 `POST /modelTF/model-manage/export`，导出沿用节点选择、MinIO 缓存准备、GPU/调度约束和归档轮询；导出结果记录到 `model_export_jobs`，并建立导出制品与模型血缘。 +- 新增 `GET /modelTF/model-eval/{task_id}/report`，优先流式返回 MinIO 报告，历史任务无归档对象时返回数据库报告兼容结果。 +- 新增 `GET /modelTF/storage/resources/{resource_type}/{resource_id}/manifest`、管理员对象清理和孤儿扫描接口；预签名上传增加过期时间和真实 SHA-256 内容校验。 +- Compute Agent 增加缓存 TTL、缓存保护窗口和元数据清单；超过 TTL 的非保护缓存会在状态检查时清理，容量淘汰会跳过保护中的资源。 +- 数据集单文件下载改为 MinIO 流式传输，避免 Backend 一次性载入完整大文件；训练、合并、导出、评测仍由统一轮询器负责重启后的归档补偿。 +- 新增 `007_platform_completion.sql`，并已加入运行时兼容迁移；主工程和 `docker/offline/src` 的源码及初始化 SQL 已同步。 +- 以上为代码闭环和单节点验证；生产级 MinIO TLS/HA、第二节点/多卡并行压力测试、全量 CI 和历史项目数据正式迁移仍属于上线前专项工作。 + +## 八、下一阶段开发计划 + +### P0：安全与数据正确性 + +1. 为预签名上传补充 Content-Type、大小上限、过期对象清理和上传完成审计；当前已完成 Key、资源写权限、对象存在性和大小校验。 +2. 将资源版本快照继续接入推理服务启动、模型导出和缓存准备的所有入口，并增加版本切换回归测试。 +3. 完成模型导出接口的后端权限、租户范围和审计闭环。 +4. 补充历史数据租户归属迁移；当前新建资源和主要任务列表已完成租户过滤，历史 NULL 租户仍按兼容策略处理。 + +### P1：跨节点可靠性 + +1. 将当前资源副本字段升级为完整 manifest，记录每个文件的校验值、大小、目标节点路径和准备时间。 +2. 完善推理模型缓存的重启对账、失效版本清理和服务级重试；训练、合并、评测归档已具备补偿轮询基础。 +3. GPU 原子分配、异常回收和任务释放已完成基础闭环；下一步补充多节点重连对账及多卡并行压力测试。 +4. 增加缓存 TTL、运行任务保护、磁盘占用告警和可视化清理入口；当前已实现可选容量上限和按访问时间淘汰。 +5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。 + +### P2：性能与用户体验 + +1. 优化页面列表接口和高频轮询，采用批量查询、短期缓存和动态退避。 +2. 将大文件上传/下载/复制改为流式或分片传输。 +3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。 +4. 处理前端离线资源警告，继续拆分首屏 chunk。 +5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。 + +### P3：工程化和上线准备 + +1. 建立正式数据库版本迁移机制和离线升级脚本。 +2. 增加 CI：前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。 +3. 增加第二个可达节点/多卡节点后执行多节点端到端并行测试；MinIO 故障注入基础测试已完成。 +4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。 +5. 建立生产运行手册，包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。 + +## 九、阶段验收标准 + +完成下一阶段后，至少应满足： + +- 用户只能看到和操作其所属租户授权的模型、数据集、训练任务、推理服务和评测任务。 +- 任何任务创建都能明确记录用户、租户、资源版本、算力节点、GPU 列表和 MinIO 对象版本。 +- 同一个节点的同一张 GPU 不能被两个活动任务同时分配。 +- MinIO 临时不可用时，任务进入可解释的等待/失败状态，并能按策略重试，页面不会无限等待。 +- Backend 或 Compute Agent 重启后，任务、缓存、GPU 分配和归档状态可以对账恢复。 +- 训练、合并、评测和推理产物都能在 MinIO 中找到，并且可以通过权限校验后的接口下载或使用。 +- 删除资源后不会继续出现在普通列表中，关联对象能够按引用状态延迟清理并留下审计记录。 +- 新数据库初始化和已有数据库迁移后，所有业务接口不再因为缺表或缺字段启动失败。 +- 离线部署不依赖外部字体、图标或 CDN，前端首屏和核心业务操作在无网络环境下可用。 + +## 十、相关文件索引 + +- 平台架构：[platform-architecture-requirements.md](./platform-architecture-requirements.md) +- 权限设计：[permissions-design.md](./permissions-design.md) +- MinIO 与 Compute 缓存方案：[minio-compute-cache-plan.md](./minio-compute-cache-plan.md) +- 平台治理菜单设计：[platform-governance-menu-design.md](./platform-governance-menu-design.md) +- 数据处理设计：[data-process-design.md](./data-process-design.md) +- 数据库初始化脚本：[../backend/app/db/sql/000_full_init.sql](../backend/app/db/sql/000_full_init.sql) +- 离线部署目录：[../docker/offline](../docker/offline) + diff --git a/docs/20260812/权限开发进度.md b/docs/20260812/权限开发进度.md new file mode 100644 index 0000000..f218503 --- /dev/null +++ b/docs/20260812/权限开发进度.md @@ -0,0 +1,447 @@ +# 权限开发进度 + +> 更新时间：2026-08-20 +> 适用范围：YG_FT 平台当前主工程、Backend、Frontend、Compute Agent、MinIO 资源访问及 `docker/offline/src` 离线源码。 +> 当前结论：权限 2.0 的基础能力已形成闭环；本轮继续完成 GPU/租户配额原子预留、租户成员邀请与接受、审批动作白名单/幂等/过期处理、GPU 节点与卡冲突校验，并补充租户详情页成员管理。在线数据库迁移已执行并核验，前端构建、后端静态检查、权限用例和基础容器健康检查通过。由于当前只有一个可达算力节点，跨租户双用户、第二节点/多卡并发、节点故障回收仍需后续专项验证，暂不能标记为“全部完成”。按工作包估算，核心权限开发约完成 94%，上线验收约完成 76%。 + +## 本轮 11 项完成情况（2026-08-19） + +| 编号 | 权限能力 | 完成内容 | 状态 | +|---|---|---|---| +| 1 | 审批决策安全 | 审批人从当前会话获取，校验指定审批人、租户管理员、禁止申请人自审，记录审批审计 | 已完成 | +| 2 | 资源访问申请 | 新增资源访问申请记录、申请权限/期限/理由、审批后自动写入 ACL、支持撤回和过期 | 已完成 | +| 3 | 审批策略执行 | 按租户、动作、资源类型匹配策略；高风险操作审批通过后可一次性重试执行 | 已完成 | +| 4 | 租户与项目隔离 | `tenant_members`、活跃租户校验、历史资源归属补齐、跨租户 ACL 主体校验、项目软删除 | 已完成 | +| 5 | 用户与角色边界 | 禁用用户立即注销会话；租户成员支持 owner/admin/member/viewer，保护最后一个 owner | 已完成 | +| 6 | 数据集权限入口 | 预览、来源、版本查询/创建/激活/删除、上传/编辑/下载统一接入资源动作权限 | 已完成 | +| 7 | 模型和推理权限 | 训练模型合并/导出分别校验 execute/download；聊天、预加载、批量、卸载绑定授权模型或推理任务 | 已完成 | +| 8 | GPU 分配申请 | 普通用户可提交 GPU 分配申请，审批通过后自动分配；管理员可直接分配 | 已完成 | +| 9 | 租户配额申请 | 租户成员可提交配额变更申请，平台管理员审批后自动更新配额 | 已完成 | +| 10 | 软删除与安全状态 | 资源删除撤销 ACL、取消待处理申请和审批；租户/项目采用软删除状态 | 已完成 | +| 11 | 前端按钮与审计 | 审批决策、ACL 编辑、访问申请/撤回按权限显示；下载、导出、审批、GPU 等敏感操作补充结构化审计 | 已完成 | + +### 数据库迁移结果 + +- 新增增量迁移：`backend/app/db/sql/009_permission_completion.sql`，可独立兼容旧库执行，并已在当前远程 PostgreSQL 执行成功。 +- 当前远程库已核验包含：`tenant_members`、`resource_access_requests`、审批策略/执行状态字段、审计结果字段、ACL 生命周期字段、项目/租户软删除字段、数据转换任务租户字段。 +- 已有用户已补齐到 `tenant_members`，当前迁移后共生成 5 条租户成员关系。 +- `backend/app/db/sql/000_full_init.sql` 已合并完整结构；离线目录继续只保留该完整脚本。 + +### 验证结果 + +- 后端相关模块 `python -m py_compile`：通过。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 路径提示和大 chunk 警告，无 TypeScript 错误。 +- `docker/offline/src` 的后端权限代码、Compute Agent、前端源码和完整初始化 SQL 已与主工程关键文件哈希一致。 +- 运行中的 Backend、Frontend、Compute API 容器已重启，健康检查通过；当前容器采用源码挂载方式，无需重建镜像即可加载本轮代码。已验证健康接口可用，未登录访问受保护接口返回 401；完整双用户、跨租户和多 GPU 回归仍待执行。 + +## 一、检查依据 + +本次按代码和初始化脚本逐项核对，主要依据如下： + +- `docs/permissions-design.md` +- `backend/app/core/auth.py` +- `backend/app/api/v1/endpoints/platform.py` +- `backend/app/modules/resource/router.py` +- `backend/app/modules/approval/router.py` +- `backend/app/modules/tenant/router.py` +- `backend/app/modules/system/router.py` +- `backend/app/db/platform_store.py` +- `backend/app/db/sql/000_full_init.sql` +- `frontend/src/stores/auth.ts` +- `frontend/src/router/index.ts` +- `frontend/src/views/governance/ResourceAclView.vue` + +检查口径不是只判断“是否有接口”，还检查了接口是否验证当前用户、是否校验资源所有权和 ACL、是否校验租户边界、审批结果是否真正改变授权、前端按钮是否与后端动作一致。 + +## 二、改造前总体进度（历史基线） + +| 能力模块 | 当前状态 | 结论 | +|---|---|---| +| 登录、密码、Token 会话 | 已实现基础能力 | 有会话过期、注销、状态校验和登录限流基础，仍需统一续期和失败审计 | +| 平台角色与权限码 | 部分实现 | `admin/operator/viewer` 和业务权限码存在，但后端部分业务只校验登录，未统一校验权限码 | +| 用户创建与管理 | 部分实现 | 管理员可创建、修改、删除和重置密码；租户范围、角色边界、邀请/申请流程未完成 | +| 租户与配额 | 部分实现 | 租户 CRUD、配额和留存策略接口为管理员专属；缺少租户成员、租户管理员和统一隔离 | +| 资源 ACL | 已实现基础能力 | 支持用户/角色的 `read/write/execute/download/delete/admin`，授权边界和跨租户限制不足 | +| 资源申请 | 未完整实现 | 没有独立的资源访问申请对象，现有审批实例不能可靠地落地 ACL 或配额变更 | +| 资源审批 | 部分实现且存在安全缺口 | 模板、实例、步骤和部分高风险操作存在；审批决策接口必须先修复越权问题 | +| 基座模型权限 | 平台共享已实现 | 登录用户可查看和使用，上传/修改/删除限制管理员；缺少按租户/用途/配额的精细控制 | +| 训练模型权限 | 部分实现 | 所有者、ACL、删除、合并、推理加载已有校验；导出下载动作和派生权限继承还不完整 | +| 数据集权限 | 部分实现且入口不一致 | 列表、详情、删除、部分下载和训练/评测使用有校验；上传、编辑、预览、版本接口仍有缺口 | +| 训练/评测/推理权限 | 部分实现 | 资源执行权和 GPU 分配已有基础校验；部分聊天、状态和历史入口没有统一鉴权 | +| GPU 与节点权限 | 基础能力已实现 | 管理员分配、用户可用 GPU 过滤、原子预留和释放已存在；配额审批和跨节点回收对账仍需完善 | +| 前端按钮级权限 | 部分实现 | 页面和菜单有基础控制，资源动作没有集中式权限决策，很多按钮依赖后端报错 | +| 审计与软删除 | 部分实现 | 主要写操作有审计，资源软删除已覆盖若干表；actor、下载、拒绝和跨租户查询仍需统一 | + +## 三、已经实现的内容 + +### 3.1 认证、角色与用户 + +- `get_current_user` 会校验 Bearer Token、用户存在性、用户状态和会话有效期；`sessions` 支持注销和过期时间。 +- `require_admin` 对管理员专属接口提供后端保护，受保护用户也具备管理员旁路能力。 +- 用户列表、创建、修改、删除、重置密码均已增加管理员依赖；用户可修改自己的密码。 +- 用户记录包含 `tenant_id`、`role`、`permissions`、`protected` 等字段，角色权限在初始化脚本中有基础种子数据。 +- 登录失败限流和旧明文密码升级已经具备基础实现。 + +### 3.2 租户、资源和 ACL + +- `tenants` 表和租户 CRUD、配额、留存策略接口已经存在，当前接口统一要求管理员。 +- 数据集、模型、训练模型、评测任务等核心资源已经具备 `created_by`、`tenant_id` 或任务 payload 中的归属信息。 +- `acls` 表支持用户和角色两类授权主体，权限包括 `read`、`write`、`execute`、`download`、`delete`、`admin`。 +- 资源 ACL 查询和全量替换接口已经存在，资源所有者或管理员可以管理 ACL。 +- 列表接口对数据集、评测、训练和推理任务已增加“本人资源 + ACL 授权资源”的过滤逻辑。 +- MinIO 预签名、资源清单、对象列表、模型制品、模型血缘和评测报告等新入口已增加登录和资源访问校验。 + +### 3.3 训练、评测、推理和 GPU + +- 训练创建会检查训练数据集的 `execute` 权限，并对用户选择的 GPU 执行分配校验。 +- 评测创建会检查训练模型、数据集的 `execute` 权限，并校验算力节点和 GPU。 +- 推理任务加载会检查任务及训练模型的执行权，同时使用 `gpu_reservations` 防止同一张 GPU 被并发占用。 +- GPU 分配接口和用户可用 GPU 查询已经存在，失败、停止、删除、节点异常等路径具备基础释放逻辑。 +- 训练模型删除、训练任务停止/删除、评测任务删除和推理任务删除已接入部分审批拦截。 +- 权重合并、导出、缓存准备和 MinIO 归档已经能记录模型、节点、租户和部分血缘信息。 + +### 3.4 审计和前端 + +- `audit_logs`、`operation_logs` 表及管理员查询/导出页面已经存在。 +- 资源 ACL 变更、租户管理、模型/数据集/任务等主要写操作已经接入审计或操作日志装饰器。 +- 前端路由和侧边栏对治理、组织、资源授权、审批、日志、算力节点等页面做了管理员限制。 +- `ResourceAclView` 已支持资源类型、用户/角色主体和多权限编辑。 + +## 四、改造前未实现或存在明显安全缺口（历史基线） + +> 本节保留本轮改造前的检查快照，不代表当前状态。当前剩余问题以“十、当前仍需开发的功能”和“十一、下一步开发计划”为准。 + +以下项目属于必须继续开发的内容，优先级高于页面样式和性能优化。 + +### 4.1 审批决策不能直接信任请求参数 + +`backend/app/modules/approval/router.py` 的审批决策接口当前没有 `get_current_user` 依赖，并且从请求体读取 `approver_id`。调用方可以伪造审批人 ID，属于高风险越权问题。 + +必须改为：服务端从当前会话取得审批人；校验其是否为当前步骤指定审批人、租户管理员或平台管理员；校验当前步骤、实例状态和申请人不能自审；审批通过后再执行对应动作或写入 ACL。 + +### 4.2 资源申请流程尚未形成 + +当前有 `approval_templates`、`approval_instances`、`approval_steps`，但没有独立的资源访问申请记录，也没有统一的“申请资源 -> 审批 -> 授权/配额变更”事务流程： + +- 创建审批实例时未统一验证资源是否存在、申请人是否属于资源租户、申请动作是否合法。 +- 审批实例批准后不会自动创建 ACL、GPU 分配或租户配额变更。 +- 不能表达申请权限、申请期限、申请原因、审批后的 ACL 权限和撤销时间。 +- 审批模板查询和详情接口没有完整的租户/角色范围控制。 + +### 4.3 租户隔离不是全量强制 + +- 当前用户只有单一 `tenant_id`，没有 `tenant_members` 或租户角色关系；无法支持租户管理员、跨租户平台管理员和成员邀请的清晰边界。 +- `filter_accessible_resource_ids` 和批量版本主要按 ACL/所有者过滤，不在统一函数中校验资源租户。 +- 数据集、评测等查询仍对 `tenant_id IS NULL` 做兼容放行；历史数据未完成归属迁移。 +- `trained_models()` 没有统一 tenant 参数，资源过滤依赖上层二次处理。 +- ACL 设置接口没有校验授权主体和资源是否属于同一租户，存在跨租户授权风险。 +- 新建模型、数据集、任务虽然多数会补默认租户，但缺少“租户必须存在且处于 active”的统一校验。 + +### 4.4 数据集权限入口不一致 + +核心列表、详情、删除、部分下载已校验，但以下文件/版本接口当前未统一接入当前用户和资源权限： + +- 文件预览和记录来源查询。 +- 文件版本列表、版本内容查询。 +- 创建、激活、删除数据集文件版本。 +- 数据集上传接口没有统一的当前用户、写权限和租户校验。 +- 数据集编辑接口没有统一的当前用户和写权限校验。 + +此外，数据集下载和单文件下载目前检查的是 `read`，没有严格使用设计中的独立 `download` 权限。 + +### 4.5 模型使用、导出和推理入口不一致 + +- 基座模型按平台共享资源处理，登录用户可以查看和使用；但模型名称查询、本地模型列表和部分本地聊天/状态/卸载入口缺少统一鉴权。 +- 训练模型列表、详情、合并和加载已有 ACL 校验，但导出接口实际属于下载/外发动作，不能只检查 `execute`。 +- 评测报告下载当前检查 `read`，应单独检查 `download` 并记录下载审计。 +- 训练模型由训练任务生成时，尚未完整实现“基座模型 + 数据集授权关系”向派生模型权限和血缘策略的统一继承。 +- 推理聊天接口没有始终绑定到已授权的推理任务、模型资源和指定算力节点，存在绕过模型使用流程的风险。 + +### 4.6 用户创建和角色边界不完整 + +- 当前只有平台管理员创建用户，普通用户不能申请加入租户，也没有邀请、审批、禁用后会话清理的完整流程。 +- 用户只能直接挂在一个 `tenant_id` 上，不能表达一个用户属于多个租户或在不同租户中拥有不同角色。 +- 后端 `create_user` 对非管理员角色会归一为普通用户，前端出现的 `operator` 角色与后端实际行为可能不一致。 +- 创建用户时缺少租户存在性、租户状态、租户用户配额和角色白名单校验。 +- 管理员可以创建管理员角色，尚未区分平台管理员和租户管理员的授予权限。 + +### 4.7 前端按钮级权限没有闭环 + +- 前端已有菜单和路由权限基础，但 `hasPermission` 没有覆盖所有业务路由动作，部分业务页对已登录用户直接开放。 +- 资源级按钮主要靠资源对象中的所有者字段判断，没有统一使用后端返回的 ACL 决策。 +- 下载、执行、删除、授权、导出、审批等动作没有统一的 `can(resource, action)` 机制。 +- 即使按钮隐藏，前端 API 模块仍可能直接调用敏感接口；必须以后端鉴权为最终边界。 + +### 4.8 审计、软删除和权限拒绝记录不完整 + +- 部分模块的 `_actor` 直接保存 Authorization Token，而不是规范的用户 ID，导致审计主体不一致。 +- 访问、下载、导出、ACL 授权、审批拒绝、GPU 分配和权限拒绝没有全部统一记录租户、资源和结果。 +- `record_visit` 公开接口容易被伪造；如果继续保留匿名访问，应明确它不是安全审计日志。 +- 资源软删除已覆盖模型、训练模型、数据集、评测任务等主要表，但关联 MinIO 对象、ACL、审批和血缘的延迟清理策略还不完整。 + +## 五、需要优化的现有流程 + +### 5.1 统一权限模型 + +将当前分散的 `is_admin`、所有者判断、ACL 查询、租户判断、GPU 判断收敛为统一服务： + +```text +认证 -> 平台/租户角色 -> 租户边界 -> 资源所有权/ACL -> 动作权限 -> GPU/配额 -> 审批 -> 审计 +``` + +每个敏感接口都应明确动作，例如 `dataset.read`、`dataset.download`、`dataset.execute`、`trained_model.export`、`inference.load`、`gpu.reserve`，禁止只使用“已登录”作为业务权限。 + +### 5.2 重新设计资源申请和审批 + +推荐流程： + +```text +申请人选择资源和动作 + -> 校验申请人所属租户和基础权限 + -> 创建 resource_access_requests + -> 根据租户/资源类型匹配审批策略 + -> 指定审批人完成审批 + -> 事务内写入 ACL/配额/GPU 预留 + -> 记录授权有效期、来源和审计 +``` + +审批拒绝、撤回、过期和资源删除都应撤销或冻结对应授权，不能仅修改审批实例状态。 + +### 5.3 模型、数据集、训练任务联合授权 + +训练、评测、推理分别使用以下最小权限： + +| 场景 | 必须具备的权限 | +|---|---| +| 查看基座模型 | `model.read`；基座模型默认平台共享 | +| 使用基座模型训练 | `model.execute` 或平台共享策略 + 数据集 `execute` | +| 下载基座模型 | 单独的 `model.download`，默认不授予 | +| 使用训练模型推理 | `trained_model.execute` | +| 下载/导出训练模型 | `trained_model.download` 或 `trained_model.export` | +| 使用数据集训练/评测 | `dataset.execute` | +| 查看数据集 | `dataset.read` | +| 下载数据集文件 | `dataset.download` | +| 创建训练任务 | 上述资源权限 + 指定节点/GPU 使用权 + 租户配额 | + +训练模型应保留创建者、租户、基座模型、数据集、训练任务和资源版本快照。派生模型默认只对创建者和同租户授权，不应因为基座模型共享而自动公开训练产物。 + +## 六、建议的数据库改造 + +当前 `000_full_init.sql` 已包含用户、角色、会话、ACL、租户、审批、审计、GPU 分配和 GPU 预留表，但要完成权限 2.0，建议增加或扩展以下结构。实施时必须同步主工程和 `docker/offline/src/backend/app/db/sql/000_full_init.sql`。 + +1. `tenant_members`：用户、租户、租户角色、状态、加入来源和有效期，解决一个用户多租户和租户管理员问题。 +2. `resource_access_requests`：申请人、租户、资源、动作、申请原因、申请权限、有效期、审批状态和最终授权记录。 +3. `acls` 增加 `tenant_id`、`granted_by`、`source_request_id`、`expires_at`、`revoked_at`，保留授权来源和自动过期能力。 +4. `approval_templates` 增加 `tenant_id`、`action`、`resource_type`、`scope`、`status`；审批步骤增加主体类型、主体 ID 和租户范围。 +5. 资源表统一补齐非空租户策略、归属用户、软删除字段和必要索引；历史 NULL 数据通过一次性迁移处理。 +6. `audit_logs` 增加结果、拒绝原因、request_id、认证会话和结构化 detail，避免把 Token 当作 actor。 + +不建议把完整 ACL 和审批状态继续塞入模型、数据集或任务 JSON 字段；JSON 可保留兼容信息，但权限判断应以结构化表为准。 + +## 七、历史开发计划（已执行） + +### 第一阶段：P0 安全修复 + +- 修复审批决策鉴权和审批人伪造问题。 +- 补齐数据集文件/版本/上传/编辑接口权限。 +- 补齐本地模型聊天、状态、卸载、模型名称查询等历史入口鉴权。 +- 统一 `download`、`execute`、`write`、`delete` 动作检查。 +- 限制 ACL 授权范围，校验主体存在、同租户和资源归属。 +- 增加至少 20 个后端权限回归用例，覆盖未登录、本人、同租户他人、跨租户、管理员和已撤销 ACL。 + +### 第二阶段：P1 租户和资源申请 + +- 引入租户成员和租户角色，明确平台管理员、租户管理员、成员、只读成员边界。 +- 开发资源访问申请接口和前端申请页面。 +- 重做审批模板匹配、审批人校验、审批结果落地 ACL、有效期和撤销流程。 +- 将配额申请、GPU 分配申请、模型导出和跨用户删除纳入审批策略。 +- 新资源强制租户归属并完成历史数据迁移。 + +### 第三阶段：P1 联合资源权限 + +- 建立模型/数据集/训练任务/评测任务/推理任务的统一资源授权服务。 +- 训练前一次性校验基座模型、数据集、节点、GPU 和租户配额。 +- 训练模型生成时写入血缘和权限来源；推理、评测、合并、导出使用同一套动作权限。 +- 版本快照、MinIO 对象、算力节点本地缓存沿用同一资源授权结果。 + +### 第四阶段：P2 前端和审计 + +- 增加统一 `can(resource, action)` 和按钮权限组件。 +- 授权和审批界面使用用户/租户/资源中文名称，展示授权来源、有效期和审批状态。 +- 规范审计 actor、租户、请求 ID、资源、动作、结果和失败原因。 +- 将权限不足、审批中、资源过期、MinIO 不可用分别展示，避免全部显示为通用 500。 + +### 第五阶段：P3 测试与上线 + +- 新库初始化、增量迁移、离线目录同步和前端构建全部纳入 CI。 +- 执行跨租户、跨用户、ACL 撤销、审批越权、软删除、MinIO 故障和 GPU 冲突专项测试。 +- 在第二个可达节点或多卡节点到位后，执行多节点、多 GPU 的权限和并发测试。 +- 补充权限运维手册：新建租户、创建用户、授权模型/数据集、审批、撤销权限、审计追踪和故障恢复。 + +## 八、验收标准 + +- 未登录用户不能访问任何模型、数据集、任务、聊天、版本、下载和审批接口。 +- 用户只能访问所属租户中本人拥有或被明确授权的资源；跨租户 ACL 默认禁止。 +- 查看、下载、执行、编辑、删除和授权是独立动作，不能用 `read` 替代所有动作。 +- 普通用户不能伪造审批人、审批其他租户资源或审批自己的申请。 +- 训练、评测和推理创建必须同时满足资源权限、GPU 权限、节点权限和租户配额。 +- 审批通过后才产生授权，审批拒绝、撤回、过期和资源删除后授权不会继续生效。 +- 基座模型共享不等于训练产物共享；训练后的模型和数据集必须按租户、所有者和 ACL 控制。 +- 前端隐藏按钮不能替代后端校验，直接调用 API 也必须返回明确的 401/403。 +- 所有敏感操作可通过用户、租户、资源、动作和请求 ID追溯到审计记录。 + +## 九、历史实施记录（已完成） + +本轮已完成权限闭环的代码实现： + +1. 统一校验当前会话、会话过期、退出状态和可用租户范围。 +2. 补齐模型、数据集、训练、评测、推理、数据处理、数据转换、算力、存储和审计入口的登录及模块权限。 +3. 新增 `tenant_members`，新建用户、项目、数据集、训练任务和数据处理任务写入当前租户与创建者。 +4. ACL 写入校验主体存在性、状态、租户边界、有效期和撤销状态；普通所有者不能授予 `delete/admin` 或跨租户权限。 +5. 新增 `resource_access_requests`；审批人由当前会话确定，禁止伪造审批人和申请人自审；审批通过后才落 ACL。 +6. 训练、评测和推理统一校验数据集、基座模型、训练模型、任务、节点和 GPU 使用权限；下载和导出使用独立的 `download` 权限。 +7. 前端认证 store 新增 `can(resource, action)`，模型管理和审批页面按权限显示操作按钮，后端 401/403 仍是最终防线。 +8. 新增 `009_permission_completion.sql`；主工程与离线目录的完整初始化 SQL 已同步且 SHA-256 一致，离线目录只保留完整初始化脚本。 + +### 本轮验证 + +- 后端权限相关文件 `py_compile` 通过。 +- 前端 `npm run build` 通过；仅保留已有字体路径和 chunk size 警告。 +- 当前 WSL 容器已确认 Backend、Frontend、Compute、Redis、MinIO 均运行；基础健康和未登录拦截已验证，历史治理测试夹具与当前鉴权/数据库行为不完全兼容，不能替代新的权限专项回归。 + +### 上线前验证 + +- 第二个可达节点或多卡节点到位后的多租户、多 GPU 并行压力测试。 +- 对已执行的 `009_permission_completion.sql` 进行旧数据租户归属、ACL、审批和软删除记录对账。 +- 更新 Backend/Frontend 容器后重新执行治理测试和权限专项测试。 + +## 十、当前复核结论（2026-08-19） + +### 10.1 已完成的基础能力 + +本轮 11 项权限改造已经形成基础闭环：会话和用户状态校验、租户成员关系、资源 ACL、资源访问申请、审批决策安全、模型/数据集/训练/评测/推理入口权限、GPU 分配申请、配额申请、软删除和前端基础按钮控制均已落地；数据库迁移和完整初始化 SQL 已同步到主工程及离线目录。 + +这些能力可以作为后续完善的基础，但“接口存在”不等于“所有资源和所有异常路径均已达到上线标准”。尤其是密钥暴露、配额实际拦截、资源列表一致性和专项测试仍需要继续处理。 + +### 10.2 仍需开发的功能 + +| 优先级 | 功能 | 当前缺口 | 处理结论 | +|---|---|---|---| +| P0 | 模型密钥和敏感信息保护 | 模型列表、详情、名称查询、创建和更新响应已移除真实 `api_key`，仍需继续排查任务详情、日志和其他敏感配置输出 | 基础闭环已完成；继续做全量敏感字段扫描和受控密钥管理 | +| P0 | 统一资源动作策略 | 已增加资源动作白名单和 `export -> download` 归一化，但各业务入口仍需逐步迁移到统一授权服务 | 第一阶段已完成；继续完成全量入口收敛 | +| P0 | 数据转换、数据处理和存储权限一致性 | 部分列表、文件、预览、版本、缓存和 MinIO 对象入口仍需逐接口核对“租户 + 所有者/ACL + 动作” | 必须完成全量入口审计，尤其是 `read/download/execute/write/delete` 的区分 | +| P0 | 审计失败链路 | 审计装饰器已记录失败结果、原因、请求 ID、会话 ID 和租户；手工审计入口仍需继续统一 | 基础闭环已完成；继续补齐所有权限拒绝和异常入口 | +| P1 | 租户成员生命周期 | 已有成员表和角色，但缺少邀请、加入申请、审批、过期、移除和前端租户切换的完整流程 | 需要开发，明确平台管理员与租户管理员的授予边界 | +| P1 | 配额强制执行 | 配额申请和审批已实现，但训练、评测、推理、存储等资源消耗尚未全部进行原子配额检查和扣减 | 需要开发配额使用量/预留量/释放量账本，不能只保存配额配置 | +| P1 | GPU 分配强校验 | 申请链路已实现，但分配前仍需统一校验节点状态、GPU 存在性、冲突、租户配额和释放对账 | 需要开发原子预留、超时回收和节点故障对账 | +| P1 | 审批策略完整性 | 需要严格限制动作白名单、资源类型、模板作用域和重复申请；过期处理不应只依赖查询触发 | 需要补充定时过期、幂等键、执行失败重试和策略管理边界 | +| P1 | 派生模型和数据血缘授权 | 基座模型、数据集、训练模型之间已有部分血缘，但权限来源、版本快照和派生资源默认授权规则还不够统一 | 需要固化“创建者 + 租户 + 显式 ACL”，禁止共享基座模型自动公开训练产物 | +| P1 | 用户软删除和对象清理 | 用户及部分关联关系仍有物理删除风险；MinIO 对象、ACL、审批、缓存的异步清理缺少统一编排 | 需要补齐软删除、撤销、延迟清理和失败重试 | +| P2 | 前端资源级权限体验 | 已有菜单/按钮级基础控制，但缺少统一 `can(resource, action)`、授权来源、有效期、审批中和 403 状态展示 | 需要开发统一权限组件和错误状态处理，后端校验仍是最终边界 | +| P2 | 权限专项测试与上线检查 | 基础构建、静态检查、健康接口已验证，尚未完成双用户、跨租户、撤销、过期、MinIO 故障、GPU 冲突的全流程矩阵 | 必须补充自动化测试、迁移回归和离线部署验收 | + +### 10.3 需要优化的设计 + +1. **从“角色判断”改为“动作授权”**：统一使用 `resource_type + resource_id + action + tenant_id + actor` 判断，平台管理员只作为明确的管理范围，不再作为各业务模块的隐式旁路。 +2. **区分平台角色和租户角色**：`users.role` 只表达平台级角色，`tenant_members.role` 表达租户内角色；禁止通过租户成员关系授予平台管理员权限。 +3. **区分查看、下载、执行、编辑、删除、授权**：`read` 不能替代 `download`，`execute` 不能替代 `export`，高风险动作必须绑定审批和审计。 +4. **统一资源列表和详情规则**：列表、详情、文件、版本、预览、缓存、下载和导出必须调用同一授权服务，避免“列表看不到但接口可访问”或“列表能看到但操作必然 403”。 +5. **权限与配额采用预留模型**：任务提交时原子预留 GPU、显存、并发数和存储额度，任务完成、失败、取消和节点失联时统一释放或对账。 +6. **审批采用可执行状态机**：申请、审批、拒绝、撤回、过期、执行中、执行失败、已执行状态分离；审批结果应有幂等执行记录，避免重复授权或重复分配。 +7. **敏感字段默认拒绝返回**：API key、对象内部凭据、节点访问凭据不能随普通资源详情返回；日志、审计和错误信息也不能泄露 Token、密码或完整连接串。 +8. **安全审计与访问统计分离**：权限成功、拒绝、下载、导出、授权、审批和异常进入不可篡改审计；页面访问统计单独存储，避免污染安全审计记录。 + +## 十一、下一步开发计划 + +### 阶段一：P0 安全封堵与统一策略 + +1. 对模型列表、详情、名称查询及相关 DTO 做 API key/凭据脱敏，增加“仅后端内部读取”的封装。（基础闭环已完成，继续扫描任务和日志输出） +2. 建立统一 `authorize(resource, action)` 服务，定义动作白名单和平台管理员、租户管理员、所有者、ACL 的判定顺序。（已完成动作白名单，继续迁移全部入口） +3. 逐项审计平台、数据处理、数据转换、存储、MinIO、模型、数据集、训练、评测、推理的列表、详情、预览、下载、导出、缓存和删除入口。 +4. 统一记录权限成功、拒绝和异常审计，补齐 `tenant_id`、`resource_id`、`action`、`request_id`、`session_id`、`reason` 和 `result`。（装饰器基础闭环已完成，继续覆盖手工记录入口） + +### 阶段二：租户成员、审批和配额闭环 + +1. 开发租户邀请/加入申请/审批/移除/过期流程及前端租户切换。 +2. 为审批模板增加动作和资源类型白名单、租户作用域、幂等键、过期任务和执行失败重试。 +3. 建立配额预留账本，训练、评测、推理、存储和 GPU 任务提交前统一原子检查；任务终态和故障恢复统一释放。 +4. 完善 GPU 节点、GPU 卡、租户、用户和任务的占用关系校验，覆盖冲突、超时、节点不可达和服务重启恢复。 + +### 阶段三：资源血缘和生命周期 + +1. 固化基座模型、数据集、数据处理结果、训练任务、训练模型、评测和推理任务的资源血缘及版本快照。 +2. 明确派生模型默认授权规则，训练产物不因基座模型共享而自动对外公开。 +3. 补齐用户、租户、资源、ACL、审批、MinIO 对象和本地缓存的软删除、撤销、延迟清理和失败重试。 + +### 阶段四：前端权限体验与测试 + +1. 开发统一资源级权限组件，按动作隐藏/禁用按钮，并展示授权来源、有效期、审批状态和明确的 401/403 原因。 +2. 将审批、ACL、审计列表中的用户、租户、资源 ID 映射为可读名称，同时保留 ID 作为详情字段。 +3. 编写并执行权限专项测试矩阵：未登录、同租户成员、资源所有者、ACL 授权、跨租户、禁用用户、会话注销、审批撤回/过期、软删除、MinIO 故障、GPU 冲突和配额不足。 +4. 在第二个可达节点或多卡节点准备后执行多节点、多 GPU 并发及故障恢复测试；完成主工程与 `docker/offline/src` 的源码、初始化 SQL、迁移和部署文档一致性检查。 + +## 十二、下一阶段完成标准 + +- 普通资源接口不再返回 API key、密码、Token 或节点访问凭据。 +- 所有资源入口都经过统一的租户边界和动作授权，跨租户访问返回明确 403。 +- 训练、评测、推理创建同时满足资源权限、GPU 预留和租户配额，失败时不会留下孤儿占用。 +- 审批只能由合法审批人执行，申请人不能自审；重复回调不会重复授权、分配或扣减配额。 +- 权限拒绝、下载、导出、授权、审批和异常均能按用户、租户、资源和请求 ID追溯。 +- 主工程和离线目录初始化新库均无缺表、缺字段；迁移可重复执行且不破坏既有数据。 +- 权限专项自动化测试通过，且完成至少一次双用户、跨租户和 MinIO 故障场景的真实容器验证。 + +## 十三、上一阶段权限闭环开发结果（2026-08-20） + +本轮围绕 P0 安全封堵完成了一个可验证的权限闭环： + +1. **模型凭据不出接口**：模型列表、详情、按名称查询、创建、更新和用途更新响应统一经过公开 DTO 脱敏，移除 `api_key`、密码、Token 等字段，仅返回 `api_key_configured` 布尔状态；后端内部评测、数据生成仍使用数据库中的真实配置。 +2. **编辑密钥不被意外清空**：前端编辑在线模型时不回填真实密钥，留空表示保留已有配置，只有管理员主动输入新值时才提交替换。 +3. **资源动作统一入口**：增加资源动作白名单 `read/write/execute/download/export/delete/admin`，并将 `export` 统一归一到下载权限，非法动作默认拒绝，避免把模块权限误当成资源权限。 +4. **失败审计闭环**：审计装饰器对成功和异常分别记录，失败记录包含结果、失败原因、租户、请求 ID、会话 ID和耗时，并对异常中的常见凭据进行脱敏。 +5. **访问统计受控**：模块访问统计只接受平台已登记的模块名，不能通过请求参数向安全审计表写入任意动作；审计 CSV 导出改为标准 CSV 编码，并补充结果、原因、请求和会话字段。 +6. **离线版本同步**：上述后端权限代码、前端模型编辑代码和权限安全回归用例已同步到 `docker/offline/src`；本轮未新增数据库字段，因此 `000_full_init.sql` 无需变更。 + +### 上一阶段验证结果 + +- 后端权限相关模块 `python -m py_compile`：通过。 +- WSL Backend 容器执行权限安全用例：8 passed；仅有 pytest 缓存目录只读警告。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 路径和 chunk size 警告。 +- 容器接口验证：未登录访问模型接口返回 401；管理员访问模型列表/详情时响应不包含 `api_key`，仅返回 `api_key_configured`。 +- Backend 容器已重启并加载当前源码；数据库无需迁移。 + +### 上一阶段未覆盖的范围（已在第十四节继续处理） + +本轮没有声称完成配额账本、GPU 原子预留、租户邀请/加入申请、审批定时过期、所有资源入口的逐接口审计，以及第二节点/多 GPU 的真实并发测试；这些仍按“十一、下一步开发计划”执行。 + +## 十四、本轮继续开发结果（2026-08-20） + +### 已完成 + +1. 新增 `tenant_quota_reservations` 配额预留账本，支持按租户统计 GPU 预留量、原子限制 GPU 配额，以及任务完成、失败、停止、删除和节点故障时释放预留。 +2. 训练、评测和推理统一接入租户 GPU 配额预留；评测/推理使用的 `gpu_reservations` 与配额账本在同一事务内创建或释放，避免只释放算力卡而遗留配额占用。 +3. 租户成员支持邀请、接受、过期、角色更新和移除；邀请不允许授予 `owner`，成员状态和租户有效性由后端校验，租户详情页补充成员管理和 GPU 配额使用量展示。 +4. 审批动作增加白名单，非法动作拒绝创建；相同申请人在同一资源上的待审批申请幂等复用；审批实例读取时自动处理过期状态，资源访问申请也避免重复创建。 +5. GPU 分配增加节点启用状态、GPU 卡存在性、用户 active 状态和同卡跨用户冲突校验；冲突返回 409，不再静默覆盖或产生重复授权。 +6. 完整初始化 SQL 增加配额预留表和索引，新增 `010_permission_quota_membership.sql` 增量迁移；主工程相关源码、前端租户 API/页面和 SQL 已同步到离线源码目录。 + +### 本轮验证 + +- WSL Backend 容器重启后，`tenant_quota_reservations` 可正常查询，默认租户配额使用量返回正常。 +- WSL Backend 容器执行 `test_permission_security.py`：3 passed。 +- 后端相关文件 `python -m py_compile`：通过。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 资源路径和大 chunk 警告。 + +### 必须后续验证的场景 + +1. 创建第二个 active 用户和第二个租户，验证邀请接受、租户切换、跨租户资源访问均按预期返回 401/403。 +2. 将租户 GPU 配额设为 1，同时提交两个需要 GPU 的训练/评测/推理任务，确认第二个任务被拒绝；第一个任务终态后确认配额可再次使用。 +3. 在同一多卡节点上让两个用户申请同一张卡，确认审批执行阶段冲突返回 409，另一张空闲卡仍可正常分配。 +4. 构造过期审批、重复提交、审批拒绝/撤回，确认不会重复创建 ACL、GPU 分配或配额预留。 +5. 准备第二个可达节点或多卡节点，执行训练、评测、推理的多节点/多 GPU 并发和节点失联回收测试。 +6. 注入 MinIO 故障，确认任务失败后 GPU 与租户配额均能释放，恢复 MinIO 后可以重新提交任务。 + +### 下一步计划 + +- 补齐数据处理、数据转换、MinIO 对象、缓存、版本和报告下载等剩余资源入口的逐接口动作授权审计。 +- 将用户物理删除改为统一软删除，并编排成员关系、ACL、审批、MinIO 对象和本地缓存的撤销与延迟清理。 +- 将上述后续验证场景加入自动化测试矩阵和离线部署验收脚本；在多节点条件满足后更新本文件的上线验收进度。 diff --git a/docs/database-migration.md b/docs/database-migration.md new file mode 100644 index 0000000..21badd0 --- /dev/null +++ b/docs/database-migration.md @@ -0,0 +1,38 @@ +# 数据库迁移说明 + +## 当前迁移文件 + +- 新库初始化：`backend/app/db/sql/000_full_init.sql` +- 已有数据库增量迁移：`backend/app/db/sql/005_storage_progress_migration.sql` +- GPU 预留迁移：`backend/app/db/sql/006_gpu_reservations.sql` +- 平台闭环迁移：`backend/app/db/sql/007_platform_completion.sql` +- 离线部署使用：`docker/offline/src/backend/app/db/sql/000_full_init.sql` + +## 执行方式 + +```bash +for migration in 005_storage_progress_migration.sql 006_gpu_reservations.sql 007_platform_completion.sql; do + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "backend/app/db/sql/$migration" +done +``` + +迁移前应完成数据库备份，并保存以下结构快照： + +```sql +SELECT table_name, column_name, data_type +FROM information_schema.columns +WHERE table_schema = 'public' +ORDER BY table_name, ordinal_position; +``` + +## 本次迁移内容 + +- 为算力节点资源副本增加 `version_id` 和 `storage_object_id`，用于记录 MinIO 版本与本地缓存对应关系。 +- 补齐 `storage_objects.metadata`、模型产物、数据集文件和数据转换任务的 MinIO 字段。 +- 补齐评测任务软删除、创建者、租户和报告对象字段。 +- 增加活动评测任务、资源副本和 MinIO 对象索引。 +- 增加 GPU 预留表，统一训练、推理、评测的卡级占用约束。 +- 增加模型导出创建者/租户/归档状态字段。 +- 增加 MinIO 对象软删除清理、缓存版本校验和访问保护字段及清理任务表。 + +`000_full_init.sql` 已包含相同的幂等变更，新增数据库直接执行初始化脚本即可；已有数据库不要依赖容器重启自动完成迁移。 diff --git a/docs/governance-user-guide.md b/docs/governance-user-guide.md index cc2052a..5e7d72b 100644 --- a/docs/governance-user-guide.md +++ b/docs/governance-user-guide.md @@ -110,7 +110,7 @@ | 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` | | 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` | | 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` | -| 系统设置 - 运行日志 | 系统/训练日志；管理员可查看审计记录、操作诊断 | `/logs` | +| 系统设置 - 运行日志 | 普通用户查看本人操作记录；管理员可查看系统/训练日志、审计记录和全量操作诊断 | `/logs` | | 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` | ### 2.4 重置用户密码 diff --git a/docs/permissions-design.md b/docs/permissions-design.md index 8af20be..b5b4a62 100644 --- a/docs/permissions-design.md +++ b/docs/permissions-design.md @@ -131,7 +131,9 @@ | `compute` | `/compute` | 算力节点 | | `hardware` | `/hardware` | 平台性能 | | `logs` | `/logs`, `/training-log/:id` | 查看日志 | -| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances`, `/logs` | 平台治理和运行日志（管理员） | +| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances` | 平台治理管理功能（管理员） | + +运行日志采用分层访问模型：拥有 `logs` 权限的普通用户可以进入 `/logs`，页面只展示其本人产生的操作记录，后端按当前用户 ID 强制过滤，忽略普通用户传入的跨用户筛选条件。管理员在同一入口保留系统日志、训练日志、审计记录和全量操作诊断能力。 --- diff --git a/docs/当前项目开发进度.md b/docs/当前项目开发进度.md index 06d9aa0..3792fb2 100644 --- a/docs/当前项目开发进度.md +++ b/docs/当前项目开发进度.md @@ -1,6 +1,6 @@ # 当前项目开发进度 -> 评估基线：2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。 +> 评估基线：2026-08-20 当前工作区代码、数据库初始化脚本、增量迁移脚本、Docker 部署文件、前端页面和 WSL 容器验证结果。 > > 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容，统一标记为“部分完成”或“未完成”。 @@ -24,14 +24,16 @@ FastAPI Backend API | 范围 | 当前状态 | 结论 | |---|---|---| -| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 | -| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 | +| 平台基础架构 | 已完成开发基线 | 前端、Backend、PostgreSQL、Redis、MinIO、Compute Agent 和离线 Docker 编排均已具备 | +| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测主要流程已闭环 | | 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入，跨节点一致性和失败恢复仍需加强 | -| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现，但完整的租户/项目隔离尚未闭环 | -| MinIO 统一存储 | 部分完成 | 大文件和模型已接入，仍存在兼容性的本地路径和部分数据双写/回退路径 | -| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 | +| 权限治理 | 功能闭环基本完成 | 登录、角色、权限码、ACL、审批、资源申请、GPU 申请和审计已实现，历史数据迁移与全量回归待完成 | +| MinIO 统一存储 | 基本完成 | 大文件、模型、产物、报告和节点缓存已接入；小型内容保留数据库属于明确的性能策略 | +| 生产可靠性 | 部分完成 | 重试、补偿、缓存治理和故障快速失败已实现，生命周期运维、高可用、备份和大规模压力测试未完成 | | 前端体验 | 基本可用，需优化 | 构建问题已持续修复，但页面响应等待、首屏体积和部分错误提示仍需优化 | +当前结论：项目已经进入“核心功能闭环后的专项验证和上线前加固阶段”，不再是单纯的功能骨架开发。尚不能宣称生产可上线，主要风险集中在历史数据迁移、跨节点多 GPU 并发、评测指标质量、MinIO 运维和全量回归测试。 + ## 二、已完成的功能 ### 2.1 平台基础与部署 @@ -55,6 +57,7 @@ FastAPI Backend API - 已建立资源 ACL 管理页面和相关接口，可对用户或角色授予资源级权限。 - 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。 - 已建立运行日志、审计日志查询页面及审计记录写入机制。 +- “运行日志”已按分层模型实现：普通用户只查看本人操作日志，管理员查看系统日志、训练日志、审计记录和全量操作诊断。 - 已加入软删除相关字段和部分删除逻辑，避免直接物理删除业务资源。 ### 2.3 算力节点与 GPU 资源 @@ -141,72 +144,61 @@ FastAPI Backend API - Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。 - 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段，这是当前已确认的小文件性能策略，不是错误，但必须统一记录来源、大小、校验值和版本。 -- 大文件在部分代码路径中仍通过 `read()` 或 `put_bytes()` 一次性读入内存，未完成流式或分片上传。 +- Compute Agent 节点侧的大文件上传已改为流式读取；Backend 端部分历史上传/下载路径仍未完成统一的分片传输。 - MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联，数据库没有直接外键，删除和数据一致性需要应用层保证。 - 删除业务资源后，对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。 -### 3.2 MinIO 预签名接口的权限边界需要加强 +### 3.2 MinIO 预签名接口的权限边界 -当前预签名接口已经存在，但 PUT 上传场景仍需要重点补强： - -- 需要根据资源类型和资源 ID 校验当前用户的写权限，而不应只校验读取权限。 -- 需要服务端生成并校验对象 Key，避免客户端任意写入其他用户或其他资源的对象路径。 -- 需要增加上传完成确认接口，校验对象实际存在、大小和校验值后再写入业务表。 -- 需要限制允许的 Bucket、Content-Type、大小和有效期。 -- 需要记录预签名创建、上传完成、失败和过期事件，便于审计。 +资源写权限、服务端对象 Key、上传完成确认和对象大小校验已经完成；仍需补充 Content-Type 白名单、对象过期回收和完整审计事件。 ### 3.3 激活版本和跨节点资源版本仍需加强 数据集已经有 `active_version_id` 和版本表，但以下场景仍需补充： -- 训练、推理和评测必须只使用资源当前激活版本，并在任务创建时固化版本 ID。 +- 训练、评测、模型对比和权重合并已在任务创建时固化资源版本 ID；推理服务启动和导出任务仍需统一补齐。 - 同一文件名的不同版本不能只依靠文件名同步，应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。 - 已创建任务在后续切换激活版本后，不能被意外切换到新版本。 -- 需要为每个准备到算力节点的资源保存版本、对象 ETag/校验值和本地路径清单。 +- 已为资源副本保存版本、对象 ID 和本地路径；对象 ETag/校验值及多文件 manifest 仍需补齐。 - 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。 -### 3.4 权限 2.0 尚未完全落地 +### 3.4 权限 2.0 已基本闭环，仍需历史迁移和全量回归 -已有用户、角色、权限码、ACL、审批和审计基础，但仍存在以下差距： +已有用户、角色、权限码、ACL、审批、资源申请、GPU 申请、审计和普通用户自助日志能力，仍存在以下差距： -- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。 -- 项目空间设计已经讨论过取消，但数据库中仍保留 `projects`、`project_members` 等历史结构，需要明确兼容策略和最终迁移方式。 -- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。 -- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。 -- 前端按钮权限已经有基础实现，但不能替代后端鉴权；仍需要对所有关键动作进行后端默认拒绝校验。 -- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。 +- 历史数据中的 NULL 租户归属仍需盘点和迁移，之后再增加更严格的数据库约束。 +- 项目空间已从前端和新业务流程移除，但 `projects`、`project_members` 等历史表仍需兼容迁移和下线方案。 +- 训练、推理、评测、模型合并、导出、缓存准备和下载接口仍需执行全量跨用户/跨租户回归。 +- 资源动作鉴权已经分散接入，仍需继续统一授权函数和后端默认拒绝策略。 +- 审批策略、配额、GPU 预留和资源 ACL 的组合规则需要继续用自动化矩阵验证。 -### 3.5 模型合并、导出和评测报告闭环不足 +### 3.5 模型合并、导出和评测报告已基本闭环 -- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立，但失败时的清理、重试和幂等性仍需加强。 -- 合并结果归档到 MinIO 的逻辑主要依赖任务完成轮询，服务重启或轮询中断时可能需要补偿扫描。 -- 模型导出任务目前有查询模型和表结构，但完整的创建、执行、进度、失败重试和下载闭环尚未完成。 -- 评测结果和报告字段已经存在，但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。 -- 评测指标配置和执行器返回指标之间仍需要强类型映射，避免前端显示为通用的 `custom`。 +- 权重合并前自动准备 Base Model 和 Adapter、失败清理、重试和 MinIO 归档路径已经接入，仍需进行更多幂等和长任务测试。 +- 模型导出已经具备创建、节点准备、CPU 执行、制品血缘、归档轮询、权限和前端入口，真实大模型多格式压力测试待完成。 +- 评测报告接口已支持 MinIO 流式返回和历史数据库兼容结果，但报告为空、报告版本和下载流程仍需回归。 +- 评测指标配置和执行器返回指标之间仍需强类型映射，避免前端显示为通用的 `custom`。 -### 3.6 GPU 资源分配需要统一到所有任务类型 +### 3.6 GPU 资源分配已统一接入，待多节点多卡验证 -- 训练已经有较完整的节点/GPU 选择和预检流程。 -- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码，但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。 -- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。 -- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。 -- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报，仍需要校验采样时间、单位、空值和任务对应关系。 +- 训练、推理和评测已经接入节点/GPU 选择、原子预留和释放流程。 +- 单节点同卡冲突已经验证；多节点、多 GPU 并行和长时间压力仍需真实环境验证。 +- 需要继续处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。 +- 训练详情中的显存使用量、GPU 使用率等指标需要继续校验采样时间、单位、空值和任务对应关系。 -## 四、尚未完成的功能 +## 四、待完成或待专项验证的功能 -以下功能在当前代码中没有形成可验收的完整闭环，或仍处于设计/基础代码阶段： +以下内容已经有代码基础或单节点验证，但尚未达到上线验收条件： -1. **完整的租户隔离和资源继承模型**：所有列表、详情、下载、缓存、训练、推理、评测和导出接口都需要统一的租户范围过滤。 -2. **项目取消后的正式数据迁移方案**：需要决定历史项目数据如何归属到用户或租户，并提供一次性迁移脚本和回滚方案。 -3. **预签名上传完成确认和对象校验**：包括 Key 白名单、ACL、大小限制、哈希/ETag 和状态回写。 -4. **MinIO 对象生命周期管理**：软删除后的延迟删除、失败重试、孤儿对象扫描、对象引用检查和管理员清理入口。 -5. **Compute Agent 缓存治理**：容量上限、LRU/TTL、运行任务保护、磁盘占用监控、缓存清单和版本校验。 -6. **统一的资源归档编排器**：训练、合并、评测和推理相关产物需要支持断点恢复、幂等重试和服务重启补偿。 -7. **模型导出完整流程**：导出任务创建、格式/量化参数、进度、失败重试、MinIO 归档和下载权限。 -8. **流式和分片文件传输**：避免大文件上传、下载和对象复制时将完整内容读入 Backend 或 Compute Agent 内存。 -9. **生产级 MinIO 安全和高可用**：默认密钥替换、TLS、网络访问控制、管理员 Console 隔离、容量监控、备份和恢复。 -10. **完整的端到端测试和持续集成**：至少覆盖单节点、多节点、多 GPU、跨用户、跨租户、版本切换、MinIO 不可用和服务重启恢复。 -11. **统一数据库迁移体系**：当前初始化 SQL 适合新库初始化，但尚未替代正式的版本化迁移工具；已有数据库更新仍需要明确迁移脚本和执行记录。 +1. **历史租户和项目数据迁移**：新数据已补齐租户/ACL 基础校验，历史 NULL 租户和项目兼容表仍需迁移、校验和下线方案。 +2. **MinIO 生命周期运维**：预签名安全、软删除、失败记录、重试、孤儿扫描和管理员清理接口已存在，自动定时清理和运行监控仍需接入。 +3. **Compute Agent 缓存生产治理**：容量上限、LRU、TTL、保护窗口、版本/校验清单已实现，运行中任务动态保护、磁盘告警和大规模缓存压力待验证。 +4. **多节点、多 GPU 并行**：节点/GPU 选择、原子预留、冲突拒绝和释放已实现，真实多节点、多卡并发尚未执行。 +5. **模型导出和评测报告质量**：创建、执行、归档、权限和下载接口已建立，真实大模型多格式导出、报告版本和指标质量仍需专项验证。 +6. **超大文件传输**：单文件流式下载和 Compute Agent 流式上传已完成，多文件 ZIP、分片上传和断点续传未完成。 +7. **生产级 MinIO 安全和高可用**：开发环境接入和故障快速失败已完成，TLS、密钥管理、Console 隔离、容量监控、备份恢复需在生产阶段实施。 +8. **全量端到端测试和持续集成**：已有前端构建、Backend 编译和存储安全 CI 基线，权限全矩阵、跨节点并行和故障注入仍需扩展。 +9. **正式数据库迁移体系**：已形成 `005` 至 `012` 幂等增量迁移和运行时兼容执行，仍需增加版本记录、迁移前检查和离线升级演练。 ## 五、需要优化的功能 @@ -265,6 +257,8 @@ FastAPI Backend API - `data_convert_tasks.storage_backend` - `eval_tasks.report_storage_object_id` +当前增量迁移脚本为 `005_storage_progress_migration.sql` 至 `012_tenant_user_hierarchy.sql`，已覆盖存储进度、GPU 预留、平台补全、权限 2.0、配额/成员关系、权限生命周期和租户/用户层级等结构。 + 离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意： - 初始化 SQL 主要用于新数据库或新数据卷；已有数据库不能仅靠重启容器自动获得全部新字段。 @@ -274,6 +268,18 @@ FastAPI Backend API ## 七、当前验证结果 +### 7.1 2026-08-19 本轮按计划落地内容 + +- P0 MinIO 预签名上传已增加资源存在性、资源写权限、管理员基座模型写权限、资源版本和对象 Key 前缀校验；新增上传完成确认接口，会校验 MinIO 对象存在、大小和状态后再标记为可用。 +- 训练、评测、模型对比和权重合并任务会保存创建时的租户信息与资源版本快照，后续切换数据集激活版本不会改变已创建任务的输入版本。 +- 数据集、评测任务、训练任务和模型对比列表增加租户范围过滤；用户表、模型/数据集创建入口补齐租户归属；数据转换资源补充所有者权限映射。 +- Compute Agent 准备缓存后会回写资源副本的版本、MinIO 对象和本地路径；缓存支持可选容量上限、按访问时间淘汰非临时文件，并提供当前占用量和上限状态。 +- 训练产物、权重合并结果和评测报告增加 MinIO 归档、失败状态记录以及 Backend 重启后的补偿轮询；离线部署源码已同步对应逻辑。 +- Compute Agent 大文件上传已改为流式读取，避免将整个文件一次性读入内存；离线 Compute Agent 同步完成。 +- 增加 `005_storage_progress_migration.sql` 增量迁移脚本，并修复 `000_full_init.sql` 中旧数据库执行时索引早于字段补齐的问题；主工程和离线初始化脚本已同步。 +- 增加 `MINIO_PRESIGN_MAX_BYTES` 配置和上传 Content-Type 白名单；首次数据库 schema 检查移出 Compute Poller 的 Uvicorn 事件循环，避免远程 PostgreSQL 慢连接拖垮健康检查；轮询相同失败改为 300 秒限频记录，并跳过已标记 offline 节点。 +- 前端 `npm run build` 已通过；容器内 MinIO Key 越权校验专项测试为 `5 passed`，Backend 和 Compute Agent 重启后均为 healthy。 + 已完成的静态和局部验证： - Backend 和离线 Backend 源码 `compileall` 检查通过。 @@ -281,45 +287,80 @@ FastAPI Backend API - 主工程和离线包初始化 SQL 已做同步检查，内容一致。 - 前端此前已完成 `npm run build` 类型错误修复，构建剩余问题主要是非阻断的资源/分包警告。 - 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。 +- 当前 WSL 运行验证中 Backend、Compute Agent、MinIO 均为 healthy；`gpu-node-02`（`172.25.179.69:19100`）连接、GPU 0 分配和任务执行均正常。`gpu-node-01` 的历史地址不可达，已通过健康检查标记为 offline，轮询器不再重复轮询其历史任务。 + +### 7.2 2026-08-19 gpu-node-02 真实流程验证 + +- 训练：使用 `qwen3.5-0.8B` + `test_data_0817`，任务 `ft_172a2da65140` 完成，GPU 0 使用期间可看到显存、利用率、温度和进程，结束后恢复 idle；24 个训练产物已归档 MinIO。 +- 资源快照：任务 `ft_d93b0fdae1c9` 创建时已保存数据集 `ds_8f6b8c5714c7` 的活动版本 `file_d550c2128722_v1`。 +- 训练曲线：任务 `ft_d93b0fdae1c9` 通过 `logging_steps=1` 生成 3 个 loss/epoch/learning-rate/grad-norm 数据点，并生成 `training_loss.png`；后端解析器已兼容带引号数字格式。 +- 权重合并：任务 `merge_1dc15a290810` 完成，基座模型路径、合并路径已回写，8 个合并模型文件归档 MinIO。 +- 推理：任务 `cmp_abdf0762869d` 在 GPU 0 加载 `qwen3.5-0.8B` 成功，对话接口返回正常；卸载后 GPU 恢复 idle。 +- 评测：任务 `eval_3c3f84263e23` 完成 2 条样本评测，报告、样本明细和基础指标已落库；GPU 恢复 idle。评审模型 HTTP 400 导致评审分数为 0，属于评审模型接口配置问题，算力评测链路本身已跑通。 + +### 7.3 多 GPU、MinIO 故障和跨用户权限专项验证 + +- 多 GPU 调度开发：新增 `gpu_reservations` 表，评测和推理在远程提交/加载前与训练统一纳入数据库原子预留；失败、停止、删除、节点不可达和卸载路径自动释放预留。当前 `gpu-node-02` 只有 GPU 0，已通过同卡“推理预留后评测抢占”测试，评测被明确拒绝，释放后 GPU 恢复 idle。多节点、多卡的真实并行测试待增加第二个可达节点和多卡节点后执行。 +- 调度锁优化：调度锁改为事务内持有并在提交前释放，避免一次调度成功后后续请求等待 30 秒 TTL；schema 初始化增加 PostgreSQL advisory lock，避免轮询器与首个请求并发初始化造成死锁。 +- MinIO 故障注入：停止 `yg-ft-minio` 后，`GET /modelTF/health` 返回 HTTP 200 且 `storage.status=unavailable`；恢复容器后返回 `storage.status=ready`。MinIO 客户端关闭默认重试链，故障健康探测耗时从约 6 秒降至约 0.3 秒。 +- 跨用户权限：创建临时用户 `qa_user_0819`，未授权访问管理员数据集返回 403；授予资源 `read/download` ACL 后列表和详情可访问；撤销 ACL 后再次返回 403；非管理员创建用户返回 403。测试用户已删除，未遗留测试 ACL 或 GPU 预留。 +- 用户管理接口已补齐管理员依赖，创建、列表、修改、删除和重置密码不再允许普通用户调用。 当前不能据此宣称“全量功能测试通过”： - 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设，需要按当前分层存储策略更新。 -- WSL Docker 运行时验证受当前环境的 `E_ACCESSDENIED` 影响，不能在本次文档生成时完成全部容器健康、数据库字段和跨节点测试。 -- 多节点、多 GPU、MinIO 临时不可用、服务重启恢复和跨用户权限测试仍需要在可用运行环境中执行。 +- 本轮已完成当前 WSL Docker 容器健康检查、`gpu-node-02` 单节点真实流程、单卡冲突、MinIO 故障恢复和跨用户 ACL 专项验证；多节点、多卡的真实并行测试仍需要第二个可达节点和多卡节点。 + +- 本轮专项安全测试为 `5 passed`；全量 pytest 仍未执行完毕，需要按测试类别拆分并设置超时。 + +### 7.4 本轮 11 项未验证能力的补齐结果 + +- 租户管理接口已统一要求管理员身份；模型制品、模型血缘、导出任务、评测报告和 MinIO 资源清单均增加资源级访问校验。 +- 新增 `POST /modelTF/model-manage/export`，导出沿用节点选择、MinIO 缓存准备、GPU/调度约束和归档轮询；导出结果记录到 `model_export_jobs`，并建立导出制品与模型血缘。 +- 新增 `GET /modelTF/model-eval/{task_id}/report`，优先流式返回 MinIO 报告，历史任务无归档对象时返回数据库报告兼容结果。 +- 新增 `GET /modelTF/storage/resources/{resource_type}/{resource_id}/manifest`、管理员对象清理和孤儿扫描接口；预签名上传增加过期时间和真实 SHA-256 内容校验。 +- Compute Agent 增加缓存 TTL、缓存保护窗口和元数据清单；超过 TTL 的非保护缓存会在状态检查时清理，容量淘汰会跳过保护中的资源。 +- 数据集单文件下载改为 MinIO 流式传输，避免 Backend 一次性载入完整大文件；训练、合并、导出、评测仍由统一轮询器负责重启后的归档补偿。 +- 新增 `007_platform_completion.sql`，并已加入运行时兼容迁移；主工程和 `docker/offline/src` 的源码及初始化 SQL 已同步。 +- 以上为代码闭环和单节点验证；生产级 MinIO TLS/HA、第二节点/多卡并行压力测试、全量 CI 和历史项目数据正式迁移仍属于上线前专项工作。 + +### 7.5 2026-08-20 普通用户运行日志验证 + +- 已修复普通用户进入“运行日志”调用管理员专属 `/log-files` 接口导致 403 的问题。 +- 普通用户现在查看本人操作日志，后端按当前用户 ID 强制过滤，忽略普通用户传入的跨用户 `user_id` 条件。 +- 管理员仍可查看系统日志、训练日志、审计记录和全量操作诊断。 +- 临时普通用户创建数据转换任务后验证：操作日志查询和统计均返回 200，返回记录全部属于该用户；测试用户和测试资源已清理。 +- 前端 `npm run build` 再次通过，离线前端静态资源和源码已同步，离线 Frontend/Backend 容器健康。 ## 八、下一阶段开发计划 -### P0：安全与数据正确性 +### P0：权限与租户隔离安全闭环 -1. 完善 MinIO 预签名 PUT 的资源写权限、对象 Key 白名单、大小/类型限制和上传完成确认。 -2. 统一任务创建时的资源版本固化，训练、推理、评测只使用已授权的激活版本快照。 -3. 逐一补齐评测、推理、模型合并、模型导出、缓存准备的后端权限校验和审计记录。 -4. 完成租户隔离查询范围，清理或兼容历史项目字段，补充数据迁移脚本。 +1. 对现有数据库执行结构差异检查，盘点所有 `tenant_id`、创建者、ACL、资源版本和历史 NULL 数据。 +2. 编写历史租户归属迁移和项目兼容数据迁移脚本，迁移前生成结构/数据快照，迁移后增加一致性检查。 +3. 建立接口权限矩阵，覆盖数据集下载、模型使用、推理、评测、权重合并、导出、缓存准备、审批和运行日志。 +4. 补齐跨用户、跨租户、已撤销 ACL、软删除资源和审批未通过场景的自动化测试。 -### P1：跨节点可靠性 +### P1：资源申请、审批和联合权限 -1. 建立资源清单/manifest，记录 MinIO 对象版本、校验值、目标节点路径和缓存状态。 -2. 完善训练、合并、评测和推理的准备、执行、归档、失败重试和服务重启补偿。 -3. 完善 GPU 原子分配、异常回收、节点重连对账和任务释放。 -4. 增加缓存容量、TTL/LRU、运行任务保护和磁盘占用监控。 -5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。 +1. 对资源申请、审批通过后的 ACL 写入、配额扣减和 GPU 预留执行事务一致性回归，重点验证重复审批和撤回。 +2. 验证租户管理员、普通成员、只读成员和平台管理员在跨租户资源、模型、数据集和算力上的差异。 +3. 验证训练创建的基座模型、数据集、GPU、训练产物之间的联合授权和血缘快照。 +4. 统一审批策略、资源 ACL、租户配额和 GPU 预留的错误提示，避免只返回 403/500。 -### P2：性能与用户体验 +### P2：前端权限体验与审计完善 -1. 优化页面列表接口和高频轮询，采用批量查询、短期缓存和动态退避。 -2. 将大文件上传/下载/复制改为流式或分片传输。 -3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。 -4. 处理前端离线资源警告，继续拆分首屏 chunk。 -5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。 +1. 完善页面级权限与资源动作权限的统一组件，重点覆盖下载、导出、执行、删除、授权和审批按钮。 +2. 运行日志保持普通用户只看本人、管理员看全量的分层模型，并补充权限拒绝和下载/导出审计字段。 +3. 优化模型推理、模型评测、训练详情、数据集和运行日志页面的首屏 loading、超时、重试和空状态。 +4. 减少列表接口重复查询，继续拆分首屏大 chunk，移除离线环境不需要的外部资源请求。 -### P3：工程化和上线准备 +### P3：可靠性、测试和上线准备 -1. 建立正式数据库版本迁移机制和离线升级脚本。 -2. 增加 CI：前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。 -3. 增加多节点端到端测试和 MinIO 故障注入测试。 -4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。 -5. 建立生产运行手册，包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。 +1. 准备第二个可达节点或多卡节点，执行训练、推理、评测并发、节点不可达和服务重启测试。 +2. 接入 MinIO 对象生命周期定时清理、容量监控、TLS、密钥管理、备份恢复和故障演练。 +3. 建立正式数据库迁移版本记录、迁移前结构检查、离线升级和回滚演练。 +4. 将前端构建、Backend 编译、权限安全、存储安全和端到端流程测试分层纳入 CI。 ## 九、阶段验收标准 diff --git a/docs/权限开发进度.md b/docs/权限开发进度.md new file mode 100644 index 0000000..38fa584 --- /dev/null +++ b/docs/权限开发进度.md @@ -0,0 +1,493 @@ +# 权限开发进度 + +> 更新时间：2026-08-20 +> 适用范围：YG_FT 平台当前主工程、Backend、Frontend、Compute Agent、MinIO 资源访问及 `docker/offline/src` 离线源码。 +> 当前结论：权限 2.0 的核心功能已形成闭环。本轮继续补齐租户成员角色在 ACL 中的实际判定、有效期租户隔离、用户软删除、资源/MinIO 对象待清理、训练模型预加载资源存在性校验和推理状态接口授权，并已执行在线数据库迁移。自动化权限/存储安全用例、前端构建、后端静态检查、容器健康和用户删除闭环均已验证。由于当前只有一个可达算力节点，跨租户双用户、第二节点/多卡并发、节点故障回收和 MinIO 故障注入仍需专项验证，因此保留为上线前场景。按工作包估算，核心权限开发约完成 98%，上线验收约完成 82%。 + +## 本轮 11 项完成情况（2026-08-19） + +| 编号 | 权限能力 | 完成内容 | 状态 | +|---|---|---|---| +| 1 | 审批决策安全 | 审批人从当前会话获取，校验指定审批人、租户管理员、禁止申请人自审，记录审批审计 | 已完成 | +| 2 | 资源访问申请 | 新增资源访问申请记录、申请权限/期限/理由、审批后自动写入 ACL、支持撤回和过期 | 已完成 | +| 3 | 审批策略执行 | 按租户、动作、资源类型匹配策略；高风险操作审批通过后可一次性重试执行 | 已完成 | +| 4 | 租户与项目隔离 | `tenant_members`、活跃租户校验、历史资源归属补齐、跨租户 ACL 主体校验、项目软删除 | 已完成 | +| 5 | 用户与角色边界 | 禁用用户立即注销会话；租户成员支持 owner/admin/member/viewer，保护最后一个 owner | 已完成 | +| 6 | 数据集权限入口 | 预览、来源、版本查询/创建/激活/删除、上传/编辑/下载统一接入资源动作权限 | 已完成 | +| 7 | 模型和推理权限 | 训练模型合并/导出分别校验 execute/download；聊天、预加载、批量、卸载绑定授权模型或推理任务 | 已完成 | +| 8 | GPU 分配申请 | 普通用户可提交 GPU 分配申请，审批通过后自动分配；管理员可直接分配 | 已完成 | +| 9 | 租户配额申请 | 租户成员可提交配额变更申请，平台管理员审批后自动更新配额 | 已完成 | +| 10 | 软删除与安全状态 | 资源删除撤销 ACL、取消待处理申请和审批；租户/项目采用软删除状态 | 已完成 | +| 11 | 前端按钮与审计 | 审批决策、ACL 编辑、访问申请/撤回按权限显示；下载、导出、审批、GPU 等敏感操作补充结构化审计 | 已完成 | + +### 数据库迁移结果 + +- 新增增量迁移：`backend/app/db/sql/009_permission_completion.sql`，可独立兼容旧库执行，并已在当前远程 PostgreSQL 执行成功。 +- 当前远程库已核验包含：`tenant_members`、`resource_access_requests`、审批策略/执行状态字段、审计结果字段、ACL 生命周期字段、项目/租户软删除字段、数据转换任务租户字段。 +- 已有用户已补齐到 `tenant_members`，当前迁移后共生成 5 条租户成员关系。 +- `backend/app/db/sql/000_full_init.sql` 已合并完整结构；离线目录继续只保留该完整脚本。 + +### 验证结果 + +- 后端相关模块 `python -m py_compile`：通过。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 路径提示和大 chunk 警告，无 TypeScript 错误。 +- `docker/offline/src` 的后端权限代码、Compute Agent、前端源码和完整初始化 SQL 已与主工程关键文件哈希一致。 +- 运行中的 Backend、Frontend、Compute API 容器已重启，健康检查通过；当前容器采用源码挂载方式，无需重建镜像即可加载本轮代码。已验证健康接口可用，未登录访问受保护接口返回 401；完整双用户、跨租户和多 GPU 回归仍待执行。 + +## 一、检查依据 + +本次按代码和初始化脚本逐项核对，主要依据如下： + +- `docs/permissions-design.md` +- `backend/app/core/auth.py` +- `backend/app/api/v1/endpoints/platform.py` +- `backend/app/modules/resource/router.py` +- `backend/app/modules/approval/router.py` +- `backend/app/modules/tenant/router.py` +- `backend/app/modules/system/router.py` +- `backend/app/db/platform_store.py` +- `backend/app/db/sql/000_full_init.sql` +- `frontend/src/stores/auth.ts` +- `frontend/src/router/index.ts` +- `frontend/src/views/governance/ResourceAclView.vue` + +检查口径不是只判断“是否有接口”，还检查了接口是否验证当前用户、是否校验资源所有权和 ACL、是否校验租户边界、审批结果是否真正改变授权、前端按钮是否与后端动作一致。 + +## 二、改造前总体进度（历史基线） + +| 能力模块 | 当前状态 | 结论 | +|---|---|---| +| 登录、密码、Token 会话 | 已实现基础能力 | 有会话过期、注销、状态校验和登录限流基础，仍需统一续期和失败审计 | +| 平台角色与权限码 | 部分实现 | `admin/operator/viewer` 和业务权限码存在，但后端部分业务只校验登录，未统一校验权限码 | +| 用户创建与管理 | 部分实现 | 管理员可创建、修改、删除和重置密码；租户范围、角色边界、邀请/申请流程未完成 | +| 租户与配额 | 部分实现 | 租户 CRUD、配额和留存策略接口为管理员专属；缺少租户成员、租户管理员和统一隔离 | +| 资源 ACL | 已实现基础能力 | 支持用户/角色的 `read/write/execute/download/delete/admin`，授权边界和跨租户限制不足 | +| 资源申请 | 未完整实现 | 没有独立的资源访问申请对象，现有审批实例不能可靠地落地 ACL 或配额变更 | +| 资源审批 | 部分实现且存在安全缺口 | 模板、实例、步骤和部分高风险操作存在；审批决策接口必须先修复越权问题 | +| 基座模型权限 | 平台共享已实现 | 登录用户可查看和使用，上传/修改/删除限制管理员；缺少按租户/用途/配额的精细控制 | +| 训练模型权限 | 部分实现 | 所有者、ACL、删除、合并、推理加载已有校验；导出下载动作和派生权限继承还不完整 | +| 数据集权限 | 部分实现且入口不一致 | 列表、详情、删除、部分下载和训练/评测使用有校验；上传、编辑、预览、版本接口仍有缺口 | +| 训练/评测/推理权限 | 部分实现 | 资源执行权和 GPU 分配已有基础校验；部分聊天、状态和历史入口没有统一鉴权 | +| GPU 与节点权限 | 基础能力已实现 | 管理员分配、用户可用 GPU 过滤、原子预留和释放已存在；配额审批和跨节点回收对账仍需完善 | +| 前端按钮级权限 | 部分实现 | 页面和菜单有基础控制，资源动作没有集中式权限决策，很多按钮依赖后端报错 | +| 审计与软删除 | 部分实现 | 主要写操作有审计，资源软删除已覆盖若干表；actor、下载、拒绝和跨租户查询仍需统一 | + +## 三、已经实现的内容 + +### 3.1 认证、角色与用户 + +- `get_current_user` 会校验 Bearer Token、用户存在性、用户状态和会话有效期；`sessions` 支持注销和过期时间。 +- `require_admin` 对管理员专属接口提供后端保护，受保护用户也具备管理员旁路能力。 +- 用户列表、创建、修改、删除、重置密码均已增加管理员依赖；用户可修改自己的密码。 +- 用户记录包含 `tenant_id`、`role`、`permissions`、`protected` 等字段，角色权限在初始化脚本中有基础种子数据。 +- 登录失败限流和旧明文密码升级已经具备基础实现。 + +### 3.2 租户、资源和 ACL + +- `tenants` 表和租户 CRUD、配额、留存策略接口已经存在，当前接口统一要求管理员。 +- 数据集、模型、训练模型、评测任务等核心资源已经具备 `created_by`、`tenant_id` 或任务 payload 中的归属信息。 +- `acls` 表支持用户和角色两类授权主体，权限包括 `read`、`write`、`execute`、`download`、`delete`、`admin`。 +- 资源 ACL 查询和全量替换接口已经存在，资源所有者或管理员可以管理 ACL。 +- 列表接口对数据集、评测、训练和推理任务已增加“本人资源 + ACL 授权资源”的过滤逻辑。 +- MinIO 预签名、资源清单、对象列表、模型制品、模型血缘和评测报告等新入口已增加登录和资源访问校验。 + +### 3.3 训练、评测、推理和 GPU + +- 训练创建会检查训练数据集的 `execute` 权限，并对用户选择的 GPU 执行分配校验。 +- 评测创建会检查训练模型、数据集的 `execute` 权限，并校验算力节点和 GPU。 +- 推理任务加载会检查任务及训练模型的执行权，同时使用 `gpu_reservations` 防止同一张 GPU 被并发占用。 +- GPU 分配接口和用户可用 GPU 查询已经存在，失败、停止、删除、节点异常等路径具备基础释放逻辑。 +- 训练模型删除、训练任务停止/删除、评测任务删除和推理任务删除已接入部分审批拦截。 +- 权重合并、导出、缓存准备和 MinIO 归档已经能记录模型、节点、租户和部分血缘信息。 + +### 3.4 审计和前端 + +- `audit_logs`、`operation_logs` 表及管理员查询/导出页面已经存在。 +- 资源 ACL 变更、租户管理、模型/数据集/任务等主要写操作已经接入审计或操作日志装饰器。 +- 前端路由和侧边栏对治理、组织、资源授权、审批、日志、算力节点等页面做了管理员限制。 +- `ResourceAclView` 已支持资源类型、用户/角色主体和多权限编辑。 + +## 四、改造前未实现或存在明显安全缺口（历史基线） + +> 本节保留本轮改造前的检查快照，不代表当前状态。当前剩余问题以“十、当前仍需开发的功能”和“十一、下一步开发计划”为准。 + +以下项目属于必须继续开发的内容，优先级高于页面样式和性能优化。 + +### 4.1 审批决策不能直接信任请求参数 + +`backend/app/modules/approval/router.py` 的审批决策接口当前没有 `get_current_user` 依赖，并且从请求体读取 `approver_id`。调用方可以伪造审批人 ID，属于高风险越权问题。 + +必须改为：服务端从当前会话取得审批人；校验其是否为当前步骤指定审批人、租户管理员或平台管理员；校验当前步骤、实例状态和申请人不能自审；审批通过后再执行对应动作或写入 ACL。 + +### 4.2 资源申请流程尚未形成 + +当前有 `approval_templates`、`approval_instances`、`approval_steps`，但没有独立的资源访问申请记录，也没有统一的“申请资源 -> 审批 -> 授权/配额变更”事务流程： + +- 创建审批实例时未统一验证资源是否存在、申请人是否属于资源租户、申请动作是否合法。 +- 审批实例批准后不会自动创建 ACL、GPU 分配或租户配额变更。 +- 不能表达申请权限、申请期限、申请原因、审批后的 ACL 权限和撤销时间。 +- 审批模板查询和详情接口没有完整的租户/角色范围控制。 + +### 4.3 租户隔离不是全量强制 + +- 当前用户只有单一 `tenant_id`，没有 `tenant_members` 或租户角色关系；无法支持租户管理员、跨租户平台管理员和成员邀请的清晰边界。 +- `filter_accessible_resource_ids` 和批量版本主要按 ACL/所有者过滤，不在统一函数中校验资源租户。 +- 数据集、评测等查询仍对 `tenant_id IS NULL` 做兼容放行；历史数据未完成归属迁移。 +- `trained_models()` 没有统一 tenant 参数，资源过滤依赖上层二次处理。 +- ACL 设置接口没有校验授权主体和资源是否属于同一租户，存在跨租户授权风险。 +- 新建模型、数据集、任务虽然多数会补默认租户，但缺少“租户必须存在且处于 active”的统一校验。 + +### 4.4 数据集权限入口不一致 + +核心列表、详情、删除、部分下载已校验，但以下文件/版本接口当前未统一接入当前用户和资源权限： + +- 文件预览和记录来源查询。 +- 文件版本列表、版本内容查询。 +- 创建、激活、删除数据集文件版本。 +- 数据集上传接口没有统一的当前用户、写权限和租户校验。 +- 数据集编辑接口没有统一的当前用户和写权限校验。 + +此外，数据集下载和单文件下载目前检查的是 `read`，没有严格使用设计中的独立 `download` 权限。 + +### 4.5 模型使用、导出和推理入口不一致 + +- 基座模型按平台共享资源处理，登录用户可以查看和使用；但模型名称查询、本地模型列表和部分本地聊天/状态/卸载入口缺少统一鉴权。 +- 训练模型列表、详情、合并和加载已有 ACL 校验，但导出接口实际属于下载/外发动作，不能只检查 `execute`。 +- 评测报告下载当前检查 `read`，应单独检查 `download` 并记录下载审计。 +- 训练模型由训练任务生成时，尚未完整实现“基座模型 + 数据集授权关系”向派生模型权限和血缘策略的统一继承。 +- 推理聊天接口没有始终绑定到已授权的推理任务、模型资源和指定算力节点，存在绕过模型使用流程的风险。 + +### 4.6 用户创建和角色边界不完整 + +- 当前只有平台管理员创建用户，普通用户不能申请加入租户，也没有邀请、审批、禁用后会话清理的完整流程。 +- 用户只能直接挂在一个 `tenant_id` 上，不能表达一个用户属于多个租户或在不同租户中拥有不同角色。 +- 后端 `create_user` 对非管理员角色会归一为普通用户，前端出现的 `operator` 角色与后端实际行为可能不一致。 +- 创建用户时缺少租户存在性、租户状态、租户用户配额和角色白名单校验。 +- 管理员可以创建管理员角色，尚未区分平台管理员和租户管理员的授予权限。 + +### 4.7 前端按钮级权限没有闭环 + +- 前端已有菜单和路由权限基础，但 `hasPermission` 没有覆盖所有业务路由动作，部分业务页对已登录用户直接开放。 +- 资源级按钮主要靠资源对象中的所有者字段判断，没有统一使用后端返回的 ACL 决策。 +- 下载、执行、删除、授权、导出、审批等动作没有统一的 `can(resource, action)` 机制。 +- 即使按钮隐藏，前端 API 模块仍可能直接调用敏感接口；必须以后端鉴权为最终边界。 + +### 4.8 审计、软删除和权限拒绝记录不完整 + +- 部分模块的 `_actor` 直接保存 Authorization Token，而不是规范的用户 ID，导致审计主体不一致。 +- 访问、下载、导出、ACL 授权、审批拒绝、GPU 分配和权限拒绝没有全部统一记录租户、资源和结果。 +- `record_visit` 公开接口容易被伪造；如果继续保留匿名访问，应明确它不是安全审计日志。 +- 资源软删除已覆盖模型、训练模型、数据集、评测任务等主要表，但关联 MinIO 对象、ACL、审批和血缘的延迟清理策略还不完整。 + +## 五、需要优化的现有流程 + +### 5.1 统一权限模型 + +将当前分散的 `is_admin`、所有者判断、ACL 查询、租户判断、GPU 判断收敛为统一服务： + +```text +认证 -> 平台/租户角色 -> 租户边界 -> 资源所有权/ACL -> 动作权限 -> GPU/配额 -> 审批 -> 审计 +``` + +每个敏感接口都应明确动作，例如 `dataset.read`、`dataset.download`、`dataset.execute`、`trained_model.export`、`inference.load`、`gpu.reserve`，禁止只使用“已登录”作为业务权限。 + +### 5.2 重新设计资源申请和审批 + +推荐流程： + +```text +申请人选择资源和动作 + -> 校验申请人所属租户和基础权限 + -> 创建 resource_access_requests + -> 根据租户/资源类型匹配审批策略 + -> 指定审批人完成审批 + -> 事务内写入 ACL/配额/GPU 预留 + -> 记录授权有效期、来源和审计 +``` + +审批拒绝、撤回、过期和资源删除都应撤销或冻结对应授权，不能仅修改审批实例状态。 + +### 5.3 模型、数据集、训练任务联合授权 + +训练、评测、推理分别使用以下最小权限： + +| 场景 | 必须具备的权限 | +|---|---| +| 查看基座模型 | `model.read`；基座模型默认平台共享 | +| 使用基座模型训练 | `model.execute` 或平台共享策略 + 数据集 `execute` | +| 下载基座模型 | 单独的 `model.download`，默认不授予 | +| 使用训练模型推理 | `trained_model.execute` | +| 下载/导出训练模型 | `trained_model.download` 或 `trained_model.export` | +| 使用数据集训练/评测 | `dataset.execute` | +| 查看数据集 | `dataset.read` | +| 下载数据集文件 | `dataset.download` | +| 创建训练任务 | 上述资源权限 + 指定节点/GPU 使用权 + 租户配额 | + +训练模型应保留创建者、租户、基座模型、数据集、训练任务和资源版本快照。派生模型默认只对创建者和同租户授权，不应因为基座模型共享而自动公开训练产物。 + +## 六、建议的数据库改造 + +当前 `000_full_init.sql` 已包含用户、角色、会话、ACL、租户、审批、审计、GPU 分配和 GPU 预留表，但要完成权限 2.0，建议增加或扩展以下结构。实施时必须同步主工程和 `docker/offline/src/backend/app/db/sql/000_full_init.sql`。 + +1. `tenant_members`：用户、租户、租户角色、状态、加入来源和有效期，解决一个用户多租户和租户管理员问题。 +2. `resource_access_requests`：申请人、租户、资源、动作、申请原因、申请权限、有效期、审批状态和最终授权记录。 +3. `acls` 增加 `tenant_id`、`granted_by`、`source_request_id`、`expires_at`、`revoked_at`，保留授权来源和自动过期能力。 +4. `approval_templates` 增加 `tenant_id`、`action`、`resource_type`、`scope`、`status`；审批步骤增加主体类型、主体 ID 和租户范围。 +5. 资源表统一补齐非空租户策略、归属用户、软删除字段和必要索引；历史 NULL 数据通过一次性迁移处理。 +6. `audit_logs` 增加结果、拒绝原因、request_id、认证会话和结构化 detail，避免把 Token 当作 actor。 + +不建议把完整 ACL 和审批状态继续塞入模型、数据集或任务 JSON 字段；JSON 可保留兼容信息，但权限判断应以结构化表为准。 + +## 七、历史开发计划（已执行） + +### 第一阶段：P0 安全修复 + +- 修复审批决策鉴权和审批人伪造问题。 +- 补齐数据集文件/版本/上传/编辑接口权限。 +- 补齐本地模型聊天、状态、卸载、模型名称查询等历史入口鉴权。 +- 统一 `download`、`execute`、`write`、`delete` 动作检查。 +- 限制 ACL 授权范围，校验主体存在、同租户和资源归属。 +- 增加至少 20 个后端权限回归用例，覆盖未登录、本人、同租户他人、跨租户、管理员和已撤销 ACL。 + +### 第二阶段：P1 租户和资源申请 + +- 引入租户成员和租户角色，明确平台管理员、租户管理员、成员、只读成员边界。 +- 开发资源访问申请接口和前端申请页面。 +- 重做审批模板匹配、审批人校验、审批结果落地 ACL、有效期和撤销流程。 +- 将配额申请、GPU 分配申请、模型导出和跨用户删除纳入审批策略。 +- 新资源强制租户归属并完成历史数据迁移。 + +### 第三阶段：P1 联合资源权限 + +- 建立模型/数据集/训练任务/评测任务/推理任务的统一资源授权服务。 +- 训练前一次性校验基座模型、数据集、节点、GPU 和租户配额。 +- 训练模型生成时写入血缘和权限来源；推理、评测、合并、导出使用同一套动作权限。 +- 版本快照、MinIO 对象、算力节点本地缓存沿用同一资源授权结果。 + +### 第四阶段：P2 前端和审计 + +- 增加统一 `can(resource, action)` 和按钮权限组件。 +- 授权和审批界面使用用户/租户/资源中文名称，展示授权来源、有效期和审批状态。 +- 规范审计 actor、租户、请求 ID、资源、动作、结果和失败原因。 +- 将权限不足、审批中、资源过期、MinIO 不可用分别展示，避免全部显示为通用 500。 + +### 第五阶段：P3 测试与上线 + +- 新库初始化、增量迁移、离线目录同步和前端构建全部纳入 CI。 +- 执行跨租户、跨用户、ACL 撤销、审批越权、软删除、MinIO 故障和 GPU 冲突专项测试。 +- 在第二个可达节点或多卡节点到位后，执行多节点、多 GPU 的权限和并发测试。 +- 补充权限运维手册：新建租户、创建用户、授权模型/数据集、审批、撤销权限、审计追踪和故障恢复。 + +## 八、验收标准 + +- 未登录用户不能访问任何模型、数据集、任务、聊天、版本、下载和审批接口。 +- 用户只能访问所属租户中本人拥有或被明确授权的资源；跨租户 ACL 默认禁止。 +- 查看、下载、执行、编辑、删除和授权是独立动作，不能用 `read` 替代所有动作。 +- 普通用户不能伪造审批人、审批其他租户资源或审批自己的申请。 +- 训练、评测和推理创建必须同时满足资源权限、GPU 权限、节点权限和租户配额。 +- 审批通过后才产生授权，审批拒绝、撤回、过期和资源删除后授权不会继续生效。 +- 基座模型共享不等于训练产物共享；训练后的模型和数据集必须按租户、所有者和 ACL 控制。 +- 前端隐藏按钮不能替代后端校验，直接调用 API 也必须返回明确的 401/403。 +- 所有敏感操作可通过用户、租户、资源、动作和请求 ID追溯到审计记录。 + +## 九、历史实施记录（已完成） + +本轮已完成权限闭环的代码实现： + +1. 统一校验当前会话、会话过期、退出状态和可用租户范围。 +2. 补齐模型、数据集、训练、评测、推理、数据处理、数据转换、算力、存储和审计入口的登录及模块权限。 +3. 新增 `tenant_members`，新建用户、项目、数据集、训练任务和数据处理任务写入当前租户与创建者。 +4. ACL 写入校验主体存在性、状态、租户边界、有效期和撤销状态；普通所有者不能授予 `delete/admin` 或跨租户权限。 +5. 新增 `resource_access_requests`；审批人由当前会话确定，禁止伪造审批人和申请人自审；审批通过后才落 ACL。 +6. 训练、评测和推理统一校验数据集、基座模型、训练模型、任务、节点和 GPU 使用权限；下载和导出使用独立的 `download` 权限。 +7. 前端认证 store 新增 `can(resource, action)`，模型管理和审批页面按权限显示操作按钮，后端 401/403 仍是最终防线。 +8. 新增 `009_permission_completion.sql`；主工程与离线目录的完整初始化 SQL 已同步且 SHA-256 一致，离线目录只保留完整初始化脚本。 + +### 本轮验证 + +- 后端权限相关文件 `py_compile` 通过。 +- 前端 `npm run build` 通过；仅保留已有字体路径和 chunk size 警告。 +- 当前 WSL 容器已确认 Backend、Frontend、Compute、Redis、MinIO 均运行；基础健康和未登录拦截已验证，历史治理测试夹具与当前鉴权/数据库行为不完全兼容，不能替代新的权限专项回归。 + +### 上线前验证 + +- 第二个可达节点或多卡节点到位后的多租户、多 GPU 并行压力测试。 +- 对已执行的 `009_permission_completion.sql` 进行旧数据租户归属、ACL、审批和软删除记录对账。 +- 更新 Backend/Frontend 容器后重新执行治理测试和权限专项测试。 + +## 十、当前复核结论（2026-08-19） + +### 10.1 已完成的基础能力 + +本轮 11 项权限改造已经形成基础闭环：会话和用户状态校验、租户成员关系、资源 ACL、资源访问申请、审批决策安全、模型/数据集/训练/评测/推理入口权限、GPU 分配申请、配额申请、软删除和前端基础按钮控制均已落地；数据库迁移和完整初始化 SQL 已同步到主工程及离线目录。 + +这些能力可以作为后续完善的基础，但“接口存在”不等于“所有资源和所有异常路径均已达到上线标准”。尤其是密钥暴露、配额实际拦截、资源列表一致性和专项测试仍需要继续处理。 + +### 10.2 仍需开发的功能 + +| 优先级 | 功能 | 当前缺口 | 处理结论 | +|---|---|---|---| +| P0 | 模型密钥和敏感信息保护 | 模型列表、详情、名称查询、创建和更新响应已移除真实 `api_key`，仍需继续排查任务详情、日志和其他敏感配置输出 | 基础闭环已完成；继续做全量敏感字段扫描和受控密钥管理 | +| P0 | 统一资源动作策略 | 已增加资源动作白名单和 `export -> download` 归一化，但各业务入口仍需逐步迁移到统一授权服务 | 第一阶段已完成；继续完成全量入口收敛 | +| P0 | 数据转换、数据处理和存储权限一致性 | 部分列表、文件、预览、版本、缓存和 MinIO 对象入口仍需逐接口核对“租户 + 所有者/ACL + 动作” | 必须完成全量入口审计，尤其是 `read/download/execute/write/delete` 的区分 | +| P0 | 审计失败链路 | 审计装饰器已记录失败结果、原因、请求 ID、会话 ID 和租户；手工审计入口仍需继续统一 | 基础闭环已完成；继续补齐所有权限拒绝和异常入口 | +| P1 | 租户成员生命周期 | 已有成员表和角色，但缺少邀请、加入申请、审批、过期、移除和前端租户切换的完整流程 | 需要开发，明确平台管理员与租户管理员的授予边界 | +| P1 | 配额强制执行 | 配额申请和审批已实现，但训练、评测、推理、存储等资源消耗尚未全部进行原子配额检查和扣减 | 需要开发配额使用量/预留量/释放量账本，不能只保存配额配置 | +| P1 | GPU 分配强校验 | 申请链路已实现，但分配前仍需统一校验节点状态、GPU 存在性、冲突、租户配额和释放对账 | 需要开发原子预留、超时回收和节点故障对账 | +| P1 | 审批策略完整性 | 需要严格限制动作白名单、资源类型、模板作用域和重复申请；过期处理不应只依赖查询触发 | 需要补充定时过期、幂等键、执行失败重试和策略管理边界 | +| P1 | 派生模型和数据血缘授权 | 基座模型、数据集、训练模型之间已有部分血缘，但权限来源、版本快照和派生资源默认授权规则还不够统一 | 需要固化“创建者 + 租户 + 显式 ACL”，禁止共享基座模型自动公开训练产物 | +| P1 | 用户软删除和对象清理 | 用户及部分关联关系仍有物理删除风险；MinIO 对象、ACL、审批、缓存的异步清理缺少统一编排 | 需要补齐软删除、撤销、延迟清理和失败重试 | +| P2 | 前端资源级权限体验 | 已有菜单/按钮级基础控制，但缺少统一 `can(resource, action)`、授权来源、有效期、审批中和 403 状态展示 | 需要开发统一权限组件和错误状态处理，后端校验仍是最终边界 | +| P2 | 权限专项测试与上线检查 | 基础构建、静态检查、健康接口已验证，尚未完成双用户、跨租户、撤销、过期、MinIO 故障、GPU 冲突的全流程矩阵 | 必须补充自动化测试、迁移回归和离线部署验收 | + +### 10.3 需要优化的设计 + +1. **从“角色判断”改为“动作授权”**：统一使用 `resource_type + resource_id + action + tenant_id + actor` 判断，平台管理员只作为明确的管理范围，不再作为各业务模块的隐式旁路。 +2. **区分平台角色和租户角色**：`users.role` 只表达平台级角色，`tenant_members.role` 表达租户内角色；禁止通过租户成员关系授予平台管理员权限。 +3. **区分查看、下载、执行、编辑、删除、授权**：`read` 不能替代 `download`，`execute` 不能替代 `export`，高风险动作必须绑定审批和审计。 +4. **统一资源列表和详情规则**：列表、详情、文件、版本、预览、缓存、下载和导出必须调用同一授权服务，避免“列表看不到但接口可访问”或“列表能看到但操作必然 403”。 +5. **权限与配额采用预留模型**：任务提交时原子预留 GPU、显存、并发数和存储额度，任务完成、失败、取消和节点失联时统一释放或对账。 +6. **审批采用可执行状态机**：申请、审批、拒绝、撤回、过期、执行中、执行失败、已执行状态分离；审批结果应有幂等执行记录，避免重复授权或重复分配。 +7. **敏感字段默认拒绝返回**：API key、对象内部凭据、节点访问凭据不能随普通资源详情返回；日志、审计和错误信息也不能泄露 Token、密码或完整连接串。 +8. **安全审计与访问统计分离**：权限成功、拒绝、下载、导出、授权、审批和异常进入不可篡改审计；页面访问统计单独存储，避免污染安全审计记录。 + +## 十一、下一步开发计划 + +### 阶段一：P0 安全封堵与统一策略 + +1. 对模型列表、详情、名称查询及相关 DTO 做 API key/凭据脱敏，增加“仅后端内部读取”的封装。（基础闭环已完成，继续扫描任务和日志输出） +2. 建立统一 `authorize(resource, action)` 服务，定义动作白名单和平台管理员、租户管理员、所有者、ACL 的判定顺序。（已完成动作白名单，继续迁移全部入口） +3. 逐项审计平台、数据处理、数据转换、存储、MinIO、模型、数据集、训练、评测、推理的列表、详情、预览、下载、导出、缓存和删除入口。 +4. 统一记录权限成功、拒绝和异常审计，补齐 `tenant_id`、`resource_id`、`action`、`request_id`、`session_id`、`reason` 和 `result`。（装饰器基础闭环已完成，继续覆盖手工记录入口） + +### 阶段二：租户成员、审批和配额闭环 + +1. 开发租户邀请/加入申请/审批/移除/过期流程及前端租户切换。 +2. 为审批模板增加动作和资源类型白名单、租户作用域、幂等键、过期任务和执行失败重试。 +3. 建立配额预留账本，训练、评测、推理、存储和 GPU 任务提交前统一原子检查；任务终态和故障恢复统一释放。 +4. 完善 GPU 节点、GPU 卡、租户、用户和任务的占用关系校验，覆盖冲突、超时、节点不可达和服务重启恢复。 + +### 阶段三：资源血缘和生命周期 + +1. 固化基座模型、数据集、数据处理结果、训练任务、训练模型、评测和推理任务的资源血缘及版本快照。 +2. 明确派生模型默认授权规则，训练产物不因基座模型共享而自动对外公开。 +3. 补齐用户、租户、资源、ACL、审批、MinIO 对象和本地缓存的软删除、撤销、延迟清理和失败重试。 + +### 阶段四：前端权限体验与测试 + +1. 开发统一资源级权限组件，按动作隐藏/禁用按钮，并展示授权来源、有效期、审批状态和明确的 401/403 原因。 +2. 将审批、ACL、审计列表中的用户、租户、资源 ID 映射为可读名称，同时保留 ID 作为详情字段。 +3. 编写并执行权限专项测试矩阵：未登录、同租户成员、资源所有者、ACL 授权、跨租户、禁用用户、会话注销、审批撤回/过期、软删除、MinIO 故障、GPU 冲突和配额不足。 +4. 在第二个可达节点或多卡节点准备后执行多节点、多 GPU 并发及故障恢复测试；完成主工程与 `docker/offline/src` 的源码、初始化 SQL、迁移和部署文档一致性检查。 + +## 十二、下一阶段完成标准 + +- 普通资源接口不再返回 API key、密码、Token 或节点访问凭据。 +- 所有资源入口都经过统一的租户边界和动作授权，跨租户访问返回明确 403。 +- 训练、评测、推理创建同时满足资源权限、GPU 预留和租户配额，失败时不会留下孤儿占用。 +- 审批只能由合法审批人执行，申请人不能自审；重复回调不会重复授权、分配或扣减配额。 +- 权限拒绝、下载、导出、授权、审批和异常均能按用户、租户、资源和请求 ID追溯。 +- 主工程和离线目录初始化新库均无缺表、缺字段；迁移可重复执行且不破坏既有数据。 +- 权限专项自动化测试通过，且完成至少一次双用户、跨租户和 MinIO 故障场景的真实容器验证。 + +## 十三、上一阶段权限闭环开发结果（2026-08-20） + +本轮围绕 P0 安全封堵完成了一个可验证的权限闭环： + +1. **模型凭据不出接口**：模型列表、详情、按名称查询、创建、更新和用途更新响应统一经过公开 DTO 脱敏，移除 `api_key`、密码、Token 等字段，仅返回 `api_key_configured` 布尔状态；后端内部评测、数据生成仍使用数据库中的真实配置。 +2. **编辑密钥不被意外清空**：前端编辑在线模型时不回填真实密钥，留空表示保留已有配置，只有管理员主动输入新值时才提交替换。 +3. **资源动作统一入口**：增加资源动作白名单 `read/write/execute/download/export/delete/admin`，并将 `export` 统一归一到下载权限，非法动作默认拒绝，避免把模块权限误当成资源权限。 +4. **失败审计闭环**：审计装饰器对成功和异常分别记录，失败记录包含结果、失败原因、租户、请求 ID、会话 ID和耗时，并对异常中的常见凭据进行脱敏。 +5. **访问统计受控**：模块访问统计只接受平台已登记的模块名，不能通过请求参数向安全审计表写入任意动作；审计 CSV 导出改为标准 CSV 编码，并补充结果、原因、请求和会话字段。 +6. **离线版本同步**：上述后端权限代码、前端模型编辑代码和权限安全回归用例已同步到 `docker/offline/src`；本轮未新增数据库字段，因此 `000_full_init.sql` 无需变更。 + +### 上一阶段验证结果 + +- 后端权限相关模块 `python -m py_compile`：通过。 +- WSL Backend 容器执行权限安全用例：8 passed；仅有 pytest 缓存目录只读警告。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 路径和 chunk size 警告。 +- 容器接口验证：未登录访问模型接口返回 401；管理员访问模型列表/详情时响应不包含 `api_key`，仅返回 `api_key_configured`。 +- Backend 容器已重启并加载当前源码；数据库无需迁移。 + +### 上一阶段未覆盖的范围（已在第十四节继续处理） + +本轮没有声称完成配额账本、GPU 原子预留、租户邀请/加入申请、审批定时过期、所有资源入口的逐接口审计，以及第二节点/多 GPU 的真实并发测试；这些仍按“十一、下一步开发计划”执行。 + +## 十四、本轮继续开发结果（2026-08-20） + +### 已完成 + +1. 新增 `tenant_quota_reservations` 配额预留账本，支持按租户统计 GPU 预留量、原子限制 GPU 配额，以及任务完成、失败、停止、删除和节点故障时释放预留。 +2. 训练、评测和推理统一接入租户 GPU 配额预留；评测/推理使用的 `gpu_reservations` 与配额账本在同一事务内创建或释放，避免只释放算力卡而遗留配额占用。 +3. 租户成员支持邀请、接受、过期、角色更新和移除；邀请不允许授予 `owner`，成员状态和租户有效性由后端校验，租户详情页补充成员管理和 GPU 配额使用量展示。 +4. 审批动作增加白名单，非法动作拒绝创建；相同申请人在同一资源上的待审批申请幂等复用；审批实例读取时自动处理过期状态，资源访问申请也避免重复创建。 +5. GPU 分配增加节点启用状态、GPU 卡存在性、用户 active 状态和同卡跨用户冲突校验；冲突返回 409，不再静默覆盖或产生重复授权。 +6. 完整初始化 SQL 增加配额预留表和索引，新增 `010_permission_quota_membership.sql` 增量迁移；主工程相关源码、前端租户 API/页面和 SQL 已同步到离线源码目录。 + +### 本轮验证 + +- WSL Backend 容器重启后，`tenant_quota_reservations` 可正常查询，默认租户配额使用量返回正常。 +- WSL Backend 容器执行 `test_permission_security.py`：3 passed。 +- 后端相关文件 `python -m py_compile`：通过。 +- 前端 `npm run build`：通过；仅保留既有 Font Awesome 资源路径和大 chunk 警告。 + +### 必须后续验证的场景 + +1. 创建第二个 active 用户和第二个租户，验证邀请接受、租户切换、跨租户资源访问均按预期返回 401/403。 +2. 将租户 GPU 配额设为 1，同时提交两个需要 GPU 的训练/评测/推理任务，确认第二个任务被拒绝；第一个任务终态后确认配额可再次使用。 +3. 在同一多卡节点上让两个用户申请同一张卡，确认审批执行阶段冲突返回 409，另一张空闲卡仍可正常分配。 +4. 构造过期审批、重复提交、审批拒绝/撤回，确认不会重复创建 ACL、GPU 分配或配额预留。 +5. 准备第二个可达节点或多卡节点，执行训练、评测、推理的多节点/多 GPU 并发和节点失联回收测试。 +6. 注入 MinIO 故障，确认任务失败后 GPU 与租户配额均能释放，恢复 MinIO 后可以重新提交任务。 + +### 下一步计划 + +- 对数据处理、数据转换、MinIO 对象、缓存、版本和报告下载入口继续执行一次接口级回归，重点验证跨租户、过期 ACL 和下载/执行动作分离。 +- 将双用户、跨租户、配额不足、同卡冲突、审批过期/幂等、MinIO 故障和节点失联场景加入可重复的自动化测试夹具；当前单节点环境先完成无需第二节点的部分。 +- 在第二个可达节点或多卡节点到位后，执行多节点、多 GPU 并发、任务失败回收和重启恢复测试，完成上线验收闭环。 + +## 十五、本轮权限全量收口结果（2026-08-20） + +### 已完成的开发 + +1. **租户成员角色真正参与资源授权**：ACL 的 `role` 主体不再只读取平台角色，改为读取资源所属租户中的有效成员角色；成员过期、禁用或不属于资源租户时不能通过角色 ACL 访问。 +2. **租户有效期隔离**：用户可用租户集合和资源授权均校验成员有效期，避免过期成员继续访问资源。 +3. **用户生命周期闭环**：用户删除改为软删除，保留用户和审计链路；同时注销会话、禁用租户成员、撤销用户 ACL、取消待审批申请、释放 GPU 分配，并禁止删除租户最后一个 owner。 +4. **资源和对象清理链路**：被删除用户创建的模型、训练模型、数据集、评测、推理、数据处理和数据转换资源统一标记删除；关联 MinIO `storage_objects` 标记为 `deleted`，进入既有清理/重试队列；资源 ACL 和安全状态同步撤销。 +5. **推理入口补强**：训练模型预加载必须引用真实且未删除的训练模型资源，普通用户不能通过任意本地路径绕过资源授权；本地推理状态查询要求普通用户提供有 `read` 权限的推理任务。 +6. **初始化和迁移完整性**：新增 `011_permission_lifecycle.sql`，完整初始化脚本同步补齐用户、评测、推理、数据转换和存储对象生命周期字段；离线目录仍只保留完整的 `000_full_init.sql`。 + +### 自动验证结果 + +- `python -m py_compile backend/app/core/auth.py backend/app/db/platform_store.py backend/app/api/v1/endpoints/platform.py`：通过。 +- `git diff --check`：通过。 +- WSL Backend 容器启动并自动执行迁移后为 `healthy`；已核对 `users`、`eval_tasks`、`compare_tasks`、`data_convert_tasks`、`storage_objects` 的新增字段均存在。 +- `test_permission_security.py` 与 `test_storage_security.py`：8 passed；仅有容器内 pytest 缓存目录只读警告。 +- 真实 API 闭环：临时用户删除后状态为 `deleted`、存在 `deleted_at`，使用原凭据登录返回 HTTP 401。 +- 前端 `npm run build`：需在本轮同步后再次执行，预期仅保留既有资源路径和 chunk size 警告；该项以最终命令结果为准。 + +### 仍需现场验证的场景 + +1. 第二个 active 用户和第二个租户的邀请、接受、租户切换及跨租户资源访问。 +2. 租户 GPU 配额为 1 时，训练/评测/推理并发预留、终态释放和服务重启恢复。 +3. 同一多卡节点的同卡冲突、不同卡并行和节点不可达后的回收。 +4. 审批过期、重复回调、拒绝、撤回和执行失败重试的真实链路。 +5. MinIO 故障注入后的任务失败、GPU/配额释放、对象清理重试和恢复后重新提交。 + +以上项目属于环境依赖型验收，不再作为代码未开发项；在当前单节点、单用户测试环境中保留为上线前验证项。 + +## 十六、前端权限页面收口结果（2026-08-20） + +本轮已完成前端权限页面的主要操作闭环： + +1. `用户权限设置`由占位页改为完整权限矩阵，支持按平台角色、账号状态和页面权限保存；已删除用户进入只读态，管理员角色自动拥有全部页面权限。 +2. 用户创建页增加页面权限配置，普通用户默认不授予组织管理和算力节点权限。 +3. 用户管理页增加权限入口、权限数量、软删除状态和危险操作禁用逻辑。 +4. 资源 ACL 页支持用户/租户角色主体、权限标签、全部撤销二次确认和加载失败提示。 +5. 审批策略页从 JSON 文本改为可视化配置指定用户、租户管理员和平台管理员审批步骤。 +6. 审批申请和访问申请页增加中文事项、资源类型、权限和状态展示，保留资源 ID 作为详情信息。 +7. 普通用户可进入审批中心的“我的申请”“访问申请”“租户邀请”，支持撤回访问申请和接受有效租户邀请。 +8. 租户成员邀请页增加邀请有效期设置；租户配额和 GPU 预留量继续在租户详情页展示。 +9. 本轮前端源码已同步至 `docker/offline/src/frontend/src`，初始化 SQL 目录仍只保留 `000_full_init.sql`。 + +前端 `npm run build` 已通过。尚需浏览器实际点击验证页面布局、不同账号菜单差异、双用户跨租户流程和多 GPU/MinIO 故障场景；这些属于环境验收，不再是页面缺少开发入口。 diff --git a/docs/权限验收测试用例.md b/docs/权限验收测试用例.md new file mode 100644 index 0000000..5faa4c6 --- /dev/null +++ b/docs/权限验收测试用例.md @@ -0,0 +1,227 @@ +# 权限验收测试用例 + +> 版本：权限 2.0 前端权限页面收口版 +> 编写日期：2026-08-20 +> 适用系统：YG_FT 微调平台主工程、Backend、Frontend、Compute Agent、PostgreSQL、Redis、MinIO +> 说明：本文件将自动化验证、单节点页面验证和多用户/多节点专项验证分开记录。 + +## 1. 验收目标 + +验证平台权限功能是否能够形成以下闭环： + +`登录认证 → 页面权限 → 租户成员 → 资源创建 → ACL/访问申请 → 审批 → 训练/评测/推理执行 → GPU/配额预留 → 任务释放 → 审计 → 软删除和对象清理` + +验收重点： + +- 前端页面是否提供完整的权限管理入口。 +- 普通用户是否可以提交访问申请、查看自己的申请和接受租户邀请。 +- 管理员是否可以配置用户页面权限、租户成员、租户配额、资源 ACL 和审批策略。 +- 页面按钮隐藏只是辅助体验，直接调用接口仍必须由后端返回 401/403。 +- 用户、租户、资源、ACL、审批、GPU、配额和 MinIO 对象状态是否能够保持一致。 + +## 2. 测试环境 + +### 2.1 服务 + +| 服务 | 地址/容器 | 验收要求 | +|---|---|---| +| Frontend | `http://127.0.0.1:16801` | 页面可打开，路由切换无白屏 | +| Backend | `http://127.0.0.1:17861/modelTF` | 健康接口返回成功 | +| Compute API | `http://172.25.179.69:19100` | 当前可达节点可进行单节点验证 | +| Redis | WSL Docker `yg-ft-redis` | 容器 healthy | +| MinIO | WSL Docker `yg-ft-minio` | 容器 healthy，可进行对象读写 | +| PostgreSQL | 当前环境实际远程 PostgreSQL | 初始化字段和增量迁移均存在 | + +### 2.2 测试账号和数据 + +准备以下测试对象，禁止使用生产账号和真实敏感数据： + +| 对象 | 建议值 | 说明 | +|---|---|---| +| 平台管理员 | 当前环境管理员账号 | 执行用户、租户、ACL、审批和配额管理 | +| 普通用户 A | 新建测试用户 | 作为资源所有者 | +| 普通用户 B | 新建测试用户 | 作为被授权用户和申请人 | +| 租户 A | 新建测试租户 | 与租户 B 隔离验证 | +| 租户 B | 新建测试租户 | 跨租户访问验证 | +| 测试数据集 | `test_data_0817` 或新的小数据集 | 用于 ACL、训练和下载验证 | +| 测试模型 | `qwen3.5-0.8B` | 用于模型使用权限和推理验证 | + +## 3. 自动化验证结果 + +以下项目已在当前开发环境执行： + +| 编号 | 验证内容 | 结果 | +|---|---|---| +| AUTO-01 | `npm run build` | 通过；仅有既有 Font Awesome 路径和 chunk size 警告 | +| AUTO-02 | `test_permission_security.py` | 通过，8 passed | +| AUTO-03 | `test_storage_security.py` | 已包含在 8 passed 中 | +| AUTO-04 | 后端权限文件 `py_compile` | 通过 | +| AUTO-05 | `git diff --check` | 通过 | +| AUTO-06 | Backend、Frontend、Compute、Redis、MinIO 容器状态 | 通过，Backend healthy | +| AUTO-07 | 迁移后字段核对 | 通过，users、eval_tasks、compare_tasks、data_convert_tasks、storage_objects 字段存在 | +| AUTO-08 | 临时用户软删除闭环 | 通过，状态为 deleted、存在 deleted_at，原凭据登录返回 401 | +| AUTO-09 | `docker/offline/src` 同步 | 通过，离线 SQL 目录只保留 `000_full_init.sql` | + +## 4. 前端页面验收 + +### 4.1 组织与权限 + +页面入口：`/organization` + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| UI-ORG-01 | 管理员打开“组织与权限” | 显示“用户与角色”“租户与配额”两个页签 | +| UI-ORG-02 | 点击“创建用户” | 进入用户创建页面，可填写账号、角色、状态和页面权限 | +| UI-ORG-03 | 创建普通用户 | 默认授予业务页面权限，不自动授予组织管理和算力节点权限 | +| UI-ORG-04 | 用户列表点击“权限” | 进入 `/user-settings/:id/permission`，显示用户角色、所属租户、状态和权限矩阵 | +| UI-ORG-05 | 修改用户页面权限并保存 | 页面提示保存成功，刷新后权限保持一致 | +| UI-ORG-06 | 将普通用户改为管理员 | 页面显示全部权限；保存时自动补齐全部权限码 | +| UI-ORG-07 | 查看已软删除用户 | 显示“已删除”，权限页只读，不能重新激活 | +| UI-ORG-08 | 普通用户访问组织与权限 | 路由跳转到无权访问页，菜单不显示管理员治理入口 | + +### 4.2 租户与配额 + +页面入口：`/organization?tab=tenants`、`/tenants/:id` + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| UI-TEN-01 | 管理员创建租户并设置 GPU、存储配额 | 租户列表出现新租户，配额显示正确 | +| UI-TEN-02 | 打开租户详情 | 显示租户信息、配额设置、GPU 预留量和成员列表 | +| UI-TEN-03 | 邀请成员并设置角色、邀请有效期 | 页面提示邀请成功，成员状态为待接受，过期时间正确 | +| UI-TEN-04 | 修改成员角色 | 成员列表角色更新，不能通过页面授予非法 owner 权限 | +| UI-TEN-05 | 移除成员 | 成员状态/关系更新，后续资源访问按后端权限拒绝 | +| UI-TEN-06 | 保存配额 | 配额配置更新，已预留 GPU 数量不超过新配额 | + +### 4.3 我的申请与租户邀请 + +页面入口：`/approval-instances?tab=mine`、`/approval-instances?tab=access`、`/approval-instances?tab=invitations` + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| UI-SELF-01 | 普通用户打开审批中心 | 可进入，不再被管理员专属路由拦截 | +| UI-SELF-02 | 打开“访问申请” | 可选择可见的数据集/训练模型，资源名称优先显示中文名称，同时保留 ID | +| UI-SELF-03 | 提交 read/execute/download 申请 | 申请出现在“我的访问申请”，状态为审批中 | +| UI-SELF-04 | 撤回待审批申请 | 二次确认后状态变为已撤回，不能再次撤回 | +| UI-SELF-05 | 打开“租户邀请” | 显示租户、角色、邀请人、有效期和状态 | +| UI-SELF-06 | 接受有效邀请 | 状态变为已加入，获得对应租户成员关系 | +| UI-SELF-07 | 接受过期邀请 | 后端拒绝，页面显示失败原因，成员关系不产生 | + +### 4.4 资源授权 + +页面入口：`/resource-acl` + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| UI-ACL-01 | 管理员选择数据集或训练模型 | 显示资源名称和资源 ID，不显示已软删除资源 | +| UI-ACL-02 | 查看资源 ACL | 显示用户/租户角色、主体类型和权限标签 | +| UI-ACL-03 | 增加用户 ACL | 可选择查看、编辑、使用、下载等权限并保存 | +| UI-ACL-04 | 增加角色 ACL | 可选择租户管理员、成员、只读角色；后端校验最终结果 | +| UI-ACL-05 | 移除全部 ACL | 页面二次确认后保存空 ACL，授权全部撤销 | +| UI-ACL-06 | ACL 查询失败 | 页面显示明确错误提示，不把失败误显示为空授权 | +| UI-ACL-07 | 普通用户打开资源授权页面 | 页面入口隐藏，直接访问接口返回 403 | + +### 4.5 审批中心 + +页面入口：`/approval-instances` + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| UI-APP-01 | 管理员打开审批申请 | 显示资源类型、申请事项、申请人、状态和当前步骤的中文信息 | +| UI-APP-02 | 管理员打开审批策略 | 可配置指定用户、租户管理员、平台管理员审批步骤，不需要输入 JSON | +| UI-APP-03 | 指定审批人审批 | 只能由当前会话指定审批人执行，审批结果和意见可见 | +| UI-APP-04 | 申请人查看自己的申请 | 只能看到自己的申请，不显示其他租户的审批数据 | +| UI-APP-05 | 审批通过 | ACL、GPU 分配或成员关系等业务效果真正落地 | +| UI-APP-06 | 审批拒绝/过期 | 状态明确显示，不能产生授权或资源占用 | + +## 5. 后端接口与页面联动验收 + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| API-AUTH-01 | 未登录访问任意受保护页面接口 | 返回 401，前端回到登录页 | +| API-AUTH-02 | 普通用户直接调用管理员接口 | 返回 403，不能仅依赖前端隐藏按钮 | +| API-AUTH-03 | 禁用用户使用旧 Token | 返回 401 或明确无效会话错误 | +| API-AUTH-04 | 已删除用户登录 | 返回 401 | +| API-TEN-01 | 用户访问其他租户资源详情 | 返回 403 或资源不存在，不泄露资源内容 | +| API-TEN-02 | 过期成员访问租户资源 | 返回 403 | +| API-ACL-01 | 没有 read 的用户读取资源 ACL | 返回 403 | +| API-ACL-02 | 只有 read 的用户执行模型推理 | 返回 403 | +| API-ACL-03 | 只有 execute 的用户下载模型 | 返回 403 | +| API-ACL-04 | ACL 授权后重新访问 | 按授权动作成功 | +| API-APP-01 | 申请人提交自己的审批决策 | 返回 403 | +| API-APP-02 | 非指定审批人提交决策 | 返回 403 | +| API-APP-03 | 重复提交相同资源访问申请 | 复用或拒绝重复待审批申请，不重复写 ACL | +| API-LIFE-01 | 删除用户 | 会话、成员、ACL、GPU 分配、待审批项和用户资源状态同步处理 | +| API-LIFE-02 | 执行存储清理 | 已删除 MinIO 对象进入清理，失败任务可重试 | + +## 6. 训练、评测、推理权限验收 + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| JOB-01 | 用户使用无权限数据集创建训练 | 预检失败，显示数据集无权限 | +| JOB-02 | 用户使用授权数据集创建训练 | 预检通过，GPU 和租户配额原子预留 | +| JOB-03 | 用户使用未授权训练模型推理 | 页面提示无权限，接口返回 403 | +| JOB-04 | 用户使用已授权训练模型推理 | 可选择有权限的算力节点和 GPU 卡 | +| JOB-05 | 用户没有 download 权限导出模型 | 操作按钮不显示，接口返回 403 | +| JOB-06 | 评测任务结束 | GPU 预留和租户配额预留释放 | +| JOB-07 | 推理任务删除 | 推理服务释放，任务软删除，ACL 状态撤销 | +| JOB-08 | 节点不可达或 MinIO 暂时不可用 | 任务进入失败/重试状态，不留下永久 GPU 或配额占用 | + +## 7. GPU、配额和多节点专项验收 + +以下用例需要第二个可达节点或一个包含多张可分配 GPU 卡的节点，当前单节点环境暂保留。 + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| GPU-01 | 租户 GPU 配额设置为 1，同时提交两个 GPU 任务 | 第二个任务被拒绝，不能超配 | +| GPU-02 | 同一节点同一张卡由两个用户申请 | 审批/分配阶段返回 409，不能重复占用 | +| GPU-03 | 同一节点选择两张空闲卡 | 任务使用页面指定的两张卡，不使用其他卡 | +| GPU-04 | 任务失败或取消 | GPU 分配和配额预留释放 | +| GPU-05 | Backend 重启后恢复任务状态 | 已完成/失败任务不重复占用；运行任务保持可对账 | +| GPU-06 | 模型训练、推理、评测选择不同节点 | 模型和数据从 MinIO 准备到指定节点，不错误派发到其他节点 | +| GPU-07 | 节点失联 | 任务进入失败或待重试，资源占用在超时后回收 | + +## 8. MinIO 故障专项验收 + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| MINIO-01 | 暂停 MinIO 容器后上传数据集 | 页面显示明确存储不可用，不显示上传成功 | +| MINIO-02 | MinIO 暂停期间预检训练 | 预检提示对象不可用，可重试 | +| MINIO-03 | MinIO 恢复后重试任务 | 对象可重新准备，任务继续或重新提交成功 | +| MINIO-04 | 删除用户后执行对象清理 | 用户资源对象标记 deleted，清理成功后标记 purged | +| MINIO-05 | 清理时对象删除失败 | storage cleanup job 保留 failed 状态和错误原因，后续可重试 | + +## 9. 离线部署验收 + +| 用例 | 操作步骤 | 预期结果 | +|---|---|---| +| OFFLINE-01 | 检查 `docker/offline/src/backend/app/db/sql` | 只存在完整的 `000_full_init.sql` | +| OFFLINE-02 | 清空数据库并执行完整初始化 SQL | 不缺权限相关表和字段，Backend 可启动 | +| OFFLINE-03 | 离线启动 Frontend | 不请求在线 CDN，页面可正常打开 | +| OFFLINE-04 | 离线启动 Backend、Compute、MinIO | 各服务使用独立网络和配置，权限接口可用 | +| OFFLINE-05 | 离线新库执行用户/租户/ACL/审批流程 | 与主工程行为一致 | + +## 10. 问题记录规则 + +每条失败用例至少记录： + +- 测试编号、执行时间、执行人和账号。 +- 页面 URL、接口 URL、请求方法和请求 ID。 +- 当前用户、租户、资源 ID、资源所属租户。 +- 实际 HTTP 状态码、页面提示、数据库状态和容器日志。 +- 是否产生了残留 ACL、GPU 分配、配额预留、MinIO 对象或审批记录。 +- 修复提交、回归结果和是否需要再次现场验证。 + +## 11. 当前验收结论 + +当前已自动验证：前端构建、后端权限安全测试、存储安全测试、数据库字段迁移、容器健康、用户软删除登录拦截和离线源码 SQL 目录结构。 + +当前前端权限页面已完成主要操作闭环： + +- 用户页面权限设置。 +- 用户创建时的页面权限配置。 +- 租户成员邀请、角色、有效期和配额展示。 +- 普通用户访问申请、我的申请和租户邀请接受。 +- 资源用户/角色 ACL 管理和全部撤销。 +- 审批申请、审批策略可视化配置和中文状态展示。 + +仍需现场验证：双用户跨租户、多 GPU 并发、第二节点调度、节点失联回收、MinIO 故障注入和完整浏览器点击流。这些属于环境依赖型验收，不代表前端页面缺少入口。 diff --git a/docs/模型评测优化设计方案.md b/docs/模型评测优化设计方案.md new file mode 100644 index 0000000..a7eda2c --- /dev/null +++ b/docs/模型评测优化设计方案.md @@ -0,0 +1,161 @@ +# 模型评测优化设计方案 + +## 1. 现状检查 + +当前模型评测菜单已经具备以下基础能力： + +- 创建评测任务：选择训练模型、评测数据集、算力节点和 GPU。 +- 任务调度：后端将模型、适配器、数据集准备到目标算力节点，再提交 Compute API 任务。 +- 模型推理：Compute Agent 使用 LLaMA-Factory 推理会话逐条生成回答。 +- 结果落库：任务完成后读取 `eval_results.json`，写入任务详情、样本结果和指标摘要。 +- 报告归档：评测输出目录可以归档到 MinIO，并通过报告接口下载。 +- 前端详情：显示综合分、通过率、样本结果、指标摘要，并对运行中的任务进行轮询。 + +当前缺陷主要集中在评分和进度链路： + +1. 创建页面的 BLEU、ROUGE、余弦指标默认全部关闭；未配置 LLM 评委时，样本没有确定性评分，容易得到 0 分。 +2. BLEU、ROUGE、余弦、LLM Judge 的返回范围和含义不统一，百分制、0-1、小量程评分混在一起。 +3. 评测模型地址直接拼接 `/v1/chat/completions`，当地址已经包含 `/v1` 时会出现 `/v1/v1`。 +4. LLM Judge 只解析少量文本格式，无法可靠解析 JSON、Markdown JSON 或 0-1 综合评价。 +5. ROUGE 对中文没有采用 `nlp-eval-demo` 的字符级中文分词策略，中文短文本容易得到失真的结果。 +6. 后端只在 Compute 任务完成后读取结果文件，运行时没有样本完成数、当前阶段和中间指标。 +7. 前端没有雷达图，用户无法直观看到 BLEU、ROUGE、语义相似度、精确匹配和 LLM Judge 等维度。 + +## 2. 目标 + +建立一条可解释、可持续轮询、兼容旧任务的评测闭环： + +```text +创建任务 -> 资源准备 -> 模型加载 -> 样本推理 -> 逐样本评分 + -> 中间进度文件 -> 后端同步 -> 页面进度与雷达图 + -> 完成报告 -> MinIO 归档 -> 详情与下载 +``` + +目标结果： + +- 所有最终展示分数统一为 0-100，避免不同指标之间直接相加造成误解。 +- 没有 LLM 评委时，仍然使用确定性指标生成样本得分和综合分，不再因为“未配置评委”自动归零。 +- 有 LLM 评委时，保留原有自定义评分区间，同时将其归一化到 0-100 展示。 +- 每个评测任务都能看到阶段、总样本数、已完成样本数、百分比和当前指标状态。 +- 详情页展示指标雷达图；指标不可用时显示原因，不把“依赖未安装”伪装成 0 分。 +- 不新增必需数据库表，继续利用 `eval_tasks.payload` 保存评测结果和进度，兼容现有数据库及离线初始化 SQL。 + +## 3. 评分设计 + +### 3.1 指标契约 + +Compute Agent 内部统一使用以下结构： + +```json +{ + "enabled": true, + "available": true, + "score": 82.5, + "max_score": 100, + "unit": "percent", + "sample_count": 3, + "error": "" +} +``` + +`score` 永远是 0-100。指标不可用时 `available=false`，`score` 可以为 null，同时保留 `error`。旧报告中只有 `score` 的结构继续兼容，后端读取时按旧结构补齐默认字段。 + +### 3.2 确定性指标 + +参考 `nlp-eval-demo` 的实现，支持： + +- BLEU：sacrebleu，结果由 0-100 转换为百分制。 +- ROUGE-1、ROUGE-2、ROUGE-L：中文按字符切分，英文按词切分，使用 F1 均值并转换为百分制。 +- 余弦相似度：TF-IDF 余弦相似度，转换为百分制。 +- 精确匹配：标准化空白和大小写后完全一致，百分制。 +- 文本相似度：SequenceMatcher，作为无外部模型时的稳定兜底指标。 +- BERTScore：仅在 `bert_score` 和模型可用时启用；下载/加载失败只标记不可用，不阻断整个评测任务。 + +精确匹配和文本相似度始终计算。用户在创建页面选择的 BLEU、ROUGE、余弦作为额外指标；如果没有选择额外指标，也用“文本相似度 + ROUGE-L（可用时）”生成确定性综合分。 + +### 3.3 无 LLM 评委时的样本分数 + +对每条样本使用可用确定性指标的平均值作为样本百分制得分： + +```text +sample_score = mean(exact_match, text_similarity, rougeL, cosine, bleu, bertscore) +``` + +其中未启用或不可用的指标不参与平均。样本通过阈值默认 60 分；如果旧维度配置的 `pass_threshold <= score_max`，先按旧量程换算为百分制。 + +### 3.4 LLM Judge + +- 兼容 OpenAI Chat Completions 接口。 +- 自动规范化 `api_url`，避免重复 `/v1`。 +- 优先解析 JSON 的 `score`、`综合评价`、`overall_score`、`dimensions` 字段，再解析 Markdown/自然语言中的评分。 +- 支持 0-1、0-5、0-100 三种返回量程；最终统一转换为 0-100。 +- API 调用失败时记录样本失败原因，不把失败当作正常 0 分；如果所有样本都调用失败，任务状态仍可完成但报告会明确提示评委不可用。 + +## 4. 进度设计 + +Compute Agent 在评测输出目录写入两个文件： + +- `eval_progress.json`：轻量进度文件，每完成一条样本更新一次。 +- `eval_results.json`：运行中写入部分结果，完成后写入完整报告。 + +进度结构： + +```json +{ + "status": "running", + "stage": "inference", + "total": 20, + "completed": 7, + "percentage": 35, + "current_index": 8, + "message": "正在生成第 8 条样本", + "updated_at": "2026-08-20T00:00:00Z" +} +``` + +后端详情接口每次轮询 Compute Agent： + +1. 任务运行中读取 `eval_progress.json`，写入 `eval_tasks.payload.progress_detail`。 +2. 读取到部分 `eval_results.json` 时同步已完成样本和基础指标，页面可以边运行边展示。 +3. Compute 任务完成后读取完整报告，再执行 MinIO 归档。 +4. Compute Agent 暂时不可达时保留最后一次进度，不因为一次轮询失败把评测任务误判为失败。 + +## 5. 前端设计 + +- 列表页增加进度列：运行中显示 `已完成/总数` 和百分比，完成后显示最终分数。 +- 详情页增加当前阶段、进度消息和进度条。 +- 详情页增加“指标雷达图”，雷达轴来自可用的 `dimension_summary` 指标，最大值统一 100。 +- 指标卡同时显示分数、通过率、样本数和不可用原因。 +- 样本结果在运行中支持逐步出现，保留现有搜索、筛选和分页。 +- 不改变已有任务创建、删除、报告下载和权限校验流程。 + +## 6. 数据库与兼容性 + +本次不新增数据库表和必需字段。评测结果继续存放在 `eval_tasks.payload` JSON 中，新增键包括： + +- `progress_detail` +- `basic_metrics` 的统一指标对象 +- `metric_summary_version` +- 样本的 `raw_score`、`raw_max_score`、`score`、`max_score` + +旧任务兼容规则：缺少进度时由 `progress` 和样本数量推导；旧的量程评分按 `score/max_score` 转换为百分制；没有基础指标时显示“历史任务未保存指标明细”。因此无需修改 `000_full_init.sql`。 + +## 7. 实施步骤 + +1. 重构 Compute Agent 评测指标、中文 ROUGE、LLM Judge 解析、评分归一化和中间结果写入。 +2. 增加 Compute Agent 进度文件读取能力，后端同步运行中进度和部分结果。 +3. 扩展后端评测任务结果兼容、失败信息和进度返回。 +4. 扩展前端类型、列表进度列、详情进度区和指标雷达图。 +5. 增加单元测试、前端构建测试和接口/运行时冒烟验证。 +6. 同步前后端、算力服务到 `docker/offline/src`，确认初始化 SQL 无需变更。 + +## 8. 验收标准 + +- 使用仅包含 `instruction/output` 的小型 JSONL 数据集，未配置 LLM Judge 时综合分不再固定为 0。 +- 中文答案能得到可解释的 ROUGE-1/2/L 分数。 +- LLM Judge 的地址为 `https://host/v1` 时请求路径仍正确。 +- 评测运行期间能看到 `completed/total` 和百分比变化。 +- 完成后详情页存在至少 3 个可用指标时显示雷达图，指标少于 3 个时显示明细降级视图。 +- 失败、依赖缺失、空答案等情况能在报告中区分，不以正常 0 分掩盖原因。 +- 现有权限、GPU 预约、MinIO 归档、报告下载和旧任务查看不受影响。 + diff --git a/docs/租户与用户权限体系梳理及纠偏流程.md b/docs/租户与用户权限体系梳理及纠偏流程.md new file mode 100644 index 0000000..b40acb7 --- /dev/null +++ b/docs/租户与用户权限体系梳理及纠偏流程.md @@ -0,0 +1,628 @@ +# 租户与用户权限体系梳理及纠偏流程 + +> 版本：v1.0 +> 梳理日期：2026-08-20 +> 适用范围：YG_FT 平台后端、前端、Compute Agent、MinIO、PostgreSQL 及离线部署目录 +> 目的：明确租户、用户、角色、权限、资源和审批之间的层级关系，识别当前实现中的概念冲突，并为后续优化提供唯一设计口径。 + +## 一、结论摘要 + +当前系统已经具备用户登录、租户字段、租户成员、资源 ACL、审批、GPU 分配和资源级动作校验，但概念来源还没有完全收敛，主要表现为： + +1. **用户和租户没有混为一个数据库对象，但业务流程中边界不清**：用户表保存 `tenant_id`，同时又通过 `tenant_members` 表表达租户关系，导致“主租户”和“成员关系”的判定规则并存。 +2. **平台角色、租户角色、页面权限码、资源动作权限同时存在，但职责没有彻底分离**：`users.role`、`users.permissions`、`tenant_members.role`、`roles.permissions` 可能产生不一致。 +3. **资源归属字段不统一**：部分资源使用独立的 `created_by`，部分任务把创建者放在 `payload.created_by`，部分资源的 `tenant_id` 允许为空，历史数据因此出现“ACL 已授权但列表不可见”的问题。 +4. **租户创建、用户创建、成员加入和资源授权是四条不同流程，目前在页面上有交叉**：创建用户时直接写入租户成员，租户的 `owner_user_id` 却没有同步建立 owner 成员关系。 +5. **项目表和项目字段仍在数据库及部分接口中保留**，但当前产品已经取消项目作为用户可见的业务隔离层，继续把 `project_id` 当权限条件会造成新的歧义。 +6. **正确的目标模型应当是**：平台管理员管理平台身份和租户；用户通过租户成员关系进入一个或多个租户；资源属于一个租户并有一个所有者；资源使用通过 ACL 动作权限；高风险动作通过审批；GPU 和配额是资源使用前的运行时约束。 + +本文件的纠偏原则是：**先统一口径，再做兼容迁移；先保证后端判定一致，再调整前端菜单和数据库约束；历史字段暂不删除，但不再新增新的业务依赖。** + +## 二、目标层级模型 + +```text +平台 Platform +│ +├── 平台身份与平台角色 +│ ├── platform_admin：平台管理员，可管理租户、用户、策略和全部资源 +│ └── platform_user：普通平台用户，不因登录自动获得资源权限 +│ +├── 租户 Tenant +│ ├── 租户基本信息、状态、配额、留存策略 +│ ├── 租户成员 TenantMember +│ │ ├── owner：租户所有者 +│ │ ├── admin：租户管理员 +│ │ ├── member：普通成员 +│ │ └── viewer：只读成员 +│ └── 租户内资源 +│ ├── 数据集 +│ ├── 基座模型 +│ ├── 训练任务与训练模型 +│ ├── 评测任务 +│ ├── 推理任务 +│ └── 数据处理/转换任务 +│ +├── 资源授权 Resource ACL +│ ├── principal：user 或 tenant role +│ ├── read：查看 +│ ├── write：编辑 +│ ├── execute：训练、推理、评测等使用 +│ ├── download：下载或导出文件 +│ ├── delete：删除 +│ └── admin：资源授权和完整管理 +│ +├── 运行时资源 +│ ├── Compute Node +│ ├── GPU +│ ├── GPU assignment：用户获批可使用哪些 GPU +│ ├── GPU reservation：任务当前占用哪些 GPU +│ └── Tenant quota reservation：租户当前预留了多少运行资源 +│ +└── 审批与审计 + ├── resource access：申请资源 ACL + ├── gpu.assign：申请算力卡 + ├── tenant.quota.update：申请租户配额变更 + ├── 高风险删除/导出/合并操作 + └── 全部结果写入审计日志 +``` + +### 2.1 四个必须分开的概念 + +| 概念 | 正确含义 | 不应承担的职责 | +|---|---|---| +| 用户 User | 可登录平台的人或服务账号 | 不代表租户，也不自动代表资源权限 | +| 租户 Tenant | 资源、成员、配额和审计的隔离边界 | 不直接作为登录账号 | +| 角色 Role | 某个范围内的一组职责 | 不应直接等同于某个用户的全部资源权限 | +| 权限 Permission | 对页面或资源的具体操作能力 | 不应只靠前端按钮控制 | + +## 三、当前实现盘点 + +### 3.1 用户身份层 + +当前主要实现位置： + +- `users` 表：保存账号、密码哈希、平台角色、状态、页面权限 JSON、主租户字段。 +- `sessions` 表：保存登录会话、过期时间和注销时间。 +- `backend/app/core/auth.py`：解析 Token、校验用户状态、校验页面权限和租户上下文。 +- `/users` 接口：当前只有管理员可以创建、修改、禁用和删除用户。 + +当前用户对象包含： + +```text +id / username / display_name / status +role / protected / permissions +tenant_id +``` + +存在的混淆： + +- `role=operator/viewer/user` 是平台用户字段，但 `tenant_members.role` 还有 `owner/admin/member/viewer`，两套角色名称不在同一层级。 +- `role=user` 在创建接口中被接受，但设计文档主要定义的是 `admin/operator/viewer/guest`，前端又同时显示“普通用户”。 +- 创建用户时直接生成 `tenant_members` 活跃成员记录；这相当于“创建用户”和“加入租户”一次完成，没有邀请、待确认或成员审批边界。 +- `users.permissions` 是用户级 JSON，`roles` 表只提供查询接口，当前没有形成“角色变更自动计算权限”的唯一来源。 +- 非管理员创建/更新用户时会被剥离 `compute` 和 `user-settings`，但初始化 SQL 中的 `u_operator` 种子数据仍包含 `compute`，代码默认值和 SQL 种子不一致。 + +### 3.2 租户层 + +当前主要实现位置： + +- `tenants` 表：保存租户名称、状态、owner_user_id、配额和留存策略。 +- `tenant_members` 表：保存租户与用户的多对多关系、租户角色、状态、邀请人和有效期。 +- `tenant_membership()`、`user_tenant_ids()`：判定当前用户的租户范围。 +- `/tenants/{tenant_id}/members`：管理员或租户 owner/admin 管理成员。 + +当前逻辑： + +```text +当前用户租户范围 = users.tenant_id + tenant_members 中 status=active 的 tenant_id +当前请求租户 = X-Tenant-ID（仅在用户属于该租户时生效） +资源租户 = 资源 tenant_id，部分任务还会从 payload 中读取 tenant_id +``` + +主要问题： + +1. `users.tenant_id` 和 `tenant_members` 都能表达租户关系，但没有明确谁是主数据、谁是兼容字段。 +2. `tenants.owner_user_id` 创建租户时写入，但创建租户流程没有自动写入对应的 `tenant_members(role='owner')`。 +3. 平台管理员创建普通用户时直接加入租户，租户管理员也可以直接添加成员，邀请/加入申请流程虽已具备接口，但不是主流程。 +4. `tenant_id` 在资源表中有的为 `NOT NULL`，有的允许 `NULL`；历史资源的 `NULL` 租户需要特殊兼容判断。 +5. 租户删除是状态删除，但成员、资源、ACL、审批、MinIO 对象和本地缓存的后续处理没有统一的生命周期编排入口。 + +### 3.3 页面权限层 + +页面权限由两部分组成： + +```text +前端：路由 meta.permission + 侧边栏过滤 + 按钮显示 +后端：MODULE_PERMISSIONS + get_current_user / require_admin +``` + +页面权限码包括： + +```text +dashboard, fine-tune, model-eval, model-inference, +model-manage, dataset, data-process, data-convert, +compute, hardware, logs, user-settings +``` + +问题： + +- 页面权限码是“能否进入模块”，不是“能否使用某一条数据或 GPU”，但历史实现中一度把 `compute` 页面权限阻断了普通用户读取自己获批 GPU 的接口，造成“有 GPU 分配但训练页面看不到”的问题。 +- 普通用户的 `compute` 管理菜单应保持隐藏，但训练、推理、评测所需的个人节点/GPU查询应当是独立的自助只读接口。 +- 前端权限控制和后端权限控制分别维护，按钮隐藏不能作为安全边界。 + +### 3.4 资源权限层 + +当前资源权限主要由以下条件组合： + +```text +平台管理员旁路 + 或 +资源属于当前租户 +且（资源所有者 == 当前用户 + 或 ACL 明确授权 + 或当前用户是租户 owner/admin） +且满足具体动作 read/write/execute/download/delete/admin +``` + +已接入资源类型包括： + +```text +dataset / model / trained_model / fine-tune / eval / inference +data_process / data_convert / compare / project（历史兼容） +``` + +当前差异： + +| 资源 | 当前所有者来源 | 当前租户来源 | 主要风险 | +|---|---|---|---| +| datasets | `created_by` | `tenant_id`，部分历史为空 | 列表、详情、训练权限可能不一致 | +| models | `created_by` | `tenant_id`，基座模型存在平台共享规则 | 基座模型共享与训练产物共享容易混淆 | +| trained_models | `created_by` | `tenant_id` | 血缘来源、默认授权需统一 | +| fine_tune_tasks | 多数在 `payload.created_by` | 多数在 payload 或派生字段 | 结构化查询和授权容易漏字段 | +| eval_tasks | `created_by` | `tenant_id` | 与数据集/模型执行权限必须同时校验 | +| compare_tasks | 表字段和 payload 并存 | 表字段和 payload 并存 | 推理任务与模型使用权限容易分裂 | +| data_process_tasks | `created_by` | `tenant_id` | 处理结果派生数据集授权不明确 | +| data_convert_tasks | `created_by` | `tenant_id` | 历史表结构和初始化脚本存在演进痕迹 | +| projects | `create_by` 等历史字段 | `tenant_id` | 当前已取消项目业务层，但表和接口仍在 | + +### 3.5 GPU 与运行时权限层 + +GPU 相关关系实际上有三层： + +```text +gpus + └── gpu_assignments：管理员批准“用户可以使用哪张卡” + └── gpu_allocations / gpu_reservations：任务运行时“当前占用了哪张卡” +``` + +正确判定顺序应为： + +1. 用户拥有租户成员资格且账号 active。 +2. 用户已经获得该 GPU 的分配或管理员旁路。 +3. 节点 online、enabled，GPU 存在且没有被其他任务占用。 +4. 租户配额可以完成原子预留。 +5. 训练、推理或评测任务提交到指定节点。 + +目前已经有分配申请、审批后写入 `gpu_assignments`、运行时预留和释放逻辑，但仍需把“审批结果、GPU 分配、配额预留、节点故障回收”统一成一个状态机。 + +## 四、当前数据库关系与问题 + +### 4.1 当前表的职责分组 + +```text +身份 +├── users +├── sessions +└── roles + +租户 +├── tenants +├── tenant_members +└── tenant_quota_reservations + +资源权限 +├── acls +├── resource_access_requests +├── approval_templates +├── approval_instances +└── approval_steps + +业务资源 +├── datasets / dataset_files / dataset_file_versions / dataset_records +├── models / trained_models / model_lineage / model_artifacts +├── fine_tune_tasks / eval_tasks / compare_tasks +├── data_process_tasks / data_convert_tasks +└── projects / project_members（历史兼容） + +运行资源 +├── compute_nodes +├── gpus +├── gpu_assignments +├── gpu_allocations +├── gpu_reservations +└── resource_replicas / storage_objects / storage_cache_jobs + +审计 +└── audit_logs +``` + +### 4.2 数据库设计中的主要问题 + +#### 问题 A：租户归属字段可空且没有统一外键 + +`models`、`trained_models`、`datasets`、`eval_tasks`、`compare_tasks`、数据处理相关表的 `tenant_id` 演进过程不同，部分字段仍允许空值。多数资源表也没有对 `tenants(id)` 的外键约束。 + +后果： + +- 资源可能被创建但没有租户归属。 +- 列表接口和详情接口的可见性不同。 +- ACL 授权是否跨租户无法仅靠数据库判断。 +- 历史数据需要特殊分支，增加权限代码复杂度。 + +#### 问题 B：所有者字段不统一 + +当前同时存在 `created_by`、`owner_id`、`create_by` 和 `payload.created_by`。这会导致： + +- 列表判断资源所有者时需要按资源类型写特殊逻辑。 +- 用户删除时无法保证所有资源都能被一致地软删除。 +- 审计目标和资源所有者可能不是同一个字段。 + +#### 问题 C：角色权限没有单一来源 + +当前存在以下可能来源： + +```text +users.role +users.permissions(JSON) +tenant_members.role +roles.permissions(JSON) +前端 PermissionCode +后端 ALL_PERMISSIONS / MODULE_PERMISSIONS +``` + +如果更新了角色但没有同步用户权限 JSON，页面和接口可能出现不同结果；如果只修改前端权限码，后端仍可能拒绝；如果只修改 `tenant_members.role`，并不会自动改变平台页面权限。 + +#### 问题 D：资源 ACL 不是所有资源的唯一授权来源 + +基座模型在当前设计中对租户用户默认共享 read/execute；资源所有者又有隐式权限；租户管理员又有管理旁路；ACL 只是其中一层。这个设计可以成立，但必须在文档和代码中严格区分： + +```text +平台共享规则 ≠ ACL 授权 +资源所有权 ≠ 租户管理员权限 +页面权限 ≠ 资源动作权限 +GPU 分配 ≠ 资源 ACL +``` + +#### 问题 E：项目字段与当前产品边界不一致 + +`projects`、`project_members`、`project_id` 仍在数据库中存在，但当前产品已经决定不把项目作为用户可见的隔离层。它们应被标记为 legacy，不应再参与新资源的权限判定，也不应在新建页面继续要求填写。 + +## 五、目标唯一口径 + +### 5.1 用户与租户的最终关系 + +```text +一个用户可以属于多个租户 +一个租户可以拥有多个用户 +用户在一次请求中只能选择一个 active_tenant +资源必须归属于 active_tenant +``` + +推荐保留： + +- `users.tenant_id`：暂时保留为兼容字段，表示用户的 primary tenant，不再作为多租户关系的唯一来源。 +- `tenant_members`：升级为租户成员关系的唯一权威来源。 +- `X-Tenant-ID`：明确表达当前请求租户，必须来自 active membership。 + +最终判定： + +```text +有效租户 = tenant_members(status=active, 未过期) +primary tenant = users.tenant_id（仅兼容和默认登录上下文） +当前租户 = 请求 X-Tenant-ID，否则 primary tenant +``` + +### 5.2 平台角色与租户角色 + +推荐收敛为两层： + +| 层级 | 字段/表 | 允许的角色 | 作用 | +|---|---|---|---| +| 平台层 | `users.platform_role`，兼容当前 `users.role` | `platform_admin`、`platform_user` | 管理租户、平台配置、平台用户和全局策略 | +| 租户层 | `tenant_members.role` | `owner`、`admin`、`member`、`viewer` | 管理本租户成员、资源和租户级审批 | + +不建议继续把 `operator`、`user`、`member` 混在一个角色字段中。过渡期可以保留旧值，但新增代码只允许通过映射函数转换： + +```text +admin/protected -> platform_admin +operator/user -> platform_user + tenant_members.member +viewer -> platform_user + tenant_members.viewer +``` + +### 5.3 页面权限与资源动作 + +页面权限只解决“能否进入模块”： + +```text +module permission: dataset / fine-tune / model-inference / ... +``` + +资源动作解决“能否操作某个资源”： + +```text +read / write / execute / download / delete / admin +``` + +统一授权函数输入应固定为： + +```text +authorize( + actor_id, + active_tenant_id, + resource_type, + resource_id, + action, +) +``` + +推荐判定顺序： + +1. 会话有效且用户 active。 +2. 当前租户 active，用户是该租户成员。 +3. 平台管理员旁路，或资源属于当前租户。 +4. 资源所有者拥有默认动作集合。 +5. 租户 owner/admin 按租户范围拥有管理动作。 +6. ACL 明确授予当前用户或租户角色对应动作。 +7. 对 download、delete、admin 等高风险动作执行额外审批检查。 + +### 5.4 资源标准字段 + +所有新资源应统一具备： + +```text +id +tenant_id NOT NULL +created_by NOT NULL +created_at +updated_at +deleted_at +deleted_by +``` + +历史 `owner_id`、`create_by` 和 `payload.created_by` 只在迁移阶段读取，最终写入标准字段。任务配置 JSON 可以保留业务参数，但不再存放权限事实。 + +## 六、标准业务流程 + +### 6.1 创建租户流程 + +```text +平台管理员 + │ + ├─ 创建租户基本信息 + ├─ 设置租户配额和状态=active + ├─ 指定或创建租户 owner + ├─ 写入 tenant_members(tenant_id, owner_user_id, owner) + ├─ 初始化审批策略 + └─ 记录 tenant.create / tenant.member.add 审计 +``` + +纠偏要求：`tenants.owner_user_id` 和 `tenant_members(role='owner')` 必须在同一事务中维护；不能只写一个。 + +### 6.2 创建用户与加入租户流程 + +#### 平台管理员创建用户 + +```text +平台管理员 + │ + ├─ 创建 users 记录 + ├─ 选择初始平台角色：platform_user 或 platform_admin + ├─ 选择 primary tenant + ├─ 写入 tenant_members + │ ├─ owner 仅平台管理员明确指定时允许 + │ └─ 普通用户默认 member 或 viewer + ├─ 生成页面权限快照（过渡期） + └─ 记录 user.create / tenant.member.add 审计 +``` + +#### 已存在用户加入新租户 + +```text +租户 owner/admin 或平台管理员 + │ + ├─ 发出邀请或创建加入申请 + ├─ tenant_members.status = pending + ├─ 被邀请用户接受 + ├─ tenant_members.status = active + └─ 用户选择 X-Tenant-ID 后访问该租户资源 +``` + +纠偏要求：不能把“创建登录用户”和“拥有某租户全部资源”理解为同一件事；新用户默认没有任何资源 ACL，也没有 GPU 分配。 + +### 6.3 创建业务资源流程 + +```text +登录用户 + │ + ├─ 会话校验 + ├─ 确定 active_tenant + ├─ 校验租户 active 和用户成员状态 + ├─ 写入资源 tenant_id + created_by + ├─ 写入资源业务表 + ├─ 写入 MinIO 对象元数据（如有文件） + ├─ 生成资源血缘/版本快照 + └─ 记录资源创建审计 +``` + +数据处理结果生成数据集时，必须继承任务的租户和创建者；不能因为是系统生成就把 `tenant_id` 和 `created_by` 留空。 + +### 6.4 资源查看和授权流程 + +```text +用户请求资源列表/详情 + │ + ├─ 页面权限检查 + ├─ active_tenant 检查 + ├─ 资源 tenant_id 检查 + ├─ 所有者 / 租户角色 / ACL 判定 + ├─ 返回资源元数据 + └─ 文件、版本、MinIO 对象再次按同一资源动作校验 +``` + +资源授权： + +```text +资源所有者或租户管理员/平台管理员 + │ + ├─ 选择用户或租户角色 + ├─ 选择 read/write/execute/download 等动作 + ├─ 设置有效期 + ├─ 写入 ACL 或创建访问审批 + └─ 记录授权来源、授权人和审计 +``` + +### 6.5 训练、推理、评测使用资源流程 + +```text +创建任务 + │ + ├─ 页面权限：fine-tune / model-inference / model-eval + ├─ active_tenant 和任务资源归属校验 + ├─ 基座模型：平台共享规则或 model.execute + ├─ 训练模型：trained_model.execute + ├─ 数据集：dataset.execute + ├─ GPU assignment：用户是否获批使用所选卡 + ├─ 节点：GPU 所属节点是否 online/enabled + ├─ 原子预留 GPU 和租户配额 + ├─ MinIO/节点缓存准备 + ├─ 提交计算任务 + └─ 成功、失败、取消、超时、节点故障统一释放预留 +``` + +### 6.6 高风险操作审批流程 + +```text +用户发起高风险操作 + │ + ├─ 检查当前用户是否有基础资源权限 + ├─ 检查是否为管理员旁路 + ├─ 匹配 tenant + action + resource_type 审批策略 + ├─ 创建 pending 审批实例 + ├─ 审批人从当前会话确定 + ├─ approved -> 幂等执行具体效果 + │ ├─ ACL 授权 + │ ├─ GPU 分配 + │ └─ 配额更新 + ├─ rejected/cancelled/expired -> 不产生资源效果 + └─ 全流程记录审计 +``` + +## 七、纠偏开发计划 + +### 阶段 0：冻结口径 + +1. 书面确认：不再把项目作为新业务隔离层。 +2. 确认平台角色只有平台管理员和平台用户两类；租户职责由 `tenant_members.role` 表达。 +3. 确认 `tenant_members` 是租户关系权威表，`users.tenant_id` 仅作为 primary tenant 兼容字段。 +4. 确认所有资源必须有 `tenant_id`、`created_by`、生命周期字段。 +5. 确认基座模型共享不等于训练模型和数据集共享。 + +### 阶段 1：身份和租户纠偏 + +1. 为平台角色建立统一映射，停止新增 `role='user'` 的特殊逻辑。 +2. 用户创建、租户创建、成员邀请、成员接受、成员移除统一走成员服务。 +3. 创建租户时同步 owner 成员记录。 +4. 租户成员变更、用户禁用、用户软删除时统一撤销会话、GPU 分配、ACL、待审批流程。 +5. 新增当前租户查询和切换接口，前端显示“当前租户”，避免仅显示用户主租户。 + +### 阶段 2：资源数据纠偏 + +1. 对所有资源表补齐并回填 `tenant_id`、`created_by`、`created_at`、`updated_at`。 +2. 将任务 payload 中的创建者和租户迁移到结构化字段。 +3. 对历史 `tenant_id IS NULL` 数据按以下顺序处理： + - 能从创建者找到租户的，回填创建者所属租户。 + - 系统生成数据集能从源任务找到租户的，继承源任务租户。 + - 仍无法确定归属的，进入待治理清单，不自动公开。 + - 管理员明确授权的历史资源可临时通过用户级 ACL 访问，直到完成归属迁移。 +4. 为资源和租户建立必要索引；在数据清洗完成后再逐步收紧 `NOT NULL` 和外键。 + +### 阶段 3：权限判定收敛 + +1. 建立统一 `authorize()` 服务，所有列表、详情、文件、版本、下载、MinIO、缓存和任务接口调用同一套判定。 +2. 页面权限只用于模块入口；资源动作必须在后端独立判断。 +3. 统一基座模型、训练模型、数据集和任务的血缘权限来源。 +4. 统一 GPU assignment、GPU reservation 和租户配额 reservation 的状态流转。 +5. 审批实例增加幂等键和过期处理任务，避免重复审批和重复执行。 + +### 阶段 4：数据库和初始化脚本 + +1. 将完整初始化 SQL 与迁移 SQL 的职责分开：新库只执行一份完整脚本，旧库执行版本化迁移。 +2. 初始化 SQL 中统一种子用户、种子角色、默认租户和租户成员关系，不能出现 operator 仍拥有普通用户不应有的 `compute` 权限而代码又剥离该权限的矛盾。 +3. 给所有需要的关联增加外键或由应用层统一保证引用完整性。 +4. 为 `tenant_id + status`、`created_by + deleted_at`、ACL 主体、审批状态和 GPU 状态建立索引。 +5. 主工程与 `docker/offline/src` 的完整 SQL、迁移、代码和部署文档做 SHA-256 一致性校验。 + +### 阶段 5：前端流程纠偏 + +1. 用户管理页面分别显示平台角色、所属租户、租户角色、页面权限和资源权限，不再把它们放在一个“权限”字段中。 +2. 租户详情页显示成员、租户角色、配额使用、审批策略和资源统计。 +3. 资源详情页显示资源所属租户、创建者、授权主体、动作、有效期和授权来源。 +4. 训练/推理/评测页面只显示当前用户有 `execute` 权限且已获批 GPU 的资源。 +5. 对“无权访问”“未授权”“审批中”“资源已删除”“租户已停用”分别展示原因，不统一显示为数据为空或 500。 + +## 八、建议的验收矩阵 + +| 场景 | 预期结果 | +|---|---| +| 用户未加入租户访问租户资源 | 403 | +| 用户属于租户但没有资源 ACL | 资源列表不可见，详情 403 | +| 用户获得 dataset.read | 数据集可见，可预览，但不能训练或上传 | +| 用户获得 dataset.execute | 数据集可用于训练/评测，但不能编辑或下载 | +| 用户获得 dataset.download | 可以下载文件，但不能训练 | +| 用户获得 trained_model.execute | 可以推理/评测，不能导出 | +| 用户获得 trained_model.download | 可以导出或下载，不能执行推理 | +| 用户获批 GPU 但没有数据集 execute | 不能创建训练任务 | +| 用户有数据集 execute 但没有 GPU assignment | 不能创建训练任务 | +| 管理员撤销 ACL | 列表、详情、执行、下载均立即重新校验 | +| 用户被禁用 | 会话失效，成员和 ACL 不能继续生效 | +| 租户被停用 | 租户资源不可创建和执行，运行任务进入待处理/失败策略 | +| 历史资源 tenant_id 为空且无 ACL | 普通用户不可见 | +| 历史资源 tenant_id 为空且有管理员直接用户 ACL | 仅指定用户在有效期内可见和按动作使用 | +| 资源删除 | 资源不可见，ACL、审批、对象清理流程被触发 | +| MinIO 不可用 | 资源元数据权限仍可判断，涉及对象的操作返回明确存储故障 | + +## 九、纠偏完成判定 + +满足以下条件，才认为租户与用户权限完成统一，而不是“接口能够调用”： + +- `tenant_members` 成为租户关系唯一权威来源，`users.tenant_id` 只作为兼容主租户。 +- 平台角色和租户角色职责分离，角色权限不再通过多个 JSON/字段重复维护。 +- 新资源全部具备非空租户和创建者，历史资源有明确归属或进入治理清单。 +- 所有资源接口、文件接口、MinIO 接口和任务接口使用统一动作授权。 +- 数据集、基座模型、训练模型和任务之间的派生关系不会自动扩大访问范围。 +- 用户创建、成员邀请、资源授权、GPU 申请、配额申请和审批都能追溯到用户、租户、资源、动作和请求 ID。 +- 初始化 SQL、迁移 SQL、在线数据库和离线源码目录的表结构与权限逻辑一致。 +- 双用户、跨租户、ACL 撤销、权限过期、用户禁用、租户停用、GPU 冲突和 MinIO 故障场景全部通过验证。 + +## 十、2026-08-20 本轮改造结果 + +本轮已按目标层级模型完成一轮可运行的后端、数据库和前端收敛： + +1. `users.platform_role` 已作为平台角色标准字段，值为 `platform_admin` 或 `platform_user`；历史 `users.role` 继续返回，供旧客户端兼容。 +2. `tenant_members` 已作为租户成员关系的权威来源。创建租户时会在同一事务中写入 `owner` 成员；创建用户时单独使用 `tenant_role` 写入成员角色，不再因为平台角色是管理员就自动成为租户 owner。 +3. 用户接口返回 `platform_role`、`tenant_memberships` 和 `tenant_count`，组织页面已区分平台身份与租户成员关系。 +4. `fine_tune_tasks` 已增加结构化 `tenant_id`、`created_by`、`deleted_at`、`deleted_by`；训练列表、详情和 owner 判定优先使用结构化字段，历史 payload 仅作回退。 +5. 新建数据集、训练、评测、推理任务会绑定当前认证租户；普通用户不能仅通过请求体伪造其他租户。平台管理员仍可为指定租户创建资源。 +6. 普通用户的资源列表继续通过租户成员、所有者和 ACL 统一过滤；项目不再出现在前端路由、组织页面和新权限判断中。 +7. 租户删除已补充 `deleted_at/deleted_by` 软删除，并同步禁用该租户的成员关系。 +8. `000_full_init.sql` 已包含本轮标准字段、索引、owner 成员修复和无项目业务依赖的种子数据；`docker/offline/src/backend/app/db/sql` 仅保留这一份完整初始化脚本。 +9. `docker/offline/src` 已同步当前 backend、frontend、compute 源码和构建后的 `frontend-dist`。 + +本轮已在线验证：数据库字段存在、普通用户身份返回、用户租户成员关系返回、租户 owner 自动建立、租户软删除、跨租户伪造创建被 403 拒绝；前端 `npm run build` 通过。项目表、项目字段及旧项目 API 仅保留历史兼容读取，不再作为新业务隔离条件。 + +## 十一、下一步开发计划 + +1. 为资源详情和资源授权页面补充租户、创建者、授权来源、有效期和动作权限的可视化信息。 +2. 将数据处理、数据转换等派生资源的租户和创建者继承改为统一服务，并清理历史 `tenant_id IS NULL` 待治理清单。 +3. 将 GPU assignment、GPU reservation、租户配额 reservation 和审批实例整理为同一状态机，补充节点故障回收验收。 +4. 执行双租户、双用户、ACL 过期/撤销、用户禁用、租户停用、MinIO 故障和多 GPU 冲突的完整回归测试。 diff --git a/frontend/src/api/modules/approval.ts b/frontend/src/api/modules/approval.ts index 575909c..af3bc2e 100644 --- a/frontend/src/api/modules/approval.ts +++ b/frontend/src/api/modules/approval.ts @@ -21,17 +21,49 @@ export interface ApprovalInstance { status: string current_step: number create_time?: string + action?: string | null + tenant_id?: string | null + execution_status?: string | null steps: Array } +export interface ResourceAccessRequest { + id: string + tenant_id: string + resource_type: string + resource_id: string + applicant_id: string + principal_type: string + principal_id: string + requested_permissions: string[] + reason?: string | null + approval_id?: string | null + status: string + expires_at?: string | null + created_at?: string + cancelled_at?: string | null + cancelled_by?: string | null +} + +export interface GpuRequestOption { + id: string + code: string + name: string + gpus: Array<{ + index: number + name: string + memory_total_gb: number + }> +} + export const getApprovalTemplates = () => get('/approvals/templates') export const createApprovalTemplate = (payload: { name: string; steps: ApprovalStep[] }) => post('/approvals/templates', payload) -export const getApprovalInstances = (status?: string) => - get('/approvals', { status }) +export const getApprovalInstances = (status?: string, mine = false) => + get('/approvals', { status, mine: mine || undefined }) export const createApprovalInstance = (payload: { template_id?: string @@ -46,5 +78,31 @@ export const getApprovalInstance = (id: string) => export const decideApproval = ( id: string, step_index: number, - payload: { approver_id: string; approved: boolean; comment?: string }, + payload: { approved: boolean; comment?: string }, ) => post(`/approvals/${id}/steps/${step_index}/decision`, payload) + +export const createResourceAccessRequest = (payload: { + resource_type: string + resource_id: string + principal_type?: 'user' | 'role' + principal_id?: string + requested_permissions: string[] + reason?: string + expires_at?: string +}) => post('/approvals/resource-access/requests', payload) + +export const getResourceAccessRequests = (status?: string) => + get('/approvals/resource-access/requests', { status }) + +export const cancelResourceAccessRequest = (id: string) => + post(`/approvals/resource-access/requests/${id}/cancel`, {}) + +export const getGpuRequestOptions = () => + get<{ nodes: GpuRequestOption[] }>('/approvals/gpu-options') + +export const requestGpuAccess = (payload: { + assignments: Array<{ node_id: string; gpu_index: number }> + reason?: string +}) => post<{ approval_required?: boolean; approval_id?: string; approval?: ApprovalInstance }>( + '/approvals/gpu-requests', payload, +) diff --git a/frontend/src/api/modules/audit.ts b/frontend/src/api/modules/audit.ts index 511d4db..09b16c2 100644 --- a/frontend/src/api/modules/audit.ts +++ b/frontend/src/api/modules/audit.ts @@ -11,6 +11,11 @@ export interface AuditLog { target_id?: string detail?: string client_ip?: string + result?: string + reason?: string + request_id?: string + session_id?: string + metadata?: string | Record time?: string } diff --git a/frontend/src/api/modules/compare.ts b/frontend/src/api/modules/compare.ts index c58f195..3d0a7ad 100644 --- a/frontend/src/api/modules/compare.ts +++ b/frontend/src/api/modules/compare.ts @@ -1,4 +1,4 @@ -import { get, post, del } from '../request' +import { get, post, del, getAuthHeaders } from '../request' import type { CompareTask, CompareModelRef } from '@/types' const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000 @@ -78,7 +78,10 @@ export const streamChatReal = (data: any): Promise => { } return fetch('/modelTF/model-compare/stream-chat', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Content-Type': 'application/json', + ...getAuthHeaders(), + }, body: JSON.stringify({ messages, temperature: data.temperature ?? 0.7, diff --git a/frontend/src/api/modules/data-convert.ts b/frontend/src/api/modules/data-convert.ts index 1b30675..4511aa6 100644 --- a/frontend/src/api/modules/data-convert.ts +++ b/frontend/src/api/modules/data-convert.ts @@ -10,7 +10,12 @@ export interface DataConvertTask { output_count: number error_message: string create_time: string - update_time?: string + update_time?: string + created_by?: string | null + creator_name?: string | null + processed_by?: string | null + processor_name?: string | null + processed_at?: string | null input_files?: Array<{ name: string; size: number }> } diff --git a/frontend/src/api/modules/model.ts b/frontend/src/api/modules/model.ts index 835212a..c27de85 100644 --- a/frontend/src/api/modules/model.ts +++ b/frontend/src/api/modules/model.ts @@ -38,6 +38,10 @@ export interface ModelExportJob { payload?: Record create_time?: string completed_at?: string + created_by?: string + tenant_id?: string + archive_status?: string + archive_error?: string } /** 模型列表 */ @@ -51,7 +55,7 @@ export const getModelByName = (name: string) => get(`/model-manage/na /** 本地模型路径列表 */ export const getLocalModels = () => - get<{ models: { path: string; name: string; source?: string }[] }>('/model-manage/local-models') + get<{ models: { path: string; name: string; storage_status?: string }[] }>('/model-manage/local-models') /** 已训练模型列表 */ export const getTrainedModels = () => @@ -97,9 +101,16 @@ export const mergeModel = (data: { output_model_name?: string }) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 }) -/** 导出已训练模型权重 */ -export const exportModelUrl = (modelName: string) => - `/modelTF/model-manage/trained-models/${encodeURIComponent(modelName)}/export` +/** 导出已训练模型权重，沿用节点缓存和 MinIO 归档流程 */ +export const exportModel = (data: { + trained_model_id?: string | number + model_name?: string + base_model_path?: string + adapter_path?: string + compute_node_id?: string + output_model_name?: string + export_quantization_bit?: 0 | 4 | 8 +}) => post('/model-manage/export', data, { timeout: 15 * 60 * 1000 }) /** 测试在线模型连通性 */ export const testOnlineModel = (data: { diff --git a/frontend/src/api/modules/tenant.ts b/frontend/src/api/modules/tenant.ts index 843fa96..0b71ddd 100644 --- a/frontend/src/api/modules/tenant.ts +++ b/frontend/src/api/modules/tenant.ts @@ -9,11 +9,37 @@ export interface Tenant { quota: Record retention_policy_id?: string | null create_time?: string + deleted_at?: string | null + deleted_by?: string | null +} + +export interface TenantMember { + tenant_id: string + user_id: string + username?: string + display_name?: string + role: 'owner' | 'admin' | 'member' | 'viewer' + status: 'active' | 'pending' | 'disabled' | 'expired' + invited_by?: string | null + joined_at?: string | null + expires_at?: string | null +} + +export interface TenantQuotaUsage { + tenant_id: string + quota: Record + gpu_limit: number + gpu_reserved: number + storage_reserved: number + reservations: number } /** 租户列表 */ export const getTenants = () => get('/tenants') +/** 当前用户可切换的 active 租户及成员角色 */ +export const getMyTenants = () => get('/tenants/mine') + /** 租户详情 */ export const getTenant = (id: string) => get(`/tenants/${id}`) @@ -28,6 +54,10 @@ export const updateTenant = (id: string, payload: Partial) => /** 删除租户 */ export const deleteTenant = (id: string) => del(`/tenants/${id}`) +/** 恢复软删除租户 */ +export const restoreTenant = (id: string) => + post(`/tenants/${id}/restore`, {}) + /** 设置租户配额 */ export const setTenantQuota = (id: string, quota: Record) => put(`/tenants/${id}/quota`, quota) @@ -35,3 +65,26 @@ export const setTenantQuota = (id: string, quota: Record) => /** 设置租户留存策略 */ export const setTenantRetention = (id: string, retention_policy_id: string) => put(`/tenants/${id}/retention-policy`, { retention_policy_id }) + +export const getTenantMembers = (id: string) => + get(`/tenants/${id}/members`) + +export const inviteTenantMember = ( + id: string, + payload: { user_id: string; role?: TenantMember['role']; expires_at?: string }, +) => post(`/tenants/${id}/members/invite`, payload) + +export const updateTenantMember = (id: string, userId: string, payload: Partial) => + put(`/tenants/${id}/members/${userId}`, payload) + +export const removeTenantMember = (id: string, userId: string) => + del(`/tenants/${id}/members/${userId}`) + +export const acceptTenantInvitation = (id: string, userId: string) => + post(`/tenants/${id}/members/${userId}/accept`, {}) + +export const getTenantInvitations = () => + get('/tenants/invitations') + +export const getTenantQuotaUsage = (id: string) => + get(`/tenants/${id}/quota/usage`) diff --git a/frontend/src/api/request.ts b/frontend/src/api/request.ts index 6b17c2d..0c080d3 100644 --- a/frontend/src/api/request.ts +++ b/frontend/src/api/request.ts @@ -70,13 +70,27 @@ function getAuthToken(): string | null { return null } +function getActiveTenantId(): string | null { + return localStorage.getItem('activeTenantId') +} + +/** Headers shared by Axios and raw fetch requests such as SSE streaming. */ +export function getAuthHeaders(): Record { + const headers: Record = {} + const token = getAuthToken() + if (token) headers.Authorization = `Bearer ${token}` + const tenantId = getActiveTenantId() + if (tenantId) headers['X-Tenant-ID'] = tenantId + return headers +} + // 请求拦截器：注入 Authorization header service.interceptors.request.use( (config) => { - const token = getAuthToken() - if (token) { + const authHeaders = getAuthHeaders() + if (Object.keys(authHeaders).length) { config.headers = config.headers || {} - config.headers['Authorization'] = `Bearer ${token}` + Object.assign(config.headers, authHeaders) } return config }, diff --git a/frontend/src/components/AppHeader.vue b/frontend/src/components/AppHeader.vue index 970eb9f..ea8e803 100644 --- a/frontend/src/components/AppHeader.vue +++ b/frontend/src/components/AppHeader.vue @@ -3,11 +3,17 @@ import { computed, ref, onMounted, onUnmounted } from 'vue' import { storeToRefs } from 'pinia' import { useSystemStore } from '@/stores/system' import { useRoute, useRouter } from 'vue-router' +import { ElMessage } from 'element-plus' +import { getMyTenants, type Tenant } from '@/api/modules/tenant' +import { useAuthStore } from '@/stores/auth' const systemStore = useSystemStore() const { metrics } = storeToRefs(systemStore) const route = useRoute() const router = useRouter() +const auth = useAuthStore() +const tenants = ref([]) +const activeTenantId = ref(localStorage.getItem('activeTenantId') || auth.currentUser?.tenant_id || (auth.isAdmin ? 'admin' : 'default')) const showBackButton = computed(() => { return route.path.split('/').filter(Boolean).length > 1 @@ -41,11 +47,27 @@ function openGuide() { window.open(guideUrl, '_blank', 'noopener,noreferrer') } +async function loadTenants() { + tenants.value = await getMyTenants().catch(() => []) + if (!tenants.value.some((tenant) => tenant.id === activeTenantId.value)) { + activeTenantId.value = tenants.value[0]?.id || (auth.isAdmin ? 'admin' : 'default') + localStorage.setItem('activeTenantId', activeTenantId.value) + } +} + +function switchTenant(value: string) { + activeTenantId.value = value + localStorage.setItem('activeTenantId', value) + ElMessage.success('当前租户已切换，正在刷新页面数据') + window.location.reload() +} + const serverIp = ref(window.location.hostname) onMounted(() => { updateTime() timer = setInterval(updateTime, 1000) + void loadTenants() }) onUnmounted(() => { @@ -102,6 +124,13 @@ onUnmounted(() => {
+
+ + + + +
+
@@ -256,6 +285,22 @@ onUnmounted(() => { background-color: #e2e8f0; } +.tenant-switcher { + display: flex; + align-items: center; + gap: 6px; + min-width: 150px; + color: #64748b; + + .fa { + color: var(--primary-color); + } + + :deep(.el-select) { + width: 132px; + } +} + .system-info { display: flex; align-items: center; diff --git a/frontend/src/components/AppSidebar.vue b/frontend/src/components/AppSidebar.vue index 64c8e8e..29eac31 100644 --- a/frontend/src/components/AppSidebar.vue +++ b/frontend/src/components/AppSidebar.vue @@ -38,6 +38,7 @@ interface MenuItem { icon: string to: string permission: PermissionCode + allowNonAdmin?: boolean } interface MenuGroup { @@ -83,7 +84,7 @@ const menuGroups: MenuGroup[] = [ items: [ { key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' }, { key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' }, - { key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' }, + { key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances?tab=mine', permission: 'user-settings', allowNonAdmin: true }, ], }, { @@ -115,10 +116,12 @@ const visibleMenuGroups = computed(() => .map((group) => ({ ...group, items: group.items.filter((item) => { - // admin 可以看到所有菜单 - if (auth.isAdmin) return true - // 非 admin 用户：仅隐藏管理员专属菜单 - return !ADMIN_ONLY_PERMISSIONS.includes(item.permission) + // admin 可以看到所有菜单 + if (auth.isAdmin) return true + // 运行日志是普通用户可选的自助权限；没有权限时不显示入口。 + if (item.permission === 'logs') return auth.hasPermission('logs') + // 非 admin 用户：仅隐藏管理员专属菜单 + return item.allowNonAdmin || !ADMIN_ONLY_PERMISSIONS.includes(item.permission) }), })) .filter((group) => group.items.length > 0), diff --git a/frontend/src/mock/users.ts b/frontend/src/mock/users.ts index ff340b3..f4a8e4f 100644 --- a/frontend/src/mock/users.ts +++ b/frontend/src/mock/users.ts @@ -43,12 +43,13 @@ function defaultUsers(): SystemUser[] { { id: 'u_admin', username: 'admin', - display_name: 'Platform Admin', + display_name: 'Admin', role: 'admin', status: 'active', permissions: allPermissions, - create_time: '2026-01-01T00:00:00Z', - protected: true, + create_time: '2026-01-01T00:00:00Z', + tenant_id: 'admin', + protected: true, }, { id: 'u_operator', @@ -57,8 +58,9 @@ function defaultUsers(): SystemUser[] { role: 'operator', status: 'active', permissions: allPermissions.filter((item) => item !== 'user-settings'), - create_time: '2026-01-01T00:00:00Z', - protected: false, + create_time: '2026-01-01T00:00:00Z', + tenant_id: 'default', + protected: false, }, ] } @@ -88,7 +90,7 @@ export function authenticateMockUser(username: string, password: string): LoginR if (!user || user.status !== 'active') { throw new UserMutationError('Invalid username or disabled account', 401) } - const expected = defaultPasswords[username] || 'platform123' + const expected = defaultPasswords[username] || '123456' if (password !== expected) { throw new UserMutationError('Invalid username or password', 401) } @@ -115,7 +117,7 @@ export function createMockUser(payload: CreateUserPayload): SystemUser { create_time: new Date().toISOString(), protected: false, } - defaultPasswords[user.username] = payload.password || 'platform123' + defaultPasswords[user.username] = payload.password || '123456' users.push(user) writeUsers(users) return user diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index ede4f42..754dc41 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -50,18 +50,6 @@ const routes: RouteRecordRaw[] = [ component: () => import('@/views/tenants/TenantDetailView.vue'), meta: { title: '租户详情', permission: 'user-settings' }, }, - { - path: 'projects', - name: 'projects', - redirect: '/organization?tab=users', - meta: { title: '组织与权限', permission: 'user-settings' }, - }, - { - path: 'projects/:id', - name: 'project-detail', - redirect: '/organization?tab=users', - meta: { title: '组织与权限', permission: 'user-settings' }, - }, { path: 'audit-logs', name: 'audit-logs', @@ -72,7 +60,7 @@ const routes: RouteRecordRaw[] = [ path: 'operation-logs', name: 'operation-logs', redirect: '/logs?tab=operations', - meta: { title: '运行日志', permission: 'user-settings' }, + meta: { title: '运行日志', permission: 'logs' }, }, { path: 'approval-templates', @@ -84,7 +72,17 @@ const routes: RouteRecordRaw[] = [ path: 'approval-instances', name: 'approval-instances', component: () => import('@/views/approvals/ApprovalCenterView.vue'), - meta: { title: '审批中心', permission: 'user-settings' }, + meta: { title: '审批中心', permission: 'user-settings', selfService: true }, + }, + { + path: 'tenant-invitations', + redirect: '/approval-instances?tab=invitations', + meta: { title: '审批中心', selfService: true }, + }, + { + path: 'resource-access-requests', + redirect: '/approval-instances?tab=access', + meta: { title: '审批中心', selfService: true }, }, { path: 'resource-acl', @@ -366,7 +364,6 @@ const permissionBySegment: Record = { organization: 'user-settings', 'user-settings': 'user-settings', tenants: 'user-settings', - projects: 'user-settings', 'audit-logs': 'user-settings', 'operation-logs': 'user-settings', 'approval-templates': 'user-settings', @@ -411,17 +408,22 @@ router.beforeEach((to, _from, next) => { if (!to.meta.skipPermission) { const permission = requiredPermission(to.path, to.meta.permission) // 仅限制管理员专属页面的访问权限 - // user-settings（组织与权限、资源授权、审批中心、运行日志）仅 admin 可访问 - if (permission === 'user-settings' && !auth.isAdmin) { + // user-settings（组织与权限、资源授权、审批中心）仅 admin 可访问 + const selfService = to.meta.selfService === true && (!to.query.tab || ['mine', 'access', 'invitations', 'compute'].includes(String(to.query.tab))) + if (permission === 'user-settings' && !auth.isAdmin && !selfService) { next({ name: 'permission-denied', replace: true }) return } // compute（算力节点/GPU 分配）仅 admin 可访问 - if (permission === 'compute' && !auth.isAdmin) { + if (permission === 'compute' && !auth.isAdmin) { + next({ name: 'permission-denied', replace: true }) + return + } + if (permission === 'logs' && !auth.isAdmin && !auth.hasPermission('logs')) { next({ name: 'permission-denied', replace: true }) return } - // 其他所有业务页面对已登录用户开放，不再检查权限码 + // 其他所有业务页面对已登录用户开放，不再检查权限码 } // 路由切换时记录业务模块访问（用于看板用户操作分布统计） diff --git a/frontend/src/stores/auth.ts b/frontend/src/stores/auth.ts index 86c04a6..98c68d9 100644 --- a/frontend/src/stores/auth.ts +++ b/frontend/src/stores/auth.ts @@ -68,6 +68,13 @@ export const useAuthStore = defineStore('auth', () => { return currentUser.value?.permissions.includes(permission) ?? false } + /** Button-level decision for a module action. The API remains the final authority. */ + function can(permission?: PermissionCode, action?: 'read' | 'write' | 'execute' | 'download' | 'delete' | 'admin') { + if (!hasPermission(permission)) return false + if (action === 'admin') return isAdmin.value + return true + } + /** 退出 */ async function logout() { const sessionId = sessionStorage.getItem(SESSION_STORAGE_KEY) @@ -88,6 +95,7 @@ export const useAuthStore = defineStore('auth', () => { isLoggedIn, isAdmin, hasPermission, + can, login, logout, } diff --git a/frontend/src/types/dataProcess.ts b/frontend/src/types/dataProcess.ts index 63e86b9..8832537 100644 --- a/frontend/src/types/dataProcess.ts +++ b/frontend/src/types/dataProcess.ts @@ -50,8 +50,11 @@ export interface DataProcessTask { filtered_count?: number duplicate_count?: number error_count?: number - creator_name?: string | null - creator?: string | null + creator_name?: string | null + creator?: string | null + processor_name?: string | null + processor?: string | null + updated_by?: string | number | null created_by?: string | number | null create_time?: string created_at?: string diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 722b186..1aabe68 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -24,7 +24,11 @@ export interface ModelItem { path?: string api_url?: string api_key?: string + /** Returned by the server without exposing the actual credential. */ + api_key_configured?: boolean online_model_name?: string + storage_status?: 'pending' | 'archiving' | 'available' | 'failed' | 'not_applicable' | string + storage_error?: string create_time?: string } @@ -42,6 +46,8 @@ export interface TrainedModel { merged?: boolean merging?: boolean merged_path?: string + created_by?: string + tenant_id?: string } /** 创建/编辑模型请求体 */ @@ -88,8 +94,10 @@ export interface DatasetItem { size?: string | number size_bytes?: number count?: number - description?: string - create_time?: string + description?: string + created_by?: string | number | null + creator_name?: string | null + create_time?: string files?: DatasetFile[] current_version_no?: number | null current_version_nos?: number[] @@ -168,6 +176,8 @@ export interface FineTuneTask { log_file?: string train_duration?: string create_time?: string + created_by?: string + tenant_id?: string } export type FineTuneStartPayload = Omit< @@ -264,6 +274,19 @@ export interface EvalTask { score?: number status?: string create_time?: string + progress?: number + progress_detail?: EvalProgress +} + +export interface EvalProgress { + status?: string + stage?: string + total?: number + completed?: number + percentage?: number + current_index?: number + message?: string + updated_at?: string } /** 启动评测时提交的可选基础指标配置。 */ @@ -291,7 +314,8 @@ export interface StartEvalPayload { gpus?: number[] compute_node_id?: string dataset_id: string | number - dimension_id: string | number + dimension_id?: string | number + dimension?: Partial data_source: 'dataset' | 'inference' leaderboard: boolean basic_metrics: BasicEvalMetricsConfig @@ -316,6 +340,8 @@ export interface EvalSampleResult { score: number max_score: number }> + raw_score?: number | null + raw_max_score?: number | null } /** 评测任务详情，包含逐样本结果和综合评价 */ @@ -334,8 +360,14 @@ export interface EvalTaskDetail extends EvalTask { score: number max_score: number pass_rate: number + sample_count?: number + available?: boolean + error?: string }> samples: EvalSampleResult[] + progress_detail?: EvalProgress + basic_metrics?: Record> + metric_summary_version?: number } export interface Dimension { @@ -446,20 +478,27 @@ export type PermissionCode = | 'logs' | 'user-settings' -export type UserRole = 'admin' | 'operator' | 'viewer' +export type UserRole = 'admin' | 'operator' | 'viewer' | 'user' -export type UserStatus = 'active' | 'disabled' +export type UserStatus = 'active' | 'disabled' | 'pending' | 'deleted' export interface SystemUser { id: string username: string display_name: string role: UserRole + /** Canonical platform scope; role is retained for legacy API clients. */ + platform_role?: 'platform_admin' | 'platform_user' status: UserStatus permissions: PermissionCode[] create_time: string last_login?: string protected?: boolean + tenant_id?: string + deleted_at?: string | null + deleted_by?: string | null + tenant_memberships?: Array<{ tenant_id: string; role: string; status: string; expires_at?: string | null }> + tenant_count?: number } export interface LoginResponse { @@ -475,6 +514,8 @@ export interface CreateUserPayload { role: UserRole status?: UserStatus permissions?: PermissionCode[] + tenant_id?: string + tenant_role?: 'owner' | 'admin' | 'member' | 'viewer' } export interface UpdateUserAccessPayload { diff --git a/frontend/src/views/approvals/ApprovalCenterView.vue b/frontend/src/views/approvals/ApprovalCenterView.vue index dd6817b..972f6bf 100644 --- a/frontend/src/views/approvals/ApprovalCenterView.vue +++ b/frontend/src/views/approvals/ApprovalCenterView.vue @@ -1,14 +1,24 @@ + + + - - - 配额设置（可选，0 表示不限制） @@ -153,9 +174,6 @@ onMounted(load) - - -