feat: 权限与日志治理完善,MinIO 独立部署与 tiktoken 离线打包适配
- 后端:强化平台/审批/资源/系统接口权限校验与操作日志,更新权限设计文档与测试用例 - 存储:新增 MinIO 独立部署适配(端口 19000/19001),外部端点与 host-gateway 互通 - 离线:打包 tiktoken cl100k_base 词表进镜像,避免无网环境联网下载 - 其他:算力节点接口微调,前端微调创建页小修,忽略 MinIO 运行时数据 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -211,3 +211,7 @@ docker/compute/data/yg-ft/logs/**
|
||||
|
||||
# Offline deployment bundle - 离线部署包(镜像、运行时等大文件,不提交)
|
||||
docker/offline/
|
||||
|
||||
# MinIO object storage data - 对象存储运行时数据,勿提交,保留目录结构
|
||||
docker/minio/data/*
|
||||
!docker/minio/data/.gitkeep
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.core.config import get_settings
|
||||
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__)
|
||||
@@ -9,10 +11,30 @@ logger = get_logger(__name__)
|
||||
|
||||
@router.get("/health")
|
||||
async def health_check() -> dict[str, object]:
|
||||
logger.info("health check requested")
|
||||
# Health endpoints are called frequently by Docker and the frontend.
|
||||
# Keep failures visible without emitting one INFO line per probe.
|
||||
logger.debug("health check requested")
|
||||
storage_status: dict[str, object] = {"enabled": get_settings().minio_enabled, "status": "disabled"}
|
||||
if get_settings().minio_enabled:
|
||||
try:
|
||||
get_object_storage().ensure_bucket()
|
||||
storage_status = {
|
||||
"enabled": True,
|
||||
"status": "ready",
|
||||
"endpoint": get_settings().minio_endpoint,
|
||||
"bucket": get_settings().minio_bucket,
|
||||
}
|
||||
except (ObjectStorageError, OSError) as exc:
|
||||
logger.warning("MinIO health check failed: %s", exc)
|
||||
storage_status = {
|
||||
"enabled": True,
|
||||
"status": "unavailable",
|
||||
"endpoint": get_settings().minio_endpoint,
|
||||
"error": str(exc),
|
||||
}
|
||||
return {
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": get_platform_store().health_metrics(),
|
||||
"data": {**get_platform_store().health_metrics(), "storage": storage_status},
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
import asyncio
|
||||
import hashlib
|
||||
import uuid
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -13,7 +14,7 @@ from fastapi.responses import PlainTextResponse, StreamingResponse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
|
||||
from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
@@ -21,6 +22,21 @@ from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_com
|
||||
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
|
||||
|
||||
router = APIRouter()
|
||||
_LOGIN_FAILURES: dict[str, list[float]] = {}
|
||||
_DASHBOARD_CACHE_TTL = 5.0
|
||||
_DASHBOARD_CACHE: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _cached_dashboard(key: str) -> dict[str, Any] | None:
|
||||
item = _DASHBOARD_CACHE.get(key)
|
||||
if not item or time.monotonic() - item["created_at"] >= _DASHBOARD_CACHE_TTL:
|
||||
return None
|
||||
return item["value"]
|
||||
|
||||
|
||||
def _store_dashboard_cache(key: str, value: dict[str, Any]) -> dict[str, Any]:
|
||||
_DASHBOARD_CACHE[key] = {"created_at": time.monotonic(), "value": value}
|
||||
return value
|
||||
|
||||
|
||||
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
|
||||
@@ -77,6 +93,27 @@ def _candidate_online_nodes(store: Any, preferred_node_id: str | None = None) ->
|
||||
return preferred + others
|
||||
|
||||
|
||||
async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id: str, node: dict[str, Any]) -> str | None:
|
||||
"""Prepare MinIO resource files on a node and return the local directory."""
|
||||
if not get_settings().minio_enabled or not resource_id:
|
||||
return None
|
||||
objects = store.storage_objects_for_resource(resource_type, resource_id)
|
||||
if not objects:
|
||||
return None
|
||||
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
||||
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
|
||||
for obj in objects:
|
||||
await client.prepare_cache({
|
||||
"resource_id": resource_id,
|
||||
"version_id": obj["version_id"],
|
||||
"download_url": get_object_storage().presigned_get(obj["object_key"]),
|
||||
"checksum_sha256": obj.get("checksum_sha256") or "",
|
||||
"byte_size": obj.get("byte_size") or 0,
|
||||
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
|
||||
})
|
||||
return f"/data/yg-ft/{root_name}/{resource_id}"
|
||||
|
||||
|
||||
def _build_messages_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Convert frontend inference payload to compute API messages format.
|
||||
|
||||
@@ -396,13 +433,20 @@ async def _fine_tune_preflight_with_job_payload(
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
ip = request.client.host if request and request.client else "unknown"
|
||||
now = time.time()
|
||||
recent = [stamp for stamp in _LOGIN_FAILURES.get(ip, []) if now - stamp < 300]
|
||||
if len(recent) >= 5:
|
||||
raise fail(429, "too many login attempts, retry later")
|
||||
user = store.login(payload.get("username", ""), payload.get("password", ""))
|
||||
if not user:
|
||||
_LOGIN_FAILURES[ip] = [*recent, now]
|
||||
raise fail(401, "invalid username or password")
|
||||
sess = store.create_session(user["id"])
|
||||
return ok({"token": f"platform-token-{user['id']}", "user": user, "session_id": sess["session_id"]})
|
||||
_LOGIN_FAILURES.pop(ip, None)
|
||||
sess = store.create_session(user["id"], ip=None)
|
||||
return ok({"token": f"platform-token-{user['id']}.{sess['session_id']}", "user": user, "session_id": sess["session_id"]})
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
@@ -422,7 +466,7 @@ async def me(request: Request) -> dict[str, Any]:
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
# token 格式: platform-token-{user_id}
|
||||
if token.startswith("platform-token-"):
|
||||
user_id = token[len("platform-token-"):]
|
||||
user_id = token[len("platform-token-"):].split(".", 1)[0]
|
||||
for u in store.users():
|
||||
if u.get("id") == user_id:
|
||||
return ok(u)
|
||||
@@ -431,9 +475,12 @@ async def me(request: Request) -> dict[str, Any]:
|
||||
|
||||
@router.get("/dashboard/overview")
|
||||
async def dashboard_overview() -> dict[str, Any]:
|
||||
cached = _cached_dashboard("overview")
|
||||
if cached is not None:
|
||||
return cached
|
||||
store = get_platform_store()
|
||||
tasks = store.tasks()
|
||||
return ok(
|
||||
return _store_dashboard_cache("overview", ok(
|
||||
{
|
||||
"models": len(store.models()),
|
||||
"datasets": len(store.datasets()),
|
||||
@@ -442,11 +489,14 @@ async def dashboard_overview() -> dict[str, Any]:
|
||||
"compute_nodes": len(store.compute_nodes()),
|
||||
"gpus": len(store.gpus()),
|
||||
}
|
||||
)
|
||||
))
|
||||
|
||||
|
||||
@router.get("/dashboard/stats")
|
||||
async def dashboard_stats() -> dict[str, Any]:
|
||||
cached = _cached_dashboard("stats")
|
||||
if cached is not None:
|
||||
return cached
|
||||
"""看板聚合数据:基于平台真实数据;缺项做合理近似。"""
|
||||
store = get_platform_store()
|
||||
tasks = store.tasks()
|
||||
@@ -603,7 +653,7 @@ async def dashboard_stats() -> dict[str, Any]:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return ok(
|
||||
return _store_dashboard_cache("stats", ok(
|
||||
{
|
||||
"online_services": sum(s["count"] for s in service_status),
|
||||
"running_tasks": len(running_ft) + eval_running,
|
||||
@@ -615,7 +665,7 @@ async def dashboard_stats() -> dict[str, Any]:
|
||||
"login_duration_rank": login_duration_rank,
|
||||
"recent_login_users": recent_login_users,
|
||||
}
|
||||
)
|
||||
))
|
||||
|
||||
|
||||
@router.get("/system-info")
|
||||
@@ -731,7 +781,12 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
|
||||
|
||||
|
||||
@router.delete("/model-manage/trained-models/{model_id}")
|
||||
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
|
||||
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("trained_model", model_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this trained model")
|
||||
pending = _require_approval_or_admin("trained_model", model_id, current_user, f"删除训练模型 {model_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_trained_model(model_id)
|
||||
return ok({"deleted": model_id, "type": type})
|
||||
|
||||
@@ -747,7 +802,9 @@ async def trained_model_lineage(model_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/model-manage/export-jobs")
|
||||
async def model_export_jobs(trained_model_id: str | None = Query(default=None)) -> dict[str, Any]:
|
||||
async def model_export_jobs(trained_model_id: str | None = Query(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if trained_model_id and not has_resource_access("trained_model", trained_model_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access export jobs")
|
||||
return ok(get_platform_store().model_export_jobs(trained_model_id))
|
||||
|
||||
|
||||
@@ -878,7 +935,7 @@ async def delete_model(model_id: str, current_user: dict = Depends(get_current_u
|
||||
|
||||
|
||||
@router.post("/model-manage/merge")
|
||||
async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("model_name") or "")
|
||||
trained_model = next(
|
||||
@@ -889,6 +946,12 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not trained_model:
|
||||
raise fail(404, "trained model not found")
|
||||
if not has_resource_access("trained_model", trained_model["id"], current_user, "execute"):
|
||||
raise fail(403, "no permission to merge this trained model")
|
||||
if payload.get("base_model_id") and not has_resource_access("model", str(payload["base_model_id"]), current_user, "execute"):
|
||||
raise fail(403, "no permission to use merge base model")
|
||||
base_model_path = payload.get("base_model_path") or (trained_model and trained_model.get("base_model_path"))
|
||||
adapter_path = (
|
||||
payload.get("adapter_path")
|
||||
@@ -906,6 +969,15 @@ async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
await _wait_for_object_storage()
|
||||
except RuntimeError as exc:
|
||||
raise fail(503, str(exc))
|
||||
try:
|
||||
prepared_base = await _prepare_resource_on_node(store, "model", str(payload.get("base_model_id") or base_model_path), node)
|
||||
if prepared_base:
|
||||
base_model_path = prepared_base
|
||||
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", str(payload.get("adapter_model_id") or (trained_model and trained_model.get("id")) or ""), node)
|
||||
if prepared_adapter:
|
||||
adapter_path = prepared_adapter
|
||||
except Exception as exc:
|
||||
raise fail(502, f"merge resource preparation failed: {exc}")
|
||||
health = node.get("health_detail") or {}
|
||||
output_root = str(health.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
|
||||
output_name = str(payload.get("output_model_name") or payload.get("merged_model_name") or f"{trained_model_id or 'model'}-merged")
|
||||
@@ -1075,6 +1147,7 @@ async def _sync_training_dataset_to_compute_node(
|
||||
"version_id": obj["version_id"],
|
||||
"download_url": url,
|
||||
"checksum_sha256": obj.get("checksum_sha256") or "",
|
||||
"byte_size": obj.get("byte_size") or 0,
|
||||
"relative_path": f"datasets/{dataset_id}/{target_name}",
|
||||
})
|
||||
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
|
||||
@@ -1258,7 +1331,15 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
|
||||
|
||||
|
||||
@router.post("/fine-tune")
|
||||
async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
if not is_admin(current_user):
|
||||
model_id = str(payload.get("base_model") or payload.get("base_model_id") or "")
|
||||
dataset_id = str(payload.get("train_dataset_id") or "")
|
||||
if model_id and not has_resource_access("model", model_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to use this base model")
|
||||
if dataset_id and not has_resource_access("dataset", dataset_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to use this dataset")
|
||||
try:
|
||||
task = get_platform_store().create_task(payload)
|
||||
return ok({"id": task["id"]})
|
||||
@@ -1275,11 +1356,16 @@ async def start_fine_tune(
|
||||
# GPU 权限校验:普通用户只能使用被分配的 GPU
|
||||
if not is_admin(current_user):
|
||||
node_id = payload.get("compute_node_id") or payload.get("node_id")
|
||||
gpu_indices = payload.get("gpus") or []
|
||||
gpu_indices = payload.get("gpu_indices")
|
||||
if gpu_indices is None:
|
||||
gpu_indices = payload.get("gpus") or []
|
||||
if node_id and gpu_indices:
|
||||
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
|
||||
raise fail(403, "无权使用所选 GPU,请联系管理员分配")
|
||||
# 记录创建者
|
||||
if node_id and not gpu_indices:
|
||||
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
|
||||
payload["strict_node_selection"] = bool(node_id)
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
try:
|
||||
return ok(await _submit_fine_tune_task(store, payload))
|
||||
@@ -1411,7 +1497,9 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.put("/fine-tune/{task_id}")
|
||||
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("fine-tune", task_id, current_user, "write"):
|
||||
raise fail(403, "no permission to update this task")
|
||||
try:
|
||||
return ok(get_platform_store().update_task(task_id, payload))
|
||||
except KeyError:
|
||||
@@ -1442,13 +1530,15 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
|
||||
|
||||
|
||||
@router.post("/fine-tune/{task_id}/retry")
|
||||
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
payload = payload or {}
|
||||
try:
|
||||
task = store.task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
if not has_resource_access("fine-tune", task_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to retry this task")
|
||||
if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
|
||||
raise fail(409, "only failed or stopped tasks can be retried without force=true")
|
||||
retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
|
||||
@@ -1544,7 +1634,7 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
|
||||
|
||||
|
||||
@router.post("/model-eval/start")
|
||||
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""Start an evaluation task: submit eval job to compute node."""
|
||||
store = get_platform_store()
|
||||
# 1. Create eval task record
|
||||
@@ -1745,12 +1835,23 @@ async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/model-compare")
|
||||
async def model_compare_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().compare_tasks())
|
||||
async def model_compare_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
tasks = get_platform_store().compare_tasks()
|
||||
if is_admin(current_user):
|
||||
return ok(tasks)
|
||||
accessible = filter_accessible_resource_ids_batch("compare", [item["id"] for item in tasks], current_user)
|
||||
return ok([item for item in tasks if item["id"] in accessible])
|
||||
|
||||
|
||||
@router.post("/model-compare")
|
||||
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
model_ids = payload.get("model_ids") or payload.get("models") or []
|
||||
if not is_admin(current_user):
|
||||
for model_id in model_ids:
|
||||
if isinstance(model_id, dict): model_id = model_id.get("id") or model_id.get("model_id")
|
||||
if model_id and not has_resource_access("model", str(model_id), current_user, "execute"):
|
||||
raise fail(403, "no permission to use inference model")
|
||||
task = get_platform_store().create_compare_task(payload)
|
||||
return ok({"id": task["id"]})
|
||||
|
||||
@@ -1766,9 +1867,12 @@ async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict
|
||||
|
||||
|
||||
@router.get("/model-compare/{task_id}")
|
||||
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
||||
async def model_compare_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().compare_task(task_id))
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
if not has_resource_access("compare", task_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access inference task")
|
||||
return ok(task)
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
@@ -1805,12 +1909,17 @@ async def _unload_from_compute_node(store: Any, task: dict[str, Any] | None = No
|
||||
|
||||
|
||||
@router.delete("/model-compare/{task_id}")
|
||||
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||||
async def model_compare_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
# 先删记录(快),再 best-effort 释放算力节点上的模型——删除绝不被卸载阻塞
|
||||
try:
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
if not has_resource_access("compare", task_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete inference task")
|
||||
pending = _require_approval_or_admin("compare", task_id, current_user, f"删除推理任务 {task_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_compare_task(task_id)
|
||||
try:
|
||||
await _unload_from_compute_node(get_platform_store(), task=task)
|
||||
@@ -1873,7 +1982,7 @@ def _invalidate_superseded_models(store: Any, task_id: str, loaded_models: list[
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/load")
|
||||
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||
async def model_compare_load(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""异步派发模型加载到算力节点,立即返回。
|
||||
|
||||
加载进度由轮询对账器(compute_poller → reconcile_inference_loads)推进:
|
||||
@@ -1883,6 +1992,8 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
store = get_platform_store()
|
||||
task = store.compare_task(task_id)
|
||||
if not has_resource_access("compare", task_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to load inference task")
|
||||
models = task.get("models") or []
|
||||
if isinstance(models, str):
|
||||
try:
|
||||
@@ -1953,10 +2064,12 @@ async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/unload")
|
||||
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||||
async def model_compare_unload(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
store = get_platform_store()
|
||||
task = store.compare_task(task_id)
|
||||
if not has_resource_access("compare", task_id, current_user, "write"):
|
||||
raise fail(403, "no permission to unload inference task")
|
||||
# 任务感知卸载:只释放该任务实际加载到的节点,短超时快速返回
|
||||
unload_result = await _unload_from_compute_node(store, task=task)
|
||||
updated = store.update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}})
|
||||
@@ -2048,7 +2161,19 @@ async def model_chat_local_unload() -> dict[str, Any]:
|
||||
if not store.is_inference_loaded(n["id"]):
|
||||
continue
|
||||
try:
|
||||
result = await ComputeNodeClient(n["api_base_url"]).inference_unload()
|
||||
client = ComputeNodeClient(n["api_base_url"])
|
||||
last_error = ""
|
||||
result = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
result = await client.inference_unload()
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001 - retry node cleanup
|
||||
last_error = str(exc)
|
||||
if attempt < 2:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
if result is None:
|
||||
raise RuntimeError(last_error or "inference unload failed")
|
||||
results.append({"node_id": n["id"], "success": True, "result": result})
|
||||
except Exception as exc: # noqa: BLE001 - best-effort unload
|
||||
results.append({"node_id": n["id"], "success": False, "error": str(exc)})
|
||||
@@ -2073,19 +2198,31 @@ async def model_chat_local_status() -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("/model-chat/trained/preload")
|
||||
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
resource_id = str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or "")
|
||||
if resource_id and not has_resource_access("trained_model", resource_id, current_user, "execute") and not has_resource_access("model", resource_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to load this model")
|
||||
"""Load a trained model (base + adapter) on the compute node for inference."""
|
||||
model_path = (payload.get("model_name_or_path") or "").strip()
|
||||
if not model_path:
|
||||
return ok({"loaded": False, "error": "model_name_or_path is required"})
|
||||
store = get_platform_store()
|
||||
node = _select_first_online_node(store)
|
||||
requested_node_id = str(payload.get("compute_node_id") or payload.get("node_id") or "")
|
||||
node = next((item for item in store.compute_nodes() if item.get("id") == requested_node_id and item.get("enabled") and item.get("scheduler_status") == "online"), None)
|
||||
if not node:
|
||||
node = _select_first_online_node(store)
|
||||
if not node:
|
||||
return ok({"loaded": False, "error": "no online compute node"})
|
||||
try:
|
||||
prepared_path = await _prepare_resource_on_node(store, "trained_model", str(payload.get("trained_model_id") or payload.get("model_id") or payload.get("resource_id") or ""), node)
|
||||
if prepared_path:
|
||||
payload = {**payload, "model_name_or_path": prepared_path}
|
||||
prepared_path = await _prepare_resource_on_node(store, "model", str(payload.get("model_id") or payload.get("resource_id") or ""), node)
|
||||
if prepared_path:
|
||||
payload = {**payload, "model_name_or_path": prepared_path}
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
||||
result = await client.inference_load(payload)
|
||||
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
|
||||
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
||||
store.mark_inference_loaded(node["id"])
|
||||
return ok(result)
|
||||
@@ -2094,8 +2231,13 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dic
|
||||
|
||||
|
||||
@router.get("/compute/nodes")
|
||||
async def compute_nodes() -> dict[str, Any]:
|
||||
return ok(get_platform_store().compute_nodes())
|
||||
async def compute_nodes(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
nodes = get_platform_store().compute_nodes()
|
||||
if is_admin(current_user):
|
||||
return ok(nodes)
|
||||
# 普通用户只看到自己被分配 GPU 的节点,避免泄露节点拓扑和未授权资源。
|
||||
assigned = {item["node_id"] for item in get_platform_store().gpu_assignments_for_user(current_user["id"])}
|
||||
return ok([node for node in nodes if node["id"] in assigned])
|
||||
|
||||
|
||||
@router.post("/storage/objects/presign")
|
||||
@@ -2154,27 +2296,90 @@ async def prepare_storage_resource(
|
||||
for obj in objects:
|
||||
url = get_object_storage().presigned_get(obj["object_key"])
|
||||
filename = Path(str(obj.get("file_name") or obj["object_key"])).name
|
||||
result = await client.prepare_cache({
|
||||
"resource_id": resource_id,
|
||||
"version_id": obj["version_id"],
|
||||
"download_url": url,
|
||||
"checksum_sha256": obj.get("checksum_sha256") or "",
|
||||
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
|
||||
})
|
||||
cache_job = store.create_storage_cache_job({"storage_object_id": obj["id"], "node_id": node_id, "direction": "download"})
|
||||
try:
|
||||
result = await client.prepare_cache({
|
||||
"resource_id": resource_id,
|
||||
"version_id": obj["version_id"],
|
||||
"download_url": url,
|
||||
"checksum_sha256": obj.get("checksum_sha256") or "",
|
||||
"byte_size": obj.get("byte_size") or 0,
|
||||
"relative_path": f"{resource_type}s/{resource_id}/{filename}",
|
||||
})
|
||||
store.update_storage_cache_job(cache_job["id"], {"status": "completed", "progress": 100, "local_path": result.get("local_path"), "completed_at": utcnow()})
|
||||
except Exception as exc:
|
||||
store.update_storage_cache_job(cache_job["id"], {"status": "failed", "error": str(exc), "completed_at": utcnow()})
|
||||
raise
|
||||
prepared.append({**result, "storage_object_id": obj["id"], "node_id": node_id})
|
||||
return ok({"resource_type": resource_type, "resource_id": resource_id, "node_id": node_id, "status": "ready", "items": prepared})
|
||||
|
||||
|
||||
@router.get("/compute/nodes/{node_id}")
|
||||
async def compute_node_detail(node_id: str) -> dict[str, Any]:
|
||||
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
|
||||
@router.get("/storage/cache/jobs/{node_id}")
|
||||
async def storage_cache_jobs(node_id: str, limit: int = Query(default=100, ge=1, le=500), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
return ok(get_platform_store().storage_cache_jobs_for_node(node_id, limit))
|
||||
|
||||
|
||||
@router.post("/storage/resources/{resource_type}/{resource_id}/archive-node/{node_id}")
|
||||
async def archive_node_files(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
node_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
"""Archive completed node files to MinIO without proxying file bytes through Backend."""
|
||||
if not has_resource_access(resource_type, resource_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to archive this resource")
|
||||
store = get_platform_store()
|
||||
if not is_admin(current_user):
|
||||
model_id = str(payload.get("model_id") or "")
|
||||
dataset_id = str(payload.get("dataset_id") or "")
|
||||
if not model_id or not has_resource_access("model", model_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to evaluate this model")
|
||||
if not dataset_id or not has_resource_access("dataset", dataset_id, current_user, "execute"):
|
||||
raise fail(403, "no permission to evaluate this dataset")
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
files = payload.get("files") or []
|
||||
if not isinstance(files, list) or not files:
|
||||
raise fail(400, "files is required")
|
||||
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
||||
archived = []
|
||||
for item in files:
|
||||
path = str(item.get("path") or "")
|
||||
name = Path(str(item.get("file_name") or Path(path).name)).name
|
||||
version_id = str(item.get("version_id") or uuid.uuid4().hex)
|
||||
object_key = str(item.get("object_key") or f"{resource_type}s/{resource_id}/versions/{version_id}/{name}")
|
||||
url = get_object_storage().presigned_put(object_key)
|
||||
result = await client.upload_file_to_url(path, url, object_key, str(item.get("content_type") or "application/octet-stream"))
|
||||
metadata = get_object_storage().stat(object_key)
|
||||
record = store.create_storage_object({
|
||||
"resource_type": resource_type, "resource_id": resource_id, "version_id": version_id,
|
||||
"bucket": get_object_storage().bucket, "object_key": object_key, "file_name": name,
|
||||
"content_type": item.get("content_type"), "checksum_sha256": result.get("checksum_sha256"),
|
||||
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0, "status": "available",
|
||||
"created_by": current_user.get("id"),
|
||||
})
|
||||
archived.append({"object": record, "node_id": node_id})
|
||||
return ok({"status": "available", "items": archived})
|
||||
|
||||
|
||||
@router.get("/compute/nodes/{node_id}")
|
||||
async def compute_node_detail(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
if not is_admin(current_user) and not any(item["node_id"] == node_id for item in store.gpu_assignments_for_user(current_user["id"])):
|
||||
raise fail(403, "no permission to access this compute node")
|
||||
return ok(node)
|
||||
|
||||
|
||||
@router.post("/compute/nodes")
|
||||
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
try:
|
||||
return ok(get_platform_store().create_compute_node(payload))
|
||||
except KeyError as exc:
|
||||
@@ -2184,7 +2389,9 @@ async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str,
|
||||
|
||||
|
||||
@router.put("/compute/nodes/{node_id}")
|
||||
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
try:
|
||||
return ok(get_platform_store().update_compute_node(node_id, payload))
|
||||
except KeyError:
|
||||
@@ -2194,7 +2401,9 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
|
||||
|
||||
|
||||
@router.delete("/compute/nodes/{node_id}")
|
||||
async def delete_compute_node(node_id: str) -> dict[str, Any]:
|
||||
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
try:
|
||||
return ok(get_platform_store().delete_compute_node(node_id))
|
||||
except KeyError:
|
||||
@@ -2204,7 +2413,9 @@ async def delete_compute_node(node_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/test-connection")
|
||||
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||||
async def test_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
store = get_platform_store()
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
@@ -2238,22 +2449,28 @@ async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/health-check")
|
||||
async def health_check_compute_node(node_id: str) -> dict[str, Any]:
|
||||
return await test_compute_node(node_id)
|
||||
async def health_check_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
return await test_compute_node(node_id, current_user)
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/enable")
|
||||
async def enable_compute_node(node_id: str) -> dict[str, Any]:
|
||||
async def enable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"}))
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/disable")
|
||||
async def disable_compute_node(node_id: str) -> dict[str, Any]:
|
||||
async def disable_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"}))
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/drain")
|
||||
async def drain_compute_node(node_id: str) -> dict[str, Any]:
|
||||
async def drain_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user):
|
||||
raise fail(403, "admin permission required")
|
||||
return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"}))
|
||||
|
||||
|
||||
@@ -2460,8 +2677,13 @@ async def compute_node_engines(node_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/compute/gpus")
|
||||
async def compute_gpus() -> dict[str, Any]:
|
||||
return ok(get_platform_store().gpus())
|
||||
async def compute_gpus(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
gpus = store.gpus()
|
||||
if is_admin(current_user):
|
||||
return ok(gpus)
|
||||
assigned = {(item["node_id"], int(item["gpu_index"])) for item in store.gpu_assignments_for_user(current_user["id"])}
|
||||
return ok([gpu for gpu in gpus if (gpu.get("node_id"), int(gpu.get("id", gpu.get("gpu_index", -1)))) in assigned])
|
||||
|
||||
|
||||
@router.get("/compute/queue")
|
||||
@@ -2485,8 +2707,10 @@ async def compute_job_detail(job_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/stop")
|
||||
async def compute_job_stop(job_id: str) -> dict[str, Any]:
|
||||
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
task = _task_for_compute_job(job_id)
|
||||
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
|
||||
raise fail(403, "no permission to stop this compute job")
|
||||
if not task:
|
||||
node = _node_for_compute_job_record(job_id)
|
||||
if not node:
|
||||
@@ -2521,13 +2745,13 @@ async def compute_job_logs(
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/retry")
|
||||
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
payload = payload or {}
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
return await retry_fine_tune(task["id"], payload)
|
||||
return await retry_fine_tune(task["id"], payload, current_user)
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/priority")
|
||||
|
||||
@@ -9,6 +9,13 @@ from app.db.platform_store import get_platform_store
|
||||
|
||||
# 无需鉴权的路径前缀(健康检查、登录等)
|
||||
PUBLIC_PATHS = ("/health", "/login", "/system-info")
|
||||
OWNER_TABLES = {
|
||||
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
|
||||
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
|
||||
"fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"),
|
||||
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
|
||||
"project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"),
|
||||
}
|
||||
|
||||
|
||||
def _extract_token(request: Request) -> str | None:
|
||||
@@ -19,6 +26,9 @@ def _extract_token(request: Request) -> str | None:
|
||||
return token[len("platform-token-"):]
|
||||
return None
|
||||
|
||||
def _session_token(user_id: str, session_id: str) -> str:
|
||||
return f"platform-token-{user_id}.{session_id}"
|
||||
|
||||
|
||||
def get_current_user(request: Request) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -33,14 +43,30 @@ def get_current_user(request: Request) -> dict[str, Any]:
|
||||
if path.endswith(prefix):
|
||||
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
|
||||
|
||||
user_id = _extract_token(request)
|
||||
if not user_id:
|
||||
token_value = _extract_token(request)
|
||||
if not token_value:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
|
||||
|
||||
store = get_platform_store()
|
||||
for u in store.users():
|
||||
if u.get("id") == user_id:
|
||||
return u
|
||||
user_id, _, session_id = token_value.partition(".")
|
||||
if session_id:
|
||||
with store.connect() as conn:
|
||||
session = conn.execute(
|
||||
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
|
||||
).fetchone()
|
||||
if not session or session["user_id"] != user_id or session["logout_at"]:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
||||
if session["expires_at"]:
|
||||
from datetime import datetime, timezone
|
||||
try:
|
||||
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
||||
except ValueError:
|
||||
pass
|
||||
with store.connect() as conn:
|
||||
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
|
||||
if user_row:
|
||||
return store._user(user_row)
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
|
||||
|
||||
|
||||
@@ -75,6 +101,23 @@ def has_resource_access(
|
||||
user_id = user.get("id")
|
||||
user_role = user.get("role")
|
||||
|
||||
owner_tables = OWNER_TABLES
|
||||
table_info = owner_tables.get(resource_type)
|
||||
if table_info and user_id:
|
||||
table, column = table_info
|
||||
with store.connect() as conn:
|
||||
row = conn.execute(f"SELECT {column} FROM {table} WHERE id=?", (resource_id,)).fetchone()
|
||||
if row:
|
||||
owner = row[column]
|
||||
if column == "payload":
|
||||
try:
|
||||
import json
|
||||
owner = json.loads(owner or "{}").get("created_by")
|
||||
except (TypeError, ValueError):
|
||||
owner = None
|
||||
if owner == user_id:
|
||||
return True
|
||||
|
||||
for entry in acls:
|
||||
# 按 user 授权
|
||||
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
|
||||
@@ -135,4 +178,40 @@ def filter_accessible_resource_ids(
|
||||
).fetchall()
|
||||
|
||||
accessible = {r["resource_id"] for r in rows}
|
||||
if resource_type in owner_tables:
|
||||
table, column = owner_tables[resource_type]
|
||||
with store.connect() as conn:
|
||||
owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall()
|
||||
accessible.update(row["id"] for row in owned)
|
||||
return [rid for rid in all_ids if rid in accessible]
|
||||
|
||||
|
||||
def filter_accessible_resource_ids_batch(
|
||||
resource_type: str,
|
||||
resource_ids: list[str],
|
||||
user: dict[str, Any],
|
||||
) -> set[str]:
|
||||
"""Filter a list endpoint with one ACL query instead of one query per row."""
|
||||
if is_admin(user):
|
||||
return set(resource_ids)
|
||||
if not resource_ids:
|
||||
return set()
|
||||
store = get_platform_store()
|
||||
placeholders = ",".join("?" for _ in resource_ids)
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
|
||||
"AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))",
|
||||
(resource_type, *resource_ids, user.get("id"), user.get("role")),
|
||||
).fetchall()
|
||||
accessible = {row["resource_id"] for row in rows}
|
||||
table_info = OWNER_TABLES.get(resource_type)
|
||||
if table_info and user.get("id"):
|
||||
table, column = table_info
|
||||
with store.connect() as conn:
|
||||
owned = conn.execute(
|
||||
f"SELECT id FROM {table} WHERE id IN ({placeholders}) AND {column}=?",
|
||||
(*resource_ids, user["id"]),
|
||||
).fetchall()
|
||||
accessible.update(row["id"] for row in owned)
|
||||
return accessible
|
||||
|
||||
@@ -61,7 +61,9 @@ class Settings:
|
||||
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
|
||||
compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5)
|
||||
minio_enabled: bool = _bool_env("MINIO_ENABLED", False)
|
||||
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
|
||||
# MinIO is an independent service and may run on another host. The
|
||||
# endpoint must therefore be reachable from the Backend container.
|
||||
minio_endpoint: str = os.getenv("MINIO_ENDPOINT", "http://host.docker.internal:19000")
|
||||
minio_access_key: str = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
|
||||
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
|
||||
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
|
||||
|
||||
@@ -210,6 +210,10 @@ def configure_logging(settings: Settings | None = None) -> None:
|
||||
logger.propagate = True
|
||||
|
||||
|
||||
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
|
||||
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
return logging.getLogger(name)
|
||||
|
||||
@@ -229,7 +233,15 @@ def setup_request_logging(app: FastAPI) -> None:
|
||||
try:
|
||||
response = await call_next(request)
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
logger.info(
|
||||
# Docker/frontend probes and polling endpoints are intentionally
|
||||
# quiet at INFO; failures remain visible at WARNING/ERROR.
|
||||
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
|
||||
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
|
||||
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
|
||||
log_method = logger.debug
|
||||
if response.status_code >= 400:
|
||||
log_method = logger.warning
|
||||
log_method(
|
||||
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
|
||||
@@ -526,8 +526,15 @@ class PlatformStore:
|
||||
"artifact_dir": "TEXT",
|
||||
"compute_node_id": "TEXT",
|
||||
"compute_node_name": "TEXT",
|
||||
"created_by": "TEXT",
|
||||
"tenant_id": "TEXT",
|
||||
"project_id": "TEXT",
|
||||
"deleted_at": "TEXT",
|
||||
"deleted_by": "TEXT",
|
||||
},
|
||||
)
|
||||
for table in ("models", "datasets", "eval_tasks"):
|
||||
self._ensure_columns(conn, table, {"deleted_at": "TEXT", "deleted_by": "TEXT", "tenant_id": "TEXT", "project_id": "TEXT"})
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"resource_replicas",
|
||||
@@ -1470,12 +1477,12 @@ class PlatformStore:
|
||||
|
||||
def delete_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM models WHERE id=?", (model_id,))
|
||||
conn.execute("UPDATE models SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", model_id))
|
||||
|
||||
def trained_models(self) -> list[dict[str, Any]]:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall()
|
||||
rows = conn.execute("SELECT * FROM trained_models WHERE deleted_at IS NULL ORDER BY create_time DESC").fetchall()
|
||||
items = []
|
||||
for row in rows:
|
||||
item = {
|
||||
@@ -1498,7 +1505,7 @@ class PlatformStore:
|
||||
|
||||
def delete_trained_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id))
|
||||
conn.execute("UPDATE trained_models SET deleted_at=?, deleted_by=? WHERE id=? OR name=?", (utcnow(), "system", model_id, model_id))
|
||||
|
||||
def datasets(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
@@ -1507,6 +1514,7 @@ class PlatformStore:
|
||||
FROM datasets dataset
|
||||
LEFT JOIN data_process_tasks task
|
||||
ON task.id=COALESCE(dataset.source_task_id, dataset.task_id)
|
||||
WHERE dataset.deleted_at IS NULL
|
||||
ORDER BY dataset.create_time DESC"""
|
||||
).fetchall()
|
||||
return [self._dataset(conn, row) for row in rows]
|
||||
@@ -1646,8 +1654,7 @@ class PlatformStore:
|
||||
|
||||
def delete_dataset(self, dataset_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,))
|
||||
conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,))
|
||||
conn.execute("UPDATE datasets SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", dataset_id))
|
||||
|
||||
def add_dataset_file(self, conn: PgConnection, dataset_id: str, name: str, content: str) -> dict[str, Any]:
|
||||
now = utcnow()
|
||||
@@ -2004,13 +2011,14 @@ class PlatformStore:
|
||||
task_id = str(payload.get("task_id") or payload.get("id"))
|
||||
current = self.task(task_id)
|
||||
merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8}
|
||||
selected_gpus = payload.get("gpus") or merged.get("gpus") or [0]
|
||||
selected_gpus = payload.get("gpus") or merged.get("gpus") or []
|
||||
process_id = int(43000 + (time.time() % 10000))
|
||||
with self.connect() as conn:
|
||||
owner = f"start:{task_id}:{uuid.uuid4().hex[:8]}"
|
||||
if not self._acquire_scheduler_lock(conn, "compute-scheduler", owner):
|
||||
raise RuntimeError("compute scheduler is busy, please retry")
|
||||
node = self._schedule_node_locked(conn, payload)
|
||||
selected_gpus = list(node.get("selected_gpus") or selected_gpus)
|
||||
sync_job_id = new_id("sync")
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -2094,7 +2102,7 @@ class PlatformStore:
|
||||
task = self.task(task_id)
|
||||
merged = {**task, **(payload or {}), "id": task_id}
|
||||
node = self.select_compute_node(merged)
|
||||
selected_gpus = merged.get("gpus") or [0]
|
||||
selected_gpus = merged.get("gpus") or []
|
||||
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
|
||||
|
||||
def prepare_compute_job_payload_from_payload(self, payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
@@ -2108,7 +2116,7 @@ class PlatformStore:
|
||||
"progress": int(payload.get("progress") or 0),
|
||||
}
|
||||
node = self.select_compute_node(transient_task)
|
||||
selected_gpus = transient_task.get("gpus") or [0]
|
||||
selected_gpus = transient_task.get("gpus") or []
|
||||
return node, self._compute_job_payload_from_task_node(transient_task, node, selected_gpus)
|
||||
|
||||
def _compute_job_payload_from_task_node(
|
||||
@@ -2260,7 +2268,7 @@ class PlatformStore:
|
||||
for item in runtime_files
|
||||
],
|
||||
"output_dir": output_dir,
|
||||
"gpus": selected_gpus or task.get("gpus") or [0],
|
||||
"gpus": selected_gpus if selected_gpus is not None else (task.get("gpus") or []),
|
||||
"compute_node_id": node["id"],
|
||||
"compute_node_code": node["code"],
|
||||
}
|
||||
@@ -2270,7 +2278,7 @@ class PlatformStore:
|
||||
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
|
||||
if not node:
|
||||
raise RuntimeError("compute node not found")
|
||||
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0])
|
||||
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [])
|
||||
|
||||
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
status_map = {
|
||||
@@ -2493,7 +2501,7 @@ class PlatformStore:
|
||||
|
||||
def delete_eval_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
|
||||
|
||||
def running_eval_tasks(self) -> list[dict[str, Any]]:
|
||||
"""Return eval tasks that have been submitted to a compute node and are still running."""
|
||||
@@ -2712,27 +2720,71 @@ class PlatformStore:
|
||||
def _node_capacity(self, node: dict[str, Any]) -> int:
|
||||
return max(1, int(node.get("max_parallel_jobs") or 1), int(node.get("gpu_count") or 0))
|
||||
|
||||
@staticmethod
|
||||
def _payload_gpu_indexes(payload: dict[str, Any]) -> list[int]:
|
||||
raw = payload.get("gpu_indices")
|
||||
if raw is None:
|
||||
raw = payload.get("gpus")
|
||||
if raw is None:
|
||||
return []
|
||||
try:
|
||||
values = [int(item) for item in raw]
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError("invalid GPU index") from exc
|
||||
if any(item < 0 for item in values):
|
||||
raise RuntimeError("GPU index must be non-negative")
|
||||
return sorted(set(values))
|
||||
|
||||
def _select_node_gpus(
|
||||
self,
|
||||
conn: PgConnection,
|
||||
node: dict[str, Any],
|
||||
requested: list[int],
|
||||
payload: dict[str, Any],
|
||||
) -> list[int]:
|
||||
available = self._node_gpu_indexes(conn, node)
|
||||
active = self._active_gpu_indexes(conn, node["id"])
|
||||
allowed = payload.get("allowed_gpu_indices")
|
||||
if allowed is not None:
|
||||
available &= {int(item) for item in allowed}
|
||||
if requested:
|
||||
selected = set(requested)
|
||||
if not selected.issubset(available):
|
||||
raise RuntimeError(f"requested GPU is not available on compute node {node['code']}")
|
||||
if selected.intersection(active):
|
||||
raise RuntimeError(f"requested GPU is busy on compute node {node['code']}")
|
||||
return sorted(selected)
|
||||
if payload.get("allow_cpu") or payload.get("device") == "cpu":
|
||||
return []
|
||||
count = max(1, int(payload.get("gpu_count") or 1))
|
||||
free = sorted(available - active)
|
||||
if len(free) < count:
|
||||
raise RuntimeError(f"compute node {node['code']} has only {len(free)} available GPU(s)")
|
||||
return free[:count]
|
||||
|
||||
def _schedule_node_locked(self, conn: PgConnection, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
|
||||
requested_gpus = [int(item) for item in payload.get("gpus") or []]
|
||||
requested_gpus = self._payload_gpu_indexes(payload)
|
||||
nodes = self._compute_nodes_locked(conn)
|
||||
candidates = [
|
||||
n
|
||||
for n in nodes
|
||||
if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < self._node_capacity(n)
|
||||
]
|
||||
if requested_gpus:
|
||||
requested_gpu_set = set(requested_gpus)
|
||||
candidates = [
|
||||
node
|
||||
for node in candidates
|
||||
if requested_gpu_set.issubset(self._node_gpu_indexes(conn, node))
|
||||
and not requested_gpu_set.intersection(self._active_gpu_indexes(conn, node["id"]))
|
||||
]
|
||||
if requested:
|
||||
selected = next((n for n in candidates if n["id"] == requested), None)
|
||||
if selected:
|
||||
return selected
|
||||
if not selected:
|
||||
raise RuntimeError("selected compute node is unavailable")
|
||||
selected["selected_gpus"] = self._select_node_gpus(conn, selected, requested_gpus, payload)
|
||||
return selected
|
||||
filtered = []
|
||||
for node in candidates:
|
||||
try:
|
||||
node["selected_gpus"] = self._select_node_gpus(conn, node, requested_gpus, payload)
|
||||
filtered.append(node)
|
||||
except RuntimeError:
|
||||
continue
|
||||
candidates = filtered
|
||||
if not candidates:
|
||||
if not nodes:
|
||||
raise RuntimeError("no available compute node: no compute node configured")
|
||||
@@ -2744,6 +2796,8 @@ class PlatformStore:
|
||||
reason = f"status={node['scheduler_status']}"
|
||||
elif node["current_running_jobs"] >= self._node_capacity(node):
|
||||
reason = f"capacity full {node['current_running_jobs']}/{self._node_capacity(node)}"
|
||||
elif requested:
|
||||
reason = "selected node unavailable"
|
||||
else:
|
||||
reason = "not selected"
|
||||
reasons.append(f"{node['code']}({reason})")
|
||||
@@ -2930,6 +2984,43 @@ class PlatformStore:
|
||||
raise KeyError(object_id)
|
||||
return dict(row)
|
||||
|
||||
def create_storage_cache_job(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
job_id = str(payload.get("id") or new_id("cache"))
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO storage_cache_jobs
|
||||
(id, storage_object_id, node_id, direction, status, progress, local_path, error, create_time, completed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(job_id, payload["storage_object_id"], payload["node_id"], payload.get("direction", "download"),
|
||||
payload.get("status", "running"), int(payload.get("progress", 0)), payload.get("local_path"),
|
||||
payload.get("error"), payload.get("create_time") or utcnow(), payload.get("completed_at")),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
def update_storage_cache_job(self, job_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed = {"status", "progress", "local_path", "error", "completed_at"}
|
||||
fields = {key: value for key, value in payload.items() if key in allowed}
|
||||
if fields:
|
||||
assignments = ", ".join(f"{key}=?" for key in fields)
|
||||
with self.connect() as conn:
|
||||
conn.execute(f"UPDATE storage_cache_jobs SET {assignments} WHERE id=?", (*fields.values(), job_id))
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM storage_cache_jobs WHERE id=?", (job_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(job_id)
|
||||
return dict(row)
|
||||
|
||||
def storage_cache_jobs_for_node(self, node_id: str, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM storage_cache_jobs WHERE node_id=? ORDER BY create_time DESC LIMIT ?",
|
||||
(node_id, max(1, min(limit, 500))),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def update_resource_replica_sync_result(
|
||||
self,
|
||||
replica_id: str,
|
||||
@@ -3613,9 +3704,9 @@ class PlatformStore:
|
||||
login_at = utcnow()
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (id, user_id, username, login_at, create_time) "
|
||||
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s)",
|
||||
(sid, user_id, user_id, login_at, login_at),
|
||||
"INSERT INTO sessions (id, user_id, username, login_at, issued_at, expires_at, create_time) "
|
||||
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s, %s, %s)",
|
||||
(sid, user_id, user_id, login_at, login_at, datetime.fromtimestamp(time.time() + 1800, timezone.utc).isoformat(), login_at),
|
||||
)
|
||||
return {"session_id": sid, "user_id": user_id, "login_at": login_at}
|
||||
|
||||
@@ -4263,6 +4354,14 @@ class PlatformStore:
|
||||
assigned = {r["gpu_index"] for r in rows}
|
||||
return all(idx in assigned for idx in gpu_indices)
|
||||
|
||||
def assigned_gpu_indexes(self, user_id: str, node_id: str) -> list[int]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT gpu_index FROM gpu_assignments WHERE user_id=? AND node_id=? ORDER BY gpu_index",
|
||||
(user_id, node_id),
|
||||
).fetchall()
|
||||
return [int(row["gpu_index"]) for row in rows]
|
||||
|
||||
# ===================== 平台治理:资源可见性过滤 =====================
|
||||
|
||||
def _filter_accessible_ids(
|
||||
|
||||
@@ -64,7 +64,12 @@ CREATE TABLE IF NOT EXISTS models (
|
||||
api_key TEXT,
|
||||
online_model_name TEXT,
|
||||
can_train INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
create_time TEXT NOT NULL,
|
||||
created_by TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
deleted_at TEXT,
|
||||
deleted_by TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trained_models (
|
||||
@@ -78,7 +83,12 @@ CREATE TABLE IF NOT EXISTS trained_models (
|
||||
merged_path TEXT,
|
||||
artifact_dir TEXT,
|
||||
compute_node_id TEXT,
|
||||
compute_node_name TEXT
|
||||
compute_node_name TEXT,
|
||||
created_by TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
deleted_at TEXT,
|
||||
deleted_by TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||
@@ -130,7 +140,12 @@ CREATE TABLE IF NOT EXISTS datasets (
|
||||
size TEXT,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
description TEXT,
|
||||
create_time TEXT NOT NULL
|
||||
create_time TEXT NOT NULL,
|
||||
created_by TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
deleted_at TEXT,
|
||||
deleted_by TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_files (
|
||||
@@ -413,6 +428,18 @@ CREATE TABLE IF NOT EXISTS acls (
|
||||
permission TEXT NOT NULL,
|
||||
create_time TEXT
|
||||
);
|
||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_acls_subject_permission
|
||||
ON acls(resource_type, resource_id, principal_type, principal_id, permission);
|
||||
CREATE INDEX IF NOT EXISTS idx_acls_resource ON acls(resource_type, resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_acls_principal ON acls(principal_type, principal_id);
|
||||
|
||||
-- ---- 权限扩展(来源:004_permissions.sql) ----
|
||||
|
||||
@@ -475,6 +502,9 @@ CREATE TABLE IF NOT EXISTS approval_steps (
|
||||
comment TEXT,
|
||||
time TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instances(applicant_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body
|
||||
from fastapi import APIRouter, Body, Depends
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.core.auth import get_current_user, is_admin
|
||||
|
||||
router = APIRouter(prefix="/approvals", tags=["approval"])
|
||||
|
||||
|
||||
@router.get("/templates")
|
||||
def list_templates() -> dict[str, Any]:
|
||||
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_templates())
|
||||
|
||||
|
||||
@router.post("/templates")
|
||||
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
def create_template(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||
if not payload.get("name"):
|
||||
raise fail(400, "name 必填")
|
||||
return ok(get_platform_store().create_approval_template(payload))
|
||||
@@ -30,7 +32,8 @@ def get_template(template_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.put("/templates/{template_id}")
|
||||
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||
try:
|
||||
return ok(get_platform_store().update_approval_template(template_id, payload))
|
||||
except KeyError:
|
||||
@@ -38,7 +41,8 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> di
|
||||
|
||||
|
||||
@router.delete("/templates/{template_id}")
|
||||
def delete_template(template_id: str) -> dict[str, Any]:
|
||||
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||
try:
|
||||
return ok(get_platform_store().delete_approval_template(template_id))
|
||||
except KeyError:
|
||||
@@ -46,13 +50,15 @@ def delete_template(template_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_instances(status: str | None = None) -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_instances(status=status))
|
||||
def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
items = get_platform_store().approval_instances(status=status)
|
||||
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
for field in ("resource_type", "resource_id", "applicant_id"):
|
||||
def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload["applicant_id"] = current_user.get("id")
|
||||
for field in ("resource_type", "resource_id"):
|
||||
if not payload.get(field):
|
||||
raise fail(400, f"{field} 必填")
|
||||
try:
|
||||
@@ -62,9 +68,12 @@ def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/{instance_id}")
|
||||
def get_instance(instance_id: str) -> dict[str, Any]:
|
||||
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().approval_instance(instance_id))
|
||||
item = get_platform_store().approval_instance(instance_id)
|
||||
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
|
||||
raise fail(403, "no permission to access approval")
|
||||
return ok(item)
|
||||
except KeyError:
|
||||
raise fail(404, "instance not found")
|
||||
|
||||
|
||||
@@ -266,3 +266,8 @@ class ComputeNodeClient:
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def upload_file_to_url(self, path: str, upload_url: str, object_key: str = "", content_type: str = "application/octet-stream") -> dict[str, Any]:
|
||||
return await self._request("POST", "/compute/files/upload-to-url", json_data={
|
||||
"path": path, "upload_url": upload_url, "object_key": object_key, "content_type": content_type,
|
||||
}, timeout=900)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from fastapi import APIRouter, Body, Request, Depends
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.core.auth import get_current_user, has_resource_access, is_admin
|
||||
|
||||
router = APIRouter(prefix="/resources", tags=["resource"])
|
||||
|
||||
@@ -16,8 +17,10 @@ def _actor(request: Request) -> str | None:
|
||||
|
||||
|
||||
@router.get("/{resource_type}/{resource_id}/acl")
|
||||
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]:
|
||||
def get_acl(resource_type: str, resource_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""查询资源 ACL,返回按主体分组的权限列表。"""
|
||||
if not has_resource_access(resource_type, resource_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access resource ACL")
|
||||
return ok(get_platform_store().resource_acl(resource_type, resource_id))
|
||||
|
||||
|
||||
@@ -27,9 +30,18 @@ def set_acl(
|
||||
resource_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
"""设置资源 ACL,body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
|
||||
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
|
||||
raise fail(403, "only resource owner or admin can update ACL")
|
||||
entries = payload.get("entries") or []
|
||||
allowed = {"read", "write", "execute", "download", "delete", "admin"}
|
||||
for entry in entries:
|
||||
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
|
||||
raise fail(400, "invalid ACL principal")
|
||||
if any(permission not in allowed for permission in entry.get("permissions") or []):
|
||||
raise fail(400, "invalid ACL permission")
|
||||
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
|
||||
get_platform_store().record_audit(
|
||||
action="resource.acl.set",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Query, Request
|
||||
from fastapi import APIRouter, Body, Query, Request, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
|
||||
from app.core.auth import get_current_user, is_admin
|
||||
|
||||
|
||||
router = APIRouter(prefix="/system", tags=["system"])
|
||||
@@ -32,13 +33,13 @@ def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
|
||||
|
||||
|
||||
@router.get("/permissions/codes")
|
||||
def permission_codes() -> dict:
|
||||
def permission_codes(current_user: dict = Depends(get_current_user)) -> dict:
|
||||
"""返回平台权限码清单(权限码接口)。"""
|
||||
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
|
||||
|
||||
|
||||
@router.get("/permissions")
|
||||
def permissions_overview() -> dict:
|
||||
def permissions_overview(current_user: dict = Depends(get_current_user)) -> dict:
|
||||
"""返回权限码清单与角色定义。"""
|
||||
store = get_platform_store()
|
||||
return {
|
||||
@@ -59,8 +60,12 @@ def audit_logs(
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
|
||||
if not is_admin(current_user):
|
||||
from app.api.v1.endpoints.platform import fail
|
||||
raise fail(403, "admin permission required")
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
@@ -85,8 +90,12 @@ def audit_logs_export(
|
||||
target_type: str | None = Query(default=None, description="目标类型"),
|
||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> StreamingResponse:
|
||||
"""审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。"""
|
||||
if not is_admin(current_user):
|
||||
from app.api.v1.endpoints.platform import fail
|
||||
raise fail(403, "admin permission required")
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
|
||||
@@ -21,8 +21,10 @@ async def run_compute_poller() -> None:
|
||||
while True:
|
||||
try:
|
||||
result = await poll_compute_jobs_once()
|
||||
if result["synced"] or result["failed"]:
|
||||
logger.info("compute jobs polled", extra={"result": result})
|
||||
if result["failed"]:
|
||||
logger.warning("compute polling reported failures", extra={"result": result})
|
||||
elif result["synced"]:
|
||||
logger.debug("compute jobs synchronized", extra={"result": result})
|
||||
except asyncio.CancelledError:
|
||||
logger.info("compute poller stopped")
|
||||
raise
|
||||
|
||||
@@ -25,6 +25,7 @@ from compute.engines.llama_factory.inference import get_inference_session
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs())
|
||||
jobs: dict[str, dict[str, Any]] = {}
|
||||
cache_locks: dict[str, asyncio.Lock] = {}
|
||||
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
|
||||
process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"))
|
||||
|
||||
@@ -620,6 +621,30 @@ def create_app() -> FastAPI:
|
||||
)
|
||||
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/upload-to-url")
|
||||
async def upload_file_to_url(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Upload one node-local artifact to a Backend-issued presigned URL."""
|
||||
source = Path(str(payload.get("path") or "")).resolve()
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")).resolve()
|
||||
if not _path_inside(data_root, source) or not source.is_file():
|
||||
raise HTTPException(status_code=400, detail="artifact path must be an existing file inside YG_FT_DATA_ROOT")
|
||||
upload_url = str(payload.get("upload_url") or "")
|
||||
if not upload_url:
|
||||
raise HTTPException(status_code=400, detail="upload_url is required")
|
||||
digest = hashlib.sha256()
|
||||
byte_size = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
|
||||
with source.open("rb") as handle:
|
||||
content = handle.read()
|
||||
digest.update(content)
|
||||
byte_size = len(content)
|
||||
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
|
||||
return {"status": "available", "path": str(source), "byte_size": byte_size, "checksum_sha256": digest.hexdigest(), "object_key": payload.get("object_key")}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/jobs")
|
||||
async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = {**payload, "require_dataset_files": True}
|
||||
@@ -645,7 +670,7 @@ def create_app() -> FastAPI:
|
||||
"status": "queued",
|
||||
"progress": 10,
|
||||
"pid": int(52000 + now() % 10000),
|
||||
"gpus": payload.get("gpus") or [0],
|
||||
"gpus": payload.get("gpus") or [],
|
||||
"created_at": now(),
|
||||
"command": command.command,
|
||||
"work_dir": command.work_dir,
|
||||
@@ -843,21 +868,46 @@ def create_app() -> FastAPI:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_target = target.with_name(f".{target.name}.part")
|
||||
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
|
||||
expected_size = int(payload.get("byte_size") or 0)
|
||||
lock = cache_locks.setdefault(str(target), asyncio.Lock())
|
||||
async with lock:
|
||||
if target.is_file() and expected_checksum:
|
||||
existing_digest = hashlib.sha256()
|
||||
with target.open("rb") as existing:
|
||||
while chunk := existing.read(1024 * 1024):
|
||||
existing_digest.update(chunk)
|
||||
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
|
||||
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
|
||||
digest = hashlib.sha256()
|
||||
byte_size = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
|
||||
async with client.stream("GET", download_url) as response:
|
||||
response.raise_for_status()
|
||||
with temp_target.open("wb") as output:
|
||||
async for chunk in response.aiter_bytes(1024 * 1024):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
byte_size += len(chunk)
|
||||
async with lock:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
digest = hashlib.sha256()
|
||||
byte_size = 0
|
||||
temp_target.unlink(missing_ok=True)
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
|
||||
async with client.stream("GET", download_url) as response:
|
||||
response.raise_for_status()
|
||||
with temp_target.open("wb") as output:
|
||||
async for chunk in response.aiter_bytes(1024 * 1024):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
byte_size += len(chunk)
|
||||
break
|
||||
except Exception:
|
||||
temp_target.unlink(missing_ok=True)
|
||||
if attempt == 2:
|
||||
raise
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
checksum = digest.hexdigest()
|
||||
if expected_checksum and checksum != expected_checksum:
|
||||
temp_target.unlink(missing_ok=True)
|
||||
raise HTTPException(status_code=502, detail="cache checksum mismatch")
|
||||
if expected_size and byte_size != expected_size:
|
||||
temp_target.unlink(missing_ok=True)
|
||||
raise HTTPException(status_code=502, detail="cache byte size mismatch")
|
||||
temp_target.replace(target)
|
||||
except HTTPException:
|
||||
raise
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
# Docker 部署说明
|
||||
|
||||
## 跨服务器部署
|
||||
|
||||
MinIO、Backend API 和 Compute API 使用各自服务器上的独立 Docker 网络,不能使用跨服务器的容器名称互访。当前 WSL 联调地址为 `172.25.179.69`:
|
||||
|
||||
```env
|
||||
# docker/app/.env
|
||||
MINIO_ENABLED=true
|
||||
MINIO_ENDPOINT=http://172.25.179.69:19000
|
||||
COMPUTE_API_BASE_URL=http://172.25.179.69:19100
|
||||
FILE_GATEWAY_BASE_URL=http://172.25.179.69:19101
|
||||
```
|
||||
|
||||
拆分到不同服务器后,将 `172.25.179.69` 替换为对应服务器 IP。MinIO 容器内部仍使用 `9000/9001`,对外使用 `19000/19001`;Backend 和 Compute API 通过外部 IP 访问 MinIO,不加入 MinIO 的 Docker 网络。
|
||||
|
||||
本目录按应用服务器和算力服务器拆分 Dockerfile 与 Docker Compose 文件。Compose 文件不包含 `build:`,不会在 `docker compose up` 时自动构建业务镜像。所有业务镜像需要先通过手动 `docker build` 构建,再由 Compose 启动。
|
||||
|
||||
## 基础镜像
|
||||
|
||||
@@ -50,10 +50,14 @@ COMPUTE_POLL_BATCH_SIZE=100
|
||||
COMPUTE_REQUEST_TIMEOUT_SECONDS=5
|
||||
|
||||
# MinIO object storage. Enable after the MinIO service is reachable.
|
||||
MINIO_ENABLED=false
|
||||
MINIO_ENDPOINT=http://minio:9000
|
||||
MINIO_API_PORT=19000
|
||||
MINIO_CONSOLE_PORT=19001
|
||||
MINIO_ENABLED=true
|
||||
# For split-server deployment, replace host.docker.internal with the MinIO
|
||||
# server address, for example http://10.10.20.30:19000.
|
||||
MINIO_ENDPOINT=http://172.25.179.69:19000
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=change_me_minio_secret
|
||||
MINIO_BUCKET=yg-ft-resources
|
||||
MINIO_SECURE=false
|
||||
STORAGE_WAIT_SECONDS=300
|
||||
|
||||
@@ -63,7 +63,9 @@ services:
|
||||
COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100}
|
||||
COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5}
|
||||
MINIO_ENABLED: ${MINIO_ENABLED:-false}
|
||||
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-http://minio:9000}
|
||||
# Use the storage server's externally reachable address. Do not use a
|
||||
# MinIO container name because storage is deployed independently.
|
||||
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-http://host.docker.internal:19000}
|
||||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
|
||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
|
||||
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
|
||||
@@ -75,6 +77,8 @@ services:
|
||||
- ../../backend:/app:ro
|
||||
- ../../runtime/app/logs/backend:/opt/yg-ft/logs/backend
|
||||
- ../../runtime/app/data:/data/yg-ft
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
- yg-ft-app
|
||||
healthcheck:
|
||||
|
||||
100256
docker/app/tiktoken/cl100k_base
Normal file
100256
docker/app/tiktoken/cl100k_base
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,9 @@
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=change_me_minio_secret
|
||||
MINIO_API_PORT=9000
|
||||
MINIO_CONSOLE_PORT=9001
|
||||
MINIO_API_PORT=19000
|
||||
MINIO_CONSOLE_PORT=19001
|
||||
MINIO_DATA_ROOT_HOST=./data
|
||||
|
||||
# Backend and Compute API use this externally reachable endpoint when MinIO
|
||||
# runs on a separate server. Keep the container's internal API port at 9000.
|
||||
MINIO_EXTERNAL_ENDPOINT=http://<storage-server-ip>:19000
|
||||
|
||||
0
docker/minio/data/.gitkeep
Normal file
0
docker/minio/data/.gitkeep
Normal file
@@ -7,8 +7,8 @@ services:
|
||||
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-change_me_minio_secret}
|
||||
ports:
|
||||
- "${MINIO_API_PORT:-9000}:9000"
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
- "${MINIO_API_PORT:-19000}:9000"
|
||||
- "${MINIO_CONSOLE_PORT:-19001}:9001"
|
||||
volumes:
|
||||
- ${MINIO_DATA_ROOT_HOST:-./data}:/data
|
||||
healthcheck:
|
||||
|
||||
423
docs/permissions-and-logging-test-cases.md
Normal file
423
docs/permissions-and-logging-test-cases.md
Normal file
@@ -0,0 +1,423 @@
|
||||
# 权限与日志改造功能测试用例
|
||||
|
||||
## 1. 测试范围
|
||||
|
||||
本文档用于验证本次权限和日志相关开发内容:
|
||||
|
||||
- 训练、评测、推理资源权限校验。
|
||||
- 模型合并和训练模型导出权限。
|
||||
- 算力节点及 GPU 分配权限。
|
||||
- 资源所有者、ACL、租户/项目归属字段。
|
||||
- 资源软删除和审计记录。
|
||||
- 登录失败限流与 Token 会话有效期。
|
||||
- 后端轮询日志降噪。
|
||||
- 数据库初始化和运行时字段迁移。
|
||||
|
||||
## 2. 测试环境
|
||||
|
||||
| 项目 | 配置 |
|
||||
|---|---|
|
||||
| 前端地址 | `http://172.25.179.69:16801` |
|
||||
| Backend 地址 | `http://172.25.179.69:17861/modelTF` |
|
||||
| MinIO 地址 | `http://172.25.179.69:19000` |
|
||||
| WSL IP | `172.25.179.69` |
|
||||
| 运行方式 | Docker Compose |
|
||||
| 数据库 | 远程 PostgreSQL |
|
||||
|
||||
## 3. 前置数据
|
||||
|
||||
准备以下账号和资源:
|
||||
|
||||
| 数据 | 要求 |
|
||||
|---|---|
|
||||
| 管理员账号 | 具有 `admin` 角色 |
|
||||
| 普通操作员 | 具有训练、评测或推理页面权限 |
|
||||
| 只读用户 | 具有 `dashboard` 或 `logs` 权限,不具有业务写权限 |
|
||||
| 测试模型 | 一个基座模型和一个训练模型 |
|
||||
| 测试数据集 | 一个训练数据集和一个评测数据集 |
|
||||
| 算力节点 | 至少一个在线节点,最好有 2 张及以上 GPU |
|
||||
| ACL 资源 | 将测试数据集授权给普通用户进行验证 |
|
||||
|
||||
## 4. 鉴权与会话测试
|
||||
|
||||
### AUTH-001 登录成功生成会话 Token
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 使用有效账号调用登录接口或登录页面。
|
||||
2. 查看响应中的 `token` 和 `session_id`。
|
||||
3. 使用 Token 调用 `/modelTF/me`。
|
||||
|
||||
**预期**
|
||||
|
||||
- 登录返回 HTTP 200。
|
||||
- Token 格式包含用户 ID 和 session ID。
|
||||
- `/me` 能返回当前用户信息。
|
||||
- `sessions` 表生成一条未注销记录。
|
||||
|
||||
### AUTH-002 无 Token 访问受保护接口
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 不携带 `Authorization` 调用数据集、模型或训练列表接口。
|
||||
|
||||
**预期**
|
||||
|
||||
- 返回 HTTP 401。
|
||||
- 不返回资源数据。
|
||||
|
||||
### AUTH-003 注销后 Token 失效
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 登录并记录 Token、`session_id`。
|
||||
2. 调用注销接口。
|
||||
3. 使用原 Token 调用 `/me` 或资源接口。
|
||||
|
||||
**预期**
|
||||
|
||||
- 注销成功。
|
||||
- `sessions.logout_at` 已写入。
|
||||
- 原 Token 返回 HTTP 401。
|
||||
|
||||
### AUTH-004 登录失败限流
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 同一客户端 IP 连续输入错误密码 5 次。
|
||||
2. 第 6 次继续登录。
|
||||
|
||||
**预期**
|
||||
|
||||
- 前 5 次返回登录失败。
|
||||
- 第 6 次返回 HTTP 429。
|
||||
- 使用正确密码也应在冷却窗口内被限制。
|
||||
- 登录成功后失败计数清除。
|
||||
|
||||
## 5. 资源所有权与 ACL
|
||||
|
||||
### ACL-001 资源所有者访问自己的资源
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 普通用户创建数据集或模型。
|
||||
2. 使用该用户查看列表和详情。
|
||||
3. 修改资源元数据。
|
||||
|
||||
**预期**
|
||||
|
||||
- 资源出现在自己的列表中。
|
||||
- 详情访问返回 HTTP 200。
|
||||
- 资源所有者可以执行允许的写操作。
|
||||
|
||||
### ACL-002 未授权用户访问资源
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 用户 A 创建数据集。
|
||||
2. 用户 B 未获得 ACL 授权时查看该数据集详情。
|
||||
|
||||
**预期**
|
||||
|
||||
- 用户 B 不应在列表中看到该资源。
|
||||
- 直接访问详情返回 HTTP 403。
|
||||
|
||||
### ACL-003 ACL 授权后访问资源
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 用户 A 或管理员给用户 B 授予 `read` 权限。
|
||||
2. 用户 B 刷新资源列表并访问详情。
|
||||
|
||||
**预期**
|
||||
|
||||
- 用户 B 可以看到并读取资源。
|
||||
- 用户 B 不能执行 `write`、`delete` 或 `execute` 操作。
|
||||
|
||||
### ACL-004 非所有者修改 ACL
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 用户 B 仅拥有资源 `read` 权限。
|
||||
2. 用户 B 调用 ACL 修改接口。
|
||||
|
||||
**预期**
|
||||
|
||||
- 返回 HTTP 403。
|
||||
- ACL 内容不发生变化。
|
||||
|
||||
### ACL-005 ACL 修改审计
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 管理员或资源所有者修改 ACL。
|
||||
2. 查询审计日志。
|
||||
|
||||
**预期**
|
||||
|
||||
- 出现 `resource.acl.set` 操作记录。
|
||||
- 记录操作者、资源类型、资源 ID 和变更详情。
|
||||
|
||||
## 6. 训练权限测试
|
||||
|
||||
### TRAIN-001 训练创建校验模型和数据集权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 普通用户选择无权限的基座模型或数据集创建训练任务。
|
||||
2. 再使用已授权模型和数据集创建训练任务。
|
||||
|
||||
**预期**
|
||||
|
||||
- 无权限资源返回 HTTP 403。
|
||||
- 已授权资源允许创建任务。
|
||||
- 任务记录包含 `created_by`。
|
||||
|
||||
### TRAIN-002 GPU 授权校验
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 管理员将 GPU 0 分配给用户 A。
|
||||
2. 用户 A 选择 GPU 0 创建训练任务。
|
||||
3. 用户 A 选择未分配的 GPU 1 创建训练任务。
|
||||
|
||||
**预期**
|
||||
|
||||
- GPU 0 可以提交。
|
||||
- GPU 1 返回 HTTP 403。
|
||||
- 未指定 GPU 时,仅从用户已授权的空闲 GPU 中自动分配。
|
||||
|
||||
### TRAIN-003 训练任务停止、重试和删除权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 用户 A 创建训练任务。
|
||||
2. 用户 B 尝试停止、重试或删除该任务。
|
||||
3. 管理员执行相同操作。
|
||||
|
||||
**预期**
|
||||
|
||||
- 用户 B 无资源权限时返回 HTTP 403。
|
||||
- 删除和停止等高风险操作按配置进入审批流程。
|
||||
- 管理员可以旁路审批执行。
|
||||
- GPU 占用在停止、失败和删除后释放。
|
||||
|
||||
## 7. 评测权限测试
|
||||
|
||||
### EVAL-001 评测创建联合权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 普通用户选择无权使用的模型创建评测。
|
||||
2. 普通用户选择无权使用的数据集创建评测。
|
||||
3. 使用同时拥有权限的模型和数据集创建评测。
|
||||
|
||||
**预期**
|
||||
|
||||
- 模型无权限返回 HTTP 403。
|
||||
- 数据集无权限返回 HTTP 403。
|
||||
- 两个资源均有 `execute` 权限时允许创建。
|
||||
- 评测任务包含 `created_by`、模型、数据集和算力节点信息。
|
||||
|
||||
### EVAL-002 评测详情和删除权限
|
||||
|
||||
**预期**
|
||||
|
||||
- 无权限用户不能查看评测详情。
|
||||
- 评测删除需要 `delete` 权限。
|
||||
- 非管理员删除高风险评测任务时触发审批。
|
||||
|
||||
## 8. 推理权限测试
|
||||
|
||||
### INFER-001 推理创建模型权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 普通用户选择无权模型创建推理任务。
|
||||
2. 选择已授权模型创建推理任务。
|
||||
|
||||
**预期**
|
||||
|
||||
- 无权模型返回 HTTP 403。
|
||||
- 有权模型允许创建。
|
||||
- 推理任务包含 `created_by`。
|
||||
|
||||
### INFER-002 推理加载和卸载权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 用户 A 创建推理任务。
|
||||
2. 用户 B 调用加载、卸载接口。
|
||||
3. 用户 A 执行加载和卸载。
|
||||
|
||||
**预期**
|
||||
|
||||
- 用户 B 返回 HTTP 403。
|
||||
- 用户 A 可以执行授权范围内的加载和卸载。
|
||||
- 卸载后 GPU 和节点状态恢复为空闲。
|
||||
|
||||
### INFER-003 推理任务删除审批
|
||||
|
||||
**预期**
|
||||
|
||||
- 非管理员删除他人推理任务被拒绝或进入审批。
|
||||
- 管理员可以直接删除。
|
||||
- 删除操作有审计日志。
|
||||
|
||||
## 9. 模型合并与导出测试
|
||||
|
||||
### MODEL-001 权重合并权限
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 普通用户选择无权限训练模型进行合并。
|
||||
2. 选择已授权训练模型进行合并。
|
||||
|
||||
**预期**
|
||||
|
||||
- 无权限返回 HTTP 403。
|
||||
- 已授权训练模型允许合并。
|
||||
- 如指定基座模型,还必须拥有基座模型 `execute` 权限。
|
||||
|
||||
### MODEL-002 训练模型删除权限
|
||||
|
||||
**预期**
|
||||
|
||||
- 只有资源所有者、ACL 授权用户或管理员可操作。
|
||||
- 删除采用软删除。
|
||||
- `deleted_at`、`deleted_by` 被写入。
|
||||
|
||||
### MODEL-003 导出任务访问权限
|
||||
|
||||
**预期**
|
||||
|
||||
- 无权用户不能查看训练模型导出任务。
|
||||
- 有权用户可以查看导出状态。
|
||||
- 导出动作应记录 `trained_model.export` 审计日志。
|
||||
|
||||
## 10. 算力节点与 GPU 管理测试
|
||||
|
||||
### GPU-001 普通用户节点可见范围
|
||||
|
||||
**预期**
|
||||
|
||||
- 普通用户只能看到被分配 GPU 所在节点。
|
||||
- 普通用户只能看到已授权 GPU。
|
||||
- 管理员可看到所有节点和 GPU。
|
||||
|
||||
### GPU-002 节点管理接口权限
|
||||
|
||||
验证节点创建、修改、删除、启用、禁用、排空、连接测试和健康检查。
|
||||
|
||||
**预期**
|
||||
|
||||
- 普通用户全部返回 HTTP 403。
|
||||
- 管理员操作成功。
|
||||
|
||||
### GPU-003 多卡自动分配
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 节点配置多张 GPU。
|
||||
2. 启动一个任务占用其中一张卡。
|
||||
3. 再启动任务并选择剩余卡。
|
||||
|
||||
**预期**
|
||||
|
||||
- 已占用 GPU 不再出现在可选列表。
|
||||
- 剩余 GPU 可以被其他任务使用。
|
||||
- 任务失败、停止或完成后 GPU 释放。
|
||||
|
||||
## 11. 软删除与数据库测试
|
||||
|
||||
### DB-001 初始化字段检查
|
||||
|
||||
执行:
|
||||
|
||||
```sql
|
||||
SELECT table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_name IN ('models', 'datasets', 'trained_models', 'eval_tasks', 'sessions')
|
||||
ORDER BY table_name, ordinal_position;
|
||||
```
|
||||
|
||||
**预期字段**
|
||||
|
||||
- 资源表存在 `created_by`、`tenant_id`、`project_id`、`deleted_at`、`deleted_by`。
|
||||
- `sessions` 存在 `issued_at`、`expires_at`、`logout_at`。
|
||||
|
||||
### DB-002 软删除列表过滤
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 删除模型、数据集或评测任务。
|
||||
2. 查询列表。
|
||||
3. 直接查询数据库记录。
|
||||
|
||||
**预期**
|
||||
|
||||
- 前端列表不再显示已删除资源。
|
||||
- 数据库记录仍存在。
|
||||
- `deleted_at` 和 `deleted_by` 有值。
|
||||
|
||||
## 12. 日志降噪测试
|
||||
|
||||
### LOG-001 正常轮询日志
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 重启 Backend 容器。
|
||||
2. 连续观察 1 分钟日志。
|
||||
|
||||
```bash
|
||||
docker logs -f --tail=200 yg-ft-backend-api
|
||||
```
|
||||
|
||||
**预期**
|
||||
|
||||
- 不再每次以 `INFO` 输出 `compute jobs polled`。
|
||||
- 健康检查成功请求不再以 `INFO` 输出应用日志。
|
||||
- 正常任务同步日志仅在 `DEBUG` 级别出现。
|
||||
|
||||
### LOG-002 异常轮询日志
|
||||
|
||||
**步骤**
|
||||
|
||||
1. 临时停止算力节点或断开节点网络。
|
||||
2. 观察 Backend 日志。
|
||||
|
||||
**预期**
|
||||
|
||||
- 轮询失败以 `WARNING` 或 `ERROR` 输出。
|
||||
- 异常包含节点、任务或错误原因。
|
||||
- 恢复节点后轮询继续工作。
|
||||
|
||||
## 13. 容器验证命令
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
docker logs --tail=200 yg-ft-backend-api
|
||||
curl -i http://172.25.179.69:17861/modelTF/health
|
||||
curl -i http://172.25.179.69:16801/
|
||||
```
|
||||
|
||||
预期 Backend 和 Frontend 均为 `healthy`,健康接口返回 HTTP 200。
|
||||
|
||||
## 14. 测试结果记录
|
||||
|
||||
| 用例编号 | 测试结果 | 实际结果 | 缺陷编号 | 测试人 | 日期 |
|
||||
|---|---|---|---|---|---|
|
||||
| AUTH-001 | □通过 □失败 | | | | |
|
||||
| AUTH-002 | □通过 □失败 | | | | |
|
||||
| AUTH-003 | □通过 □失败 | | | | |
|
||||
| AUTH-004 | □通过 □失败 | | | | |
|
||||
| ACL-001 | □通过 □失败 | | | | |
|
||||
| TRAIN-001 | □通过 □失败 | | | | |
|
||||
| TRAIN-002 | □通过 □失败 | | | | |
|
||||
| EVAL-001 | □通过 □失败 | | | | |
|
||||
| INFER-001 | □通过 □失败 | | | | |
|
||||
| MODEL-001 | □通过 □失败 | | | | |
|
||||
| GPU-001 | □通过 □失败 | | | | |
|
||||
| DB-001 | □通过 □失败 | | | | |
|
||||
| LOG-001 | □通过 □失败 | | | | |
|
||||
| LOG-002 | □通过 □失败 | | | | |
|
||||
122
docs/permissions-and-logging-test-results.md
Normal file
122
docs/permissions-and-logging-test-results.md
Normal file
@@ -0,0 +1,122 @@
|
||||
# 权限与日志改造自动化测试结果
|
||||
|
||||
## 1. 测试时间
|
||||
|
||||
2026-08-12
|
||||
|
||||
## 2. 测试环境
|
||||
|
||||
| 项目 | 地址/状态 |
|
||||
|---|---|
|
||||
| Frontend | `http://172.25.179.69:16801` |
|
||||
| Backend | `http://172.25.179.69:17861/modelTF` |
|
||||
| WSL | `172.25.179.69` |
|
||||
| Docker | 已运行 |
|
||||
|
||||
## 3. 自动执行结果
|
||||
|
||||
| 用例 | 结果 | 实际结果 |
|
||||
|---|---|---|
|
||||
| 容器状态 | 通过 | Compute、Frontend、Backend、Redis、MinIO 均为 `healthy` |
|
||||
| Backend 健康检查 | 通过 | `/modelTF/health` 返回 HTTP 200 |
|
||||
| Frontend 首页 | 通过 | `/` 返回 HTTP 200 |
|
||||
| 未登录访问数据集接口 | 通过 | `/modelTF/dataset-manage` 返回 HTTP 401 |
|
||||
| Redis 容器状态 | 通过 | 容器状态为 healthy |
|
||||
| 轮询正常日志降噪 | 通过 | 最近 120 秒未发现 `compute jobs polled` 或 `health check requested` 高频日志 |
|
||||
| 轮询异常日志保留 | 未触发 | 当前未人为停止算力节点,未产生轮询失败日志 |
|
||||
| 前端构建 | 通过 | `npm run build` 成功 |
|
||||
| 后端编译 | 通过 | 相关 Python 模块 `py_compile` 成功 |
|
||||
| Git 差异检查 | 通过 | `git diff --check` 无格式错误 |
|
||||
|
||||
## 4. 数据库字段迁移
|
||||
|
||||
### 静态检查结果
|
||||
|
||||
初始化 SQL 和运行时迁移逻辑已包含以下字段:
|
||||
|
||||
- `models.deleted_at`
|
||||
- `models.deleted_by`
|
||||
- `models.tenant_id`
|
||||
- `models.project_id`
|
||||
- `datasets.deleted_at`
|
||||
- `datasets.deleted_by`
|
||||
- `datasets.tenant_id`
|
||||
- `datasets.project_id`
|
||||
- `trained_models.deleted_at`
|
||||
- `trained_models.deleted_by`
|
||||
- `trained_models.tenant_id`
|
||||
- `trained_models.project_id`
|
||||
- `eval_tasks.deleted_at`
|
||||
- `eval_tasks.deleted_by`
|
||||
- `sessions.issued_at`
|
||||
- `sessions.expires_at`
|
||||
|
||||
### 远程数据库检查状态
|
||||
|
||||
已通过当前 Backend 容器使用一次性 PostgreSQL 连接检查远程数据库,字段迁移已完成。
|
||||
|
||||
确认存在:
|
||||
|
||||
- `models`: `created_by`、`tenant_id`、`project_id`、`deleted_at`、`deleted_by`
|
||||
- `datasets`: `created_by`、`tenant_id`、`project_id`、`deleted_at`、`deleted_by`
|
||||
- `trained_models`: `created_by`、`tenant_id`、`project_id`、`deleted_at`、`deleted_by`
|
||||
- `eval_tasks`: `created_by`、`tenant_id`、`project_id`、`deleted_at`、`deleted_by`
|
||||
- `sessions`: `issued_at`、`expires_at`、`logout_at`
|
||||
|
||||
使用的检查 SQL:
|
||||
|
||||
```sql
|
||||
SELECT table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_name IN ('models', 'datasets', 'trained_models', 'eval_tasks', 'sessions')
|
||||
ORDER BY table_name, ordinal_position;
|
||||
```
|
||||
|
||||
## 5. Redis 说明
|
||||
|
||||
已从当前环境确认 Redis 实际密码配置为 `Tvhrf659WaX-S1B8FG6c2kSZK07XTv82`,使用该密码验证返回 `PONG`。
|
||||
|
||||
项目配置已同步为:
|
||||
|
||||
```env
|
||||
REDIS_PASSWORD=Tvhrf659WaX-S1B8FG6c2kSZK07XTv82
|
||||
REDIS_URL=redis://:Tvhrf659WaX-S1B8FG6c2kSZK07XTv82@redis:6379/0
|
||||
```
|
||||
|
||||
后续验证应从当前 Docker 环境读取实际密码后执行:
|
||||
|
||||
```bash
|
||||
docker compose config
|
||||
docker exec yg-ft-redis printenv REDISCLI_AUTH
|
||||
docker exec yg-ft-redis redis-cli -a "$REDISCLI_AUTH" ping
|
||||
```
|
||||
|
||||
预期返回:
|
||||
|
||||
```text
|
||||
PONG
|
||||
```
|
||||
|
||||
## 6. 未自动执行的破坏性用例
|
||||
|
||||
以下用例需要测试账号、测试资源或人工确认,未自动执行,以避免影响现有数据:
|
||||
|
||||
- ACL 授权和撤销。
|
||||
- 训练任务创建、停止、重试和删除。
|
||||
- 评测任务启动和删除。
|
||||
- 推理模型加载、卸载和删除。
|
||||
- 模型权重合并和导出。
|
||||
- GPU 分配、占用和释放。
|
||||
- 软删除后资源恢复和数据完整性。
|
||||
- 人为停止算力节点验证轮询失败恢复。
|
||||
- 登录失败 5 次后的限流验证。
|
||||
- Token 过期和注销后的访问验证。
|
||||
|
||||
## 7. 当前结论
|
||||
|
||||
当前服务基础可用,页面和 Backend 健康接口正常,未登录鉴权正常,日志降噪逻辑生效,前后端构建通过。
|
||||
|
||||
当前仍需人工或使用专用测试数据验证:
|
||||
|
||||
1. 权限矩阵中的创建、执行、删除、审批和 GPU 占用场景。
|
||||
2. Token 过期和登录失败限流场景。
|
||||
@@ -803,3 +803,88 @@ async function loadGpus() {
|
||||
4. **前端适配**:GPU 下拉过滤、资源授权按钮、权限管理页面优化
|
||||
5. **审批扩展**:在删除/停止接口中接入 `_require_approval_or_admin`
|
||||
6. **测试补充**:扩展 `test_governance.py` 覆盖 GPU 分配、资源过滤、审批扩展场景
|
||||
## 16. 2.0 权限增强基线(MinIO、缓存与跨节点场景)
|
||||
|
||||
### 16.1 默认拒绝
|
||||
|
||||
所有受保护接口采用 deny by default。权限判定依次执行:认证、租户边界、项目成员关系、页面权限、资源动作权限、审批校验和审计记录。任何一步无法确定时返回 403,不得因为字段缺失、资源不存在或 ACL 查询异常而自动放行。
|
||||
|
||||
### 16.2 资源归属和继承
|
||||
|
||||
资源统一使用 `tenant_id`、`project_id`、`created_by`、`visibility` 表达边界。训练任务继承模型、数据集和项目边界;训练模型继承训练任务边界;合并模型继承被合并模型边界;评测必须同时校验模型和数据集;推理必须校验模型、项目和算力节点。
|
||||
|
||||
### 16.3 MinIO 对象权限
|
||||
|
||||
- 所有对象访问必须经过 Backend 鉴权,前端不得持有 MinIO 密钥。
|
||||
- Compute API 只使用 Backend 签发的预签名 URL。
|
||||
- 预签名 URL 默认有效期不超过 15 分钟,上传和下载分别签发。
|
||||
- 生成 URL 前必须校验资源权限、对象状态和版本归属。
|
||||
- bucket 由服务端配置,禁止客户端提交任意 bucket。
|
||||
- 禁止通过修改 `object_key`、`version_id` 或文件名越权访问对象。
|
||||
- 删除对象使用 `deleting -> deleted` 状态,失败时保留错误信息并可重试。
|
||||
|
||||
### 16.4 缓存和算力节点权限
|
||||
|
||||
- 缓存是资源副本,不产生新的资源所有权。
|
||||
- 只有拥有源模型或数据集 `execute` 权限的用户才能触发缓存。
|
||||
- 用户不能直接调用 Compute API 的缓存、文件、上传和推理管理接口。
|
||||
- Compute API 只接受 Backend 服务令牌,不能转发用户 Token。
|
||||
- 产物归档必须校验任务、节点、资源和项目关联关系。
|
||||
- 任务运行期间缓存引用不可被普通用户清理。
|
||||
- 缓存清理只能删除节点副本,不得删除 MinIO 正式对象。
|
||||
|
||||
### 16.5 训练、合并、推理和评测动作矩阵
|
||||
|
||||
| 动作 | 必要权限 | 额外约束 |
|
||||
|---|---|---|
|
||||
| 创建训练 | 项目 `write` + 模型/数据集 `execute` | GPU、配额和节点权限同时通过 |
|
||||
| 查看训练 | 任务 `read` | 日志、曲线、checkpoint 继承任务权限 |
|
||||
| 停止训练 | 任务 `write` 或 `admin` | 停止他人任务需要审批或管理员权限 |
|
||||
| 权重合并 | 训练模型 `execute` | 使用绑定节点或有权限的指定节点 |
|
||||
| 归档训练产物 | 任务 `write` | 只能归档任务输出目录内文件 |
|
||||
| 创建推理 | 模型 `execute` | 节点、GPU 配额和项目权限通过 |
|
||||
| 删除推理 | 推理服务 `delete` 或管理员 | 释放 GPU 和缓存引用 |
|
||||
| 创建评测 | 模型/数据集 `execute` | 两个资源必须在允许范围内 |
|
||||
| 下载报告 | 评测任务 `read` + 报告 `download` | 预签名 URL 短时有效 |
|
||||
|
||||
### 16.6 服务身份和密钥边界
|
||||
|
||||
| 身份 | 用途 | 禁止事项 |
|
||||
|---|---|---|
|
||||
| 用户 Token | 调用 Backend 业务接口 | 直接调用 Compute 或 MinIO |
|
||||
| Backend 服务令牌 | 调用 Compute API | 返回前端或写入任务参数 |
|
||||
| MinIO 管理密钥 | Backend 对象操作 | 注入浏览器、Compute 容器或日志 |
|
||||
| Compute 节点身份 | 节点心跳和任务执行 | 访问其他节点本地路径 |
|
||||
|
||||
生产环境禁止使用默认凭据;服务令牌必须从环境变量或密钥管理系统读取并脱敏记录。
|
||||
|
||||
### 16.7 必须补充的接口保护
|
||||
|
||||
```text
|
||||
POST /storage/objects/presign
|
||||
POST /storage/resources/{type}/{id}/prepare/{node_id}
|
||||
POST /storage/resources/{type}/{id}/archive-node/{node_id}
|
||||
GET /storage/cache/jobs/{node_id}
|
||||
POST /model-manage/merge
|
||||
POST /model-chat/local/preload
|
||||
POST /model-chat/trained/preload
|
||||
POST /model-chat/local/unload
|
||||
POST /model-compare/{task_id}/load
|
||||
POST /model-compare/{task_id}/unload
|
||||
```
|
||||
|
||||
请求体中的所有资源 ID 都必须校验。`node_id`、`model_id`、`dataset_id`、`task_id` 不一致时返回 403 或 409。
|
||||
|
||||
### 16.8 权限审计验收用例
|
||||
|
||||
必须覆盖:用户 A 不能读取用户 B 资源;项目 A 不能使用项目 B 数据集;评测必须同时拥有模型和数据集 `execute` 权限;修改对象 key、版本或资源 ID 不能获取预签名 URL;普通用户不能调用 Compute API;产物路径不能越出任务数据根目录;用户不能清理其他项目正在使用的缓存;无节点权限时推理返回 403;删除、停止、合并和归档他人资源按规则进入审批。
|
||||
|
||||
## 17. 2.0 实施顺序
|
||||
|
||||
1. 建立统一 `authorize_resource_action()` 和 `authorize_task_resources()` 后端辅助函数。
|
||||
2. 为模型、数据集、训练、推理、评测和对象接口补齐租户/项目过滤。
|
||||
3. 统一预签名 URL 权限校验、有效期和审计日志。
|
||||
4. 为 Compute API 增加服务令牌、节点归属和路径范围校验。
|
||||
5. 将 GPU、缓存、配额和节点选择校验合并到任务创建事务中。
|
||||
6. 增加跨资源权限测试和越权回归测试。
|
||||
7. 前端仅负责隐藏操作按钮,最终权限以 Backend 返回为准。
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<script setup lang="ts">
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, computed, onMounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
||||
|
||||
@@ -100,7 +100,7 @@ export function buildFineTunePayload(
|
||||
}
|
||||
|
||||
export function buildFineTuneCommand(form: FineTuneFormModel, gpus: number[]) {
|
||||
const gpuIds = gpus.length ? gpus.join(',') : '0'
|
||||
const gpuIds = gpus.length ? gpus.join(',') : '<auto>'
|
||||
const stage = form.train_type === 'DPO' ? 'dpo' : form.train_type === 'CPT' ? 'pt' : 'sft'
|
||||
const lines = [
|
||||
`CUDA_VISIBLE_DEVICES=${gpuIds} llamafactory-cli train`,
|
||||
|
||||
Reference in New Issue
Block a user