YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合

Browse Source 
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请
- 存储: MinIO 存储进度迁移、对象存储安全加固与测试
- 计算: GPU 资源预留、compute 轮询与同步增强
- 权限: permission v2 迁移、权限安全验收测试
- 日志: 后端运行日志中文说明、操作日志整合
- 数据处理/评测: 数据转换与模型评测优化

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-21 09:49:48 +08:00
parent 080ef6ab00
commit 6f0e82f351
94 changed files with 9547 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
backend/app/api/v1/endpoints/data_process.py
View File

@@ -29,12 +29,18 @@ from fastapi import (
Header,
HTTPException,
Query,
Request,
UploadFile,
)
from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.core.auth import (
filter_accessible_resource_ids,
get_current_user,
has_resource_access,
is_admin,
)
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
@@ -110,7 +116,6 @@ from app.schemas.data_process import (
ResultUpdate,
)
router = APIRouter(prefix="/data-process")
logger = logging.getLogger(__name__)
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
MAX_SOURCE_FILE_COUNT = 20
@@ -163,6 +168,27 @@ def fail(status_code: int, message: str) -> HTTPException:
)
def _authorize_data_process_request(
request: Request,
task_id: str | None = None,
current_user: dict[str, Any] = Depends(get_current_user),
) -> None:
"""Apply resource ACL and tenant checks to every task-scoped endpoint."""
if not task_id or is_admin(current_user):
return
permission = "read" if request.method in {"GET", "HEAD"} else "write"
if any(marker in request.url.path for marker in ("/start", "/generate", "/regenerate", "/repeat", "/publish", "/external/")):
permission = "execute"
if not has_resource_access("data_process", task_id, current_user, permission):
raise fail(403, "no permission to access this data process task")
router = APIRouter(
prefix="/data-process",
dependencies=[Depends(_authorize_data_process_request)],
)
@contextmanager
def api_errors() -> Iterator[None]:
try:
@@ -887,7 +913,7 @@ def list_tasks(
keyword=keyword,
status=status,
process_type=process_type,
tenant_id=tenant_id,
tenant_id=tenant_id if is_admin(current_user) else (current_user.get("tenant_id") or "default"),
project_id=project_id,
)
# #4 资源 ACL 过滤：admin 放行，普通用户只看到自己被授权的数据处理任务
@@ -908,9 +934,15 @@ def list_tasks(
def create_task(
payload: DataProcessTaskCreate,
store: DataProcessStore = Depends(get_data_process_store),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
with api_errors():
task = store.create_task(payload.model_dump(mode="json"))
values = payload.model_dump(mode="json")
values["created_by"] = current_user.get("id")
values["owner_id"] = current_user.get("id")
values["tenant_id"] = current_user.get("tenant_id") or "default"
get_platform_store().assert_active_tenant(values["tenant_id"])
task = store.create_task(values)
return ok(task, "data process task created")

1245
backend/app/api/v1/endpoints/platform.py
View File

File diff suppressed because it is too large Load Diff

73
backend/app/core/audit.py
View File

@@ -21,7 +21,7 @@ from typing import Any, Callable, Optional, TypeVar
from fastapi import Request
from app.core.logging import get_logger, request_id_var
from app.core.logging import get_client_ip, get_logger, mask_sensitive_string, request_id_var
logger = get_logger("app.audit")
@@ -67,12 +67,25 @@ def audit_log(
detail=detail,
trace_id=trace_id,
duration_ms=elapsed_ms,
kwargs=kwargs,
args=args,
)
return result
except Exception:
logger.error(
"审计日志记录失败 action=%s", action, exc_info=True
except Exception as exc:
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=_extract_target_id(None, kwargs, extract_target_id),
detail=_build_detail(detail_template, kwargs),
trace_id=trace_id,
duration_ms=(time.perf_counter() - started_at) * 1000,
result="failure",
reason=_safe_exception_reason(exc),
kwargs=kwargs,
args=args,
)
logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc))
raise
return async_wrapper # type: ignore
@@ -94,12 +107,25 @@ def audit_log(
detail=detail,
trace_id=trace_id,
duration_ms=elapsed_ms,
kwargs=kwargs,
args=args,
)
return result
except Exception:
logger.error(
"审计日志记录失败 action=%s", action, exc_info=True
except Exception as exc:
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=_extract_target_id(None, kwargs, extract_target_id),
detail=_build_detail(detail_template, kwargs),
trace_id=trace_id,
duration_ms=(time.perf_counter() - started_at) * 1000,
result="failure",
reason=_safe_exception_reason(exc),
kwargs=kwargs,
args=args,
)
logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc))
raise
return sync_wrapper # type: ignore
@@ -144,18 +170,36 @@ def _record_audit(
detail: str,
trace_id: str,
duration_ms: float,
*,
kwargs: dict[str, Any] | None = None,
args: tuple[Any, ...] = (),
result: str = "success",
reason: str | None = None,
) -> None:
"""通过已有的 record_audit 方法写入审计日志"""
try:
from app.db.platform_store import get_platform_store
store = get_platform_store()
kwargs = kwargs or {}
request = _extract_request(args, kwargs)
current_user = kwargs.get("current_user") or kwargs.get("user") or {}
request_id = request.headers.get("X-Request-ID") if request else None
request_id = request_id or trace_id
client_ip = get_client_ip(request) or None
detail_text = f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}"
store.record_audit(
action=action,
actor_id=actor_id,
target_type=target_type or None,
target_id=target_id,
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
tenant_id=str(current_user.get("tenant_id") or "") or None,
detail=mask_sensitive_string(detail_text),
result=result,
reason=mask_sensitive_string(reason or "") or None,
request_id=request_id,
session_id=str(current_user.get("session_id") or "") or None,
ip=client_ip,
)
except Exception:
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
@@ -170,6 +214,19 @@ def _extract_actor_id(kwargs: dict) -> Optional[str]:
return None
def _extract_request(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Request | None:
for value in tuple(kwargs.values()) + tuple(args):
if isinstance(value, Request):
return value
return None
def _safe_exception_reason(exc: Exception) -> str:
"""Keep audit failures useful without recording credentials or tokens."""
value = getattr(exc, "detail", None) or str(exc) or exc.__class__.__name__
return mask_sensitive_string(str(value))[:500]
# ==================== 预定义的审计操作常量 ====================
class AuditActions:

527
backend/app/core/auth.py
View File

@@ -2,20 +2,76 @@
from __future__ import annotations
from typing import Any
import json
from fastapi import Depends, HTTPException, Query, Request, status
from app.db.platform_store import get_platform_store
from app.core.logging import get_client_ip
# 无需鉴权的路径前缀（健康检查、登录等）
PUBLIC_PATHS = ("/health", "/login", "/system-info")
OWNER_TABLES = {
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
"fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"),
"fine-tune": ("fine_tune_tasks", "created_by"), "fine_tune_task": ("fine_tune_tasks", "created_by"),
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
"project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"),
"project": ("projects", "create_by"), "data_process": ("data_process_tasks", "created_by"),
"data_convert": ("data_convert_tasks", "created_by"),
}
RESOURCE_TABLES = {
"dataset": "datasets",
"model": "models",
"trained_model": "trained_models",
"eval": "eval_tasks",
"fine-tune": "fine_tune_tasks",
"fine_tune_task": "fine_tune_tasks",
"compare": "compare_tasks",
"inference": "compare_tasks",
"project": "projects",
"data_process": "data_process_tasks",
"data_convert": "data_convert_tasks",
}
MODULE_PERMISSIONS = {
"dashboard": "dashboard",
"fine-tune": "fine-tune",
"model-eval": "model-eval",
"model-compare": "model-inference",
"model-inference": "model-inference",
"model-chat": "model-inference",
"model-manage": "model-manage",
"dataset-manage": "dataset",
"data-process": "data-process",
"data-convert": "data-convert",
"compute": "compute",
"hardware": "hardware",
"users": "user-settings",
}
# These endpoints are the user-facing compute view used by training,
# inference, and evaluation forms. They only return the current user's
# assigned nodes/GPUs in the endpoint implementation, so they must remain
# available after an approval without granting access to the admin compute
# management page.
SELF_SERVICE_COMPUTE_PATHS = {
"/compute/nodes",
"/compute/gpus",
"/compute/my-gpus",
}
# Resource actions are deliberately kept separate from module permissions.
# A user may be allowed to open a module while still lacking the action on a
# specific resource (for example, download or delete).
RESOURCE_ACTIONS = frozenset({
"read",
"write",
"execute",
"download",
"export",
"delete",
"admin",
})
RESOURCE_ACTION_ALIASES = {"export": "download"}
def _extract_token(request: Request) -> str | None:
@@ -30,6 +86,26 @@ def _session_token(user_id: str, session_id: str) -> str:
return f"platform-token-{user_id}.{session_id}"
def _record_auth_event(actor_id: str | None, action: str, reason: str, request: Request) -> None:
"""Best-effort security audit for authentication and permission denials."""
try:
get_platform_store().record_audit(
action=action,
actor_id=actor_id,
target_type="auth",
target_id=request.url.path,
detail=reason,
result="denied",
reason=reason,
request_id=request.headers.get("X-Request-ID"),
ip=get_client_ip(request) or None,
)
except Exception:
# An audit failure must never turn an authentication decision into an
# accidental allow or an unrelated 500 response.
pass
def get_current_user(request: Request) -> dict[str, Any]:
"""
FastAPI 依赖：解析当前登录用户。
@@ -55,31 +131,219 @@ def get_current_user(request: Request) -> dict[str, Any]:
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
).fetchone()
if not session or session["user_id"] != user_id or session["logout_at"]:
_record_auth_event(user_id, "auth.session.denied", "session expired or logged out", request)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
if session["expires_at"]:
from datetime import datetime, timezone
try:
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
_record_auth_event(user_id, "auth.session.denied", "session expired", request)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
except ValueError:
pass
with store.connect() as conn:
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
if user_row:
return store._user(user_row)
user = store._user(user_row)
if user.get("status") != "active":
_record_auth_event(user_id, "auth.user.disabled", "user is not active", request)
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user disabled")
requested_tenant = request.headers.get("X-Tenant-ID", "").strip()
if requested_tenant and (is_admin(user) or requested_tenant in user_tenant_ids(user)):
user["tenant_id"] = requested_tenant
# Keep the session identifier in request context so business audit
# records can be traced back to the exact login session.
if session_id:
user["session_id"] = session_id
path_parts = path.strip("/").split("/")
# The API may be mounted directly at /modelTF or behind /api/v1/modelTF.
# Locate the first known module segment instead of relying on a fixed index.
segment = next((part for part in path_parts if part in MODULE_PERMISSIONS), "")
required = MODULE_PERMISSIONS.get(segment)
relative_path = "/" + "/".join(path_parts[path_parts.index(segment):]) if segment else path
self_service_compute = relative_path.rstrip("/") in SELF_SERVICE_COMPUTE_PATHS
if (
required
and not is_admin(user)
and required not in (user.get("permissions") or [])
and not self_service_compute
):
_record_auth_event(user_id, "auth.permission.denied", f"missing permission: {required}", request)
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"missing permission: {required}")
return user
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
"""FastAPI 依赖：要求当前用户是管理员（role=admin 或 protected）。"""
if current_user.get("role") == "admin" or current_user.get("protected"):
"""FastAPI 依赖：要求当前用户是平台管理员。"""
if is_admin(current_user):
return current_user
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
def require_tenant_admin(
tenant_id: str,
current_user: dict[str, Any] = Depends(get_current_user),
) -> dict[str, Any]:
"""Allow platform admins and active owner/admin tenant members."""
if is_admin(current_user) or is_tenant_admin(current_user, tenant_id):
return current_user
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant admin permission required")
def is_admin(user: dict[str, Any]) -> bool:
"""判断用户是否为管理员（admin 角色或 protected 标记）。"""
return user.get("role") == "admin" or user.get("protected", False)
"""判断用户是否为平台管理员；兼容历史 role=admin/protected 数据。"""
return (
user.get("platform_role") == "platform_admin"
or user.get("role") == "admin"
or user.get("protected", False)
)
def user_tenant_ids(user: dict[str, Any]) -> set[str]:
"""Return the tenant scope of a user.
Existing installations keep the primary tenant on ``users.tenant_id``.
``tenant_members`` is optional during the migration and adds memberships
when the permission v2 schema is available.
"""
if is_admin(user):
return {"*"}
primary_tenant = str(user.get("tenant_id") or "default")
result: set[str] = set()
user_id = user.get("id")
if not user_id:
return result
try:
store = get_platform_store()
with store.connect() as conn:
rows = conn.execute(
"SELECT tenant_id FROM tenant_members "
"WHERE user_id=? AND status='active' "
"AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)",
(user_id,),
).fetchall()
result.update(str(row["tenant_id"]) for row in rows if row.get("tenant_id"))
if not result:
active_tenant = conn.execute(
"SELECT id FROM tenants WHERE id=? "
"AND COALESCE(status, 'active')='active' "
"AND COALESCE(deleted_at, '')=''",
(primary_tenant,),
).fetchone()
if active_tenant:
result.add(primary_tenant)
except Exception:
# Older databases are upgraded lazily. The primary users.tenant_id
# remains a valid fallback until the additive table is available.
result.add(primary_tenant)
return result
def tenant_membership(user: dict[str, Any], tenant_id: str | None = None) -> dict[str, Any] | None:
"""Return the active membership for the selected tenant, if any."""
if is_admin(user):
return {"tenant_id": tenant_id or user.get("tenant_id") or "default", "role": "owner", "status": "active"}
target = str(tenant_id or user.get("tenant_id") or "default")
try:
with get_platform_store().connect() as conn:
row = conn.execute(
"SELECT tenant_id, role, status, expires_at FROM tenant_members "
"WHERE tenant_id=? AND user_id=? AND status='active'",
(target, user.get("id")),
).fetchone()
if not row:
return None
if row.get("expires_at"):
from datetime import datetime, timezone
try:
if datetime.fromisoformat(str(row["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
return None
except ValueError:
return None
return dict(row)
except Exception:
return None
def is_tenant_admin(user: dict[str, Any], tenant_id: str | None = None) -> bool:
membership = tenant_membership(user, tenant_id)
return bool(membership and membership.get("role") in {"owner", "admin"})
def is_tenant_admin_for_resource(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool:
if is_admin(user) or resource_type == "model":
return False
tenant_id = resource_tenant_id(resource_type, resource)
return bool(tenant_id and is_tenant_admin(user, tenant_id))
def bind_active_tenant(payload: dict[str, Any], user: dict[str, Any]) -> dict[str, Any]:
"""Bind a new resource to the authenticated tenant context.
Platform administrators may explicitly create a resource in another
active tenant. Ordinary users can only use the tenant selected by the
authenticated session/X-Tenant-ID header, never a client-supplied tenant
id alone.
"""
requested = str(payload.get("tenant_id") or user.get("tenant_id") or "default")
if not is_admin(user) and requested not in user_tenant_ids(user):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant access denied")
payload["tenant_id"] = requested if is_admin(user) else str(user.get("tenant_id") or requested)
return payload
def resource_record(resource_type: str, resource_id: str) -> dict[str, Any] | None:
"""Load a resource row for authorization without exposing storage details."""
table = RESOURCE_TABLES.get(resource_type)
if not table or not resource_id:
return None
store = get_platform_store()
try:
with store.connect() as conn:
row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (resource_id,)).fetchone()
except Exception:
return None
if not row:
return None
result = dict(row)
if result.get("deleted_at"):
return None
return result
def resource_tenant_id(resource_type: str, resource: dict[str, Any]) -> str | None:
"""Resolve tenant ownership, including legacy JSON-backed task rows."""
# Administrator-created base models are platform shared. Online models
# created by ordinary users retain tenant scope, so their visibility can
# be limited to the owner and members of that tenant.
if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}:
return None
tenant_id = resource.get("tenant_id")
if tenant_id:
return str(tenant_id)
payload = resource.get("payload")
if payload:
try:
data = json.loads(payload) if isinstance(payload, str) else payload
if isinstance(data, dict) and data.get("tenant_id"):
return str(data["tenant_id"])
except (TypeError, ValueError, json.JSONDecodeError):
pass
return None
def resource_in_user_tenant(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool:
if is_admin(user):
return True
if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}:
return True
tenant_id = resource_tenant_id(resource_type, resource)
if not tenant_id:
# A missing tenant is not an implicit shared scope. The migration must
# assign historical rows before ordinary users can access them.
return False
return "*" in user_tenant_ids(user) or tenant_id in user_tenant_ids(user)
def has_resource_access(
@@ -93,16 +357,57 @@ def has_resource_access(
- admin/protected 用户直接放行（旁路）。
- 其他用户检查 acls 表中是否有对应授权。
"""
if user.get("role") == "admin" or user.get("protected"):
if permission not in RESOURCE_ACTIONS:
return False
permission = RESOURCE_ACTION_ALIASES.get(permission, permission)
if is_admin(user):
return True
store = get_platform_store()
resource = resource_record(resource_type, resource_id)
if not resource:
return False
# Some historical datasets were created before tenant_id was introduced.
# They are not shared by default, but an explicit user ACL granted by an
# administrator is still a valid compatibility path for that legacy row.
legacy_unscoped = resource_type != "model" and not resource_tenant_id(resource_type, resource)
if not resource_in_user_tenant(resource_type, resource, user) and not legacy_unscoped:
return False
if resource_type == "model":
model_source = str(resource.get("model_source") or "").lower()
if permission in {"read", "execute"}:
# Local/registered base models are platform resources. For online
# models, only administrator-created records are global; a normal
# user's online model is shared with the active tenant below.
if model_source not in {"api", "online"}:
return True
creator_id = str(resource.get("created_by") or "")
if creator_id:
with store.connect() as conn:
creator = conn.execute(
"SELECT platform_role, role, protected FROM users WHERE id=?",
(creator_id,),
).fetchone()
if creator and (
creator.get("platform_role") == "platform_admin"
or creator.get("role") == "admin"
or bool(creator.get("protected"))
):
return True
# The tenant check above has already rejected models outside the
# user's active tenant. Same-tenant online models are usable.
if resource_tenant_id(resource_type, resource) in user_tenant_ids(user):
return True
if is_tenant_admin_for_resource(resource_type, resource, user):
return True
acls = store.get_acl(resource_type, resource_id)
user_id = user.get("id")
user_role = user.get("role")
# ACL role principals are tenant roles, not platform roles. Falling back
# to the platform role keeps compatibility with old ACL records.
membership = tenant_membership(user, resource_tenant_id(resource_type, resource))
user_role = (membership or {}).get("role") or user.get("role")
owner_tables = OWNER_TABLES
table_info = owner_tables.get(resource_type)
table_info = OWNER_TABLES.get(resource_type)
if table_info and user_id:
table, column = table_info
with store.connect() as conn:
@@ -111,7 +416,6 @@ def has_resource_access(
owner = row[column]
if column == "payload":
try:
import json
owner = json.loads(owner or "{}").get("created_by")
except (TypeError, ValueError):
owner = None
@@ -119,6 +423,15 @@ def has_resource_access(
return True
for entry in acls:
if entry.get("revoked_at"):
continue
if entry.get("expires_at"):
from datetime import datetime, timezone
try:
if datetime.fromisoformat(str(entry["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
continue
except ValueError:
pass
# 按 user 授权
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
if _permission_covers(entry.get("permission"), permission):
@@ -154,48 +467,13 @@ def filter_accessible_resource_ids(
- admin 直接返回全部。
- 普通用户查 acls 表取交集。
"""
if user.get("role") == "admin" or user.get("protected"):
if is_admin(user):
return all_ids
if not all_ids:
return []
store = get_platform_store()
user_id = user.get("id")
user_role = user.get("role")
# 查询该用户在该资源类型下有 read 权限的所有 resource_id
with store.connect() as conn:
rows = conn.execute(
"""
SELECT DISTINCT resource_id FROM acls
WHERE resource_type=? AND (
(principal_type='user' AND principal_id=?)
OR (principal_type='role' AND principal_id=?)
)
""",
(resource_type, user_id, user_role),
).fetchall()
accessible = {r["resource_id"] for r in rows}
if resource_type in OWNER_TABLES:
table, column = OWNER_TABLES[resource_type]
if column == "payload":
# payload 是 JSON 字符串，需要查出后解析 created_by
with store.connect() as conn:
owned = conn.execute(f"SELECT id, {column} FROM {table}").fetchall()
for row in owned:
try:
import json
payload = json.loads(row[column] or "{}")
if payload.get("created_by") == user_id:
accessible.add(row["id"])
except (TypeError, ValueError):
pass
else:
with store.connect() as conn:
owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall()
accessible.update(row["id"] for row in owned)
accessible = filter_accessible_resource_ids_batch(resource_type, all_ids, user)
return [rid for rid in all_ids if rid in accessible]
@@ -204,37 +482,134 @@ def filter_accessible_resource_ids_batch(
resource_ids: list[str],
user: dict[str, Any],
) -> set[str]:
"""Filter a list endpoint with one ACL query instead of one query per row."""
"""Filter a list endpoint with a bounded set of SQL queries.
The previous implementation called ``has_resource_access`` once per
resource. Each call loaded the resource, tenant membership and ACL again,
which made ordinary-user list pages slow on a remote PostgreSQL server.
"""
if is_admin(user):
return set(resource_ids)
if not resource_ids:
ids = list(dict.fromkeys(str(item) for item in resource_ids if item))
if not ids:
return set()
table = RESOURCE_TABLES.get(resource_type)
if not table:
return set()
placeholders = ",".join("?" for _ in ids)
primary_tenant = str(user.get("tenant_id") or "default")
tenant_ids = {primary_tenant}
tenant_roles: dict[str, str] = {primary_tenant: str(user.get("role") or "")}
user_id = str(user.get("id") or "")
store = get_platform_store()
placeholders = ",".join("?" for _ in resource_ids)
with store.connect() as conn:
rows = conn.execute(
f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
"AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))",
(resource_type, *resource_ids, user.get("id"), user.get("role")),
memberships = conn.execute(
"SELECT tenant_id, role FROM tenant_members "
"WHERE user_id=? AND status='active' "
"AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)",
(user_id,),
).fetchall()
accessible = {row["resource_id"] for row in rows}
table_info = OWNER_TABLES.get(resource_type)
if table_info and user.get("id"):
table, column = table_info
with store.connect() as conn:
owned = conn.execute(
f"SELECT id, {column} FROM {table} WHERE id IN ({placeholders})",
(*resource_ids,),
).fetchall()
for row in owned:
owner = row[column]
# 如果列是 payload（JSON），需要解析后提取 created_by
if column == "payload":
for membership in memberships:
tenant_id = str(membership["tenant_id"] or "")
if tenant_id:
tenant_ids.add(tenant_id)
tenant_roles[tenant_id] = str(membership["role"] or "")
rows = conn.execute(
f"SELECT * FROM {table} WHERE id IN ({placeholders})", tuple(ids)
).fetchall()
row_by_id = {str(row["id"]): dict(row) for row in rows}
acl_rows = conn.execute(
"SELECT resource_id, principal_type, principal_id, permission, expires_at "
f"FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
"AND (revoked_at IS NULL OR revoked_at='')",
(resource_type, *ids),
).fetchall()
acl_by_resource: dict[str, list[dict[str, Any]]] = {}
for row in acl_rows:
acl_by_resource.setdefault(str(row["resource_id"]), []).append(dict(row))
allowed: set[str] = set()
for resource_id in ids:
resource = row_by_id.get(resource_id)
if not resource or resource.get("deleted_at"):
continue
if resource_type == "model":
model_source = str(resource.get("model_source") or "").lower()
if model_source not in {"api", "online"}:
allowed.add(resource_id)
continue
creator_id = str(resource.get("created_by") or "")
if creator_id:
with store.connect() as conn:
creator = conn.execute(
"SELECT platform_role, role, protected FROM users WHERE id=?",
(creator_id,),
).fetchone()
if creator and (
creator.get("platform_role") == "platform_admin"
or creator.get("role") == "admin"
or bool(creator.get("protected"))
):
allowed.add(resource_id)
continue
tenant_id = resource_tenant_id(resource_type, resource)
if tenant_id and tenant_id in tenant_ids:
allowed.add(resource_id)
continue
tenant_id = resource_tenant_id(resource_type, resource)
if not tenant_id:
# Legacy rows without a tenant are never implicitly visible. Only
# a direct user ACL can expose one to its explicitly named user;
# role ACLs remain blocked until the row is tenant-migrated.
for entry in acl_by_resource.get(resource_id, []):
expires_at = entry.get("expires_at")
if expires_at:
try:
from datetime import datetime, timezone
if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
continue
except ValueError:
continue
if (
entry.get("principal_type") == "user"
and entry.get("principal_id") == user_id
and _permission_covers(entry.get("permission"), "read")
):
allowed.add(resource_id)
break
continue
if tenant_id not in tenant_ids:
continue
if tenant_roles.get(tenant_id) in {"owner", "admin"}:
allowed.add(resource_id)
continue
owner = resource.get("created_by")
if resource_type in {"eval", "fine-tune", "fine_tune_task", "compare", "inference"} and resource.get("payload"):
try:
payload = json.loads(resource["payload"]) if isinstance(resource["payload"], str) else resource["payload"]
if isinstance(payload, dict) and payload.get("created_by"):
owner = payload["created_by"]
except (TypeError, ValueError, json.JSONDecodeError):
pass
if owner == user_id:
allowed.add(resource_id)
continue
role = tenant_roles.get(tenant_id) or str(user.get("role") or "")
for entry in acl_by_resource.get(resource_id, []):
expires_at = entry.get("expires_at")
if expires_at:
try:
import json
owner = json.loads(owner or "{}").get("created_by")
except (TypeError, ValueError):
owner = None
if owner == user["id"]:
accessible.add(row["id"])
return accessible
from datetime import datetime, timezone
if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
continue
except ValueError:
continue
user_match = entry.get("principal_type") == "user" and entry.get("principal_id") == user_id
role_match = entry.get("principal_type") == "role" and entry.get("principal_id") == role
if (user_match or role_match) and _permission_covers(entry.get("permission"), "read"):
allowed.add(resource_id)
break
return allowed

1
backend/app/core/config.py
View File

@@ -71,6 +71,7 @@ class Settings:
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
# MinIO round trips. Larger files remain the shared canonical objects.
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
minio_presign_max_bytes: int = _int_env("MINIO_PRESIGN_MAX_BYTES", 1024 * 1024 * 1024 * 1024)
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")

35
backend/app/core/logging.py
View File

@@ -17,6 +17,18 @@ from fastapi import FastAPI, Request
from app.core.config import Settings, get_settings
request_id_var: ContextVar[str] = ContextVar("request_id", default="-")
client_ip_var: ContextVar[str] = ContextVar("client_ip", default="")
def get_client_ip(request: Request | None) -> str:
"""获取客户端地址，兼容前置反向代理传递的真实地址。"""
if request is None:
return ""
for header in ("X-Real-IP", "X-Forwarded-For"):
value = request.headers.get(header, "")
if value:
return value.split(",", 1)[0].strip()
return request.client.host if request.client else ""
# ==================== 敏感数据脱敏规则 ====================
@@ -65,14 +77,21 @@ def mask_sensitive_string(text: str) -> str:
"""从文本中脱敏常见敏感信息"""
if not text:
return text
# Mask the value as well as the key. Replacing only ``api_key=`` would
# still leak the credential in audit messages and exception text.
assignment_pattern = (
r"(Bearer\s+|(?:api[-_]?key|access[_-]?token|refresh[_-]?token|"
r"secret[_-]?key|password|private[_-]?key|token)\s*[:=]\s*)"
r"(\"[^\"]*\"|'[^']*'|[^\s,;]+)"
)
try:
text = re.sub(assignment_pattern, r"\1***", text, flags=re.IGNORECASE)
except re.error:
pass
patterns = [
(r'Bearer\s+[A-Za-z0-9\-._]+', '***'),
(r'token\s*[:=]\s*', '***'),
(r'password\s*[:=]\s*', '***'),
(r'secret[_-]?key\s*[:=]', '***'),
(r'api[-_]?key\s*[:=]', '***'),
(r'private[_-]?key\s*[:=]', '***'),
(r'Bearer\s+[A-Za-z0-9\-._]+', 'Bearer ***'),
(r'\d{11}', r'\d{3}\*\d{4}'), # 手机号/身份证
(r'1[3-9]\d{9}', r'1\*{3}\*{4}'), # 手机号
]
@@ -335,6 +354,7 @@ def setup_request_logging(app: FastAPI) -> None:
async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def]
request_id = request.headers.get("X-Request-ID") or str(uuid4())
token = request_id_var.set(request_id)
ip_token = client_ip_var.set(get_client_ip(request))
started_at = time.perf_counter()
try:
response = await call_next(request)
@@ -411,6 +431,7 @@ def setup_request_logging(app: FastAPI) -> None:
raise
finally:
request_id_var.reset(token)
client_ip_var.reset(ip_token)
# ==================== 配置函数 ====================

4
backend/app/core/op_log.py
View File

@@ -31,7 +31,7 @@ from typing import Any, Callable, Optional, TypeVar
from fastapi import Request
from app.core.logging import get_logger, request_id_var
from app.core.logging import get_client_ip, get_logger, request_id_var
from app.db.platform_store import get_platform_store, new_id, utcnow
logger = get_logger("app.op_log")
@@ -351,7 +351,7 @@ def _write_log(
req_method = None
req_path = None
if request:
client_ip = request.client.host if request.client else None
client_ip = get_client_ip(request) or None
req_method = request.method
req_path = request.url.path

1831
backend/app/db/platform_store.py
View File

File diff suppressed because it is too large Load Diff

335
backend/app/db/sql/000_full_init.sql
View File

@@ -4,7 +4,7 @@
-- 用途：切换到新的 PG 数据集时，一次性创建平台运行所需的全部数据库对象与
-- 基础种子数据（幂等，可重复执行）。
--
-- 覆盖范围（与运行时代码实际使用的表一致）：
-- 覆盖范围（与运行时代码实际使用的表一致，离线新库只需执行本文件）：
-- 001_platform_runtime.sql 平台核心表
-- 002_governance.sql 治理表（租户 / 审批 / 审计 / 留存）
-- 003_tenant_quota.sql 租户配额列
@@ -16,10 +16,9 @@
-- 说明：
-- * 本脚本通过 psql 执行，包含 DO $$ ... $$ 块与事务，不能用应用的
-- executescript()（按分号切分）执行。
-- * 应用启动时 PlatformStore.ensure_schema() 只会自动执行
-- 001 / 002_governance / 003_tenant_quota；数据处理表需另跑
-- 002_data_process.sql（本脚本已包含）。应用首次启动还会自动补充
-- admin/operator 种子用户（本脚本已包含，二选一即可）。
-- * 本文件已经合并平台运行、治理、数据处理、权限、MinIO、GPU 预留、
-- 缓存治理和数据转换等全部初始化对象；全新数据库无需再执行其他 SQL。
-- 应用启动时的 ensure_schema() 仅作为兼容兜底，不是离线初始化前置条件。
-- * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS，
-- 可在已初始化的库上安全重复执行。
--
@@ -45,12 +44,20 @@ CREATE TABLE IF NOT EXISTS users (
password_hash TEXT NOT NULL,
display_name TEXT NOT NULL,
role TEXT NOT NULL,
platform_role TEXT NOT NULL DEFAULT 'platform_user',
status TEXT NOT NULL,
permissions TEXT NOT NULL,
create_time TEXT NOT NULL,
last_login TEXT,
protected INTEGER NOT NULL DEFAULT 0
protected INTEGER NOT NULL DEFAULT 0,
tenant_id TEXT NOT NULL DEFAULT 'default',
deleted_at TEXT,
deleted_by TEXT
);
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at);
CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at);
CREATE TABLE IF NOT EXISTS models (
id TEXT PRIMARY KEY,
@@ -68,6 +75,9 @@ CREATE TABLE IF NOT EXISTS models (
created_by TEXT,
tenant_id TEXT,
project_id TEXT,
storage_status TEXT NOT NULL DEFAULT 'pending',
storage_error TEXT,
storage_version_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
@@ -131,6 +141,10 @@ CREATE TABLE IF NOT EXISTS model_export_jobs (
create_time TEXT NOT NULL,
completed_at TEXT
);
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending';
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT;
CREATE TABLE IF NOT EXISTS datasets (
id TEXT PRIMARY KEY,
@@ -209,8 +223,16 @@ CREATE TABLE IF NOT EXISTS fine_tune_tasks (
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
gpus TEXT NOT NULL,
sync_job_id TEXT,
compute_job_id TEXT
compute_job_id TEXT,
tenant_id TEXT NOT NULL DEFAULT 'default',
created_by TEXT,
deleted_at TEXT,
deleted_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active
ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator
ON fine_tune_tasks(created_by, deleted_at, create_time DESC);
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
id TEXT PRIMARY KEY,
@@ -273,6 +295,8 @@ CREATE TABLE IF NOT EXISTS resource_replicas (
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
version_id TEXT,
storage_object_id TEXT,
local_path TEXT NOT NULL,
status TEXT NOT NULL,
sync_status TEXT NOT NULL,
@@ -310,6 +334,11 @@ CREATE TABLE IF NOT EXISTS storage_objects (
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
);
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
id TEXT PRIMARY KEY,
@@ -323,6 +352,23 @@ CREATE TABLE IF NOT EXISTS storage_cache_jobs (
create_time TEXT NOT NULL,
completed_at TEXT
);
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT;
CREATE TABLE IF NOT EXISTS storage_cleanup_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
action TEXT NOT NULL DEFAULT 'delete',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
@@ -332,8 +378,16 @@ CREATE TABLE IF NOT EXISTS eval_tasks (
name TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE TABLE IF NOT EXISTS eval_dimensions (
id TEXT PRIMARY KEY,
@@ -349,8 +403,17 @@ CREATE TABLE IF NOT EXISTS compare_tasks (
name TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL,
create_time TEXT NOT NULL
create_time TEXT NOT NULL,
created_by TEXT,
tenant_id TEXT,
deleted_at TEXT,
deleted_by TEXT
);
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at);
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
@@ -368,6 +431,24 @@ CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status);
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running');
-- 评测/推理等非训练任务的统一 GPU 原子预留。训练任务继续使用 gpu_allocations。
CREATE TABLE IF NOT EXISTS gpu_reservations (
id TEXT PRIMARY KEY,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
gpu_index INTEGER NOT NULL,
owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')),
owner_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')),
create_time TEXT NOT NULL,
released_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner
ON gpu_reservations(owner_type, owner_id, status);
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status
ON gpu_reservations(node_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active
ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved';
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
@@ -393,6 +474,9 @@ CREATE TABLE IF NOT EXISTS projects (
create_by TEXT,
updated_at TEXT
);
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at);
CREATE TABLE IF NOT EXISTS project_members (
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
@@ -433,6 +517,15 @@ CREATE TABLE IF NOT EXISTS acls (
);
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_status TEXT NOT NULL DEFAULT 'pending';
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_error TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_version_id TEXT;
UPDATE models SET storage_status = CASE
WHEN model_source IN ('api', 'online') THEN 'not_applicable'
WHEN storage_status IS NULL OR storage_status = '' THEN 'pending'
ELSE storage_status
END
WHERE storage_status IS NULL OR storage_status = '' OR model_source IN ('api', 'online');
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
@@ -462,6 +555,7 @@ ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
-- ============================================================================
-- 二、治理表（来源：002_governance.sql）
@@ -477,6 +571,15 @@ CREATE TABLE IF NOT EXISTS tenants (
retention_policy_id TEXT,
create_time TEXT
);
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at);
INSERT INTO tenants (id, name, code, status, quota, create_time)
VALUES ('default', '默认租户', 'default', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
ON CONFLICT (id) DO NOTHING;
INSERT INTO tenants (id, name, code, status, quota, create_time)
VALUES ('admin', '管理员租户', 'admin', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
ON CONFLICT (id) DO UPDATE SET name='管理员租户', code='admin', status='active', deleted_at=NULL, deleted_by=NULL;
CREATE TABLE IF NOT EXISTS approval_templates (
id TEXT PRIMARY KEY,
@@ -534,6 +637,34 @@ CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
-- ---- 操作日志（接口操作审计） ----
CREATE TABLE IF NOT EXISTS operation_logs (
id TEXT PRIMARY KEY,
user_id TEXT,
username TEXT,
module TEXT,
action TEXT,
target_type TEXT,
target_id TEXT,
target_name TEXT,
status TEXT NOT NULL,
error_message TEXT,
error_type TEXT,
error_traceback TEXT,
func_name TEXT,
detail TEXT,
client_ip TEXT,
request_method TEXT,
request_path TEXT,
trace_id TEXT,
duration_ms REAL,
create_time TEXT
);
CREATE INDEX IF NOT EXISTS idx_operation_logs_user_time ON operation_logs(user_id, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_operation_logs_module_time ON operation_logs(module, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_operation_logs_status ON operation_logs(status, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_operation_logs_create_time ON operation_logs(create_time DESC);
CREATE TABLE IF NOT EXISTS retention_policies (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -876,29 +1007,201 @@ CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_ta
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_by TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_at TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at);
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT;
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_002 ON resource_replicas(storage_object_id, node_id);
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员
-- 六、权限 2.0：租户成员、资源申请、ACL 生命周期和审批动作
-- ============================================================================
CREATE TABLE IF NOT EXISTS tenant_members (
tenant_id TEXT NOT NULL,
user_id TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'member',
status TEXT NOT NULL DEFAULT 'active',
invited_by TEXT,
joined_at TEXT,
expires_at TEXT,
PRIMARY KEY (tenant_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
SELECT COALESCE(u.tenant_id, 'default'), u.id,
CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END,
'active', COALESCE(u.create_time, NOW()::text)
FROM users u
ON CONFLICT (tenant_id, user_id) DO NOTHING;
-- GPU/存储任务使用的租户配额原子预留账本。任务提交时预留，终态或故障回收。
CREATE TABLE IF NOT EXISTS tenant_quota_reservations (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
owner_type TEXT NOT NULL,
owner_id TEXT NOT NULL,
gpu_count INTEGER NOT NULL DEFAULT 0,
storage_bytes BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'reserved',
create_time TEXT NOT NULL,
released_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant
ON tenant_quota_reservations(tenant_id, status);
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner
ON tenant_quota_reservations(owner_type, owner_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner
ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved';
CREATE TABLE IF NOT EXISTS resource_access_requests (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
applicant_id TEXT NOT NULL,
principal_type TEXT NOT NULL DEFAULT 'user',
principal_id TEXT NOT NULL,
requested_permissions TEXT NOT NULL DEFAULT '[]',
reason TEXT,
approval_id TEXT,
status TEXT NOT NULL DEFAULT 'pending',
expires_at TEXT,
created_at TEXT NOT NULL,
decided_at TEXT,
decided_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource
ON resource_access_requests(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant
ON resource_access_requests(applicant_id, status);
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at);
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';
-- ============================================================================
-- 七、权限 2.0 完整闭环：审批策略、执行状态、结构化审计与历史归属
-- ============================================================================
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant';
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active';
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT;
CREATE INDEX IF NOT EXISTS idx_approval_templates_match
ON approval_templates(tenant_id, action, resource_type, status);
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending';
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT;
CREATE INDEX IF NOT EXISTS idx_approval_instances_execution
ON approval_instances(status, execution_status, action, resource_type, resource_id);
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT;
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT;
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry
ON resource_access_requests(status, expires_at);
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success';
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT;
CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id);
CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time);
UPDATE datasets d
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = '');
UPDATE models m
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
UPDATE trained_models m
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
UPDATE eval_tasks e
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = '');
-- ============================================================================
-- 八、种子数据：初始管理员 / 操作员
-- 应用首次启动（ensure_seed_data）也会自动创建；此处提供以便脱离应用直接初始化。
-- 密码：admin / admin123，operator / operator123（上线前请改密）。
-- ============================================================================
INSERT INTO users
(id, username, password_hash, display_name, role, status, permissions, create_time, protected)
(id, username, password_hash, display_name, role, platform_role, status, permissions, create_time, protected, tenant_id)
VALUES
(
'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd',
'Platform Admin', 'admin', 'active',
'Admin', 'admin', 'platform_admin', 'active',
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]',
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1, 'admin'
),
(
'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125',
'Platform Operator', 'operator', 'active',
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]',
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0
'Platform Operator', 'operator', 'platform_user', 'active',
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","hardware","logs"]',
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0, 'default'
)
ON CONFLICT (username) DO NOTHING;
UPDATE users SET display_name='Admin', tenant_id='admin' WHERE username='admin';
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
VALUES
('admin', 'u_admin', 'owner', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
('default', 'u_operator', 'member', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
ON CONFLICT (tenant_id, user_id) DO NOTHING;
UPDATE tenant_members SET status='disabled' WHERE tenant_id='default' AND user_id='u_admin';
-- Canonical hierarchy fields for new and historical rows. Project columns are
-- intentionally retained only as compatibility data and are not used here.
UPDATE users
SET platform_role = CASE WHEN role = 'admin' OR COALESCE(protected, 0) = 1
THEN 'platform_admin' ELSE 'platform_user' END
WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user');
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text)
FROM tenants t
JOIN users u ON u.id = t.owner_user_id
WHERE t.owner_user_id IS NOT NULL AND COALESCE(t.status, 'active') = 'active'
ON CONFLICT (tenant_id, user_id) DO UPDATE SET role='owner', status='active';
UPDATE tenant_members tm
SET role='member'
FROM users u
WHERE tm.user_id=u.id AND tm.role='owner'
AND (u.role <> 'admin' AND COALESCE(u.protected, 0)=0)
AND NOT EXISTS (SELECT 1 FROM tenants t WHERE t.id=tm.tenant_id AND t.owner_user_id=tm.user_id);
CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC);
COMMIT;

32
backend/app/db/sql/005_storage_progress_migration.sql Normal file
View File

@@ -0,0 +1,32 @@
-- YG Fine-Tune Platform additive migration: MinIO completion and cache manifest.
-- Execute with psql against an existing database after taking a schema backup.
-- The statements are idempotent and are also included in 000_full_init.sql.
BEGIN;
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT;
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS project_id TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_005
ON resource_replicas(storage_object_id, node_id);
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource_005
ON storage_objects(resource_type, resource_id, version_id, status);
CREATE INDEX IF NOT EXISTS idx_eval_tasks_active_005
ON eval_tasks(create_time DESC) WHERE deleted_at IS NULL;
COMMIT;

21
backend/app/db/sql/006_gpu_reservations.sql Normal file
View File

@@ -0,0 +1,21 @@
-- 原子 GPU 预留：评测和推理与训练统一纳入调度占用模型。
BEGIN;
CREATE TABLE IF NOT EXISTS gpu_reservations (
id TEXT PRIMARY KEY,
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
gpu_index INTEGER NOT NULL,
owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')),
owner_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')),
create_time TEXT NOT NULL,
released_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner
ON gpu_reservations(owner_type, owner_id, status);
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status
ON gpu_reservations(node_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active
ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved';
COMMIT;

33
backend/app/db/sql/007_platform_completion.sql Normal file
View File

@@ -0,0 +1,33 @@
-- 平台可靠性闭环：导出权限、对象清理、缓存 manifest 元数据。
BEGIN;
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending';
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT;
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT;
CREATE TABLE IF NOT EXISTS storage_cleanup_jobs (
id TEXT PRIMARY KEY,
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
action TEXT NOT NULL DEFAULT 'delete',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
create_time TEXT NOT NULL,
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time);
COMMIT;

53
backend/app/db/sql/008_permission_v2.sql Normal file
View File

@@ -0,0 +1,53 @@
-- 权限 2.0：租户成员、资源申请、ACL 生命周期和审批动作
CREATE TABLE IF NOT EXISTS tenant_members (
tenant_id TEXT NOT NULL,
user_id TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'member',
status TEXT NOT NULL DEFAULT 'active',
invited_by TEXT,
joined_at TEXT,
expires_at TEXT,
PRIMARY KEY (tenant_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
CREATE TABLE IF NOT EXISTS resource_access_requests (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
applicant_id TEXT NOT NULL,
principal_type TEXT NOT NULL DEFAULT 'user',
principal_id TEXT NOT NULL,
requested_permissions TEXT NOT NULL DEFAULT '[]',
reason TEXT,
approval_id TEXT,
status TEXT NOT NULL DEFAULT 'pending',
expires_at TEXT,
created_at TEXT NOT NULL,
decided_at TEXT,
decided_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource
ON resource_access_requests(resource_type, resource_id, status);
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant
ON resource_access_requests(applicant_id, status);
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at);
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';

116
backend/app/db/sql/009_permission_completion.sql Normal file
View File

@@ -0,0 +1,116 @@
-- 权限 2.0 完整闭环：审批策略、执行状态、结构化审计与历史租户归属
-- 该迁移可独立执行：兼容仅执行过 000_full_init.sql、或未执行 008 的旧数据库。
CREATE TABLE IF NOT EXISTS tenant_members (
tenant_id TEXT NOT NULL,
user_id TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'member',
status TEXT NOT NULL DEFAULT 'active',
invited_by TEXT,
joined_at TEXT,
expires_at TEXT,
PRIMARY KEY (tenant_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
SELECT COALESCE(u.tenant_id, 'default'), u.id,
CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END,
'active', COALESCE(u.create_time, NOW()::text)
FROM users u
ON CONFLICT (tenant_id, user_id) DO NOTHING;
CREATE TABLE IF NOT EXISTS resource_access_requests (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL DEFAULT 'default',
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
applicant_id TEXT NOT NULL,
principal_type TEXT NOT NULL DEFAULT 'user',
principal_id TEXT NOT NULL,
requested_permissions TEXT NOT NULL DEFAULT '[]',
reason TEXT,
approval_id TEXT,
status TEXT NOT NULL DEFAULT 'pending',
expires_at TEXT,
created_at TEXT NOT NULL DEFAULT to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
decided_at TEXT,
decided_by TEXT
);
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE models ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant';
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active';
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT;
CREATE INDEX IF NOT EXISTS idx_approval_templates_match
ON approval_templates(tenant_id, action, resource_type, status);
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending';
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT;
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT;
CREATE INDEX IF NOT EXISTS idx_approval_instances_execution
ON approval_instances(status, execution_status, action, resource_type, resource_id);
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT;
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT;
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry
ON resource_access_requests(status, expires_at);
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at);
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT;
CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at);
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success';
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT;
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT;
CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id);
CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time);
-- 历史资源按创建者租户补齐归属。无法识别的资源保留 default，后续由管理员复核。
UPDATE datasets d
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = '');
UPDATE models m
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
UPDATE trained_models m
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
UPDATE eval_tasks e
SET tenant_id = COALESCE(u.tenant_id, 'default')
FROM users u
WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = '');

21
backend/app/db/sql/010_permission_quota_membership.sql Normal file
View File

@@ -0,0 +1,21 @@
-- Permission 2.0: atomic tenant quota reservations for GPU-backed tasks.
-- This migration is additive and safe to run repeatedly.
CREATE TABLE IF NOT EXISTS tenant_quota_reservations (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
owner_type TEXT NOT NULL,
owner_id TEXT NOT NULL,
gpu_count INTEGER NOT NULL DEFAULT 0,
storage_bytes BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'reserved',
create_time TEXT NOT NULL,
released_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant
ON tenant_quota_reservations(tenant_id, status);
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner
ON tenant_quota_reservations(owner_type, owner_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner
ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved';

20
backend/app/db/sql/011_permission_lifecycle.sql Normal file
View File

@@ -0,0 +1,20 @@
-- Permission 2.0: user and inference task lifecycle tombstones.
-- Additive migration, safe to execute repeatedly.
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at);

63
backend/app/db/sql/012_tenant_user_hierarchy.sql Normal file
View File

@@ -0,0 +1,63 @@
-- Tenant/user hierarchy alignment. Additive and safe to run repeatedly.
-- New business flows use tenant_members, platform_role, tenant_id and
-- created_by. Legacy role/project fields remain for compatibility only.
ALTER TABLE users ADD COLUMN IF NOT EXISTS platform_role TEXT NOT NULL DEFAULT 'platform_user';
UPDATE users
SET platform_role = CASE
WHEN role = 'admin' OR COALESCE(protected, 0) = 1 THEN 'platform_admin'
ELSE 'platform_user'
END
WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user');
CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at);
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
UPDATE fine_tune_tasks
SET tenant_id = COALESCE(NULLIF(tenant_id, ''), payload::json->>'tenant_id', 'default'),
created_by = COALESCE(NULLIF(created_by, ''), payload::json->>'created_by')
WHERE tenant_id IS NULL OR tenant_id = '' OR created_by IS NULL OR created_by = '';
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active
ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator
ON fine_tune_tasks(created_by, deleted_at, create_time DESC);
INSERT INTO tenants (id, name, code, status, quota, create_time)
VALUES ('default', '默认租户', 'default', 'active', '{}', NOW()::text)
ON CONFLICT (id) DO UPDATE
SET status = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN 'active' ELSE tenants.status END,
deleted_at = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_at END,
deleted_by = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_by END;
-- Make the tenant owner relationship explicit and repair older tenant rows.
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text)
FROM tenants t
JOIN users u ON u.id = t.owner_user_id
WHERE t.owner_user_id IS NOT NULL
AND COALESCE(t.status, 'active') = 'active'
ON CONFLICT (tenant_id, user_id) DO UPDATE
SET role = 'owner', status = 'active';
-- Existing account creation used admin as a tenant owner. Platform role and
-- tenant role are separate, so keep only explicit tenant owners as owners.
UPDATE tenant_members tm
SET role = 'member'
FROM users u
WHERE tm.user_id = u.id
AND tm.role = 'owner'
AND (u.role <> 'admin' AND COALESCE(u.protected, 0) = 0)
AND NOT EXISTS (
SELECT 1 FROM tenants t
WHERE t.id = tm.tenant_id AND t.owner_user_id = tm.user_id
);
-- Project is no longer a business isolation boundary. Keep historical columns
-- readable, but make tenant-scoped queries the only supported new path.
CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC);

324
backend/app/modules/approval/router.py
View File

@@ -1,17 +1,105 @@
from __future__ import annotations
import json
from fastapi import APIRouter, Body, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, is_admin
from app.core.auth import (
get_current_user,
has_resource_access,
is_tenant_admin,
is_admin,
resource_in_user_tenant,
resource_record,
)
router = APIRouter(prefix="/approvals", tags=["approval"])
# Approval actions are deliberately finite. A caller must not be able to
# create an arbitrary approval that no executor or audit policy understands.
ALLOWED_APPROVAL_ACTIONS = {
"resource.access",
"gpu.assign",
"tenant.quota.update",
"tenant.member.add",
"tenant.member.remove",
"model.use",
"dataset.use",
"dataset.delete",
"trained_model.merge",
"trained_model.export",
"trained_model.delete",
"fine_tune.stop",
"fine_tune.delete",
"eval.delete",
"inference.delete",
"project.archive",
"project.delete",
}
def _available_gpu_options() -> list[dict[str, Any]]:
"""Return only online nodes and currently unassigned, idle GPUs."""
store = get_platform_store()
nodes = store.compute_nodes()
gpus = store.gpus()
assigned = {(str(item.get("node_id")), int(item.get("gpu_index"))) for item in store.gpu_assignments()}
by_node: dict[str, list[dict[str, Any]]] = {}
for gpu in gpus:
node_id = str(gpu.get("node_id") or "")
index = int(gpu.get("id") or 0)
if gpu.get("status") != "idle" or (node_id, index) in assigned:
continue
by_node.setdefault(node_id, []).append({
"index": index,
"name": gpu.get("name") or "GPU",
"memory_total_gb": float(gpu.get("memory_total_gb") or 0),
})
result = []
for node in nodes:
node_id = str(node.get("id") or "")
if not node.get("enabled") or node.get("scheduler_status") != "online" or not by_node.get(node_id):
continue
result.append({
"id": node_id,
"code": node.get("code") or node_id,
"name": node.get("name") or node.get("code") or node_id,
"gpus": sorted(by_node[node_id], key=lambda item: item["index"]),
})
return result
def _validate_gpu_request(assignments: Any) -> list[dict[str, Any]]:
if not isinstance(assignments, list) or not assignments:
raise fail(400, "assignments 不能为空")
available = {
(node["id"], gpu["index"])
for node in _available_gpu_options()
for gpu in node["gpus"]
}
normalized = []
seen: set[tuple[str, int]] = set()
for item in assignments:
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
raise fail(400, "每项必须包含 node_id 和 gpu_index")
try:
key = (str(item["node_id"]), int(item["gpu_index"]))
except (TypeError, ValueError):
raise fail(400, "gpu_index 必须是整数")
if key not in available:
raise fail(409, f"GPU {key[0]}:{key[1]} 当前不可申请")
if key not in seen:
seen.add(key)
normalized.append({"node_id": key[0], "gpu_index": key[1]})
return normalized
@router.get("/templates")
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().approval_templates())
@@ -20,11 +108,31 @@ def create_template(payload: dict[str, Any] = Body(...), current_user: dict = De
if not is_admin(current_user): raise fail(403, "admin permission required")
if not payload.get("name"):
raise fail(400, "name 必填")
return ok(get_platform_store().create_approval_template(payload))
steps = payload.get("steps") or []
if not isinstance(steps, list) or any(not isinstance(step, dict) for step in steps):
raise fail(400, "steps 格式无效")
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
payload = {
**payload,
"created_by": current_user.get("id"),
"tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default",
"scope": payload.get("scope") or "tenant",
"status": payload.get("status") or "active",
}
template = get_platform_store().create_approval_template(payload)
get_platform_store().record_audit(
action="approval.template.create", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template["id"],
tenant_id=template.get("tenant_id"),
)
return ok(template)
@router.get("/templates/{template_id}")
def get_template(template_id: str) -> dict[str, Any]:
def get_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try:
return ok(get_platform_store().approval_template(template_id))
except KeyError:
@@ -34,8 +142,16 @@ def get_template(template_id: str) -> dict[str, Any]:
@router.put("/templates/{template_id}")
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
try:
return ok(get_platform_store().update_approval_template(template_id, payload))
template = get_platform_store().update_approval_template(template_id, payload)
get_platform_store().record_audit(
action="approval.template.update", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template_id,
tenant_id=template.get("tenant_id"), detail=f"fields={','.join(payload.keys())}",
)
return ok(template)
except KeyError:
raise fail(404, "template not found")
@@ -44,15 +160,77 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...), curre
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try:
return ok(get_platform_store().delete_approval_template(template_id))
template = get_platform_store().delete_approval_template(template_id)
get_platform_store().record_audit(
action="approval.template.delete", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template_id,
tenant_id=template.get("tenant_id"),
)
return ok(template)
except KeyError:
raise fail(404, "template not found")
@router.get("")
def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
items = get_platform_store().approval_instances(status=status)
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
def list_instances(
status: str | None = None,
mine: bool = False,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
user_id = current_user.get("id")
items = get_platform_store().approval_instances(
status=status,
applicant_id=user_id if mine and not is_admin(current_user) else None,
)
if is_admin(current_user):
return ok(items)
if mine:
return ok(items)
visible = []
for item in items:
if item.get("applicant_id") == user_id:
visible.append(item)
continue
if any(step.get("approver_id") == user_id and step.get("status") == "pending" for step in item.get("steps", [])):
visible.append(item)
continue
if is_tenant_admin(current_user, item.get("tenant_id")) and item.get("status") == "pending":
visible.append(item)
return ok(visible)
@router.get("/gpu-options")
def gpu_request_options(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Self-service GPU options; does not expose the admin compute page."""
return ok({"nodes": _available_gpu_options()})
@router.post("/gpu-requests")
def create_gpu_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
assignments = _validate_gpu_request(payload.get("assignments"))
user_id = str(current_user.get("id") or "")
if is_admin(current_user):
try:
return ok({"approval_required": False, "assignments": get_platform_store().assign_gpus(
[{**item, "user_id": user_id} for item in assignments], assigned_by=user_id,
)})
except ValueError as exc:
raise fail(409, str(exc))
normalized = [{**item, "user_id": user_id} for item in assignments]
instance = get_platform_store().create_approval_instance({
"resource_type": "gpu",
"resource_id": f"batch:{user_id}",
"applicant_id": user_id,
"action": "gpu.assign",
"tenant_id": current_user.get("tenant_id") or "default",
"reason": json.dumps({"assignments": normalized, "reason": payload.get("reason")}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
detail=f"count={len(normalized)}",
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.post("")
@@ -61,6 +239,24 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
for field in ("resource_type", "resource_id"):
if not payload.get(field):
raise fail(400, f"{field} 必填")
resource = resource_record(str(payload["resource_type"]), str(payload["resource_id"]))
if not resource and not is_admin(current_user):
raise fail(404, "resource not found")
action = str(payload.get("action") or "")
if not action or action not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
if action != "resource.access" and not is_admin(current_user) and not has_resource_access(
str(payload["resource_type"]), str(payload["resource_id"]), current_user, "read"
):
raise fail(403, "no permission to request approval for this resource")
if resource and not is_admin(current_user) and not resource_in_user_tenant(str(payload["resource_type"]), resource, current_user):
raise fail(403, "resource belongs to another tenant")
requested = payload.get("requested_permissions") or []
allowed = {"read", "write", "execute", "download"}
if any(permission not in allowed for permission in requested):
raise fail(400, "invalid requested permission")
payload["tenant_id"] = current_user.get("tenant_id") or "default"
payload["requested_permissions"] = requested
try:
return ok(get_platform_store().create_approval_instance(payload))
except KeyError:
@@ -71,7 +267,7 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
item = get_platform_store().approval_instance(instance_id)
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id") and not is_tenant_admin(current_user, item.get("tenant_id")):
raise fail(403, "no permission to access approval")
return ok(item)
except KeyError:
@@ -83,18 +279,118 @@ def decide(
instance_id: str,
step_index: int,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not payload.get("approver_id"):
raise fail(400, "approver_id 必填")
try:
return ok(
get_platform_store().decide_approval_step(
instance = get_platform_store().approval_instance(instance_id)
if instance.get("applicant_id") == current_user.get("id"):
raise fail(403, "applicant cannot approve own request")
step = next((item for item in instance.get("steps", []) if int(item.get("step_index", -1)) == step_index), None)
if not step and is_admin(current_user) and not instance.get("steps"):
step = {"approver_id": None, "status": "pending"}
if not step:
raise fail(404, "approval step not found")
designated = step.get("approver_id")
if not is_admin(current_user) and designated != current_user.get("id") and not (
step.get("approver_type") == "admin" and is_tenant_admin(current_user, instance.get("tenant_id"))
):
raise fail(403, "current user is not the designated approver")
if instance.get("action") == "tenant.quota.update" and not is_admin(current_user):
raise fail(403, "only platform administrator can approve tenant quota changes")
submitted_approver = payload.get("approver_id")
if submitted_approver and submitted_approver != current_user.get("id"):
raise fail(403, "approver_id must match current session")
store = get_platform_store()
result = store.decide_approval_step(
instance_id,
step_index,
approver_id=payload["approver_id"],
approver_id=str(current_user.get("id")),
approved=bool(payload.get("approved", False)),
comment=payload.get("comment"),
)
if result.get("status") == "approved" and result.get("execution_status") == "ready":
try:
effect = store.apply_approval_effect(result, str(current_user.get("id") or ""))
if effect is not None:
result = store.approval_instance(instance_id)
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE approval_instances SET execution_status='failed', execution_error=? WHERE id=?",
(str(exc), instance_id),
)
raise fail(409, f"审批已通过，但执行失败：{exc}")
store.record_audit(
action="approval.decision", actor_id=current_user.get("id"),
target_type="approval_instance", target_id=instance_id,
tenant_id=result.get("tenant_id"), result="success" if payload.get("approved") else "rejected",
detail=f"step={step_index}", reason=payload.get("comment"),
)
return ok(result)
except (KeyError, ValueError) as e:
raise fail(400, str(e))
@router.post("/resource-access/requests")
def create_resource_access_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
resource_type = str(payload.get("resource_type") or "")
resource_id = str(payload.get("resource_id") or "")
resource = resource_record(resource_type, resource_id)
if not resource:
raise fail(404, "resource not found")
if not is_admin(current_user) and not resource_in_user_tenant(resource_type, resource, current_user):
raise fail(403, "resource belongs to another tenant")
permissions = payload.get("requested_permissions") or ["read"]
allowed = {"read", "write", "execute", "download"}
if not permissions or any(permission not in allowed for permission in permissions):
raise fail(400, "invalid requested permissions")
result = get_platform_store().create_resource_access_request({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"principal_type": "user",
"principal_id": current_user.get("id"),
"requested_permissions": permissions,
"reason": payload.get("reason"),
"template_id": payload.get("template_id"),
"tenant_id": current_user.get("tenant_id") or "default",
"expires_at": payload.get("expires_at"),
})
get_platform_store().record_audit(
action="resource.access.request", actor_id=current_user.get("id"),
target_type=resource_type, target_id=resource_id,
tenant_id=current_user.get("tenant_id") or "default",
detail=f"permissions={','.join(permissions)}",
)
return ok(result)
@router.get("/resource-access/requests")
def list_resource_access_requests(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
items = get_platform_store().resource_access_requests(
user_id=None if is_admin(current_user) else current_user.get("id"),
status=status,
)
return ok(items)
@router.post("/resource-access/requests/{request_id}/cancel")
def cancel_resource_access_request(request_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
item = get_platform_store().cancel_resource_access_request(
request_id,
str(current_user.get("id") or ""),
is_admin_actor=is_admin(current_user),
)
except KeyError:
raise fail(404, "access request not found")
except PermissionError as exc:
raise fail(403, str(exc))
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="resource.access.cancel", actor_id=current_user.get("id"),
target_type="resource_access_request", target_id=request_id,
tenant_id=item.get("tenant_id"),
)
return ok(item)

182
backend/app/modules/compute_gateway/sync.py
View File

@@ -14,6 +14,18 @@ from app.modules.storage.minio_store import get_object_storage
MAX_STARTING_ATTEMPTS = 40
def _extract_job_failure_reason(log_text: str, limit: int = 2000) -> str:
"""Return a concise actionable reason from a failed Compute job log."""
lines = [line.strip() for line in str(log_text or "").splitlines() if line.strip()]
if not lines:
return ""
markers = ("[eval] FAILED", "Traceback", "RuntimeError", "Error:", "ERROR")
for index in range(len(lines) - 1, -1, -1):
if any(marker in lines[index] for marker in markers):
return "\n".join(lines[index : index + 8])[-limit:]
return "\n".join(lines[-8:])[-limit:]
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
@@ -27,12 +39,15 @@ async def _archive_node_directory(
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
if source == data_root:
raise RuntimeError("refuse to archive compute data root; output_dir must be a task subdirectory")
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
max_files = 10000
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
@@ -49,6 +64,8 @@ async def _archive_node_directory(
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
if len(archived) >= max_files:
raise RuntimeError(f"archive file count exceeds limit {max_files}")
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
@@ -115,11 +132,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = "compute node deleted"
store.mark_inference_unloaded(item.get("node_id") or "")
store.release_external_gpus("inference", str(task["id"]), item.get("node_id"))
continue
if not node.get("enabled") or node.get("scheduler_status") != "online":
item["status"] = "error"
item["error"] = "compute node offline"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
continue
try:
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
@@ -128,6 +147,7 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = f"compute node unreachable: {exc}"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
continue
node_status = status.get("status")
if node_status == "ready":
@@ -139,11 +159,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = status.get("error") or "model load failed on compute node"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
elif node_status == "idle":
# 节点重启导致已加载模型丢失
item["status"] = "error"
item["error"] = "model disappeared from compute node (node may have restarted)"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
# node_status == "loading" -> 保持 starting，下轮再查
if dirty:
if any(i.get("status") in {"ready", "running"} for i in items):
@@ -157,11 +179,16 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
return reconciled
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
async def fetch_eval_result_content(
client: ComputeNodeClient,
node: dict[str, Any],
job: dict[str, Any],
file_name: str = "eval_results.json",
) -> dict[str, Any] | None:
output_dir = job.get("output_dir")
if not output_dir:
return None
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
full_path = f"{str(output_dir).rstrip('/')}/{file_name}"
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
@@ -175,11 +202,36 @@ async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, A
return payload if isinstance(payload, dict) else None
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
async def fetch_eval_progress_content(
client: ComputeNodeClient,
node: dict[str, Any],
job: dict[str, Any],
) -> dict[str, Any] | None:
return await fetch_eval_result_content(client, node, job, "eval_progress.json")
async def poll_compute_jobs_once(store: Any | None = None) -> dict[str, Any]:
store = store or get_platform_store()
synced: list[dict[str, Any]] = []
failed: list[dict[str, str]] = []
for task in store.running_compute_tasks():
online_nodes = {
str(node.get("id"))
for node in store.compute_nodes()
if node.get("enabled") and node.get("scheduler_status") in {"online", "draining"}
}
training_tasks = {str(task["id"]): task for task in store.running_compute_tasks()}
# Completed tasks whose MinIO archive was interrupted remain eligible for
# reconciliation after a Backend restart or a transient node failure.
if get_settings().minio_enabled:
for task in store.tasks():
if task.get("status") != "completed" or not task.get("compute_job_id"):
continue
if (
str(task.get("archive_status") or "") != "completed"
and str(task.get("compute_node_id")) in online_nodes
):
training_tasks.setdefault(str(task["id"]), task)
for task in training_tasks.values():
node = _node_for_task(task)
if not node:
failed.append({"task_id": task["id"], "error": "compute node not found"})
@@ -215,26 +267,47 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
try:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
store.update_task(task["id"], {
"archive_status": "completed",
"archive_object_ids": [str(item["id"]) for item in archived],
"archive_error": "",
})
except Exception as archive_exc:
store.update_task(task["id"], {
"archive_status": "pending",
"archive_error": str(archive_exc)[:2000],
})
raise
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
for record in store.active_standalone_compute_jobs():
standalone_jobs = {
str(record["id"]): record
for record in store.active_standalone_compute_jobs()
if str(record.get("node_id")) in online_nodes
}
if get_settings().minio_enabled:
for record in store.standalone_compute_jobs_pending_archive():
if str(record.get("node_id")) in online_nodes:
standalone_jobs.setdefault(str(record["id"]), record)
for record in standalone_jobs.values():
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
if not node:
failed.append({"job_id": record["id"], "error": "compute node not found"})
@@ -266,12 +339,32 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
store.update_compute_job_archive(record["id"], "completed", [str(item["id"]) for item in archived])
except Exception as exc: # noqa: BLE001 - keep polling other jobs
try:
store.update_compute_job_archive(record["id"], "pending", [], str(exc)[:2000])
except Exception:
pass
failed.append({"job_id": record["id"], "error": str(exc)})
# ── Eval job sync ────────────────────────────────────────────────
eval_synced = 0
for eval_task in store.running_eval_tasks():
eval_tasks = {str(task["id"]): task for task in store.running_eval_tasks()}
if get_settings().minio_enabled:
# A completed evaluation can win the race with the poller: its status
# is persisted before the report archive finishes. Keep such tasks in
# the reconciliation set until the report object is available.
for task in store.eval_tasks():
if (
task.get("status") == "completed"
and task.get("compute_job_id")
and str(task.get("archive_status") or "") != "completed"
and str(task.get("compute_node_id")) in online_nodes
):
eval_tasks.setdefault(str(task["id"]), task)
for eval_task in eval_tasks.values():
if str(eval_task.get("compute_node_id")) not in online_nodes:
continue
node = next(
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
None,
@@ -283,15 +376,39 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
client = ComputeNodeClient(node["api_base_url"])
job = await client.get_job(eval_task["compute_job_id"])
result_content = None
# Try to read eval_results.json from the job output directory
# Read live progress and partial results while the evaluator is running.
if job.get("status") in {"queued", "running"} and job.get("output_dir"):
try:
progress_content = await fetch_eval_progress_content(client, node, job)
if progress_content:
store.update_eval_task(
eval_task["id"],
{
"progress_detail": progress_content,
"progress": progress_content.get("percentage", eval_task.get("progress", 0)),
},
)
except Exception:
pass
try:
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
result_content = None
# Try to read eval_results.json from the job output directory on completion.
if job.get("status") == "completed" and job.get("output_dir"):
try:
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
pass
if job.get("status") in {"failed", "stopped"} and not job.get("error"):
try:
failure_logs = await client.job_logs(eval_task["compute_job_id"], tail_lines=120)
job["error"] = _extract_job_failure_reason(str(failure_logs.get("content") or ""))
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
archived = await _archive_node_directory(
store,
client,
node,
@@ -301,9 +418,28 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
report_object = next(
(item for item in archived if Path(str(item.get("file_name") or "")).name == "eval_results.json"),
archived[0] if archived else None,
)
store.update_eval_task(eval_task["id"], {
"report_storage_object_id": str(report_object["id"]) if report_object else "",
"archive_status": "completed",
"archive_object_ids": [str(item["id"]) for item in archived],
"archive_error": "",
})
# 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001
try:
current_eval = store.eval_task(eval_task["id"])
except Exception:
current_eval = eval_task
if current_eval.get("status") == "completed":
try:
store.update_eval_task(eval_task["id"], {"archive_status": "pending", "archive_error": str(exc)[:2000]})
except Exception:
pass
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
# ── Inference load reconciliation ─────────────────────────────────────

85
backend/app/modules/data_convert/router.py
View File

@@ -6,11 +6,11 @@ import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi import APIRouter, Body, Depends, File, Request, UploadFile
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.auth import get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
@@ -18,7 +18,26 @@ from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
def _authorize_data_convert_request(
request: Request,
task_id: str | None = None,
current_user: dict[str, Any] = Depends(get_current_user),
) -> None:
"""Protect every task-scoped conversion endpoint with resource ACL."""
if not task_id or is_admin(current_user):
return
permission = "read" if request.method in {"GET", "HEAD"} else "write"
if request.url.path.endswith("/run") or request.url.path.endswith("/import-as-dataset"):
permission = "execute"
if not has_resource_access("data_convert", task_id, current_user, permission):
raise fail(403, "no permission to access this data convert task")
router = APIRouter(
prefix="/data-convert",
tags=["data-convert"],
dependencies=[Depends(_authorize_data_convert_request)],
)
# 存储根目录
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
@@ -207,24 +226,32 @@ def list_tasks(
if is_admin(current_user):
# 管理员可见全部
rows = conn.execute(
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
"FROM data_convert_tasks task "
"LEFT JOIN users creator ON creator.id=task.created_by "
"LEFT JOIN users processor ON processor.id=task.processed_by "
"WHERE task.deleted_at IS NULL "
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
(page_size, (page - 1) * page_size),
).fetchall()
total = conn.execute(
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
).fetchone()[0]
else:
# 普通用户只能看到自己创建的
# 普通用户只能看到本租户且由自己创建的任务；跨租户 ACL 通过任务级依赖访问。
user_id = current_user.get("id")
rows = conn.execute(
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s "
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
(user_id, page_size, (page - 1) * page_size),
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
"FROM data_convert_tasks task "
"LEFT JOIN users creator ON creator.id=task.created_by "
"LEFT JOIN users processor ON processor.id=task.processed_by "
"WHERE task.deleted_at IS NULL AND task.tenant_id=%s AND task.created_by=%s "
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
(current_user.get("tenant_id") or "default", user_id, page_size, (page - 1) * page_size),
).fetchall()
total = conn.execute(
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s",
(user_id,)
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND tenant_id=%s AND created_by=%s",
(current_user.get("tenant_id") or "default", user_id,)
).fetchone()[0]
return ok({"items": [dict(r) for r in rows], "total": total})
@@ -243,11 +270,16 @@ def create_task(
description = str(payload.get("description") or "").strip()
user_id = current_user.get("id")
store = get_platform_store()
tenant_id = current_user.get("tenant_id") or "default"
try:
store.assert_active_tenant(tenant_id)
except ValueError as exc:
raise fail(400, str(exc))
with store.connect() as conn:
conn.execute(
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) "
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by, tenant_id) "
"VALUES (%s, %s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id, tenant_id),
)
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
@@ -318,8 +350,8 @@ async def upload_source_files(
# 标记上传完成
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='uploaded', update_time=NOW() WHERE id=%s",
(task_id,),
"UPDATE data_convert_tasks SET status='uploaded', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(current_user.get("id"), task_id),
)
# 自动转换并导入数据集
try:
@@ -332,8 +364,8 @@ async def upload_source_files(
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
(input_count, output_count, task_id),
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(input_count, output_count, current_user.get("id"), task_id),
)
# 自动导入数据集
content = output.decode("utf-8")
@@ -348,6 +380,7 @@ async def upload_source_files(
"count": output_count,
"description": f"由数据类型转换任务 {task_id} 自动导入",
"created_by": task.get("created_by") or current_user.get("id"),
"tenant_id": task.get("tenant_id") or current_user.get("tenant_id") or "default",
})
dataset_id = dataset["id"]
with store.connect() as conn:
@@ -375,8 +408,8 @@ async def upload_source_files(
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
(str(exc)[:500], task_id),
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(str(exc)[:500], current_user.get("id"), task_id),
)
return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]})
@@ -396,8 +429,8 @@ def run_convert(
store = get_platform_store()
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='running', error_message='', update_time=NOW() WHERE id=%s",
(task_id,),
"UPDATE data_convert_tasks SET status='running', error_message='', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(current_user.get("id"), task_id),
)
try:
if _minio_enabled():
@@ -408,14 +441,14 @@ def run_convert(
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
(input_count, output_count, task_id),
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(input_count, output_count, current_user.get("id"), task_id),
)
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
(str(exc)[:500], task_id),
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(str(exc)[:500], current_user.get("id"), task_id),
)
raise fail(500, f"convert failed: {exc}")
return ok(_get_task(task_id))

8
backend/app/modules/data_process/store/tasks.py
View File

@@ -84,10 +84,14 @@ class TasksMixin:
rows = conn.execute(
f"""
SELECT task.*,
creator.display_name AS creator_name,
processor.display_name AS processor_name,
(SELECT COUNT(*) FROM data_process_source_files source_file
WHERE source_file.task_id=task.id
AND source_file.deleted_at IS NULL) AS source_file_count
FROM data_process_tasks task
LEFT JOIN users creator ON creator.id=task.created_by
LEFT JOIN users processor ON processor.id=task.updated_by
WHERE {where}
ORDER BY task.created_at DESC, task.id DESC
LIMIT %s OFFSET %s
@@ -410,6 +414,8 @@ class TasksMixin:
row = conn.execute(
"""
SELECT task.*,
creator.display_name AS creator_name,
processor.display_name AS processor_name,
(SELECT COUNT(*) FROM data_process_source_files source
WHERE source.task_id=task.id AND source.deleted_at IS NULL)
AS source_file_count,
@@ -442,6 +448,8 @@ class TasksMixin:
ELSE NULL
END AS duration_seconds
FROM data_process_tasks task
LEFT JOIN users creator ON creator.id=task.created_by
LEFT JOIN users processor ON processor.id=task.updated_by
WHERE task.id=%s AND task.deleted_at IS NULL
""",
(task_id,),

55
backend/app/modules/gpu/router.py
View File

@@ -1,25 +1,18 @@
"""GPU 算力分配管理路由。"""
from __future__ import annotations
import json
from typing import Any
from fastapi import APIRouter, Body, Depends, Request
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.auth import get_current_user, is_admin, user_tenant_ids
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/compute", tags=["gpu-assignment"])
def _actor_id(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
if token.startswith("platform-token-"):
return token[len("platform-token-"):]
return None
@router.get("/gpu-assignments")
def list_assignments(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""查看全部分配关系（仅 admin）。"""
@@ -40,8 +33,11 @@ def assign_gpus(
assignments = payload.get("assignments") or []
if not assignments:
raise fail(400, "assignments 不能为空")
actor = _actor_id(request) if request else None
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
actor = current_user.get("id")
try:
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="gpu.assign",
actor_id=actor,
@@ -51,6 +47,41 @@ def assign_gpus(
return ok(result)
@router.post("/gpu-assignments/request")
def request_gpu_assignment(
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
assignments = payload.get("assignments") or []
if not assignments:
raise fail(400, "assignments 不能为空")
if is_admin(current_user):
try:
return ok(get_platform_store().assign_gpus(assignments, assigned_by=current_user.get("id")))
except ValueError as exc:
raise fail(409, str(exc))
user_id = str(current_user.get("id") or "")
normalized = []
for item in assignments:
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
raise fail(400, "每项必须包含 node_id 和 gpu_index")
normalized.append({**item, "user_id": user_id})
instance = get_platform_store().create_approval_instance({
"resource_type": "gpu",
"resource_id": f"batch:{user_id}",
"applicant_id": user_id,
"action": "gpu.assign",
"tenant_id": current_user.get("tenant_id") or "default",
"reason": json.dumps({"assignments": normalized}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
detail=f"count={len(normalized)}",
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.delete("/gpu-assignments/{assignment_id}")
def unassign_gpu(
assignment_id: str,
@@ -61,7 +92,7 @@ def unassign_gpu(
if not is_admin(current_user):
raise fail(403, "admin permission required")
get_platform_store().unassign_gpu(assignment_id)
actor = _actor_id(request) if request else None
actor = current_user.get("id")
get_platform_store().record_audit(
action="gpu.unassign",
actor_id=actor,

34
backend/app/modules/project/router.py
View File

@@ -32,16 +32,24 @@ def _require_approval_or_admin(
resource_id: str,
current_user: dict[str, Any],
action_desc: str = "",
action: str = "project.change",
) -> dict[str, Any] | None:
"""高风险操作审批旁路：admin 直接放行，普通用户创建审批实例（code=202）。"""
if is_admin(current_user):
return None
if not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "no permission to request this project change")
store = get_platform_store()
if store.consume_approved_approval(resource_type, resource_id, str(current_user.get("id") or ""), action):
return None
instance = store.create_approval_instance({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"template_id": None,
"action": action,
"tenant_id": current_user.get("tenant_id") or "default",
"reason": action_desc,
})
return {
"code": 202,
@@ -68,12 +76,20 @@ def list_projects(
@router.post("")
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_project(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user) and str(payload.get("tenant_id") or current_user.get("tenant_id") or "default") != str(current_user.get("tenant_id") or "default"):
raise fail(403, "cannot create project in another tenant")
payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
payload.setdefault("create_by", current_user.get("id"))
store = get_platform_store()
try:
store.assert_active_tenant(payload["tenant_id"])
except ValueError as exc:
raise fail(400, str(exc))
proj = store.create_project(payload)
store.record_audit(
action="project.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=proj["id"],
tenant_id=proj.get("tenant_id"),
@@ -109,7 +125,7 @@ def update_project(
raise fail(404, "project not found")
store.record_audit(
action="project.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
@@ -125,7 +141,7 @@ def archive_project(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}", "project.archive")
if pending:
return pending
store = get_platform_store()
@@ -135,7 +151,7 @@ def archive_project(
raise fail(404, "project not found")
store.record_audit(
action="project.archive",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
@@ -150,14 +166,14 @@ def delete_project(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}", "project.delete")
if pending:
return pending
store = get_platform_store()
store.delete_project(project_id)
store.record_audit(
action="project.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
)
@@ -190,7 +206,7 @@ def add_member(
raise fail(404, "project not found")
store.record_audit(
action="project.member.add",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project.member",
target_id=project_id,
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
@@ -215,7 +231,7 @@ def update_member(
raise fail(404, "project or member not found")
store.record_audit(
action="project.member.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project.member",
target_id=project_id,
detail=f"user_id={user_id},role={payload.get('role')}",

50
backend/app/modules/resource/router.py
View File

@@ -5,16 +5,21 @@ from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, has_resource_access, is_admin
from app.core.auth import (
get_current_user,
has_resource_access,
is_admin,
resource_record,
resource_tenant_id,
user_tenant_ids,
)
from app.core.audit import audit_log, AuditActions
router = APIRouter(prefix="/resources", tags=["resource"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
def _actor(request: Request, current_user: dict[str, Any]) -> str | None:
return str(current_user.get("id") or "") or None
@router.get("/{resource_type}/{resource_id}/acl")
@@ -39,19 +44,48 @@ def set_acl(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""设置资源 ACL，body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
resource = resource_record(resource_type, resource_id)
if not resource and not is_admin(current_user):
raise fail(404, "resource not found")
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "only resource owner or admin can update ACL")
entries = payload.get("entries") or []
allowed = {"read", "write", "execute", "download", "delete", "admin"}
owner_allowed = {"read", "write", "execute", "download"}
tenant_id = resource_tenant_id(resource_type, resource) if resource else None
tenant_ids = user_tenant_ids(current_user)
for entry in entries:
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
raise fail(400, "invalid ACL principal")
if any(permission not in allowed for permission in entry.get("permissions") or []):
permissions = set(entry.get("permissions") or [])
if any(permission not in allowed for permission in permissions):
raise fail(400, "invalid ACL permission")
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
if not is_admin(current_user) and permissions - owner_allowed:
raise fail(403, "resource owners cannot grant delete or admin permission")
if entry.get("principal_type") == "user":
with get_platform_store().connect() as conn:
principal = conn.execute(
"SELECT id, tenant_id, status FROM users WHERE id=?",
(entry["principal_id"],),
).fetchone()
if not principal or principal.get("status") != "active":
raise fail(400, "ACL user does not exist or is inactive")
principal_tenant = str(principal.get("tenant_id") or "default")
if not is_admin(current_user) and tenant_id and principal_tenant not in tenant_ids:
raise fail(403, "cannot grant resource access across tenants")
elif not is_admin(current_user):
# Role ACLs are global in the legacy schema and therefore cannot
# be safely scoped to one tenant by a normal resource owner.
raise fail(403, "only administrators can grant role-based ACLs")
result = get_platform_store().set_resource_acl(
resource_type,
resource_id,
entries,
granted_by=str(current_user.get("id") or "") or None,
)
get_platform_store().record_audit(
action="resource.acl.set",
actor_id=_actor(request) if request else None,
actor_id=_actor(request, current_user) if request else current_user.get("id"),
target_type=resource_type,
target_id=resource_id,
detail=f"entries={len(entries)}",

20
backend/app/modules/retention/router.py
View File

@@ -1,10 +1,11 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from fastapi import APIRouter, Body, Request, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import require_admin
router = APIRouter(prefix="/retention-policies", tags=["retention"])
@@ -16,18 +17,18 @@ def _actor(request: Request) -> str | None:
@router.get("")
def list_policies() -> dict[str, Any]:
def list_policies(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().retention_policies())
@router.post("")
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
if not payload.get("name"):
raise fail(400, "name 必填")
policy = get_platform_store().create_retention_policy(payload)
get_platform_store().record_audit(
action="retention.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy["id"],
detail=f"name={policy.get('name')}",
@@ -36,7 +37,7 @@ def create_policy(payload: dict[str, Any] = Body(...), request: Request = None)
@router.get("/{policy_id}")
def get_policy(policy_id: str) -> dict[str, Any]:
def get_policy(policy_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().retention_policy(policy_id))
except KeyError:
@@ -45,7 +46,8 @@ def get_policy(policy_id: str) -> dict[str, Any]:
@router.put("/{policy_id}")
def update_policy(
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None,
current_user: dict = Depends(require_admin),
) -> dict[str, Any]:
store = get_platform_store()
try:
@@ -54,7 +56,7 @@ def update_policy(
raise fail(404, "retention policy not found")
store.record_audit(
action="retention.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy_id,
detail=f"fields={','.join(payload.keys())}",
@@ -63,12 +65,12 @@ def update_policy(
@router.delete("/{policy_id}")
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
def delete_policy(policy_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
store.delete_retention_policy(policy_id)
store.record_audit(
action="retention.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy_id,
)

18
backend/app/modules/storage/minio_store.py
View File

@@ -8,6 +8,7 @@ from typing import Any
from minio import Minio
from minio.error import S3Error
import urllib3
from app.core.config import get_settings
@@ -20,7 +21,20 @@ class MinioObjectStorage:
def __init__(self) -> None:
settings = get_settings()
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure)
# MinIO outages must fail fast; higher-level workflows own the retry
# policy and should not wait through urllib3's default retry chain.
http_client = urllib3.PoolManager(
cert_reqs="CERT_REQUIRED" if settings.minio_secure else "CERT_NONE",
timeout=urllib3.Timeout(connect=2.0, read=10.0),
retries=False,
)
self.client = Minio(
endpoint,
access_key=settings.minio_access_key,
secret_key=settings.minio_secret_key,
secure=settings.minio_secure,
http_client=http_client,
)
self.bucket = settings.minio_bucket
def _ensure_enabled(self) -> None:
@@ -32,7 +46,7 @@ class MinioObjectStorage:
try:
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(self.bucket)
except S3Error as exc:
except Exception as exc: # noqa: BLE001 - normalize network/client failures
raise ObjectStorageError(str(exc)) from exc
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:

94
backend/app/modules/system/router.py
View File

@@ -1,33 +1,56 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Query, Request, Depends
import csv
import io
from fastapi import APIRouter, Body, HTTPException, Query, Request, Depends
from fastapi.responses import StreamingResponse
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
from app.core.auth import get_current_user, is_admin
from app.core.logging import get_client_ip
router = APIRouter(prefix="/system", tags=["system"])
_VISIT_MODULES = {
"dashboard",
"fine-tune",
"model-eval",
"model-inference",
"model-manage",
"dataset",
"data-process",
"data-convert",
}
@router.post("/audit/visit")
def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
def record_visit(
payload: dict = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict:
"""记录用户访问业务模块的行为，用于看板用户操作分布统计。"""
action = str(payload.get("action") or payload.get("module") or "").strip()
if not action:
return {"code": 0, "message": "ok", "data": {"recorded": False}}
actor_id = ""
if request is not None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
if token.startswith("platform-token-"):
actor_id = token[len("platform-token-"):]
# Visit statistics are intentionally limited to known module names. This
# endpoint must not become a free-form audit-log injection point.
if action not in _VISIT_MODULES:
return {"code": 0, "message": "ok", "data": {"recorded": False}}
actor_id = current_user.get("id")
get_platform_store().record_audit(
action=action,
actor_id=actor_id or None,
target_type="module",
target_id=action,
detail=str(payload.get("detail") or ""),
tenant_id=str(current_user.get("tenant_id") or "") or None,
session_id=str(current_user.get("session_id") or "") or None,
request_id=(request.headers.get("X-Request-ID") if request else None),
detail="module visit",
metadata={"source": "frontend", "detail_length": len(str(payload.get("detail") or ""))},
ip=get_client_ip(request) or None,
)
return {"code": 0, "message": "ok", "data": {"recorded": True}}
@@ -117,13 +140,22 @@ def audit_logs_export(
offset=0,
)
items = result["items"]
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
header = ",".join(columns) + "\n"
columns = [
"time", "tenant_id", "project_id", "actor_id", "action", "target_type",
"target_id", "detail", "client_ip", "result", "reason", "request_id",
"session_id", "metadata",
]
def iter_rows():
yield header
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow(columns)
yield buffer.getvalue()
for row in items:
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
buffer.seek(0)
buffer.truncate(0)
writer.writerow([row.get(c, "") or "" for c in columns])
yield buffer.getvalue()
return StreamingResponse(
iter_rows(),
@@ -134,6 +166,16 @@ def audit_logs_export(
# ===================== 操作日志 =====================
def _operation_log_scope(current_user: dict, conditions: list[str], params: list) -> None:
"""校验操作日志权限，并为普通用户追加本人范围。"""
if is_admin(current_user):
return
if "logs" not in (current_user.get("permissions") or []):
raise HTTPException(status_code=403, detail="missing permission: logs")
conditions.append("user_id = %s")
params.append(str(current_user.get("id") or ""))
@router.get("/operation-logs")
def operation_logs(
user_id: str | None = Query(default=None, description="按用户 ID 筛选"),
@@ -147,14 +189,12 @@ def operation_logs(
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""操作日志查询：按用户/模块/动作/状态/关键字/时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
"""操作日志查询：管理员查全量，普通用户只能查本人记录。"""
store = get_platform_store()
conditions = []
params: list = []
if user_id:
_operation_log_scope(current_user, conditions, params)
if user_id and is_admin(current_user):
conditions.append("user_id = %s")
params.append(user_id)
if module:
@@ -192,13 +232,11 @@ def operation_logs_stats(
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
) -> dict:
"""操作日志统计：总操作数、成功数、失败数、失败率、各模块失败分布、最近错误列表。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
"""操作日志统计：管理员统计全量，普通用户统计本人记录。"""
store = get_platform_store()
conditions = []
params: list = []
_operation_log_scope(current_user, conditions, params)
if start_time:
conditions.append("create_time >= %s")
params.append(start_time)
@@ -260,13 +298,17 @@ def operation_logs_stats(
@router.get("/operation-logs/modules")
def operation_log_modules(current_user: dict = Depends(get_current_user)) -> dict:
"""返回操作日志中出现的模块列表（用于筛选下拉框）。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store()
conditions: list[str] = []
params: list = []
_operation_log_scope(current_user, conditions, params)
where = " WHERE " + " AND ".join(conditions) if conditions else ""
with store.connect() as conn:
rows = conn.execute(
"SELECT DISTINCT module FROM operation_logs WHERE module IS NOT NULL ORDER BY module"
f"SELECT DISTINCT module FROM operation_logs{where}"
+ (" AND" if where else " WHERE")
+ " module IS NOT NULL ORDER BY module",
tuple(params),
).fetchall()
modules = [{"value": r["module"], "label": r["module"]} for r in rows]
return {"code": 0, "message": "ok", "data": modules}

238
backend/app/modules/tenant/router.py
View File

@@ -1,10 +1,12 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
import json
from fastapi import APIRouter, Body, Depends, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import is_admin, require_admin, require_tenant_admin, get_current_user, user_tenant_ids
router = APIRouter(prefix="/tenants", tags=["tenant"])
@@ -16,20 +18,33 @@ def _actor(request: Request) -> str | None:
@router.get("")
def list_tenants() -> dict[str, Any]:
def list_tenants(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().tenants())
@router.get("/invitations")
def my_invitations(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().tenant_invitations(str(current_user.get("id") or "")))
@router.get("/mine")
def my_tenants(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().user_tenants(str(current_user.get("id") or ""), include_all=is_admin(current_user)))
@router.post("")
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
payload = {**payload, "owner_user_id": payload.get("owner_user_id") or current_user.get("id")}
try:
tenant = store.create_tenant(payload)
except KeyError as e:
raise fail(400, f"missing field: {e}")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant["id"],
tenant_id=tenant["id"],
@@ -39,7 +54,7 @@ def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None)
@router.get("/{tenant_id}")
def get_tenant(tenant_id: str) -> dict[str, Any]:
def get_tenant(tenant_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().tenant(tenant_id))
except KeyError:
@@ -47,7 +62,7 @@ def get_tenant(tenant_id: str) -> dict[str, Any]:
@router.put("/{tenant_id}")
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.update_tenant(tenant_id, payload)
@@ -55,7 +70,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -65,7 +80,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
@router.put("/{tenant_id}/quota")
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_quota(tenant_id, payload)
@@ -73,7 +88,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.quota.set",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -82,7 +97,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
@router.put("/{tenant_id}/retention-policy")
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
@@ -90,7 +105,7 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.retention.set",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -99,18 +114,213 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
@router.delete("/{tenant_id}")
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
def delete_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.delete_tenant(tenant_id)
tenant = store.delete_tenant(tenant_id, str(current_user.get("id") or "system"))
except KeyError:
raise fail(404, "tenant not found")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"name={tenant.get('name')}",
)
return ok(tenant)
@router.post("/{tenant_id}/restore")
def restore_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.restore_tenant(tenant_id, str(current_user.get("id") or "system"))
except KeyError:
raise fail(404, "tenant not found")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.restore",
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"name={tenant.get('name')}",
)
return ok(tenant)
@router.get("/{tenant_id}/quota/usage")
def quota_usage(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
return ok(get_platform_store().tenant_quota_usage(tenant_id))
@router.post("/{tenant_id}/quota/request")
def request_quota_change(tenant_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user) and tenant_id not in user_tenant_ids(current_user):
raise fail(403, "tenant access denied")
try:
get_platform_store().assert_active_tenant(tenant_id)
except ValueError as exc:
raise fail(400, str(exc))
quota = payload.get("quota")
if not isinstance(quota, dict):
raise fail(400, "quota must be an object")
instance = get_platform_store().create_approval_instance({
"resource_type": "tenant",
"resource_id": tenant_id,
"applicant_id": current_user.get("id"),
"action": "tenant.quota.update",
"tenant_id": tenant_id,
"reason": json.dumps({"quota": quota}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="tenant.quota.request", actor_id=current_user.get("id"),
target_type="tenant", target_id=tenant_id, tenant_id=tenant_id,
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.get("/{tenant_id}/members")
def list_members(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().tenant_members(tenant_id))
except KeyError:
raise fail(404, "tenant not found")
@router.post("/{tenant_id}/members")
def add_member(
tenant_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
if not payload.get("user_id"):
raise fail(400, "user_id 必填")
if not is_admin(current_user) and payload.get("role") == "owner":
raise fail(403, "only platform administrator can grant owner role")
try:
member = get_platform_store().add_tenant_member(
tenant_id,
str(payload["user_id"]),
str(payload.get("role") or "member"),
current_user.get("id"),
)
except KeyError:
raise fail(404, "tenant or user not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.add",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{payload['user_id']}",
tenant_id=tenant_id,
)
return ok(member)
@router.post("/{tenant_id}/members/invite")
def invite_member(
tenant_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
user_id = str(payload.get("user_id") or "")
if not user_id:
raise fail(400, "user_id 必填")
if payload.get("role") == "owner":
raise fail(403, "tenant invitations cannot grant owner role")
try:
member = get_platform_store().invite_tenant_member(
tenant_id,
user_id,
str(payload.get("role") or "member"),
current_user.get("id"),
payload.get("expires_at"),
)
except KeyError:
raise fail(404, "tenant or active user not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.invite",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
detail=f"role={member.get('role')};expires_at={member.get('expires_at')}",
)
return ok(member)
@router.put("/{tenant_id}/members/{user_id}")
def update_member(
tenant_id: str,
user_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
try:
if not is_admin(current_user) and payload.get("role") == "owner":
raise fail(403, "only platform administrator can grant owner role")
member = get_platform_store().update_tenant_member(tenant_id, user_id, payload)
get_platform_store().record_audit(
action="tenant.member.update",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
detail=f"fields={','.join(payload.keys())}",
)
return ok(member)
except KeyError:
raise fail(404, "tenant member not found")
except ValueError as exc:
raise fail(400, str(exc))
@router.delete("/{tenant_id}/members/{user_id}")
def remove_member(tenant_id: str, user_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
try:
get_platform_store().remove_tenant_member(tenant_id, user_id)
except KeyError:
raise fail(404, "tenant member not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.remove",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
)
return ok({"tenant_id": tenant_id, "user_id": user_id, "removed": True})
@router.post("/{tenant_id}/members/{user_id}/accept")
def accept_invitation(
tenant_id: str,
user_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not is_admin(current_user) and str(current_user.get("id") or "") != user_id:
raise fail(403, "only the invited user can accept this invitation")
try:
member = get_platform_store().accept_tenant_invitation(tenant_id, user_id)
except KeyError:
raise fail(404, "tenant invitation not found")
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="tenant.member.accept",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
)
return ok(member)

30
backend/app/workers/compute_poller.py
View File

@@ -1,9 +1,11 @@
from __future__ import annotations
import asyncio
import time
from app.core.config import get_settings
from app.core.logging import get_logger
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.sync import poll_compute_jobs_once
@@ -18,11 +20,33 @@ async def run_compute_poller() -> None:
interval = max(3, settings.compute_poll_interval_seconds)
logger.info("compute poller started", extra={"interval_seconds": interval})
# PlatformStore may run additive schema checks against a remote PostgreSQL
# server on first use. Keep that startup work off the Uvicorn event loop so
# health checks and normal API requests can still respond while the DB is
# unavailable or slow.
store = None
last_failure_signature = ""
last_failure_logged_at = 0.0
await asyncio.sleep(1)
while True:
try:
result = await poll_compute_jobs_once()
if store is None:
store = await asyncio.to_thread(get_platform_store)
result = await poll_compute_jobs_once(store)
if result["failed"]:
logger.warning("compute polling reported failures", extra={"result": result})
signature = "|".join(sorted({
str(item.get("error") or "")[:120]
for item in result["failed"]
}))
now = time.monotonic()
if signature != last_failure_signature or now - last_failure_logged_at >= 300:
logger.warning(
"compute polling reported failures count=%d first_error=%s",
len(result["failed"]),
signature[:500],
)
last_failure_signature = signature
last_failure_logged_at = now
elif result["synced"]:
logger.debug("compute jobs synchronized", extra={"result": result})
except asyncio.CancelledError:
@@ -30,4 +54,6 @@ async def run_compute_poller() -> None:
raise
except Exception as exc: # noqa: BLE001 - keep background polling alive
logger.exception("compute poller failed", extra={"error": str(exc)})
if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
store = None
await asyncio.sleep(interval)

1
backend/requirements.txt
View File

@@ -9,6 +9,7 @@ alembic>=1.13.1
redis>=5.0.4
httpx>=0.27.0
minio>=7.2.7
urllib3>=2.0.7
PyJWT>=2.8.0
passlib[bcrypt]>=1.7.4
python-dotenv>=1.0.1

30
backend/tests/test_permission_security.py Normal file
View File

@@ -0,0 +1,30 @@
from __future__ import annotations
from app.api.v1.endpoints.platform import _public_model
from app.core.audit import _safe_exception_reason
from app.core.auth import RESOURCE_ACTIONS, RESOURCE_ACTION_ALIASES
def test_public_model_never_exposes_provider_credentials() -> None:
result = _public_model({
"id": "m_1",
"name": "online",
"api_url": "https://example.invalid/v1",
"api_key": "secret-value",
})
assert "api_key" not in result
assert result["api_key_configured"] is True
assert result["name"] == "online"
def test_resource_action_registry_keeps_export_separate_from_module_permissions() -> None:
assert "download" in RESOURCE_ACTIONS
assert "execute" in RESOURCE_ACTIONS
assert RESOURCE_ACTION_ALIASES["export"] == "download"
def test_audit_exception_reason_masks_credentials() -> None:
reason = _safe_exception_reason(ValueError("api_key=secret-value"))
assert "secret-value" not in reason
assert "***" in reason

26
backend/tests/test_storage_security.py Normal file
View File

@@ -0,0 +1,26 @@
from __future__ import annotations
import pytest
from app.api.v1.endpoints.platform import _validate_storage_key
def test_storage_key_is_scoped_to_resource_version() -> None:
assert (
_validate_storage_key("dataset", "ds_123", "v1", None, "train.jsonl")
== "datasets/ds_123/versions/v1/train.jsonl"
)
@pytest.mark.parametrize(
"object_key",
[
"models/other/versions/v1/model.bin",
"datasets/ds_123/versions/v1/../secret.bin",
"datasets/ds_123/versions/v1/../../secret.bin",
"/datasets/ds_123/versions/v1/model.bin",
],
)
def test_storage_key_rejects_escape_or_cross_resource_paths(object_key: str) -> None:
with pytest.raises(ValueError):
_validate_storage_key("dataset", "ds_123", "v1", object_key, None)