YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合

Browse Source 
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请
- 存储: MinIO 存储进度迁移、对象存储安全加固与测试
- 计算: GPU 资源预留、compute 轮询与同步增强
- 权限: permission v2 迁移、权限安全验收测试
- 日志: 后端运行日志中文说明、操作日志整合
- 数据处理/评测: 数据转换与模型评测优化

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-21 09:49:48 +08:00
parent 080ef6ab00
commit 6f0e82f351
94 changed files with 9547 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
backend/app/modules/approval/router.py
View File

@@ -1,17 +1,105 @@
from __future__ import annotations
import json
from fastapi import APIRouter, Body, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, is_admin
from app.core.auth import (
get_current_user,
has_resource_access,
is_tenant_admin,
is_admin,
resource_in_user_tenant,
resource_record,
)
router = APIRouter(prefix="/approvals", tags=["approval"])
# Approval actions are deliberately finite. A caller must not be able to
# create an arbitrary approval that no executor or audit policy understands.
ALLOWED_APPROVAL_ACTIONS = {
"resource.access",
"gpu.assign",
"tenant.quota.update",
"tenant.member.add",
"tenant.member.remove",
"model.use",
"dataset.use",
"dataset.delete",
"trained_model.merge",
"trained_model.export",
"trained_model.delete",
"fine_tune.stop",
"fine_tune.delete",
"eval.delete",
"inference.delete",
"project.archive",
"project.delete",
}
def _available_gpu_options() -> list[dict[str, Any]]:
"""Return only online nodes and currently unassigned, idle GPUs."""
store = get_platform_store()
nodes = store.compute_nodes()
gpus = store.gpus()
assigned = {(str(item.get("node_id")), int(item.get("gpu_index"))) for item in store.gpu_assignments()}
by_node: dict[str, list[dict[str, Any]]] = {}
for gpu in gpus:
node_id = str(gpu.get("node_id") or "")
index = int(gpu.get("id") or 0)
if gpu.get("status") != "idle" or (node_id, index) in assigned:
continue
by_node.setdefault(node_id, []).append({
"index": index,
"name": gpu.get("name") or "GPU",
"memory_total_gb": float(gpu.get("memory_total_gb") or 0),
})
result = []
for node in nodes:
node_id = str(node.get("id") or "")
if not node.get("enabled") or node.get("scheduler_status") != "online" or not by_node.get(node_id):
continue
result.append({
"id": node_id,
"code": node.get("code") or node_id,
"name": node.get("name") or node.get("code") or node_id,
"gpus": sorted(by_node[node_id], key=lambda item: item["index"]),
})
return result
def _validate_gpu_request(assignments: Any) -> list[dict[str, Any]]:
if not isinstance(assignments, list) or not assignments:
raise fail(400, "assignments 不能为空")
available = {
(node["id"], gpu["index"])
for node in _available_gpu_options()
for gpu in node["gpus"]
}
normalized = []
seen: set[tuple[str, int]] = set()
for item in assignments:
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
raise fail(400, "每项必须包含 node_id 和 gpu_index")
try:
key = (str(item["node_id"]), int(item["gpu_index"]))
except (TypeError, ValueError):
raise fail(400, "gpu_index 必须是整数")
if key not in available:
raise fail(409, f"GPU {key[0]}:{key[1]} 当前不可申请")
if key not in seen:
seen.add(key)
normalized.append({"node_id": key[0], "gpu_index": key[1]})
return normalized
@router.get("/templates")
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
return ok(get_platform_store().approval_templates())
@@ -20,11 +108,31 @@ def create_template(payload: dict[str, Any] = Body(...), current_user: dict = De
if not is_admin(current_user): raise fail(403, "admin permission required")
if not payload.get("name"):
raise fail(400, "name 必填")
return ok(get_platform_store().create_approval_template(payload))
steps = payload.get("steps") or []
if not isinstance(steps, list) or any(not isinstance(step, dict) for step in steps):
raise fail(400, "steps 格式无效")
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
payload = {
**payload,
"created_by": current_user.get("id"),
"tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default",
"scope": payload.get("scope") or "tenant",
"status": payload.get("status") or "active",
}
template = get_platform_store().create_approval_template(payload)
get_platform_store().record_audit(
action="approval.template.create", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template["id"],
tenant_id=template.get("tenant_id"),
)
return ok(template)
@router.get("/templates/{template_id}")
def get_template(template_id: str) -> dict[str, Any]:
def get_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
try:
return ok(get_platform_store().approval_template(template_id))
except KeyError:
@@ -34,8 +142,16 @@ def get_template(template_id: str) -> dict[str, Any]:
@router.put("/templates/{template_id}")
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
try:
return ok(get_platform_store().update_approval_template(template_id, payload))
template = get_platform_store().update_approval_template(template_id, payload)
get_platform_store().record_audit(
action="approval.template.update", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template_id,
tenant_id=template.get("tenant_id"), detail=f"fields={','.join(payload.keys())}",
)
return ok(template)
except KeyError:
raise fail(404, "template not found")
@@ -44,15 +160,77 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...), curre
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): raise fail(403, "admin permission required")
try:
return ok(get_platform_store().delete_approval_template(template_id))
template = get_platform_store().delete_approval_template(template_id)
get_platform_store().record_audit(
action="approval.template.delete", actor_id=current_user.get("id"),
target_type="approval_template", target_id=template_id,
tenant_id=template.get("tenant_id"),
)
return ok(template)
except KeyError:
raise fail(404, "template not found")
@router.get("")
def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
items = get_platform_store().approval_instances(status=status)
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
def list_instances(
status: str | None = None,
mine: bool = False,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
user_id = current_user.get("id")
items = get_platform_store().approval_instances(
status=status,
applicant_id=user_id if mine and not is_admin(current_user) else None,
)
if is_admin(current_user):
return ok(items)
if mine:
return ok(items)
visible = []
for item in items:
if item.get("applicant_id") == user_id:
visible.append(item)
continue
if any(step.get("approver_id") == user_id and step.get("status") == "pending" for step in item.get("steps", [])):
visible.append(item)
continue
if is_tenant_admin(current_user, item.get("tenant_id")) and item.get("status") == "pending":
visible.append(item)
return ok(visible)
@router.get("/gpu-options")
def gpu_request_options(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Self-service GPU options; does not expose the admin compute page."""
return ok({"nodes": _available_gpu_options()})
@router.post("/gpu-requests")
def create_gpu_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
assignments = _validate_gpu_request(payload.get("assignments"))
user_id = str(current_user.get("id") or "")
if is_admin(current_user):
try:
return ok({"approval_required": False, "assignments": get_platform_store().assign_gpus(
[{**item, "user_id": user_id} for item in assignments], assigned_by=user_id,
)})
except ValueError as exc:
raise fail(409, str(exc))
normalized = [{**item, "user_id": user_id} for item in assignments]
instance = get_platform_store().create_approval_instance({
"resource_type": "gpu",
"resource_id": f"batch:{user_id}",
"applicant_id": user_id,
"action": "gpu.assign",
"tenant_id": current_user.get("tenant_id") or "default",
"reason": json.dumps({"assignments": normalized, "reason": payload.get("reason")}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
detail=f"count={len(normalized)}",
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.post("")
@@ -61,6 +239,24 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
for field in ("resource_type", "resource_id"):
if not payload.get(field):
raise fail(400, f"{field} 必填")
resource = resource_record(str(payload["resource_type"]), str(payload["resource_id"]))
if not resource and not is_admin(current_user):
raise fail(404, "resource not found")
action = str(payload.get("action") or "")
if not action or action not in ALLOWED_APPROVAL_ACTIONS:
raise fail(400, "不支持的审批动作")
if action != "resource.access" and not is_admin(current_user) and not has_resource_access(
str(payload["resource_type"]), str(payload["resource_id"]), current_user, "read"
):
raise fail(403, "no permission to request approval for this resource")
if resource and not is_admin(current_user) and not resource_in_user_tenant(str(payload["resource_type"]), resource, current_user):
raise fail(403, "resource belongs to another tenant")
requested = payload.get("requested_permissions") or []
allowed = {"read", "write", "execute", "download"}
if any(permission not in allowed for permission in requested):
raise fail(400, "invalid requested permission")
payload["tenant_id"] = current_user.get("tenant_id") or "default"
payload["requested_permissions"] = requested
try:
return ok(get_platform_store().create_approval_instance(payload))
except KeyError:
@@ -71,7 +267,7 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
item = get_platform_store().approval_instance(instance_id)
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id") and not is_tenant_admin(current_user, item.get("tenant_id")):
raise fail(403, "no permission to access approval")
return ok(item)
except KeyError:
@@ -83,18 +279,118 @@ def decide(
instance_id: str,
step_index: int,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not payload.get("approver_id"):
raise fail(400, "approver_id 必填")
try:
return ok(
get_platform_store().decide_approval_step(
instance = get_platform_store().approval_instance(instance_id)
if instance.get("applicant_id") == current_user.get("id"):
raise fail(403, "applicant cannot approve own request")
step = next((item for item in instance.get("steps", []) if int(item.get("step_index", -1)) == step_index), None)
if not step and is_admin(current_user) and not instance.get("steps"):
step = {"approver_id": None, "status": "pending"}
if not step:
raise fail(404, "approval step not found")
designated = step.get("approver_id")
if not is_admin(current_user) and designated != current_user.get("id") and not (
step.get("approver_type") == "admin" and is_tenant_admin(current_user, instance.get("tenant_id"))
):
raise fail(403, "current user is not the designated approver")
if instance.get("action") == "tenant.quota.update" and not is_admin(current_user):
raise fail(403, "only platform administrator can approve tenant quota changes")
submitted_approver = payload.get("approver_id")
if submitted_approver and submitted_approver != current_user.get("id"):
raise fail(403, "approver_id must match current session")
store = get_platform_store()
result = store.decide_approval_step(
instance_id,
step_index,
approver_id=payload["approver_id"],
approver_id=str(current_user.get("id")),
approved=bool(payload.get("approved", False)),
comment=payload.get("comment"),
)
if result.get("status") == "approved" and result.get("execution_status") == "ready":
try:
effect = store.apply_approval_effect(result, str(current_user.get("id") or ""))
if effect is not None:
result = store.approval_instance(instance_id)
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE approval_instances SET execution_status='failed', execution_error=? WHERE id=?",
(str(exc), instance_id),
)
raise fail(409, f"审批已通过，但执行失败：{exc}")
store.record_audit(
action="approval.decision", actor_id=current_user.get("id"),
target_type="approval_instance", target_id=instance_id,
tenant_id=result.get("tenant_id"), result="success" if payload.get("approved") else "rejected",
detail=f"step={step_index}", reason=payload.get("comment"),
)
return ok(result)
except (KeyError, ValueError) as e:
raise fail(400, str(e))
@router.post("/resource-access/requests")
def create_resource_access_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
resource_type = str(payload.get("resource_type") or "")
resource_id = str(payload.get("resource_id") or "")
resource = resource_record(resource_type, resource_id)
if not resource:
raise fail(404, "resource not found")
if not is_admin(current_user) and not resource_in_user_tenant(resource_type, resource, current_user):
raise fail(403, "resource belongs to another tenant")
permissions = payload.get("requested_permissions") or ["read"]
allowed = {"read", "write", "execute", "download"}
if not permissions or any(permission not in allowed for permission in permissions):
raise fail(400, "invalid requested permissions")
result = get_platform_store().create_resource_access_request({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"principal_type": "user",
"principal_id": current_user.get("id"),
"requested_permissions": permissions,
"reason": payload.get("reason"),
"template_id": payload.get("template_id"),
"tenant_id": current_user.get("tenant_id") or "default",
"expires_at": payload.get("expires_at"),
})
get_platform_store().record_audit(
action="resource.access.request", actor_id=current_user.get("id"),
target_type=resource_type, target_id=resource_id,
tenant_id=current_user.get("tenant_id") or "default",
detail=f"permissions={','.join(permissions)}",
)
return ok(result)
@router.get("/resource-access/requests")
def list_resource_access_requests(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
items = get_platform_store().resource_access_requests(
user_id=None if is_admin(current_user) else current_user.get("id"),
status=status,
)
return ok(items)
@router.post("/resource-access/requests/{request_id}/cancel")
def cancel_resource_access_request(request_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
item = get_platform_store().cancel_resource_access_request(
request_id,
str(current_user.get("id") or ""),
is_admin_actor=is_admin(current_user),
)
except KeyError:
raise fail(404, "access request not found")
except PermissionError as exc:
raise fail(403, str(exc))
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="resource.access.cancel", actor_id=current_user.get("id"),
target_type="resource_access_request", target_id=request_id,
tenant_id=item.get("tenant_id"),
)
return ok(item)

182
backend/app/modules/compute_gateway/sync.py
View File

@@ -14,6 +14,18 @@ from app.modules.storage.minio_store import get_object_storage
MAX_STARTING_ATTEMPTS = 40
def _extract_job_failure_reason(log_text: str, limit: int = 2000) -> str:
"""Return a concise actionable reason from a failed Compute job log."""
lines = [line.strip() for line in str(log_text or "").splitlines() if line.strip()]
if not lines:
return ""
markers = ("[eval] FAILED", "Traceback", "RuntimeError", "Error:", "ERROR")
for index in range(len(lines) - 1, -1, -1):
if any(marker in lines[index] for marker in markers):
return "\n".join(lines[index : index + 8])[-limit:]
return "\n".join(lines[-8:])[-limit:]
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
@@ -27,12 +39,15 @@ async def _archive_node_directory(
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
if source == data_root:
raise RuntimeError("refuse to archive compute data root; output_dir must be a task subdirectory")
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
max_files = 10000
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
@@ -49,6 +64,8 @@ async def _archive_node_directory(
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
if len(archived) >= max_files:
raise RuntimeError(f"archive file count exceeds limit {max_files}")
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
@@ -115,11 +132,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = "compute node deleted"
store.mark_inference_unloaded(item.get("node_id") or "")
store.release_external_gpus("inference", str(task["id"]), item.get("node_id"))
continue
if not node.get("enabled") or node.get("scheduler_status") != "online":
item["status"] = "error"
item["error"] = "compute node offline"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
continue
try:
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
@@ -128,6 +147,7 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = f"compute node unreachable: {exc}"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
continue
node_status = status.get("status")
if node_status == "ready":
@@ -139,11 +159,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
item["status"] = "error"
item["error"] = status.get("error") or "model load failed on compute node"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
elif node_status == "idle":
# 节点重启导致已加载模型丢失
item["status"] = "error"
item["error"] = "model disappeared from compute node (node may have restarted)"
store.mark_inference_unloaded(node["id"])
store.release_external_gpus("inference", str(task["id"]), node["id"])
# node_status == "loading" -> 保持 starting，下轮再查
if dirty:
if any(i.get("status") in {"ready", "running"} for i in items):
@@ -157,11 +179,16 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
return reconciled
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
async def fetch_eval_result_content(
client: ComputeNodeClient,
node: dict[str, Any],
job: dict[str, Any],
file_name: str = "eval_results.json",
) -> dict[str, Any] | None:
output_dir = job.get("output_dir")
if not output_dir:
return None
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
full_path = f"{str(output_dir).rstrip('/')}/{file_name}"
data_root = "/data/yg-ft/"
if full_path.startswith(data_root):
full_path = full_path[len(data_root):]
@@ -175,11 +202,36 @@ async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, A
return payload if isinstance(payload, dict) else None
async def poll_compute_jobs_once() -> dict[str, Any]:
store = get_platform_store()
async def fetch_eval_progress_content(
client: ComputeNodeClient,
node: dict[str, Any],
job: dict[str, Any],
) -> dict[str, Any] | None:
return await fetch_eval_result_content(client, node, job, "eval_progress.json")
async def poll_compute_jobs_once(store: Any | None = None) -> dict[str, Any]:
store = store or get_platform_store()
synced: list[dict[str, Any]] = []
failed: list[dict[str, str]] = []
for task in store.running_compute_tasks():
online_nodes = {
str(node.get("id"))
for node in store.compute_nodes()
if node.get("enabled") and node.get("scheduler_status") in {"online", "draining"}
}
training_tasks = {str(task["id"]): task for task in store.running_compute_tasks()}
# Completed tasks whose MinIO archive was interrupted remain eligible for
# reconciliation after a Backend restart or a transient node failure.
if get_settings().minio_enabled:
for task in store.tasks():
if task.get("status") != "completed" or not task.get("compute_job_id"):
continue
if (
str(task.get("archive_status") or "") != "completed"
and str(task.get("compute_node_id")) in online_nodes
):
training_tasks.setdefault(str(task["id"]), task)
for task in training_tasks.values():
node = _node_for_task(task)
if not node:
failed.append({"task_id": task["id"], "error": "compute node not found"})
@@ -215,26 +267,47 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
try:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
store.update_task(task["id"], {
"archive_status": "completed",
"archive_object_ids": [str(item["id"]) for item in archived],
"archive_error": "",
})
except Exception as archive_exc:
store.update_task(task["id"], {
"archive_status": "pending",
"archive_error": str(archive_exc)[:2000],
})
raise
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
for record in store.active_standalone_compute_jobs():
standalone_jobs = {
str(record["id"]): record
for record in store.active_standalone_compute_jobs()
if str(record.get("node_id")) in online_nodes
}
if get_settings().minio_enabled:
for record in store.standalone_compute_jobs_pending_archive():
if str(record.get("node_id")) in online_nodes:
standalone_jobs.setdefault(str(record["id"]), record)
for record in standalone_jobs.values():
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
if not node:
failed.append({"job_id": record["id"], "error": "compute node not found"})
@@ -266,12 +339,32 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
store.update_compute_job_archive(record["id"], "completed", [str(item["id"]) for item in archived])
except Exception as exc: # noqa: BLE001 - keep polling other jobs
try:
store.update_compute_job_archive(record["id"], "pending", [], str(exc)[:2000])
except Exception:
pass
failed.append({"job_id": record["id"], "error": str(exc)})
# ── Eval job sync ────────────────────────────────────────────────
eval_synced = 0
for eval_task in store.running_eval_tasks():
eval_tasks = {str(task["id"]): task for task in store.running_eval_tasks()}
if get_settings().minio_enabled:
# A completed evaluation can win the race with the poller: its status
# is persisted before the report archive finishes. Keep such tasks in
# the reconciliation set until the report object is available.
for task in store.eval_tasks():
if (
task.get("status") == "completed"
and task.get("compute_job_id")
and str(task.get("archive_status") or "") != "completed"
and str(task.get("compute_node_id")) in online_nodes
):
eval_tasks.setdefault(str(task["id"]), task)
for eval_task in eval_tasks.values():
if str(eval_task.get("compute_node_id")) not in online_nodes:
continue
node = next(
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
None,
@@ -283,15 +376,39 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
client = ComputeNodeClient(node["api_base_url"])
job = await client.get_job(eval_task["compute_job_id"])
result_content = None
# Try to read eval_results.json from the job output directory
# Read live progress and partial results while the evaluator is running.
if job.get("status") in {"queued", "running"} and job.get("output_dir"):
try:
progress_content = await fetch_eval_progress_content(client, node, job)
if progress_content:
store.update_eval_task(
eval_task["id"],
{
"progress_detail": progress_content,
"progress": progress_content.get("percentage", eval_task.get("progress", 0)),
},
)
except Exception:
pass
try:
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
result_content = None
# Try to read eval_results.json from the job output directory on completion.
if job.get("status") == "completed" and job.get("output_dir"):
try:
result_content = await fetch_eval_result_content(client, node, job)
except Exception:
pass
if job.get("status") in {"failed", "stopped"} and not job.get("error"):
try:
failure_logs = await client.job_logs(eval_task["compute_job_id"], tail_lines=120)
job["error"] = _extract_job_failure_reason(str(failure_logs.get("content") or ""))
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
archived = await _archive_node_directory(
store,
client,
node,
@@ -301,9 +418,28 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
report_object = next(
(item for item in archived if Path(str(item.get("file_name") or "")).name == "eval_results.json"),
archived[0] if archived else None,
)
store.update_eval_task(eval_task["id"], {
"report_storage_object_id": str(report_object["id"]) if report_object else "",
"archive_status": "completed",
"archive_object_ids": [str(item["id"]) for item in archived],
"archive_error": "",
})
# 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001
try:
current_eval = store.eval_task(eval_task["id"])
except Exception:
current_eval = eval_task
if current_eval.get("status") == "completed":
try:
store.update_eval_task(eval_task["id"], {"archive_status": "pending", "archive_error": str(exc)[:2000]})
except Exception:
pass
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
# ── Inference load reconciliation ─────────────────────────────────────

85
backend/app/modules/data_convert/router.py
View File

@@ -6,11 +6,11 @@ import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi import APIRouter, Body, Depends, File, Request, UploadFile
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.auth import get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
@@ -18,7 +18,26 @@ from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
def _authorize_data_convert_request(
request: Request,
task_id: str | None = None,
current_user: dict[str, Any] = Depends(get_current_user),
) -> None:
"""Protect every task-scoped conversion endpoint with resource ACL."""
if not task_id or is_admin(current_user):
return
permission = "read" if request.method in {"GET", "HEAD"} else "write"
if request.url.path.endswith("/run") or request.url.path.endswith("/import-as-dataset"):
permission = "execute"
if not has_resource_access("data_convert", task_id, current_user, permission):
raise fail(403, "no permission to access this data convert task")
router = APIRouter(
prefix="/data-convert",
tags=["data-convert"],
dependencies=[Depends(_authorize_data_convert_request)],
)
# 存储根目录
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
@@ -207,24 +226,32 @@ def list_tasks(
if is_admin(current_user):
# 管理员可见全部
rows = conn.execute(
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
"FROM data_convert_tasks task "
"LEFT JOIN users creator ON creator.id=task.created_by "
"LEFT JOIN users processor ON processor.id=task.processed_by "
"WHERE task.deleted_at IS NULL "
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
(page_size, (page - 1) * page_size),
).fetchall()
total = conn.execute(
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
).fetchone()[0]
else:
# 普通用户只能看到自己创建的
# 普通用户只能看到本租户且由自己创建的任务；跨租户 ACL 通过任务级依赖访问。
user_id = current_user.get("id")
rows = conn.execute(
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s "
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
(user_id, page_size, (page - 1) * page_size),
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
"FROM data_convert_tasks task "
"LEFT JOIN users creator ON creator.id=task.created_by "
"LEFT JOIN users processor ON processor.id=task.processed_by "
"WHERE task.deleted_at IS NULL AND task.tenant_id=%s AND task.created_by=%s "
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
(current_user.get("tenant_id") or "default", user_id, page_size, (page - 1) * page_size),
).fetchall()
total = conn.execute(
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s",
(user_id,)
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND tenant_id=%s AND created_by=%s",
(current_user.get("tenant_id") or "default", user_id,)
).fetchone()[0]
return ok({"items": [dict(r) for r in rows], "total": total})
@@ -243,11 +270,16 @@ def create_task(
description = str(payload.get("description") or "").strip()
user_id = current_user.get("id")
store = get_platform_store()
tenant_id = current_user.get("tenant_id") or "default"
try:
store.assert_active_tenant(tenant_id)
except ValueError as exc:
raise fail(400, str(exc))
with store.connect() as conn:
conn.execute(
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) "
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by, tenant_id) "
"VALUES (%s, %s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id, tenant_id),
)
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
@@ -318,8 +350,8 @@ async def upload_source_files(
# 标记上传完成
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='uploaded', update_time=NOW() WHERE id=%s",
(task_id,),
"UPDATE data_convert_tasks SET status='uploaded', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(current_user.get("id"), task_id),
)
# 自动转换并导入数据集
try:
@@ -332,8 +364,8 @@ async def upload_source_files(
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
(input_count, output_count, task_id),
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(input_count, output_count, current_user.get("id"), task_id),
)
# 自动导入数据集
content = output.decode("utf-8")
@@ -348,6 +380,7 @@ async def upload_source_files(
"count": output_count,
"description": f"由数据类型转换任务 {task_id} 自动导入",
"created_by": task.get("created_by") or current_user.get("id"),
"tenant_id": task.get("tenant_id") or current_user.get("tenant_id") or "default",
})
dataset_id = dataset["id"]
with store.connect() as conn:
@@ -375,8 +408,8 @@ async def upload_source_files(
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
(str(exc)[:500], task_id),
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(str(exc)[:500], current_user.get("id"), task_id),
)
return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]})
@@ -396,8 +429,8 @@ def run_convert(
store = get_platform_store()
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='running', error_message='', update_time=NOW() WHERE id=%s",
(task_id,),
"UPDATE data_convert_tasks SET status='running', error_message='', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(current_user.get("id"), task_id),
)
try:
if _minio_enabled():
@@ -408,14 +441,14 @@ def run_convert(
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
(input_count, output_count, task_id),
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(input_count, output_count, current_user.get("id"), task_id),
)
except Exception as exc:
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
(str(exc)[:500], task_id),
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
(str(exc)[:500], current_user.get("id"), task_id),
)
raise fail(500, f"convert failed: {exc}")
return ok(_get_task(task_id))

8
backend/app/modules/data_process/store/tasks.py
View File

@@ -84,10 +84,14 @@ class TasksMixin:
rows = conn.execute(
f"""
SELECT task.*,
creator.display_name AS creator_name,
processor.display_name AS processor_name,
(SELECT COUNT(*) FROM data_process_source_files source_file
WHERE source_file.task_id=task.id
AND source_file.deleted_at IS NULL) AS source_file_count
FROM data_process_tasks task
LEFT JOIN users creator ON creator.id=task.created_by
LEFT JOIN users processor ON processor.id=task.updated_by
WHERE {where}
ORDER BY task.created_at DESC, task.id DESC
LIMIT %s OFFSET %s
@@ -410,6 +414,8 @@ class TasksMixin:
row = conn.execute(
"""
SELECT task.*,
creator.display_name AS creator_name,
processor.display_name AS processor_name,
(SELECT COUNT(*) FROM data_process_source_files source
WHERE source.task_id=task.id AND source.deleted_at IS NULL)
AS source_file_count,
@@ -442,6 +448,8 @@ class TasksMixin:
ELSE NULL
END AS duration_seconds
FROM data_process_tasks task
LEFT JOIN users creator ON creator.id=task.created_by
LEFT JOIN users processor ON processor.id=task.updated_by
WHERE task.id=%s AND task.deleted_at IS NULL
""",
(task_id,),

55
backend/app/modules/gpu/router.py
View File

@@ -1,25 +1,18 @@
"""GPU 算力分配管理路由。"""
from __future__ import annotations
import json
from typing import Any
from fastapi import APIRouter, Body, Depends, Request
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.auth import get_current_user, is_admin, user_tenant_ids
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/compute", tags=["gpu-assignment"])
def _actor_id(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
if token.startswith("platform-token-"):
return token[len("platform-token-"):]
return None
@router.get("/gpu-assignments")
def list_assignments(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""查看全部分配关系（仅 admin）。"""
@@ -40,8 +33,11 @@ def assign_gpus(
assignments = payload.get("assignments") or []
if not assignments:
raise fail(400, "assignments 不能为空")
actor = _actor_id(request) if request else None
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
actor = current_user.get("id")
try:
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="gpu.assign",
actor_id=actor,
@@ -51,6 +47,41 @@ def assign_gpus(
return ok(result)
@router.post("/gpu-assignments/request")
def request_gpu_assignment(
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
assignments = payload.get("assignments") or []
if not assignments:
raise fail(400, "assignments 不能为空")
if is_admin(current_user):
try:
return ok(get_platform_store().assign_gpus(assignments, assigned_by=current_user.get("id")))
except ValueError as exc:
raise fail(409, str(exc))
user_id = str(current_user.get("id") or "")
normalized = []
for item in assignments:
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
raise fail(400, "每项必须包含 node_id 和 gpu_index")
normalized.append({**item, "user_id": user_id})
instance = get_platform_store().create_approval_instance({
"resource_type": "gpu",
"resource_id": f"batch:{user_id}",
"applicant_id": user_id,
"action": "gpu.assign",
"tenant_id": current_user.get("tenant_id") or "default",
"reason": json.dumps({"assignments": normalized}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
detail=f"count={len(normalized)}",
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.delete("/gpu-assignments/{assignment_id}")
def unassign_gpu(
assignment_id: str,
@@ -61,7 +92,7 @@ def unassign_gpu(
if not is_admin(current_user):
raise fail(403, "admin permission required")
get_platform_store().unassign_gpu(assignment_id)
actor = _actor_id(request) if request else None
actor = current_user.get("id")
get_platform_store().record_audit(
action="gpu.unassign",
actor_id=actor,

34
backend/app/modules/project/router.py
View File

@@ -32,16 +32,24 @@ def _require_approval_or_admin(
resource_id: str,
current_user: dict[str, Any],
action_desc: str = "",
action: str = "project.change",
) -> dict[str, Any] | None:
"""高风险操作审批旁路：admin 直接放行，普通用户创建审批实例（code=202）。"""
if is_admin(current_user):
return None
if not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "no permission to request this project change")
store = get_platform_store()
if store.consume_approved_approval(resource_type, resource_id, str(current_user.get("id") or ""), action):
return None
instance = store.create_approval_instance({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"template_id": None,
"action": action,
"tenant_id": current_user.get("tenant_id") or "default",
"reason": action_desc,
})
return {
"code": 202,
@@ -68,12 +76,20 @@ def list_projects(
@router.post("")
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_project(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user) and str(payload.get("tenant_id") or current_user.get("tenant_id") or "default") != str(current_user.get("tenant_id") or "default"):
raise fail(403, "cannot create project in another tenant")
payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
payload.setdefault("create_by", current_user.get("id"))
store = get_platform_store()
try:
store.assert_active_tenant(payload["tenant_id"])
except ValueError as exc:
raise fail(400, str(exc))
proj = store.create_project(payload)
store.record_audit(
action="project.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=proj["id"],
tenant_id=proj.get("tenant_id"),
@@ -109,7 +125,7 @@ def update_project(
raise fail(404, "project not found")
store.record_audit(
action="project.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
@@ -125,7 +141,7 @@ def archive_project(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}", "project.archive")
if pending:
return pending
store = get_platform_store()
@@ -135,7 +151,7 @@ def archive_project(
raise fail(404, "project not found")
store.record_audit(
action="project.archive",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
@@ -150,14 +166,14 @@ def delete_project(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}", "project.delete")
if pending:
return pending
store = get_platform_store()
store.delete_project(project_id)
store.record_audit(
action="project.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project",
target_id=project_id,
)
@@ -190,7 +206,7 @@ def add_member(
raise fail(404, "project not found")
store.record_audit(
action="project.member.add",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project.member",
target_id=project_id,
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
@@ -215,7 +231,7 @@ def update_member(
raise fail(404, "project or member not found")
store.record_audit(
action="project.member.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="project.member",
target_id=project_id,
detail=f"user_id={user_id},role={payload.get('role')}",

50
backend/app/modules/resource/router.py
View File

@@ -5,16 +5,21 @@ from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import get_current_user, has_resource_access, is_admin
from app.core.auth import (
get_current_user,
has_resource_access,
is_admin,
resource_record,
resource_tenant_id,
user_tenant_ids,
)
from app.core.audit import audit_log, AuditActions
router = APIRouter(prefix="/resources", tags=["resource"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
def _actor(request: Request, current_user: dict[str, Any]) -> str | None:
return str(current_user.get("id") or "") or None
@router.get("/{resource_type}/{resource_id}/acl")
@@ -39,19 +44,48 @@ def set_acl(
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""设置资源 ACL，body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
resource = resource_record(resource_type, resource_id)
if not resource and not is_admin(current_user):
raise fail(404, "resource not found")
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
raise fail(403, "only resource owner or admin can update ACL")
entries = payload.get("entries") or []
allowed = {"read", "write", "execute", "download", "delete", "admin"}
owner_allowed = {"read", "write", "execute", "download"}
tenant_id = resource_tenant_id(resource_type, resource) if resource else None
tenant_ids = user_tenant_ids(current_user)
for entry in entries:
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
raise fail(400, "invalid ACL principal")
if any(permission not in allowed for permission in entry.get("permissions") or []):
permissions = set(entry.get("permissions") or [])
if any(permission not in allowed for permission in permissions):
raise fail(400, "invalid ACL permission")
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
if not is_admin(current_user) and permissions - owner_allowed:
raise fail(403, "resource owners cannot grant delete or admin permission")
if entry.get("principal_type") == "user":
with get_platform_store().connect() as conn:
principal = conn.execute(
"SELECT id, tenant_id, status FROM users WHERE id=?",
(entry["principal_id"],),
).fetchone()
if not principal or principal.get("status") != "active":
raise fail(400, "ACL user does not exist or is inactive")
principal_tenant = str(principal.get("tenant_id") or "default")
if not is_admin(current_user) and tenant_id and principal_tenant not in tenant_ids:
raise fail(403, "cannot grant resource access across tenants")
elif not is_admin(current_user):
# Role ACLs are global in the legacy schema and therefore cannot
# be safely scoped to one tenant by a normal resource owner.
raise fail(403, "only administrators can grant role-based ACLs")
result = get_platform_store().set_resource_acl(
resource_type,
resource_id,
entries,
granted_by=str(current_user.get("id") or "") or None,
)
get_platform_store().record_audit(
action="resource.acl.set",
actor_id=_actor(request) if request else None,
actor_id=_actor(request, current_user) if request else current_user.get("id"),
target_type=resource_type,
target_id=resource_id,
detail=f"entries={len(entries)}",

20
backend/app/modules/retention/router.py
View File

@@ -1,10 +1,11 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from fastapi import APIRouter, Body, Request, Depends
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import require_admin
router = APIRouter(prefix="/retention-policies", tags=["retention"])
@@ -16,18 +17,18 @@ def _actor(request: Request) -> str | None:
@router.get("")
def list_policies() -> dict[str, Any]:
def list_policies(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().retention_policies())
@router.post("")
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
if not payload.get("name"):
raise fail(400, "name 必填")
policy = get_platform_store().create_retention_policy(payload)
get_platform_store().record_audit(
action="retention.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy["id"],
detail=f"name={policy.get('name')}",
@@ -36,7 +37,7 @@ def create_policy(payload: dict[str, Any] = Body(...), request: Request = None)
@router.get("/{policy_id}")
def get_policy(policy_id: str) -> dict[str, Any]:
def get_policy(policy_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().retention_policy(policy_id))
except KeyError:
@@ -45,7 +46,8 @@ def get_policy(policy_id: str) -> dict[str, Any]:
@router.put("/{policy_id}")
def update_policy(
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None,
current_user: dict = Depends(require_admin),
) -> dict[str, Any]:
store = get_platform_store()
try:
@@ -54,7 +56,7 @@ def update_policy(
raise fail(404, "retention policy not found")
store.record_audit(
action="retention.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy_id,
detail=f"fields={','.join(payload.keys())}",
@@ -63,12 +65,12 @@ def update_policy(
@router.delete("/{policy_id}")
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
def delete_policy(policy_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
store.delete_retention_policy(policy_id)
store.record_audit(
action="retention.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="retention_policy",
target_id=policy_id,
)

18
backend/app/modules/storage/minio_store.py
View File

@@ -8,6 +8,7 @@ from typing import Any
from minio import Minio
from minio.error import S3Error
import urllib3
from app.core.config import get_settings
@@ -20,7 +21,20 @@ class MinioObjectStorage:
def __init__(self) -> None:
settings = get_settings()
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure)
# MinIO outages must fail fast; higher-level workflows own the retry
# policy and should not wait through urllib3's default retry chain.
http_client = urllib3.PoolManager(
cert_reqs="CERT_REQUIRED" if settings.minio_secure else "CERT_NONE",
timeout=urllib3.Timeout(connect=2.0, read=10.0),
retries=False,
)
self.client = Minio(
endpoint,
access_key=settings.minio_access_key,
secret_key=settings.minio_secret_key,
secure=settings.minio_secure,
http_client=http_client,
)
self.bucket = settings.minio_bucket
def _ensure_enabled(self) -> None:
@@ -32,7 +46,7 @@ class MinioObjectStorage:
try:
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(self.bucket)
except S3Error as exc:
except Exception as exc: # noqa: BLE001 - normalize network/client failures
raise ObjectStorageError(str(exc)) from exc
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:

94
backend/app/modules/system/router.py
View File

@@ -1,33 +1,56 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Query, Request, Depends
import csv
import io
from fastapi import APIRouter, Body, HTTPException, Query, Request, Depends
from fastapi.responses import StreamingResponse
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
from app.core.auth import get_current_user, is_admin
from app.core.logging import get_client_ip
router = APIRouter(prefix="/system", tags=["system"])
_VISIT_MODULES = {
"dashboard",
"fine-tune",
"model-eval",
"model-inference",
"model-manage",
"dataset",
"data-process",
"data-convert",
}
@router.post("/audit/visit")
def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
def record_visit(
payload: dict = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict:
"""记录用户访问业务模块的行为，用于看板用户操作分布统计。"""
action = str(payload.get("action") or payload.get("module") or "").strip()
if not action:
return {"code": 0, "message": "ok", "data": {"recorded": False}}
actor_id = ""
if request is not None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
if token.startswith("platform-token-"):
actor_id = token[len("platform-token-"):]
# Visit statistics are intentionally limited to known module names. This
# endpoint must not become a free-form audit-log injection point.
if action not in _VISIT_MODULES:
return {"code": 0, "message": "ok", "data": {"recorded": False}}
actor_id = current_user.get("id")
get_platform_store().record_audit(
action=action,
actor_id=actor_id or None,
target_type="module",
target_id=action,
detail=str(payload.get("detail") or ""),
tenant_id=str(current_user.get("tenant_id") or "") or None,
session_id=str(current_user.get("session_id") or "") or None,
request_id=(request.headers.get("X-Request-ID") if request else None),
detail="module visit",
metadata={"source": "frontend", "detail_length": len(str(payload.get("detail") or ""))},
ip=get_client_ip(request) or None,
)
return {"code": 0, "message": "ok", "data": {"recorded": True}}
@@ -117,13 +140,22 @@ def audit_logs_export(
offset=0,
)
items = result["items"]
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
header = ",".join(columns) + "\n"
columns = [
"time", "tenant_id", "project_id", "actor_id", "action", "target_type",
"target_id", "detail", "client_ip", "result", "reason", "request_id",
"session_id", "metadata",
]
def iter_rows():
yield header
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow(columns)
yield buffer.getvalue()
for row in items:
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
buffer.seek(0)
buffer.truncate(0)
writer.writerow([row.get(c, "") or "" for c in columns])
yield buffer.getvalue()
return StreamingResponse(
iter_rows(),
@@ -134,6 +166,16 @@ def audit_logs_export(
# ===================== 操作日志 =====================
def _operation_log_scope(current_user: dict, conditions: list[str], params: list) -> None:
"""校验操作日志权限，并为普通用户追加本人范围。"""
if is_admin(current_user):
return
if "logs" not in (current_user.get("permissions") or []):
raise HTTPException(status_code=403, detail="missing permission: logs")
conditions.append("user_id = %s")
params.append(str(current_user.get("id") or ""))
@router.get("/operation-logs")
def operation_logs(
user_id: str | None = Query(default=None, description="按用户 ID 筛选"),
@@ -147,14 +189,12 @@ def operation_logs(
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""操作日志查询：按用户/模块/动作/状态/关键字/时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
"""操作日志查询：管理员查全量，普通用户只能查本人记录。"""
store = get_platform_store()
conditions = []
params: list = []
if user_id:
_operation_log_scope(current_user, conditions, params)
if user_id and is_admin(current_user):
conditions.append("user_id = %s")
params.append(user_id)
if module:
@@ -192,13 +232,11 @@ def operation_logs_stats(
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
) -> dict:
"""操作日志统计：总操作数、成功数、失败数、失败率、各模块失败分布、最近错误列表。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
"""操作日志统计：管理员统计全量，普通用户统计本人记录。"""
store = get_platform_store()
conditions = []
params: list = []
_operation_log_scope(current_user, conditions, params)
if start_time:
conditions.append("create_time >= %s")
params.append(start_time)
@@ -260,13 +298,17 @@ def operation_logs_stats(
@router.get("/operation-logs/modules")
def operation_log_modules(current_user: dict = Depends(get_current_user)) -> dict:
"""返回操作日志中出现的模块列表（用于筛选下拉框）。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
store = get_platform_store()
conditions: list[str] = []
params: list = []
_operation_log_scope(current_user, conditions, params)
where = " WHERE " + " AND ".join(conditions) if conditions else ""
with store.connect() as conn:
rows = conn.execute(
"SELECT DISTINCT module FROM operation_logs WHERE module IS NOT NULL ORDER BY module"
f"SELECT DISTINCT module FROM operation_logs{where}"
+ (" AND" if where else " WHERE")
+ " module IS NOT NULL ORDER BY module",
tuple(params),
).fetchall()
modules = [{"value": r["module"], "label": r["module"]} for r in rows]
return {"code": 0, "message": "ok", "data": modules}

238
backend/app/modules/tenant/router.py
View File

@@ -1,10 +1,12 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
import json
from fastapi import APIRouter, Body, Depends, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
from app.core.auth import is_admin, require_admin, require_tenant_admin, get_current_user, user_tenant_ids
router = APIRouter(prefix="/tenants", tags=["tenant"])
@@ -16,20 +18,33 @@ def _actor(request: Request) -> str | None:
@router.get("")
def list_tenants() -> dict[str, Any]:
def list_tenants(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().tenants())
@router.get("/invitations")
def my_invitations(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().tenant_invitations(str(current_user.get("id") or "")))
@router.get("/mine")
def my_tenants(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
return ok(get_platform_store().user_tenants(str(current_user.get("id") or ""), include_all=is_admin(current_user)))
@router.post("")
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
payload = {**payload, "owner_user_id": payload.get("owner_user_id") or current_user.get("id")}
try:
tenant = store.create_tenant(payload)
except KeyError as e:
raise fail(400, f"missing field: {e}")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.create",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant["id"],
tenant_id=tenant["id"],
@@ -39,7 +54,7 @@ def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None)
@router.get("/{tenant_id}")
def get_tenant(tenant_id: str) -> dict[str, Any]:
def get_tenant(tenant_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().tenant(tenant_id))
except KeyError:
@@ -47,7 +62,7 @@ def get_tenant(tenant_id: str) -> dict[str, Any]:
@router.put("/{tenant_id}")
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.update_tenant(tenant_id, payload)
@@ -55,7 +70,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.update",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -65,7 +80,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
@router.put("/{tenant_id}/quota")
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_quota(tenant_id, payload)
@@ -73,7 +88,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.quota.set",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -82,7 +97,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
@router.put("/{tenant_id}/retention-policy")
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
@@ -90,7 +105,7 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.retention.set",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
@@ -99,18 +114,213 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
@router.delete("/{tenant_id}")
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
def delete_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.delete_tenant(tenant_id)
tenant = store.delete_tenant(tenant_id, str(current_user.get("id") or "system"))
except KeyError:
raise fail(404, "tenant not found")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.delete",
actor_id=_actor(request) if request else None,
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"name={tenant.get('name')}",
)
return ok(tenant)
@router.post("/{tenant_id}/restore")
def restore_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.restore_tenant(tenant_id, str(current_user.get("id") or "system"))
except KeyError:
raise fail(404, "tenant not found")
except ValueError as exc:
raise fail(400, str(exc))
store.record_audit(
action="tenant.restore",
actor_id=current_user.get("id"),
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"name={tenant.get('name')}",
)
return ok(tenant)
@router.get("/{tenant_id}/quota/usage")
def quota_usage(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
return ok(get_platform_store().tenant_quota_usage(tenant_id))
@router.post("/{tenant_id}/quota/request")
def request_quota_change(tenant_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user) and tenant_id not in user_tenant_ids(current_user):
raise fail(403, "tenant access denied")
try:
get_platform_store().assert_active_tenant(tenant_id)
except ValueError as exc:
raise fail(400, str(exc))
quota = payload.get("quota")
if not isinstance(quota, dict):
raise fail(400, "quota must be an object")
instance = get_platform_store().create_approval_instance({
"resource_type": "tenant",
"resource_id": tenant_id,
"applicant_id": current_user.get("id"),
"action": "tenant.quota.update",
"tenant_id": tenant_id,
"reason": json.dumps({"quota": quota}, ensure_ascii=False),
})
get_platform_store().record_audit(
action="tenant.quota.request", actor_id=current_user.get("id"),
target_type="tenant", target_id=tenant_id, tenant_id=tenant_id,
)
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
@router.get("/{tenant_id}/members")
def list_members(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().tenant_members(tenant_id))
except KeyError:
raise fail(404, "tenant not found")
@router.post("/{tenant_id}/members")
def add_member(
tenant_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
if not payload.get("user_id"):
raise fail(400, "user_id 必填")
if not is_admin(current_user) and payload.get("role") == "owner":
raise fail(403, "only platform administrator can grant owner role")
try:
member = get_platform_store().add_tenant_member(
tenant_id,
str(payload["user_id"]),
str(payload.get("role") or "member"),
current_user.get("id"),
)
except KeyError:
raise fail(404, "tenant or user not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.add",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{payload['user_id']}",
tenant_id=tenant_id,
)
return ok(member)
@router.post("/{tenant_id}/members/invite")
def invite_member(
tenant_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
user_id = str(payload.get("user_id") or "")
if not user_id:
raise fail(400, "user_id 必填")
if payload.get("role") == "owner":
raise fail(403, "tenant invitations cannot grant owner role")
try:
member = get_platform_store().invite_tenant_member(
tenant_id,
user_id,
str(payload.get("role") or "member"),
current_user.get("id"),
payload.get("expires_at"),
)
except KeyError:
raise fail(404, "tenant or active user not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.invite",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
detail=f"role={member.get('role')};expires_at={member.get('expires_at')}",
)
return ok(member)
@router.put("/{tenant_id}/members/{user_id}")
def update_member(
tenant_id: str,
user_id: str,
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(require_tenant_admin),
) -> dict[str, Any]:
try:
if not is_admin(current_user) and payload.get("role") == "owner":
raise fail(403, "only platform administrator can grant owner role")
member = get_platform_store().update_tenant_member(tenant_id, user_id, payload)
get_platform_store().record_audit(
action="tenant.member.update",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
detail=f"fields={','.join(payload.keys())}",
)
return ok(member)
except KeyError:
raise fail(404, "tenant member not found")
except ValueError as exc:
raise fail(400, str(exc))
@router.delete("/{tenant_id}/members/{user_id}")
def remove_member(tenant_id: str, user_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
try:
get_platform_store().remove_tenant_member(tenant_id, user_id)
except KeyError:
raise fail(404, "tenant member not found")
except ValueError as exc:
raise fail(400, str(exc))
get_platform_store().record_audit(
action="tenant.member.remove",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
)
return ok({"tenant_id": tenant_id, "user_id": user_id, "removed": True})
@router.post("/{tenant_id}/members/{user_id}/accept")
def accept_invitation(
tenant_id: str,
user_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not is_admin(current_user) and str(current_user.get("id") or "") != user_id:
raise fail(403, "only the invited user can accept this invitation")
try:
member = get_platform_store().accept_tenant_invitation(tenant_id, user_id)
except KeyError:
raise fail(404, "tenant invitation not found")
except ValueError as exc:
raise fail(409, str(exc))
get_platform_store().record_audit(
action="tenant.member.accept",
actor_id=current_user.get("id"),
target_type="tenant_member",
target_id=f"{tenant_id}:{user_id}",
tenant_id=tenant_id,
)
return ok(member)