This commit is contained in:
wangjiming
2026-08-03 09:34:08 +08:00
parent b975de02da
commit 15c4223f2c
43 changed files with 4498 additions and 234 deletions

View File

@@ -34,6 +34,7 @@ from fastapi import (
from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -815,18 +816,33 @@ def list_tasks(
keyword: str | None = Query(default=None),
status: DataProcessStatus | None = Query(default=None),
process_type: ProcessType | None = Query(default=None),
tenant_id: str | None = Query(default=None),
project_id: str | None = Query(default=None),
store: DataProcessStore = Depends(get_data_process_store),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
with api_errors():
return ok(
store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
)
tasks = store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
tenant_id=tenant_id,
project_id=project_id,
)
# #4 资源 ACL 过滤admin 放行,普通用户只看到自己被授权的数据处理任务
items = tasks.get("items", [])
if not is_admin(current_user) and items:
accessible_ids = set(
filter_accessible_resource_ids(
"data-process", [t["id"] for t in items], current_user
)
)
items = [t for t in items if t["id"] in accessible_ids]
tasks["items"] = items
tasks["total"] = len(items)
return ok(tasks)
@router.post("")
@@ -2282,13 +2298,11 @@ def regenerate_results_batch(
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
# httpx.Client 支持跨线程复用,批次内共享连接池可减少重复建连开销。
with (
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor,
):
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor:
futures = {
executor.submit(
_regenerate_result_in_place,

View File

@@ -2,14 +2,16 @@
import json
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Body, File, HTTPException, Query, UploadFile
from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import PlainTextResponse, StreamingResponse
import httpx
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
@@ -35,6 +37,35 @@ def fail(status_code: int, message: str) -> HTTPException:
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
def _require_approval_or_admin(
resource_type: str,
resource_id: str,
current_user: dict[str, Any],
action_desc: str = "",
) -> dict[str, Any] | None:
"""
高风险操作审批旁路:
- admin 用户直接放行(返回 None
- 普通用户创建审批实例返回审批待定响应code=202非 None
code=202 使前端响应拦截器走业务错误分支,弹提示并 reject
避免前端误认为删除成功。
"""
if is_admin(current_user):
return None
store = get_platform_store()
instance = store.create_approval_instance({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"template_id": None,
})
return {
"code": 202,
"message": f"操作已提交审批,等待管理员批准:{action_desc}",
"data": {"approval_required": True, "approval_id": instance["id"]},
}
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -231,8 +262,18 @@ async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/me")
async def me() -> dict[str, Any]:
return ok(get_platform_store().users()[0])
async def me(request: Request) -> dict[str, Any]:
"""根据 Authorization header 中的 token 返回当前登录用户信息"""
store = get_platform_store()
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
# token 格式: platform-token-{user_id}
if token.startswith("platform-token-"):
user_id = token[len("platform-token-"):]
for u in store.users():
if u.get("id") == user_id:
return ok(u)
raise fail(401, "invalid or missing token")
@router.get("/dashboard/overview")
@@ -251,6 +292,183 @@ async def dashboard_overview() -> dict[str, Any]:
)
@router.get("/dashboard/stats")
async def dashboard_stats() -> dict[str, Any]:
"""看板聚合数据:基于平台真实数据;缺项做合理近似。"""
store = get_platform_store()
tasks = store.tasks()
users = store.users()
nodes = store.compute_nodes()
datasets = store.datasets()
eval_tasks = store.eval_tasks()
# 数据处理任务总数(来自 data_process 模块)
try:
from app.modules.data_process.store import get_data_process_store
dp_store = get_data_process_store()
dp_result = dp_store.list_tasks(page=1, page_size=1)
dp_count = int(dp_result.get("total", 0))
except Exception:
dp_count = 0
running_statuses = {"syncing", "queued", "running"}
running_ft = [t for t in tasks if t.get("status") in running_statuses]
failed_ft = [t for t in tasks if t.get("status") == "failed"]
all_ft = tasks # 全部训练任务(含已完成/异常)
online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
# 近 7 天训练统计(按创建日期分桶)
now = datetime.now(timezone.utc)
train_by_day: dict[str, int] = {}
for t in tasks:
ct = t.get("create_time")
if ct:
train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
training_7d = []
for i in range(6, -1, -1):
day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
training_7d.append(
{
"date": day[5:],
"train": train_by_day.get(day, 0),
"gpu": sum(len(t.get("gpus") or []) for t in running_ft),
"accuracy": None,
}
)
# 服务状态 —— 与界面实际数据对齐
service_status = [
{
"type": "模型推理",
"status": "error" if (nodes and not online_nodes) else ("busy" if (nodes and len(online_nodes) < len(nodes)) else "normal"),
"count": len(online_nodes),
},
{
"type": "模型训练",
"status": "error" if failed_ft else ("busy" if running_ft else "normal"),
"count": len(all_ft),
},
{
"type": "模型评测",
"status": "normal",
"count": len(eval_tasks),
},
{
"type": "数据处理",
"status": "normal" if not failed_ft else "busy",
"count": dp_count,
},
]
# 训练任务状态归一化
status_map = {
"syncing": "running",
"queued": "running",
"running": "running",
"pending": "pending",
"paused": "pending",
"completed": "completed",
"failed": "failed",
"error": "failed",
"cancelled": "failed",
"stopped": "failed",
}
training_tasks = [
{
"id": t.get("id"),
"name": t.get("name"),
"status": status_map.get(t.get("status"), "pending"),
"train_type": t.get("train_type") or t.get("trainType") or "",
"train_method": t.get("train_method") or t.get("trainMethod") or "",
"base_model": t.get("base_model") or t.get("baseModel") or "",
"progress": t.get("progress", 0),
"accuracy": t.get("accuracy"),
"started_at": (t.get("create_time") or "")[:16],
}
for t in tasks[:8]
]
# 用户操作分布:统计平台全部操作(含治理模块)
MODULE_LABELS = [
("data-process", "数据处理"),
("data_process", "数据处理"),
("dataset", "数据集管理"),
("fine-tune", "模型训练"),
("fine_tune", "模型训练"),
("model-eval", "模型评测"),
("eval", "模型评测"),
("model-inference", "模型推理"),
("inference", "模型推理"),
("model-manage", "模型管理"),
("model", "模型管理"),
("trained", "模型管理"),
# 治理模块操作
("tenant", "租户与项目"),
("project", "租户与项目"),
("approval", "租户与项目"),
("acl", "租户与项目"),
("user", "用户管理"),
("role", "用户管理"),
]
OP_ORDER = [
"数据集管理",
"数据处理",
"模型训练",
"模型评测",
"模型推理",
"模型管理",
"租户与项目",
"用户管理",
]
def _op_module(action: str) -> str | None:
a = (action or "").lower()
for prefix, label in MODULE_LABELS:
if a.startswith(prefix):
return label
return None
audit = store.audit_logs(limit=1000)
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
for log in audit.get("items", []):
label = _op_module(log.get("action") or "")
if label:
op_counter[label] += 1
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
# 最近登录用户
recent = sorted(
[u for u in users if u.get("last_login")],
key=lambda u: u["last_login"],
reverse=True,
)[:5]
recent_login_users = [
{
"user": u.get("display_name") or u.get("username"),
"role": u.get("role"),
"last_login": (u.get("last_login") or "")[:16],
}
for u in recent
]
# 登录时长排行(本月)
login_duration_rank = store.login_duration_rank()
return ok(
{
"online_services": sum(s["count"] for s in service_status),
"running_tasks": len(running_ft),
"pending_alerts": 0,
"training_7d": training_7d,
"service_status": service_status,
"training_tasks": training_tasks,
"operation_distribution": operation_distribution,
"login_duration_rank": login_duration_rank,
"recent_login_users": recent_login_users,
}
)
@router.get("/system-info")
async def system_info() -> dict[str, Any]:
return ok(get_platform_store().system_info())
@@ -285,6 +503,21 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
raise fail(400, str(exc))
@router.post("/users/{user_id}/reset-password")
async def reset_user_password(
user_id: str,
payload: dict[str, Any] = Body(default={}),
) -> dict[str, Any]:
new_password = payload.get("password") or "Platform@123"
try:
get_platform_store().reset_password(user_id, new_password)
return ok({"reset": user_id})
except KeyError:
raise fail(404, "user not found")
except ValueError as exc:
raise fail(400, str(exc))
@router.get("/model-manage/local-models")
async def local_models() -> dict[str, Any]:
store = get_platform_store()
@@ -348,8 +581,13 @@ async def model_by_name(name: str) -> dict[str, Any]:
@router.get("/model-manage")
async def model_list() -> dict[str, Any]:
return ok(get_platform_store().models())
async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
models = get_platform_store().models()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(models)
# 普通用户只返回有 ACL 授权的模型
accessible = set(filter_accessible_resource_ids("model", [m["id"] for m in models], current_user))
return ok([m for m in models if m["id"] in accessible])
@router.post("/model-manage")
@@ -365,11 +603,14 @@ async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/model-manage/{model_id}")
async def model_detail(model_id: str) -> dict[str, Any]:
async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().model(model_id))
model = get_platform_store().model(model_id)
except KeyError:
raise fail(404, "model not found")
if not has_resource_access("model", model_id, current_user, "read"):
raise fail(403, "no permission to access this model")
return ok(model)
@router.put("/model-manage/{model_id}")
@@ -389,7 +630,12 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
@router.delete("/model-manage/{model_id}")
async def delete_model(model_id: str) -> dict[str, Any]:
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this model")
pending = _require_approval_or_admin("model", model_id, current_user, f"删除模型 {model_id}")
if pending:
return pending
get_platform_store().delete_model(model_id)
return ok({"deleted": model_id})
@@ -651,8 +897,12 @@ async def download_dataset_file(dataset_id: str, file_id: str, version_id: str |
@router.get("/dataset-manage")
async def dataset_list() -> dict[str, Any]:
return ok(get_platform_store().datasets())
async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
datasets = get_platform_store().datasets()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(datasets)
accessible = set(filter_accessible_resource_ids("dataset", [d["id"] for d in datasets], current_user))
return ok([d for d in datasets if d["id"] in accessible])
@router.post("/dataset-manage")
@@ -662,11 +912,14 @@ async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/dataset-manage/{dataset_id}")
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().dataset(dataset_id))
dataset = get_platform_store().dataset(dataset_id)
except KeyError:
raise fail(404, "dataset not found")
if not has_resource_access("dataset", dataset_id, current_user, "read"):
raise fail(403, "no permission to access this dataset")
return ok(dataset)
@router.put("/dataset-manage/{dataset_id}")
@@ -678,7 +931,12 @@ async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -
@router.delete("/dataset-manage/{dataset_id}")
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("dataset", dataset_id, current_user, "delete"):
raise fail(403, "no permission to delete this dataset")
pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}")
if pending:
return pending
get_platform_store().delete_dataset(dataset_id)
return ok({"deleted": dataset_id})
@@ -703,8 +961,12 @@ async def tensorboard_start() -> dict[str, Any]:
@router.get("/fine-tune")
async def fine_tune_list() -> dict[str, Any]:
return ok(get_platform_store().tasks())
async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
tasks = get_platform_store().tasks()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(tasks)
accessible = set(filter_accessible_resource_ids("fine-tune", [t["id"] for t in tasks], current_user))
return ok([t for t in tasks if t["id"] in accessible])
@router.post("/fine-tune")
@@ -897,7 +1159,12 @@ async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(de
@router.delete("/fine-tune/{task_id}")
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this task")
pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}")
if pending:
return pending
get_platform_store().delete_task(task_id)
return ok({"deleted": task_id})
@@ -937,16 +1204,23 @@ async def fine_tune_metrics(task_id: str) -> dict[str, Any]:
@router.get("/model-eval")
async def model_eval_list() -> dict[str, Any]:
return ok(get_platform_store().eval_tasks())
async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
tasks = get_platform_store().eval_tasks()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(tasks)
accessible = set(filter_accessible_resource_ids("eval", [t["id"] for t in tasks], current_user))
return ok([t for t in tasks if t["id"] in accessible])
@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str) -> dict[str, Any]:
async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().eval_task(task_id))
task = get_platform_store().eval_task(task_id)
except KeyError:
raise fail(404, "eval task not found")
if not has_resource_access("eval", task_id, current_user, "read"):
raise fail(403, "no permission to access this eval task")
return ok(task)
@router.post("/model-eval/start")
@@ -956,7 +1230,12 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
@router.delete("/model-eval/{task_id}")
async def model_eval_delete(task_id: str) -> dict[str, Any]:
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("eval", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this eval task")
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
if pending:
return pending
get_platform_store().delete_eval_task(task_id)
return ok({"deleted": task_id})

View File

@@ -3,8 +3,20 @@
from app.api.v1.endpoints.data_process import router as data_process_router
from app.api.v1.endpoints.platform import router as platform_router
from app.api.v1.endpoints.health import router as health_router
from app.modules.tenant.router import router as tenant_router
from app.modules.project.router import router as project_router
from app.modules.approval.router import router as approval_router
from app.modules.system.router import router as system_router
from app.modules.retention.router import router as retention_router
from app.modules.resource.router import router as resource_router
api_router = APIRouter()
api_router.include_router(health_router, tags=["health"])
api_router.include_router(data_process_router, tags=["data-process"])
api_router.include_router(platform_router, tags=["platform"])
api_router.include_router(system_router, tags=["system"])
api_router.include_router(tenant_router, tags=["tenant"])
api_router.include_router(project_router, tags=["project"])
api_router.include_router(approval_router, tags=["approval"])
api_router.include_router(retention_router, tags=["retention"])
api_router.include_router(resource_router, tags=["resource"])