update
This commit is contained in:
@@ -34,6 +34,7 @@ from fastapi import (
|
||||
from fastapi.responses import StreamingResponse
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
||||
from app.modules.data_process.algorithms import (
|
||||
ParsedText,
|
||||
canonical_record_json,
|
||||
@@ -815,18 +816,33 @@ def list_tasks(
|
||||
keyword: str | None = Query(default=None),
|
||||
status: DataProcessStatus | None = Query(default=None),
|
||||
process_type: ProcessType | None = Query(default=None),
|
||||
tenant_id: str | None = Query(default=None),
|
||||
project_id: str | None = Query(default=None),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
return ok(
|
||||
store.list_tasks(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
process_type=process_type,
|
||||
)
|
||||
tasks = store.list_tasks(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
process_type=process_type,
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
)
|
||||
# #4 资源 ACL 过滤:admin 放行,普通用户只看到自己被授权的数据处理任务
|
||||
items = tasks.get("items", [])
|
||||
if not is_admin(current_user) and items:
|
||||
accessible_ids = set(
|
||||
filter_accessible_resource_ids(
|
||||
"data-process", [t["id"] for t in items], current_user
|
||||
)
|
||||
)
|
||||
items = [t for t in items if t["id"] in accessible_ids]
|
||||
tasks["items"] = items
|
||||
tasks["total"] = len(items)
|
||||
return ok(tasks)
|
||||
|
||||
|
||||
@router.post("")
|
||||
@@ -2282,13 +2298,11 @@ def regenerate_results_batch(
|
||||
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||
)
|
||||
# httpx.Client 支持跨线程复用,批次内共享连接池可减少重复建连开销。
|
||||
with (
|
||||
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
|
||||
ThreadPoolExecutor(
|
||||
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||
thread_name_prefix="data-result-regeneration",
|
||||
) as executor,
|
||||
):
|
||||
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
|
||||
ThreadPoolExecutor(
|
||||
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||
thread_name_prefix="data-result-regeneration",
|
||||
) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
_regenerate_result_in_place,
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Body, File, HTTPException, Query, UploadFile
|
||||
from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import PlainTextResponse, StreamingResponse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
@@ -35,6 +37,35 @@ def fail(status_code: int, message: str) -> HTTPException:
|
||||
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
||||
|
||||
|
||||
def _require_approval_or_admin(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
current_user: dict[str, Any],
|
||||
action_desc: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""
|
||||
高风险操作审批旁路:
|
||||
- admin 用户直接放行(返回 None)
|
||||
- 普通用户创建审批实例,返回审批待定响应(code=202,非 None)
|
||||
code=202 使前端响应拦截器走业务错误分支,弹提示并 reject,
|
||||
避免前端误认为删除成功。
|
||||
"""
|
||||
if is_admin(current_user):
|
||||
return None
|
||||
store = get_platform_store()
|
||||
instance = store.create_approval_instance({
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"applicant_id": current_user.get("id"),
|
||||
"template_id": None,
|
||||
})
|
||||
return {
|
||||
"code": 202,
|
||||
"message": f"操作已提交审批,等待管理员批准:{action_desc}",
|
||||
"data": {"approval_required": True, "approval_id": instance["id"]},
|
||||
}
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
@@ -231,8 +262,18 @@ async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def me() -> dict[str, Any]:
|
||||
return ok(get_platform_store().users()[0])
|
||||
async def me(request: Request) -> dict[str, Any]:
|
||||
"""根据 Authorization header 中的 token 返回当前登录用户信息"""
|
||||
store = get_platform_store()
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
# token 格式: platform-token-{user_id}
|
||||
if token.startswith("platform-token-"):
|
||||
user_id = token[len("platform-token-"):]
|
||||
for u in store.users():
|
||||
if u.get("id") == user_id:
|
||||
return ok(u)
|
||||
raise fail(401, "invalid or missing token")
|
||||
|
||||
|
||||
@router.get("/dashboard/overview")
|
||||
@@ -251,6 +292,183 @@ async def dashboard_overview() -> dict[str, Any]:
|
||||
)
|
||||
|
||||
|
||||
@router.get("/dashboard/stats")
|
||||
async def dashboard_stats() -> dict[str, Any]:
|
||||
"""看板聚合数据:基于平台真实数据;缺项做合理近似。"""
|
||||
store = get_platform_store()
|
||||
tasks = store.tasks()
|
||||
users = store.users()
|
||||
nodes = store.compute_nodes()
|
||||
datasets = store.datasets()
|
||||
eval_tasks = store.eval_tasks()
|
||||
# 数据处理任务总数(来自 data_process 模块)
|
||||
try:
|
||||
from app.modules.data_process.store import get_data_process_store
|
||||
|
||||
dp_store = get_data_process_store()
|
||||
dp_result = dp_store.list_tasks(page=1, page_size=1)
|
||||
dp_count = int(dp_result.get("total", 0))
|
||||
except Exception:
|
||||
dp_count = 0
|
||||
|
||||
running_statuses = {"syncing", "queued", "running"}
|
||||
running_ft = [t for t in tasks if t.get("status") in running_statuses]
|
||||
failed_ft = [t for t in tasks if t.get("status") == "failed"]
|
||||
all_ft = tasks # 全部训练任务(含已完成/异常)
|
||||
online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
|
||||
|
||||
# 近 7 天训练统计(按创建日期分桶)
|
||||
now = datetime.now(timezone.utc)
|
||||
train_by_day: dict[str, int] = {}
|
||||
for t in tasks:
|
||||
ct = t.get("create_time")
|
||||
if ct:
|
||||
train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
|
||||
training_7d = []
|
||||
for i in range(6, -1, -1):
|
||||
day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
training_7d.append(
|
||||
{
|
||||
"date": day[5:],
|
||||
"train": train_by_day.get(day, 0),
|
||||
"gpu": sum(len(t.get("gpus") or []) for t in running_ft),
|
||||
"accuracy": None,
|
||||
}
|
||||
)
|
||||
|
||||
# 服务状态 —— 与界面实际数据对齐
|
||||
service_status = [
|
||||
{
|
||||
"type": "模型推理",
|
||||
"status": "error" if (nodes and not online_nodes) else ("busy" if (nodes and len(online_nodes) < len(nodes)) else "normal"),
|
||||
"count": len(online_nodes),
|
||||
},
|
||||
{
|
||||
"type": "模型训练",
|
||||
"status": "error" if failed_ft else ("busy" if running_ft else "normal"),
|
||||
"count": len(all_ft),
|
||||
},
|
||||
{
|
||||
"type": "模型评测",
|
||||
"status": "normal",
|
||||
"count": len(eval_tasks),
|
||||
},
|
||||
{
|
||||
"type": "数据处理",
|
||||
"status": "normal" if not failed_ft else "busy",
|
||||
"count": dp_count,
|
||||
},
|
||||
]
|
||||
|
||||
# 训练任务状态归一化
|
||||
status_map = {
|
||||
"syncing": "running",
|
||||
"queued": "running",
|
||||
"running": "running",
|
||||
"pending": "pending",
|
||||
"paused": "pending",
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
"error": "failed",
|
||||
"cancelled": "failed",
|
||||
"stopped": "failed",
|
||||
}
|
||||
training_tasks = [
|
||||
{
|
||||
"id": t.get("id"),
|
||||
"name": t.get("name"),
|
||||
"status": status_map.get(t.get("status"), "pending"),
|
||||
"train_type": t.get("train_type") or t.get("trainType") or "",
|
||||
"train_method": t.get("train_method") or t.get("trainMethod") or "",
|
||||
"base_model": t.get("base_model") or t.get("baseModel") or "",
|
||||
"progress": t.get("progress", 0),
|
||||
"accuracy": t.get("accuracy"),
|
||||
"started_at": (t.get("create_time") or "")[:16],
|
||||
}
|
||||
for t in tasks[:8]
|
||||
]
|
||||
|
||||
# 用户操作分布:统计平台全部操作(含治理模块)
|
||||
MODULE_LABELS = [
|
||||
("data-process", "数据处理"),
|
||||
("data_process", "数据处理"),
|
||||
("dataset", "数据集管理"),
|
||||
("fine-tune", "模型训练"),
|
||||
("fine_tune", "模型训练"),
|
||||
("model-eval", "模型评测"),
|
||||
("eval", "模型评测"),
|
||||
("model-inference", "模型推理"),
|
||||
("inference", "模型推理"),
|
||||
("model-manage", "模型管理"),
|
||||
("model", "模型管理"),
|
||||
("trained", "模型管理"),
|
||||
# 治理模块操作
|
||||
("tenant", "租户与项目"),
|
||||
("project", "租户与项目"),
|
||||
("approval", "租户与项目"),
|
||||
("acl", "租户与项目"),
|
||||
("user", "用户管理"),
|
||||
("role", "用户管理"),
|
||||
]
|
||||
OP_ORDER = [
|
||||
"数据集管理",
|
||||
"数据处理",
|
||||
"模型训练",
|
||||
"模型评测",
|
||||
"模型推理",
|
||||
"模型管理",
|
||||
"租户与项目",
|
||||
"用户管理",
|
||||
]
|
||||
|
||||
def _op_module(action: str) -> str | None:
|
||||
a = (action or "").lower()
|
||||
for prefix, label in MODULE_LABELS:
|
||||
if a.startswith(prefix):
|
||||
return label
|
||||
return None
|
||||
|
||||
audit = store.audit_logs(limit=1000)
|
||||
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
|
||||
for log in audit.get("items", []):
|
||||
label = _op_module(log.get("action") or "")
|
||||
if label:
|
||||
op_counter[label] += 1
|
||||
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
|
||||
|
||||
# 最近登录用户
|
||||
recent = sorted(
|
||||
[u for u in users if u.get("last_login")],
|
||||
key=lambda u: u["last_login"],
|
||||
reverse=True,
|
||||
)[:5]
|
||||
recent_login_users = [
|
||||
{
|
||||
"user": u.get("display_name") or u.get("username"),
|
||||
"role": u.get("role"),
|
||||
"last_login": (u.get("last_login") or "")[:16],
|
||||
}
|
||||
for u in recent
|
||||
]
|
||||
|
||||
# 登录时长排行(本月)
|
||||
login_duration_rank = store.login_duration_rank()
|
||||
|
||||
return ok(
|
||||
{
|
||||
"online_services": sum(s["count"] for s in service_status),
|
||||
"running_tasks": len(running_ft),
|
||||
"pending_alerts": 0,
|
||||
"training_7d": training_7d,
|
||||
"service_status": service_status,
|
||||
"training_tasks": training_tasks,
|
||||
"operation_distribution": operation_distribution,
|
||||
"login_duration_rank": login_duration_rank,
|
||||
"recent_login_users": recent_login_users,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/system-info")
|
||||
async def system_info() -> dict[str, Any]:
|
||||
return ok(get_platform_store().system_info())
|
||||
@@ -285,6 +503,21 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
|
||||
raise fail(400, str(exc))
|
||||
|
||||
|
||||
@router.post("/users/{user_id}/reset-password")
|
||||
async def reset_user_password(
|
||||
user_id: str,
|
||||
payload: dict[str, Any] = Body(default={}),
|
||||
) -> dict[str, Any]:
|
||||
new_password = payload.get("password") or "Platform@123"
|
||||
try:
|
||||
get_platform_store().reset_password(user_id, new_password)
|
||||
return ok({"reset": user_id})
|
||||
except KeyError:
|
||||
raise fail(404, "user not found")
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
|
||||
|
||||
@router.get("/model-manage/local-models")
|
||||
async def local_models() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
@@ -348,8 +581,13 @@ async def model_by_name(name: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/model-manage")
|
||||
async def model_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().models())
|
||||
async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
models = get_platform_store().models()
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return ok(models)
|
||||
# 普通用户只返回有 ACL 授权的模型
|
||||
accessible = set(filter_accessible_resource_ids("model", [m["id"] for m in models], current_user))
|
||||
return ok([m for m in models if m["id"] in accessible])
|
||||
|
||||
|
||||
@router.post("/model-manage")
|
||||
@@ -365,11 +603,14 @@ async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/model-manage/{model_id}")
|
||||
async def model_detail(model_id: str) -> dict[str, Any]:
|
||||
async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().model(model_id))
|
||||
model = get_platform_store().model(model_id)
|
||||
except KeyError:
|
||||
raise fail(404, "model not found")
|
||||
if not has_resource_access("model", model_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this model")
|
||||
return ok(model)
|
||||
|
||||
|
||||
@router.put("/model-manage/{model_id}")
|
||||
@@ -389,7 +630,12 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
|
||||
|
||||
|
||||
@router.delete("/model-manage/{model_id}")
|
||||
async def delete_model(model_id: str) -> dict[str, Any]:
|
||||
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("model", model_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this model")
|
||||
pending = _require_approval_or_admin("model", model_id, current_user, f"删除模型 {model_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_model(model_id)
|
||||
return ok({"deleted": model_id})
|
||||
|
||||
@@ -651,8 +897,12 @@ async def download_dataset_file(dataset_id: str, file_id: str, version_id: str |
|
||||
|
||||
|
||||
@router.get("/dataset-manage")
|
||||
async def dataset_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().datasets())
|
||||
async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
datasets = get_platform_store().datasets()
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return ok(datasets)
|
||||
accessible = set(filter_accessible_resource_ids("dataset", [d["id"] for d in datasets], current_user))
|
||||
return ok([d for d in datasets if d["id"] in accessible])
|
||||
|
||||
|
||||
@router.post("/dataset-manage")
|
||||
@@ -662,11 +912,14 @@ async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/dataset-manage/{dataset_id}")
|
||||
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
|
||||
async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().dataset(dataset_id))
|
||||
dataset = get_platform_store().dataset(dataset_id)
|
||||
except KeyError:
|
||||
raise fail(404, "dataset not found")
|
||||
if not has_resource_access("dataset", dataset_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this dataset")
|
||||
return ok(dataset)
|
||||
|
||||
|
||||
@router.put("/dataset-manage/{dataset_id}")
|
||||
@@ -678,7 +931,12 @@ async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -
|
||||
|
||||
|
||||
@router.delete("/dataset-manage/{dataset_id}")
|
||||
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
|
||||
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("dataset", dataset_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this dataset")
|
||||
pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_dataset(dataset_id)
|
||||
return ok({"deleted": dataset_id})
|
||||
|
||||
@@ -703,8 +961,12 @@ async def tensorboard_start() -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/fine-tune")
|
||||
async def fine_tune_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().tasks())
|
||||
async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
tasks = get_platform_store().tasks()
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return ok(tasks)
|
||||
accessible = set(filter_accessible_resource_ids("fine-tune", [t["id"] for t in tasks], current_user))
|
||||
return ok([t for t in tasks if t["id"] in accessible])
|
||||
|
||||
|
||||
@router.post("/fine-tune")
|
||||
@@ -897,7 +1159,12 @@ async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(de
|
||||
|
||||
|
||||
@router.delete("/fine-tune/{task_id}")
|
||||
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
|
||||
async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("fine-tune", task_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this task")
|
||||
pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
@@ -937,16 +1204,23 @@ async def fine_tune_metrics(task_id: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
@router.get("/model-eval")
|
||||
async def model_eval_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().eval_tasks())
|
||||
async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
tasks = get_platform_store().eval_tasks()
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return ok(tasks)
|
||||
accessible = set(filter_accessible_resource_ids("eval", [t["id"] for t in tasks], current_user))
|
||||
return ok([t for t in tasks if t["id"] in accessible])
|
||||
|
||||
|
||||
@router.get("/model-eval/{task_id}")
|
||||
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
||||
async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().eval_task(task_id))
|
||||
task = get_platform_store().eval_task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "eval task not found")
|
||||
if not has_resource_access("eval", task_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this eval task")
|
||||
return ok(task)
|
||||
|
||||
|
||||
@router.post("/model-eval/start")
|
||||
@@ -956,7 +1230,12 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
|
||||
|
||||
|
||||
@router.delete("/model-eval/{task_id}")
|
||||
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
||||
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("eval", task_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this eval task")
|
||||
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
|
||||
if pending:
|
||||
return pending
|
||||
get_platform_store().delete_eval_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user