feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请 - 存储: MinIO 存储进度迁移、对象存储安全加固与测试 - 计算: GPU 资源预留、compute 轮询与同步增强 - 权限: permission v2 迁移、权限安全验收测试 - 日志: 后端运行日志中文说明、操作日志整合 - 数据处理/评测: 数据转换与模型评测优化 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -6,11 +6,11 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, File, UploadFile
|
||||
from fastapi import APIRouter, Body, Depends, File, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, Response
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.core.auth import get_current_user, is_admin
|
||||
from app.core.auth import get_current_user, has_resource_access, is_admin
|
||||
from app.core.config import get_settings
|
||||
from app.core.op_log import op_log, OpModule, OpAction
|
||||
from app.db.platform_store import get_platform_store, new_id
|
||||
@@ -18,7 +18,26 @@ from app.modules.storage.minio_store import get_object_storage
|
||||
from app.modules.storage.policy import should_store_in_minio
|
||||
|
||||
|
||||
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
|
||||
def _authorize_data_convert_request(
|
||||
request: Request,
|
||||
task_id: str | None = None,
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
) -> None:
|
||||
"""Protect every task-scoped conversion endpoint with resource ACL."""
|
||||
if not task_id or is_admin(current_user):
|
||||
return
|
||||
permission = "read" if request.method in {"GET", "HEAD"} else "write"
|
||||
if request.url.path.endswith("/run") or request.url.path.endswith("/import-as-dataset"):
|
||||
permission = "execute"
|
||||
if not has_resource_access("data_convert", task_id, current_user, permission):
|
||||
raise fail(403, "no permission to access this data convert task")
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/data-convert",
|
||||
tags=["data-convert"],
|
||||
dependencies=[Depends(_authorize_data_convert_request)],
|
||||
)
|
||||
|
||||
# 存储根目录
|
||||
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
|
||||
@@ -207,24 +226,32 @@ def list_tasks(
|
||||
if is_admin(current_user):
|
||||
# 管理员可见全部
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
|
||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
||||
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
|
||||
"FROM data_convert_tasks task "
|
||||
"LEFT JOIN users creator ON creator.id=task.created_by "
|
||||
"LEFT JOIN users processor ON processor.id=task.processed_by "
|
||||
"WHERE task.deleted_at IS NULL "
|
||||
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
|
||||
(page_size, (page - 1) * page_size),
|
||||
).fetchall()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
# 普通用户只能看到自己创建的
|
||||
# 普通用户只能看到本租户且由自己创建的任务；跨租户 ACL 通过任务级依赖访问。
|
||||
user_id = current_user.get("id")
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s "
|
||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
||||
(user_id, page_size, (page - 1) * page_size),
|
||||
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
|
||||
"FROM data_convert_tasks task "
|
||||
"LEFT JOIN users creator ON creator.id=task.created_by "
|
||||
"LEFT JOIN users processor ON processor.id=task.processed_by "
|
||||
"WHERE task.deleted_at IS NULL AND task.tenant_id=%s AND task.created_by=%s "
|
||||
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
|
||||
(current_user.get("tenant_id") or "default", user_id, page_size, (page - 1) * page_size),
|
||||
).fetchall()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s",
|
||||
(user_id,)
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND tenant_id=%s AND created_by=%s",
|
||||
(current_user.get("tenant_id") or "default", user_id,)
|
||||
).fetchone()[0]
|
||||
return ok({"items": [dict(r) for r in rows], "total": total})
|
||||
|
||||
@@ -243,11 +270,16 @@ def create_task(
|
||||
description = str(payload.get("description") or "").strip()
|
||||
user_id = current_user.get("id")
|
||||
store = get_platform_store()
|
||||
tenant_id = current_user.get("tenant_id") or "default"
|
||||
try:
|
||||
store.assert_active_tenant(tenant_id)
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) "
|
||||
"VALUES (%s, %s, %s, %s, %s)",
|
||||
(task_id, name, description, output_filename, user_id),
|
||||
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by, tenant_id) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(task_id, name, description, output_filename, user_id, tenant_id),
|
||||
)
|
||||
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
|
||||
if not _minio_enabled():
|
||||
@@ -318,8 +350,8 @@ async def upload_source_files(
|
||||
# 标记上传完成
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='uploaded', update_time=NOW() WHERE id=%s",
|
||||
(task_id,),
|
||||
"UPDATE data_convert_tasks SET status='uploaded', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(current_user.get("id"), task_id),
|
||||
)
|
||||
# 自动转换并导入数据集
|
||||
try:
|
||||
@@ -332,8 +364,8 @@ async def upload_source_files(
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='completed', "
|
||||
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
|
||||
(input_count, output_count, task_id),
|
||||
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(input_count, output_count, current_user.get("id"), task_id),
|
||||
)
|
||||
# 自动导入数据集
|
||||
content = output.decode("utf-8")
|
||||
@@ -348,6 +380,7 @@ async def upload_source_files(
|
||||
"count": output_count,
|
||||
"description": f"由数据类型转换任务 {task_id} 自动导入",
|
||||
"created_by": task.get("created_by") or current_user.get("id"),
|
||||
"tenant_id": task.get("tenant_id") or current_user.get("tenant_id") or "default",
|
||||
})
|
||||
dataset_id = dataset["id"]
|
||||
with store.connect() as conn:
|
||||
@@ -375,8 +408,8 @@ async def upload_source_files(
|
||||
except Exception as exc:
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
|
||||
(str(exc)[:500], task_id),
|
||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(str(exc)[:500], current_user.get("id"), task_id),
|
||||
)
|
||||
return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]})
|
||||
|
||||
@@ -396,8 +429,8 @@ def run_convert(
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='running', error_message='', update_time=NOW() WHERE id=%s",
|
||||
(task_id,),
|
||||
"UPDATE data_convert_tasks SET status='running', error_message='', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(current_user.get("id"), task_id),
|
||||
)
|
||||
try:
|
||||
if _minio_enabled():
|
||||
@@ -408,14 +441,14 @@ def run_convert(
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='completed', "
|
||||
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
|
||||
(input_count, output_count, task_id),
|
||||
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(input_count, output_count, current_user.get("id"), task_id),
|
||||
)
|
||||
except Exception as exc:
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
|
||||
(str(exc)[:500], task_id),
|
||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||
(str(exc)[:500], current_user.get("id"), task_id),
|
||||
)
|
||||
raise fail(500, f"convert failed: {exc}")
|
||||
return ok(_get_task(task_id))
|
||||
|
||||
Reference in New Issue
Block a user