修改用户设置的新增用户的权限点击操作
This commit is contained in:
@@ -16,6 +16,7 @@ import httpx
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, filter_accessible_resource_ids_batch, get_current_user, has_resource_access, is_admin
|
||||
from app.core.config import get_settings
|
||||
from app.core.audit import audit_log, AuditActions
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
|
||||
@@ -884,6 +885,11 @@ async def test_online_model(payload: dict[str, Any] = Body(...)) -> dict[str, An
|
||||
|
||||
|
||||
@router.post("/model-manage")
|
||||
@audit_log(
|
||||
action=AuditActions.CREATE_MODEL,
|
||||
target_type="model",
|
||||
detail_template="创建模型: {name}",
|
||||
)
|
||||
async def create_model(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
try:
|
||||
@@ -908,6 +914,11 @@ async def model_detail(model_id: str, current_user: dict = Depends(get_current_u
|
||||
|
||||
|
||||
@router.put("/model-manage/{model_id}")
|
||||
@audit_log(
|
||||
action=AuditActions.UPDATE_MODEL,
|
||||
target_type="model",
|
||||
detail_template="更新模型: {model_id}",
|
||||
)
|
||||
async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_model(model_id, payload))
|
||||
@@ -1266,6 +1277,11 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s
|
||||
|
||||
|
||||
@router.post("/dataset-manage")
|
||||
@audit_log(
|
||||
action=AuditActions.CREATE_DATASET,
|
||||
target_type="dataset",
|
||||
detail_template="创建数据集: {name}",
|
||||
)
|
||||
async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
dataset = get_platform_store().create_dataset(payload)
|
||||
@@ -1284,6 +1300,11 @@ async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_curre
|
||||
|
||||
|
||||
@router.put("/dataset-manage/{dataset_id}")
|
||||
@audit_log(
|
||||
action=AuditActions.UPDATE_DATASET,
|
||||
target_type="dataset",
|
||||
detail_template="更新数据集: {dataset_id}",
|
||||
)
|
||||
async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_dataset(dataset_id, payload))
|
||||
@@ -1292,6 +1313,11 @@ async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -
|
||||
|
||||
|
||||
@router.delete("/dataset-manage/{dataset_id}")
|
||||
@audit_log(
|
||||
action=AuditActions.DELETE_DATASET,
|
||||
target_type="dataset",
|
||||
detail_template="删除数据集: {dataset_id}",
|
||||
)
|
||||
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("dataset", dataset_id, current_user, "delete"):
|
||||
raise fail(403, "no permission to delete this dataset")
|
||||
@@ -1331,6 +1357,11 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
|
||||
|
||||
|
||||
@router.post("/fine-tune")
|
||||
@audit_log(
|
||||
action=AuditActions.CREATE_FINE_TUNE,
|
||||
target_type="fine_tune",
|
||||
detail_template="创建微调任务: {name}",
|
||||
)
|
||||
async def create_fine_tune(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
payload.setdefault("created_by", current_user.get("id"))
|
||||
if not is_admin(current_user):
|
||||
|
||||
199
backend/app/core/audit.py
Normal file
199
backend/app/core/audit.py
Normal file
@@ -0,0 +1,199 @@
|
||||
"""
|
||||
审计日志装饰器模块
|
||||
|
||||
提供 @audit_log 装饰器,用于自动记录关键业务操作的审计日志。
|
||||
|
||||
使用示例:
|
||||
from app.core.audit import audit_log
|
||||
|
||||
@audit_log(action="create_dataset", target_type="dataset")
|
||||
async def create_dataset(request: Request, ...):
|
||||
...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Optional, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.core.logging import get_logger, request_id_var
|
||||
|
||||
logger = get_logger("app.audit")
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def audit_log(
|
||||
action: str,
|
||||
target_type: str = "",
|
||||
*,
|
||||
detail_template: str = "",
|
||||
extract_target_id: Optional[Callable[[Any], str]] = None,
|
||||
) -> Callable[[F], F]:
|
||||
"""
|
||||
审计日志装饰器
|
||||
|
||||
Args:
|
||||
action: 操作类型,如 create_dataset、update_model 等
|
||||
target_type: 目标资源类型,如 dataset、model 等
|
||||
detail_template: 日志详情模板(支持 format 参数)
|
||||
extract_target_id: 从返回值中提取目标 ID 的函数
|
||||
|
||||
Returns:
|
||||
装饰后的函数
|
||||
"""
|
||||
|
||||
def decorator(func: F) -> F:
|
||||
if asyncio.iscoroutinefunction(func):
|
||||
@functools.wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
started_at = time.perf_counter()
|
||||
trace_id = request_id_var.get("-")
|
||||
try:
|
||||
result = await func(*args, **kwargs)
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
target_id = _extract_target_id(result, kwargs, extract_target_id)
|
||||
detail = _build_detail(detail_template, kwargs)
|
||||
_record_audit(
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
detail=detail,
|
||||
trace_id=trace_id,
|
||||
duration_ms=elapsed_ms,
|
||||
)
|
||||
return result
|
||||
except Exception:
|
||||
logger.error(
|
||||
"审计日志记录失败 action=%s", action, exc_info=True
|
||||
)
|
||||
raise
|
||||
|
||||
return async_wrapper # type: ignore
|
||||
else:
|
||||
@functools.wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
started_at = time.perf_counter()
|
||||
trace_id = request_id_var.get("-")
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
target_id = _extract_target_id(result, kwargs, extract_target_id)
|
||||
detail = _build_detail(detail_template, kwargs)
|
||||
_record_audit(
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
detail=detail,
|
||||
trace_id=trace_id,
|
||||
duration_ms=elapsed_ms,
|
||||
)
|
||||
return result
|
||||
except Exception:
|
||||
logger.error(
|
||||
"审计日志记录失败 action=%s", action, exc_info=True
|
||||
)
|
||||
raise
|
||||
|
||||
return sync_wrapper # type: ignore
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _extract_target_id(
|
||||
result: Any, kwargs: dict, extractor: Optional[Callable[[Any], str]]
|
||||
) -> Optional[str]:
|
||||
"""从返回值或 kwargs 中提取目标 ID"""
|
||||
if extractor:
|
||||
try:
|
||||
return extractor(result)
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(result, dict):
|
||||
return result.get("id")
|
||||
# 尝试从路径参数中提取
|
||||
for key in ("dataset_id", "model_id", "task_id", "resource_id"):
|
||||
val = kwargs.get(key)
|
||||
if val:
|
||||
return str(val)
|
||||
return None
|
||||
|
||||
|
||||
def _build_detail(template: str, kwargs: dict) -> str:
|
||||
"""构建审计详情"""
|
||||
if not template:
|
||||
return ""
|
||||
try:
|
||||
return template.format(**kwargs)
|
||||
except (KeyError, IndexError):
|
||||
return template
|
||||
|
||||
|
||||
def _record_audit(
|
||||
action: str,
|
||||
target_type: str,
|
||||
target_id: Optional[str],
|
||||
detail: str,
|
||||
trace_id: str,
|
||||
duration_ms: float,
|
||||
) -> None:
|
||||
"""通过已有的 record_audit 方法写入审计日志"""
|
||||
try:
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
store = get_platform_store()
|
||||
store.record_audit(
|
||||
action=action,
|
||||
target_type=target_type or None,
|
||||
target_id=target_id,
|
||||
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
|
||||
)
|
||||
except Exception:
|
||||
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
|
||||
|
||||
|
||||
# ==================== 预定义的审计操作常量 ====================
|
||||
|
||||
class AuditActions:
|
||||
"""预定义的审计操作类型"""
|
||||
# 数据集操作
|
||||
CREATE_DATASET = "create_dataset"
|
||||
UPDATE_DATASET = "update_dataset"
|
||||
DELETE_DATASET = "delete_dataset"
|
||||
|
||||
# 模型操作
|
||||
CREATE_MODEL = "create_model"
|
||||
UPDATE_MODEL = "update_model"
|
||||
DELETE_MODEL = "delete_model"
|
||||
|
||||
# 微调任务
|
||||
CREATE_FINE_TUNE = "create_fine_tune"
|
||||
UPDATE_FINE_TUNE = "update_fine_tune"
|
||||
DELETE_FINE_TUNE = "delete_fine_tune"
|
||||
|
||||
# 推理任务
|
||||
CREATE_INFERENCE = "create_inference"
|
||||
UPDATE_INFERENCE = "update_inference"
|
||||
DELETE_INFERENCE = "delete_inference"
|
||||
|
||||
# 用户管理
|
||||
CREATE_USER = "create_user"
|
||||
UPDATE_USER = "update_user"
|
||||
DELETE_USER = "delete_user"
|
||||
|
||||
# 租户管理
|
||||
CREATE_TENANT = "create_tenant"
|
||||
UPDATE_TENANT = "update_tenant"
|
||||
DELETE_TENANT = "delete_tenant"
|
||||
|
||||
# 权限授权
|
||||
GRANT_ACL = "grant_acl"
|
||||
REVOKE_ACL = "revoke_acl"
|
||||
|
||||
# 系统配置
|
||||
UPDATE_CONFIG = "update_config"
|
||||
@@ -8,7 +8,7 @@ from logging import Handler, LogRecord
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
from typing import Any, Callable, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
@@ -17,6 +17,74 @@ from app.core.config import Settings, get_settings
|
||||
|
||||
request_id_var: ContextVar[str] = ContextVar("request_id", default="-")
|
||||
|
||||
# ==================== 敏感数据脱敏规则 ====================
|
||||
|
||||
SENSITIVE_PATTERNS: dict[str, Callable | str] = {
|
||||
"token": "***",
|
||||
"password": "***",
|
||||
"access_token": "***",
|
||||
"refresh_token": "***",
|
||||
"secret_key": "***",
|
||||
"authorization": "***",
|
||||
"bearer": "***",
|
||||
"api_key": "***",
|
||||
"private_key": "***",
|
||||
}
|
||||
|
||||
def mask_value(key: str, value: Any) -> str:
|
||||
"""对单个值进行脱敏处理"""
|
||||
if value is None:
|
||||
return ""
|
||||
str_val = str(value)
|
||||
|
||||
handler = SENSITIVE_PATTERNS.get(key)
|
||||
if callable(handler):
|
||||
return handler(str_val)
|
||||
elif isinstance(handler, str):
|
||||
# 支持正则替换模式,如 r"1\d{3}\d{4}"
|
||||
try:
|
||||
return re.sub(handler, "***", str_val)
|
||||
except re.error:
|
||||
return "***"
|
||||
return handler
|
||||
|
||||
|
||||
def mask_sensitive_dict(data: dict) -> dict:
|
||||
"""递归脱敏字典中的敏感字段"""
|
||||
if not data or not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
result = {}
|
||||
for key, value in data.items():
|
||||
result[key] = mask_value(key, value)
|
||||
return result
|
||||
|
||||
|
||||
def mask_sensitive_string(text: str) -> str:
|
||||
"""从文本中脱敏常见敏感信息"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
patterns = [
|
||||
(r'Bearer\s+[A-Za-z0-9\-._]+', '***'),
|
||||
(r'token\s*[:=]\s*', '***'),
|
||||
(r'password\s*[:=]\s*', '***'),
|
||||
(r'secret[_-]?key\s*[:=]', '***'),
|
||||
(r'api[-_]?key\s*[:=]', '***'),
|
||||
(r'private[_-]?key\s*[:=]', '***'),
|
||||
(r'\d{11}', r'\d{3}\*\d{4}'), # 手机号/身份证
|
||||
(r'1[3-9]\d{9}', r'1\*{3}\*{4}'), # 手机号
|
||||
]
|
||||
|
||||
for pattern, replacement in patterns:
|
||||
try:
|
||||
text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
|
||||
except re.error:
|
||||
pass
|
||||
return text
|
||||
|
||||
|
||||
# ==================== RequestId Filter ====================
|
||||
|
||||
class RequestIdFilter(logging.Filter):
|
||||
def filter(self, record: LogRecord) -> bool:
|
||||
@@ -24,8 +92,30 @@ class RequestIdFilter(logging.Filter):
|
||||
return True
|
||||
|
||||
|
||||
# ==================== Enhanced JSON Formatter ====================
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
"""Format one JSON object per line for ELK/Filebeat collection."""
|
||||
"""
|
||||
增强的 JSON 日志格式化器,支持结构化字段输出。
|
||||
|
||||
输出示例:
|
||||
{
|
||||
"@timestamp": "2026-08-17T18:30:00.123Z",
|
||||
"level": "INFO",
|
||||
"logger": "dataset.router",
|
||||
"message": "数据集创建成功",
|
||||
"module": "dataset.router",
|
||||
"function": "create_dataset",
|
||||
"file": "dataset/router.py",
|
||||
"line": 45,
|
||||
"process": 12345,
|
||||
"thread": "MainThread",
|
||||
"request_id": "req-abc123",
|
||||
"user_id": "u_admin",
|
||||
"client_ip": "192.168.1.100",
|
||||
"extra": {...}
|
||||
}
|
||||
"""
|
||||
|
||||
def format(self, record: LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
@@ -44,13 +134,26 @@ class JsonLogFormatter(logging.Formatter):
|
||||
"thread_name": record.threadName,
|
||||
"request_id": getattr(record, "request_id", "-"),
|
||||
}
|
||||
|
||||
# 从 record 中提取额外字段(通过 extra 参数传入)
|
||||
for attr in ("user_id", "client_ip", "target_type", "target_id",
|
||||
"duration_ms", "status_code", "error"):
|
||||
val = getattr(record, attr, None)
|
||||
if val is not None:
|
||||
payload[attr] = val
|
||||
|
||||
# 处理异常信息
|
||||
if record.exc_info:
|
||||
payload["exception"] = self.formatException(record.exc_info)
|
||||
if record.stack_info:
|
||||
payload["stack"] = self.formatStack(record.stack_info)
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
# ==================== DateSizeRotatingFileHandler ====================
|
||||
# (保持不变,已有实现)
|
||||
|
||||
class DateSizeRotatingFileHandler(Handler):
|
||||
"""Rotate log files by date and size while keeping date in every file name."""
|
||||
|
||||
@@ -160,6 +263,114 @@ class DateSizeRotatingFileHandler(Handler):
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
# ==================== Structured Logger 封装 ====================
|
||||
|
||||
class StructuredLogger:
|
||||
"""
|
||||
结构化日志记录器,提供统一的日志接口。
|
||||
|
||||
使用方式:
|
||||
logger = get_structured_logger('dataset.router')
|
||||
logger.info('创建数据集', dataset_id='ds_123')
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, module: str = ""):
|
||||
self.logger = logging.getLogger(name)
|
||||
self.name = name
|
||||
self.module = module
|
||||
|
||||
@property
|
||||
def trace_id(self) -> str:
|
||||
return request_id_var.get("-")
|
||||
|
||||
def info(self, message: str, **extra: Any) -> None:
|
||||
self._log("INFO", message, **extra)
|
||||
|
||||
def warning(self, message: str, **extra: Any) -> None:
|
||||
self._log("WARNING", message, **extra)
|
||||
|
||||
def error(self, message: str, **extra: Any) -> None:
|
||||
self._log("ERROR", message, **extra)
|
||||
|
||||
def debug(self, message: str, **extra: Any) -> None:
|
||||
self._log("DEBUG", message, **extra)
|
||||
|
||||
def _log(self, level: str, message: str, **extra: Any) -> None:
|
||||
"""统一日志记录方法"""
|
||||
log_entry: dict[str, Any] = {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"level": level,
|
||||
"logger": self.name,
|
||||
"module": self.module,
|
||||
"message": message,
|
||||
"trace_id": self.trace_id,
|
||||
"extra": extra,
|
||||
}
|
||||
self.logger.log(getattr(logging, level, logging.INFO), json.dumps(log_entry, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def get_structured_logger(name: str, module: str = "") -> StructuredLogger:
|
||||
"""获取结构化日志记录器"""
|
||||
return StructuredLogger(name, module)
|
||||
|
||||
|
||||
# ==================== 快捷函数 ====================
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""获取标准 Python logger"""
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
def set_request_id(request_id: str) -> None:
|
||||
"""设置当前请求的追踪 ID"""
|
||||
request_id_var.set(request_id)
|
||||
|
||||
|
||||
def setup_request_logging(app: FastAPI) -> None:
|
||||
"""配置 FastAPI 请求日志中间件"""
|
||||
logger = get_logger("app.access")
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def]
|
||||
request_id = request.headers.get("X-Request-ID") or str(uuid4())
|
||||
token = request_id_var.set(request_id)
|
||||
started_at = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
|
||||
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
|
||||
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
|
||||
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
|
||||
log_method = logger.debug
|
||||
if response.status_code >= 400:
|
||||
log_method = logger.warning
|
||||
log_method(
|
||||
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
response.status_code,
|
||||
elapsed_ms,
|
||||
request.client.host if request.client else "-",
|
||||
)
|
||||
response.headers["X-Request-ID"] = request_id
|
||||
return response
|
||||
except Exception:
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
logger.exception(
|
||||
"request failed method=%s path=%s duration_ms=%.2f client=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
elapsed_ms,
|
||||
request.client.host if request.client else "-",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
request_id_var.reset(token)
|
||||
|
||||
|
||||
# ==================== 配置函数 ====================
|
||||
|
||||
def configure_logging(settings: Settings | None = None) -> None:
|
||||
settings = settings or get_settings()
|
||||
|
||||
@@ -209,57 +420,5 @@ def configure_logging(settings: Settings | None = None) -> None:
|
||||
logger.handlers.clear()
|
||||
logger.propagate = True
|
||||
|
||||
|
||||
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
|
||||
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
def set_request_id(request_id: str) -> None:
|
||||
request_id_var.set(request_id)
|
||||
|
||||
|
||||
def setup_request_logging(app: FastAPI) -> None:
|
||||
logger = get_logger("app.access")
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def]
|
||||
request_id = request.headers.get("X-Request-ID") or str(uuid4())
|
||||
token = request_id_var.set(request_id)
|
||||
started_at = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
# Docker/frontend probes and polling endpoints are intentionally
|
||||
# quiet at INFO; failures remain visible at WARNING/ERROR.
|
||||
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
|
||||
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
|
||||
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
|
||||
log_method = logger.debug
|
||||
if response.status_code >= 400:
|
||||
log_method = logger.warning
|
||||
log_method(
|
||||
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
response.status_code,
|
||||
elapsed_ms,
|
||||
request.client.host if request.client else "-",
|
||||
)
|
||||
response.headers["X-Request-ID"] = request_id
|
||||
return response
|
||||
except Exception:
|
||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||
logger.exception(
|
||||
"request failed method=%s path=%s duration_ms=%.2f client=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
elapsed_ms,
|
||||
request.client.host if request.client else "-",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
request_id_var.reset(token)
|
||||
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
|
||||
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
|
||||
|
||||
@@ -555,6 +555,8 @@ class PlatformStore:
|
||||
extra_path = schema_dir / extra
|
||||
if extra_path.exists():
|
||||
conn.executescript(extra_path.read_text(encoding="utf-8"))
|
||||
# data_convert_tasks 表补充 created_by 字段(用于数据隔离)
|
||||
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
|
||||
|
||||
def _column_names(self, conn: PgConnection, table_name: str) -> set[str]:
|
||||
columns = conn.execute(
|
||||
@@ -1285,7 +1287,13 @@ class PlatformStore:
|
||||
|
||||
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
user_id = new_id("u")
|
||||
permissions = payload.get("permissions") or (ALL_PERMISSIONS if payload.get("role") == "admin" else ["dashboard"])
|
||||
role = payload.get("role", "user")
|
||||
if role == "admin":
|
||||
permissions = ALL_PERMISSIONS
|
||||
else:
|
||||
# 普通用户:默认拥有所有业务权限,仅排除 user-settings 和 compute
|
||||
role = "user"
|
||||
permissions = [p for p in ALL_PERMISSIONS if p not in ("user-settings", "compute")]
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -1298,7 +1306,7 @@ class PlatformStore:
|
||||
payload["username"],
|
||||
hash_password(payload.get("password", "platform123")),
|
||||
payload.get("display_name") or payload["username"],
|
||||
payload.get("role", "viewer"),
|
||||
role,
|
||||
payload.get("status", "active"),
|
||||
json_dumps(permissions),
|
||||
utcnow(),
|
||||
@@ -1318,8 +1326,8 @@ class PlatformStore:
|
||||
# 管理员权限不可更改,必须是全部
|
||||
perms = ALL_PERMISSIONS
|
||||
else:
|
||||
# 非 admin 用户不能拥有 user-settings 权限
|
||||
perms = [p for p in (perms or []) if p != "user-settings"]
|
||||
# 非 admin 用户不能拥有 user-settings 和 compute 权限
|
||||
perms = [p for p in (perms or []) if p not in ("user-settings", "compute")]
|
||||
payload = {**payload, "permissions": perms}
|
||||
values = {
|
||||
"role": payload.get("role", row["role"]),
|
||||
|
||||
@@ -51,10 +51,18 @@ CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
||||
-- 幂等升级 audit_logs 表:新增字段(已存在则跳过)
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS trace_id TEXT;
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_method TEXT;
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_path TEXT;
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS status_code INTEGER;
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS duration_ms REAL;
|
||||
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS extra JSONB;
|
||||
|
||||
-- 索引(已存在则跳过)
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_trace ON audit_logs(trace_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_actor_time ON audit_logs(actor_id, time);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_target ON audit_logs(target_type, target_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
id TEXT PRIMARY KEY,
|
||||
|
||||
@@ -9,7 +9,7 @@ from fastapi import APIRouter, Body, Depends, File, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.core.auth import get_current_user
|
||||
from app.core.auth import get_current_user, is_admin
|
||||
from app.db.platform_store import get_platform_store, new_id
|
||||
|
||||
|
||||
@@ -63,14 +63,28 @@ def list_tasks(
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
|
||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
||||
(page_size, (page - 1) * page_size),
|
||||
).fetchall()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
|
||||
).fetchone()[0]
|
||||
if is_admin(current_user):
|
||||
# 管理员可见全部
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
|
||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
||||
(page_size, (page - 1) * page_size),
|
||||
).fetchall()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
# 普通用户只能看到自己创建的
|
||||
user_id = current_user.get("id")
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s "
|
||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
||||
(user_id, page_size, (page - 1) * page_size),
|
||||
).fetchall()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s",
|
||||
(user_id,)
|
||||
).fetchone()[0]
|
||||
return ok({"items": [dict(r) for r in rows], "total": total})
|
||||
|
||||
|
||||
@@ -85,12 +99,13 @@ def create_task(
|
||||
task_id = new_id("dct")
|
||||
output_filename = _safe_output_filename(payload.get("output_filename"))
|
||||
description = str(payload.get("description") or "").strip()
|
||||
user_id = current_user.get("id")
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO data_convert_tasks (id, name, description, output_filename) "
|
||||
"VALUES (%s, %s, %s, %s)",
|
||||
(task_id, name, description, output_filename),
|
||||
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) "
|
||||
"VALUES (%s, %s, %s, %s, %s)",
|
||||
(task_id, name, description, output_filename, user_id),
|
||||
)
|
||||
# 创建目录
|
||||
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.core.auth import get_current_user, has_resource_access, is_admin
|
||||
from app.core.audit import audit_log, AuditActions
|
||||
|
||||
router = APIRouter(prefix="/resources", tags=["resource"])
|
||||
|
||||
@@ -25,6 +26,11 @@ def get_acl(resource_type: str, resource_id: str, current_user: dict = Depends(g
|
||||
|
||||
|
||||
@router.put("/{resource_type}/{resource_id}/acl")
|
||||
@audit_log(
|
||||
action=AuditActions.GRANT_ACL,
|
||||
target_type="",
|
||||
detail_template="设置资源授权: {resource_type}/{resource_id}",
|
||||
)
|
||||
def set_acl(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
|
||||
412
docs/生产级日志系统方案.md
Normal file
412
docs/生产级日志系统方案.md
Normal file
@@ -0,0 +1,412 @@
|
||||
# 生产级日志系统设计方案
|
||||
|
||||
> 版本:v1.0
|
||||
> 日期:2026-08-17
|
||||
> 状态:待评审
|
||||
|
||||
---
|
||||
|
||||
## 一、现状分析
|
||||
|
||||
### 1.1 当前日志架构
|
||||
|
||||
```
|
||||
┌─────────────┐
|
||||
│ FastAPI │ ← 请求入口
|
||||
└──────┬──────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────┐
|
||||
│ Logging │ ← Python logging 模块
|
||||
│ Middleware │
|
||||
└──────┬──────┘
|
||||
│
|
||||
├──────────────────┬──────────────────┐
|
||||
▼ ▼
|
||||
┌─────────────┐ ┌─────────────┐
|
||||
│ Console │ │ File │ ← 输出目标
|
||||
│ (开发环境) │ │ (JSON格式) │
|
||||
└─────────────┘ └─────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────┐
|
||||
│ audit_logs │ ← 审计日志表
|
||||
│ (PostgreSQL) │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
### 1.2 现有组件
|
||||
|
||||
| 组件 | 文件路径 | 功能 |
|
||||
|------|----------|------|
|
||||
| `logging.py` | `backend/app/core/` | 日志配置、JSON 格式化、按日期/大小轮转 |
|
||||
| `platform_store.py` | `backend/app/db/` | `record_audit()` 审计日志写入 |
|
||||
| `002_governance.sql` | `backend/app/db/sql/` | `audit_logs` 表结构 |
|
||||
|
||||
### 1.3 存在的问题
|
||||
|
||||
| 问题 | 影响 | 严重程度 |
|
||||
|------|------|----------|
|
||||
| **无结构化日志分级** | DEBUG/INFO/WARNING/ERROR 全部混在一起,无法按级别过滤查看 | 🔴 高 |
|
||||
| **无请求链路追踪** | 一个请求从进入到返回经过哪些服务/函数,无法串联 | 🔴 高 |
|
||||
| **审计日志与业务耦合** | 各模块手动调用 `record_audit()`,容易遗漏 | 🟡 中 |
|
||||
| **无敏感数据脱敏** | 用户 token、密码等可能明文记录 | 🔴 高 |
|
||||
| **无日志聚合查询** | 无法按用户/时间范围/操作类型快速检索 | 🟡 中 |
|
||||
| **无告警通知** | 系统异常无法主动推送通知 | 🟡 中 |
|
||||
| **日志文件无归档策略** | 只有简单的过期删除,无压缩归档 | 🟢 低 |
|
||||
|
||||
---
|
||||
|
||||
## 二、设计目标
|
||||
|
||||
### 2.1 核心原则
|
||||
|
||||
1. **结构化** - 日志有固定 schema,便于机器解析和查询
|
||||
2. **可追溯** - 每个请求有唯一 ID,可串联完整调用链路
|
||||
3. **分级输出** - 不同环境输出不同级别,生产环境不输出 DEBUG
|
||||
4. **安全合规** - 敏感数据自动脱敏(token、密码、手机号等)
|
||||
5. **高性能** - 日志写入不影响业务接口性能(异步写入)
|
||||
6. **可观测** - 支持快速检索、统计、告警
|
||||
|
||||
### 2.2 日志分级标准
|
||||
|
||||
| 级别 | 使用场景 | 示例 | 生产环境 |
|
||||
|------|----------|------|:--------:|
|
||||
| **DEBUG** | 开发调试 | 变量值、SQL 语句、完整堆栈 | ❌ 不输出 |
|
||||
| **INFO** | 正常流程记录 | 任务创建成功、用户登录 | ✅ 记录 |
|
||||
| **WARNING** | 可恢复异常 | 重试操作、参数校验失败、资源不足 | ✅ 记录 |
|
||||
| **ERROR** | 需要人工介入 | 数据库连接失败、第三方 API 超时 | ✅ 记录 + 告警 |
|
||||
| **CRITICAL** | 系统不可用 | 磁盘满、主节点宕机 | ✅ 记录 + 立即告警 |
|
||||
|
||||
---
|
||||
|
||||
## 三、技术方案
|
||||
|
||||
### 3.1 整体架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ 应用层 (Application Layer) │
|
||||
├─────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||
│ │ 数据集管理 │ │ 微调训练 │ │ 模型推理 │ │ 用户认证 │ ... │
|
||||
│ └─────┬────┘ └─────┬────┘ └─────┬────┘ └─────┬────┘ │
|
||||
│ │ │ │ │ │
|
||||
│ └────────────┴───────────┴──────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────┐ │
|
||||
│ │ Structured │ ← 结构化日志中间件 │
|
||||
│ │ Logger │ │
|
||||
│ └──────┬───────┘ │
|
||||
│ │ │
|
||||
│ ┌────────────┬────────────┬─────────────┐ │
|
||||
│ ▼ ▼ ▼ │ │
|
||||
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────┐ │
|
||||
│ │ Console │ │ File │ │ 审计DB │ │ 告警 │ │
|
||||
│ │ (开发) │ │ (JSON) │ │ (PG) │ │(可选) │ │
|
||||
│ └──────────┘ └──────────┘ └──────────┘ └────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ 可观测层 (Observability) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ ┌───────────┐ ┌───────────┐ ┌───────────┐ │
|
||||
│ │ Grafana │ │ Kibana │ │ PagerDuty │ ... │
|
||||
│ │ (查询) │ │ (分析) │ │ (告警) │ │
|
||||
│ └───────────┘ └───────────┘ └───────────┘ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 3.2 日志 Schema 设计
|
||||
|
||||
#### 3.2.1 应用日志 (app.log)
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2026-08-17T10:30:00.000Z",
|
||||
"level": "INFO",
|
||||
"trace_id": "req-abc123",
|
||||
"parent_span_id": "span-xyz789", // OpenTelemetry Span
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"path": "/dataset-manage",
|
||||
"client_ip": "192.168.1.100",
|
||||
"user_agent": "Mozilla/5.0...",
|
||||
"user_id": "u_admin"
|
||||
},
|
||||
"module": "dataset.router",
|
||||
"function": "create_dataset",
|
||||
"message": "数据集创建成功",
|
||||
"extra": {
|
||||
"dataset_id": "ds_abc123",
|
||||
"dataset_name": "训练数据"
|
||||
},
|
||||
"duration_ms": 125,
|
||||
"status_code": 200,
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
#### 3.2.2 审计日志 (audit_logs 表)
|
||||
|
||||
```sql
|
||||
-- 已有表结构(保持不变)
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
actor_id TEXT, -- 操作人
|
||||
action TEXT, -- 操作类型: create/delete/update/acl.set/login...
|
||||
target_type TEXT, -- 资源类型: dataset/model/fine-tune/user...
|
||||
target_id TEXT, -- 资源 ID
|
||||
detail TEXT, -- 详细信息 JSON
|
||||
client_ip TEXT, -- 客户端 IP
|
||||
time TEXT, -- 操作时间
|
||||
|
||||
-- 新增字段
|
||||
trace_id TEXT, -- 关联应用日志的请求追踪 ID
|
||||
request_method TEXT, -- HTTP 方法
|
||||
request_path TEXT, -- 请求路径
|
||||
status_code INTEGER, -- 响应状态码
|
||||
duration_ms REAL, -- 耗时(ms)
|
||||
extra JSONB -- 扩展信息
|
||||
);
|
||||
|
||||
-- 新增索引
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_trace ON audit_logs(trace_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_actor_time ON audit_logs(actor_id, time);
|
||||
```
|
||||
|
||||
### 3.3 日志中间件设计
|
||||
|
||||
```python
|
||||
# backend/app/core/logging.py 新增
|
||||
|
||||
class StructuredLogger:
|
||||
"""结构化日志记录器"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.logger = logging.getLogger(name)
|
||||
self.trace_id = context_var.get("trace_id")
|
||||
|
||||
def info(self, msg: str, **kwargs):
|
||||
self._log("INFO", msg, **kwargs)
|
||||
|
||||
def warning(self, msg: str, **kwargs):
|
||||
self._log("WARNING", msg, **kwargs)
|
||||
|
||||
def error(self, msg: str, **kwargs):
|
||||
self._log("ERROR", msg, **kwargs)
|
||||
|
||||
def _log(self, level: str, msg: str,
|
||||
user_id: str = None,
|
||||
target_type: str = None,
|
||||
target_id: str = None,
|
||||
duration_ms: float = None,
|
||||
status_code: int = None,
|
||||
error: Exception = None,
|
||||
**extra):
|
||||
"""统一日志记录方法"""
|
||||
log_entry = {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"level": level,
|
||||
"trace_id": self.trace_id.get(),
|
||||
"request": {
|
||||
"user_id": user_id or current_user_id(),
|
||||
"client_ip": client_ip(),
|
||||
# ...
|
||||
},
|
||||
"module": calling_module,
|
||||
"message": msg,
|
||||
"target": {
|
||||
"type": target_type,
|
||||
"id": target_id,
|
||||
},
|
||||
"extra": extra,
|
||||
"duration_ms": duration_ms,
|
||||
"error": format_exception(error) if error else None,
|
||||
}
|
||||
|
||||
# 1. 写入控制台/文件
|
||||
self.logger.log(level, json.dumps(log_entry))
|
||||
|
||||
# 2. 异步写入审计表(如果需要)
|
||||
if level in ("WARNING", "ERROR", "CRITICAL"):
|
||||
async_write_audit(log_entry)
|
||||
```
|
||||
|
||||
### 3.4 装饰器模式(推荐)
|
||||
|
||||
使用 Python 裁饰器自动记录,避免手动调用:
|
||||
|
||||
```python
|
||||
# backend/app/core/log_decorator.py
|
||||
|
||||
def audit_log(action: str, target_type: str = ""):
|
||||
"""审计日志装饰器"""
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
result = await func(*args, **kwargs)
|
||||
|
||||
# 自动记录审计日志
|
||||
record_audit(
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=kwargs.get('id') or result.get('id'),
|
||||
detail=f"params={kwargs}"
|
||||
)
|
||||
return result
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
# 使用示例
|
||||
@audit_log("dataset.create", "dataset")
|
||||
async def create_dataset(...):
|
||||
# 业务逻辑
|
||||
pass
|
||||
```
|
||||
|
||||
### 3.5 敏感数据脱敏规则
|
||||
|
||||
```python
|
||||
# backend/app/core/masking.py
|
||||
|
||||
SENSITIVE_FIELDS = {
|
||||
"token": "***",
|
||||
"password": "***",
|
||||
"phone": lambda x: f"{x[:3]}****{x[-4:]}",
|
||||
"email": lambda x: x[0] + "***" + x.split("@")[1] if "@" in x else "***",
|
||||
"id_card": lambda x: f"{x[:6]}********{x[-4:]}",
|
||||
}
|
||||
|
||||
def mask_sensitive(data: dict) -> dict:
|
||||
"""递归脱敏字典中的敏感字段"""
|
||||
for key, value in data.items():
|
||||
if key in SENSITIVE_FIELDS:
|
||||
data[key] = SENSITIVE_FIELDS[key](value) if callable(SENSITIVE_FIELDS[key]) else "***"
|
||||
elif isinstance(value, dict):
|
||||
mask_sensitive(value)
|
||||
return data
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 四、实施计划
|
||||
|
||||
### 4.1 Phase 1:基础增强(1-2 天)
|
||||
|
||||
- [ ] **P1-1** 升级 `JsonLogFormatter`,增加 `trace_id` 字段
|
||||
- [ ] **P1-2** 新增 `StructuredLogger` 封装类
|
||||
- [ ] **P1-3** 统一所有模块的日志格式为 JSON
|
||||
- [ ] **P1-4** 实现 `mask_sensitive()` 脱敏函数
|
||||
- [ ] **P1-5** 审计日志表新增 `trace_id`、`duration_ms` 字段
|
||||
|
||||
### 4.2 Phase 2:自动化(2-3 天)
|
||||
|
||||
- [ ] **P2-1** 编写 `@audit_log` 装饰器
|
||||
- [ ] **P2-2** 为关键业务接口添加装饰器:
|
||||
- 数据集 CRUD
|
||||
- 模型 CRUD
|
||||
- 微调任务创建/删除
|
||||
- 用户登录/登出
|
||||
- ACL 授权变更
|
||||
- [ ] **P2-3** 实现日志异步写入队列(避免影响性能)
|
||||
|
||||
### 4.3 Phase 3:可观测性(3-5 天)
|
||||
|
||||
- [ ] **P3-1** 集成 ELK Stack 或 Loki(可选)
|
||||
- [ ] **P3-2** 编写 Grafana 仪表板:
|
||||
- 请求量趋势图
|
||||
- 错误率统计
|
||||
- 慢接口 TOP10
|
||||
- 用户操作审计面板
|
||||
- [ ] [ ] **P3-3** 实现告警规则(错误率超阈值触发)
|
||||
|
||||
---
|
||||
|
||||
## 五、配置示例
|
||||
|
||||
### 5.1 日志配置 (settings)
|
||||
|
||||
```yaml
|
||||
# config.yaml 或 .env
|
||||
LOGGING:
|
||||
level: INFO # 生产环境用 INFO,开发用 DEBUG
|
||||
dir: ./logs
|
||||
file_prefix: app
|
||||
max_bytes: 50MB # 单文件最大 50MB
|
||||
retention_days: 30 # 保留 30 天
|
||||
error_prefix: error # 错误日志单独文件
|
||||
json: true # JSON 格式输出
|
||||
|
||||
AUDIT:
|
||||
enabled: true
|
||||
auto_record: true # 是否自动记录(通过装饰器)
|
||||
sensitive_mask: true # 启用敏感数据脱敏
|
||||
```
|
||||
|
||||
### 5.2 日志输出示例
|
||||
|
||||
**控制台输出(开发环境):**
|
||||
```
|
||||
2026-08-17 18:30:00.123 | INFO | pid=12345 | MainThread | req=req-abc | dataset.router:create_dataset | dataset/router.py:45 | 数据集创建成功 {"dataset_id":"ds_abc"}
|
||||
```
|
||||
|
||||
**文件输出(JSON 格式):**
|
||||
```json
|
||||
{"@timestamp":"2026-08-17T18:30:00.123Z","level":"INFO","logger":"dataset.router","message":"数据集创建成功","module":"dataset.router","function":"create_dataset","file":"dataset/router.py","line":45,"process":12345,"thread":"MainThread","request_id":"req-abc","extra":{"dataset_id":"ds_abc"}}
|
||||
```
|
||||
|
||||
**审计日志查询 SQL:**
|
||||
```sql
|
||||
-- 查询某用户最近7天的所有操作
|
||||
SELECT time, action, target_type, target_id, detail, client_ip
|
||||
FROM audit_logs
|
||||
WHERE actor_id = 'u_admin'
|
||||
AND time >= now() - interval '7 days'
|
||||
ORDER BY time DESC;
|
||||
|
||||
-- 查询某资源的授权变更历史
|
||||
SELECT * FROM audit_logs
|
||||
WHERE action LIKE '%acl%'
|
||||
AND target_id = 'ds_abc123'
|
||||
ORDER BY time DESC;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 六、附录
|
||||
|
||||
### A. 日志关键字段说明
|
||||
|
||||
| 字段 | 类型 | 说明 | 示例 |
|
||||
|------|------|------|------|
|
||||
| `trace_id` | string | 请求唯一标识,用于串联一次请求的所有日志 | `req-uuid-1234` |
|
||||
| `parent_span_id` | string | 父 Span ID(用于分布式追踪) | `span-parent-5678` |
|
||||
| `actor_id` | string | 操作人用户 ID | `u_admin` |
|
||||
| `action` | string | 操作动作 | `dataset.create`, `model.delete`, `login.success` |
|
||||
| `target_type` | string | 操作的资源类型 | `dataset`, `trained_model`, `user` |
|
||||
| `target_id` | string | 资源 ID | `ds_abc123` |
|
||||
| `detail` | string/json | 操作详情 | `{"name": "训练数据", "type": "train"}` |
|
||||
| `client_ip` | string | 客户端 IP | `192.168.1.100` |
|
||||
| `duration_ms` | real | 接口耗时(ms) | `125.5` |
|
||||
| `status_code` | int | HTTP 状态码 | `200`, `404`, `500` |
|
||||
|
||||
### B. 推荐的 Python 日志库对比
|
||||
|
||||
| 库 | 特点 | 适用场景 |
|
||||
|-----|------|---------|
|
||||
| `structlog` | 结构化日志,高性能 | 推荐 ✅ |
|
||||
| `loguru` | 简单易用,自动配置 | 小型项目 |
|
||||
| `logging` | Python 标准库 | 当前已使用 |
|
||||
|
||||
### C. 参考链接
|
||||
|
||||
- [Python logging cookbook](https://docs.python.org/3/howto/logging.html)
|
||||
- [ELK Stack 官方文档](https://www.elastic.co/guide/index.html)
|
||||
- [OpenTelemetry 规范](https://opentelemetry.io/docs/)
|
||||
@@ -1,22 +0,0 @@
|
||||
4.2.3 评估工作台
|
||||
负责训练后模型的质量评测和问题诊断。这是平台闭环的核心环节。
|
||||
|
||||
|
||||
评估流程图
|
||||
1.评估方式
|
||||
评估默认使用本地部署的 LLM 作为评审模型,不依赖外部 API。评审模型对每条测试数据从四个子维度打分:核心事实正确性、信息完整性、无幻觉、格式合规性(可选)。汇总为三档判定:正确、部分正确、错误。
|
||||
用户可以选择额外启动人工复核——平台按错误类型分层抽样建议五十到两百条题目,用户在界面上逐条确认或修改 LLM 评审的判定。人工复核的结果用于校准 LLM 评审——一致率超过百分之八十时 LLM 评审结果标记为"可信",低于百分之六十时标记为"以人工为准"。
|
||||
平台同时提供自动指标作为参考——如关键字段匹配率。这些指标不单独作为判定依据,仅作为快速参考。
|
||||
|
||||
2.错误诊断
|
||||
评审模型在完成评分后,额外输出一个错误分类标签。标签从五种固定类型中选择:混淆(模型回答的值像是另一个实体的属性值)、不完整(事实正确但缺少部分信息)、格式偏差(语义正确但措辞与预期不符)、幻觉(回答中存在标准答案没有的内容)、其他(不属于以上任何类型)。
|
||||
这种分类方式简单可落地——它是一个固定枚举的分类任务,评审模型的prompt 中已包含每种类型的定义和判别示例,不需要额外的自然语言聚类或机器学习算法。
|
||||
|
||||
3.评估报告
|
||||
评估完成后自动生成报告,分为四个部分:总览面板:整体得分和各维度通过率。如果做了人工复核,展示 LLM 评审和人工判定的一致率及可信度标记。
|
||||
错误分类面板:按五种分类标签分组的错误列表,每组展示数量和占比。点击展开可查看具体错误样例(问题、标准答案、模型回答、评审模型的原因描述)。
|
||||
修复建议面板:根据错误分类的统计分布,自动生成方向性建议。如"混淆"类错误占比最高时,建议检查训练数据中指令相似但答案不同的样本对;"不完整"类占比最高时,建议统一同类问题的答案详略标准。某类错误的绝对数量不足百分之三时不单独给建议,"其他"类占比最高时提示用户人工分析错误样例。
|
||||
迭代对比面板:如果存在上轮评估记录,展示两轮各分类标签的数量变化。标注"混淆类错误从35条降到14条,下降60%,修复可能生效"。
|
||||
|
||||
4.建议有效性的验证
|
||||
平台在每次评估完成后将分类标签的统计数据(每种标签的数量和占比)存入评估记录。下一轮评估的对应数据与之对比,计算差值和变化百分比。某类错误数量下降超过百分之二十,标注"调整可能生效";变化不足百分之十,标注"调整可能未生效或生效不显著";反向上升则标注"建议检查本次调整方向是否正确"。
|
||||
@@ -2,7 +2,7 @@
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import { Plus, Delete, Refresh } from '@element-plus/icons-vue'
|
||||
import type { TagProps, UploadRequestOptions } from 'element-plus'
|
||||
import type { TagProps, UploadRequestOptions, UploadFile } from 'element-plus'
|
||||
import PageCard from '@/components/PageCard.vue'
|
||||
import {
|
||||
getDataConvertTasks,
|
||||
@@ -16,6 +16,8 @@ const loading = ref(false)
|
||||
const tasks = ref<DataConvertTask[]>([])
|
||||
const showCreate = ref(false)
|
||||
const form = ref({ name: '', outputName: 'converted-data' })
|
||||
const fileList = ref<UploadFile[]>([])
|
||||
const creating = ref(false)
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
@@ -27,21 +29,91 @@ async function load() {
|
||||
}
|
||||
}
|
||||
|
||||
// 文件上传前的校验(仅校验文件格式)
|
||||
function beforeUpload(file: UploadFile) {
|
||||
// 检查文件类型
|
||||
const isJson = file.name.endsWith('.json') || file.raw?.type === 'application/json'
|
||||
if (!isJson) {
|
||||
ElMessage.error('只能上传 .json 格式的文件')
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// 手动点击"创建并上传"
|
||||
async function submitCreate() {
|
||||
if (!form.value.name) {
|
||||
// 校验任务名称
|
||||
if (!form.value.name || !form.value.name.trim()) {
|
||||
ElMessage.warning('请填写任务名称')
|
||||
return
|
||||
}
|
||||
await createDataConvertTask({
|
||||
name: form.value.name,
|
||||
output_filename: form.value.outputName + '.jsonl',
|
||||
})
|
||||
ElMessage.success('任务创建成功')
|
||||
showCreate.value = false
|
||||
form.value = { name: '', outputName: 'converted-data' }
|
||||
load()
|
||||
// 检查名称是否重复
|
||||
const exists = tasks.value.some((t) => t.name === form.value.name.trim())
|
||||
if (exists) {
|
||||
ElMessage.error(`数据集管理中已存在名为「${form.value.name}」的任务,请换一个名称`)
|
||||
return
|
||||
}
|
||||
// 检查是否选择了文件
|
||||
if (!fileList.value || fileList.value.length === 0) {
|
||||
ElMessage.warning('请选择要上传的 JSON 文件')
|
||||
return
|
||||
}
|
||||
|
||||
creating.value = true
|
||||
try {
|
||||
// 1. 创建转换任务
|
||||
const task = await createDataConvertTask({
|
||||
name: form.value.name.trim(),
|
||||
output_filename: form.value.outputName.trim() + '.jsonl',
|
||||
})
|
||||
|
||||
// 2. 获取任务 ID
|
||||
const taskId = (task as any)?.id || task?.id || (task as any)?.data?.id
|
||||
if (!taskId) {
|
||||
throw new Error('创建任务失败,服务端未返回任务 ID')
|
||||
}
|
||||
|
||||
// 3. 上传文件到刚创建的任务
|
||||
const file = fileList.value[0].raw
|
||||
if (!file) {
|
||||
throw new Error('文件信息丢失,请重新选择文件')
|
||||
}
|
||||
|
||||
const res = await uploadSourceFiles(taskId, [file])
|
||||
const data = (res as any)?.data || res
|
||||
|
||||
if (data?.auto_converted) {
|
||||
ElMessage.success(
|
||||
`创建成功!文件已上传并自动转换完成(输入 ${data.input_count} 条 / 输出 ${data.output_count} 条),结果已导入数据集`,
|
||||
)
|
||||
} else if (data?.error) {
|
||||
ElMessage.error(`文件上传成功但转换失败:${data.error}`)
|
||||
} else {
|
||||
ElMessage.warning('文件已上传,等待后台转换处理...')
|
||||
}
|
||||
|
||||
// 关闭弹窗并刷新列表
|
||||
showCreate.value = false
|
||||
resetForm()
|
||||
load()
|
||||
} catch (e: any) {
|
||||
// 根据错误类型给出更清晰的提示
|
||||
const msg = e?.message || e?.toString() || '未知错误'
|
||||
if (msg.includes('409') || msg.includes('conflict') || msg.includes('已存在') || msg.includes('duplicate')) {
|
||||
ElMessage.error(`数据集管理中已存在名为「${form.value.name}」的任务,请换一个名称`)
|
||||
} else if (msg.includes('400')) {
|
||||
ElMessage.error(`参数错误:${msg}`)
|
||||
} else if (msg.includes('403') || msg.includes('权限')) {
|
||||
ElMessage.error(`没有权限执行此操作:${msg}`)
|
||||
} else {
|
||||
ElMessage.error(`操作失败:${msg}`)
|
||||
}
|
||||
} finally {
|
||||
creating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 自定义上传(表格中的上传按钮仍使用此方法)
|
||||
async function customUpload(options: UploadRequestOptions) {
|
||||
const taskId = options.data?.taskId as string
|
||||
if (!taskId) {
|
||||
@@ -59,11 +131,22 @@ async function customUpload(options: UploadRequestOptions) {
|
||||
ElMessage.warning('上传完成,但转换失败:' + (data?.error || '未知错误'))
|
||||
}
|
||||
load()
|
||||
} catch {
|
||||
ElMessage.error('上传失败')
|
||||
} catch (e: any) {
|
||||
ElMessage.error('上传失败:' + (e?.message || '未知错误'))
|
||||
}
|
||||
}
|
||||
|
||||
// 重置表单
|
||||
function resetForm() {
|
||||
form.value = { name: '', outputName: 'converted-data' }
|
||||
fileList.value = []
|
||||
}
|
||||
|
||||
// 移除已选文件
|
||||
function handleRemoveFile(file: UploadFile) {
|
||||
fileList.value = fileList.value.filter((f) => f.uid !== file.uid)
|
||||
}
|
||||
|
||||
async function handleDelete(task: DataConvertTask) {
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
@@ -136,21 +219,42 @@ onMounted(load)
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
|
||||
<!-- 新建任务弹窗 -->
|
||||
<el-dialog v-model="showCreate" title="新建转换任务" width="480px">
|
||||
<!-- 新建任务弹窗(一步到位:填写信息 + 上传文件) -->
|
||||
<el-dialog v-model="showCreate" title="新建转换任务" width="520px" :close-on-click-modal="false">
|
||||
<el-form label-width="100px">
|
||||
<el-form-item label="任务名称" required>
|
||||
<el-input v-model="form.name" placeholder="请输入任务名称" />
|
||||
<el-input v-model="form.name" placeholder="请输入任务名称" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="输出文件名">
|
||||
<el-input v-model="form.outputName" placeholder="converted-data">
|
||||
<el-input v-model="form.outputName" placeholder="converted-data" clearable>
|
||||
<template #append>.jsonl</template>
|
||||
</el-input>
|
||||
</el-form-item>
|
||||
<el-form-item label="上传文件" required>
|
||||
<el-upload
|
||||
ref="uploadRef"
|
||||
v-model:file-list="fileList"
|
||||
:auto-upload="false"
|
||||
:limit="1"
|
||||
accept=".json"
|
||||
:before-upload="beforeUpload"
|
||||
:on-remove="handleRemoveFile"
|
||||
:disabled="creating"
|
||||
drag
|
||||
>
|
||||
<el-icon class="el-icon--upload"><Plus /></el-icon>
|
||||
<div class="el-upload__text">将 JSON 文件拖到此处,或<em>点击上传</em></div>
|
||||
<template #tip>
|
||||
<div class="el-upload__tip">仅支持 .json 格式文件,点击"创建并上传"按钮后自动创建任务并上传文件</div>
|
||||
</template>
|
||||
</el-upload>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="showCreate = false">取消</el-button>
|
||||
<el-button type="primary" @click="submitCreate">创建</el-button>
|
||||
<el-button @click="showCreate = false" :disabled="creating">取消</el-button>
|
||||
<el-button type="primary" :loading="creating" :disabled="!form.name || fileList.length === 0" @click="submitCreate">
|
||||
{{ creating ? '处理中...' : '创建并上传' }}
|
||||
</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</PageCard>
|
||||
@@ -162,4 +266,10 @@ onMounted(load)
|
||||
gap: 10px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
:deep(.el-upload-dragger) {
|
||||
width: 100%;
|
||||
.el-upload__text {
|
||||
padding: 20px 0;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -3,7 +3,7 @@ import { reactive, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { createUser } from '@/api/modules/system'
|
||||
import type { CreateUserPayload, PermissionCode } from '@/types'
|
||||
import type { CreateUserPayload } from '@/types'
|
||||
|
||||
const router = useRouter()
|
||||
const submitting = ref(false)
|
||||
@@ -12,26 +12,11 @@ const form = reactive<CreateUserPayload>({
|
||||
username: '',
|
||||
display_name: '',
|
||||
password: 'platform123',
|
||||
role: 'viewer',
|
||||
role: 'user',
|
||||
status: 'active',
|
||||
permissions: ['dashboard'],
|
||||
permissions: [],
|
||||
})
|
||||
|
||||
const permissionOptions: PermissionCode[] = [
|
||||
'dashboard',
|
||||
'fine-tune',
|
||||
'model-eval',
|
||||
'model-inference',
|
||||
'model-manage',
|
||||
'dataset',
|
||||
'data-process',
|
||||
'data-convert',
|
||||
'compute',
|
||||
'hardware',
|
||||
'logs',
|
||||
'user-settings',
|
||||
]
|
||||
|
||||
async function submit() {
|
||||
submitting.value = true
|
||||
try {
|
||||
@@ -49,19 +34,18 @@ async function submit() {
|
||||
<h1>创建用户</h1>
|
||||
<el-form :model="form" label-width="110px" class="user-form">
|
||||
<el-form-item label="账号">
|
||||
<el-input v-model="form.username" />
|
||||
<el-input v-model="form.username" placeholder="登录用户名" />
|
||||
</el-form-item>
|
||||
<el-form-item label="显示名称">
|
||||
<el-input v-model="form.display_name" />
|
||||
<el-input v-model="form.display_name" placeholder="如:张三" />
|
||||
</el-form-item>
|
||||
<el-form-item label="初始密码">
|
||||
<el-input v-model="form.password" type="password" show-password />
|
||||
</el-form-item>
|
||||
<el-form-item label="角色">
|
||||
<el-select v-model="form.role">
|
||||
<el-select v-model="form.role" style="width: 100%">
|
||||
<el-option label="管理员" value="admin" />
|
||||
<el-option label="操作员" value="operator" />
|
||||
<el-option label="观察员" value="viewer" />
|
||||
<el-option label="普通用户" value="user" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="状态">
|
||||
@@ -70,10 +54,13 @@ async function submit() {
|
||||
<el-radio value="disabled">禁用</el-radio>
|
||||
</el-radio-group>
|
||||
</el-form-item>
|
||||
<el-form-item label="页面权限">
|
||||
<el-checkbox-group v-model="form.permissions">
|
||||
<el-checkbox v-for="item in permissionOptions" :key="item" :value="item">{{ item }}</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
<el-form-item label="权限说明">
|
||||
<el-alert type="info" :closable="false" show-icon>
|
||||
<template #title>
|
||||
<span v-if="form.role === 'admin'">管理员:拥有全部权限,包括用户管理、平台治理、算力节点</span>
|
||||
<span v-else>普通用户:可见服务看板、模型服务、数据治理、其他工具、平台性能、查看日志。数据集和微调模型仅创建者和被授权用户可见。</span>
|
||||
</template>
|
||||
</el-alert>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-button @click="router.back()">返回</el-button>
|
||||
@@ -89,6 +76,6 @@ async function submit() {
|
||||
}
|
||||
|
||||
.user-form {
|
||||
max-width: 760px;
|
||||
max-width: 640px;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import { onMounted, reactive, ref } from 'vue'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import {
|
||||
changeMyPassword,
|
||||
deleteUser,
|
||||
getUsers,
|
||||
resetUserPassword,
|
||||
@@ -81,39 +80,11 @@ async function confirmResetPwd() {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 用户自改密码 ----------
|
||||
const myPwdDialog = reactive({ visible: false, oldPassword: '', newPassword: '', saving: false })
|
||||
function openChangeMyPwd() {
|
||||
myPwdDialog.oldPassword = ''
|
||||
myPwdDialog.newPassword = ''
|
||||
myPwdDialog.visible = true
|
||||
}
|
||||
async function confirmChangeMyPwd() {
|
||||
if (!myPwdDialog.oldPassword.trim() || !myPwdDialog.newPassword.trim()) {
|
||||
ElMessage.warning('请填写旧密码和新密码')
|
||||
return
|
||||
}
|
||||
if (myPwdDialog.newPassword.length < 6) {
|
||||
ElMessage.warning('新密码至少 6 位')
|
||||
return
|
||||
}
|
||||
myPwdDialog.saving = true
|
||||
try {
|
||||
await changeMyPassword(myPwdDialog.oldPassword.trim(), myPwdDialog.newPassword.trim())
|
||||
ElMessage.success('密码修改成功')
|
||||
myPwdDialog.visible = false
|
||||
} catch {
|
||||
ElMessage.error('密码修改失败,请检查旧密码是否正确')
|
||||
} finally {
|
||||
myPwdDialog.saving = false
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 删除 ----------
|
||||
async function removeUser(row: SystemUser) {
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
`确定删除用户 “${row.display_name}(${row.username})” 吗?该操作不可恢复。`,
|
||||
`确定删除用户 "${row.display_name}(${row.username})" 吗?该操作不可恢复。`,
|
||||
'删除用户',
|
||||
{ type: 'warning', confirmButtonText: '删除', cancelButtonText: '取消' },
|
||||
)
|
||||
@@ -139,7 +110,6 @@ async function removeUser(row: SystemUser) {
|
||||
<p>管理平台账号、角色状态与登录密码。</p>
|
||||
</div>
|
||||
<div>
|
||||
<el-button @click="openChangeMyPwd">修改密码</el-button>
|
||||
<el-button type="primary" @click="$router.push('/user-settings/create')">创建用户</el-button>
|
||||
</div>
|
||||
</header>
|
||||
@@ -147,7 +117,13 @@ async function removeUser(row: SystemUser) {
|
||||
<el-table :data="users" border>
|
||||
<el-table-column prop="username" label="账号" min-width="140" />
|
||||
<el-table-column prop="display_name" label="显示名称" min-width="160" />
|
||||
<el-table-column prop="role" label="角色" width="120" />
|
||||
<el-table-column prop="role" label="角色" width="120">
|
||||
<template #default="{ row }">
|
||||
<el-tag :type="asSystemUser(row).role === 'admin' ? 'danger' : 'info'" size="small">
|
||||
{{ asSystemUser(row).role === 'admin' ? '管理员' : '普通用户' }}
|
||||
</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="状态" width="130">
|
||||
<template #default="{ row }">
|
||||
<el-tag :type="statusTagType(asSystemUser(row).status)" size="small">{{ statusLabel(asSystemUser(row).status) }}</el-tag>
|
||||
@@ -189,22 +165,6 @@ async function removeUser(row: SystemUser) {
|
||||
<el-button type="primary" :loading="pwdDialog.saving" @click="confirmResetPwd">确定重置</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
|
||||
<!-- 修改自己的密码 -->
|
||||
<el-dialog v-model="myPwdDialog.visible" title="修改密码" width="420px">
|
||||
<el-form label-width="80px">
|
||||
<el-form-item label="旧密码">
|
||||
<el-input v-model="myPwdDialog.oldPassword" placeholder="请输入当前密码" show-password />
|
||||
</el-form-item>
|
||||
<el-form-item label="新密码">
|
||||
<el-input v-model="myPwdDialog.newPassword" placeholder="至少 6 位" show-password />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="myPwdDialog.visible = false">取消</el-button>
|
||||
<el-button type="primary" :loading="myPwdDialog.saving" @click="confirmChangeMyPwd">确认修改</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user