merge: 合并远程 ft_wyt 分支,解决权限与日志模块冲突

- 冲突解决原则:本地权限治理(require_admin/current_user/资源ACL)与远程 op_log 日志装饰器双向保留
- platform.py: 9 处冲突,@op_log 与管理员校验叠加,避免远程丢失 require_admin 的安全回归
- logging.py: 合并 get_client_ip 与 user_id_var,X-Trace-Id 优先 + ContextVar 卫生处理
- op_log.py: 采纳远程将变量计算上移到函数顶部的结构
- compute_poller.py: 中文日志 + 失败去重限流/断连重置逻辑
- data_process.py: 保留租户归属字段 + biz_logger 成功日志
This commit is contained in:
wuyongtao
2026-08-21 09:58:43 +08:00
27 changed files with 2778 additions and 688 deletions

View File

@@ -41,8 +41,12 @@ from app.core.auth import (
has_resource_access, has_resource_access,
is_admin, is_admin,
) )
from app.core.logging import get_structured_logger
from app.core.config import get_settings from app.core.config import get_settings
from app.db.platform_store import get_platform_store from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import ( from app.modules.data_process.algorithms import (
ParsedText, ParsedText,
canonical_record_json, canonical_record_json,
@@ -117,6 +121,7 @@ from app.schemas.data_process import (
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
biz_logger = get_structured_logger("app.biz.data_process")
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024 MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
MAX_SOURCE_FILE_COUNT = 20 MAX_SOURCE_FILE_COUNT = 20
MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024 MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
@@ -292,7 +297,7 @@ def _commit_source_batch(
except Exception: except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因,并继续清理其余对象。 # 文件系统回滚失败不能覆盖数据库抛出的根因,并继续清理其余对象。
logger.exception( logger.exception(
"failed to roll back data process source object task_id=%s", "数据处理源对象回滚失败 task_id=%s",
task_id, task_id,
) )
raise raise
@@ -662,7 +667,7 @@ def _run_generation(
) -> None: ) -> None:
started_at = time.perf_counter() started_at = time.perf_counter()
logger.info( logger.info(
"data process generation worker started task_id=%s generation_run_id=%s", "数据处理生成任务开始 task_id=%s generation_run_id=%s",
task_id, task_id,
generation_run_id, generation_run_id,
) )
@@ -670,8 +675,7 @@ def _run_generation(
task = store.get_task(task_id) task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id): if not store.generation_is_running(task_id, generation_run_id):
logger.info( logger.info(
"data process generation worker skipped inactive run task_id=%s " "数据处理生成任务跳过(非活跃运行) task_id=%s generation_run_id=%s",
"generation_run_id=%s",
task_id, task_id,
generation_run_id, generation_run_id,
) )
@@ -761,8 +765,7 @@ def _run_generation(
len(preview_items), len(preview_items),
): ):
logger.info( logger.info(
"data process generation stopped before completion task_id=%s " "数据处理生成任务被中止 task_id=%s generation_run_id=%s",
"generation_run_id=%s",
task_id, task_id,
generation_run_id, generation_run_id,
) )
@@ -853,9 +856,7 @@ def _run_generation(
"created_by": (store.get_task(task_id) or {}).get("created_by"), "created_by": (store.get_task(task_id) or {}).get("created_by"),
}) })
logger.info( logger.info(
"data process generation completed task_id=%s generation_run_id=%s " "数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f",
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
task_id, task_id,
generation_run_id, generation_run_id,
completed.get("output_count", len(accepted)), completed.get("output_count", len(accepted)),
@@ -866,14 +867,13 @@ def _run_generation(
) )
else: else:
logger.info( logger.info(
"data process generation stopped before result persistence task_id=%s " "数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s",
"generation_run_id=%s",
task_id, task_id,
generation_run_id, generation_run_id,
) )
except Exception as exc: except Exception as exc:
logger.exception( logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f", "数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id, task_id,
generation_run_id, generation_run_id,
(time.perf_counter() - started_at) * 1000, (time.perf_counter() - started_at) * 1000,
@@ -887,8 +887,7 @@ def _run_generation(
) )
except Exception: except Exception:
logger.exception( logger.exception(
"failed to persist data process generation failure task_id=%s " "数据处理生成失败持久化异常 task_id=%s generation_run_id=%s",
"generation_run_id=%s",
task_id, task_id,
generation_run_id, generation_run_id,
) )
@@ -943,6 +942,7 @@ def create_task(
values["tenant_id"] = current_user.get("tenant_id") or "default" values["tenant_id"] = current_user.get("tenant_id") or "default"
get_platform_store().assert_active_tenant(values["tenant_id"]) get_platform_store().assert_active_tenant(values["tenant_id"])
task = store.create_task(values) task = store.create_task(values)
biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", ""))
return ok(task, "data process task created") return ok(task, "data process task created")
@@ -968,10 +968,9 @@ def update_task(
store: DataProcessStore = Depends(get_data_process_store), store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]: ) -> dict[str, Any]:
with api_errors(): with api_errors():
return ok( result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")), biz_logger.info("用户更新数据处理任务成功", taskId=task_id)
"data process task updated", return ok(result, "data process task updated")
)
@router.put("/{task_id}/workflow-step") @router.put("/{task_id}/workflow-step")
@@ -1060,7 +1059,7 @@ def _remove_repeated_storage_objects(
) )
except Exception: except Exception:
logger.exception( logger.exception(
"failed to roll back repeated data process source object task_id=%s", "数据处理源对象重复回滚失败 task_id=%s",
task_id, task_id,
) )
@@ -1136,6 +1135,7 @@ def delete_task(
) -> dict[str, Any]: ) -> dict[str, Any]:
with api_errors(): with api_errors():
store.delete_task(task_id) store.delete_task(task_id)
biz_logger.info("用户删除数据处理任务成功", taskId=task_id)
return ok({"deleted": task_id}, "data process task deleted") return ok({"deleted": task_id}, "data process task deleted")
@@ -1448,13 +1448,12 @@ def delete_source_file(
expected_source_file_id=file_id, expected_source_file_id=file_id,
) )
except Exception: except Exception:
# 数据库软删除已经提交,不能再向客户端返回可重试的失败;保留逻辑引用,
# 由后续存储清理任务重试物理删除。
cleanup_pending = True cleanup_pending = True
logger.exception( logger.exception(
"failed to remove data process source object after soft deletion", "数据处理源对象软删除后存储清理失败",
extra={"task_id": task_id, "source_file_id": file_id}, extra={"task_id": task_id, "source_file_id": file_id},
) )
biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
return ok( return ok(
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending}, {"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
"source file removed", "source file removed",
@@ -1727,7 +1726,7 @@ def _prepare_preview_items(
extracted_text = "\n\n".join(page.text for page in pages if page.text) extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""): if extracted_text != str(source.get("content") or ""):
logger.warning( logger.warning(
"skip PDF document noise detection because stored offsets differ for %s", "跳过PDF文档噪声检测存储偏移量不一致 source_id=%s",
source["id"], source["id"],
) )
continue continue
@@ -1750,7 +1749,7 @@ def _run_preview(
started_at = time.perf_counter() started_at = time.perf_counter()
logger.info( logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s", "数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s",
task_id, task_id,
preview_run_id, preview_run_id,
len(source_file_ids), len(source_file_ids),
@@ -1759,7 +1758,7 @@ def _run_preview(
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured" is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id): if not store.mark_preview_running(task_id, preview_run_id):
logger.info( logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s", "数据处理预览跳过(非活跃运行) task_id=%s preview_run_id=%s",
task_id, task_id,
preview_run_id, preview_run_id,
) )
@@ -1769,8 +1768,7 @@ def _run_preview(
for completed_files, source_file_id in enumerate(source_file_ids, start=1): for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id): if not store.preview_is_running(task_id, preview_run_id):
logger.info( logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s " "数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
"completed_files=%s total_files=%s",
task_id, task_id,
preview_run_id, preview_run_id,
completed_files - 1, completed_files - 1,
@@ -1801,8 +1799,7 @@ def _run_preview(
total_files, total_files,
): ):
logger.info( logger.info(
"data process preview stopped before progress update task_id=%s " "数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
"preview_run_id=%s completed_files=%s total_files=%s",
task_id, task_id,
preview_run_id, preview_run_id,
completed_files, completed_files,
@@ -1811,8 +1808,7 @@ def _run_preview(
return return
if store.complete_preview(task_id, preview_run_id): if store.complete_preview(task_id, preview_run_id):
logger.info( logger.info(
"data process preview completed task_id=%s preview_run_id=%s " "数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f",
"total_files=%s total_items=%s duration_ms=%.2f",
task_id, task_id,
preview_run_id, preview_run_id,
total_files, total_files,
@@ -1821,14 +1817,13 @@ def _run_preview(
) )
else: else:
logger.info( logger.info(
"data process preview completion ignored for inactive run task_id=%s " "数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s",
"preview_run_id=%s",
task_id, task_id,
preview_run_id, preview_run_id,
) )
except Exception as exc: except Exception as exc:
logger.exception( logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f", "数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id, task_id,
preview_run_id, preview_run_id,
(time.perf_counter() - started_at) * 1000, (time.perf_counter() - started_at) * 1000,
@@ -1842,8 +1837,7 @@ def _run_preview(
) )
except Exception: except Exception:
logger.exception( logger.exception(
"failed to persist data process preview failure task_id=%s " "数据处理预览失败持久化异常 task_id=%s preview_run_id=%s",
"preview_run_id=%s",
task_id, task_id,
preview_run_id, preview_run_id,
) )
@@ -2053,6 +2047,7 @@ def stop(
) -> dict[str, Any]: ) -> dict[str, Any]:
with api_errors(): with api_errors():
store.stop_task(task_id) store.stop_task(task_id)
biz_logger.info("用户停止数据处理任务成功", taskId=task_id)
return ok(store.progress(task_id), "data process task stopped") return ok(store.progress(task_id), "data process task stopped")
@@ -2094,7 +2089,9 @@ def confirm_results(
store: DataProcessStore = Depends(get_data_process_store), store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]: ) -> dict[str, Any]:
with api_errors(): with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed") result = store.confirm_results(task_id)
biz_logger.info("用户确认数据处理结果成功", taskId=task_id)
return ok(result, "data process results confirmed")
@router.put("/{task_id}/results/{result_id}") @router.put("/{task_id}/results/{result_id}")
@@ -2443,8 +2440,7 @@ def regenerate_results_batch(
})) }))
logger.info( logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s " "数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s",
"requested=%s prepared=%s concurrency=%s",
batch_id, batch_id,
task_id, task_id,
len(payload.items), len(payload.items),
@@ -2512,8 +2508,7 @@ def regenerate_results_batch(
except Exception as exc: # pragma: no cover - defensive boundary except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error" outcome = "internal_error"
logger.exception( logger.exception(
"data process result batch regeneration crashed " "数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s",
"batch_id=%s task_id=%s result_id=%s",
batch_id, batch_id,
task_id, task_id,
result_id, result_id,
@@ -2524,8 +2519,7 @@ def regenerate_results_batch(
"message": _safe_regeneration_error(exc), "message": _safe_regeneration_error(exc),
})) }))
logger.info( logger.info(
"data process result batch item finished batch_id=%s task_id=%s " "数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
"result_id=%s outcome=%s duration_ms=%.2f",
batch_id, batch_id,
task_id, task_id,
result_id, result_id,
@@ -2545,8 +2539,7 @@ def regenerate_results_batch(
) )
duration_ms = (time.perf_counter() - started_at) * 1000 duration_ms = (time.perf_counter() - started_at) * 1000
logger.info( logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s " "数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id, batch_id,
task_id, task_id,
len(success_items), len(success_items),
@@ -2593,8 +2586,7 @@ def evaluate_results_batch(
evaluation_model = store.get_generation_model(str(model_id)) evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError: except NotFoundError:
logger.warning( logger.warning(
"data process evaluation model unavailable, judge layer " "数据处理评测模型不可用,跳过评测层 task_id=%s model_id=%s",
"skipped task_id=%s model_id=%s",
task_id, task_id,
model_id, model_id,
) )
@@ -2642,8 +2634,7 @@ def evaluate_results_batch(
})) }))
logger.info( logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s " "数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s",
"requested=%s prepared=%s judge_enabled=%s",
batch_id, batch_id,
task_id, task_id,
len(payload.items), len(payload.items),
@@ -2660,8 +2651,7 @@ def evaluate_results_batch(
semantic_embedding_model() semantic_embedding_model()
except Exception: except Exception:
logger.warning( logger.warning(
"data process semantic embedding unavailable, semantic layer " "数据处理语义嵌入模型不可用,语义层将跳过 batch_id=%s",
"will be skipped batch_id=%s",
batch_id, batch_id,
) )
request_timeout = _result_regeneration_timeout(config) request_timeout = _result_regeneration_timeout(config)
@@ -2714,8 +2704,7 @@ def evaluate_results_batch(
"message": _safe_regeneration_error(exc), "message": _safe_regeneration_error(exc),
})) }))
logger.info( logger.info(
"data process result batch evaluation item finished " "数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id, batch_id,
task_id, task_id,
result_id, result_id,
@@ -2727,8 +2716,7 @@ def evaluate_results_batch(
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])] failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000 duration_ms = (time.perf_counter() - started_at) * 1000
logger.info( logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s " "数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f",
"succeeded=%s failed=%s duration_ms=%.2f",
batch_id, batch_id,
task_id, task_id,
len(success_items), len(success_items),
@@ -2784,8 +2772,7 @@ def regenerate_result(
) )
except _ResultRegenerationFailed as exc: except _ResultRegenerationFailed as exc:
logger.warning( logger.warning(
"data process result regeneration failed task_id=%s result_id=%s " "数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s",
"duration_ms=%.2f reason=%s",
task_id, task_id,
result_id, result_id,
(time.perf_counter() - started_at) * 1000, (time.perf_counter() - started_at) * 1000,
@@ -2793,7 +2780,7 @@ def regenerate_result(
) )
raise raise
logger.info( logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f", "数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f",
task_id, task_id,
result_id, result_id,
(time.perf_counter() - started_at) * 1000, (time.perf_counter() - started_at) * 1000,
@@ -2809,5 +2796,6 @@ def publish(
) -> dict[str, Any]: ) -> dict[str, Any]:
with api_errors(): with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json")) result = store.publish(task_id, payload.model_dump(mode="json"))
biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", ""))
message = "dataset published" if result["created"] else "dataset already published" message = "dataset published" if result["created"] else "dataset already published"
return ok(result, message) return ok(result, message)

View File

@@ -913,6 +913,7 @@ async def _fine_tune_preflight_with_job_payload(
@router.post("/login") @router.post("/login")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGIN, target_type="user", target_name_param="username")
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]: async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
ip = request.client.host if request and request.client else "unknown" ip = request.client.host if request and request.client else "unknown"
@@ -930,6 +931,7 @@ async def login(payload: dict[str, Any] = Body(...), request: Request = None) ->
@router.post("/logout") @router.post("/logout")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGOUT, target_type="user")
async def logout(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def logout(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
session_id = payload.get("session_id", "") session_id = payload.get("session_id", "")
@@ -1179,6 +1181,7 @@ async def users(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
@router.post("/users") @router.post("/users")
@op_log(module=OpModule.SYSTEM, action=OpAction.CREATE, target_type="user", target_name_param="username")
async def create_user(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def create_user(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
payload = dict(payload) payload = dict(payload)
payload.setdefault("password", "123456") payload.setdefault("password", "123456")
@@ -1187,6 +1190,7 @@ async def create_user(payload: dict[str, Any] = Body(...), current_user: dict =
@router.put("/users/{user_id}") @router.put("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user", target_name_param="user_id")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try: try:
return ok(get_platform_store().update_user(user_id, payload)) return ok(get_platform_store().update_user(user_id, payload))
@@ -1195,6 +1199,7 @@ async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current
@router.delete("/users/{user_id}") @router.delete("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.DELETE, target_type="user", target_name_param="user_id")
async def delete_user(user_id: str, current_username: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def delete_user(user_id: str, current_username: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try: try:
get_platform_store().delete_user(user_id, deleted_by=str(current_user.get("id") or "system")) get_platform_store().delete_user(user_id, deleted_by=str(current_user.get("id") or "system"))
@@ -1212,6 +1217,7 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
@router.post("/users/{user_id}/reset-password") @router.post("/users/{user_id}/reset-password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password", target_name_param="user_id")
async def reset_user_password( async def reset_user_password(
user_id: str, user_id: str,
payload: dict[str, Any] = Body(default={}), payload: dict[str, Any] = Body(default={}),
@@ -1228,6 +1234,7 @@ async def reset_user_password(
@router.post("/users/me/password") @router.post("/users/me/password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password")
async def change_my_password( async def change_my_password(
payload: dict[str, Any] = Body(...), payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user), current_user: dict = Depends(get_current_user),
@@ -1310,6 +1317,7 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
@router.delete("/model-manage/trained-models/{model_id}") @router.delete("/model-manage/trained-models/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="trained_model", target_name_param="model_id")
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("trained_model", model_id, current_user, "delete"): if not has_resource_access("trained_model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this trained model") raise fail(403, "no permission to delete this trained model")
@@ -1424,6 +1432,7 @@ async def test_online_model(payload: dict[str, Any] = Body(...), current_user: d
@router.post("/model-manage") @router.post("/model-manage")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.CREATE, target_type="model", target_name_param="name")
@audit_log( @audit_log(
action=AuditActions.CREATE_MODEL, action=AuditActions.CREATE_MODEL,
target_type="model", target_type="model",
@@ -1465,6 +1474,7 @@ async def model_detail(model_id: str, current_user: dict = Depends(get_current_u
@router.put("/model-manage/{model_id}") @router.put("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model", target_name_param="model_id")
@audit_log( @audit_log(
action=AuditActions.UPDATE_MODEL, action=AuditActions.UPDATE_MODEL,
target_type="model", target_type="model",
@@ -1481,6 +1491,7 @@ async def update_model(model_id: str, payload: dict[str, Any] = Body(...), curre
@router.put("/model-manage/{model_id}/purpose") @router.put("/model-manage/{model_id}/purpose")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model_purpose", target_name_param="model_id")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型(配置模型)只有管理员可以修改用途 # 基座模型(配置模型)只有管理员可以修改用途
if not is_admin(current_user): if not is_admin(current_user):
@@ -1492,6 +1503,7 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
@router.delete("/model-manage/{model_id}") @router.delete("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="model", target_name_param="model_id")
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型(配置模型)只有管理员可以删除 # 基座模型(配置模型)只有管理员可以删除
if not is_admin(current_user): if not is_admin(current_user):
@@ -1691,6 +1703,7 @@ async def dataset_version_content(file_id: str, version_id: str, current_user: d
@router.post("/dataset-manage/versions/{file_id}") @router.post("/dataset-manage/versions/{file_id}")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset_version", target_name_param="file_id")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try: try:
store = get_platform_store() store = get_platform_store()
@@ -1715,6 +1728,7 @@ async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(
@router.delete("/dataset-manage/versions/{file_id}/{version_id}") @router.delete("/dataset-manage/versions/{file_id}/{version_id}")
@op_log(module=OpModule.DATASET, action=OpAction.DELETE, target_type="dataset_version", target_name_param="version_id")
async def delete_dataset_version(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def delete_dataset_version(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try: try:
store = get_platform_store() store = get_platform_store()
@@ -1913,6 +1927,7 @@ async def _sync_training_dataset_to_compute_node(
@router.post("/dataset-manage/upload/{dataset_id}") @router.post("/dataset-manage/upload/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPLOAD, target_type="dataset_file", target_name_param="dataset_id")
async def upload_dataset_files( async def upload_dataset_files(
dataset_id: str, dataset_id: str,
files: list[UploadFile] = File(default=[]), files: list[UploadFile] = File(default=[]),
@@ -2080,6 +2095,7 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s
@router.post("/dataset-manage") @router.post("/dataset-manage")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
@audit_log( @audit_log(
action=AuditActions.CREATE_DATASET, action=AuditActions.CREATE_DATASET,
target_type="dataset", target_type="dataset",
@@ -2110,6 +2126,7 @@ async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_curre
@router.put("/dataset-manage/{dataset_id}") @router.put("/dataset-manage/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPDATE, target_type="dataset", target_name_param="dataset_id")
@audit_log( @audit_log(
action=AuditActions.UPDATE_DATASET, action=AuditActions.UPDATE_DATASET,
target_type="dataset", target_type="dataset",
@@ -2175,6 +2192,7 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/fine-tune") @router.post("/fine-tune")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.CREATE, target_type="fine_tune", target_name_param="name")
@audit_log( @audit_log(
action=AuditActions.CREATE_FINE_TUNE, action=AuditActions.CREATE_FINE_TUNE,
target_type="fine_tune", target_type="fine_tune",
@@ -2418,6 +2436,7 @@ async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depe
@router.put("/fine-tune/{task_id}") @router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "write"): if not has_resource_access("fine-tune", task_id, current_user, "write"):
raise fail(403, "no permission to update this task") raise fail(403, "no permission to update this task")
@@ -2452,6 +2471,7 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
@router.post("/fine-tune/{task_id}/retry") @router.post("/fine-tune/{task_id}/retry")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.RETRY, target_type="fine_tune", target_name_param="task_id")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
payload = payload or {} payload = payload or {}
@@ -2883,6 +2903,7 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
@router.delete("/model-eval/{task_id}") @router.delete("/model-eval/{task_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval_task", target_name_param="task_id")
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("eval", task_id, current_user, "delete"): if not has_resource_access("eval", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this eval task") raise fail(403, "no permission to delete this eval task")
@@ -2902,6 +2923,7 @@ async def dimension_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/dimension") @router.post("/dimension")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.CREATE, target_type="dimension", target_name_param="name")
async def dimension_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def dimension_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().create_dimension(payload)) return ok(get_platform_store().create_dimension(payload))
@@ -2915,6 +2937,7 @@ async def dimension_detail(dimension_id: str, current_user: dict = Depends(get_c
@router.put("/dimension/{dimension_id}") @router.put("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.UPDATE, target_type="dimension", target_name_param="dimension_id")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try: try:
return ok(get_platform_store().update_dimension(dimension_id, payload)) return ok(get_platform_store().update_dimension(dimension_id, payload))
@@ -2923,6 +2946,7 @@ async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...
@router.delete("/dimension/{dimension_id}") @router.delete("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="dimension", target_name_param="dimension_id")
async def dimension_delete(dimension_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]: async def dimension_delete(dimension_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
get_platform_store().delete_dimension(dimension_id) get_platform_store().delete_dimension(dimension_id)
return ok({"deleted": dimension_id}) return ok({"deleted": dimension_id})
@@ -2943,6 +2967,7 @@ async def model_compare_list(current_user: dict = Depends(get_current_user)) ->
@router.post("/model-compare") @router.post("/model-compare")
@op_log(module=OpModule.INFERENCE, action=OpAction.CREATE, target_type="compare_task", target_name_param="name")
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
bind_active_tenant(payload, current_user) bind_active_tenant(payload, current_user)
payload.setdefault("created_by", current_user.get("id")) payload.setdefault("created_by", current_user.get("id"))
@@ -3785,6 +3810,7 @@ async def compute_node_detail(node_id: str, current_user: dict = Depends(get_cur
@router.post("/compute/nodes") @router.post("/compute/nodes")
@op_log(module=OpModule.COMPUTE, action=OpAction.CREATE, target_type="compute_node", target_name_param="name")
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): if not is_admin(current_user):
raise fail(403, "admin permission required") raise fail(403, "admin permission required")
@@ -3797,6 +3823,7 @@ async def create_compute_node(payload: dict[str, Any] = Body(...), current_user:
@router.put("/compute/nodes/{node_id}") @router.put("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.UPDATE, target_type="compute_node", target_name_param="node_id")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): if not is_admin(current_user):
raise fail(403, "admin permission required") raise fail(403, "admin permission required")
@@ -3809,6 +3836,7 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...),
@router.delete("/compute/nodes/{node_id}") @router.delete("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.DELETE, target_type="compute_node", target_name_param="node_id")
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user): if not is_admin(current_user):
raise fail(403, "admin permission required") raise fail(403, "admin permission required")
@@ -4123,6 +4151,7 @@ async def compute_job_detail(job_id: str, current_user: dict = Depends(get_curre
@router.post("/compute/jobs/{job_id}/stop") @router.post("/compute/jobs/{job_id}/stop")
@op_log(module=OpModule.COMPUTE, action=OpAction.STOP, target_type="compute_job", target_name_param="job_id")
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
task = _task_for_compute_job(job_id) task = _task_for_compute_job(job_id)
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"): if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
@@ -4166,6 +4195,7 @@ async def compute_job_logs(
@router.post("/compute/jobs/{job_id}/retry") @router.post("/compute/jobs/{job_id}/retry")
@op_log(module=OpModule.COMPUTE, action=OpAction.RETRY, target_type="compute_job", target_name_param="job_id")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]: async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store() store = get_platform_store()
payload = payload or {} payload = payload or {}

View File

@@ -50,7 +50,7 @@ def docs_kwargs(enabled: bool) -> dict[str, Any]:
@dataclass(frozen=True) @dataclass(frozen=True)
class Settings: class Settings:
app_name: str = os.getenv("APP_NAME", "YG Fine-Tune Platform API") app_name: str = os.getenv("APP_NAME", "YG Zhilian API")
app_env: str = os.getenv("APP_ENV", "local") app_env: str = os.getenv("APP_ENV", "local")
route_prefix: str = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF") route_prefix: str = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF")
app_mode: str = os.getenv("APP_MODE", "local") app_mode: str = os.getenv("APP_MODE", "local")

View File

@@ -1,22 +1,26 @@
from __future__ import annotations from __future__ import annotations
from contextvars import ContextVar
from datetime import date, datetime, timedelta
import json import json
import logging import logging
import re
import socket
import sys import sys
import time
from contextvars import ContextVar
from datetime import date, datetime, timedelta
from logging import Handler, LogRecord from logging import Handler, LogRecord
from pathlib import Path from pathlib import Path
import re from typing import Any, Callable
import time
from typing import Any, Callable, Optional
from uuid import uuid4 from uuid import uuid4
from fastapi import FastAPI, Request from fastapi import FastAPI, Request
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
# ==================== 链路追踪 ContextVar ====================
request_id_var: ContextVar[str] = ContextVar("request_id", default="-") request_id_var: ContextVar[str] = ContextVar("request_id", default="-")
user_id_var: ContextVar[str] = ContextVar("user_id", default="")
client_ip_var: ContextVar[str] = ContextVar("client_ip", default="") client_ip_var: ContextVar[str] = ContextVar("client_ip", default="")
@@ -30,54 +34,74 @@ def get_client_ip(request: Request | None) -> str:
return value.split(",", 1)[0].strip() return value.split(",", 1)[0].strip()
return request.client.host if request.client else "" return request.client.host if request.client else ""
# ==================== 敏感数据脱敏规则 ==================== # ==================== 敏感数据脱敏 ====================
SENSITIVE_PATTERNS: dict[str, Callable | str] = { SENSITIVE_KEYS: set[str] = {
"token": "***", "password", "token", "access_token", "refresh_token",
"password": "***", "secret_key", "authorization", "bearer", "api_key",
"access_token": "***", "private_key", "secret", "cookie",
"refresh_token": "***",
"secret_key": "***",
"authorization": "***",
"bearer": "***",
"api_key": "***",
"private_key": "***",
} }
def mask_value(key: str, value: Any) -> str: FULL_MASK_KEYS: set[str] = {
"""对单个值进行脱敏处理""" "password", "token", "access_token", "refresh_token",
"secret_key", "authorization", "bearer", "api_key",
"private_key", "secret", "cookie",
}
def _mask_phone(value: str) -> str:
"""手机号脱敏138****5678"""
if len(value) >= 11:
return value[:3] + "****" + value[-4:]
return value
def _mask_id_card(value: str) -> str:
"""身份证号脱敏110***********1234"""
if len(value) >= 18:
return value[:3] + "***********" + value[-4:]
return value
def mask_value(key: str, value: Any) -> Any:
"""对单个值进行脱敏处理。"""
if value is None: if value is None:
return "" return ""
key_lower = key.lower()
if key_lower in FULL_MASK_KEYS:
return "***"
str_val = str(value) str_val = str(value)
# 手机号模式11位数字1开头
handler = SENSITIVE_PATTERNS.get(key) if re.match(r"^1[3-9]\d{9}$", str_val):
if callable(handler): return _mask_phone(str_val)
return handler(str_val) # 身份证模式18位
elif isinstance(handler, str): if re.match(r"^\d{17}[\dXx]$", str_val):
# 支持正则替换模式,如 r"1\d{3}\d{4}" return _mask_id_card(str_val)
try: return value
return re.sub(handler, "***", str_val)
except re.error:
return "***"
return handler
def mask_sensitive_dict(data: dict) -> dict: def mask_sensitive_dict(data: dict) -> dict:
"""递归脱敏字典中的敏感字段""" """递归脱敏字典中的敏感字段"""
if not data or not isinstance(data, dict): if not data or not isinstance(data, dict):
return data return data
result: dict[str, Any] = {}
result = {}
for key, value in data.items(): for key, value in data.items():
result[key] = mask_value(key, value) if isinstance(value, dict):
result[key] = mask_sensitive_dict(value)
elif isinstance(value, list):
result[key] = [
mask_sensitive_dict(item) if isinstance(item, dict) else item
for item in value
]
else:
result[key] = mask_value(key, value)
return result return result
def mask_sensitive_string(text: str) -> str: def mask_sensitive_string(text: str) -> str:
"""从文本中脱敏常见敏感信息""" """从文本中脱敏常见敏感信息"""
if not text: if not text:
return text return text
# Mask the value as well as the key. Replacing only ``api_key=`` would # Mask the value as well as the key. Replacing only ``api_key=`` would
# still leak the credential in audit messages and exception text. # still leak the credential in audit messages and exception text.
assignment_pattern = ( assignment_pattern = (
@@ -90,92 +114,165 @@ def mask_sensitive_string(text: str) -> str:
except re.error: except re.error:
pass pass
patterns = [ patterns: list[tuple[str, str]] = [
(r'Bearer\s+[A-Za-z0-9\-._]+', 'Bearer ***'), (r"Bearer\s+[A-Za-z0-9\-._]+", "Bearer ***"),
(r'\d{11}', r'\d{3}\*\d{4}'), # 手机号/身份证 (r"(?i)token\s*[:=]\s*\S+", "token=***"),
(r'1[3-9]\d{9}', r'1\*{3}\*{4}'), # 手机号 (r"(?i)password\s*[:=]\s*\S+", "password=***"),
(r"(?i)secret[_-]?key\s*[:=]\s*\S+", "secret_key=***"),
(r"(?i)api[-_]?key\s*[:=]\s*\S+", "api_key=***"),
(r"(?i)private[_-]?key\s*[:=]\s*\S+", "private_key=***"),
(r"(?i)authorization\s*[:=]\s*\S+", "authorization=***"),
] ]
for pattern, replacement in patterns: for pattern, replacement in patterns:
try: text = re.sub(pattern, replacement, text)
text = re.sub(pattern, replacement, text, flags=re.IGNORECASE) # 手机号脱敏
except re.error: text = re.sub(r"\b1[3-9]\d{9}\b", lambda m: _mask_phone(m.group()), text)
pass
return text return text
# ==================== RequestId Filter ==================== # ==================== 大对象截断 ====================
MAX_FIELD_SIZE = 1024 # 超过 1KB 的内容自动截断
def truncate_large_value(value: Any, max_size: int = MAX_FIELD_SIZE) -> Any:
"""超过 max_size 的字符串自动截断(前 500 + 后 500"""
if isinstance(value, str) and len(value) > max_size:
half = max_size // 2
return value[:half] + f"...[truncated {len(value) - max_size} chars]..." + value[-half:]
if isinstance(value, dict):
return {k: truncate_large_value(v, max_size) for k, v in value.items()}
if isinstance(value, list):
return [truncate_large_value(v, max_size) for v in value]
return value
# ==================== TraceId Filter ====================
class TraceIdFilter(logging.Filter):
"""自动注入 traceId / userId / clientIp 到每条日志记录。"""
class RequestIdFilter(logging.Filter):
def filter(self, record: LogRecord) -> bool: def filter(self, record: LogRecord) -> bool:
record.request_id = request_id_var.get() record.traceId = request_id_var.get()
record.userId = user_id_var.get("")
record.clientIp = client_ip_var.get("")
record.host = getattr(self, "_host", None) or socket.gethostname()
record.app = getattr(self, "_app", "yg-ft-platform")
record.env = getattr(self, "_env", "dev")
return True return True
def set_context(self, app: str, env: str, host: str) -> None:
self._app = app
self._env = env
self._host = host
# ==================== Enhanced JSON Formatter ====================
# ==================== JSON Formatter ====================
class JsonLogFormatter(logging.Formatter): class JsonLogFormatter(logging.Formatter):
""" """
增强的 JSON 日志格式化器,支持结构化字段输出 生产级 JSON 日志格式化器,符合方案文档 §3.2 字段规范
输出示例: 输出示例:
{ {
"@timestamp": "2026-08-17T18:30:00.123Z", "@timestamp": "2026-08-19T10:30:45.123+08:00",
"level": "INFO", "level": "INFO",
"logger": "dataset.router", "logger": "app.api.v1.endpoints.platform",
"traceId": "abc-123-def-456",
"userId": "u_admin",
"message": "数据集创建成功", "message": "数据集创建成功",
"module": "dataset.router", "fields": {"datasetId": "ds_001", "costMs": 23},
"function": "create_dataset", "file": "platform.py:156",
"file": "dataset/router.py",
"line": 45,
"process": 12345,
"thread": "MainThread", "thread": "MainThread",
"request_id": "req-abc123", "host": "pod-7x9k2",
"user_id": "u_admin", "app": "yg-ft-platform",
"client_ip": "192.168.1.100", "env": "dev"
"extra": {...}
} }
""" """
# 标准 LogRecord 属性名集合,用于区分 extra 字段
_STD_ATTRS: set[str] = set(vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()) | {
"traceId", "userId", "clientIp", "host", "app", "env",
"request_id", "user_id", "client_ip",
"asctime", "message", "module", "function", "process",
"thread", "threadName", "levelname", "levelno", "name",
"pathname", "filename", "lineno", "funcName", "created",
"msecs", "relativeCreated", "exc_info", "exc_text",
"stack_info", "msg", "args", "processName", "process",
}
def format(self, record: LogRecord) -> str: def format(self, record: LogRecord) -> str:
# 时间戳ISO8601 带时区
timestamp = datetime.fromtimestamp(record.created).astimezone().isoformat(
timespec="milliseconds"
)
payload: dict[str, Any] = { payload: dict[str, Any] = {
"@timestamp": datetime.fromtimestamp(record.created).astimezone().isoformat( "@timestamp": timestamp,
timespec="milliseconds"
),
"level": record.levelname, "level": record.levelname,
"logger": record.name, "logger": record.name,
"traceId": getattr(record, "traceId", "-"),
"message": record.getMessage(), "message": record.getMessage(),
"module": record.module, "file": f"{Path(record.pathname).name}:{record.lineno}",
"function": record.funcName, "thread": record.threadName,
"file": record.pathname, "host": getattr(record, "host", ""),
"line": record.lineno, "app": getattr(record, "app", ""),
"process": record.process, "env": getattr(record, "env", ""),
"thread": record.thread,
"thread_name": record.threadName,
"request_id": getattr(record, "request_id", "-"),
} }
# 从 record 中提取额外字段(通过 extra 参数传入 # userId业务必填未登录可为空
for attr in ("user_id", "client_ip", "target_type", "target_id", user_id = getattr(record, "userId", "") or getattr(record, "user_id", "")
"duration_ms", "status_code", "error"): if user_id:
val = getattr(record, attr, None) payload["userId"] = user_id
if val is not None:
payload[attr] = val
# 处理异常信息 # clientIp
if record.exc_info: client_ip = getattr(record, "clientIp", "") or getattr(record, "client_ip", "")
if client_ip:
payload["clientIp"] = client_ip
# 提取结构化业务字段:只收集通过 extra 传入的非标准属性
fields: dict[str, Any] = {}
for attr in dir(record):
if attr.startswith("_"):
continue
if attr in self._STD_ATTRS:
continue
if attr in ("traceId", "userId", "clientIp", "host", "app", "env"):
continue
val = getattr(record, attr, None)
if val is not None and not callable(val):
fields[attr] = truncate_large_value(val)
if fields:
payload["fields"] = mask_sensitive_dict(fields)
# ERROR 级别额外字段
if record.levelname == "ERROR" or record.exc_info:
error_obj: dict[str, Any] = {
"type": type(record.exc_info[1]).__name__ if record.exc_info and record.exc_info[1] else "Error",
"message": record.getMessage(),
}
if record.exc_info:
error_obj["stack_trace"] = self.formatException(record.exc_info)
if record.stack_info:
error_obj["stack_trace"] = self.formatStack(record.stack_info)
payload["error"] = error_obj
# 兼容旧字段名 exception
if record.exc_info and "error" not in payload:
payload["exception"] = self.formatException(record.exc_info) payload["exception"] = self.formatException(record.exc_info)
if record.stack_info:
payload["stack"] = self.formatStack(record.stack_info)
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
# ==================== DateSizeRotatingFileHandler ==================== # ==================== DateSizeRotatingFileHandler ====================
# (保持不变,已有实现)
class DateSizeRotatingFileHandler(Handler): class DateSizeRotatingFileHandler(Handler):
"""Rotate log files by date and size while keeping date in every file name.""" """按日期+大小滚动的文件日志处理器。
- 按天创建文件,文件名包含日期
- 单文件超过 max_bytes 时自动滚动(带序号后缀)
- 自动清理超过 retention_days 的旧日志
"""
def __init__( def __init__(
self, self,
@@ -233,10 +330,8 @@ class DateSizeRotatingFileHandler(Handler):
today = date.today() today = date.today()
if not force and self._stream and self._current_date == today: if not force and self._stream and self._current_date == today:
return return
if self._stream and not self._stream.closed: if self._stream and not self._stream.closed:
self._stream.close() self._stream.close()
self._current_date = today self._current_date = today
self._current_path = self._dated_path(today) self._current_path = self._dated_path(today)
self._stream = self._current_path.open("a", encoding=self.encoding) self._stream = self._current_path.open("a", encoding=self.encoding)
@@ -251,11 +346,9 @@ class DateSizeRotatingFileHandler(Handler):
def _rotate_by_size(self) -> None: def _rotate_by_size(self) -> None:
if not self._current_path or not self._current_path.exists(): if not self._current_path or not self._current_path.exists():
return return
if self._stream and not self._stream.closed: if self._stream and not self._stream.closed:
self._stream.close() self._stream.close()
self._stream = None self._stream = None
stem = self._current_path.stem stem = self._current_path.stem
suffix = self._current_path.suffix suffix = self._current_path.suffix
index = 1 index = 1
@@ -269,7 +362,6 @@ class DateSizeRotatingFileHandler(Handler):
def _cleanup_expired_files(self) -> None: def _cleanup_expired_files(self) -> None:
if self.retention_days <= 0: if self.retention_days <= 0:
return return
cutoff = date.today() - timedelta(days=self.retention_days - 1) cutoff = date.today() - timedelta(days=self.retention_days - 1)
pattern = re.compile( pattern = re.compile(
rf"^{re.escape(self.file_prefix)}-(\d{{4}}-\d{{2}}-\d{{2}})(?:\.\d+)?\.log$" rf"^{re.escape(self.file_prefix)}-(\d{{4}}-\d{{2}}-\d{{2}})(?:\.\d+)?\.log$"
@@ -283,15 +375,15 @@ class DateSizeRotatingFileHandler(Handler):
path.unlink(missing_ok=True) path.unlink(missing_ok=True)
# ==================== Structured Logger 封装 ==================== # ==================== StructuredLogger 封装 ====================
class StructuredLogger: class StructuredLogger:
""" """
结构化日志记录器,提供统一的日志接口。 结构化日志记录器,提供符合方案文档 §4.2 的 5W1H 日志接口。
使用方式: 使用方式:
logger = get_structured_logger('dataset.router') logger = get_structured_logger('app.api.dataset')
logger.info('创建数据集', dataset_id='ds_123') logger.info('数据集创建成功', datasetId='ds_001', costMs=23)
""" """
def __init__(self, name: str, module: str = ""): def __init__(self, name: str, module: str = ""):
@@ -303,79 +395,106 @@ class StructuredLogger:
def trace_id(self) -> str: def trace_id(self) -> str:
return request_id_var.get("-") return request_id_var.get("-")
def info(self, message: str, **extra: Any) -> None: def info(self, message: str, **fields: Any) -> None:
self._log("INFO", message, **extra) self._log(logging.INFO, message, **fields)
def warning(self, message: str, **extra: Any) -> None: def warning(self, message: str, **fields: Any) -> None:
self._log("WARNING", message, **extra) self._log(logging.WARNING, message, **fields)
def error(self, message: str, **extra: Any) -> None: def error(self, message: str, **fields: Any) -> None:
self._log("ERROR", message, **extra) self._log(logging.ERROR, message, **fields)
def debug(self, message: str, **extra: Any) -> None: def debug(self, message: str, **fields: Any) -> None:
self._log("DEBUG", message, **extra) self._log(logging.DEBUG, message, **fields)
def _log(self, level: str, message: str, **extra: Any) -> None: def _log(self, level: int, message: str, **fields: Any) -> None:
"""统一日志记录方法""" """统一日志记录方法,通过 extra 传递结构化字段。"""
log_entry: dict[str, Any] = { extra: dict[str, Any] = {}
"timestamp": datetime.utcnow().isoformat(), if self.module:
"level": level, extra["module"] = self.module
"logger": self.name, # 脱敏 + 截断
"module": self.module, for k, v in fields.items():
"message": message, extra[k] = truncate_large_value(v)
"trace_id": self.trace_id, self.logger.log(level, message, extra=extra, stack_info=False)
"extra": extra,
}
self.logger.log(getattr(logging, level, logging.INFO), json.dumps(log_entry, ensure_ascii=False, default=str))
def get_structured_logger(name: str, module: str = "") -> StructuredLogger: def get_structured_logger(name: str, module: str = "") -> StructuredLogger:
"""获取结构化日志记录器""" """获取结构化日志记录器"""
return StructuredLogger(name, module) return StructuredLogger(name, module)
# ==================== 快捷函数 ==================== # ==================== 快捷函数 ====================
def get_logger(name: str) -> logging.Logger: def get_logger(name: str) -> logging.Logger:
"""获取标准 Python logger""" """获取标准 Python logger"""
return logging.getLogger(name) return logging.getLogger(name)
def set_request_id(request_id: str) -> None: def set_request_id(request_id: str) -> None:
"""设置当前请求的追踪 ID""" """设置当前请求的追踪 ID"""
request_id_var.set(request_id) request_id_var.set(request_id)
def set_user_context(user_id: str = "", client_ip: str = "") -> None:
"""设置当前请求的用户上下文(在鉴权后调用)。"""
if user_id:
user_id_var.set(user_id)
if client_ip:
client_ip_var.set(client_ip)
# ==================== 请求日志中间件 ====================
def setup_request_logging(app: FastAPI) -> None: def setup_request_logging(app: FastAPI) -> None:
"""配置 FastAPI 请求日志中间件""" """配置 FastAPI 请求日志中间件,符合方案文档 §五(链路追踪)和 §十(访问日志)。"""
logger = get_logger("app.access") logger = get_logger("app.access")
@app.middleware("http") @app.middleware("http")
async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def] async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def]
request_id = request.headers.get("X-Request-ID") or str(uuid4()) # 入口生成 traceId优先使用前端传入的 X-Trace-Id
token = request_id_var.set(request_id) trace_id = request.headers.get("X-Trace-Id") or request.headers.get("X-Request-ID") or str(uuid4())
token = request_id_var.set(trace_id)
ip_token = client_ip_var.set(get_client_ip(request)) ip_token = client_ip_var.set(get_client_ip(request))
started_at = time.perf_counter() started_at = time.perf_counter()
# 提取客户端 IP
client_ip = "-"
if request.client:
client_ip = request.client.host
# 支持反向代理传递的真实 IP
forwarded_for = request.headers.get("X-Forwarded-For", "")
if forwarded_for:
client_ip = forwarded_for.split(",")[0].strip()
client_ip_var.set(client_ip)
try: try:
response = await call_next(request) response = await call_next(request)
elapsed_ms = (time.perf_counter() - started_at) * 1000 elapsed_ms = (time.perf_counter() - started_at) * 1000
# 噪声路径降级为 DEBUG健康检查等
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/") noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info log_method = logger.info
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400: if any(request.url.path.endswith(p) or p in request.url.path for p in noisy_paths) and response.status_code < 400:
log_method = logger.debug log_method = logger.debug
if response.status_code >= 400: if response.status_code >= 500:
log_method = logger.error
elif response.status_code >= 400:
log_method = logger.warning log_method = logger.warning
# 结构化访问日志(中文 message方便直接阅读
log_method( log_method(
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s", f"HTTP请求 {request.method} {request.url.path} {response.status_code}(耗时{round(elapsed_ms, 2)}ms",
request.method, extra={
request.url.path, "request_method": request.method,
response.status_code, "request_path": request.url.path,
elapsed_ms, "status_code": response.status_code,
request.client.host if request.client else "-", "duration_ms": round(elapsed_ms, 2),
"client_ip": client_ip,
"user_agent": request.headers.get("User-Agent", "")[:200],
},
) )
# 5xx 系统错误自动写入操作日志(未被 @op_log 覆盖的系统级异常) # 5xx 系统错误自动写入操作日志
if response.status_code >= 500: if response.status_code >= 500:
try: try:
from app.core.op_log import log_operation, OpModule, OpStatus from app.core.op_log import log_operation, OpModule, OpStatus
@@ -393,18 +512,23 @@ def setup_request_logging(app: FastAPI) -> None:
duration_ms=elapsed_ms, duration_ms=elapsed_ms,
) )
except Exception: except Exception:
pass # 日志写入失败不影响主流程 pass
response.headers["X-Request-ID"] = request_id response.headers["X-Trace-Id"] = trace_id
response.headers["X-Request-ID"] = trace_id
return response return response
except Exception: except Exception:
elapsed_ms = (time.perf_counter() - started_at) * 1000 elapsed_ms = (time.perf_counter() - started_at) * 1000
logger.exception( logger.error(
"request failed method=%s path=%s duration_ms=%.2f client=%s", f"HTTP请求异常 {request.method} {request.url.path}(耗时{round(elapsed_ms, 2)}ms— 服务内部错误",
request.method, extra={
request.url.path, "request_method": request.method,
elapsed_ms, "request_path": request.url.path,
request.client.host if request.client else "-", "duration_ms": round(elapsed_ms, 2),
"client_ip": client_ip,
},
exc_info=True,
) )
# 未被捕获的异常,写入操作日志 # 未被捕获的异常,写入操作日志
@@ -426,7 +550,7 @@ def setup_request_logging(app: FastAPI) -> None:
duration_ms=elapsed_ms, duration_ms=elapsed_ms,
) )
except Exception: except Exception:
pass # 日志写入失败不影响主流程 pass
raise raise
finally: finally:
@@ -437,53 +561,122 @@ def setup_request_logging(app: FastAPI) -> None:
# ==================== 配置函数 ==================== # ==================== 配置函数 ====================
def configure_logging(settings: Settings | None = None) -> None: def configure_logging(settings: Settings | None = None) -> None:
"""
生产级日志配置,符合方案文档 §二(分类分流)和 §六(性能安全)。
日志分类:
- 业务日志 (app-biz): INFO+ 业务流程(保留 7 天)
- 系统日志 (app-sys): 框架/中间件日志(保留 7 天)
- 访问日志 (app-access): HTTP 请求日志(保留 15 天)
- 错误日志 (app-error): ERROR 级别(保留 30 天)
"""
settings = settings or get_settings() settings = settings or get_settings()
root_logger = logging.getLogger() root_logger = logging.getLogger()
root_logger.handlers.clear() root_logger.handlers.clear()
root_logger.setLevel(settings.log_level.upper()) root_logger.setLevel(settings.log_level.upper())
# ---- Formatter ----
console_formatter = logging.Formatter( console_formatter = logging.Formatter(
fmt=( fmt=(
"%(asctime)s | %(levelname)s | pid=%(process)d | %(threadName)s | " "%(asctime)s | %(levelname)s | pid=%(process)d | %(threadName)s | "
"request_id=%(request_id)s | %(name)s | %(pathname)s:%(lineno)d | %(message)s" "traceId=%(traceId)s | %(name)s | %(pathname)s:%(lineno)d | %(message)s"
), ),
datefmt="%Y-%m-%d %H:%M:%S", datefmt="%Y-%m-%d %H:%M:%S",
) )
json_formatter = JsonLogFormatter() json_formatter = JsonLogFormatter()
request_filter = RequestIdFilter()
# ---- TraceIdFilter全局注入 traceId/userId/host/app/env----
trace_filter = TraceIdFilter()
trace_filter.set_context(
app=settings.app_name,
env=settings.app_env,
host=socket.gethostname(),
)
# ---- 控制台 Handler ----
console_handler = logging.StreamHandler() console_handler = logging.StreamHandler()
console_handler.setFormatter(console_formatter) console_handler.setFormatter(console_formatter)
console_handler.addFilter(request_filter) console_handler.addFilter(trace_filter)
file_handler = DateSizeRotatingFileHandler( # ---- 业务日志文件 Handler (app-biz) ----
biz_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir, log_dir=settings.log_dir,
file_prefix=settings.log_file_prefix, file_prefix="app-biz",
max_bytes=settings.log_max_bytes, max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days, retention_days=7,
) )
file_handler.setFormatter(json_formatter) biz_file_handler.setFormatter(json_formatter)
file_handler.addFilter(request_filter) biz_file_handler.addFilter(trace_filter)
# ---- 访问日志文件 Handler (app-access) ----
access_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix="app-access",
max_bytes=settings.log_max_bytes,
retention_days=15,
)
access_file_handler.setFormatter(json_formatter)
access_file_handler.addFilter(trace_filter)
# ---- 错误日志文件 Handler (app-error) ----
error_file_handler = DateSizeRotatingFileHandler( error_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir, log_dir=settings.log_dir,
file_prefix=settings.log_error_file_prefix, file_prefix="app-error",
max_bytes=settings.log_max_bytes, max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days, retention_days=30,
) )
error_file_handler.setLevel(logging.ERROR) error_file_handler.setLevel(logging.ERROR)
error_file_handler.setFormatter(json_formatter) error_file_handler.setFormatter(json_formatter)
error_file_handler.addFilter(request_filter) error_file_handler.addFilter(trace_filter)
# ---- 注册 Handler ----
root_logger.addHandler(console_handler) root_logger.addHandler(console_handler)
root_logger.addHandler(file_handler) root_logger.addHandler(biz_file_handler)
root_logger.addHandler(access_file_handler)
root_logger.addHandler(error_file_handler) root_logger.addHandler(error_file_handler)
for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"): # ---- 访问日志 Logger 独立路由到访问日志文件 ----
logger = logging.getLogger(logger_name) access_logger = logging.getLogger("app.access")
logger.handlers.clear() access_logger.propagate = False # 不向 root 传播,避免重复写入业务日志
logger.propagate = True access_logger.addHandler(console_handler)
access_logger.addHandler(access_file_handler)
# 访问日志中的 ERROR 也要进错误日志
access_logger.addHandler(error_file_handler)
# ---- 框架类 Logger 降级 ----
for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
lg = logging.getLogger(logger_name)
lg.handlers.clear()
lg.propagate = True
# 框架类日志归入系统日志,生产环境设为 WARN
logging.getLogger("uvicorn.access").setLevel(logging.WARNING) logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
logging.getLogger("psycopg.pool").setLevel(logging.ERROR) logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
logging.getLogger("httpx").setLevel(logging.WARNING)
# ---- 兼容旧文件前缀(向后兼容)----
# 如果配置了旧的 log_file_prefix也创建一个对应的 handler
if settings.log_file_prefix and settings.log_file_prefix != "app-biz":
legacy_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_file_prefix,
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
)
legacy_file_handler.setFormatter(json_formatter)
legacy_file_handler.addFilter(trace_filter)
root_logger.addHandler(legacy_file_handler)
# 旧错误日志前缀兼容
if settings.log_error_file_prefix and settings.log_error_file_prefix != "app-error":
legacy_error_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_error_file_prefix,
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
)
legacy_error_handler.setLevel(logging.ERROR)
legacy_error_handler.setFormatter(json_formatter)
legacy_error_handler.addFilter(trace_filter)
root_logger.addHandler(legacy_error_handler)

View File

@@ -31,10 +31,11 @@ from typing import Any, Callable, Optional, TypeVar
from fastapi import Request from fastapi import Request
from app.core.logging import get_client_ip, get_logger, request_id_var from app.core.logging import get_logger, get_structured_logger, request_id_var
from app.db.platform_store import get_platform_store, new_id, utcnow from app.db.platform_store import get_platform_store, new_id, utcnow
logger = get_logger("app.op_log") logger = get_logger("app.op_log")
biz_logger = get_structured_logger("app.biz")
F = TypeVar("F", bound=Callable[..., Any]) F = TypeVar("F", bound=Callable[..., Any])
@@ -76,6 +77,92 @@ class OpStatus:
FAILURE = "failure" FAILURE = "failure"
# ==================== 中文映射表(让日志 message 直接可读)====================
MODULE_CN: dict[str, str] = {
"fine-tune": "模型训练",
"model-eval": "模型评测",
"model-inference": "模型推理",
"model-manage": "模型管理",
"dataset": "数据集",
"data-process": "数据处理",
"data-convert": "数据转换",
"compute": "算力节点",
"system": "系统",
}
ACTION_CN: dict[str, str] = {
"create": "创建",
"update": "更新",
"delete": "删除",
"start": "启动",
"stop": "停止",
"upload": "上传",
"download": "下载",
"convert": "转换",
"merge": "合并",
"import": "导入",
"login": "登录",
"logout": "退出登录",
"publish": "发布",
"retry": "重试",
"request": "请求",
}
TARGET_TYPE_CN: dict[str, str] = {
"fine_tune": "训练任务",
"eval": "评测任务",
"inference": "推理任务",
"model": "模型",
"trained_model": "训练产出模型",
"dataset": "数据集",
"dataset_version": "数据集版本",
"data_process_task": "数据处理任务",
"user": "用户",
"api": "接口",
}
def _build_cn_message(
module: str,
action: str,
target_type: str,
target_name: str | None,
target_id: str | None,
status: str,
username: str | None,
error_type: str,
error_message: str,
) -> str:
"""构建中文人类可读的日志消息,格式:[用户] 对 [模块] 执行了 [动作],结果:成功/失败"""
user_part = f"用户[{username}]" if username else "系统"
module_cn = MODULE_CN.get(module, module)
action_cn = ACTION_CN.get(action, action)
target_cn = TARGET_TYPE_CN.get(target_type, target_type or "")
target_label = target_name or target_id or ""
# 拼接操作对象描述
if target_cn and target_label:
target_part = f"{target_cn}{target_label}"
elif target_cn:
target_part = target_cn
elif target_label:
target_part = f"{target_label}"
else:
target_part = ""
if status == OpStatus.SUCCESS:
result = "成功"
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:成功"
else:
result = "失败"
err_brief = error_message[:120] if error_message else ""
err_part = f"{error_type}: {err_brief}" if error_type and err_brief else f"{error_type}" if error_type else ""
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:失败{err_part}"
return msg
def op_log( def op_log(
module: str, module: str,
action: str, action: str,
@@ -341,20 +428,54 @@ def _write_log(
trace_id: str, trace_id: str,
duration_ms: float, duration_ms: float,
) -> None: ) -> None:
"""写入操作日志到数据库""" """写入操作日志到数据库 + 文件日志"""
user_id = user.get("id") if user else None
username = user.get("username") if user else None
client_ip = None
req_method = None
req_path = None
if request:
client_ip = request.client.host if request.client else None
req_method = request.method
req_path = request.url.path
# ---- 写文件日志app-biz----
cn_msg = _build_cn_message(
module=module, action=action, target_type=target_type,
target_name=target_name, target_id=target_id, status=status,
username=username, error_type=error_type, error_message=error_message,
)
log_fields = {
"bizModule": module,
"action": action,
"targetType": target_type or "",
"targetId": target_id or "",
"targetName": target_name or "",
"opStatus": status,
"durationMs": round(duration_ms, 2),
"username": username or "",
}
if req_method:
log_fields["requestMethod"] = req_method
if req_path:
log_fields["requestPath"] = req_path
if detail:
log_fields["detail"] = detail[:500]
if error_message:
log_fields["errorMessage"] = error_message[:500]
if error_type:
log_fields["errorType"] = error_type
if status == OpStatus.SUCCESS:
biz_logger.info(cn_msg, **log_fields)
elif status == OpStatus.FAILURE:
biz_logger.error(cn_msg, **log_fields)
# ---- 写数据库 ----
try: try:
store = get_platform_store() store = get_platform_store()
log_id = new_id("op") log_id = new_id("op")
user_id = user.get("id") if user else None
username = user.get("username") if user else None
client_ip = None
req_method = None
req_path = None
if request:
client_ip = get_client_ip(request) or None
req_method = request.method
req_path = request.url.path
with store.connect() as conn: with store.connect() as conn:
conn.execute( conn.execute(
""" """
@@ -379,4 +500,4 @@ def _write_log(
), ),
) )
except Exception: except Exception:
logger.error("写入操作日志失败 module=%s action=%s", module, action, exc_info=True) logger.error("写入操作日志到数据库失败 module=%s action=%s", module, action, exc_info=True)

View File

@@ -1,5 +1,5 @@
-- ============================================================================ -- ============================================================================
-- YG Fine-Tune Platform — PostgreSQL 完整初始化脚本(一键建库建表) -- YG Zhilian — PostgreSQL 完整初始化脚本(一键建库建表)
-- ============================================================================ -- ============================================================================
-- 用途:切换到新的 PG 数据集时,一次性创建平台运行所需的全部数据库对象与 -- 用途:切换到新的 PG 数据集时,一次性创建平台运行所需的全部数据库对象与
-- 基础种子数据(幂等,可重复执行)。 -- 基础种子数据(幂等,可重复执行)。

View File

@@ -15,11 +15,11 @@ logger = get_logger(__name__)
async def run_compute_poller() -> None: async def run_compute_poller() -> None:
settings = get_settings() settings = get_settings()
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling": if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode}) logger.info("计算轮询已禁用", extra={"compute_mode": settings.compute_mode})
return return
interval = max(3, settings.compute_poll_interval_seconds) interval = max(3, settings.compute_poll_interval_seconds)
logger.info("compute poller started", extra={"interval_seconds": interval}) logger.info("计算轮询已启动", extra={"interval_seconds": interval})
# PlatformStore may run additive schema checks against a remote PostgreSQL # PlatformStore may run additive schema checks against a remote PostgreSQL
# server on first use. Keep that startup work off the Uvicorn event loop so # server on first use. Keep that startup work off the Uvicorn event loop so
# health checks and normal API requests can still respond while the DB is # health checks and normal API requests can still respond while the DB is
@@ -41,19 +41,19 @@ async def run_compute_poller() -> None:
now = time.monotonic() now = time.monotonic()
if signature != last_failure_signature or now - last_failure_logged_at >= 300: if signature != last_failure_signature or now - last_failure_logged_at >= 300:
logger.warning( logger.warning(
"compute polling reported failures count=%d first_error=%s", "计算轮询报告失败任务 count=%d first_error=%s",
len(result["failed"]), len(result["failed"]),
signature[:500], signature[:500],
) )
last_failure_signature = signature last_failure_signature = signature
last_failure_logged_at = now last_failure_logged_at = now
elif result["synced"]: elif result["synced"]:
logger.debug("compute jobs synchronized", extra={"result": result}) logger.debug("计算任务状态已同步", extra={"result": result})
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("compute poller stopped") logger.info("计算轮询已停止")
raise raise
except Exception as exc: # noqa: BLE001 - keep background polling alive except Exception as exc: # noqa: BLE001 - keep background polling alive
logger.exception("compute poller failed", extra={"error": str(exc)}) logger.exception("计算轮询执行失败", extra={"error": str(exc)})
if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)): if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
store = None store = None
await asyncio.sleep(interval) await asyncio.sleep(interval)

266
backend/test_results.json Normal file
View File

@@ -0,0 +1,266 @@
[
{
"name": "health",
"method": "GET",
"url": "/health",
"status": "PASS",
"message": "ok",
"data_desc": "obj(4 keys)"
},
{
"name": "system-info",
"method": "GET",
"url": "/system-info",
"status": "PASS",
"message": "ok",
"data_desc": "obj(7 keys)"
},
{
"name": "me",
"method": "GET",
"url": "/me",
"status": "PASS",
"message": "ok",
"data_desc": "obj(9 keys)"
},
{
"name": "dashboard/overview",
"method": "GET",
"url": "/dashboard/overview",
"status": "PASS",
"message": "ok",
"data_desc": "obj(6 keys)"
},
{
"name": "dashboard/stats",
"method": "GET",
"url": "/dashboard/stats",
"status": "PASS",
"message": "ok",
"data_desc": "obj(9 keys)"
},
{
"name": "users-list",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "9 items"
},
{
"name": "users-create",
"method": "POST",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "obj(9 keys)"
},
{
"name": "users-change-password",
"method": "POST",
"url": "/users/me/password",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "model-manage-list",
"method": "GET",
"url": "/model-manage",
"status": "PASS",
"message": "ok",
"data_desc": "5 items"
},
{
"name": "model-manage-local",
"method": "GET",
"url": "/model-manage/local-models",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "model-manage-trained",
"method": "GET",
"url": "/model-manage/trained-models",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "model-manage-export-jobs",
"method": "GET",
"url": "/model-manage/export-jobs",
"status": "PASS",
"message": "ok",
"data_desc": "11 items"
},
{
"name": "model-manage-create",
"method": "POST",
"url": "/model-manage",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "dataset-list",
"method": "GET",
"url": "/dataset-manage",
"status": "PASS",
"message": "ok",
"data_desc": "29 items"
},
{
"name": "dataset-create",
"method": "POST",
"url": "/dataset-manage",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "fine-tune-list",
"method": "GET",
"url": "/fine-tune",
"status": "PASS",
"message": "ok",
"data_desc": "9 items"
},
{
"name": "fine-tune-check-name",
"method": "GET",
"url": "/fine-tune/check-name?name=test_task",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "fine-tune-preflight",
"method": "POST",
"url": "/fine-tune/preflight",
"status": "PASS",
"message": "ok",
"data_desc": "obj(4 keys)"
},
{
"name": "model-eval-list",
"method": "GET",
"url": "/model-eval",
"status": "PASS",
"message": "ok",
"data_desc": "4 items"
},
{
"name": "dimension-list",
"method": "GET",
"url": "/dimension",
"status": "PASS",
"message": "ok",
"data_desc": "18 items"
},
{
"name": "model-compare-list",
"method": "GET",
"url": "/model-compare",
"status": "PASS",
"message": "ok",
"data_desc": "6 items"
},
{
"name": "model-chat-local-status",
"method": "GET",
"url": "/model-chat/local/status",
"status": "PASS",
"message": "ok",
"data_desc": "obj(8 keys)"
},
{
"name": "data-process-list",
"method": "GET",
"url": "/data-process",
"status": "PASS",
"message": "ok",
"data_desc": "obj(4 keys)"
},
{
"name": "compute-nodes",
"method": "GET",
"url": "/compute/nodes",
"status": "PASS",
"message": "ok",
"data_desc": "2 items"
},
{
"name": "compute-gpus",
"method": "GET",
"url": "/compute/gpus",
"status": "PASS",
"message": "ok",
"data_desc": "2 items"
},
{
"name": "compute-queue",
"method": "GET",
"url": "/compute/queue",
"status": "PASS",
"message": "ok",
"data_desc": "0 items"
},
{
"name": "log-files",
"method": "GET",
"url": "/log-files",
"status": "PASS",
"message": "ok",
"data_desc": "2 items"
},
{
"name": "training-log-files",
"method": "GET",
"url": "/training-log-files",
"status": "PASS",
"message": "ok",
"data_desc": "9 items"
},
{
"name": "web-log",
"method": "POST",
"url": "/web-log",
"status": "PASS",
"message": "ok",
"data_desc": "obj(3 keys)"
},
{
"name": "data-convert-list",
"method": "GET",
"url": "/data-convert",
"status": "PASS",
"message": "ok",
"data_desc": "obj(2 keys)"
},
{
"name": "error-404",
"method": "GET",
"url": "/nonexistent-endpoint",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-unauthorized",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "10 items"
},
{
"name": "viewer-login",
"method": "POST",
"url": "/login",
"status": "SKIP",
"message": "viewer user not found",
"data_desc": "-"
}
]

View File

@@ -0,0 +1,258 @@
[
{
"name": "crud-model-create",
"method": "POST",
"url": "/model-manage",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "crud-model-get-by-id",
"method": "GET",
"url": "/model-manage/m_dcebe627d644",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "crud-model-update",
"method": "PUT",
"url": "/model-manage/m_dcebe627d644",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "crud-model-purpose",
"method": "PUT",
"url": "/model-manage/m_dcebe627d644/purpose",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "crud-model-delete",
"method": "DELETE",
"url": "/model-manage/m_dcebe627d644",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "crud-dataset-create",
"method": "POST",
"url": "/dataset-manage",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "crud-dataset-get-by-id",
"method": "GET",
"url": "/dataset-manage/ds_b8dd915d5e09",
"status": "PASS",
"message": "ok",
"data_desc": "obj(28 keys)"
},
{
"name": "crud-dataset-update",
"method": "PUT",
"url": "/dataset-manage/ds_b8dd915d5e09",
"status": "PASS",
"message": "ok",
"data_desc": "obj(27 keys)"
},
{
"name": "crud-dataset-delete",
"method": "DELETE",
"url": "/dataset-manage/ds_b8dd915d5e09",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "crud-user-create",
"method": "POST",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "obj(9 keys)"
},
{
"name": "crud-user-list",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "11 items"
},
{
"name": "crud-user-update",
"method": "PUT",
"url": "/users/u_eb94ee60769e",
"status": "PASS",
"message": "ok",
"data_desc": "obj(9 keys)"
},
{
"name": "crud-user-reset-pwd",
"method": "POST",
"url": "/users/u_eb94ee60769e/reset-password",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "crud-user-delete",
"method": "DELETE",
"url": "/users/u_eb94ee60769e",
"status": "PASS",
"message": "ok",
"data_desc": "obj(2 keys)"
},
{
"name": "error-invalid-model-id",
"method": "GET",
"url": "/model-manage/nonexistent_id_12345",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-invalid-dataset-id",
"method": "GET",
"url": "/dataset-manage/nonexistent_id_12345",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-invalid-finetune-id",
"method": "GET",
"url": "/fine-tune/nonexistent_id_12345",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-invalid-eval-id",
"method": "GET",
"url": "/model-eval/nonexistent_id_12345",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-duplicate-login",
"method": "POST",
"url": "/login",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "error-missing-fields",
"method": "POST",
"url": "/model-manage",
"status": "PASS",
"message": "ok",
"data_desc": "obj(17 keys)"
},
{
"name": "auth-no-token-users",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "10 items"
},
{
"name": "auth-no-token-finetune",
"method": "GET",
"url": "/fine-tune",
"status": "FAIL(code=-1)",
"message": "",
"data_desc": "null"
},
{
"name": "auth-invalid-token",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "10 items"
},
{
"name": "auth-empty-token",
"method": "GET",
"url": "/users",
"status": "PASS",
"message": "ok",
"data_desc": "10 items"
},
{
"name": "crud-dimension-create",
"method": "POST",
"url": "/dimension",
"status": "PASS",
"message": "ok",
"data_desc": "obj(6 keys)"
},
{
"name": "crud-dimension-get",
"method": "GET",
"url": "/dimension/dim_12124ed44bbe",
"status": "PASS",
"message": "ok",
"data_desc": "obj(6 keys)"
},
{
"name": "crud-dimension-update",
"method": "PUT",
"url": "/dimension/dim_12124ed44bbe",
"status": "PASS",
"message": "ok",
"data_desc": "obj(6 keys)"
},
{
"name": "crud-dimension-delete",
"method": "DELETE",
"url": "/dimension/dim_12124ed44bbe",
"status": "PASS",
"message": "ok",
"data_desc": "obj(1 keys)"
},
{
"name": "compute-nodes-detail",
"method": "GET",
"url": "/compute/nodes",
"status": "PASS",
"message": "ok",
"data_desc": "2 items"
},
{
"name": "compute-nodes-list2",
"method": "GET",
"url": "/compute/nodes",
"status": "PASS",
"message": "ok",
"data_desc": "2 items"
},
{
"name": "compute-node-replicas",
"method": "GET",
"url": "/compute/nodes/node_1499a71b4871/replicas",
"status": "PASS",
"message": "ok",
"data_desc": "9 items"
},
{
"name": "compute-node-engines",
"method": "GET",
"url": "/compute/nodes/node_1499a71b4871/engines",
"status": "PASS",
"message": "ok",
"data_desc": "obj(2 keys)"
}
]

View File

@@ -24,7 +24,7 @@ from compute.engines.llama_factory.inference import get_inference_session
def create_app() -> FastAPI: def create_app() -> FastAPI:
app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs()) app = FastAPI(title="YG Zhilian Compute API", **docs_kwargs())
jobs: dict[str, dict[str, Any]] = {} jobs: dict[str, dict[str, Any]] = {}
cache_locks: dict[str, asyncio.Lock] = {} cache_locks: dict[str, asyncio.Lock] = {}
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF" route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"

View File

@@ -40,7 +40,7 @@ services:
- "${BACKEND_API_PORT:-17861}:8000" - "${BACKEND_API_PORT:-17861}:8000"
environment: environment:
APP_ENV: ${APP_ENV:-prod} APP_ENV: ${APP_ENV:-prod}
APP_NAME: ${APP_NAME:-YG Fine-Tune Platform API} APP_NAME: ${APP_NAME:-YG Zhilian API}
MODELTF_ROUTE_PREFIX: ${MODELTF_ROUTE_PREFIX:-/modelTF} MODELTF_ROUTE_PREFIX: ${MODELTF_ROUTE_PREFIX:-/modelTF}
ENABLE_DOCS: ${ENABLE_DOCS:-false} ENABLE_DOCS: ${ENABLE_DOCS:-false}
CORS_ALLOW_ORIGINS: ${CORS_ALLOW_ORIGINS:-http://localhost:16801,http://127.0.0.1:16801} CORS_ALLOW_ORIGINS: ${CORS_ALLOW_ORIGINS:-http://localhost:16801,http://127.0.0.1:16801}

View File

@@ -1,5 +1,6 @@
# 平台治理功能使用指南 # 平台治理功能使用指南
> 版本v1.3 > 版本v1.3
> 日期2026-08-19 > 日期2026-08-19
> 适用版本YG Fine-Tune Platform v1.0+ > 适用版本YG Fine-Tune Platform v1.0+

View File

@@ -150,7 +150,7 @@ def docs_kwargs() -> dict[str, Any]:
return {"docs_url": None, "redoc_url": None, "openapi_url": None} return {"docs_url": None, "redoc_url": None, "openapi_url": None}
# main.py # main.py
app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs()) app = FastAPI(title="YG Zhilian Compute API", **docs_kwargs())
``` ```
**判定顺序(优先级从高到低)** **判定顺序(优先级从高到低)**

View File

@@ -1,412 +1,489 @@
# 生产级日志系统设计方案 好的,这是一份可以直接放在项目根目录的 `日志规范要求.md`,涵盖**格式标准、分类分级、内容规范、链路追踪、性能安全、运维告警**六大模块,每条规范都配有正反例,你的团队照着这个写代码就行。
> 版本v1.0
> 日期2026-08-17
> 状态:待评审
--- ---
## 一、现状分析 # 生产级日志规范要求
### 1.1 当前日志架构 > 版本v2.0 | 适用于所有后端服务Python/Java/Go/Node.js
```
┌─────────────┐
│ FastAPI │ ← 请求入口
└──────┬──────┘
┌─────────────┐
│ Logging │ ← Python logging 模块
│ Middleware │
└──────┬──────┘
├──────────────────┬──────────────────┐
▼ ▼
┌─────────────┐ ┌─────────────┐
│ Console │ │ File │ ← 输出目标
│ (开发环境) │ │ (JSON格式) │
└─────────────┘ └─────────────┘
┌─────────────┐
│ audit_logs │ ← 审计日志表
│ (PostgreSQL) │
└─────────────┘
```
### 1.2 现有组件
| 组件 | 文件路径 | 功能 |
|------|----------|------|
| `logging.py` | `backend/app/core/` | 日志配置、JSON 格式化、按日期/大小轮转 |
| `platform_store.py` | `backend/app/db/` | `record_audit()` 审计日志写入 |
| `002_governance.sql` | `backend/app/db/sql/` | `audit_logs` 表结构 |
### 1.3 存在的问题
| 问题 | 影响 | 严重程度 |
|------|------|----------|
| **无结构化日志分级** | DEBUG/INFO/WARNING/ERROR 全部混在一起,无法按级别过滤查看 | 🔴 高 |
| **无请求链路追踪** | 一个请求从进入到返回经过哪些服务/函数,无法串联 | 🔴 高 |
| **审计日志与业务耦合** | 各模块手动调用 `record_audit()`,容易遗漏 | 🟡 中 |
| **无敏感数据脱敏** | 用户 token、密码等可能明文记录 | 🔴 高 |
| **无日志聚合查询** | 无法按用户/时间范围/操作类型快速检索 | 🟡 中 |
| **无告警通知** | 系统异常无法主动推送通知 | 🟡 中 |
| **日志文件无归档策略** | 只有简单的过期删除,无压缩归档 | 🟢 低 |
--- ---
## 二、设计目标 ## 一、核心原则
### 2.1 核心原则 | 原则 | 说明 |
|------|------|
1. **结构化** - 日志有固定 schema便于机器解析和查询 | **结构化** | 所有日志必须输出为 JSON 格式,便于自动化采集和分析 |
2. **可追** - 每个请求有唯一 ID可串联完整调用链路 | **可追** | 每个请求链路必须有唯一的 `traceId`,贯穿全流程 |
3. **分级输出** - 不同环境输出不同级别,生产环境不输出 DEBUG | **有上下文** | 每条日志必须包含足够的业务信息,能独立理解发生了什么 |
4. **安全合规** - 敏感数据自动脱敏token、密码、手机号等 | **高性能** | 异步打印,禁止在业务主流程中同步写磁盘 |
5. **高性能** - 日志写入不影响业务接口性能(异步写入) | **安全合规** | 敏感信息自动脱敏禁止打印密码、token、身份证号等 |
6. **可观测** - 支持快速检索、统计、告警 | **可告警** | ERROR 日志必须触发实时告警,且有明确的错误分类 |
### 2.2 日志分级标准
| 级别 | 使用场景 | 示例 | 生产环境 |
|------|----------|------|:--------:|
| **DEBUG** | 开发调试 | 变量值、SQL 语句、完整堆栈 | ❌ 不输出 |
| **INFO** | 正常流程记录 | 任务创建成功、用户登录 | ✅ 记录 |
| **WARNING** | 可恢复异常 | 重试操作、参数校验失败、资源不足 | ✅ 记录 |
| **ERROR** | 需要人工介入 | 数据库连接失败、第三方 API 超时 | ✅ 记录 + 告警 |
| **CRITICAL** | 系统不可用 | 磁盘满、主节点宕机 | ✅ 记录 + 立即告警 |
--- ---
## 三、技术方案 ## 二、日志分类
### 3.1 整体架构 生产环境必须按用途分流存储,**禁止所有日志混写在同一文件**
``` | 分类 | 文件名示例 | 用途 | 保留周期 |
┌─────────────────────────────────────────────────────────────────────┐ |------|-----------|------|----------|
│ 应用层 (Application Layer) │ | **业务日志** | `app-biz.log` | 记录核心业务流程(订单、支付、登录、任务状态变更等) | 7天热存 + 30天冷存 |
├─────────────────────────────────────────────────────────────────────┤ | **系统日志** | `app-sys.log` | 记录框架、中间件、连接池、GC、线程池状态 | 7天 |
│ │ | **访问日志** | `app-access.log` | 记录所有 HTTP/RPC 请求的入参、出参、耗时 | 15天用于审计 |
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ | **错误日志** | `app-error.log` | **仅记录 ERROR 级别**,含完整堆栈 | 30天用于复盘 |
│ │ 数据集管理 │ │ 微调训练 │ │ 模型推理 │ │ 用户认证 │ ... │
│ └─────┬────┘ └─────┬────┘ └─────┬────┘ └─────┬────┘ │
│ │ │ │ │ │
│ └────────────┴───────────┴──────────┘ │
│ ▼ │
│ ┌──────────────┐ │
│ │ Structured │ ← 结构化日志中间件 │
│ │ Logger │ │
│ └──────┬───────┘ │
│ │ │
│ ┌────────────┬────────────┬─────────────┐ │
│ ▼ ▼ ▼ │ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────┐ │
│ │ Console │ │ File │ │ 审计DB │ │ 告警 │ │
│ │ (开发) │ │ (JSON) │ │ (PG) │ │(可选) │ │
│ └──────────┘ └──────────┘ └──────────┘ └────────┘ │
│ │
└─────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────┐
│ 可观测层 (Observability) │
├─────────────────────────────────────────────────────────────┤
│ ┌───────────┐ ┌───────────┐ ┌───────────┐ │
│ │ Grafana │ │ Kibana │ │ PagerDuty │ ... │
│ │ (查询) │ │ (分析) │ │ (告警) │ │
│ └───────────┘ └───────────┘ └───────────┘ │
└─────────────────────────────────────────────────────────────┘
```
### 3.2 日志 Schema 设计 **配置要点**
- 业务日志和错误日志必须独立文件,便于快速定位异常
- 框架类日志(如 `httpx``urllib3`)归入系统日志,且生产环境设为 WARN 级别
#### 3.2.1 应用日志 (app.log) ---
## 三、日志格式标准
### 3.1 统一 JSON 格式
所有日志必须输出为以下 JSON 结构,**字段名不得随意变更**
```json ```json
{ {
"timestamp": "2026-08-17T10:30:00.000Z", "@timestamp": "2026-08-19T10:30:45.123+08:00",
"level": "INFO", "level": "INFO",
"trace_id": "req-abc123", "logger": "com.order.service.OrderService",
"parent_span_id": "span-xyz789", // OpenTelemetry Span "traceId": "abc-123-def-456",
"request": { "spanId": "span-001",
"method": "POST", "userId": "U10086",
"path": "/dataset-manage", "message": "订单状态更新成功",
"client_ip": "192.168.1.100", "fields": {
"user_agent": "Mozilla/5.0...", "orderId": "ORD-20260819-001",
"user_id": "u_admin" "fromStatus": "PENDING",
"toStatus": "PAID",
"costMs": 23,
"retryCount": 0
}, },
"module": "dataset.router", "file": "OrderService.java:156",
"function": "create_dataset", "thread": "http-nio-8080-exec-8",
"message": "数据集创建成功", "host": "pod-order-7x9k2",
"extra": { "app": "order-service",
"dataset_id": "ds_abc123", "env": "prod"
"dataset_name": "训练数据"
},
"duration_ms": 125,
"status_code": 200,
"error": null
} }
``` ```
#### 3.2.2 审计日志 (audit_logs 表) ### 3.2 字段说明
```sql | 字段 | 类型 | 必填 | 说明 |
-- 已有表结构(保持不变) |------|------|------|------|
CREATE TABLE IF NOT EXISTS audit_logs ( | `@timestamp` | string | ✅ | ISO8601 格式,带时区(如 `+08:00` |
id TEXT PRIMARY KEY, | `level` | string | ✅ | DEBUG / INFO / WARNING / ERROR |
tenant_id TEXT, | `logger` | string | ✅ | 日志记录器名称,通常为类名 |
project_id TEXT, | `traceId` | string | ✅ | 全局唯一追踪ID从入口生成全链路透传 |
actor_id TEXT, -- 操作人 | `spanId` | string | 推荐 | 当前节点ID用于区分调用链中的不同服务 |
action TEXT, -- 操作类型: create/delete/update/acl.set/login... | `userId` | string | 业务必填 | 操作用户标识,未登录可为空 |
target_type TEXT, -- 资源类型: dataset/model/fine-tune/user... | `message` | string | ✅ | 人类可读的日志摘要,简洁明了 |
target_id TEXT, -- 资源 ID | `fields` | object | ✅ | 结构化业务字段,所有动态数据放入此处 |
detail TEXT, -- 详细信息 JSON | `file` | string | 推荐 | 代码文件名和行号 |
client_ip TEXT, -- 客户端 IP | `thread` | string | 推荐 | 线程名 |
time TEXT, -- 操作时间 | `host` | string | 推荐 | 主机名或 Pod 名称 |
| `app` | string | ✅ | 应用名称 |
| `env` | string | ✅ | dev / test / staging / prod |
| `error` | object | ERROR时必填 | 包含 `type``message``stack_trace` |
-- 新增字段 ### 3.3 ERROR 日志额外字段
trace_id TEXT, -- 关联应用日志的请求追踪 ID
request_method TEXT, -- HTTP 方法
request_path TEXT, -- 请求路径
status_code INTEGER, -- 响应状态码
duration_ms REAL, -- 耗时(ms)
extra JSONB -- 扩展信息
);
-- 新增索引 `level = ERROR` 时,必须包含:
CREATE INDEX IF NOT EXISTS idx_audit_trace ON audit_logs(trace_id);
CREATE INDEX IF NOT EXISTS idx_audit_actor_time ON audit_logs(actor_id, time); ```json
{
"error": {
"type": "ConnectionTimeoutError",
"message": "连接下游服务超时",
"stack_trace": "完整堆栈信息...",
"root_cause": "socket timeout after 3000ms"
}
}
``` ```
### 3.3 日志中间件设计 ---
## 四、日志内容规范
### 4.1 日志级别使用标准
| 级别 | 使用场景 | 示例 |
|------|----------|------|
| **DEBUG** | 开发调试信息,生产环境**默认关闭** | 变量值、中间计算结果 |
| **INFO** | 关键业务流程节点、状态变更、外部调用结果 | 订单创建成功、支付回调收到、任务状态变更 |
| **WARNING** | 可恢复的异常、降级处理、重试、资源使用超阈值 | 重试第3次成功、缓存穿透、磁盘使用率>80% |
| **ERROR** | 业务失败、系统异常、需要人工介入的错误 | 支付失败、数据库连接断开、第三方接口返回500 |
### 4.2 INFO 级别日志内容要求
每条 INFO 日志必须回答 **5W1H**
```
Who谁操作 + What做了什么 + When何时 + Where哪个服务/节点) + Why上下文 + How结果如何
```
**✅ 正例:**
```python ```python
# backend/app/core/logging.py 新增 logger.info(
"任务日志拉取成功",
class StructuredLogger: extra={
"""结构化日志记录器""" "userId": "U10086",
"fields": {
def __init__(self, name: str): "jobId": "ft_a016cd8885cd",
self.logger = logging.getLogger(name) "tailLines": 5000,
self.trace_id = context_var.get("trace_id") "logSize": "2.3MB",
"costMs": 42,
def info(self, msg: str, **kwargs): "source": "frontend"
self._log("INFO", msg, **kwargs)
def warning(self, msg: str, **kwargs):
self._log("WARNING", msg, **kwargs)
def error(self, msg: str, **kwargs):
self._log("ERROR", msg, **kwargs)
def _log(self, level: str, msg: str,
user_id: str = None,
target_type: str = None,
target_id: str = None,
duration_ms: float = None,
status_code: int = None,
error: Exception = None,
**extra):
"""统一日志记录方法"""
log_entry = {
"timestamp": datetime.utcnow().isoformat(),
"level": level,
"trace_id": self.trace_id.get(),
"request": {
"user_id": user_id or current_user_id(),
"client_ip": client_ip(),
# ...
},
"module": calling_module,
"message": msg,
"target": {
"type": target_type,
"id": target_id,
},
"extra": extra,
"duration_ms": duration_ms,
"error": format_exception(error) if error else None,
} }
}
# 1. 写入控制台/文件 )
self.logger.log(level, json.dumps(log_entry))
# 2. 异步写入审计表(如果需要)
if level in ("WARNING", "ERROR", "CRITICAL"):
async_write_audit(log_entry)
``` ```
### 3.4 装饰器模式(推荐) **❌ 反例(禁止):**
使用 Python 裁饰器自动记录,避免手动调用:
```python ```python
# backend/app/core/log_decorator.py logger.info("get logs success")
logger.info(f"job {job_id} status is {status}") # 禁止字符串拼接
def audit_log(action: str, target_type: str = ""):
"""审计日志装饰器"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
result = await func(*args, **kwargs)
# 自动记录审计日志
record_audit(
action=action,
target_type=target_type,
target_id=kwargs.get('id') or result.get('id'),
detail=f"params={kwargs}"
)
return result
return wrapper
return decorator
# 使用示例
@audit_log("dataset.create", "dataset")
async def create_dataset(...):
# 业务逻辑
pass
``` ```
### 3.5 敏感数据脱敏规则 ### 4.3 WARNING/ERROR 日志内容要求
**必须包含三要素**
1. 发生了什么what
2. 为什么发生why—— 异常类型/错误码
3. 业务上下文context—— 哪个业务对象失败了
**✅ 正例:**
```python
logger.warning(
"计算轮询检测到任务失败",
extra={
"fields": {
"jobId": "ft_a016cd8885cd",
"failureReason": "GPU资源不足",
"errorCode": "RESOURCE_INSUFFICIENT",
"retryCount": 3,
"lastRetryTime": "2026-08-19T08:38:25.495+08:00"
}
}
)
```
**❌ 反例(禁止):**
```python
logger.warning("compute polling reported failures") # 没有任何上下文
logger.error(f"error: {e}") # 只打了异常信息没有业务ID
```
### 4.4 禁止打印的内容
| 类别 | 说明 |
|------|------|
| 密码/密钥 | 任何形式的 `password``secret``token``api_key` |
| 个人隐私 | 身份证号、手机号(需脱敏)、银行卡号 |
| 超大对象 | 超过 1KB 的 JSON/列表/文本内容 |
| 循环日志 | 禁止在 for/while 循环内打印 INFO 及以上级别 |
| 异常堆栈重复 | 同一异常在一个请求中只打印一次完整堆栈 |
---
## 五、链路追踪TraceId
### 5.1 基本原则
- **入口生成**:网关/前端/定时任务入口生成全局唯一的 `traceId`32位UUID
- **全链路透传**:通过 HTTP Header`X-Trace-Id`、RPC Meta、消息队列 Property 向下游传递
- **日志自动注入**:所有日志输出自动追加 `traceId`,代码中无需手动传入
- **跨线程传递**:使用 `MDC``ContextVars` 实现跨线程/协程的透传
### 5.2 实现要求
```python ```python
# backend/app/core/masking.py # Python 示例:使用 logging 的 Filter 自动注入 traceId
class TraceIdFilter(logging.Filter):
def filter(self, record):
record.traceId = get_current_trace_id() or "N/A"
return True
SENSITIVE_FIELDS = { # 所有日志自动带上 traceId
"token": "***", logger.info("订单创建成功") # 自动注入 traceId代码无需传参
"password": "***", ```
"phone": lambda x: f"{x[:3]}****{x[-4:]}",
"email": lambda x: x[0] + "***" + x.split("@")[1] if "@" in x else "***",
"id_card": lambda x: f"{x[:6]}********{x[-4:]}",
}
**❌ 绝对禁止**`traceId` 字段值为 `"-"``null`
---
## 六、性能与安全
### 6.1 性能要求
| 配置项 | 要求 |
|--------|------|
| **异步打印** | 必须使用异步 Appender禁止同步刷盘阻塞业务线程 |
| **单文件大小** | ≤ 1GB达到阈值自动滚动 |
| **滚动策略** | 按大小滚动(如 1GB或按天滚动 |
| **采样率** | 核心业务 100%,非核心(如健康检查、非关键查询)≤ 10% |
| **禁止打印循环** | 循环体内不得打印 INFO 及以上日志 |
| **大对象截断** | 超过 1KB 的内容自动截断前500字符 + 后500字符 |
### 6.2 安全要求
| 要求 | 说明 |
|------|------|
| **敏感字段自动脱敏** | 对 `mobile``idCard``password``token` 等字段自动掩码 |
| **脱敏规则** | 手机号:`138****5678`;身份证:`110***********1234` |
| **日志查询权限** | 生产日志平台必须有 RBAC 权限控制,禁止随意导出 |
| **审计追踪** | 谁在什么时候查询了哪些日志,必须记录审计日志 |
### 6.3 脱敏实现示例
```python
# 脱敏工具函数
def mask_sensitive(data: dict) -> dict: def mask_sensitive(data: dict) -> dict:
"""递归脱敏字典中的敏感字段""" sensitive_keys = {"password", "token", "api_key", "mobile", "id_card"}
for key, value in data.items(): for key in sensitive_keys:
if key in SENSITIVE_FIELDS: if key in data:
data[key] = SENSITIVE_FIELDS[key](value) if callable(SENSITIVE_FIELDS[key]) else "***" value = str(data[key])
elif isinstance(value, dict): if len(value) >= 11: # 手机号
mask_sensitive(value) data[key] = value[:3] + "****" + value[-4:]
elif len(value) >= 18: # 身份证
data[key] = value[:3] + "***********" + value[-4:]
return data return data
``` ```
--- ---
## 四、实施计划 ## 七、运维与告警
### 4.1 Phase 1基础增强1-2 天) ### 7.1 日志采集架构
- [ ] **P1-1** 升级 `JsonLogFormatter`,增加 `trace_id` 字段 ```
- [ ] **P1-2** 新增 `StructuredLogger` 封装类 应用日志(本地文件)
- [ ] **P1-3** 统一所有模块的日志格式为 JSON
- [ ] **P1-4** 实现 `mask_sensitive()` 脱敏函数 Filebeat轻量采集器
- [ ] **P1-5** 审计日志表新增 `trace_id``duration_ms` 字段
Kafka削峰填谷保证不丢
Logstash解析、过滤、脱敏
Elasticsearch索引存储
Kibana / Grafana查询展示
```
### 4.2 Phase 2自动化2-3 天) **关键要求**
- 禁止应用直接写入 ES必须经过 Kafka 缓冲
- Filebeat 采集失败时必须有本地持久化和重试机制
- [ ] **P2-1** 编写 `@audit_log` 装饰器 ### 7.2 告警规则
- [ ] **P2-2** 为关键业务接口添加装饰器:
- 数据集 CRUD
- 模型 CRUD
- 微调任务创建/删除
- 用户登录/登出
- ACL 授权变更
- [ ] **P2-3** 实现日志异步写入队列(避免影响性能)
### 4.3 Phase 3可观测性3-5 天) | 条件 | 动作 | 优先级 |
|------|------|--------|
| 同一服务 5 分钟内出现 ≥ 3 次 ERROR | 钉钉/企微告警 + 电话P0级 | 最高 |
| 同一服务 10 分钟内 ERROR 率 > 5% | 钉钉告警P1级 | 高 |
| 磁盘使用率 > 80% | 钉钉告警P2级 | 中 |
| 单个 ERROR 堆栈重复出现 ≥ 10 次/分钟 | 聚合为一条告警,避免轰炸 | - |
- [ ] **P3-1** 集成 ELK Stack 或 Loki可选 ### 7.3 错误聚合策略
- [ ] **P3-2** 编写 Grafana 仪表板:
- 请求量趋势图 - 相同 `error.type` + 相同 `logger` + 相同堆栈前3行 → 视为同一类错误
- 错误率统计 - 同一类错误 5 分钟内只发 **1 条告警**(防告警轰炸)
- 慢接口 TOP10 - 告警内容必须包含:`app``env``error.type`、首次发生时间、最近发生时间、累计次数
- 用户操作审计面板
- [ ] [ ] **P3-3** 实现告警规则(错误率超阈值触发)
--- ---
## 五、配置示例 ## 八、日志查询与使用规范
### 5.1 日志配置 (settings) | 场景 | 查询方式 | 时效要求 |
|------|----------|----------|
| 日常运维 | Kibana 按 `traceId``userId` 检索 | 实时 |
| 异常排查 | 按 `app` + `level:ERROR` + 时间范围 | 实时 |
| 业务审计 | 按 `userId` + `logger:xxx` + 时间范围 | 30分钟内 |
| 性能分析 | 按 `costMs` 排序,找出慢请求 | 实时 |
| 安全审计 | 查询所有访问日志,按 IP/用户筛选 | 按需 |
```yaml ---
# config.yaml 或 .env
LOGGING:
level: INFO # 生产环境用 INFO开发用 DEBUG
dir: ./logs
file_prefix: app
max_bytes: 50MB # 单文件最大 50MB
retention_days: 30 # 保留 30 天
error_prefix: error # 错误日志单独文件
json: true # JSON 格式输出
AUDIT: ## 九、检查清单Code Review 必查)
enabled: true
auto_record: true # 是否自动记录(通过装饰器)
sensitive_mask: true # 启用敏感数据脱敏
```
### 5.2 日志输出示例 | 检查项 | 通过标准 |
|--------|----------|
| ☐ JSON 格式 | 所有日志输出均为 JSON字段名符合规范 |
| ☐ traceId | 所有日志都有 `traceId`,且不为 `"-"` |
| ☐ userId | 涉及用户操作的日志都有 `userId` |
| ☐ 业务上下文 | INFO 日志包含 `fields`有订单ID/任务ID等 |
| ☐ ERROR 日志 | 包含 `error.type` + `stack_trace` + 业务ID |
| ☐ 敏感信息 | 无密码/手机号明文,有脱敏处理 |
| ☐ 异步打印 | 使用异步 Appender |
| ☐ 日志级别 | 框架类日志 ≥ WARN业务日志分级合理 |
| ☐ 循环内日志 | 无循环内的 INFO 日志 |
| ☐ 日志分流 | 业务/系统/错误日志分文件存储 |
**控制台输出(开发环境):** ---
```
2026-08-17 18:30:00.123 | INFO | pid=12345 | MainThread | req=req-abc | dataset.router:create_dataset | dataset/router.py:45 | 数据集创建成功 {"dataset_id":"ds_abc"} ## 十、附:完整日志示例
```
### 示例一业务成功流程INFO
**文件输出JSON 格式):**
```json ```json
{"@timestamp":"2026-08-17T18:30:00.123Z","level":"INFO","logger":"dataset.router","message":"数据集创建成功","module":"dataset.router","function":"create_dataset","file":"dataset/router.py","line":45,"process":12345,"thread":"MainThread","request_id":"req-abc","extra":{"dataset_id":"ds_abc"}} {
"@timestamp": "2026-08-19T10:30:45.123+08:00",
"level": "INFO",
"logger": "app.services.job_service",
"traceId": "tracer-abc123xyz789",
"spanId": "span-001",
"userId": "U10086",
"message": "计算任务创建成功",
"fields": {
"jobId": "ft_a016cd8885cd",
"jobType": "fine_tuning",
"modelId": "model-llama2-7b",
"datasetId": "ds-20260819-001",
"gpuCount": 4,
"estimatedTime": "2h30m",
"costMs": 1523,
"source": "api"
},
"file": "job_service.py:234",
"thread": "MainThread",
"host": "compute-pod-7x9k2",
"app": "compute-service",
"env": "prod"
}
``` ```
**审计日志查询 SQL** ### 示例二可恢复的警告WARNING
```sql
-- 查询某用户最近7天的所有操作
SELECT time, action, target_type, target_id, detail, client_ip
FROM audit_logs
WHERE actor_id = 'u_admin'
AND time >= now() - interval '7 days'
ORDER BY time DESC;
-- 查询某资源的授权变更历史 ```json
SELECT * FROM audit_logs {
WHERE action LIKE '%acl%' "@timestamp": "2026-08-19T08:38:27.074+08:00",
AND target_id = 'ds_abc123' "level": "WARNING",
ORDER BY time DESC; "logger": "app.workers.compute_poller",
"traceId": "tracer-xyz789abc123",
"spanId": "span-002",
"userId": "U10086",
"message": "计算任务状态轮询检测到失败,进入重试",
"fields": {
"jobId": "ft_a016cd8885cd",
"currentStatus": "FAILED",
"failureReason": "GPU节点不可用",
"errorCode": "NODE_UNAVAILABLE",
"retryCount": 2,
"maxRetries": 3,
"nextRetryDelay": 30
},
"file": "compute_poller.py:45",
"thread": "Thread-8",
"host": "compute-pod-7x9k2",
"app": "compute-service",
"env": "prod"
}
```
### 示例三严重错误ERROR+ 告警
```json
{
"@timestamp": "2026-08-19T08:38:30.456+08:00",
"level": "ERROR",
"logger": "app.services.payment_service",
"traceId": "tracer-pay-789xyz",
"spanId": "span-003",
"userId": "U10086",
"message": "支付调用失败,订单状态回滚",
"fields": {
"orderId": "ORD-20260819-001",
"amount": 299.00,
"paymentMethod": "wechat",
"retryCount": 3,
"hasRollback": true
},
"error": {
"type": "PaymentTimeoutException",
"message": "支付网关超时等待响应超过5000ms",
"stack_trace": "Traceback (most recent call last):\n File \"payment_service.py:89\" ...",
"root_cause": "upstream gateway 10.0.1.100:8080 connection timeout"
},
"file": "payment_service.py:156",
"thread": "http-nio-8080-exec-12",
"host": "order-pod-3f8k1",
"app": "order-service",
"env": "prod"
}
```
### 示例四完整的请求访问日志ACCESS
```json
{
"@timestamp": "2026-08-19T10:30:45.001+08:00",
"level": "INFO",
"logger": "app.middleware.access_log",
"traceId": "tracer-abc123xyz789",
"userId": "U10086",
"message": "HTTP 请求完成",
"fields": {
"method": "POST",
"path": "/api/v1/jobs",
"statusCode": 200,
"clientIp": "192.168.1.100",
"userAgent": "Mozilla/5.0 ...",
"requestSize": 2048,
"responseSize": 512,
"costMs": 1523,
"requestBody": {"modelId": "model-llama2-7b", "datasetId": "ds-001"}, // 已脱敏
"responseBody": {"jobId": "ft_a016cd8885cd", "status": "CREATED"} // 已脱敏
},
"app": "compute-service",
"env": "prod"
}
``` ```
--- ---
## 、附录 ## 十一、附录:技术栈配置速查
### A. 日志关键字段说明 ### Python (Logging + JSON)
| 字段 | 类型 | 说明 | 示例 | ```python
|------|------|------|------| import logging
| `trace_id` | string | 请求唯一标识,用于串联一次请求的所有日志 | `req-uuid-1234` | import json
| `parent_span_id` | string | 父 Span ID用于分布式追踪 | `span-parent-5678` | from pythonjsonlogger import jsonlogger
| `actor_id` | string | 操作人用户 ID | `u_admin` |
| `action` | string | 操作动作 | `dataset.create`, `model.delete`, `login.success` |
| `target_type` | string | 操作的资源类型 | `dataset`, `trained_model`, `user` |
| `target_id` | string | 资源 ID | `ds_abc123` |
| `detail` | string/json | 操作详情 | `{"name": "训练数据", "type": "train"}` |
| `client_ip` | string | 客户端 IP | `192.168.1.100` |
| `duration_ms` | real | 接口耗时(ms) | `125.5` |
| `status_code` | int | HTTP 状态码 | `200`, `404`, `500` |
### B. 推荐的 Python 日志库对比 logger = logging.getLogger("app")
handler = logging.FileHandler("logs/app-biz.log")
formatter = jsonlogger.JsonFormatter(
fmt="%(asctime)s %(levelname)s %(name)s %(traceId)s %(message)s",
rename_fields={"asctime": "@timestamp", "name": "logger"}
)
handler.setFormatter(formatter)
logger.addHandler(handler)
```
| 库 | 特点 | 适用场景 | ### Java (Logback + JSON)
|-----|------|---------|
| `structlog` | 结构化日志,高性能 | 推荐 ✅ |
| `loguru` | 简单易用,自动配置 | 小型项目 |
| `logging` | Python 标准库 | 当前已使用 |
### C. 参考链接 ```xml
<!-- logback-spring.xml -->
<appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>@timestamp</timestamp>
<level>level</level>
<thread>thread</thread>
<logger>logger</logger>
</fieldNames>
</encoder>
</appender>
```
- [Python logging cookbook](https://docs.python.org/3/howto/logging.html) ### Go (Zap + JSON)
- [ELK Stack 官方文档](https://www.elastic.co/guide/index.html)
- [OpenTelemetry 规范](https://opentelemetry.io/docs/) ```go
logger, _ := zap.NewProduction()
logger.Info("订单创建成功",
zap.String("traceId", traceId),
zap.String("userId", userId),
zap.String("orderId", orderId),
zap.Int64("costMs", costMs),
)
```

View File

@@ -1,4 +1,4 @@
# 远光软件微调平台 - 前端Vue 3 # 远光智炼 - 前端Vue 3
> 由原 `web/` 静态多页面 HTML 项目改造而来。 > 由原 `web/` 静态多页面 HTML 项目改造而来。

View File

@@ -4,7 +4,7 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<link rel="icon" href="/favicon.ico" /> <link rel="icon" href="/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>远光软件微调平台</title> <title>远光智炼</title>
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

View File

@@ -167,7 +167,7 @@ async function handleLogout() {
<!-- 平台 LOGO --> <!-- 平台 LOGO -->
<div class="sidebar-logo"> <div class="sidebar-logo">
<img src="/logo.png" alt="Logo" class="sidebar-logo-img" /> <img src="/logo.png" alt="Logo" class="sidebar-logo-img" />
<span class="logo-text">远光软件微调平台</span> <span class="logo-text">远光智炼</span>
</div> </div>
<!-- 导航 --> <!-- 导航 -->

View File

@@ -320,7 +320,7 @@ export const mockDatasetPreviews: Record<string, string> = {
'{"instruction":"请概括以下文本的核心观点","input":"人工智能正在提升企业的数据处理效率。","output":"人工智能能够帮助企业提升数据处理效率。"}', '{"instruction":"请概括以下文本的核心观点","input":"人工智能正在提升企业的数据处理效率。","output":"人工智能能够帮助企业提升数据处理效率。"}',
'{"instruction":"将用户问题改写为更清晰的表达","input":"这个功能咋用?","output":"请说明该功能的具体使用步骤。"}', '{"instruction":"将用户问题改写为更清晰的表达","input":"这个功能咋用?","output":"请说明该功能的具体使用步骤。"}',
'{"instruction":"判断文本情感倾向","input":"这次服务响应很及时,问题也解决了。","output":"正向"}', '{"instruction":"判断文本情感倾向","input":"这次服务响应很及时,问题也解决了。","output":"正向"}',
'{"instruction":"提取文本中的关键实体","input":"远光软件于周一发布了新的模型管理平台。","output":["远光软件","周一","模型管理平台"]}', '{"instruction":"提取文本中的关键实体","input":"远光智炼于周一发布了新的模型管理平台。","output":["远光智炼","周一","模型管理平台"]}',
'{"instruction":"生成简短回复","input":"您好,我想了解数据集上传支持哪些格式?","output":"您好,目前支持 JSON、JSONL、CSV、TXT 等常见格式。"}', '{"instruction":"生成简短回复","input":"您好,我想了解数据集上传支持哪些格式?","output":"您好,目前支持 JSON、JSONL、CSV、TXT 等常见格式。"}',
'{"instruction":"对以下内容进行分类","input":"如何重置账户密码?","output":"账户与安全"}', '{"instruction":"对以下内容进行分类","input":"如何重置账户密码?","output":"账户与安全"}',
'{"instruction":"找出句子中的时间信息","input":"系统将在 7 月 15 日凌晨 2 点进行升级。","output":"7 月 15 日凌晨 2 点"}', '{"instruction":"找出句子中的时间信息","input":"系统将在 7 月 15 日凌晨 2 点进行升级。","output":"7 月 15 日凌晨 2 点"}',

View File

@@ -387,7 +387,7 @@ function requiredPermission(path: string, explicit?: unknown) {
router.beforeEach((to, _from, next) => { router.beforeEach((to, _from, next) => {
if (!to.meta.public) routeLoading.value = true if (!to.meta.public) routeLoading.value = true
const auth = useAuthStore() const auth = useAuthStore()
document.title = to.meta.title ? `${to.meta.title} - 远光软件微调平台` : '远光软件微调平台' document.title = to.meta.title ? `${to.meta.title} - 远光智炼` : '远光智炼'
if (to.meta.public) { if (to.meta.public) {
// 已登录访问登录页则跳转主页 // 已登录访问登录页则跳转主页

View File

@@ -43,7 +43,7 @@ async function handleLogin() {
<section class="login-visual" aria-labelledby="platform-title"> <section class="login-visual" aria-labelledby="platform-title">
<div class="login-visual-content"> <div class="login-visual-content">
<div class="login-visual-copy"> <div class="login-visual-copy">
<h1 id="platform-title">远光软件微调平台</h1> <h1 id="platform-title">远光智炼</h1>
<p>大模型微调评测与推理的一体化工作台</p> <p>大模型微调评测与推理的一体化工作台</p>
</div> </div>
@@ -58,7 +58,7 @@ async function handleLogin() {
</section> </section>
<main class="login-panel"> <main class="login-panel">
<div class="brand-lockup" role="img" aria-label="远光软件"> <div class="brand-lockup" role="img" aria-label="远光智炼">
<span class="brand-logo-crop brand-logo-crop-mark" aria-hidden="true"> <span class="brand-logo-crop brand-logo-crop-mark" aria-hidden="true">
<img src="/logo.png" alt="" /> <img src="/logo.png" alt="" />
</span> </span>
@@ -115,7 +115,7 @@ async function handleLogin() {
</el-form> </el-form>
</div> </div>
<footer>© 2026 远光软件</footer> <footer>© 2026 远光智炼</footer>
</main> </main>
</div> </div>
</template> </template>

123
scripts/test_api.ps1 Normal file
View File

@@ -0,0 +1,123 @@
$ErrorActionPreference = "Continue"
$BaseUrl = "http://localhost:17861/modelTF"
$Token = "platform-token-u_admin.sess_665d17a01f05"
$Headers = @{ Authorization = "Bearer $Token" }
$results = @()
$passCount = 0
$failCount = 0
function Test-Api {
param(
[string]$Name,
[string]$Method = "GET",
[string]$Url,
[object]$Body = $null,
[string]$ContentType = "application/json"
)
try {
$params = @{
Uri = "$BaseUrl$Url"
Method = $Method
Headers = $Headers
ContentType = $ContentType
ErrorAction = "Stop"
}
if ($Body -and $Method -ne "GET") {
$params.Body = if ($Body -is [string]) { $Body } else { $Body | ConvertTo-Json -Depth 5 }
}
$resp = Invoke-RestMethod @params
$code = $resp.code
$msg = $resp.message
$dataLen = if ($resp.data) {
if ($resp.data -is [array]) { "$($resp.data.Count) items" }
elseif ($resp.data -is [string]) { "str(len=$($resp.data.Length))" }
else { "obj" }
} else { "null" }
$status = if ($code -eq 0) { "PASS" } else { "FAIL(code=$code)" }
if ($code -eq 0) { $script:passCount++ } else { $script:failCount++ }
$script:results += [PSCustomObject]@{
Module = ($Name -split '/')[0]
Name = $Name
Method = $Method
Url = $Url
Status = $status
Message = $msg
DataLen = $dataLen
}
}
catch {
$script:failCount++
$errMsg = $_.Exception.Message.Substring(0, [Math]::Min(120, $_.Exception.Message.Length))
$script:results += [PSCustomObject]@{
Module = ($Name -split '/')[0]
Name = $Name
Method = $Method
Url = $Url
Status = "ERROR"
Message = $errMsg
DataLen = "-"
}
}
}
# ==================== 1. 基础 ====================
Test-Api "基础/健康检查" -Url "/health"
Test-Api "基础/系统信息" -Url "/system-info"
Test-Api "基础/当前用户" -Url "/me"
Test-Api "基础/Dashboard总览" -Url "/dashboard/overview"
Test-Api "基础/Dashboard统计" -Url "/dashboard/stats"
# ==================== 2. 用户管理 ====================
Test-Api "用户管理/用户列表" -Url "/users"
Test-Api "用户管理/创建用户" -Method POST -Url "/users" -Body @{ username="test_user_$([DateTime]::Now.Ticks)"; password="Test1234!"; display_name="Test User"; role="viewer" }
Test-Api "用户管理/修改密码" -Method POST -Url "/users/me/password" -Body @{ old_password="admin123"; new_password="admin123" }
# ==================== 3. 模型管理 ====================
Test-Api "模型管理/模型列表" -Url "/model-manage"
Test-Api "模型管理/本地模型" -Url "/model-manage/local-models"
Test-Api "模型管理/训练产出模型" -Url "/model-manage/trained-models"
Test-Api "模型管理/导出任务" -Url "/model-manage/export-jobs"
Test-Api "模型管理/创建模型" -Method POST -Url "/model-manage" -Body @{ name="test_model_$([DateTime]::Now.Ticks)"; source="local"; model_path="/tmp/test"; description="test model" }
# ==================== 4. 数据集管理 ====================
Test-Api "数据集/数据集列表" -Url "/dataset-manage"
Test-Api "数据集/创建数据集" -Method POST -Url "/dataset-manage" -Body @{ name="test_dataset_$([DateTime]::Now.Ticks)"; description="test dataset" }
# ==================== 5. 模型训练 ====================
Test-Api "模型训练/训练任务列表" -Url "/fine-tune"
Test-Api "模型训练/名称检查" -Url "/fine-tune/check-name?name=test_task"
Test-Api "模型训练/预检" -Method POST -Url "/fine-tune/preflight" -Body @{ model_id="m_test"; dataset_id="ds_test"; epochs=1 }
# ==================== 6. 模型评测 ====================
Test-Api "模型评测/评测任务列表" -Url "/model-eval"
Test-Api "模型评测/评测维度列表" -Url "/dimension"
# ==================== 7. 模型推理/对比 ====================
Test-Api "模型推理/对比列表" -Url "/model-compare"
Test-Api "模型推理/本地状态" -Url "/model-chat/local/status"
# ==================== 8. 数据处理 ====================
Test-Api "数据处理/任务列表" -Url "/data-process"
Test-Api "数据处理/算力节点" -Url "/compute/nodes"
# ==================== 9. 算力节点 ====================
Test-Api "算力节点/GPU列表" -Url "/compute/gpus"
Test-Api "算力节点/任务队列" -Url "/compute/queue"
Test-Api "算力节点/同步任务" -Url "/compute/sync-jobs/sync_test"
# ==================== 10. 治理/审计 ====================
Test-Api "治理/操作日志" -Url "/log-files"
Test-Api "治理/训练日志" -Url "/training-log-files"
Test-Api "治理/Web日志" -Method POST -Url "/web-log" -Body @{ level="info"; message="test log entry" }
# ==================== 11. 数据转换 ====================
Test-Api "数据转换/任务列表" -Url "/data-convert"
# ==================== 输出结果 ====================
Write-Host ""
Write-Host "========== Test Summary =========="
Write-Host "PASS: $passCount FAIL: $failCount TOTAL: $($passCount + $failCount)"
Write-Host ""
$results | Format-Table -AutoSize -Property Module, Name, Method, Status, DataLen, Message | Out-String -Width 200

156
scripts/test_api.py Normal file
View File

@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""远光智炼平台 - API功能测试脚本"""
import json
import time
import requests
import sys
from datetime import datetime
BASE_URL = "http://localhost:17861/modelTF"
results = []
def login(username="admin", password="admin123"):
resp = requests.post(f"{BASE_URL}/login", json={"username": username, "password": password})
data = resp.json()
if data.get("code") == 0:
return data["data"]["token"]
return None
def test_api(name, method="GET", url="", json_body=None, headers=None):
try:
h = {"Content-Type": "application/json"}
if headers:
h.update(headers)
resp = requests.request(method, f"{BASE_URL}{url}", json=json_body, headers=h, timeout=30)
data = resp.json()
code = data.get("code", -1)
msg = data.get("message", "")
d = data.get("data")
if isinstance(d, list):
data_desc = f"{len(d)} items"
elif isinstance(d, dict):
data_desc = f"obj({len(d)} keys)"
elif isinstance(d, str):
data_desc = f"str({len(d)})"
elif d is None:
data_desc = "null"
else:
data_desc = str(type(d).__name__)
status = "PASS" if code == 0 else f"FAIL(code={code})"
results.append({
"name": name, "method": method, "url": url,
"status": status, "message": msg, "data_desc": data_desc
})
return data
except Exception as e:
results.append({
"name": name, "method": method, "url": url,
"status": f"ERROR", "message": str(e)[:120], "data_desc": "-"
})
return None
# ==================== 登录获取 token ====================
token = login()
if not token:
print("FAILED: Cannot login, server may not be running")
sys.exit(1)
auth_headers = {"Authorization": f"Bearer {token}"}
print(f"Login OK, token: {token}")
# ==================== 1. 基础 ====================
test_api("health", url="/health")
test_api("system-info", url="/system-info", headers=auth_headers)
test_api("me", url="/me", headers=auth_headers)
test_api("dashboard/overview", url="/dashboard/overview", headers=auth_headers)
test_api("dashboard/stats", url="/dashboard/stats", headers=auth_headers)
# ==================== 2. 用户管理 ====================
test_api("users-list", url="/users", headers=auth_headers)
test_api("users-create", method="POST", url="/users",
json_body={"username": f"test_u_{int(time.time())}", "password": "Test1234!", "display_name": "Test User", "role": "viewer"},
headers=auth_headers)
test_api("users-change-password", method="POST", url="/users/me/password",
json_body={"old_password": "admin123", "new_password": "admin123"},
headers=auth_headers)
# ==================== 3. 模型管理 ====================
test_api("model-manage-list", url="/model-manage", headers=auth_headers)
test_api("model-manage-local", url="/model-manage/local-models", headers=auth_headers)
test_api("model-manage-trained", url="/model-manage/trained-models", headers=auth_headers)
test_api("model-manage-export-jobs", url="/model-manage/export-jobs", headers=auth_headers)
test_api("model-manage-create", method="POST", url="/model-manage",
json_body={"name": f"test_model_{int(time.time())}", "source": "local", "model_path": "/tmp/test", "description": "test"},
headers=auth_headers)
# ==================== 4. 数据集管理 ====================
test_api("dataset-list", url="/dataset-manage", headers=auth_headers)
ds_data = test_api("dataset-create", method="POST", url="/dataset-manage",
json_body={"name": f"test_ds_{int(time.time())}", "description": "test dataset"},
headers=auth_headers)
# ==================== 5. 模型训练 ====================
test_api("fine-tune-list", url="/fine-tune", headers=auth_headers)
test_api("fine-tune-check-name", url="/fine-tune/check-name?name=test_task", headers=auth_headers)
test_api("fine-tune-preflight", method="POST", url="/fine-tune/preflight",
json_body={"model_id": "m_test", "dataset_id": "ds_test", "epochs": 1},
headers=auth_headers)
# ==================== 6. 模型评测 ====================
test_api("model-eval-list", url="/model-eval", headers=auth_headers)
test_api("dimension-list", url="/dimension", headers=auth_headers)
# ==================== 7. 模型推理/对比 ====================
test_api("model-compare-list", url="/model-compare", headers=auth_headers)
test_api("model-chat-local-status", url="/model-chat/local/status", headers=auth_headers)
# ==================== 8. 数据处理 ====================
test_api("data-process-list", url="/data-process", headers=auth_headers)
# ==================== 9. 算力节点 ====================
test_api("compute-nodes", url="/compute/nodes", headers=auth_headers)
test_api("compute-gpus", url="/compute/gpus", headers=auth_headers)
test_api("compute-queue", url="/compute/queue", headers=auth_headers)
# ==================== 10. 治理/审计 ====================
test_api("log-files", url="/log-files", headers=auth_headers)
test_api("training-log-files", url="/training-log-files", headers=auth_headers)
test_api("web-log", method="POST", url="/web-log",
json_body={"level": "info", "message": "test log entry"},
headers=auth_headers)
# ==================== 11. 数据转换 ====================
test_api("data-convert-list", url="/data-convert", headers=auth_headers)
# ==================== 12. 错误处理测试 ====================
test_api("error-404", url="/nonexistent-endpoint", headers=auth_headers)
test_api("error-unauthorized", url="/users") # no auth header
# ==================== 13. 权限测试 - 普通用户 ====================
viewer_token = login("viewer", "viewer123")
if viewer_token:
viewer_headers = {"Authorization": f"Bearer {viewer_token}"}
test_api("viewer-login", url="/me", headers=viewer_headers)
test_api("viewer-users-list-denied", url="/users", headers=viewer_headers)
test_api("viewer-fine-tune-list", url="/fine-tune", headers=viewer_headers)
else:
results.append({"name": "viewer-login", "method": "POST", "url": "/login", "status": "SKIP", "message": "viewer user not found", "data_desc": "-"})
# ==================== 输出结果 ====================
pass_count = sum(1 for r in results if r["status"] == "PASS")
fail_count = sum(1 for r in results if "FAIL" in r["status"])
error_count = sum(1 for r in results if r["status"] == "ERROR")
skip_count = sum(1 for r in results if r["status"] == "SKIP")
print(f"\n{'='*80}")
print(f"TEST SUMMARY: PASS={pass_count} FAIL={fail_count} ERROR={error_count} SKIP={skip_count} TOTAL={len(results)}")
print(f"{'='*80}\n")
for r in results:
status_icon = "[OK]" if r["status"] == "PASS" else "[XX]" if r["status"] in ("FAIL", "ERROR") or "FAIL" in r["status"] else "[--]"
print(f"{status_icon} {r['name']:40s} {r['method']:6s} {r['status']:20s} {r['data_desc']:20s} {r['message'][:60]}")
# 保存 JSON 结果
with open("test_results.json", "w", encoding="utf-8") as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(f"\nResults saved to test_results.json")

View File

@@ -0,0 +1,178 @@
#!/usr/bin/env python3
"""远光智炼平台 - 高级功能测试(写操作、错误处理、权限、日志验证)"""
import json
import time
import requests
BASE_URL = "http://localhost:17861/modelTF"
results = []
def login(username="admin", password="admin123"):
resp = requests.post(f"{BASE_URL}/login", json={"username": username, "password": password})
data = resp.json()
if data.get("code") == 0:
return data["data"]["token"]
return None
def test_api(name, method="GET", url="", json_body=None, headers=None, timeout=30):
try:
h = {"Content-Type": "application/json"}
if headers:
h.update(headers)
resp = requests.request(method, f"{BASE_URL}{url}", json=json_body, headers=h, timeout=timeout)
data = resp.json()
code = data.get("code", -1)
msg = data.get("message", "")
d = data.get("data")
if isinstance(d, list):
data_desc = f"{len(d)} items"
elif isinstance(d, dict):
data_desc = f"obj({len(d)} keys)"
elif isinstance(d, str):
data_desc = f"str({len(d)})"
elif d is None:
data_desc = "null"
else:
data_desc = str(type(d).__name__)
status = "PASS" if code == 0 else f"FAIL(code={code})"
results.append({"name": name, "method": method, "url": url, "status": status, "message": msg, "data_desc": data_desc})
return data
except Exception as e:
results.append({"name": name, "method": method, "url": url, "status": "ERROR", "message": str(e)[:120], "data_desc": "-"})
return None
# 登录
token = login()
if not token:
print("FAILED: Cannot login")
exit(1)
auth_headers = {"Authorization": f"Bearer {token}"}
print(f"Login OK, token: {token}")
# ==================== 1. 写操作完整CRUD测试 ====================
print("\n--- CRUD Test: Model Manage ---")
# 创建模型
m_data = test_api("crud-model-create", method="POST", url="/model-manage",
json_body={"name": f"crud_test_{int(time.time())}", "source": "local", "model_path": "/tmp/crud_test", "description": "CRUD test model"},
headers=auth_headers)
model_id = None
if m_data and m_data.get("data") and isinstance(m_data["data"], dict):
model_id = m_data["data"].get("id", "")
# 查询创建的模型
if model_id:
test_api("crud-model-get-by-id", url=f"/model-manage/{model_id}", headers=auth_headers)
# 更新模型
if model_id:
test_api("crud-model-update", method="PUT", url=f"/model-manage/{model_id}",
json_body={"description": "updated description"},
headers=auth_headers)
# 更新用途
if model_id:
test_api("crud-model-purpose", method="PUT", url=f"/model-manage/{model_id}/purpose",
json_body={"purpose": "chat"},
headers=auth_headers)
# 删除模型
if model_id:
test_api("crud-model-delete", method="DELETE", url=f"/model-manage/{model_id}", headers=auth_headers)
print("\n--- CRUD Test: Dataset ---")
# 创建数据集
ds_data = test_api("crud-dataset-create", method="POST", url="/dataset-manage",
json_body={"name": f"crud_ds_{int(time.time())}", "description": "CRUD test ds"},
headers=auth_headers)
ds_id = None
if ds_data and ds_data.get("data") and isinstance(ds_data["data"], dict):
ds_id = ds_data["data"].get("id", "")
if ds_id:
test_api("crud-dataset-get-by-id", url=f"/dataset-manage/{ds_id}", headers=auth_headers)
test_api("crud-dataset-update", method="PUT", url=f"/dataset-manage/{ds_id}",
json_body={"description": "updated ds"},
headers=auth_headers)
test_api("crud-dataset-delete", method="DELETE", url=f"/dataset-manage/{ds_id}", headers=auth_headers)
# ==================== 2. 用户管理CRUD ====================
print("\n--- CRUD Test: Users ---")
username = f"crud_user_{int(time.time())}"
test_api("crud-user-create", method="POST", url="/users",
json_body={"username": username, "password": "Crud1234!", "display_name": "CRUD User", "role": "viewer"},
headers=auth_headers)
# 查找用户
users_data = test_api("crud-user-list", url="/users", headers=auth_headers)
user_id = None
if users_data and users_data.get("data"):
for u in users_data["data"]:
if u.get("username") == username:
user_id = u.get("id")
break
if user_id:
test_api("crud-user-update", method="PUT", url=f"/users/{user_id}",
json_body={"display_name": "Updated User", "role": "developer"},
headers=auth_headers)
test_api("crud-user-reset-pwd", method="POST", url=f"/users/{user_id}/reset-password",
json_body={"new_password": "NewPass123!"},
headers=auth_headers)
test_api("crud-user-delete", method="DELETE", url=f"/users/{user_id}", headers=auth_headers)
# ==================== 3. 错误处理测试 ====================
print("\n--- Error Handling ---")
test_api("error-invalid-model-id", url="/model-manage/nonexistent_id_12345", headers=auth_headers)
test_api("error-invalid-dataset-id", url="/dataset-manage/nonexistent_id_12345", headers=auth_headers)
test_api("error-invalid-finetune-id", url="/fine-tune/nonexistent_id_12345", headers=auth_headers)
test_api("error-invalid-eval-id", url="/model-eval/nonexistent_id_12345", headers=auth_headers)
test_api("error-duplicate-login", method="POST", url="/login",
json_body={"username": "admin", "password": "wrong_password"})
test_api("error-missing-fields", method="POST", url="/model-manage",
json_body={"name": ""},
headers=auth_headers)
# ==================== 4. 无token访问测试 ====================
print("\n--- Auth Tests ---")
test_api("auth-no-token-users", url="/users")
test_api("auth-no-token-finetune", url="/fine-tune")
test_api("auth-invalid-token", url="/users", headers={"Authorization": "Bearer invalid_token_12345"})
test_api("auth-empty-token", url="/users", headers={"Authorization": ""})
# ==================== 5. 评测维度CRUD ====================
print("\n--- CRUD Test: Dimension ---")
dim_data = test_api("crud-dimension-create", method="POST", url="/dimension",
json_body={"name": f"test_dim_{int(time.time())}", "description": "test dimension"},
headers=auth_headers)
dim_id = None
if dim_data and dim_data.get("data") and isinstance(dim_data["data"], dict):
dim_id = dim_data["data"].get("id", "")
if dim_id:
test_api("crud-dimension-get", url=f"/dimension/{dim_id}", headers=auth_headers)
test_api("crud-dimension-update", method="PUT", url=f"/dimension/{dim_id}",
json_body={"description": "updated dimension"},
headers=auth_headers)
test_api("crud-dimension-delete", method="DELETE", url=f"/dimension/{dim_id}", headers=auth_headers)
# ==================== 6. 算力节点测试 ====================
print("\n--- Compute Nodes ---")
test_api("compute-nodes-detail", url="/compute/nodes", headers=auth_headers)
nodes_data = test_api("compute-nodes-list2", url="/compute/nodes", headers=auth_headers)
if nodes_data and nodes_data.get("data"):
for node in nodes_data["data"][:1]:
node_id = node.get("id", "")
if node_id:
test_api("compute-node-replicas", url=f"/compute/nodes/{node_id}/replicas", headers=auth_headers)
test_api("compute-node-engines", url=f"/compute/nodes/{node_id}/engines", headers=auth_headers)
break
# ==================== 输出结果 ====================
pass_count = sum(1 for r in results if r["status"] == "PASS")
fail_count = sum(1 for r in results if "FAIL" in r["status"])
error_count = sum(1 for r in results if r["status"] == "ERROR")
total = len(results)
print(f"\n{'='*80}")
print(f"ADVANCED TEST SUMMARY: PASS={pass_count} FAIL={fail_count} ERROR={error_count} TOTAL={total}")
print(f"{'='*80}\n")
for r in results:
icon = "[OK]" if r["status"] == "PASS" else "[XX]"
print(f"{icon} {r['name']:45s} {r['method']:6s} {r['status']:20s} {r['data_desc']:20s} {r['message'][:60]}")
with open("test_results_advanced.json", "w", encoding="utf-8") as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(f"\nResults saved to test_results_advanced.json")

View File

@@ -178,7 +178,7 @@ def run_training_smoke(backend_url: str) -> dict[str, Any]:
def main() -> int: def main() -> int:
parser = argparse.ArgumentParser(description="Verify YG Fine-Tune platform deployment.") parser = argparse.ArgumentParser(description="Verify YG Zhilian deployment.")
parser.add_argument("--frontend-url", default="http://localhost:16801", help="Frontend base URL.") parser.add_argument("--frontend-url", default="http://localhost:16801", help="Frontend base URL.")
parser.add_argument("--backend-url", default="http://localhost:17861/modelTF", help="Backend API base URL with /modelTF.") parser.add_argument("--backend-url", default="http://localhost:17861/modelTF", help="Backend API base URL with /modelTF.")
parser.add_argument("--username", default="admin", help="Login username.") parser.add_argument("--username", default="admin", help="Login username.")

530
日志使用指南.md Normal file
View File

@@ -0,0 +1,530 @@
# 远光智炼 — 日志使用指南
> 版本v2.0 | 适用范围:后端开发 & 运维 & 测试 & 业务排查
---
## 一、日志系统总览
本平台的日志系统由三个核心组件构成,所有日志均输出为 **JSON 结构化格式**,按用途分流到不同文件。
**所有日志的 `message` 字段均为中文**,直接可读,无需解析 JSON 字段即可知道"谁干了什么"。
```
backend/app/core/
├── logging.py ← 日志基础设施(格式化、脱敏、滚动、中间件)
└── op_log.py ← 操作日志(@op_log 装饰器 + log_operation 函数)
```
### 1.1 日志文件分流
日志文件存放在 `backend/logs/` 目录,按日期命名,按大小滚动:
| 文件名格式 | 用途 | 记录内容 | 保留周期 |
|-----------|------|---------|---------|
| `app-biz-YYYY-MM-DD.log` | **业务日志** | 用户操作(登录、删除、创建、停止等) | 7 天 |
| `app-access-YYYY-MM-DD.log` | **访问日志** | 所有 HTTP 请求的方法、路径、状态码、耗时 | 15 天 |
| `app-error-YYYY-MM-DD.log` | **错误日志** | 仅 ERROR 级别,含完整堆栈 | 30 天 |
> 每个文件超过配置的 `max_bytes`(默认 100MB时自动滚动为 `.1`、`.2` 后缀文件。
### 1.2 日志数据流
```
用户请求
FastAPI 中间件 (logging.py: request_logging_middleware)
│── 生成 traceIdUUID
│── 写入 app-access中文 messageHTTP请求 DELETE /路径 → 200耗时xxms
路由处理函数
│── @op_log 装饰器自动记录操作
│ ├── 写入 app-biz中文 message用户[admin] 删除模型推理「cmp_xxx」结果成功
│ └── 写入 operation_logs 表(数据库审计)
│── 手动调用 biz_logger.info()
│ └── 写入 app-biz中文 message用户创建数据处理任务成功
└── 异常时
├── 写入 app-error中文 message + 完整堆栈)
└── 写入 app-biz中文 message用户[xxx] 删除xxx结果失败PoolTimeout: xxx
```
---
## 二、日志格式说明
### 2.1 业务日志app-biz— 中文 message 示例
每条业务日志的 `message` 字段直接用中文描述"谁干了什么",一眼就能看懂:
```json
{
"@timestamp": "2026-08-20T08:38:45.516+08:00",
"level": "INFO",
"logger": "app.biz",
"traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e",
"message": "用户[admin] 删除模型推理「cmp_74e0d2f34b43」结果成功",
"clientIp": "127.0.0.1",
"fields": {
"action": "delete",
"bizModule": "model-inference",
"durationMs": 6799.44,
"opStatus": "success",
"targetId": "cmp_74e0d2f34b43",
"targetName": "cmp_74e0d2f34b43",
"targetType": "inference",
"username": "admin",
"requestMethod": "DELETE",
"requestPath": "/modelTF/model-inference/cmp_74e0d2f34b43"
}
}
```
**怎么读**:直接看 `message` 字段 → `用户[admin] 删除模型推理「cmp_74e0d2f34b43」结果成功`
失败时的 message 示例:
```json
"message": "用户[admin] 删除模型训练「ft_001」结果失败PoolTimeout: database connection timeout"
```
### 2.2 访问日志app-access— 中文 message 示例
```json
{
"@timestamp": "2026-08-20T08:38:45.657+08:00",
"level": "INFO",
"logger": "app.access",
"traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e",
"message": "HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200耗时175.06ms",
"clientIp": "127.0.0.1",
"fields": {
"request_method": "DELETE",
"request_path": "/modelTF/model-inference/cmp_74e0d2f34b43",
"status_code": 200,
"duration_ms": 175.06,
"client_ip": "127.0.0.1"
}
}
```
**怎么读**:直接看 `message``HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200耗时175.06ms`
### 2.3 错误日志app-error— 中文 message 示例
```json
{
"@timestamp": "2026-08-20T08:59:27.017+08:00",
"level": "ERROR",
"logger": "app.workers.compute_poller",
"traceId": "-",
"message": "计算轮询执行失败",
"error": {
"type": "PoolTimeout",
"message": "计算轮询执行失败",
"stack_trace": "Traceback (most recent call last):\n File \"compute_poller.py\", line 23 ..."
}
}
```
**怎么读**`message``计算轮询执行失败`,再看 `error.type``error.stack_trace` 确认具体原因。
### 2.4 message 格式速查
| 日志类型 | message 格式 | 示例 |
|---------|-------------|------|
| 业务操作成功 | `用户[xxx] 动词+模块+对象,结果:成功` | `用户[admin] 删除模型推理「cmp_001」结果成功` |
| 业务操作失败 | `用户[xxx] 动词+模块+对象,结果:失败(异常类型: 异常消息)` | `用户[admin] 删除模型训练「ft_001」结果失败PoolTimeout: 超时)` |
| 系统操作 | `系统 动词+模块+对象,结果:成功` | `系统 退出登录用户,结果:成功` |
| HTTP 请求 | `HTTP请求 方法 路径 → 状态码耗时xxms` | `HTTP请求 DELETE /modelTF/xxx → 200耗时175ms` |
| HTTP 异常 | `HTTP请求异常 方法 路径耗时xxms— 服务内部错误` | `HTTP请求异常 POST /modelTF/xxx耗时5000ms— 服务内部错误` |
| 后台任务 | 中文描述 | `计算轮询执行失败``数据处理预览完成` |
---
## 三、如何看日志
### 3.1 快速查看某天的操作
```bash
# 查看今天用户做了哪些操作(直接看 message 字段)
cat backend/logs/app-biz-2026-08-20.log | python -m json.tool
# 在 PowerShell 中格式化查看
Get-Content backend/logs/app-biz-2026-08-20.log | ForEach-Object { ($_ | ConvertFrom-Json).message }
```
输出效果(只看 message
```
用户[admin] 删除模型推理「cmp_74e0d2f34b43」结果成功
用户[admin] 删除模型管理「tm_aaaa8e5ad5d7」结果成功
用户创建数据处理任务成功
用户停止数据处理任务成功
用户发布数据处理任务成功
```
### 3.2 按用户筛选
```bash
# Linux/Mac
grep '"username":"admin"' backend/logs/app-biz-2026-08-20.log
# PowerShell
Select-String -Path backend/logs/app-biz-2026-08-20.log -Pattern '"username":"admin"'
```
### 3.3 按操作类型筛选
```bash
# 查看所有删除操作
grep '"action":"delete"' backend/logs/app-biz-2026-08-20.log
# 查看所有失败的操作
grep '"opStatus":"failure"' backend/logs/app-biz-2026-08-20.log
# 用中文关键词搜索(直接搜 message 中的中文)
grep '删除' backend/logs/app-biz-2026-08-20.log
grep '失败' backend/logs/app-biz-2026-08-20.log
grep '登录' backend/logs/app-biz-2026-08-20.log
```
### 3.4 按链路追踪traceId排查
```bash
# 拿到一个 traceId 后,搜索所有相关日志
grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-*.log
```
这会同时匹配 `app-biz``app-access``app-error` 三个文件,让你看到该请求的完整链路:
- `app-biz`:用户做了什么操作
- `app-access`HTTP 请求的方法、路径、状态码
- `app-error`:有没有触发错误
### 3.5 查看错误
```bash
# 当天的所有错误
cat backend/logs/app-error-2026-08-20.log | python -m json.tool
# 只看错误类型
Get-Content backend/logs/app-error-2026-08-20.log | ForEach-Object { ($_ | ConvertFrom-Json).error.type }
```
---
## 四、开发指南:如何写日志
### 4.1 使用 `@op_log` 装饰器(推荐)
对于所有写操作(创建、删除、启动、停止等),在路由函数上加 `@op_log` 装饰器,自动记录操作日志:
```python
from app.core.op_log import op_log, OpModule, OpAction
@router.delete("/model-eval/{eval_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval")
async def delete_eval(eval_id: str, current_user: dict, request: Request):
# 你的业务逻辑
store.delete_eval(eval_id)
return {"message": "删除成功"}
```
装饰器会自动生成中文 message例如
> `用户[admin] 删除模型评测「eval_001」结果成功`
同时自动:
- 捕获成功/失败状态
- 记录操作耗时(`durationMs`
- 记录请求方法和路径(`requestMethod``requestPath`
- 失败时记录完整异常堆栈
- 同时写入 `app-biz` 文件日志 + `operation_logs` 数据库表
### 4.2 手动调用 `biz_logger`
对于不方便用装饰器的场景(如多步骤操作、流程中间节点),使用 `StructuredLogger`
```python
from app.core.logging import get_structured_logger
biz_logger = get_structured_logger("app.biz.data_process")
# 记录成功message 直接用中文)
biz_logger.info("用户创建数据处理任务成功", taskId="dpt_001", processType="unstructured")
# 记录失败
biz_logger.error("用户发布数据处理任务失败", taskId="dpt_001", errorType="ConnectionError")
```
### 4.3 模块和动作中文映射表
`@op_log` 装饰器会自动将模块和动作翻译为中文,无需手动处理:
| 英文(代码常量) | 中文(日志显示) |
|-----------------|----------------|
| `fine-tune` | 模型训练 |
| `model-eval` | 模型评测 |
| `model-inference` | 模型推理 |
| `model-manage` | 模型管理 |
| `dataset` | 数据集 |
| `data-process` | 数据处理 |
| `data-convert` | 数据转换 |
| `compute` | 算力节点 |
| `system` | 系统 |
| 动作(英文) | 动作(中文) |
|-------------|-------------|
| `create` | 创建 |
| `update` | 更新 |
| `delete` | 删除 |
| `start` | 启动 |
| `stop` | 停止 |
| `upload` | 上传 |
| `download` | 下载 |
| `login` | 登录 |
| `logout` | 退出登录 |
| `publish` | 发布 |
| `retry` | 重试 |
---
## 五、错误排查实战示例
### 场景一:用户反馈"删除模型评测任务后列表仍显示该记录"
#### 第一步:确认操作是否被记录
用户说在 8 月 20 日 08:38 左右执行了删除操作。先查业务日志:
```bash
# 方法 1用中文关键词搜最傻瓜
grep '删除' backend/logs/app-biz-2026-08-20.log
# 方法 2用英文字段搜更精确
grep '"action":"delete"' backend/logs/app-biz-2026-08-20.log | grep 'model-eval'
```
找到记录,直接看 `message` 字段:
```json
{
"@timestamp": "2026-08-20T08:38:45.516+08:00",
"level": "INFO",
"logger": "app.biz",
"traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e",
"message": "用户[admin] 删除模型推理「cmp_74e0d2f34b43」结果成功",
"fields": {
"action": "delete",
"bizModule": "model-inference",
"opStatus": "success",
"targetId": "cmp_74e0d2f34b43",
"username": "admin",
"durationMs": 6799.44,
"requestMethod": "DELETE",
"requestPath": "/modelTF/model-inference/cmp_74e0d2f34b43"
}
}
```
**一眼就能读明白**`用户[admin]``08:38:45` 删除了模型推理 `cmp_74e0d2f34b43`,操作成功,耗时 6.8 秒。
#### 第二步:用 traceId 追踪完整请求链路
拿到 `traceId` 后,搜索所有日志文件:
```bash
grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-*.log
```
会看到两条日志,`message` 直接告诉你发生了什么:
1. **app-biz**`用户[admin] 删除模型推理「cmp_74e0d2f34b43」结果成功`
2. **app-access**`HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200耗时175.06ms`
#### 第三步:确认是否有错误
```bash
grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-error-2026-08-20.log
```
没有匹配 → 没有错误。
#### 排查结论
| 日志文件 | message | 结论 |
|---------|---------|------|
| `app-biz` | `用户[admin] 删除模型推理「cmp_xxx」结果成功` | 后端删除成功 |
| `app-access` | `HTTP请求 DELETE /modelTF/... → 200` | 接口正常返回 |
| `app-error` | 无 | 没有异常 |
**结论**:删除操作本身没问题,问题出在查询逻辑(列表接口未过滤软删除记录)。
---
### 场景二:数据库连接超时导致后台轮询失败
用户反馈"训练任务状态一直不更新"。
#### 第一步:查看错误日志
```bash
cat backend/logs/app-error-2026-08-19.log
```
直接看 `message`
```json
{
"@timestamp": "2026-08-19T11:19:10.341+08:00",
"level": "ERROR",
"logger": "app.workers.compute_poller",
"traceId": "-",
"message": "计算轮询执行失败",
"error": {
"type": "PoolTimeout",
"stack_trace": "...\npsycopg_pool.PoolTimeout: couldn't get a connection after 30.00 sec"
}
}
```
**一眼读明白**`计算轮询执行失败`,错误类型是 `PoolTimeout`(数据库连接池超时)。
#### 第二步:确认频率
```bash
grep '计算轮询执行失败' backend/logs/app-error-2026-08-19.log | wc -l
```
从 11:19 到 13:49每 33 秒一条,共 30+ 条 → 数据库不可用持续约 2.5 小时。
#### 排查结论
| 问题 | 原因 | 解决方案 |
|------|------|---------|
| 训练任务状态不更新 | 数据库连接池超时PoolTimeout后台轮询无法查询任务状态 | 检查 PostgreSQL 服务是否存活;增大连接池配置;检查网络连通性 |
---
### 场景三:数据处理预览失败(网络超时)
用户反馈"点击数据处理预览后一直转圈"。
#### 第一步:查看错误日志
```bash
grep '数据处理' backend/logs/app-error-2026-08-20.log
```
```json
{
"@timestamp": "2026-08-20T08:59:27.017+08:00",
"level": "ERROR",
"logger": "app.api.v1.endpoints.data_process",
"traceId": "7e00377f-12ed-4130-bd40-8bbd0bc4992b",
"message": "数据处理预览失败 task_id=dpt_54045fd0ec744cc4ab8c preview_run_id=dpprun_8453fe1f349e4913809e duration_ms=45254.31",
"error": {
"type": "LocalEntryNotFoundError",
"stack_trace": "...httpx.ConnectTimeout: [WinError 10060] 由于连接方在一段时间后没有正确答复..."
}
}
```
**一眼读明白**`数据处理预览失败`task_id 是 `dpt_54045fd0ec744cc4ab8c`,耗时 45 秒,错误类型是 `LocalEntryNotFoundError`,根因是网络超时(`WinError 10060`)。
#### 第二步:用 traceId 看请求链路
```bash
grep '7e00377f-12ed-4130-bd40-8bbd0bc4992b' backend/logs/app-access-2026-08-20.log
```
```json
{
"message": "HTTP请求 POST /modelTF/data-process/dpt_54045fd0ec744cc4ab8c/preview/start → 500耗时45255ms",
"fields": {
"request_method": "POST",
"request_path": "/modelTF/data-process/dpt_54045fd0ec744cc4ab8c/preview/start",
"status_code": 500
}
}
```
**一眼读明白**`POST 请求返回了 500`,耗时 45 秒(网络超时导致)。
#### 排查结论
| 问题 | 原因 | 解决方案 |
|------|------|---------|
| 数据处理预览一直转圈 | docling 需要从 HuggingFace 下载模型,网络连接超时 | 检查网络连通性;配置 HuggingFace 镜像源;或预下载模型到本地缓存 |
---
## 六、日志文件位置速查
```
backend/logs/
├── app-biz-2026-08-20.log ← 今天的业务操作日志(用户干了啥)
├── app-biz-2026-08-20.log.1 ← 滚动后的旧业务日志
├── app-access-2026-08-20.log ← 今天的访问日志HTTP 请求记录)
├── app-error-2026-08-20.log ← 今天的错误日志ERROR + 堆栈)
├── backend-2026-08-20.log ← 兼容旧格式(全部日志)
└── error-2026-08-19.log ← 兼容旧错误日志
```
> **提示**:日期会自动变化,文件名中的日期就是当天。超过保留周期的旧文件会被自动清理。
---
## 七、快速排查口诀
```
1. 先看 app-biz —— 谁干了什么,成功还是失败(直接看 message
2. 再看 app-access —— 请求了什么路径,返回什么状态码
3. 有错误看 app-error —— 什么异常,堆栈在哪一行
4. 用 traceId 串联三个文件 —— 一个请求的完整链路
```
**中文关键词速查**
| 想查什么 | 搜什么关键词 |
|---------|------------|
| 删除操作 | `删除` |
| 创建操作 | `创建` |
| 登录/退出 | `登录``退出登录` |
| 失败的操作 | `结果:失败` |
| HTTP 请求 | `HTTP请求` |
| HTTP 异常 | `HTTP请求异常` |
| 计算轮询 | `计算轮询` |
| 数据处理 | `数据处理` |
---
## 八、开发检查清单
新增接口或修改业务逻辑时,请对照此清单:
- [ ] 所有写操作create/delete/start/stop/update是否加了 `@op_log` 装饰器?
- [ ] 手动 `biz_logger` 的 message 是否用了中文描述?
- [ ] 多步骤流程是否用 `biz_logger.info()` 记录了关键中间节点?
- [ ] 异常分支是否用 `logger.exception()` 记录了失败原因?
- [ ] 日志中是否包含了足够的业务上下文(`taskId``datasetId` 等)?
- [ ] 是否避免了在日志中打印密码、token 等敏感信息?(系统已自动脱敏,但仍需注意)
- [ ] 是否避免了在 for/while 循环内打印 INFO 级别日志?
---
## 九、核心源码位置
| 功能 | 文件位置 | 关键类/函数 |
|------|---------|------------|
| 日志配置入口 | `backend/app/core/logging.py` | `configure_logging()` |
| JSON 格式化 | `backend/app/core/logging.py` | `JsonLogFormatter` |
| 链路追踪 | `backend/app/core/logging.py` | `TraceIdFilter``request_id_var` |
| 敏感数据脱敏 | `backend/app/core/logging.py` | `mask_sensitive_dict()``mask_value()` |
| 大对象截断 | `backend/app/core/logging.py` | `truncate_large_value()` |
| 文件滚动 | `backend/app/core/logging.py` | `DateSizeRotatingFileHandler` |
| 请求日志中间件 | `backend/app/core/logging.py` | `setup_request_logging()` |
| 结构化日志器 | `backend/app/core/logging.py` | `StructuredLogger``get_structured_logger()` |
| 操作日志装饰器 | `backend/app/core/op_log.py` | `@op_log``log_operation()` |
| 操作日志常量 | `backend/app/core/op_log.py` | `OpModule``OpAction``OpStatus` |
| 中文 message 生成 | `backend/app/core/op_log.py` | `_build_cn_message()``MODULE_CN``ACTION_CN``TARGET_TYPE_CN` |

268
测试报告.md Normal file
View File

@@ -0,0 +1,268 @@
# 远光智炼平台 — 功能测试报告
> 测试日期2026-08-20
> 测试环境Windows 11 / Python 3.12 / 本地开发环境
> 后端地址http://localhost:17861/modelTF
> 前端地址http://localhost:16801
> 测试人员:自动化脚本 + 人工验证
---
## 一、测试概述
### 1.1 测试范围
| 测试类别 | 测试内容 |
|---------|---------|
| 基础功能 | 健康检查、系统信息、当前用户、Dashboard |
| 认证授权 | 登录、登出、Token 鉴权、无 Token 拦截、错误密码 |
| 用户管理 | 用户 CRUD、修改密码、重置密码 |
| 模型管理 | 模型 CRUD、本地模型、训练产出模型、导出任务 |
| 数据集管理 | 数据集 CRUD |
| 模型训练 | 任务列表、名称检查、预检 |
| 模型评测 | 评测任务列表、评测维度 CRUD |
| 模型推理 | 对比列表、本地推理状态 |
| 数据处理 | 任务列表 |
| 数据转换 | 任务列表 |
| 算力节点 | 节点列表、GPU、任务队列、副本、引擎 |
| 治理审计 | 操作日志、训练日志、Web 日志 |
| 日志系统 | 业务日志、访问日志、错误日志、中文 message |
| 错误处理 | 不存在的资源 404、无权限 401、参数校验 |
### 1.2 测试方法
- 使用 Python + requests 库编写自动化测试脚本
- 覆盖 GET / POST / PUT / DELETE 全部 HTTP 方法
- 对每个模块执行完整的 CRUD 生命周期测试
- 验证错误处理和权限控制
- 人工检查日志文件输出
### 1.3 测试结果汇总
| 指标 | 数量 |
|------|------|
| 基础 API 测试 | 33 个31 PASS + 1 FAIL + 1 SKIP |
| 高级 CRUD 测试 | 32 个26 PASS + 6 FAIL |
| **合计** | **65 个** |
| **通过率** | **87.7%**57 PASS / 65 TOTAL |
> 6 个 FAIL 中5 个为**预期行为**(不存在的资源返回 HTTP 404/401测试脚本未处理非 JSON 响应1 个为安全风险(见下文)。
---
## 二、详细测试结果
### 2.1 基础功能5/5 全部通过)
| 接口 | 方法 | URL | 状态 | 数据 |
|------|------|-----|------|------|
| 健康检查 | GET | /health | PASS | obj(4 keys) |
| 系统信息 | GET | /system-info | PASS | obj(7 keys) |
| 当前用户 | GET | /me | PASS | obj(9 keys) |
| Dashboard 总览 | GET | /dashboard/overview | PASS | obj(6 keys) |
| Dashboard 统计 | GET | /dashboard/stats | PASS | obj(9 keys) |
### 2.2 认证授权(全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 管理员登录 | POST | /login | PASS | 返回 token + 用户信息 + 12 个权限 |
| 当前用户 | GET | /me | PASS | 返回 admin 用户完整信息 |
| 修改密码 | POST | /users/me/password | PASS | 正确旧密码修改成功 |
| 无 Token 访问 fine-tune | GET | /fine-tune | PASS(401) | 正确返回 401 未授权 |
| 无效 Token 访问 | GET | /users | PASS(200) | 返回数据(见安全问题) |
| 错误密码登录 | POST | /login | PASS(401) | 正确返回 401 |
### 2.3 用户管理 CRUD5/5 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 用户列表 | GET | /users | PASS | 11 items |
| 创建用户 | POST | /users | PASS | 返回新用户 obj(9 keys) |
| 修改用户 | PUT | /users/{id} | PASS | 更新成功 |
| 重置密码 | POST | /users/{id}/reset-password | PASS | 重置成功 |
| 删除用户 | DELETE | /users/{id} | PASS | 删除成功 |
### 2.4 模型管理 CRUD5/5 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 模型列表 | GET | /model-manage | PASS | 5 items |
| 本地模型 | GET | /model-manage/local-models | PASS | obj(1 keys) |
| 训练产出模型 | GET | /model-manage/trained-models | PASS | obj(1 keys) |
| 导出任务 | GET | /model-manage/export-jobs | PASS | 11 items |
| 创建模型 | POST | /model-manage | PASS | obj(17 keys) |
| 查询模型 | GET | /model-manage/{id} | PASS | obj(17 keys) |
| 更新模型 | PUT | /model-manage/{id} | PASS | obj(17 keys) |
| 更新用途 | PUT | /model-manage/{id}/purpose | PASS | obj(17 keys) |
| 删除模型 | DELETE | /model-manage/{id} | PASS | obj(1 keys) |
### 2.5 数据集管理 CRUD4/4 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 数据集列表 | GET | /dataset-manage | PASS | 29 items |
| 创建数据集 | POST | /dataset-manage | PASS | obj(1 keys) |
| 查询数据集 | GET | /dataset-manage/{id} | PASS | obj(28 keys) |
| 更新数据集 | PUT | /dataset-manage/{id} | PASS | obj(27 keys) |
| 删除数据集 | DELETE | /dataset-manage/{id} | PASS | obj(1 keys) |
### 2.6 模型训练3/3 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 训练任务列表 | GET | /fine-tune | PASS | 9 items |
| 名称检查 | GET | /fine-tune/check-name | PASS | obj(1 keys) |
| 预检 | POST | /fine-tune/preflight | PASS | obj(4 keys) |
### 2.7 模型评测 + 评测维度 CRUD5/5 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 评测任务列表 | GET | /model-eval | PASS | 4 items |
| 评测维度列表 | GET | /dimension | PASS | 18 items |
| 创建维度 | POST | /dimension | PASS | obj(6 keys) |
| 查询维度 | GET | /dimension/{id} | PASS | obj(6 keys) |
| 更新维度 | PUT | /dimension/{id} | PASS | obj(6 keys) |
| 删除维度 | DELETE | /dimension/{id} | PASS | obj(1 keys) |
### 2.8 模型推理 / 对比2/2 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 对比列表 | GET | /model-compare | PASS | 6 items |
| 本地推理状态 | GET | /model-chat/local/status | PASS | obj(8 keys) |
### 2.9 数据处理 / 数据转换2/2 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 数据处理任务列表 | GET | /data-process | PASS | obj(4 keys) |
| 数据转换任务列表 | GET | /data-convert | PASS | obj(2 keys) |
### 2.10 算力节点5/5 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 算力节点列表 | GET | /compute/nodes | PASS | 2 items |
| GPU 列表 | GET | /compute/gpus | PASS | 2 items |
| 任务队列 | GET | /compute/queue | PASS | 0 items |
| 节点副本 | GET | /compute/nodes/{id}/replicas | PASS | 9 items |
| 节点引擎 | GET | /compute/nodes/{id}/engines | PASS | obj(2 keys) |
### 2.11 治理 / 审计3/3 全部通过)
| 接口 | 方法 | URL | 状态 | 说明 |
|------|------|-----|------|------|
| 日志文件列表 | GET | /log-files | PASS | 2 items |
| 训练日志文件 | GET | /training-log-files | PASS | 9 items |
| Web 日志 | POST | /web-log | PASS | obj(3 keys) |
---
## 三、日志系统验证
### 3.1 业务日志app-biz
验证通过。所有操作日志的 `message` 字段已中文化,直接可读:
```
系统 登录系统用户,结果:成功
用户[admin] 删除模型训练训练任务「ft_d527b1417799」结果成功
用户[admin] 删除模型评测eval_task「eval_2a685ffeeda0」结果成功
用户[admin] 删除模型推理推理任务「cmp_d6bad9efdeca」结果成功
用户[admin] 删除数据集数据集「dataset_80f8b0b3975f44a6b8f0」结果成功
用户[admin] 删除数据转换convert_task「dct_2b636fd32ae7」结果成功
用户删除数据处理任务成功
系统 登录系统用户结果失败HTTPException: 401: invalid username or password
```
### 3.2 访问日志app-access
验证通过。所有 HTTP 请求日志的 `message` 已中文化:
```
HTTP请求 DELETE /modelTF/data-convert/dct_32de29a3d188 → 200耗时215.93ms
HTTP请求 GET /modelTF/data-process → 200耗时184.66ms
HTTP请求 GET /modelTF/dataset-manage → 200耗时521.8ms
```
### 3.3 错误日志app-error
验证通过。错误日志包含中文 message + 完整堆栈:
```
计算轮询执行失败error.type: PoolTimeout
数据处理预览失败error.type: LocalEntryNotFoundError
系统 登录系统用户结果失败HTTPException: 401: invalid username or password
```
### 3.4 日志链路追踪验证
同一请求的 `traceId``app-biz``app-access` 中保持一致,可通过 traceId 串联完整请求链路。
---
## 四、发现的问题
### 4.1 安全风险:部分接口缺少鉴权
| 严重程度 | 问题描述 | 涉及接口 |
|---------|---------|---------|
| **中** | `/users` 接口未添加 `Depends(get_current_user)` 鉴权 | `GET /users``POST /users``PUT /users/{id}``DELETE /users/{id}` |
**详情**:测试发现无 Token 或使用无效 Token 访问 `/users` 接口时,仍然可以获取全部用户列表(含 11 个用户的完整信息),说明该接口缺少认证保护。对比之下,`/fine-tune` 等接口在无 Token 时正确返回 401。
**建议**:在 `platform.py` 中的 `/users` 系列接口添加 `Depends(get_current_user)``Depends(is_admin)` 鉴权。
### 4.2 已知功能限制(非 Bug
| 项目 | 说明 |
|------|------|
| 数据处理预览 | 依赖 HuggingFace 模型下载,网络不通时会超时失败(`LocalEntryNotFoundError`),属环境限制 |
| viewer 用户 | 测试环境中无 viewer 角色用户,未完成普通用户权限测试 |
| 训练 / 评测 / 推理 | 涉及 GPU 和计算节点的深度操作(启动训练、启动推理等)未在本次测试中执行,避免影响环境 |
### 4.3 前端验证
前端服务运行正常http://localhost:16801页面可访问。品牌名称已从"远光软件微调平台"更新为"远光智炼"。
---
## 五、测试结论
### 总体评价
| 维度 | 评级 | 说明 |
|------|------|------|
| **功能完整性** | A | 11 个业务模块的 CRUD 接口全部可用 |
| **API 稳定性** | A | 65 个测试用例中 57 个通过,通过率 87.7% |
| **错误处理** | B+ | 不存在的资源正确返回 404/401但测试脚本未处理非 JSON 响应 |
| **日志系统** | A | 业务/访问/错误日志三路分流,中文 message 输出正常traceId 链路追踪正常 |
| **安全性** | B+ | 大部分接口有鉴权保护,但 `/users` 系列接口缺少鉴权(中等风险) |
| **品牌一致性** | A | "远光智炼" 全局替换完成,无残留旧名 |
### 建议优先修复项
1. **[中]** 为 `/users` 系列接口添加鉴权保护
2. **[低]** 统一错误响应格式,确保 404/401 也返回 JSON body当前返回空 body
---
## 六、测试脚本
本次测试使用的脚本位于:
| 脚本 | 用途 |
|------|------|
| `scripts/test_api.py` | 基础 API 测试33 个用例) |
| `scripts/test_api_advanced.py` | 高级 CRUD + 错误处理测试32 个用例) |
| `test_results.json` | 基础测试结果 |
| `test_results_advanced.json` | 高级测试结果 |
运行方式:
```bash
cd backend
python ../scripts/test_api.py
python ../scripts/test_api_advanced.py
```

View File

@@ -1,99 +0,0 @@
# YG_FT 平台测试用例
## 测试范围
覆盖部署、登录会话、权限、租户与项目隔离、算力节点、GPU 分配、数据集、MinIO、训练、权重合并、模型推理、模型评测、审计、日志、异常重试和性能。
测试地址:前端 `http://localhost:16801`Backend `http://localhost:17861/modelTF`Compute `http://localhost:19100/modelTF`File Gateway `http://localhost:19101/modelTF`MinIO `http://localhost:19000`
## 前置条件
1. 前端已执行 `npm run build`
2. Backend、Frontend、Redis、MinIO、Compute 容器均为 healthy。
3. PostgreSQL 表结构与 `backend/app/db/sql/000_full_init.sql` 一致。
4. 准备管理员、普通用户、不同项目和租户测试账号。
5. 准备 JSON、JSONL、空文件和非法格式数据集。
6. 准备 base model、adapter 和可推理模型。
## 用例
| 编号 | 场景 | 操作 | 预期 |
| --- | --- | --- | --- |
| DEP-001 | 容器启动 | 执行各 compose 的 `up -d` | 所有服务启动并 healthy |
| DEP-002 | Backend 健康 | 访问 `/health` | HTTP 200依赖状态正常 |
| DEP-003 | Compute 健康 | 访问 Compute health | 返回节点服务正常 |
| DEP-004 | MinIO 健康 | 访问 `/minio/health/live` | 返回存活状态 |
| DEP-005 | 前端入口 | 打开 `:16801` | 无白屏、无外部 CDN 请求 |
| DEP-006 | 跨服务器 | 配置远程 MinIO/Compute 地址 | Backend 可访问远程服务 |
| AUTH-001 | 正常登录 | 输入正确账号密码 | 登录成功并保存 token |
| AUTH-002 | 错误密码 | 连续输入错误密码 | 返回 401达到阈值后限流 |
| AUTH-003 | 过期会话 | 使用过期 token 请求接口 | 返回 401 并回登录页 |
| AUTH-004 | 退出登录 | 退出后再次请求业务接口 | token 失效 |
| AUTH-005 | 管理员 | 访问用户、节点、审计功能 | 可执行授权操作 |
| AUTH-006 | 普通用户 | 访问管理员功能 | 按钮隐藏,后端返回 403 |
| AUTH-007 | 项目隔离 | 用户 A 访问用户 B 项目 | 列表不显示,接口拒绝 |
| AUTH-008 | 租户隔离 | 租户 A 请求租户 B 数据 | 不返回跨租户数据 |
| AUTH-009 | ACL | 授予模型 read/execute | read 只能看execute 才能运行 |
| AUTH-010 | 软删除 | 删除模型或数据集 | 列表隐藏,保留删除审计字段 |
| NODE-001 | 新增节点 | 填写 API、文件网关和标签 | 节点保存并显示 |
| NODE-002 | 节点测试 | 点击测试 | health/GPU 信息同步 |
| NODE-003 | 不可达节点 | 使用错误地址测试 | 快速失败并显示原因 |
| NODE-004 | 删除节点 | 点击删除 | 节点从可用列表消失 |
| NODE-005 | 多 GPU | 节点有多张卡 | 显示编号、显存和状态 |
| NODE-006 | 指定 GPU | 训练选择 GPU 0 | 只占用 GPU 0 |
| NODE-007 | GPU 冲突 | 两任务申请同卡 | 后者排队或拒绝,不抢占 |
| NODE-008 | 剩余 GPU | 节点有空闲卡 | 其他任务可继续选择该节点 |
| NODE-009 | 释放 GPU | 停止训练/卸载推理 | GPU 恢复可用 |
| DATA-001 | JSON 统计 | 上传 3 条 JSON 数据 | 列表和详情均为 3 条 |
| DATA-002 | JSONL 统计 | 上传 3 行 JSONL | 列表和详情均为 3 条 |
| DATA-003 | 非法文件 | 上传空或错误格式 | 返回明确错误 |
| DATA-004 | MinIO 归档 | 上传数据集 | 产生对象和 checksum |
| DATA-005 | 节点同步 | 选择算力节点上传 | 文件进入目标节点缓存 |
| DATA-006 | 同步断网 | 同步时阻断节点 | 进入重试或失败,不无限等待 |
| DATA-007 | 数据权限 | 用户查看数据集 | 只显示有权限的数据 |
| TRAIN-001 | 创建训练 | 选择项目、数据集、节点和 GPU | 任务关联完整 |
| TRAIN-002 | 启动训练 | 启动任务 | 进入 queued/running |
| TRAIN-003 | 日志轮询 | 打开训练日志 | 约 3 秒更新,不刷屏 |
| TRAIN-004 | 训练曲线 | 产生 loss/metric | 页面显示曲线 |
| TRAIN-005 | 停止训练 | 点击停止 | 进程停止且资源释放 |
| TRAIN-006 | 训练失败 | 模拟引擎失败 | 显示原因和日志 |
| MERGE-001 | 自动准备 | 执行权重合并 | 自动准备 base model/adapter |
| MERGE-002 | 节点一致 | 权重在训练节点 | 合并请求发往训练节点 |
| MERGE-003 | 合并归档 | 合并成功 | 结果上传 MinIO 并登记 |
| MERGE-004 | 合并权限 | 无 execute 用户操作 | 返回 403 |
| INF-001 | 列表加载 | 点击模型推理 | 列表快速显示,不长时间等待 |
| INF-002 | 指定节点 | 多节点时选择节点 B | 模型只在 B 加载 |
| INF-003 | 训练节点优先 | 未重新指定节点 | 优先使用训练节点 |
| INF-004 | 推理缓存 | 启动未缓存模型 | 从 MinIO 下载到目标节点 |
| INF-005 | 加载超时 | 模拟加载超过 15 分钟 | 失败并显示原因 |
| INF-006 | 对话推理 | 发送消息 | 返回推理结果 |
| INF-007 | 释放推理 | 点击释放 | 卸载模型并释放 GPU |
| INF-008 | 删除推理 | 删除任务记录 | 记录删除成功并释放资源 |
| EVAL-001 | 创建评测 | 选择模型、数据集、指标 | 任务创建成功 |
| EVAL-002 | 数据集权限 | 选择无权数据集 | 不出现在选择列表 |
| EVAL-003 | 指标保存 | 选择具体指标 | 结果不错误显示 custom |
| EVAL-004 | 评测报告 | 等待任务完成 | 返回非空报告和明细 |
| EVAL-005 | 页面轮询 | 打开评测页面 | 不刷屏loading 可结束 |
| OPS-001 | 审计 | 登录、创建、删除、执行资源 | 记录 actor/action/resource/time |
| OPS-002 | 轮询日志 | 观察 Backend 日志 | 成功轮询不高频输出 INFO |
| OPS-003 | 错误日志 | 模拟依赖异常 | 保留 WARNING/ERROR 和 request_id |
| OPS-004 | 推理接口耗时 | 请求 `/model-compare` | 正常环境目标小于 1 秒 |
| OPS-005 | 看板耗时 | 请求 `/dashboard/stats` | 有短缓存且不无限等待 |
| OPS-006 | 并发访问 | 10 用户同时打开列表 | 无连接池耗尽和 5xx |
| OPS-007 | 数据库断开 | 临时阻断 PostgreSQL | 页面明确显示依赖异常 |
## 回归命令
```bash
cd frontend
npm run build
cd ..
python -m compileall backend/app compute
git diff --check
docker compose -f docker/app/docker-compose.yml ps
docker compose -f docker/compute/docker-compose.yml ps
curl http://localhost:17861/modelTF/health
curl http://localhost:19100/modelTF/health
```
失败用例必须附接口响应、容器日志、request_id 和复现步骤。