merge: 合并远程 ft_wyt 分支,解决权限与日志模块冲突

- 冲突解决原则:本地权限治理(require_admin/current_user/资源ACL)与远程 op_log 日志装饰器双向保留
- platform.py: 9 处冲突,@op_log 与管理员校验叠加,避免远程丢失 require_admin 的安全回归
- logging.py: 合并 get_client_ip 与 user_id_var,X-Trace-Id 优先 + ContextVar 卫生处理
- op_log.py: 采纳远程将变量计算上移到函数顶部的结构
- compute_poller.py: 中文日志 + 失败去重限流/断连重置逻辑
- data_process.py: 保留租户归属字段 + biz_logger 成功日志
This commit is contained in:
wuyongtao
2026-08-21 09:58:43 +08:00
27 changed files with 2778 additions and 688 deletions

View File

@@ -41,8 +41,12 @@ from app.core.auth import (
has_resource_access,
is_admin,
)
from app.core.logging import get_structured_logger
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -117,6 +121,7 @@ from app.schemas.data_process import (
)
logger = logging.getLogger(__name__)
biz_logger = get_structured_logger("app.biz.data_process")
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
MAX_SOURCE_FILE_COUNT = 20
MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
@@ -292,7 +297,7 @@ def _commit_source_batch(
except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因,并继续清理其余对象。
logger.exception(
"failed to roll back data process source object task_id=%s",
"数据处理源对象回滚失败 task_id=%s",
task_id,
)
raise
@@ -662,7 +667,7 @@ def _run_generation(
) -> None:
started_at = time.perf_counter()
logger.info(
"data process generation worker started task_id=%s generation_run_id=%s",
"数据处理生成任务开始 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -670,8 +675,7 @@ def _run_generation(
task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id):
logger.info(
"data process generation worker skipped inactive run task_id=%s "
"generation_run_id=%s",
"数据处理生成任务跳过(非活跃运行) task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -761,8 +765,7 @@ def _run_generation(
len(preview_items),
):
logger.info(
"data process generation stopped before completion task_id=%s "
"generation_run_id=%s",
"数据处理生成任务被中止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -853,9 +856,7 @@ def _run_generation(
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
"数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f",
task_id,
generation_run_id,
completed.get("output_count", len(accepted)),
@@ -866,14 +867,13 @@ def _run_generation(
)
else:
logger.info(
"data process generation stopped before result persistence task_id=%s "
"generation_run_id=%s",
"数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
except Exception as exc:
logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
"数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id,
generation_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -887,8 +887,7 @@ def _run_generation(
)
except Exception:
logger.exception(
"failed to persist data process generation failure task_id=%s "
"generation_run_id=%s",
"数据处理生成失败持久化异常 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -943,6 +942,7 @@ def create_task(
values["tenant_id"] = current_user.get("tenant_id") or "default"
get_platform_store().assert_active_tenant(values["tenant_id"])
task = store.create_task(values)
biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", ""))
return ok(task, "data process task created")
@@ -968,10 +968,9 @@ def update_task(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(
store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")),
"data process task updated",
)
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
biz_logger.info("用户更新数据处理任务成功", taskId=task_id)
return ok(result, "data process task updated")
@router.put("/{task_id}/workflow-step")
@@ -1060,7 +1059,7 @@ def _remove_repeated_storage_objects(
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
"数据处理源对象重复回滚失败 task_id=%s",
task_id,
)
@@ -1136,6 +1135,7 @@ def delete_task(
) -> dict[str, Any]:
with api_errors():
store.delete_task(task_id)
biz_logger.info("用户删除数据处理任务成功", taskId=task_id)
return ok({"deleted": task_id}, "data process task deleted")
@@ -1448,13 +1448,12 @@ def delete_source_file(
expected_source_file_id=file_id,
)
except Exception:
# 数据库软删除已经提交,不能再向客户端返回可重试的失败;保留逻辑引用,
# 由后续存储清理任务重试物理删除。
cleanup_pending = True
logger.exception(
"failed to remove data process source object after soft deletion",
"数据处理源对象软删除后存储清理失败",
extra={"task_id": task_id, "source_file_id": file_id},
)
biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
return ok(
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
"source file removed",
@@ -1727,7 +1726,7 @@ def _prepare_preview_items(
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
logger.warning(
"skip PDF document noise detection because stored offsets differ for %s",
"跳过PDF文档噪声检测存储偏移量不一致 source_id=%s",
source["id"],
)
continue
@@ -1750,7 +1749,7 @@ def _run_preview(
started_at = time.perf_counter()
logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
"数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s",
task_id,
preview_run_id,
len(source_file_ids),
@@ -1759,7 +1758,7 @@ def _run_preview(
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
"数据处理预览跳过(非活跃运行) task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -1769,8 +1768,7 @@ def _run_preview(
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id):
logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s "
"completed_files=%s total_files=%s",
"数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files - 1,
@@ -1801,8 +1799,7 @@ def _run_preview(
total_files,
):
logger.info(
"data process preview stopped before progress update task_id=%s "
"preview_run_id=%s completed_files=%s total_files=%s",
"数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files,
@@ -1811,8 +1808,7 @@ def _run_preview(
return
if store.complete_preview(task_id, preview_run_id):
logger.info(
"data process preview completed task_id=%s preview_run_id=%s "
"total_files=%s total_items=%s duration_ms=%.2f",
"数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f",
task_id,
preview_run_id,
total_files,
@@ -1821,14 +1817,13 @@ def _run_preview(
)
else:
logger.info(
"data process preview completion ignored for inactive run task_id=%s "
"preview_run_id=%s",
"数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
except Exception as exc:
logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
"数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id,
preview_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -1842,8 +1837,7 @@ def _run_preview(
)
except Exception:
logger.exception(
"failed to persist data process preview failure task_id=%s "
"preview_run_id=%s",
"数据处理预览失败持久化异常 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -2053,6 +2047,7 @@ def stop(
) -> dict[str, Any]:
with api_errors():
store.stop_task(task_id)
biz_logger.info("用户停止数据处理任务成功", taskId=task_id)
return ok(store.progress(task_id), "data process task stopped")
@@ -2094,7 +2089,9 @@ def confirm_results(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed")
result = store.confirm_results(task_id)
biz_logger.info("用户确认数据处理结果成功", taskId=task_id)
return ok(result, "data process results confirmed")
@router.put("/{task_id}/results/{result_id}")
@@ -2443,8 +2440,7 @@ def regenerate_results_batch(
}))
logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s "
"requested=%s prepared=%s concurrency=%s",
"数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s",
batch_id,
task_id,
len(payload.items),
@@ -2512,8 +2508,7 @@ def regenerate_results_batch(
except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error"
logger.exception(
"data process result batch regeneration crashed "
"batch_id=%s task_id=%s result_id=%s",
"数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s",
batch_id,
task_id,
result_id,
@@ -2524,8 +2519,7 @@ def regenerate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch item finished batch_id=%s task_id=%s "
"result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2545,8 +2539,7 @@ def regenerate_results_batch(
)
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
"数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2593,8 +2586,7 @@ def evaluate_results_batch(
evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError:
logger.warning(
"data process evaluation model unavailable, judge layer "
"skipped task_id=%s model_id=%s",
"数据处理评测模型不可用,跳过评测层 task_id=%s model_id=%s",
task_id,
model_id,
)
@@ -2642,8 +2634,7 @@ def evaluate_results_batch(
}))
logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s "
"requested=%s prepared=%s judge_enabled=%s",
"数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s",
batch_id,
task_id,
len(payload.items),
@@ -2660,8 +2651,7 @@ def evaluate_results_batch(
semantic_embedding_model()
except Exception:
logger.warning(
"data process semantic embedding unavailable, semantic layer "
"will be skipped batch_id=%s",
"数据处理语义嵌入模型不可用,语义层将跳过 batch_id=%s",
batch_id,
)
request_timeout = _result_regeneration_timeout(config)
@@ -2714,8 +2704,7 @@ def evaluate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation item finished "
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2727,8 +2716,7 @@ def evaluate_results_batch(
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s duration_ms=%.2f",
"数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2784,8 +2772,7 @@ def regenerate_result(
)
except _ResultRegenerationFailed as exc:
logger.warning(
"data process result regeneration failed task_id=%s result_id=%s "
"duration_ms=%.2f reason=%s",
"数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2793,7 +2780,7 @@ def regenerate_result(
)
raise
logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
"数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2809,5 +2796,6 @@ def publish(
) -> dict[str, Any]:
with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json"))
biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", ""))
message = "dataset published" if result["created"] else "dataset already published"
return ok(result, message)

View File

@@ -913,6 +913,7 @@ async def _fine_tune_preflight_with_job_payload(
@router.post("/login")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGIN, target_type="user", target_name_param="username")
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
ip = request.client.host if request and request.client else "unknown"
@@ -930,6 +931,7 @@ async def login(payload: dict[str, Any] = Body(...), request: Request = None) ->
@router.post("/logout")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGOUT, target_type="user")
async def logout(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
session_id = payload.get("session_id", "")
@@ -1179,6 +1181,7 @@ async def users(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
@router.post("/users")
@op_log(module=OpModule.SYSTEM, action=OpAction.CREATE, target_type="user", target_name_param="username")
async def create_user(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
payload = dict(payload)
payload.setdefault("password", "123456")
@@ -1187,6 +1190,7 @@ async def create_user(payload: dict[str, Any] = Body(...), current_user: dict =
@router.put("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user", target_name_param="user_id")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_user(user_id, payload))
@@ -1195,6 +1199,7 @@ async def update_user(user_id: str, payload: dict[str, Any] = Body(...), current
@router.delete("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.DELETE, target_type="user", target_name_param="user_id")
async def delete_user(user_id: str, current_username: str | None = Query(default=None), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
get_platform_store().delete_user(user_id, deleted_by=str(current_user.get("id") or "system"))
@@ -1212,6 +1217,7 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
@router.post("/users/{user_id}/reset-password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password", target_name_param="user_id")
async def reset_user_password(
user_id: str,
payload: dict[str, Any] = Body(default={}),
@@ -1228,6 +1234,7 @@ async def reset_user_password(
@router.post("/users/me/password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password")
async def change_my_password(
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
@@ -1310,6 +1317,7 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
@router.delete("/model-manage/trained-models/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="trained_model", target_name_param="model_id")
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("trained_model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this trained model")
@@ -1424,6 +1432,7 @@ async def test_online_model(payload: dict[str, Any] = Body(...), current_user: d
@router.post("/model-manage")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.CREATE, target_type="model", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_MODEL,
target_type="model",
@@ -1465,6 +1474,7 @@ async def model_detail(model_id: str, current_user: dict = Depends(get_current_u
@router.put("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model", target_name_param="model_id")
@audit_log(
action=AuditActions.UPDATE_MODEL,
target_type="model",
@@ -1481,6 +1491,7 @@ async def update_model(model_id: str, payload: dict[str, Any] = Body(...), curre
@router.put("/model-manage/{model_id}/purpose")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model_purpose", target_name_param="model_id")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型(配置模型)只有管理员可以修改用途
if not is_admin(current_user):
@@ -1492,6 +1503,7 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
@router.delete("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="model", target_name_param="model_id")
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型(配置模型)只有管理员可以删除
if not is_admin(current_user):
@@ -1691,6 +1703,7 @@ async def dataset_version_content(file_id: str, version_id: str, current_user: d
@router.post("/dataset-manage/versions/{file_id}")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset_version", target_name_param="file_id")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
store = get_platform_store()
@@ -1715,6 +1728,7 @@ async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(
@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
@op_log(module=OpModule.DATASET, action=OpAction.DELETE, target_type="dataset_version", target_name_param="version_id")
async def delete_dataset_version(file_id: str, version_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
store = get_platform_store()
@@ -1913,6 +1927,7 @@ async def _sync_training_dataset_to_compute_node(
@router.post("/dataset-manage/upload/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPLOAD, target_type="dataset_file", target_name_param="dataset_id")
async def upload_dataset_files(
dataset_id: str,
files: list[UploadFile] = File(default=[]),
@@ -2080,6 +2095,7 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s
@router.post("/dataset-manage")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_DATASET,
target_type="dataset",
@@ -2110,6 +2126,7 @@ async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_curre
@router.put("/dataset-manage/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPDATE, target_type="dataset", target_name_param="dataset_id")
@audit_log(
action=AuditActions.UPDATE_DATASET,
target_type="dataset",
@@ -2175,6 +2192,7 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/fine-tune")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.CREATE, target_type="fine_tune", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_FINE_TUNE,
target_type="fine_tune",
@@ -2418,6 +2436,7 @@ async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depe
@router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "write"):
raise fail(403, "no permission to update this task")
@@ -2452,6 +2471,7 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
@router.post("/fine-tune/{task_id}/retry")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.RETRY, target_type="fine_tune", target_name_param="task_id")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}
@@ -2883,6 +2903,7 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
@router.delete("/model-eval/{task_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval_task", target_name_param="task_id")
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("eval", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this eval task")
@@ -2902,6 +2923,7 @@ async def dimension_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/dimension")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.CREATE, target_type="dimension", target_name_param="name")
async def dimension_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
return ok(get_platform_store().create_dimension(payload))
@@ -2915,6 +2937,7 @@ async def dimension_detail(dimension_id: str, current_user: dict = Depends(get_c
@router.put("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.UPDATE, target_type="dimension", target_name_param="dimension_id")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(require_admin)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_dimension(dimension_id, payload))
@@ -2923,6 +2946,7 @@ async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...
@router.delete("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="dimension", target_name_param="dimension_id")
async def dimension_delete(dimension_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
get_platform_store().delete_dimension(dimension_id)
return ok({"deleted": dimension_id})
@@ -2943,6 +2967,7 @@ async def model_compare_list(current_user: dict = Depends(get_current_user)) ->
@router.post("/model-compare")
@op_log(module=OpModule.INFERENCE, action=OpAction.CREATE, target_type="compare_task", target_name_param="name")
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
bind_active_tenant(payload, current_user)
payload.setdefault("created_by", current_user.get("id"))
@@ -3785,6 +3810,7 @@ async def compute_node_detail(node_id: str, current_user: dict = Depends(get_cur
@router.post("/compute/nodes")
@op_log(module=OpModule.COMPUTE, action=OpAction.CREATE, target_type="compute_node", target_name_param="name")
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -3797,6 +3823,7 @@ async def create_compute_node(payload: dict[str, Any] = Body(...), current_user:
@router.put("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.UPDATE, target_type="compute_node", target_name_param="node_id")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -3809,6 +3836,7 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...),
@router.delete("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.DELETE, target_type="compute_node", target_name_param="node_id")
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -4123,6 +4151,7 @@ async def compute_job_detail(job_id: str, current_user: dict = Depends(get_curre
@router.post("/compute/jobs/{job_id}/stop")
@op_log(module=OpModule.COMPUTE, action=OpAction.STOP, target_type="compute_job", target_name_param="job_id")
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
task = _task_for_compute_job(job_id)
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
@@ -4166,6 +4195,7 @@ async def compute_job_logs(
@router.post("/compute/jobs/{job_id}/retry")
@op_log(module=OpModule.COMPUTE, action=OpAction.RETRY, target_type="compute_job", target_name_param="job_id")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}

View File

@@ -50,7 +50,7 @@ def docs_kwargs(enabled: bool) -> dict[str, Any]:
@dataclass(frozen=True)
class Settings:
app_name: str = os.getenv("APP_NAME", "YG Fine-Tune Platform API")
app_name: str = os.getenv("APP_NAME", "YG Zhilian API")
app_env: str = os.getenv("APP_ENV", "local")
route_prefix: str = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF")
app_mode: str = os.getenv("APP_MODE", "local")

View File

@@ -1,22 +1,26 @@
from __future__ import annotations
from contextvars import ContextVar
from datetime import date, datetime, timedelta
import json
import logging
import re
import socket
import sys
import time
from contextvars import ContextVar
from datetime import date, datetime, timedelta
from logging import Handler, LogRecord
from pathlib import Path
import re
import time
from typing import Any, Callable, Optional
from typing import Any, Callable
from uuid import uuid4
from fastapi import FastAPI, Request
from app.core.config import Settings, get_settings
# ==================== 链路追踪 ContextVar ====================
request_id_var: ContextVar[str] = ContextVar("request_id", default="-")
user_id_var: ContextVar[str] = ContextVar("user_id", default="")
client_ip_var: ContextVar[str] = ContextVar("client_ip", default="")
@@ -30,54 +34,74 @@ def get_client_ip(request: Request | None) -> str:
return value.split(",", 1)[0].strip()
return request.client.host if request.client else ""
# ==================== 敏感数据脱敏规则 ====================
# ==================== 敏感数据脱敏 ====================
SENSITIVE_PATTERNS: dict[str, Callable | str] = {
"token": "***",
"password": "***",
"access_token": "***",
"refresh_token": "***",
"secret_key": "***",
"authorization": "***",
"bearer": "***",
"api_key": "***",
"private_key": "***",
SENSITIVE_KEYS: set[str] = {
"password", "token", "access_token", "refresh_token",
"secret_key", "authorization", "bearer", "api_key",
"private_key", "secret", "cookie",
}
def mask_value(key: str, value: Any) -> str:
"""对单个值进行脱敏处理"""
FULL_MASK_KEYS: set[str] = {
"password", "token", "access_token", "refresh_token",
"secret_key", "authorization", "bearer", "api_key",
"private_key", "secret", "cookie",
}
def _mask_phone(value: str) -> str:
"""手机号脱敏138****5678"""
if len(value) >= 11:
return value[:3] + "****" + value[-4:]
return value
def _mask_id_card(value: str) -> str:
"""身份证号脱敏110***********1234"""
if len(value) >= 18:
return value[:3] + "***********" + value[-4:]
return value
def mask_value(key: str, value: Any) -> Any:
"""对单个值进行脱敏处理。"""
if value is None:
return ""
key_lower = key.lower()
if key_lower in FULL_MASK_KEYS:
return "***"
str_val = str(value)
handler = SENSITIVE_PATTERNS.get(key)
if callable(handler):
return handler(str_val)
elif isinstance(handler, str):
# 支持正则替换模式,如 r"1\d{3}\d{4}"
try:
return re.sub(handler, "***", str_val)
except re.error:
return "***"
return handler
# 手机号模式11位数字1开头
if re.match(r"^1[3-9]\d{9}$", str_val):
return _mask_phone(str_val)
# 身份证模式18位
if re.match(r"^\d{17}[\dXx]$", str_val):
return _mask_id_card(str_val)
return value
def mask_sensitive_dict(data: dict) -> dict:
"""递归脱敏字典中的敏感字段"""
"""递归脱敏字典中的敏感字段"""
if not data or not isinstance(data, dict):
return data
result = {}
result: dict[str, Any] = {}
for key, value in data.items():
result[key] = mask_value(key, value)
if isinstance(value, dict):
result[key] = mask_sensitive_dict(value)
elif isinstance(value, list):
result[key] = [
mask_sensitive_dict(item) if isinstance(item, dict) else item
for item in value
]
else:
result[key] = mask_value(key, value)
return result
def mask_sensitive_string(text: str) -> str:
"""从文本中脱敏常见敏感信息"""
"""从文本中脱敏常见敏感信息"""
if not text:
return text
# Mask the value as well as the key. Replacing only ``api_key=`` would
# still leak the credential in audit messages and exception text.
assignment_pattern = (
@@ -90,92 +114,165 @@ def mask_sensitive_string(text: str) -> str:
except re.error:
pass
patterns = [
(r'Bearer\s+[A-Za-z0-9\-._]+', 'Bearer ***'),
(r'\d{11}', r'\d{3}\*\d{4}'), # 手机号/身份证
(r'1[3-9]\d{9}', r'1\*{3}\*{4}'), # 手机号
patterns: list[tuple[str, str]] = [
(r"Bearer\s+[A-Za-z0-9\-._]+", "Bearer ***"),
(r"(?i)token\s*[:=]\s*\S+", "token=***"),
(r"(?i)password\s*[:=]\s*\S+", "password=***"),
(r"(?i)secret[_-]?key\s*[:=]\s*\S+", "secret_key=***"),
(r"(?i)api[-_]?key\s*[:=]\s*\S+", "api_key=***"),
(r"(?i)private[_-]?key\s*[:=]\s*\S+", "private_key=***"),
(r"(?i)authorization\s*[:=]\s*\S+", "authorization=***"),
]
for pattern, replacement in patterns:
try:
text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
except re.error:
pass
text = re.sub(pattern, replacement, text)
# 手机号脱敏
text = re.sub(r"\b1[3-9]\d{9}\b", lambda m: _mask_phone(m.group()), text)
return text
# ==================== RequestId Filter ====================
# ==================== 大对象截断 ====================
MAX_FIELD_SIZE = 1024 # 超过 1KB 的内容自动截断
def truncate_large_value(value: Any, max_size: int = MAX_FIELD_SIZE) -> Any:
"""超过 max_size 的字符串自动截断(前 500 + 后 500"""
if isinstance(value, str) and len(value) > max_size:
half = max_size // 2
return value[:half] + f"...[truncated {len(value) - max_size} chars]..." + value[-half:]
if isinstance(value, dict):
return {k: truncate_large_value(v, max_size) for k, v in value.items()}
if isinstance(value, list):
return [truncate_large_value(v, max_size) for v in value]
return value
# ==================== TraceId Filter ====================
class TraceIdFilter(logging.Filter):
"""自动注入 traceId / userId / clientIp 到每条日志记录。"""
class RequestIdFilter(logging.Filter):
def filter(self, record: LogRecord) -> bool:
record.request_id = request_id_var.get()
record.traceId = request_id_var.get()
record.userId = user_id_var.get("")
record.clientIp = client_ip_var.get("")
record.host = getattr(self, "_host", None) or socket.gethostname()
record.app = getattr(self, "_app", "yg-ft-platform")
record.env = getattr(self, "_env", "dev")
return True
def set_context(self, app: str, env: str, host: str) -> None:
self._app = app
self._env = env
self._host = host
# ==================== Enhanced JSON Formatter ====================
# ==================== JSON Formatter ====================
class JsonLogFormatter(logging.Formatter):
"""
增强的 JSON 日志格式化器,支持结构化字段输出
生产级 JSON 日志格式化器,符合方案文档 §3.2 字段规范
输出示例:
{
"@timestamp": "2026-08-17T18:30:00.123Z",
"@timestamp": "2026-08-19T10:30:45.123+08:00",
"level": "INFO",
"logger": "dataset.router",
"logger": "app.api.v1.endpoints.platform",
"traceId": "abc-123-def-456",
"userId": "u_admin",
"message": "数据集创建成功",
"module": "dataset.router",
"function": "create_dataset",
"file": "dataset/router.py",
"line": 45,
"process": 12345,
"fields": {"datasetId": "ds_001", "costMs": 23},
"file": "platform.py:156",
"thread": "MainThread",
"request_id": "req-abc123",
"user_id": "u_admin",
"client_ip": "192.168.1.100",
"extra": {...}
"host": "pod-7x9k2",
"app": "yg-ft-platform",
"env": "dev"
}
"""
# 标准 LogRecord 属性名集合,用于区分 extra 字段
_STD_ATTRS: set[str] = set(vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()) | {
"traceId", "userId", "clientIp", "host", "app", "env",
"request_id", "user_id", "client_ip",
"asctime", "message", "module", "function", "process",
"thread", "threadName", "levelname", "levelno", "name",
"pathname", "filename", "lineno", "funcName", "created",
"msecs", "relativeCreated", "exc_info", "exc_text",
"stack_info", "msg", "args", "processName", "process",
}
def format(self, record: LogRecord) -> str:
# 时间戳ISO8601 带时区
timestamp = datetime.fromtimestamp(record.created).astimezone().isoformat(
timespec="milliseconds"
)
payload: dict[str, Any] = {
"@timestamp": datetime.fromtimestamp(record.created).astimezone().isoformat(
timespec="milliseconds"
),
"@timestamp": timestamp,
"level": record.levelname,
"logger": record.name,
"traceId": getattr(record, "traceId", "-"),
"message": record.getMessage(),
"module": record.module,
"function": record.funcName,
"file": record.pathname,
"line": record.lineno,
"process": record.process,
"thread": record.thread,
"thread_name": record.threadName,
"request_id": getattr(record, "request_id", "-"),
"file": f"{Path(record.pathname).name}:{record.lineno}",
"thread": record.threadName,
"host": getattr(record, "host", ""),
"app": getattr(record, "app", ""),
"env": getattr(record, "env", ""),
}
# 从 record 中提取额外字段(通过 extra 参数传入
for attr in ("user_id", "client_ip", "target_type", "target_id",
"duration_ms", "status_code", "error"):
# userId业务必填未登录可为空
user_id = getattr(record, "userId", "") or getattr(record, "user_id", "")
if user_id:
payload["userId"] = user_id
# clientIp
client_ip = getattr(record, "clientIp", "") or getattr(record, "client_ip", "")
if client_ip:
payload["clientIp"] = client_ip
# 提取结构化业务字段:只收集通过 extra 传入的非标准属性
fields: dict[str, Any] = {}
for attr in dir(record):
if attr.startswith("_"):
continue
if attr in self._STD_ATTRS:
continue
if attr in ("traceId", "userId", "clientIp", "host", "app", "env"):
continue
val = getattr(record, attr, None)
if val is not None:
payload[attr] = val
# 处理异常信息
if record.exc_info:
if val is not None and not callable(val):
fields[attr] = truncate_large_value(val)
if fields:
payload["fields"] = mask_sensitive_dict(fields)
# ERROR 级别额外字段
if record.levelname == "ERROR" or record.exc_info:
error_obj: dict[str, Any] = {
"type": type(record.exc_info[1]).__name__ if record.exc_info and record.exc_info[1] else "Error",
"message": record.getMessage(),
}
if record.exc_info:
error_obj["stack_trace"] = self.formatException(record.exc_info)
if record.stack_info:
error_obj["stack_trace"] = self.formatStack(record.stack_info)
payload["error"] = error_obj
# 兼容旧字段名 exception
if record.exc_info and "error" not in payload:
payload["exception"] = self.formatException(record.exc_info)
if record.stack_info:
payload["stack"] = self.formatStack(record.stack_info)
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
# ==================== DateSizeRotatingFileHandler ====================
# (保持不变,已有实现)
class DateSizeRotatingFileHandler(Handler):
"""Rotate log files by date and size while keeping date in every file name."""
"""按日期+大小滚动的文件日志处理器。
- 按天创建文件,文件名包含日期
- 单文件超过 max_bytes 时自动滚动(带序号后缀)
- 自动清理超过 retention_days 的旧日志
"""
def __init__(
self,
@@ -233,10 +330,8 @@ class DateSizeRotatingFileHandler(Handler):
today = date.today()
if not force and self._stream and self._current_date == today:
return
if self._stream and not self._stream.closed:
self._stream.close()
self._current_date = today
self._current_path = self._dated_path(today)
self._stream = self._current_path.open("a", encoding=self.encoding)
@@ -251,11 +346,9 @@ class DateSizeRotatingFileHandler(Handler):
def _rotate_by_size(self) -> None:
if not self._current_path or not self._current_path.exists():
return
if self._stream and not self._stream.closed:
self._stream.close()
self._stream = None
stem = self._current_path.stem
suffix = self._current_path.suffix
index = 1
@@ -269,7 +362,6 @@ class DateSizeRotatingFileHandler(Handler):
def _cleanup_expired_files(self) -> None:
if self.retention_days <= 0:
return
cutoff = date.today() - timedelta(days=self.retention_days - 1)
pattern = re.compile(
rf"^{re.escape(self.file_prefix)}-(\d{{4}}-\d{{2}}-\d{{2}})(?:\.\d+)?\.log$"
@@ -283,99 +375,126 @@ class DateSizeRotatingFileHandler(Handler):
path.unlink(missing_ok=True)
# ==================== Structured Logger 封装 ====================
# ==================== StructuredLogger 封装 ====================
class StructuredLogger:
"""
结构化日志记录器,提供统一的日志接口。
结构化日志记录器,提供符合方案文档 §4.2 的 5W1H 日志接口。
使用方式:
logger = get_structured_logger('dataset.router')
logger.info('创建数据集', dataset_id='ds_123')
logger = get_structured_logger('app.api.dataset')
logger.info('数据集创建成功', datasetId='ds_001', costMs=23)
"""
def __init__(self, name: str, module: str = ""):
self.logger = logging.getLogger(name)
self.name = name
self.module = module
@property
def trace_id(self) -> str:
return request_id_var.get("-")
def info(self, message: str, **extra: Any) -> None:
self._log("INFO", message, **extra)
def warning(self, message: str, **extra: Any) -> None:
self._log("WARNING", message, **extra)
def info(self, message: str, **fields: Any) -> None:
self._log(logging.INFO, message, **fields)
def error(self, message: str, **extra: Any) -> None:
self._log("ERROR", message, **extra)
def warning(self, message: str, **fields: Any) -> None:
self._log(logging.WARNING, message, **fields)
def debug(self, message: str, **extra: Any) -> None:
self._log("DEBUG", message, **extra)
def error(self, message: str, **fields: Any) -> None:
self._log(logging.ERROR, message, **fields)
def _log(self, level: str, message: str, **extra: Any) -> None:
"""统一日志记录方法"""
log_entry: dict[str, Any] = {
"timestamp": datetime.utcnow().isoformat(),
"level": level,
"logger": self.name,
"module": self.module,
"message": message,
"trace_id": self.trace_id,
"extra": extra,
}
self.logger.log(getattr(logging, level, logging.INFO), json.dumps(log_entry, ensure_ascii=False, default=str))
def debug(self, message: str, **fields: Any) -> None:
self._log(logging.DEBUG, message, **fields)
def _log(self, level: int, message: str, **fields: Any) -> None:
"""统一日志记录方法,通过 extra 传递结构化字段。"""
extra: dict[str, Any] = {}
if self.module:
extra["module"] = self.module
# 脱敏 + 截断
for k, v in fields.items():
extra[k] = truncate_large_value(v)
self.logger.log(level, message, extra=extra, stack_info=False)
def get_structured_logger(name: str, module: str = "") -> StructuredLogger:
"""获取结构化日志记录器"""
"""获取结构化日志记录器"""
return StructuredLogger(name, module)
# ==================== 快捷函数 ====================
def get_logger(name: str) -> logging.Logger:
"""获取标准 Python logger"""
"""获取标准 Python logger"""
return logging.getLogger(name)
def set_request_id(request_id: str) -> None:
"""设置当前请求的追踪 ID"""
"""设置当前请求的追踪 ID"""
request_id_var.set(request_id)
def set_user_context(user_id: str = "", client_ip: str = "") -> None:
"""设置当前请求的用户上下文(在鉴权后调用)。"""
if user_id:
user_id_var.set(user_id)
if client_ip:
client_ip_var.set(client_ip)
# ==================== 请求日志中间件 ====================
def setup_request_logging(app: FastAPI) -> None:
"""配置 FastAPI 请求日志中间件"""
"""配置 FastAPI 请求日志中间件,符合方案文档 §五(链路追踪)和 §十(访问日志)。"""
logger = get_logger("app.access")
@app.middleware("http")
async def request_logging_middleware(request: Request, call_next): # type: ignore[no-untyped-def]
request_id = request.headers.get("X-Request-ID") or str(uuid4())
token = request_id_var.set(request_id)
# 入口生成 traceId优先使用前端传入的 X-Trace-Id
trace_id = request.headers.get("X-Trace-Id") or request.headers.get("X-Request-ID") or str(uuid4())
token = request_id_var.set(trace_id)
ip_token = client_ip_var.set(get_client_ip(request))
started_at = time.perf_counter()
# 提取客户端 IP
client_ip = "-"
if request.client:
client_ip = request.client.host
# 支持反向代理传递的真实 IP
forwarded_for = request.headers.get("X-Forwarded-For", "")
if forwarded_for:
client_ip = forwarded_for.split(",")[0].strip()
client_ip_var.set(client_ip)
try:
response = await call_next(request)
elapsed_ms = (time.perf_counter() - started_at) * 1000
# 噪声路径降级为 DEBUG健康检查等
noisy_paths = ("/health", "/system-info", "/compute/jobs/", "/model-eval/", "/model-compare/")
log_method = logger.debug if request.method == "GET" and response.status_code < 400 else logger.info
if any(request.url.path.endswith(path) or path in request.url.path for path in noisy_paths) and response.status_code < 400:
log_method = logger.info
if any(request.url.path.endswith(p) or p in request.url.path for p in noisy_paths) and response.status_code < 400:
log_method = logger.debug
if response.status_code >= 400:
if response.status_code >= 500:
log_method = logger.error
elif response.status_code >= 400:
log_method = logger.warning
# 结构化访问日志(中文 message方便直接阅读
log_method(
"request completed method=%s path=%s status_code=%s duration_ms=%.2f client=%s",
request.method,
request.url.path,
response.status_code,
elapsed_ms,
request.client.host if request.client else "-",
f"HTTP请求 {request.method} {request.url.path} {response.status_code}(耗时{round(elapsed_ms, 2)}ms",
extra={
"request_method": request.method,
"request_path": request.url.path,
"status_code": response.status_code,
"duration_ms": round(elapsed_ms, 2),
"client_ip": client_ip,
"user_agent": request.headers.get("User-Agent", "")[:200],
},
)
# 5xx 系统错误自动写入操作日志(未被 @op_log 覆盖的系统级异常)
# 5xx 系统错误自动写入操作日志
if response.status_code >= 500:
try:
from app.core.op_log import log_operation, OpModule, OpStatus
@@ -393,18 +512,23 @@ def setup_request_logging(app: FastAPI) -> None:
duration_ms=elapsed_ms,
)
except Exception:
pass # 日志写入失败不影响主流程
pass
response.headers["X-Request-ID"] = request_id
response.headers["X-Trace-Id"] = trace_id
response.headers["X-Request-ID"] = trace_id
return response
except Exception:
elapsed_ms = (time.perf_counter() - started_at) * 1000
logger.exception(
"request failed method=%s path=%s duration_ms=%.2f client=%s",
request.method,
request.url.path,
elapsed_ms,
request.client.host if request.client else "-",
logger.error(
f"HTTP请求异常 {request.method} {request.url.path}(耗时{round(elapsed_ms, 2)}ms— 服务内部错误",
extra={
"request_method": request.method,
"request_path": request.url.path,
"duration_ms": round(elapsed_ms, 2),
"client_ip": client_ip,
},
exc_info=True,
)
# 未被捕获的异常,写入操作日志
@@ -426,7 +550,7 @@ def setup_request_logging(app: FastAPI) -> None:
duration_ms=elapsed_ms,
)
except Exception:
pass # 日志写入失败不影响主流程
pass
raise
finally:
@@ -437,53 +561,122 @@ def setup_request_logging(app: FastAPI) -> None:
# ==================== 配置函数 ====================
def configure_logging(settings: Settings | None = None) -> None:
"""
生产级日志配置,符合方案文档 §二(分类分流)和 §六(性能安全)。
日志分类:
- 业务日志 (app-biz): INFO+ 业务流程(保留 7 天)
- 系统日志 (app-sys): 框架/中间件日志(保留 7 天)
- 访问日志 (app-access): HTTP 请求日志(保留 15 天)
- 错误日志 (app-error): ERROR 级别(保留 30 天)
"""
settings = settings or get_settings()
root_logger = logging.getLogger()
root_logger.handlers.clear()
root_logger.setLevel(settings.log_level.upper())
# ---- Formatter ----
console_formatter = logging.Formatter(
fmt=(
"%(asctime)s | %(levelname)s | pid=%(process)d | %(threadName)s | "
"request_id=%(request_id)s | %(name)s | %(pathname)s:%(lineno)d | %(message)s"
"traceId=%(traceId)s | %(name)s | %(pathname)s:%(lineno)d | %(message)s"
),
datefmt="%Y-%m-%d %H:%M:%S",
)
json_formatter = JsonLogFormatter()
request_filter = RequestIdFilter()
# ---- TraceIdFilter全局注入 traceId/userId/host/app/env----
trace_filter = TraceIdFilter()
trace_filter.set_context(
app=settings.app_name,
env=settings.app_env,
host=socket.gethostname(),
)
# ---- 控制台 Handler ----
console_handler = logging.StreamHandler()
console_handler.setFormatter(console_formatter)
console_handler.addFilter(request_filter)
console_handler.addFilter(trace_filter)
file_handler = DateSizeRotatingFileHandler(
# ---- 业务日志文件 Handler (app-biz) ----
biz_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_file_prefix,
file_prefix="app-biz",
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
retention_days=7,
)
file_handler.setFormatter(json_formatter)
file_handler.addFilter(request_filter)
biz_file_handler.setFormatter(json_formatter)
biz_file_handler.addFilter(trace_filter)
# ---- 访问日志文件 Handler (app-access) ----
access_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix="app-access",
max_bytes=settings.log_max_bytes,
retention_days=15,
)
access_file_handler.setFormatter(json_formatter)
access_file_handler.addFilter(trace_filter)
# ---- 错误日志文件 Handler (app-error) ----
error_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_error_file_prefix,
file_prefix="app-error",
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
retention_days=30,
)
error_file_handler.setLevel(logging.ERROR)
error_file_handler.setFormatter(json_formatter)
error_file_handler.addFilter(request_filter)
error_file_handler.addFilter(trace_filter)
# ---- 注册 Handler ----
root_logger.addHandler(console_handler)
root_logger.addHandler(file_handler)
root_logger.addHandler(biz_file_handler)
root_logger.addHandler(access_file_handler)
root_logger.addHandler(error_file_handler)
for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
logger = logging.getLogger(logger_name)
logger.handlers.clear()
logger.propagate = True
# ---- 访问日志 Logger 独立路由到访问日志文件 ----
access_logger = logging.getLogger("app.access")
access_logger.propagate = False # 不向 root 传播,避免重复写入业务日志
access_logger.addHandler(console_handler)
access_logger.addHandler(access_file_handler)
# 访问日志中的 ERROR 也要进错误日志
access_logger.addHandler(error_file_handler)
# ---- 框架类 Logger 降级 ----
for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
lg = logging.getLogger(logger_name)
lg.handlers.clear()
lg.propagate = True
# 框架类日志归入系统日志,生产环境设为 WARN
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
logging.getLogger("psycopg.pool").setLevel(logging.ERROR)
logging.getLogger("httpx").setLevel(logging.WARNING)
# ---- 兼容旧文件前缀(向后兼容)----
# 如果配置了旧的 log_file_prefix也创建一个对应的 handler
if settings.log_file_prefix and settings.log_file_prefix != "app-biz":
legacy_file_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_file_prefix,
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
)
legacy_file_handler.setFormatter(json_formatter)
legacy_file_handler.addFilter(trace_filter)
root_logger.addHandler(legacy_file_handler)
# 旧错误日志前缀兼容
if settings.log_error_file_prefix and settings.log_error_file_prefix != "app-error":
legacy_error_handler = DateSizeRotatingFileHandler(
log_dir=settings.log_dir,
file_prefix=settings.log_error_file_prefix,
max_bytes=settings.log_max_bytes,
retention_days=settings.log_retention_days,
)
legacy_error_handler.setLevel(logging.ERROR)
legacy_error_handler.setFormatter(json_formatter)
legacy_error_handler.addFilter(trace_filter)
root_logger.addHandler(legacy_error_handler)

View File

@@ -31,10 +31,11 @@ from typing import Any, Callable, Optional, TypeVar
from fastapi import Request
from app.core.logging import get_client_ip, get_logger, request_id_var
from app.core.logging import get_logger, get_structured_logger, request_id_var
from app.db.platform_store import get_platform_store, new_id, utcnow
logger = get_logger("app.op_log")
biz_logger = get_structured_logger("app.biz")
F = TypeVar("F", bound=Callable[..., Any])
@@ -76,6 +77,92 @@ class OpStatus:
FAILURE = "failure"
# ==================== 中文映射表(让日志 message 直接可读)====================
MODULE_CN: dict[str, str] = {
"fine-tune": "模型训练",
"model-eval": "模型评测",
"model-inference": "模型推理",
"model-manage": "模型管理",
"dataset": "数据集",
"data-process": "数据处理",
"data-convert": "数据转换",
"compute": "算力节点",
"system": "系统",
}
ACTION_CN: dict[str, str] = {
"create": "创建",
"update": "更新",
"delete": "删除",
"start": "启动",
"stop": "停止",
"upload": "上传",
"download": "下载",
"convert": "转换",
"merge": "合并",
"import": "导入",
"login": "登录",
"logout": "退出登录",
"publish": "发布",
"retry": "重试",
"request": "请求",
}
TARGET_TYPE_CN: dict[str, str] = {
"fine_tune": "训练任务",
"eval": "评测任务",
"inference": "推理任务",
"model": "模型",
"trained_model": "训练产出模型",
"dataset": "数据集",
"dataset_version": "数据集版本",
"data_process_task": "数据处理任务",
"user": "用户",
"api": "接口",
}
def _build_cn_message(
module: str,
action: str,
target_type: str,
target_name: str | None,
target_id: str | None,
status: str,
username: str | None,
error_type: str,
error_message: str,
) -> str:
"""构建中文人类可读的日志消息,格式:[用户] 对 [模块] 执行了 [动作],结果:成功/失败"""
user_part = f"用户[{username}]" if username else "系统"
module_cn = MODULE_CN.get(module, module)
action_cn = ACTION_CN.get(action, action)
target_cn = TARGET_TYPE_CN.get(target_type, target_type or "")
target_label = target_name or target_id or ""
# 拼接操作对象描述
if target_cn and target_label:
target_part = f"{target_cn}{target_label}"
elif target_cn:
target_part = target_cn
elif target_label:
target_part = f"{target_label}"
else:
target_part = ""
if status == OpStatus.SUCCESS:
result = "成功"
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:成功"
else:
result = "失败"
err_brief = error_message[:120] if error_message else ""
err_part = f"{error_type}: {err_brief}" if error_type and err_brief else f"{error_type}" if error_type else ""
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:失败{err_part}"
return msg
def op_log(
module: str,
action: str,
@@ -341,20 +428,54 @@ def _write_log(
trace_id: str,
duration_ms: float,
) -> None:
"""写入操作日志到数据库"""
"""写入操作日志到数据库 + 文件日志"""
user_id = user.get("id") if user else None
username = user.get("username") if user else None
client_ip = None
req_method = None
req_path = None
if request:
client_ip = request.client.host if request.client else None
req_method = request.method
req_path = request.url.path
# ---- 写文件日志app-biz----
cn_msg = _build_cn_message(
module=module, action=action, target_type=target_type,
target_name=target_name, target_id=target_id, status=status,
username=username, error_type=error_type, error_message=error_message,
)
log_fields = {
"bizModule": module,
"action": action,
"targetType": target_type or "",
"targetId": target_id or "",
"targetName": target_name or "",
"opStatus": status,
"durationMs": round(duration_ms, 2),
"username": username or "",
}
if req_method:
log_fields["requestMethod"] = req_method
if req_path:
log_fields["requestPath"] = req_path
if detail:
log_fields["detail"] = detail[:500]
if error_message:
log_fields["errorMessage"] = error_message[:500]
if error_type:
log_fields["errorType"] = error_type
if status == OpStatus.SUCCESS:
biz_logger.info(cn_msg, **log_fields)
elif status == OpStatus.FAILURE:
biz_logger.error(cn_msg, **log_fields)
# ---- 写数据库 ----
try:
store = get_platform_store()
log_id = new_id("op")
user_id = user.get("id") if user else None
username = user.get("username") if user else None
client_ip = None
req_method = None
req_path = None
if request:
client_ip = get_client_ip(request) or None
req_method = request.method
req_path = request.url.path
with store.connect() as conn:
conn.execute(
"""
@@ -379,4 +500,4 @@ def _write_log(
),
)
except Exception:
logger.error("写入操作日志失败 module=%s action=%s", module, action, exc_info=True)
logger.error("写入操作日志到数据库失败 module=%s action=%s", module, action, exc_info=True)

View File

@@ -1,5 +1,5 @@
-- ============================================================================
-- YG Fine-Tune Platform — PostgreSQL 完整初始化脚本(一键建库建表)
-- YG Zhilian — PostgreSQL 完整初始化脚本(一键建库建表)
-- ============================================================================
-- 用途:切换到新的 PG 数据集时,一次性创建平台运行所需的全部数据库对象与
-- 基础种子数据(幂等,可重复执行)。

View File

@@ -15,11 +15,11 @@ logger = get_logger(__name__)
async def run_compute_poller() -> None:
settings = get_settings()
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode})
logger.info("计算轮询已禁用", extra={"compute_mode": settings.compute_mode})
return
interval = max(3, settings.compute_poll_interval_seconds)
logger.info("compute poller started", extra={"interval_seconds": interval})
logger.info("计算轮询已启动", extra={"interval_seconds": interval})
# PlatformStore may run additive schema checks against a remote PostgreSQL
# server on first use. Keep that startup work off the Uvicorn event loop so
# health checks and normal API requests can still respond while the DB is
@@ -41,19 +41,19 @@ async def run_compute_poller() -> None:
now = time.monotonic()
if signature != last_failure_signature or now - last_failure_logged_at >= 300:
logger.warning(
"compute polling reported failures count=%d first_error=%s",
"计算轮询报告失败任务 count=%d first_error=%s",
len(result["failed"]),
signature[:500],
)
last_failure_signature = signature
last_failure_logged_at = now
elif result["synced"]:
logger.debug("compute jobs synchronized", extra={"result": result})
logger.debug("计算任务状态已同步", extra={"result": result})
except asyncio.CancelledError:
logger.info("compute poller stopped")
logger.info("计算轮询已停止")
raise
except Exception as exc: # noqa: BLE001 - keep background polling alive
logger.exception("compute poller failed", extra={"error": str(exc)})
logger.exception("计算轮询执行失败", extra={"error": str(exc)})
if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
store = None
await asyncio.sleep(interval)