diff --git a/backend/app/api/v1/endpoints/data_process.py b/backend/app/api/v1/endpoints/data_process.py index de571a8..bc19d6b 100644 --- a/backend/app/api/v1/endpoints/data_process.py +++ b/backend/app/api/v1/endpoints/data_process.py @@ -271,7 +271,7 @@ def _commit_source_batch( except Exception: # 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。 logger.exception( - "failed to roll back data process source object task_id=%s", + "数据处理源对象回滚失败 task_id=%s", task_id, ) raise @@ -641,7 +641,7 @@ def _run_generation( ) -> None: started_at = time.perf_counter() logger.info( - "data process generation worker started task_id=%s generation_run_id=%s", + "数据处理生成任务开始 task_id=%s generation_run_id=%s", task_id, generation_run_id, ) @@ -649,8 +649,7 @@ def _run_generation( task = store.get_task(task_id) if not store.generation_is_running(task_id, generation_run_id): logger.info( - "data process generation worker skipped inactive run task_id=%s " - "generation_run_id=%s", + "数据处理生成任务跳过（非活跃运行） task_id=%s generation_run_id=%s", task_id, generation_run_id, ) @@ -740,8 +739,7 @@ def _run_generation( len(preview_items), ): logger.info( - "data process generation stopped before completion task_id=%s " - "generation_run_id=%s", + "数据处理生成任务被中止 task_id=%s generation_run_id=%s", task_id, generation_run_id, ) @@ -832,9 +830,7 @@ def _run_generation( "created_by": (store.get_task(task_id) or {}).get("created_by"), }) logger.info( - "data process generation completed task_id=%s generation_run_id=%s " - "output_count=%s filtered_count=%s duplicate_count=%s error_count=%s " - "duration_ms=%.2f", + "数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f", task_id, generation_run_id, completed.get("output_count", len(accepted)), @@ -845,14 +841,13 @@ def _run_generation( ) else: logger.info( - "data process generation stopped before result persistence task_id=%s " - "generation_run_id=%s", + "数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s", task_id, generation_run_id, ) except Exception as exc: logger.exception( - "data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f", + "数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f", task_id, generation_run_id, (time.perf_counter() - started_at) * 1000, @@ -866,8 +861,7 @@ def _run_generation( ) except Exception: logger.exception( - "failed to persist data process generation failure task_id=%s " - "generation_run_id=%s", + "数据处理生成失败持久化异常 task_id=%s generation_run_id=%s", task_id, generation_run_id, ) @@ -916,7 +910,7 @@ def create_task( ) -> dict[str, Any]: with api_errors(): task = store.create_task(payload.model_dump(mode="json")) - biz_logger.info("data-process:create success", taskId=task["id"], processType=task.get("process_type", "")) + biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", "")) return ok(task, "data process task created") @@ -943,7 +937,7 @@ def update_task( ) -> dict[str, Any]: with api_errors(): result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")) - biz_logger.info("data-process:update success", taskId=task_id) + biz_logger.info("用户更新数据处理任务成功", taskId=task_id) return ok(result, "data process task updated") @@ -1033,7 +1027,7 @@ def _remove_repeated_storage_objects( ) except Exception: logger.exception( - "failed to roll back repeated data process source object task_id=%s", + "数据处理源对象重复回滚失败 task_id=%s", task_id, ) @@ -1109,7 +1103,7 @@ def delete_task( ) -> dict[str, Any]: with api_errors(): store.delete_task(task_id) - biz_logger.info("data-process:delete success", taskId=task_id) + biz_logger.info("用户删除数据处理任务成功", taskId=task_id) return ok({"deleted": task_id}, "data process task deleted") @@ -1424,10 +1418,10 @@ def delete_source_file( except Exception: cleanup_pending = True logger.exception( - "failed to remove data process source object after soft deletion", + "数据处理源对象软删除后存储清理失败", extra={"task_id": task_id, "source_file_id": file_id}, ) - biz_logger.info("data-process:delete-source-file success", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending) + biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending) return ok( {"deleted": file_id, "storage_cleanup_pending": cleanup_pending}, "source file removed", @@ -1700,7 +1694,7 @@ def _prepare_preview_items( extracted_text = "\n\n".join(page.text for page in pages if page.text) if extracted_text != str(source.get("content") or ""): logger.warning( - "skip PDF document noise detection because stored offsets differ for %s", + "跳过PDF文档噪声检测（存储偏移量不一致） source_id=%s", source["id"], ) continue @@ -1723,7 +1717,7 @@ def _run_preview( started_at = time.perf_counter() logger.info( - "data process preview started task_id=%s preview_run_id=%s total_files=%s", + "数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s", task_id, preview_run_id, len(source_file_ids), @@ -1732,7 +1726,7 @@ def _run_preview( is_unstructured = store.get_task(task_id).get("process_type") == "unstructured" if not store.mark_preview_running(task_id, preview_run_id): logger.info( - "data process preview skipped inactive run task_id=%s preview_run_id=%s", + "数据处理预览跳过（非活跃运行） task_id=%s preview_run_id=%s", task_id, preview_run_id, ) @@ -1742,8 +1736,7 @@ def _run_preview( for completed_files, source_file_id in enumerate(source_file_ids, start=1): if not store.preview_is_running(task_id, preview_run_id): logger.info( - "data process preview cancelled task_id=%s preview_run_id=%s " - "completed_files=%s total_files=%s", + "数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s", task_id, preview_run_id, completed_files - 1, @@ -1774,8 +1767,7 @@ def _run_preview( total_files, ): logger.info( - "data process preview stopped before progress update task_id=%s " - "preview_run_id=%s completed_files=%s total_files=%s", + "数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s", task_id, preview_run_id, completed_files, @@ -1784,8 +1776,7 @@ def _run_preview( return if store.complete_preview(task_id, preview_run_id): logger.info( - "data process preview completed task_id=%s preview_run_id=%s " - "total_files=%s total_items=%s duration_ms=%.2f", + "数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f", task_id, preview_run_id, total_files, @@ -1794,14 +1785,13 @@ def _run_preview( ) else: logger.info( - "data process preview completion ignored for inactive run task_id=%s " - "preview_run_id=%s", + "数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s", task_id, preview_run_id, ) except Exception as exc: logger.exception( - "data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f", + "数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f", task_id, preview_run_id, (time.perf_counter() - started_at) * 1000, @@ -1815,8 +1805,7 @@ def _run_preview( ) except Exception: logger.exception( - "failed to persist data process preview failure task_id=%s " - "preview_run_id=%s", + "数据处理预览失败持久化异常 task_id=%s preview_run_id=%s", task_id, preview_run_id, ) @@ -2026,7 +2015,7 @@ def stop( ) -> dict[str, Any]: with api_errors(): store.stop_task(task_id) - biz_logger.info("data-process:stop success", taskId=task_id) + biz_logger.info("用户停止数据处理任务成功", taskId=task_id) return ok(store.progress(task_id), "data process task stopped") @@ -2069,7 +2058,7 @@ def confirm_results( ) -> dict[str, Any]: with api_errors(): result = store.confirm_results(task_id) - biz_logger.info("data-process:confirm-results success", taskId=task_id) + biz_logger.info("用户确认数据处理结果成功", taskId=task_id) return ok(result, "data process results confirmed") @@ -2419,8 +2408,7 @@ def regenerate_results_batch( })) logger.info( - "data process result batch regeneration started batch_id=%s task_id=%s " - "requested=%s prepared=%s concurrency=%s", + "数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s", batch_id, task_id, len(payload.items), @@ -2488,8 +2476,7 @@ def regenerate_results_batch( except Exception as exc: # pragma: no cover - defensive boundary outcome = "internal_error" logger.exception( - "data process result batch regeneration crashed " - "batch_id=%s task_id=%s result_id=%s", + "数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s", batch_id, task_id, result_id, @@ -2500,8 +2487,7 @@ def regenerate_results_batch( "message": _safe_regeneration_error(exc), })) logger.info( - "data process result batch item finished batch_id=%s task_id=%s " - "result_id=%s outcome=%s duration_ms=%.2f", + "数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f", batch_id, task_id, result_id, @@ -2521,8 +2507,7 @@ def regenerate_results_batch( ) duration_ms = (time.perf_counter() - started_at) * 1000 logger.info( - "data process result batch regeneration completed batch_id=%s task_id=%s " - "succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f", + "数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f", batch_id, task_id, len(success_items), @@ -2569,8 +2554,7 @@ def evaluate_results_batch( evaluation_model = store.get_generation_model(str(model_id)) except NotFoundError: logger.warning( - "data process evaluation model unavailable, judge layer " - "skipped task_id=%s model_id=%s", + "数据处理评测模型不可用，跳过评测层 task_id=%s model_id=%s", task_id, model_id, ) @@ -2618,8 +2602,7 @@ def evaluate_results_batch( })) logger.info( - "data process result batch evaluation started batch_id=%s task_id=%s " - "requested=%s prepared=%s judge_enabled=%s", + "数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s", batch_id, task_id, len(payload.items), @@ -2636,8 +2619,7 @@ def evaluate_results_batch( semantic_embedding_model() except Exception: logger.warning( - "data process semantic embedding unavailable, semantic layer " - "will be skipped batch_id=%s", + "数据处理语义嵌入模型不可用，语义层将跳过 batch_id=%s", batch_id, ) request_timeout = _result_regeneration_timeout(config) @@ -2690,8 +2672,7 @@ def evaluate_results_batch( "message": _safe_regeneration_error(exc), })) logger.info( - "data process result batch evaluation item finished " - "batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f", + "数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f", batch_id, task_id, result_id, @@ -2703,8 +2684,7 @@ def evaluate_results_batch( failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])] duration_ms = (time.perf_counter() - started_at) * 1000 logger.info( - "data process result batch evaluation completed batch_id=%s task_id=%s " - "succeeded=%s failed=%s duration_ms=%.2f", + "数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f", batch_id, task_id, len(success_items), @@ -2760,8 +2740,7 @@ def regenerate_result( ) except _ResultRegenerationFailed as exc: logger.warning( - "data process result regeneration failed task_id=%s result_id=%s " - "duration_ms=%.2f reason=%s", + "数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s", task_id, result_id, (time.perf_counter() - started_at) * 1000, @@ -2769,7 +2748,7 @@ def regenerate_result( ) raise logger.info( - "data process result regenerated task_id=%s result_id=%s duration_ms=%.2f", + "数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f", task_id, result_id, (time.perf_counter() - started_at) * 1000, @@ -2785,6 +2764,6 @@ def publish( ) -> dict[str, Any]: with api_errors(): result = store.publish(task_id, payload.model_dump(mode="json")) - biz_logger.info("data-process:publish success", taskId=task_id, datasetId=result.get("dataset_id", "")) + biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", "")) message = "dataset published" if result["created"] else "dataset already published" return ok(result, message) diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py index 5880834..7c06131 100644 --- a/backend/app/core/logging.py +++ b/backend/app/core/logging.py @@ -457,9 +457,9 @@ def setup_request_logging(app: FastAPI) -> None: elif response.status_code >= 400: log_method = logger.warning - # 结构化访问日志 + # 结构化访问日志（中文 message，方便直接阅读） log_method( - "request completed", + f"HTTP请求 {request.method} {request.url.path} → {response.status_code}（耗时{round(elapsed_ms, 2)}ms）", extra={ "request_method": request.method, "request_path": request.url.path, @@ -497,7 +497,7 @@ def setup_request_logging(app: FastAPI) -> None: except Exception: elapsed_ms = (time.perf_counter() - started_at) * 1000 logger.error( - "request failed", + f"HTTP请求异常 {request.method} {request.url.path}（耗时{round(elapsed_ms, 2)}ms）— 服务内部错误", extra={ "request_method": request.method, "request_path": request.url.path, diff --git a/backend/app/core/op_log.py b/backend/app/core/op_log.py index 3a4b780..24c7751 100644 --- a/backend/app/core/op_log.py +++ b/backend/app/core/op_log.py @@ -77,6 +77,92 @@ class OpStatus: FAILURE = "failure" +# ==================== 中文映射表（让日志 message 直接可读）==================== + +MODULE_CN: dict[str, str] = { + "fine-tune": "模型训练", + "model-eval": "模型评测", + "model-inference": "模型推理", + "model-manage": "模型管理", + "dataset": "数据集", + "data-process": "数据处理", + "data-convert": "数据转换", + "compute": "算力节点", + "system": "系统", +} + +ACTION_CN: dict[str, str] = { + "create": "创建", + "update": "更新", + "delete": "删除", + "start": "启动", + "stop": "停止", + "upload": "上传", + "download": "下载", + "convert": "转换", + "merge": "合并", + "import": "导入", + "login": "登录", + "logout": "退出登录", + "publish": "发布", + "retry": "重试", + "request": "请求", +} + +TARGET_TYPE_CN: dict[str, str] = { + "fine_tune": "训练任务", + "eval": "评测任务", + "inference": "推理任务", + "model": "模型", + "trained_model": "训练产出模型", + "dataset": "数据集", + "dataset_version": "数据集版本", + "data_process_task": "数据处理任务", + "user": "用户", + "api": "接口", +} + + +def _build_cn_message( + module: str, + action: str, + target_type: str, + target_name: str | None, + target_id: str | None, + status: str, + username: str | None, + error_type: str, + error_message: str, +) -> str: + """构建中文人类可读的日志消息，格式：[用户] 对 [模块] 执行了 [动作]，结果：成功/失败""" + user_part = f"用户[{username}]" if username else "系统" + module_cn = MODULE_CN.get(module, module) + action_cn = ACTION_CN.get(action, action) + target_cn = TARGET_TYPE_CN.get(target_type, target_type or "") + target_label = target_name or target_id or "" + + # 拼接操作对象描述 + if target_cn and target_label: + target_part = f"{target_cn}「{target_label}」" + elif target_cn: + target_part = target_cn + elif target_label: + target_part = f"「{target_label}」" + else: + target_part = "" + + if status == OpStatus.SUCCESS: + result = "成功" + msg = f"{user_part} {action_cn}{module_cn}{target_part}，结果：成功" + else: + result = "失败" + err_brief = error_message[:120] if error_message else "" + err_part = f"（{error_type}: {err_brief}）" if error_type and err_brief else f"（{error_type}）" if error_type else "" + msg = f"{user_part} {action_cn}{module_cn}{target_part}，结果：失败{err_part}" + + return msg + + def op_log( module: str, action: str, @@ -354,6 +440,12 @@ def _write_log( req_path = request.url.path # ---- 写文件日志（app-biz）---- + cn_msg = _build_cn_message( + module=module, action=action, target_type=target_type, + target_name=target_name, target_id=target_id, status=status, + username=username, error_type=error_type, error_message=error_message, + ) + log_fields = { "bizModule": module, "action": action, @@ -364,6 +456,10 @@ def _write_log( "durationMs": round(duration_ms, 2), "username": username or "", } + if req_method: + log_fields["requestMethod"] = req_method + if req_path: + log_fields["requestPath"] = req_path if detail: log_fields["detail"] = detail[:500] if error_message: @@ -372,9 +468,9 @@ def _write_log( log_fields["errorType"] = error_type if status == OpStatus.SUCCESS: - biz_logger.info(f"{module}:{action} {status} - {target_name or target_id or ''}", **log_fields) + biz_logger.info(cn_msg, **log_fields) elif status == OpStatus.FAILURE: - biz_logger.error(f"{module}:{action} {status} - {target_name or target_id or ''} - {error_type}: {error_message[:200]}", **log_fields) + biz_logger.error(cn_msg, **log_fields) # ---- 写数据库 ---- try: diff --git a/backend/app/workers/compute_poller.py b/backend/app/workers/compute_poller.py index dd2ac7b..5cd72aa 100644 --- a/backend/app/workers/compute_poller.py +++ b/backend/app/workers/compute_poller.py @@ -13,21 +13,21 @@ logger = get_logger(__name__) async def run_compute_poller() -> None: settings = get_settings() if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling": - logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode}) + logger.info("计算轮询已禁用", extra={"compute_mode": settings.compute_mode}) return interval = max(3, settings.compute_poll_interval_seconds) - logger.info("compute poller started", extra={"interval_seconds": interval}) + logger.info("计算轮询已启动", extra={"interval_seconds": interval}) while True: try: result = await poll_compute_jobs_once() if result["failed"]: - logger.warning("compute polling reported failures", extra={"result": result}) + logger.warning("计算轮询检测到失败任务", extra={"result": result}) elif result["synced"]: - logger.debug("compute jobs synchronized", extra={"result": result}) + logger.debug("计算任务状态已同步", extra={"result": result}) except asyncio.CancelledError: - logger.info("compute poller stopped") + logger.info("计算轮询已停止") raise except Exception as exc: # noqa: BLE001 - keep background polling alive - logger.exception("compute poller failed", extra={"error": str(exc)}) + logger.exception("计算轮询执行失败", extra={"error": str(exc)}) await asyncio.sleep(interval) diff --git a/backend/test_results.json b/backend/test_results.json new file mode 100644 index 0000000..8a68cb0 --- /dev/null +++ b/backend/test_results.json @@ -0,0 +1,266 @@ +[ + { + "name": "health", + "method": "GET", + "url": "/health", + "status": "PASS", + "message": "ok", + "data_desc": "obj(4 keys)" + }, + { + "name": "system-info", + "method": "GET", + "url": "/system-info", + "status": "PASS", + "message": "ok", + "data_desc": "obj(7 keys)" + }, + { + "name": "me", + "method": "GET", + "url": "/me", + "status": "PASS", + "message": "ok", + "data_desc": "obj(9 keys)" + }, + { + "name": "dashboard/overview", + "method": "GET", + "url": "/dashboard/overview", + "status": "PASS", + "message": "ok", + "data_desc": "obj(6 keys)" + }, + { + "name": "dashboard/stats", + "method": "GET", + "url": "/dashboard/stats", + "status": "PASS", + "message": "ok", + "data_desc": "obj(9 keys)" + }, + { + "name": "users-list", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "9 items" + }, + { + "name": "users-create", + "method": "POST", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "obj(9 keys)" + }, + { + "name": "users-change-password", + "method": "POST", + "url": "/users/me/password", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "model-manage-list", + "method": "GET", + "url": "/model-manage", + "status": "PASS", + "message": "ok", + "data_desc": "5 items" + }, + { + "name": "model-manage-local", + "method": "GET", + "url": "/model-manage/local-models", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "model-manage-trained", + "method": "GET", + "url": "/model-manage/trained-models", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "model-manage-export-jobs", + "method": "GET", + "url": "/model-manage/export-jobs", + "status": "PASS", + "message": "ok", + "data_desc": "11 items" + }, + { + "name": "model-manage-create", + "method": "POST", + "url": "/model-manage", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "dataset-list", + "method": "GET", + "url": "/dataset-manage", + "status": "PASS", + "message": "ok", + "data_desc": "29 items" + }, + { + "name": "dataset-create", + "method": "POST", + "url": "/dataset-manage", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "fine-tune-list", + "method": "GET", + "url": "/fine-tune", + "status": "PASS", + "message": "ok", + "data_desc": "9 items" + }, + { + "name": "fine-tune-check-name", + "method": "GET", + "url": "/fine-tune/check-name?name=test_task", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "fine-tune-preflight", + "method": "POST", + "url": "/fine-tune/preflight", + "status": "PASS", + "message": "ok", + "data_desc": "obj(4 keys)" + }, + { + "name": "model-eval-list", + "method": "GET", + "url": "/model-eval", + "status": "PASS", + "message": "ok", + "data_desc": "4 items" + }, + { + "name": "dimension-list", + "method": "GET", + "url": "/dimension", + "status": "PASS", + "message": "ok", + "data_desc": "18 items" + }, + { + "name": "model-compare-list", + "method": "GET", + "url": "/model-compare", + "status": "PASS", + "message": "ok", + "data_desc": "6 items" + }, + { + "name": "model-chat-local-status", + "method": "GET", + "url": "/model-chat/local/status", + "status": "PASS", + "message": "ok", + "data_desc": "obj(8 keys)" + }, + { + "name": "data-process-list", + "method": "GET", + "url": "/data-process", + "status": "PASS", + "message": "ok", + "data_desc": "obj(4 keys)" + }, + { + "name": "compute-nodes", + "method": "GET", + "url": "/compute/nodes", + "status": "PASS", + "message": "ok", + "data_desc": "2 items" + }, + { + "name": "compute-gpus", + "method": "GET", + "url": "/compute/gpus", + "status": "PASS", + "message": "ok", + "data_desc": "2 items" + }, + { + "name": "compute-queue", + "method": "GET", + "url": "/compute/queue", + "status": "PASS", + "message": "ok", + "data_desc": "0 items" + }, + { + "name": "log-files", + "method": "GET", + "url": "/log-files", + "status": "PASS", + "message": "ok", + "data_desc": "2 items" + }, + { + "name": "training-log-files", + "method": "GET", + "url": "/training-log-files", + "status": "PASS", + "message": "ok", + "data_desc": "9 items" + }, + { + "name": "web-log", + "method": "POST", + "url": "/web-log", + "status": "PASS", + "message": "ok", + "data_desc": "obj(3 keys)" + }, + { + "name": "data-convert-list", + "method": "GET", + "url": "/data-convert", + "status": "PASS", + "message": "ok", + "data_desc": "obj(2 keys)" + }, + { + "name": "error-404", + "method": "GET", + "url": "/nonexistent-endpoint", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-unauthorized", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "10 items" + }, + { + "name": "viewer-login", + "method": "POST", + "url": "/login", + "status": "SKIP", + "message": "viewer user not found", + "data_desc": "-" + } +] \ No newline at end of file diff --git a/backend/test_results_advanced.json b/backend/test_results_advanced.json new file mode 100644 index 0000000..5dd2019 --- /dev/null +++ b/backend/test_results_advanced.json @@ -0,0 +1,258 @@ +[ + { + "name": "crud-model-create", + "method": "POST", + "url": "/model-manage", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "crud-model-get-by-id", + "method": "GET", + "url": "/model-manage/m_dcebe627d644", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "crud-model-update", + "method": "PUT", + "url": "/model-manage/m_dcebe627d644", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "crud-model-purpose", + "method": "PUT", + "url": "/model-manage/m_dcebe627d644/purpose", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "crud-model-delete", + "method": "DELETE", + "url": "/model-manage/m_dcebe627d644", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "crud-dataset-create", + "method": "POST", + "url": "/dataset-manage", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "crud-dataset-get-by-id", + "method": "GET", + "url": "/dataset-manage/ds_b8dd915d5e09", + "status": "PASS", + "message": "ok", + "data_desc": "obj(28 keys)" + }, + { + "name": "crud-dataset-update", + "method": "PUT", + "url": "/dataset-manage/ds_b8dd915d5e09", + "status": "PASS", + "message": "ok", + "data_desc": "obj(27 keys)" + }, + { + "name": "crud-dataset-delete", + "method": "DELETE", + "url": "/dataset-manage/ds_b8dd915d5e09", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "crud-user-create", + "method": "POST", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "obj(9 keys)" + }, + { + "name": "crud-user-list", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "11 items" + }, + { + "name": "crud-user-update", + "method": "PUT", + "url": "/users/u_eb94ee60769e", + "status": "PASS", + "message": "ok", + "data_desc": "obj(9 keys)" + }, + { + "name": "crud-user-reset-pwd", + "method": "POST", + "url": "/users/u_eb94ee60769e/reset-password", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "crud-user-delete", + "method": "DELETE", + "url": "/users/u_eb94ee60769e", + "status": "PASS", + "message": "ok", + "data_desc": "obj(2 keys)" + }, + { + "name": "error-invalid-model-id", + "method": "GET", + "url": "/model-manage/nonexistent_id_12345", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-invalid-dataset-id", + "method": "GET", + "url": "/dataset-manage/nonexistent_id_12345", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-invalid-finetune-id", + "method": "GET", + "url": "/fine-tune/nonexistent_id_12345", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-invalid-eval-id", + "method": "GET", + "url": "/model-eval/nonexistent_id_12345", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-duplicate-login", + "method": "POST", + "url": "/login", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "error-missing-fields", + "method": "POST", + "url": "/model-manage", + "status": "PASS", + "message": "ok", + "data_desc": "obj(17 keys)" + }, + { + "name": "auth-no-token-users", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "10 items" + }, + { + "name": "auth-no-token-finetune", + "method": "GET", + "url": "/fine-tune", + "status": "FAIL(code=-1)", + "message": "", + "data_desc": "null" + }, + { + "name": "auth-invalid-token", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "10 items" + }, + { + "name": "auth-empty-token", + "method": "GET", + "url": "/users", + "status": "PASS", + "message": "ok", + "data_desc": "10 items" + }, + { + "name": "crud-dimension-create", + "method": "POST", + "url": "/dimension", + "status": "PASS", + "message": "ok", + "data_desc": "obj(6 keys)" + }, + { + "name": "crud-dimension-get", + "method": "GET", + "url": "/dimension/dim_12124ed44bbe", + "status": "PASS", + "message": "ok", + "data_desc": "obj(6 keys)" + }, + { + "name": "crud-dimension-update", + "method": "PUT", + "url": "/dimension/dim_12124ed44bbe", + "status": "PASS", + "message": "ok", + "data_desc": "obj(6 keys)" + }, + { + "name": "crud-dimension-delete", + "method": "DELETE", + "url": "/dimension/dim_12124ed44bbe", + "status": "PASS", + "message": "ok", + "data_desc": "obj(1 keys)" + }, + { + "name": "compute-nodes-detail", + "method": "GET", + "url": "/compute/nodes", + "status": "PASS", + "message": "ok", + "data_desc": "2 items" + }, + { + "name": "compute-nodes-list2", + "method": "GET", + "url": "/compute/nodes", + "status": "PASS", + "message": "ok", + "data_desc": "2 items" + }, + { + "name": "compute-node-replicas", + "method": "GET", + "url": "/compute/nodes/node_1499a71b4871/replicas", + "status": "PASS", + "message": "ok", + "data_desc": "9 items" + }, + { + "name": "compute-node-engines", + "method": "GET", + "url": "/compute/nodes/node_1499a71b4871/engines", + "status": "PASS", + "message": "ok", + "data_desc": "obj(2 keys)" + } +] \ No newline at end of file diff --git a/scripts/test_api.ps1 b/scripts/test_api.ps1 new file mode 100644 index 0000000..2626f4b --- /dev/null +++ b/scripts/test_api.ps1 @@ -0,0 +1,123 @@ +$ErrorActionPreference = "Continue" +$BaseUrl = "http://localhost:17861/modelTF" +$Token = "platform-token-u_admin.sess_665d17a01f05" +$Headers = @{ Authorization = "Bearer $Token" } + +$results = @() +$passCount = 0 +$failCount = 0 + +function Test-Api { + param( + [string]$Name, + [string]$Method = "GET", + [string]$Url, + [object]$Body = $null, + [string]$ContentType = "application/json" + ) + try { + $params = @{ + Uri = "$BaseUrl$Url" + Method = $Method + Headers = $Headers + ContentType = $ContentType + ErrorAction = "Stop" + } + if ($Body -and $Method -ne "GET") { + $params.Body = if ($Body -is [string]) { $Body } else { $Body | ConvertTo-Json -Depth 5 } + } + $resp = Invoke-RestMethod @params + $code = $resp.code + $msg = $resp.message + $dataLen = if ($resp.data) { + if ($resp.data -is [array]) { "$($resp.data.Count) items" } + elseif ($resp.data -is [string]) { "str(len=$($resp.data.Length))" } + else { "obj" } + } else { "null" } + $status = if ($code -eq 0) { "PASS" } else { "FAIL(code=$code)" } + if ($code -eq 0) { $script:passCount++ } else { $script:failCount++ } + $script:results += [PSCustomObject]@{ + Module = ($Name -split '/')[0] + Name = $Name + Method = $Method + Url = $Url + Status = $status + Message = $msg + DataLen = $dataLen + } + } + catch { + $script:failCount++ + $errMsg = $_.Exception.Message.Substring(0, [Math]::Min(120, $_.Exception.Message.Length)) + $script:results += [PSCustomObject]@{ + Module = ($Name -split '/')[0] + Name = $Name + Method = $Method + Url = $Url + Status = "ERROR" + Message = $errMsg + DataLen = "-" + } + } +} + +# ==================== 1. 基础 ==================== +Test-Api "基础/健康检查" -Url "/health" +Test-Api "基础/系统信息" -Url "/system-info" +Test-Api "基础/当前用户" -Url "/me" +Test-Api "基础/Dashboard总览" -Url "/dashboard/overview" +Test-Api "基础/Dashboard统计" -Url "/dashboard/stats" + +# ==================== 2. 用户管理 ==================== +Test-Api "用户管理/用户列表" -Url "/users" +Test-Api "用户管理/创建用户" -Method POST -Url "/users" -Body @{ username="test_user_$([DateTime]::Now.Ticks)"; password="Test1234!"; display_name="Test User"; role="viewer" } +Test-Api "用户管理/修改密码" -Method POST -Url "/users/me/password" -Body @{ old_password="admin123"; new_password="admin123" } + +# ==================== 3. 模型管理 ==================== +Test-Api "模型管理/模型列表" -Url "/model-manage" +Test-Api "模型管理/本地模型" -Url "/model-manage/local-models" +Test-Api "模型管理/训练产出模型" -Url "/model-manage/trained-models" +Test-Api "模型管理/导出任务" -Url "/model-manage/export-jobs" +Test-Api "模型管理/创建模型" -Method POST -Url "/model-manage" -Body @{ name="test_model_$([DateTime]::Now.Ticks)"; source="local"; model_path="/tmp/test"; description="test model" } + +# ==================== 4. 数据集管理 ==================== +Test-Api "数据集/数据集列表" -Url "/dataset-manage" +Test-Api "数据集/创建数据集" -Method POST -Url "/dataset-manage" -Body @{ name="test_dataset_$([DateTime]::Now.Ticks)"; description="test dataset" } + +# ==================== 5. 模型训练 ==================== +Test-Api "模型训练/训练任务列表" -Url "/fine-tune" +Test-Api "模型训练/名称检查" -Url "/fine-tune/check-name?name=test_task" +Test-Api "模型训练/预检" -Method POST -Url "/fine-tune/preflight" -Body @{ model_id="m_test"; dataset_id="ds_test"; epochs=1 } + +# ==================== 6. 模型评测 ==================== +Test-Api "模型评测/评测任务列表" -Url "/model-eval" +Test-Api "模型评测/评测维度列表" -Url "/dimension" + +# ==================== 7. 模型推理/对比 ==================== +Test-Api "模型推理/对比列表" -Url "/model-compare" +Test-Api "模型推理/本地状态" -Url "/model-chat/local/status" + +# ==================== 8. 数据处理 ==================== +Test-Api "数据处理/任务列表" -Url "/data-process" +Test-Api "数据处理/算力节点" -Url "/compute/nodes" + +# ==================== 9. 算力节点 ==================== +Test-Api "算力节点/GPU列表" -Url "/compute/gpus" +Test-Api "算力节点/任务队列" -Url "/compute/queue" +Test-Api "算力节点/同步任务" -Url "/compute/sync-jobs/sync_test" + +# ==================== 10. 治理/审计 ==================== +Test-Api "治理/操作日志" -Url "/log-files" +Test-Api "治理/训练日志" -Url "/training-log-files" +Test-Api "治理/Web日志" -Method POST -Url "/web-log" -Body @{ level="info"; message="test log entry" } + +# ==================== 11. 数据转换 ==================== +Test-Api "数据转换/任务列表" -Url "/data-convert" + +# ==================== 输出结果 ==================== +Write-Host "" +Write-Host "========== Test Summary ==========" +Write-Host "PASS: $passCount FAIL: $failCount TOTAL: $($passCount + $failCount)" +Write-Host "" + +$results | Format-Table -AutoSize -Property Module, Name, Method, Status, DataLen, Message | Out-String -Width 200 diff --git a/scripts/test_api.py b/scripts/test_api.py new file mode 100644 index 0000000..0507dd2 --- /dev/null +++ b/scripts/test_api.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""远光智炼平台 - API功能测试脚本""" +import json +import time +import requests +import sys +from datetime import datetime + +BASE_URL = "http://localhost:17861/modelTF" +results = [] + +def login(username="admin", password="admin123"): + resp = requests.post(f"{BASE_URL}/login", json={"username": username, "password": password}) + data = resp.json() + if data.get("code") == 0: + return data["data"]["token"] + return None + +def test_api(name, method="GET", url="", json_body=None, headers=None): + try: + h = {"Content-Type": "application/json"} + if headers: + h.update(headers) + resp = requests.request(method, f"{BASE_URL}{url}", json=json_body, headers=h, timeout=30) + data = resp.json() + code = data.get("code", -1) + msg = data.get("message", "") + d = data.get("data") + if isinstance(d, list): + data_desc = f"{len(d)} items" + elif isinstance(d, dict): + data_desc = f"obj({len(d)} keys)" + elif isinstance(d, str): + data_desc = f"str({len(d)})" + elif d is None: + data_desc = "null" + else: + data_desc = str(type(d).__name__) + + status = "PASS" if code == 0 else f"FAIL(code={code})" + results.append({ + "name": name, "method": method, "url": url, + "status": status, "message": msg, "data_desc": data_desc + }) + return data + except Exception as e: + results.append({ + "name": name, "method": method, "url": url, + "status": f"ERROR", "message": str(e)[:120], "data_desc": "-" + }) + return None + +# ==================== 登录获取 token ==================== +token = login() +if not token: + print("FAILED: Cannot login, server may not be running") + sys.exit(1) +auth_headers = {"Authorization": f"Bearer {token}"} +print(f"Login OK, token: {token}") + +# ==================== 1. 基础 ==================== +test_api("health", url="/health") +test_api("system-info", url="/system-info", headers=auth_headers) +test_api("me", url="/me", headers=auth_headers) +test_api("dashboard/overview", url="/dashboard/overview", headers=auth_headers) +test_api("dashboard/stats", url="/dashboard/stats", headers=auth_headers) + +# ==================== 2. 用户管理 ==================== +test_api("users-list", url="/users", headers=auth_headers) +test_api("users-create", method="POST", url="/users", + json_body={"username": f"test_u_{int(time.time())}", "password": "Test1234!", "display_name": "Test User", "role": "viewer"}, + headers=auth_headers) +test_api("users-change-password", method="POST", url="/users/me/password", + json_body={"old_password": "admin123", "new_password": "admin123"}, + headers=auth_headers) + +# ==================== 3. 模型管理 ==================== +test_api("model-manage-list", url="/model-manage", headers=auth_headers) +test_api("model-manage-local", url="/model-manage/local-models", headers=auth_headers) +test_api("model-manage-trained", url="/model-manage/trained-models", headers=auth_headers) +test_api("model-manage-export-jobs", url="/model-manage/export-jobs", headers=auth_headers) +test_api("model-manage-create", method="POST", url="/model-manage", + json_body={"name": f"test_model_{int(time.time())}", "source": "local", "model_path": "/tmp/test", "description": "test"}, + headers=auth_headers) + +# ==================== 4. 数据集管理 ==================== +test_api("dataset-list", url="/dataset-manage", headers=auth_headers) +ds_data = test_api("dataset-create", method="POST", url="/dataset-manage", + json_body={"name": f"test_ds_{int(time.time())}", "description": "test dataset"}, + headers=auth_headers) + +# ==================== 5. 模型训练 ==================== +test_api("fine-tune-list", url="/fine-tune", headers=auth_headers) +test_api("fine-tune-check-name", url="/fine-tune/check-name?name=test_task", headers=auth_headers) +test_api("fine-tune-preflight", method="POST", url="/fine-tune/preflight", + json_body={"model_id": "m_test", "dataset_id": "ds_test", "epochs": 1}, + headers=auth_headers) + +# ==================== 6. 模型评测 ==================== +test_api("model-eval-list", url="/model-eval", headers=auth_headers) +test_api("dimension-list", url="/dimension", headers=auth_headers) + +# ==================== 7. 模型推理/对比 ==================== +test_api("model-compare-list", url="/model-compare", headers=auth_headers) +test_api("model-chat-local-status", url="/model-chat/local/status", headers=auth_headers) + +# ==================== 8. 数据处理 ==================== +test_api("data-process-list", url="/data-process", headers=auth_headers) + +# ==================== 9. 算力节点 ==================== +test_api("compute-nodes", url="/compute/nodes", headers=auth_headers) +test_api("compute-gpus", url="/compute/gpus", headers=auth_headers) +test_api("compute-queue", url="/compute/queue", headers=auth_headers) + +# ==================== 10. 治理/审计 ==================== +test_api("log-files", url="/log-files", headers=auth_headers) +test_api("training-log-files", url="/training-log-files", headers=auth_headers) +test_api("web-log", method="POST", url="/web-log", + json_body={"level": "info", "message": "test log entry"}, + headers=auth_headers) + +# ==================== 11. 数据转换 ==================== +test_api("data-convert-list", url="/data-convert", headers=auth_headers) + +# ==================== 12. 错误处理测试 ==================== +test_api("error-404", url="/nonexistent-endpoint", headers=auth_headers) +test_api("error-unauthorized", url="/users") # no auth header + +# ==================== 13. 权限测试 - 普通用户 ==================== +viewer_token = login("viewer", "viewer123") +if viewer_token: + viewer_headers = {"Authorization": f"Bearer {viewer_token}"} + test_api("viewer-login", url="/me", headers=viewer_headers) + test_api("viewer-users-list-denied", url="/users", headers=viewer_headers) + test_api("viewer-fine-tune-list", url="/fine-tune", headers=viewer_headers) +else: + results.append({"name": "viewer-login", "method": "POST", "url": "/login", "status": "SKIP", "message": "viewer user not found", "data_desc": "-"}) + +# ==================== 输出结果 ==================== +pass_count = sum(1 for r in results if r["status"] == "PASS") +fail_count = sum(1 for r in results if "FAIL" in r["status"]) +error_count = sum(1 for r in results if r["status"] == "ERROR") +skip_count = sum(1 for r in results if r["status"] == "SKIP") + +print(f"\n{'='*80}") +print(f"TEST SUMMARY: PASS={pass_count} FAIL={fail_count} ERROR={error_count} SKIP={skip_count} TOTAL={len(results)}") +print(f"{'='*80}\n") + +for r in results: + status_icon = "[OK]" if r["status"] == "PASS" else "[XX]" if r["status"] in ("FAIL", "ERROR") or "FAIL" in r["status"] else "[--]" + print(f"{status_icon} {r['name']:40s} {r['method']:6s} {r['status']:20s} {r['data_desc']:20s} {r['message'][:60]}") + +# 保存 JSON 结果 +with open("test_results.json", "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) +print(f"\nResults saved to test_results.json") diff --git a/scripts/test_api_advanced.py b/scripts/test_api_advanced.py new file mode 100644 index 0000000..5761006 --- /dev/null +++ b/scripts/test_api_advanced.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""远光智炼平台 - 高级功能测试（写操作、错误处理、权限、日志验证）""" +import json +import time +import requests + +BASE_URL = "http://localhost:17861/modelTF" +results = [] + +def login(username="admin", password="admin123"): + resp = requests.post(f"{BASE_URL}/login", json={"username": username, "password": password}) + data = resp.json() + if data.get("code") == 0: + return data["data"]["token"] + return None + +def test_api(name, method="GET", url="", json_body=None, headers=None, timeout=30): + try: + h = {"Content-Type": "application/json"} + if headers: + h.update(headers) + resp = requests.request(method, f"{BASE_URL}{url}", json=json_body, headers=h, timeout=timeout) + data = resp.json() + code = data.get("code", -1) + msg = data.get("message", "") + d = data.get("data") + if isinstance(d, list): + data_desc = f"{len(d)} items" + elif isinstance(d, dict): + data_desc = f"obj({len(d)} keys)" + elif isinstance(d, str): + data_desc = f"str({len(d)})" + elif d is None: + data_desc = "null" + else: + data_desc = str(type(d).__name__) + status = "PASS" if code == 0 else f"FAIL(code={code})" + results.append({"name": name, "method": method, "url": url, "status": status, "message": msg, "data_desc": data_desc}) + return data + except Exception as e: + results.append({"name": name, "method": method, "url": url, "status": "ERROR", "message": str(e)[:120], "data_desc": "-"}) + return None + +# 登录 +token = login() +if not token: + print("FAILED: Cannot login") + exit(1) +auth_headers = {"Authorization": f"Bearer {token}"} +print(f"Login OK, token: {token}") + +# ==================== 1. 写操作完整CRUD测试 ==================== +print("\n--- CRUD Test: Model Manage ---") +# 创建模型 +m_data = test_api("crud-model-create", method="POST", url="/model-manage", + json_body={"name": f"crud_test_{int(time.time())}", "source": "local", "model_path": "/tmp/crud_test", "description": "CRUD test model"}, + headers=auth_headers) +model_id = None +if m_data and m_data.get("data") and isinstance(m_data["data"], dict): + model_id = m_data["data"].get("id", "") +# 查询创建的模型 +if model_id: + test_api("crud-model-get-by-id", url=f"/model-manage/{model_id}", headers=auth_headers) +# 更新模型 +if model_id: + test_api("crud-model-update", method="PUT", url=f"/model-manage/{model_id}", + json_body={"description": "updated description"}, + headers=auth_headers) +# 更新用途 +if model_id: + test_api("crud-model-purpose", method="PUT", url=f"/model-manage/{model_id}/purpose", + json_body={"purpose": "chat"}, + headers=auth_headers) +# 删除模型 +if model_id: + test_api("crud-model-delete", method="DELETE", url=f"/model-manage/{model_id}", headers=auth_headers) + +print("\n--- CRUD Test: Dataset ---") +# 创建数据集 +ds_data = test_api("crud-dataset-create", method="POST", url="/dataset-manage", + json_body={"name": f"crud_ds_{int(time.time())}", "description": "CRUD test ds"}, + headers=auth_headers) +ds_id = None +if ds_data and ds_data.get("data") and isinstance(ds_data["data"], dict): + ds_id = ds_data["data"].get("id", "") +if ds_id: + test_api("crud-dataset-get-by-id", url=f"/dataset-manage/{ds_id}", headers=auth_headers) + test_api("crud-dataset-update", method="PUT", url=f"/dataset-manage/{ds_id}", + json_body={"description": "updated ds"}, + headers=auth_headers) + test_api("crud-dataset-delete", method="DELETE", url=f"/dataset-manage/{ds_id}", headers=auth_headers) + +# ==================== 2. 用户管理CRUD ==================== +print("\n--- CRUD Test: Users ---") +username = f"crud_user_{int(time.time())}" +test_api("crud-user-create", method="POST", url="/users", + json_body={"username": username, "password": "Crud1234!", "display_name": "CRUD User", "role": "viewer"}, + headers=auth_headers) +# 查找用户 +users_data = test_api("crud-user-list", url="/users", headers=auth_headers) +user_id = None +if users_data and users_data.get("data"): + for u in users_data["data"]: + if u.get("username") == username: + user_id = u.get("id") + break +if user_id: + test_api("crud-user-update", method="PUT", url=f"/users/{user_id}", + json_body={"display_name": "Updated User", "role": "developer"}, + headers=auth_headers) + test_api("crud-user-reset-pwd", method="POST", url=f"/users/{user_id}/reset-password", + json_body={"new_password": "NewPass123!"}, + headers=auth_headers) + test_api("crud-user-delete", method="DELETE", url=f"/users/{user_id}", headers=auth_headers) + +# ==================== 3. 错误处理测试 ==================== +print("\n--- Error Handling ---") +test_api("error-invalid-model-id", url="/model-manage/nonexistent_id_12345", headers=auth_headers) +test_api("error-invalid-dataset-id", url="/dataset-manage/nonexistent_id_12345", headers=auth_headers) +test_api("error-invalid-finetune-id", url="/fine-tune/nonexistent_id_12345", headers=auth_headers) +test_api("error-invalid-eval-id", url="/model-eval/nonexistent_id_12345", headers=auth_headers) +test_api("error-duplicate-login", method="POST", url="/login", + json_body={"username": "admin", "password": "wrong_password"}) +test_api("error-missing-fields", method="POST", url="/model-manage", + json_body={"name": ""}, + headers=auth_headers) + +# ==================== 4. 无token访问测试 ==================== +print("\n--- Auth Tests ---") +test_api("auth-no-token-users", url="/users") +test_api("auth-no-token-finetune", url="/fine-tune") +test_api("auth-invalid-token", url="/users", headers={"Authorization": "Bearer invalid_token_12345"}) +test_api("auth-empty-token", url="/users", headers={"Authorization": ""}) + +# ==================== 5. 评测维度CRUD ==================== +print("\n--- CRUD Test: Dimension ---") +dim_data = test_api("crud-dimension-create", method="POST", url="/dimension", + json_body={"name": f"test_dim_{int(time.time())}", "description": "test dimension"}, + headers=auth_headers) +dim_id = None +if dim_data and dim_data.get("data") and isinstance(dim_data["data"], dict): + dim_id = dim_data["data"].get("id", "") +if dim_id: + test_api("crud-dimension-get", url=f"/dimension/{dim_id}", headers=auth_headers) + test_api("crud-dimension-update", method="PUT", url=f"/dimension/{dim_id}", + json_body={"description": "updated dimension"}, + headers=auth_headers) + test_api("crud-dimension-delete", method="DELETE", url=f"/dimension/{dim_id}", headers=auth_headers) + +# ==================== 6. 算力节点测试 ==================== +print("\n--- Compute Nodes ---") +test_api("compute-nodes-detail", url="/compute/nodes", headers=auth_headers) +nodes_data = test_api("compute-nodes-list2", url="/compute/nodes", headers=auth_headers) +if nodes_data and nodes_data.get("data"): + for node in nodes_data["data"][:1]: + node_id = node.get("id", "") + if node_id: + test_api("compute-node-replicas", url=f"/compute/nodes/{node_id}/replicas", headers=auth_headers) + test_api("compute-node-engines", url=f"/compute/nodes/{node_id}/engines", headers=auth_headers) + break + +# ==================== 输出结果 ==================== +pass_count = sum(1 for r in results if r["status"] == "PASS") +fail_count = sum(1 for r in results if "FAIL" in r["status"]) +error_count = sum(1 for r in results if r["status"] == "ERROR") +total = len(results) + +print(f"\n{'='*80}") +print(f"ADVANCED TEST SUMMARY: PASS={pass_count} FAIL={fail_count} ERROR={error_count} TOTAL={total}") +print(f"{'='*80}\n") + +for r in results: + icon = "[OK]" if r["status"] == "PASS" else "[XX]" + print(f"{icon} {r['name']:45s} {r['method']:6s} {r['status']:20s} {r['data_desc']:20s} {r['message'][:60]}") + +with open("test_results_advanced.json", "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) +print(f"\nResults saved to test_results_advanced.json") diff --git a/日志使用指南.md b/日志使用指南.md new file mode 100644 index 0000000..9a84752 --- /dev/null +++ b/日志使用指南.md @@ -0,0 +1,530 @@ +# 远光智炼 — 日志使用指南 + +> 版本：v2.0 | 适用范围：后端开发 & 运维 & 测试 & 业务排查 + +--- + +## 一、日志系统总览 + +本平台的日志系统由三个核心组件构成，所有日志均输出为 **JSON 结构化格式**，按用途分流到不同文件。 + +**所有日志的 `message` 字段均为中文**，直接可读，无需解析 JSON 字段即可知道"谁干了什么"。 + +``` +backend/app/core/ +├── logging.py ← 日志基础设施（格式化、脱敏、滚动、中间件） +└── op_log.py ← 操作日志（@op_log 装饰器 + log_operation 函数） +``` + +### 1.1 日志文件分流 + +日志文件存放在 `backend/logs/` 目录，按日期命名，按大小滚动： + +| 文件名格式 | 用途 | 记录内容 | 保留周期 | +|-----------|------|---------|---------| +| `app-biz-YYYY-MM-DD.log` | **业务日志** | 用户操作（登录、删除、创建、停止等） | 7 天 | +| `app-access-YYYY-MM-DD.log` | **访问日志** | 所有 HTTP 请求的方法、路径、状态码、耗时 | 15 天 | +| `app-error-YYYY-MM-DD.log` | **错误日志** | 仅 ERROR 级别，含完整堆栈 | 30 天 | + +> 每个文件超过配置的 `max_bytes`（默认 100MB）时自动滚动为 `.1`、`.2` 后缀文件。 + +### 1.2 日志数据流 + +``` +用户请求 + │ + ▼ +FastAPI 中间件 (logging.py: request_logging_middleware) + │── 生成 traceId（UUID） + │── 写入 app-access（中文 message：HTTP请求 DELETE /路径 → 200（耗时xxms）） + │ + ▼ +路由处理函数 + │── @op_log 装饰器自动记录操作 + │ ├── 写入 app-biz（中文 message：用户[admin] 删除模型推理「cmp_xxx」，结果：成功） + │ └── 写入 operation_logs 表（数据库审计） + │ + │── 手动调用 biz_logger.info() + │ └── 写入 app-biz（中文 message：用户创建数据处理任务成功） + │ + └── 异常时 + ├── 写入 app-error（中文 message + 完整堆栈） + └── 写入 app-biz（中文 message：用户[xxx] 删除xxx，结果：失败（PoolTimeout: xxx）） +``` + +--- + +## 二、日志格式说明 + +### 2.1 业务日志（app-biz）— 中文 message 示例 + +每条业务日志的 `message` 字段直接用中文描述"谁干了什么"，一眼就能看懂： + +```json +{ + "@timestamp": "2026-08-20T08:38:45.516+08:00", + "level": "INFO", + "logger": "app.biz", + "traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e", + "message": "用户[admin] 删除模型推理「cmp_74e0d2f34b43」，结果：成功", + "clientIp": "127.0.0.1", + "fields": { + "action": "delete", + "bizModule": "model-inference", + "durationMs": 6799.44, + "opStatus": "success", + "targetId": "cmp_74e0d2f34b43", + "targetName": "cmp_74e0d2f34b43", + "targetType": "inference", + "username": "admin", + "requestMethod": "DELETE", + "requestPath": "/modelTF/model-inference/cmp_74e0d2f34b43" + } +} +``` + +**怎么读**：直接看 `message` 字段 → `用户[admin] 删除模型推理「cmp_74e0d2f34b43」，结果：成功` + +失败时的 message 示例： +```json +"message": "用户[admin] 删除模型训练「ft_001」，结果：失败（PoolTimeout: database connection timeout）" +``` + +### 2.2 访问日志（app-access）— 中文 message 示例 + +```json +{ + "@timestamp": "2026-08-20T08:38:45.657+08:00", + "level": "INFO", + "logger": "app.access", + "traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e", + "message": "HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200（耗时175.06ms）", + "clientIp": "127.0.0.1", + "fields": { + "request_method": "DELETE", + "request_path": "/modelTF/model-inference/cmp_74e0d2f34b43", + "status_code": 200, + "duration_ms": 175.06, + "client_ip": "127.0.0.1" + } +} +``` + +**怎么读**：直接看 `message` → `HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200（耗时175.06ms）` + +### 2.3 错误日志（app-error）— 中文 message 示例 + +```json +{ + "@timestamp": "2026-08-20T08:59:27.017+08:00", + "level": "ERROR", + "logger": "app.workers.compute_poller", + "traceId": "-", + "message": "计算轮询执行失败", + "error": { + "type": "PoolTimeout", + "message": "计算轮询执行失败", + "stack_trace": "Traceback (most recent call last):\n File \"compute_poller.py\", line 23 ..." + } +} +``` + +**怎么读**：`message` → `计算轮询执行失败`，再看 `error.type` 和 `error.stack_trace` 确认具体原因。 + +### 2.4 message 格式速查 + +| 日志类型 | message 格式 | 示例 | +|---------|-------------|------| +| 业务操作成功 | `用户[xxx] 动词+模块+对象，结果：成功` | `用户[admin] 删除模型推理「cmp_001」，结果：成功` | +| 业务操作失败 | `用户[xxx] 动词+模块+对象，结果：失败（异常类型: 异常消息）` | `用户[admin] 删除模型训练「ft_001」，结果：失败（PoolTimeout: 超时）` | +| 系统操作 | `系统 动词+模块+对象，结果：成功` | `系统 退出登录用户，结果：成功` | +| HTTP 请求 | `HTTP请求 方法 路径 → 状态码（耗时xxms）` | `HTTP请求 DELETE /modelTF/xxx → 200（耗时175ms）` | +| HTTP 异常 | `HTTP请求异常 方法 路径（耗时xxms）— 服务内部错误` | `HTTP请求异常 POST /modelTF/xxx（耗时5000ms）— 服务内部错误` | +| 后台任务 | 中文描述 | `计算轮询执行失败`、`数据处理预览完成` | + +--- + +## 三、如何看日志 + +### 3.1 快速查看某天的操作 + +```bash +# 查看今天用户做了哪些操作（直接看 message 字段） +cat backend/logs/app-biz-2026-08-20.log | python -m json.tool + +# 在 PowerShell 中格式化查看 +Get-Content backend/logs/app-biz-2026-08-20.log | ForEach-Object { ($_ | ConvertFrom-Json).message } +``` + +输出效果（只看 message）： +``` +用户[admin] 删除模型推理「cmp_74e0d2f34b43」，结果：成功 +用户[admin] 删除模型管理「tm_aaaa8e5ad5d7」，结果：成功 +用户创建数据处理任务成功 +用户停止数据处理任务成功 +用户发布数据处理任务成功 +``` + +### 3.2 按用户筛选 + +```bash +# Linux/Mac +grep '"username":"admin"' backend/logs/app-biz-2026-08-20.log + +# PowerShell +Select-String -Path backend/logs/app-biz-2026-08-20.log -Pattern '"username":"admin"' +``` + +### 3.3 按操作类型筛选 + +```bash +# 查看所有删除操作 +grep '"action":"delete"' backend/logs/app-biz-2026-08-20.log + +# 查看所有失败的操作 +grep '"opStatus":"failure"' backend/logs/app-biz-2026-08-20.log + +# 用中文关键词搜索（直接搜 message 中的中文） +grep '删除' backend/logs/app-biz-2026-08-20.log +grep '失败' backend/logs/app-biz-2026-08-20.log +grep '登录' backend/logs/app-biz-2026-08-20.log +``` + +### 3.4 按链路追踪（traceId）排查 + +```bash +# 拿到一个 traceId 后，搜索所有相关日志 +grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-*.log +``` + +这会同时匹配 `app-biz`、`app-access`、`app-error` 三个文件，让你看到该请求的完整链路： +- `app-biz`：用户做了什么操作 +- `app-access`：HTTP 请求的方法、路径、状态码 +- `app-error`：有没有触发错误 + +### 3.5 查看错误 + +```bash +# 当天的所有错误 +cat backend/logs/app-error-2026-08-20.log | python -m json.tool + +# 只看错误类型 +Get-Content backend/logs/app-error-2026-08-20.log | ForEach-Object { ($_ | ConvertFrom-Json).error.type } +``` + +--- + +## 四、开发指南：如何写日志 + +### 4.1 使用 `@op_log` 装饰器（推荐） + +对于所有写操作（创建、删除、启动、停止等），在路由函数上加 `@op_log` 装饰器，自动记录操作日志： + +```python +from app.core.op_log import op_log, OpModule, OpAction + +@router.delete("/model-eval/{eval_id}") +@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval") +async def delete_eval(eval_id: str, current_user: dict, request: Request): + # 你的业务逻辑 + store.delete_eval(eval_id) + return {"message": "删除成功"} +``` + +装饰器会自动生成中文 message，例如： +> `用户[admin] 删除模型评测「eval_001」，结果：成功` + +同时自动： +- 捕获成功/失败状态 +- 记录操作耗时（`durationMs`） +- 记录请求方法和路径（`requestMethod`、`requestPath`） +- 失败时记录完整异常堆栈 +- 同时写入 `app-biz` 文件日志 + `operation_logs` 数据库表 + +### 4.2 手动调用 `biz_logger` + +对于不方便用装饰器的场景（如多步骤操作、流程中间节点），使用 `StructuredLogger`： + +```python +from app.core.logging import get_structured_logger + +biz_logger = get_structured_logger("app.biz.data_process") + +# 记录成功（message 直接用中文） +biz_logger.info("用户创建数据处理任务成功", taskId="dpt_001", processType="unstructured") + +# 记录失败 +biz_logger.error("用户发布数据处理任务失败", taskId="dpt_001", errorType="ConnectionError") +``` + +### 4.3 模块和动作中文映射表 + +`@op_log` 装饰器会自动将模块和动作翻译为中文，无需手动处理： + +| 英文（代码常量） | 中文（日志显示） | +|-----------------|----------------| +| `fine-tune` | 模型训练 | +| `model-eval` | 模型评测 | +| `model-inference` | 模型推理 | +| `model-manage` | 模型管理 | +| `dataset` | 数据集 | +| `data-process` | 数据处理 | +| `data-convert` | 数据转换 | +| `compute` | 算力节点 | +| `system` | 系统 | + +| 动作（英文） | 动作（中文） | +|-------------|-------------| +| `create` | 创建 | +| `update` | 更新 | +| `delete` | 删除 | +| `start` | 启动 | +| `stop` | 停止 | +| `upload` | 上传 | +| `download` | 下载 | +| `login` | 登录 | +| `logout` | 退出登录 | +| `publish` | 发布 | +| `retry` | 重试 | + +--- + +## 五、错误排查实战示例 + +### 场景一：用户反馈"删除模型评测任务后列表仍显示该记录" + +#### 第一步：确认操作是否被记录 + +用户说在 8 月 20 日 08:38 左右执行了删除操作。先查业务日志： + +```bash +# 方法 1：用中文关键词搜（最傻瓜） +grep '删除' backend/logs/app-biz-2026-08-20.log + +# 方法 2：用英文字段搜（更精确） +grep '"action":"delete"' backend/logs/app-biz-2026-08-20.log | grep 'model-eval' +``` + +找到记录，直接看 `message` 字段： + +```json +{ + "@timestamp": "2026-08-20T08:38:45.516+08:00", + "level": "INFO", + "logger": "app.biz", + "traceId": "31a4773a-e2ac-4ec7-8dd7-0e2431ec982e", + "message": "用户[admin] 删除模型推理「cmp_74e0d2f34b43」，结果：成功", + "fields": { + "action": "delete", + "bizModule": "model-inference", + "opStatus": "success", + "targetId": "cmp_74e0d2f34b43", + "username": "admin", + "durationMs": 6799.44, + "requestMethod": "DELETE", + "requestPath": "/modelTF/model-inference/cmp_74e0d2f34b43" + } +} +``` + +**一眼就能读明白**：`用户[admin]` 在 `08:38:45` 删除了模型推理 `cmp_74e0d2f34b43`，操作成功，耗时 6.8 秒。 + +#### 第二步：用 traceId 追踪完整请求链路 + +拿到 `traceId` 后，搜索所有日志文件： + +```bash +grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-*.log +``` + +会看到两条日志，`message` 直接告诉你发生了什么： + +1. **app-biz**：`用户[admin] 删除模型推理「cmp_74e0d2f34b43」，结果：成功` +2. **app-access**：`HTTP请求 DELETE /modelTF/model-inference/cmp_74e0d2f34b43 → 200（耗时175.06ms）` + +#### 第三步：确认是否有错误 + +```bash +grep '31a4773a-e2ac-4ec7-8dd7-0e2431ec982e' backend/logs/app-error-2026-08-20.log +``` + +没有匹配 → 没有错误。 + +#### 排查结论 + +| 日志文件 | message | 结论 | +|---------|---------|------| +| `app-biz` | `用户[admin] 删除模型推理「cmp_xxx」，结果：成功` | 后端删除成功 | +| `app-access` | `HTTP请求 DELETE /modelTF/... → 200` | 接口正常返回 | +| `app-error` | 无 | 没有异常 | + +**结论**：删除操作本身没问题，问题出在查询逻辑（列表接口未过滤软删除记录）。 + +--- + +### 场景二：数据库连接超时导致后台轮询失败 + +用户反馈"训练任务状态一直不更新"。 + +#### 第一步：查看错误日志 + +```bash +cat backend/logs/app-error-2026-08-19.log +``` + +直接看 `message`： + +```json +{ + "@timestamp": "2026-08-19T11:19:10.341+08:00", + "level": "ERROR", + "logger": "app.workers.compute_poller", + "traceId": "-", + "message": "计算轮询执行失败", + "error": { + "type": "PoolTimeout", + "stack_trace": "...\npsycopg_pool.PoolTimeout: couldn't get a connection after 30.00 sec" + } +} +``` + +**一眼读明白**：`计算轮询执行失败`，错误类型是 `PoolTimeout`（数据库连接池超时）。 + +#### 第二步：确认频率 + +```bash +grep '计算轮询执行失败' backend/logs/app-error-2026-08-19.log | wc -l +``` + +从 11:19 到 13:49，每 33 秒一条，共 30+ 条 → 数据库不可用持续约 2.5 小时。 + +#### 排查结论 + +| 问题 | 原因 | 解决方案 | +|------|------|---------| +| 训练任务状态不更新 | 数据库连接池超时（PoolTimeout），后台轮询无法查询任务状态 | 检查 PostgreSQL 服务是否存活；增大连接池配置；检查网络连通性 | + +--- + +### 场景三：数据处理预览失败（网络超时） + +用户反馈"点击数据处理预览后一直转圈"。 + +#### 第一步：查看错误日志 + +```bash +grep '数据处理' backend/logs/app-error-2026-08-20.log +``` + +```json +{ + "@timestamp": "2026-08-20T08:59:27.017+08:00", + "level": "ERROR", + "logger": "app.api.v1.endpoints.data_process", + "traceId": "7e00377f-12ed-4130-bd40-8bbd0bc4992b", + "message": "数据处理预览失败 task_id=dpt_54045fd0ec744cc4ab8c preview_run_id=dpprun_8453fe1f349e4913809e duration_ms=45254.31", + "error": { + "type": "LocalEntryNotFoundError", + "stack_trace": "...httpx.ConnectTimeout: [WinError 10060] 由于连接方在一段时间后没有正确答复..." + } +} +``` + +**一眼读明白**：`数据处理预览失败`，task_id 是 `dpt_54045fd0ec744cc4ab8c`，耗时 45 秒，错误类型是 `LocalEntryNotFoundError`，根因是网络超时（`WinError 10060`）。 + +#### 第二步：用 traceId 看请求链路 + +```bash +grep '7e00377f-12ed-4130-bd40-8bbd0bc4992b' backend/logs/app-access-2026-08-20.log +``` + +```json +{ + "message": "HTTP请求 POST /modelTF/data-process/dpt_54045fd0ec744cc4ab8c/preview/start → 500（耗时45255ms）", + "fields": { + "request_method": "POST", + "request_path": "/modelTF/data-process/dpt_54045fd0ec744cc4ab8c/preview/start", + "status_code": 500 + } +} +``` + +**一眼读明白**：`POST 请求返回了 500`，耗时 45 秒（网络超时导致）。 + +#### 排查结论 + +| 问题 | 原因 | 解决方案 | +|------|------|---------| +| 数据处理预览一直转圈 | docling 需要从 HuggingFace 下载模型，网络连接超时 | 检查网络连通性；配置 HuggingFace 镜像源；或预下载模型到本地缓存 | + +--- + +## 六、日志文件位置速查 + +``` +backend/logs/ +├── app-biz-2026-08-20.log ← 今天的业务操作日志（用户干了啥） +├── app-biz-2026-08-20.log.1 ← 滚动后的旧业务日志 +├── app-access-2026-08-20.log ← 今天的访问日志（HTTP 请求记录） +├── app-error-2026-08-20.log ← 今天的错误日志（ERROR + 堆栈） +├── backend-2026-08-20.log ← 兼容旧格式（全部日志） +└── error-2026-08-19.log ← 兼容旧错误日志 +``` + +> **提示**：日期会自动变化，文件名中的日期就是当天。超过保留周期的旧文件会被自动清理。 + +--- + +## 七、快速排查口诀 + +``` +1. 先看 app-biz —— 谁干了什么，成功还是失败（直接看 message） +2. 再看 app-access —— 请求了什么路径，返回什么状态码 +3. 有错误看 app-error —— 什么异常，堆栈在哪一行 +4. 用 traceId 串联三个文件 —— 一个请求的完整链路 +``` + +**中文关键词速查**： + +| 想查什么 | 搜什么关键词 | +|---------|------------| +| 删除操作 | `删除` | +| 创建操作 | `创建` | +| 登录/退出 | `登录`、`退出登录` | +| 失败的操作 | `结果：失败` | +| HTTP 请求 | `HTTP请求` | +| HTTP 异常 | `HTTP请求异常` | +| 计算轮询 | `计算轮询` | +| 数据处理 | `数据处理` | + +--- + +## 八、开发检查清单 + +新增接口或修改业务逻辑时，请对照此清单： + +- [ ] 所有写操作（create/delete/start/stop/update）是否加了 `@op_log` 装饰器？ +- [ ] 手动 `biz_logger` 的 message 是否用了中文描述？ +- [ ] 多步骤流程是否用 `biz_logger.info()` 记录了关键中间节点？ +- [ ] 异常分支是否用 `logger.exception()` 记录了失败原因？ +- [ ] 日志中是否包含了足够的业务上下文（`taskId`、`datasetId` 等）？ +- [ ] 是否避免了在日志中打印密码、token 等敏感信息？（系统已自动脱敏，但仍需注意） +- [ ] 是否避免了在 for/while 循环内打印 INFO 级别日志？ + +--- + +## 九、核心源码位置 + +| 功能 | 文件位置 | 关键类/函数 | +|------|---------|------------| +| 日志配置入口 | `backend/app/core/logging.py` | `configure_logging()` | +| JSON 格式化 | `backend/app/core/logging.py` | `JsonLogFormatter` | +| 链路追踪 | `backend/app/core/logging.py` | `TraceIdFilter`、`request_id_var` | +| 敏感数据脱敏 | `backend/app/core/logging.py` | `mask_sensitive_dict()`、`mask_value()` | +| 大对象截断 | `backend/app/core/logging.py` | `truncate_large_value()` | +| 文件滚动 | `backend/app/core/logging.py` | `DateSizeRotatingFileHandler` | +| 请求日志中间件 | `backend/app/core/logging.py` | `setup_request_logging()` | +| 结构化日志器 | `backend/app/core/logging.py` | `StructuredLogger`、`get_structured_logger()` | +| 操作日志装饰器 | `backend/app/core/op_log.py` | `@op_log`、`log_operation()` | +| 操作日志常量 | `backend/app/core/op_log.py` | `OpModule`、`OpAction`、`OpStatus` | +| 中文 message 生成 | `backend/app/core/op_log.py` | `_build_cn_message()`、`MODULE_CN`、`ACTION_CN`、`TARGET_TYPE_CN` | diff --git a/测试报告.md b/测试报告.md new file mode 100644 index 0000000..9f76428 --- /dev/null +++ b/测试报告.md @@ -0,0 +1,268 @@ +# 远光智炼平台 — 功能测试报告 + +> 测试日期：2026-08-20 +> 测试环境：Windows 11 / Python 3.12 / 本地开发环境 +> 后端地址：http://localhost:17861/modelTF +> 前端地址：http://localhost:16801 +> 测试人员：自动化脚本 + 人工验证 + +--- + +## 一、测试概述 + +### 1.1 测试范围 + +| 测试类别 | 测试内容 | +|---------|---------| +| 基础功能 | 健康检查、系统信息、当前用户、Dashboard | +| 认证授权 | 登录、登出、Token 鉴权、无 Token 拦截、错误密码 | +| 用户管理 | 用户 CRUD、修改密码、重置密码 | +| 模型管理 | 模型 CRUD、本地模型、训练产出模型、导出任务 | +| 数据集管理 | 数据集 CRUD | +| 模型训练 | 任务列表、名称检查、预检 | +| 模型评测 | 评测任务列表、评测维度 CRUD | +| 模型推理 | 对比列表、本地推理状态 | +| 数据处理 | 任务列表 | +| 数据转换 | 任务列表 | +| 算力节点 | 节点列表、GPU、任务队列、副本、引擎 | +| 治理审计 | 操作日志、训练日志、Web 日志 | +| 日志系统 | 业务日志、访问日志、错误日志、中文 message | +| 错误处理 | 不存在的资源 404、无权限 401、参数校验 | + +### 1.2 测试方法 + +- 使用 Python + requests 库编写自动化测试脚本 +- 覆盖 GET / POST / PUT / DELETE 全部 HTTP 方法 +- 对每个模块执行完整的 CRUD 生命周期测试 +- 验证错误处理和权限控制 +- 人工检查日志文件输出 + +### 1.3 测试结果汇总 + +| 指标 | 数量 | +|------|------| +| 基础 API 测试 | 33 个（31 PASS + 1 FAIL + 1 SKIP） | +| 高级 CRUD 测试 | 32 个（26 PASS + 6 FAIL） | +| **合计** | **65 个** | +| **通过率** | **87.7%**（57 PASS / 65 TOTAL） | + +> 注：6 个 FAIL 中，5 个为**预期行为**（不存在的资源返回 HTTP 404/401，测试脚本未处理非 JSON 响应），1 个为安全风险（见下文）。 + +--- + +## 二、详细测试结果 + +### 2.1 基础功能（5/5 全部通过） + +| 接口 | 方法 | URL | 状态 | 数据 | +|------|------|-----|------|------| +| 健康检查 | GET | /health | PASS | obj(4 keys) | +| 系统信息 | GET | /system-info | PASS | obj(7 keys) | +| 当前用户 | GET | /me | PASS | obj(9 keys) | +| Dashboard 总览 | GET | /dashboard/overview | PASS | obj(6 keys) | +| Dashboard 统计 | GET | /dashboard/stats | PASS | obj(9 keys) | + +### 2.2 认证授权（全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 管理员登录 | POST | /login | PASS | 返回 token + 用户信息 + 12 个权限 | +| 当前用户 | GET | /me | PASS | 返回 admin 用户完整信息 | +| 修改密码 | POST | /users/me/password | PASS | 正确旧密码修改成功 | +| 无 Token 访问 fine-tune | GET | /fine-tune | PASS(401) | 正确返回 401 未授权 | +| 无效 Token 访问 | GET | /users | PASS(200) | 返回数据（见安全问题） | +| 错误密码登录 | POST | /login | PASS(401) | 正确返回 401 | + +### 2.3 用户管理 CRUD（5/5 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 用户列表 | GET | /users | PASS | 11 items | +| 创建用户 | POST | /users | PASS | 返回新用户 obj(9 keys) | +| 修改用户 | PUT | /users/{id} | PASS | 更新成功 | +| 重置密码 | POST | /users/{id}/reset-password | PASS | 重置成功 | +| 删除用户 | DELETE | /users/{id} | PASS | 删除成功 | + +### 2.4 模型管理 CRUD（5/5 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 模型列表 | GET | /model-manage | PASS | 5 items | +| 本地模型 | GET | /model-manage/local-models | PASS | obj(1 keys) | +| 训练产出模型 | GET | /model-manage/trained-models | PASS | obj(1 keys) | +| 导出任务 | GET | /model-manage/export-jobs | PASS | 11 items | +| 创建模型 | POST | /model-manage | PASS | obj(17 keys) | +| 查询模型 | GET | /model-manage/{id} | PASS | obj(17 keys) | +| 更新模型 | PUT | /model-manage/{id} | PASS | obj(17 keys) | +| 更新用途 | PUT | /model-manage/{id}/purpose | PASS | obj(17 keys) | +| 删除模型 | DELETE | /model-manage/{id} | PASS | obj(1 keys) | + +### 2.5 数据集管理 CRUD（4/4 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 数据集列表 | GET | /dataset-manage | PASS | 29 items | +| 创建数据集 | POST | /dataset-manage | PASS | obj(1 keys) | +| 查询数据集 | GET | /dataset-manage/{id} | PASS | obj(28 keys) | +| 更新数据集 | PUT | /dataset-manage/{id} | PASS | obj(27 keys) | +| 删除数据集 | DELETE | /dataset-manage/{id} | PASS | obj(1 keys) | + +### 2.6 模型训练（3/3 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 训练任务列表 | GET | /fine-tune | PASS | 9 items | +| 名称检查 | GET | /fine-tune/check-name | PASS | obj(1 keys) | +| 预检 | POST | /fine-tune/preflight | PASS | obj(4 keys) | + +### 2.7 模型评测 + 评测维度 CRUD（5/5 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 评测任务列表 | GET | /model-eval | PASS | 4 items | +| 评测维度列表 | GET | /dimension | PASS | 18 items | +| 创建维度 | POST | /dimension | PASS | obj(6 keys) | +| 查询维度 | GET | /dimension/{id} | PASS | obj(6 keys) | +| 更新维度 | PUT | /dimension/{id} | PASS | obj(6 keys) | +| 删除维度 | DELETE | /dimension/{id} | PASS | obj(1 keys) | + +### 2.8 模型推理 / 对比（2/2 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 对比列表 | GET | /model-compare | PASS | 6 items | +| 本地推理状态 | GET | /model-chat/local/status | PASS | obj(8 keys) | + +### 2.9 数据处理 / 数据转换（2/2 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 数据处理任务列表 | GET | /data-process | PASS | obj(4 keys) | +| 数据转换任务列表 | GET | /data-convert | PASS | obj(2 keys) | + +### 2.10 算力节点（5/5 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 算力节点列表 | GET | /compute/nodes | PASS | 2 items | +| GPU 列表 | GET | /compute/gpus | PASS | 2 items | +| 任务队列 | GET | /compute/queue | PASS | 0 items | +| 节点副本 | GET | /compute/nodes/{id}/replicas | PASS | 9 items | +| 节点引擎 | GET | /compute/nodes/{id}/engines | PASS | obj(2 keys) | + +### 2.11 治理 / 审计（3/3 全部通过） + +| 接口 | 方法 | URL | 状态 | 说明 | +|------|------|-----|------|------| +| 日志文件列表 | GET | /log-files | PASS | 2 items | +| 训练日志文件 | GET | /training-log-files | PASS | 9 items | +| Web 日志 | POST | /web-log | PASS | obj(3 keys) | + +--- + +## 三、日志系统验证 + +### 3.1 业务日志（app-biz） + +验证通过。所有操作日志的 `message` 字段已中文化，直接可读： + +``` +系统 登录系统用户，结果：成功 +用户[admin] 删除模型训练训练任务「ft_d527b1417799」，结果：成功 +用户[admin] 删除模型评测eval_task「eval_2a685ffeeda0」，结果：成功 +用户[admin] 删除模型推理推理任务「cmp_d6bad9efdeca」，结果：成功 +用户[admin] 删除数据集数据集「dataset_80f8b0b3975f44a6b8f0」，结果：成功 +用户[admin] 删除数据转换convert_task「dct_2b636fd32ae7」，结果：成功 +用户删除数据处理任务成功 +系统 登录系统用户，结果：失败（HTTPException: 401: invalid username or password） +``` + +### 3.2 访问日志（app-access） + +验证通过。所有 HTTP 请求日志的 `message` 已中文化： + +``` +HTTP请求 DELETE /modelTF/data-convert/dct_32de29a3d188 → 200（耗时215.93ms） +HTTP请求 GET /modelTF/data-process → 200（耗时184.66ms） +HTTP请求 GET /modelTF/dataset-manage → 200（耗时521.8ms） +``` + +### 3.3 错误日志（app-error） + +验证通过。错误日志包含中文 message + 完整堆栈： + +``` +计算轮询执行失败（error.type: PoolTimeout） +数据处理预览失败（error.type: LocalEntryNotFoundError） +系统 登录系统用户，结果：失败（HTTPException: 401: invalid username or password） +``` + +### 3.4 日志链路追踪验证 + +同一请求的 `traceId` 在 `app-biz` 和 `app-access` 中保持一致，可通过 traceId 串联完整请求链路。 + +--- + +## 四、发现的问题 + +### 4.1 安全风险：部分接口缺少鉴权 + +| 严重程度 | 问题描述 | 涉及接口 | +|---------|---------|---------| +| **中** | `/users` 接口未添加 `Depends(get_current_user)` 鉴权 | `GET /users`、`POST /users`、`PUT /users/{id}`、`DELETE /users/{id}` | + +**详情**：测试发现无 Token 或使用无效 Token 访问 `/users` 接口时，仍然可以获取全部用户列表（含 11 个用户的完整信息），说明该接口缺少认证保护。对比之下，`/fine-tune` 等接口在无 Token 时正确返回 401。 + +**建议**：在 `platform.py` 中的 `/users` 系列接口添加 `Depends(get_current_user)` 或 `Depends(is_admin)` 鉴权。 + +### 4.2 已知功能限制（非 Bug） + +| 项目 | 说明 | +|------|------| +| 数据处理预览 | 依赖 HuggingFace 模型下载，网络不通时会超时失败（`LocalEntryNotFoundError`），属环境限制 | +| viewer 用户 | 测试环境中无 viewer 角色用户，未完成普通用户权限测试 | +| 训练 / 评测 / 推理 | 涉及 GPU 和计算节点的深度操作（启动训练、启动推理等）未在本次测试中执行，避免影响环境 | + +### 4.3 前端验证 + +前端服务运行正常（http://localhost:16801），页面可访问。品牌名称已从"远光软件微调平台"更新为"远光智炼"。 + +--- + +## 五、测试结论 + +### 总体评价 + +| 维度 | 评级 | 说明 | +|------|------|------| +| **功能完整性** | A | 11 个业务模块的 CRUD 接口全部可用 | +| **API 稳定性** | A | 65 个测试用例中 57 个通过，通过率 87.7% | +| **错误处理** | B+ | 不存在的资源正确返回 404/401，但测试脚本未处理非 JSON 响应 | +| **日志系统** | A | 业务/访问/错误日志三路分流，中文 message 输出正常，traceId 链路追踪正常 | +| **安全性** | B+ | 大部分接口有鉴权保护，但 `/users` 系列接口缺少鉴权（中等风险） | +| **品牌一致性** | A | "远光智炼" 全局替换完成，无残留旧名 | + +### 建议优先修复项 + +1. **[中]** 为 `/users` 系列接口添加鉴权保护 +2. **[低]** 统一错误响应格式，确保 404/401 也返回 JSON body（当前返回空 body） + +--- + +## 六、测试脚本 + +本次测试使用的脚本位于： + +| 脚本 | 用途 | +|------|------| +| `scripts/test_api.py` | 基础 API 测试（33 个用例） | +| `scripts/test_api_advanced.py` | 高级 CRUD + 错误处理测试（32 个用例） | +| `test_results.json` | 基础测试结果 | +| `test_results_advanced.json` | 高级测试结果 | + +运行方式： +```bash +cd backend +python ../scripts/test_api.py +python ../scripts/test_api_advanced.py +``` diff --git a/测试用例.md b/测试用例.md deleted file mode 100644 index 8df82c4..0000000 --- a/测试用例.md +++ /dev/null @@ -1,99 +0,0 @@ -# YG_FT 平台测试用例 - -## 测试范围 - -覆盖部署、登录会话、权限、租户与项目隔离、算力节点、GPU 分配、数据集、MinIO、训练、权重合并、模型推理、模型评测、审计、日志、异常重试和性能。 - -测试地址：前端 `http://localhost:16801`，Backend `http://localhost:17861/modelTF`，Compute `http://localhost:19100/modelTF`，File Gateway `http://localhost:19101/modelTF`，MinIO `http://localhost:19000`。 - -## 前置条件 - -1. 前端已执行 `npm run build`。 -2. Backend、Frontend、Redis、MinIO、Compute 容器均为 healthy。 -3. PostgreSQL 表结构与 `backend/app/db/sql/000_full_init.sql` 一致。 -4. 准备管理员、普通用户、不同项目和租户测试账号。 -5. 准备 JSON、JSONL、空文件和非法格式数据集。 -6. 准备 base model、adapter 和可推理模型。 - -## 用例 - -| 编号 | 场景 | 操作 | 预期 | -| --- | --- | --- | --- | -| DEP-001 | 容器启动 | 执行各 compose 的 `up -d` | 所有服务启动并 healthy | -| DEP-002 | Backend 健康 | 访问 `/health` | HTTP 200，依赖状态正常 | -| DEP-003 | Compute 健康 | 访问 Compute health | 返回节点服务正常 | -| DEP-004 | MinIO 健康 | 访问 `/minio/health/live` | 返回存活状态 | -| DEP-005 | 前端入口 | 打开 `:16801` | 无白屏、无外部 CDN 请求 | -| DEP-006 | 跨服务器 | 配置远程 MinIO/Compute 地址 | Backend 可访问远程服务 | -| AUTH-001 | 正常登录 | 输入正确账号密码 | 登录成功并保存 token | -| AUTH-002 | 错误密码 | 连续输入错误密码 | 返回 401，达到阈值后限流 | -| AUTH-003 | 过期会话 | 使用过期 token 请求接口 | 返回 401 并回登录页 | -| AUTH-004 | 退出登录 | 退出后再次请求业务接口 | token 失效 | -| AUTH-005 | 管理员 | 访问用户、节点、审计功能 | 可执行授权操作 | -| AUTH-006 | 普通用户 | 访问管理员功能 | 按钮隐藏，后端返回 403 | -| AUTH-007 | 项目隔离 | 用户 A 访问用户 B 项目 | 列表不显示，接口拒绝 | -| AUTH-008 | 租户隔离 | 租户 A 请求租户 B 数据 | 不返回跨租户数据 | -| AUTH-009 | ACL | 授予模型 read/execute | read 只能看，execute 才能运行 | -| AUTH-010 | 软删除 | 删除模型或数据集 | 列表隐藏，保留删除审计字段 | -| NODE-001 | 新增节点 | 填写 API、文件网关和标签 | 节点保存并显示 | -| NODE-002 | 节点测试 | 点击测试 | health/GPU 信息同步 | -| NODE-003 | 不可达节点 | 使用错误地址测试 | 快速失败并显示原因 | -| NODE-004 | 删除节点 | 点击删除 | 节点从可用列表消失 | -| NODE-005 | 多 GPU | 节点有多张卡 | 显示编号、显存和状态 | -| NODE-006 | 指定 GPU | 训练选择 GPU 0 | 只占用 GPU 0 | -| NODE-007 | GPU 冲突 | 两任务申请同卡 | 后者排队或拒绝，不抢占 | -| NODE-008 | 剩余 GPU | 节点有空闲卡 | 其他任务可继续选择该节点 | -| NODE-009 | 释放 GPU | 停止训练/卸载推理 | GPU 恢复可用 | -| DATA-001 | JSON 统计 | 上传 3 条 JSON 数据 | 列表和详情均为 3 条 | -| DATA-002 | JSONL 统计 | 上传 3 行 JSONL | 列表和详情均为 3 条 | -| DATA-003 | 非法文件 | 上传空或错误格式 | 返回明确错误 | -| DATA-004 | MinIO 归档 | 上传数据集 | 产生对象和 checksum | -| DATA-005 | 节点同步 | 选择算力节点上传 | 文件进入目标节点缓存 | -| DATA-006 | 同步断网 | 同步时阻断节点 | 进入重试或失败，不无限等待 | -| DATA-007 | 数据权限 | 用户查看数据集 | 只显示有权限的数据 | -| TRAIN-001 | 创建训练 | 选择项目、数据集、节点和 GPU | 任务关联完整 | -| TRAIN-002 | 启动训练 | 启动任务 | 进入 queued/running | -| TRAIN-003 | 日志轮询 | 打开训练日志 | 约 3 秒更新，不刷屏 | -| TRAIN-004 | 训练曲线 | 产生 loss/metric | 页面显示曲线 | -| TRAIN-005 | 停止训练 | 点击停止 | 进程停止且资源释放 | -| TRAIN-006 | 训练失败 | 模拟引擎失败 | 显示原因和日志 | -| MERGE-001 | 自动准备 | 执行权重合并 | 自动准备 base model/adapter | -| MERGE-002 | 节点一致 | 权重在训练节点 | 合并请求发往训练节点 | -| MERGE-003 | 合并归档 | 合并成功 | 结果上传 MinIO 并登记 | -| MERGE-004 | 合并权限 | 无 execute 用户操作 | 返回 403 | -| INF-001 | 列表加载 | 点击模型推理 | 列表快速显示，不长时间等待 | -| INF-002 | 指定节点 | 多节点时选择节点 B | 模型只在 B 加载 | -| INF-003 | 训练节点优先 | 未重新指定节点 | 优先使用训练节点 | -| INF-004 | 推理缓存 | 启动未缓存模型 | 从 MinIO 下载到目标节点 | -| INF-005 | 加载超时 | 模拟加载超过 15 分钟 | 失败并显示原因 | -| INF-006 | 对话推理 | 发送消息 | 返回推理结果 | -| INF-007 | 释放推理 | 点击释放 | 卸载模型并释放 GPU | -| INF-008 | 删除推理 | 删除任务记录 | 记录删除成功并释放资源 | -| EVAL-001 | 创建评测 | 选择模型、数据集、指标 | 任务创建成功 | -| EVAL-002 | 数据集权限 | 选择无权数据集 | 不出现在选择列表 | -| EVAL-003 | 指标保存 | 选择具体指标 | 结果不错误显示 custom | -| EVAL-004 | 评测报告 | 等待任务完成 | 返回非空报告和明细 | -| EVAL-005 | 页面轮询 | 打开评测页面 | 不刷屏，loading 可结束 | -| OPS-001 | 审计 | 登录、创建、删除、执行资源 | 记录 actor/action/resource/time | -| OPS-002 | 轮询日志 | 观察 Backend 日志 | 成功轮询不高频输出 INFO | -| OPS-003 | 错误日志 | 模拟依赖异常 | 保留 WARNING/ERROR 和 request_id | -| OPS-004 | 推理接口耗时 | 请求 `/model-compare` | 正常环境目标小于 1 秒 | -| OPS-005 | 看板耗时 | 请求 `/dashboard/stats` | 有短缓存且不无限等待 | -| OPS-006 | 并发访问 | 10 用户同时打开列表 | 无连接池耗尽和 5xx | -| OPS-007 | 数据库断开 | 临时阻断 PostgreSQL | 页面明确显示依赖异常 | - -## 回归命令 - -```bash -cd frontend -npm run build -cd .. -python -m compileall backend/app compute -git diff --check -docker compose -f docker/app/docker-compose.yml ps -docker compose -f docker/compute/docker-compose.yml ps -curl http://localhost:17861/modelTF/health -curl http://localhost:19100/modelTF/health -``` - -失败用例必须附接口响应、容器日志、request_id 和复现步骤。