YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增后端日志，且更改了平台名字

Browse Source
This commit is contained in:
wangjiming
2026-08-19 17:26:49 +08:00
parent c3e96ae61b
commit 78680a5c5c
18 changed files with 865 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/app/api/v1/endpoints/data_process.py
View File

@@ -35,6 +35,7 @@ from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.core.logging import get_structured_logger
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -103,6 +104,7 @@ from app.schemas.data_process import (
router = APIRouter(prefix="/data-process")
logger = logging.getLogger(__name__)
biz_logger = get_structured_logger("app.biz.data_process")
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
MAX_SOURCE_FILE_COUNT = 20
MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
@@ -854,6 +856,7 @@ def create_task(
) -> dict[str, Any]:
with api_errors():
task = store.create_task(payload.model_dump(mode="json"))
biz_logger.info("data-process:create success", taskId=task["id"], processType=task.get("process_type", ""))
return ok(task, "data process task created")
@@ -879,10 +882,9 @@ def update_task(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(
store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")),
"data process task updated",
)
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
biz_logger.info("data-process:update success", taskId=task_id)
return ok(result, "data process task updated")
@router.put("/{task_id}/workflow-step")
@@ -1047,6 +1049,7 @@ def delete_task(
) -> dict[str, Any]:
with api_errors():
store.delete_task(task_id)
biz_logger.info("data-process:delete success", taskId=task_id)
return ok({"deleted": task_id}, "data process task deleted")
@@ -1359,13 +1362,12 @@ def delete_source_file(
expected_source_file_id=file_id,
)
except Exception:
# 数据库软删除已经提交，不能再向客户端返回可重试的失败；保留逻辑引用，
# 由后续存储清理任务重试物理删除。
cleanup_pending = True
logger.exception(
"failed to remove data process source object after soft deletion",
extra={"task_id": task_id, "source_file_id": file_id},
)
biz_logger.info("data-process:delete-source-file success", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
return ok(
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
"source file removed",
@@ -1964,6 +1966,7 @@ def stop(
) -> dict[str, Any]:
with api_errors():
store.stop_task(task_id)
biz_logger.info("data-process:stop success", taskId=task_id)
return ok(store.progress(task_id), "data process task stopped")
@@ -2005,7 +2008,9 @@ def confirm_results(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed")
result = store.confirm_results(task_id)
biz_logger.info("data-process:confirm-results success", taskId=task_id)
return ok(result, "data process results confirmed")
@router.put("/{task_id}/results/{result_id}")
@@ -2499,5 +2504,6 @@ def publish(
) -> dict[str, Any]:
with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json"))
biz_logger.info("data-process:publish success", taskId=task_id, datasetId=result.get("dataset_id", ""))
message = "dataset published" if result["created"] else "dataset already published"
return ok(result, message)

30
backend/app/api/v1/endpoints/platform.py
View File

@@ -471,6 +471,7 @@ async def _fine_tune_preflight_with_job_payload(
@router.post("/login")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGIN, target_type="user", target_name_param="username")
async def login(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
ip = request.client.host if request and request.client else "unknown"
@@ -488,6 +489,7 @@ async def login(payload: dict[str, Any] = Body(...), request: Request = None) ->
@router.post("/logout")
@op_log(module=OpModule.SYSTEM, action=OpAction.LOGOUT, target_type="user")
async def logout(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
store = get_platform_store()
session_id = payload.get("session_id", "")
@@ -717,11 +719,13 @@ async def users() -> dict[str, Any]:
@router.post("/users")
@op_log(module=OpModule.SYSTEM, action=OpAction.CREATE, target_type="user", target_name_param="username")
async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok(get_platform_store().create_user(payload))
@router.put("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user", target_name_param="user_id")
async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_user(user_id, payload))
@@ -730,6 +734,7 @@ async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict
@router.delete("/users/{user_id}")
@op_log(module=OpModule.SYSTEM, action=OpAction.DELETE, target_type="user", target_name_param="user_id")
async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]:
try:
get_platform_store().delete_user(user_id)
@@ -741,6 +746,7 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
@router.post("/users/{user_id}/reset-password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password", target_name_param="user_id")
async def reset_user_password(
user_id: str,
payload: dict[str, Any] = Body(default={}),
@@ -756,6 +762,7 @@ async def reset_user_password(
@router.post("/users/me/password")
@op_log(module=OpModule.SYSTEM, action=OpAction.UPDATE, target_type="user_password")
async def change_my_password(
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
@@ -819,6 +826,7 @@ async def trained_models(current_user: dict = Depends(get_current_user)) -> dict
@router.delete("/model-manage/trained-models/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="trained_model", target_name_param="model_id")
async def delete_trained_model(model_id: str, type: str = Query(default="merged"), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("trained_model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this trained model")
@@ -922,6 +930,7 @@ async def test_online_model(payload: dict[str, Any] = Body(...)) -> dict[str, An
@router.post("/model-manage")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.CREATE, target_type="model", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_MODEL,
target_type="model",
@@ -950,6 +959,7 @@ async def model_detail(model_id: str, current_user: dict = Depends(get_current_u
@router.put("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model", target_name_param="model_id")
@audit_log(
action=AuditActions.UPDATE_MODEL,
target_type="model",
@@ -966,6 +976,7 @@ async def update_model(model_id: str, payload: dict[str, Any] = Body(...), curre
@router.put("/model-manage/{model_id}/purpose")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.UPDATE, target_type="model_purpose", target_name_param="model_id")
async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型（配置模型）只有管理员可以修改用途
if not is_admin(current_user):
@@ -977,6 +988,7 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
@router.delete("/model-manage/{model_id}")
@op_log(module=OpModule.MODEL_MANAGE, action=OpAction.DELETE, target_type="model", target_name_param="model_id")
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# 基座模型（配置模型）只有管理员可以删除
if not is_admin(current_user):
@@ -1100,6 +1112,7 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An
@router.post("/dataset-manage/versions/{file_id}")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset_version", target_name_param="file_id")
async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().create_file_version(file_id, payload))
@@ -1116,6 +1129,7 @@ async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(
@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
@op_log(module=OpModule.DATASET, action=OpAction.DELETE, target_type="dataset_version", target_name_param="version_id")
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().delete_file_version(file_id, version_id))
@@ -1281,6 +1295,7 @@ async def _sync_training_dataset_to_compute_node(
@router.post("/dataset-manage/upload/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPLOAD, target_type="dataset_file", target_name_param="dataset_id")
async def upload_dataset_files(
dataset_id: str,
files: list[UploadFile] = File(default=[]),
@@ -1403,6 +1418,7 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s
@router.post("/dataset-manage")
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_DATASET,
target_type="dataset",
@@ -1430,6 +1446,7 @@ async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_curre
@router.put("/dataset-manage/{dataset_id}")
@op_log(module=OpModule.DATASET, action=OpAction.UPDATE, target_type="dataset", target_name_param="dataset_id")
@audit_log(
action=AuditActions.UPDATE_DATASET,
target_type="dataset",
@@ -1491,6 +1508,7 @@ async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict
@router.post("/fine-tune")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.CREATE, target_type="fine_tune", target_name_param="name")
@audit_log(
action=AuditActions.CREATE_FINE_TUNE,
target_type="fine_tune",
@@ -1662,6 +1680,7 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
@router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "write"):
raise fail(403, "no permission to update this task")
@@ -1696,6 +1715,7 @@ async def stop_fine_tune_alt(task_id: str, current_user: dict = Depends(get_curr
@router.post("/fine-tune/{task_id}/retry")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.RETRY, target_type="fine_tune", target_name_param="task_id")
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}
@@ -1964,6 +1984,7 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
@router.delete("/model-eval/{task_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="eval_task", target_name_param="task_id")
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("eval", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this eval task")
@@ -1980,6 +2001,7 @@ async def dimension_list() -> dict[str, Any]:
@router.post("/dimension")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.CREATE, target_type="dimension", target_name_param="name")
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
return ok(get_platform_store().create_dimension(payload))
@@ -1993,6 +2015,7 @@ async def dimension_detail(dimension_id: str) -> dict[str, Any]:
@router.put("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.UPDATE, target_type="dimension", target_name_param="dimension_id")
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_dimension(dimension_id, payload))
@@ -2001,6 +2024,7 @@ async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...
@router.delete("/dimension/{dimension_id}")
@op_log(module=OpModule.MODEL_EVAL, action=OpAction.DELETE, target_type="dimension", target_name_param="dimension_id")
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
get_platform_store().delete_dimension(dimension_id)
return ok({"deleted": dimension_id})
@@ -2019,6 +2043,7 @@ async def model_compare_list(current_user: dict = Depends(get_current_user)) ->
@router.post("/model-compare")
@op_log(module=OpModule.INFERENCE, action=OpAction.CREATE, target_type="compare_task", target_name_param="name")
async def model_compare_create(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
payload.setdefault("created_by", current_user.get("id"))
model_ids = payload.get("model_ids") or payload.get("models") or []
@@ -2574,6 +2599,7 @@ async def compute_node_detail(node_id: str, current_user: dict = Depends(get_cur
@router.post("/compute/nodes")
@op_log(module=OpModule.COMPUTE, action=OpAction.CREATE, target_type="compute_node", target_name_param="name")
async def create_compute_node(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -2586,6 +2612,7 @@ async def create_compute_node(payload: dict[str, Any] = Body(...), current_user:
@router.put("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.UPDATE, target_type="compute_node", target_name_param="node_id")
async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -2598,6 +2625,7 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...),
@router.delete("/compute/nodes/{node_id}")
@op_log(module=OpModule.COMPUTE, action=OpAction.DELETE, target_type="compute_node", target_name_param="node_id")
async def delete_compute_node(node_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not is_admin(current_user):
raise fail(403, "admin permission required")
@@ -2904,6 +2932,7 @@ async def compute_job_detail(job_id: str) -> dict[str, Any]:
@router.post("/compute/jobs/{job_id}/stop")
@op_log(module=OpModule.COMPUTE, action=OpAction.STOP, target_type="compute_job", target_name_param="job_id")
async def compute_job_stop(job_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
task = _task_for_compute_job(job_id)
if task and not has_resource_access("fine-tune", task["id"], current_user, "write"):
@@ -2942,6 +2971,7 @@ async def compute_job_logs(
@router.post("/compute/jobs/{job_id}/retry")
@op_log(module=OpModule.COMPUTE, action=OpAction.RETRY, target_type="compute_job", target_name_param="job_id")
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
store = get_platform_store()
payload = payload or {}