YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: 合并远程 ft_wyt 分支，解决权限与日志模块冲突

Browse Source 
- 冲突解决原则：本地权限治理(require_admin/current_user/资源ACL)与远程 op_log 日志装饰器双向保留
- platform.py: 9 处冲突，@op_log 与管理员校验叠加，避免远程丢失 require_admin 的安全回归
- logging.py: 合并 get_client_ip 与 user_id_var，X-Trace-Id 优先 + ContextVar 卫生处理
- op_log.py: 采纳远程将变量计算上移到函数顶部的结构
- compute_poller.py: 中文日志 + 失败去重限流/断连重置逻辑
- data_process.py: 保留租户归属字段 + biz_logger 成功日志
This commit is contained in:
wuyongtao
2026-08-21 09:58:43 +08:00
parent 6f0e82f351 29d8a506cd
commit 3b9361c237
27 changed files with 2778 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
backend/app/api/v1/endpoints/data_process.py
View File

@@ -41,8 +41,12 @@ from app.core.auth import (
has_resource_access,
is_admin,
)
from app.core.logging import get_structured_logger
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -117,6 +121,7 @@ from app.schemas.data_process import (
)
logger = logging.getLogger(__name__)
biz_logger = get_structured_logger("app.biz.data_process")
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
MAX_SOURCE_FILE_COUNT = 20
MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
@@ -292,7 +297,7 @@ def _commit_source_batch(
except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。
logger.exception(
"failed to roll back data process source object task_id=%s",
"数据处理源对象回滚失败 task_id=%s",
task_id,
)
raise
@@ -662,7 +667,7 @@ def _run_generation(
) -> None:
started_at = time.perf_counter()
logger.info(
"data process generation worker started task_id=%s generation_run_id=%s",
"数据处理生成任务开始 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -670,8 +675,7 @@ def _run_generation(
task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id):
logger.info(
"data process generation worker skipped inactive run task_id=%s "
"generation_run_id=%s",
"数据处理生成任务跳过（非活跃运行） task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -761,8 +765,7 @@ def _run_generation(
len(preview_items),
):
logger.info(
"data process generation stopped before completion task_id=%s "
"generation_run_id=%s",
"数据处理生成任务被中止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -853,9 +856,7 @@ def _run_generation(
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
"数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f",
task_id,
generation_run_id,
completed.get("output_count", len(accepted)),
@@ -866,14 +867,13 @@ def _run_generation(
)
else:
logger.info(
"data process generation stopped before result persistence task_id=%s "
"generation_run_id=%s",
"数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
except Exception as exc:
logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
"数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id,
generation_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -887,8 +887,7 @@ def _run_generation(
)
except Exception:
logger.exception(
"failed to persist data process generation failure task_id=%s "
"generation_run_id=%s",
"数据处理生成失败持久化异常 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -943,6 +942,7 @@ def create_task(
values["tenant_id"] = current_user.get("tenant_id") or "default"
get_platform_store().assert_active_tenant(values["tenant_id"])
task = store.create_task(values)
biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", ""))
return ok(task, "data process task created")
@@ -968,10 +968,9 @@ def update_task(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(
store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")),
"data process task updated",
)
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
biz_logger.info("用户更新数据处理任务成功", taskId=task_id)
return ok(result, "data process task updated")
@router.put("/{task_id}/workflow-step")
@@ -1060,7 +1059,7 @@ def _remove_repeated_storage_objects(
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
"数据处理源对象重复回滚失败 task_id=%s",
task_id,
)
@@ -1136,6 +1135,7 @@ def delete_task(
) -> dict[str, Any]:
with api_errors():
store.delete_task(task_id)
biz_logger.info("用户删除数据处理任务成功", taskId=task_id)
return ok({"deleted": task_id}, "data process task deleted")
@@ -1448,13 +1448,12 @@ def delete_source_file(
expected_source_file_id=file_id,
)
except Exception:
# 数据库软删除已经提交，不能再向客户端返回可重试的失败；保留逻辑引用，
# 由后续存储清理任务重试物理删除。
cleanup_pending = True
logger.exception(
"failed to remove data process source object after soft deletion",
"数据处理源对象软删除后存储清理失败",
extra={"task_id": task_id, "source_file_id": file_id},
)
biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
return ok(
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
"source file removed",
@@ -1727,7 +1726,7 @@ def _prepare_preview_items(
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
logger.warning(
"skip PDF document noise detection because stored offsets differ for %s",
"跳过PDF文档噪声检测（存储偏移量不一致） source_id=%s",
source["id"],
)
continue
@@ -1750,7 +1749,7 @@ def _run_preview(
started_at = time.perf_counter()
logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
"数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s",
task_id,
preview_run_id,
len(source_file_ids),
@@ -1759,7 +1758,7 @@ def _run_preview(
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
"数据处理预览跳过（非活跃运行） task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -1769,8 +1768,7 @@ def _run_preview(
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id):
logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s "
"completed_files=%s total_files=%s",
"数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files - 1,
@@ -1801,8 +1799,7 @@ def _run_preview(
total_files,
):
logger.info(
"data process preview stopped before progress update task_id=%s "
"preview_run_id=%s completed_files=%s total_files=%s",
"数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files,
@@ -1811,8 +1808,7 @@ def _run_preview(
return
if store.complete_preview(task_id, preview_run_id):
logger.info(
"data process preview completed task_id=%s preview_run_id=%s "
"total_files=%s total_items=%s duration_ms=%.2f",
"数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f",
task_id,
preview_run_id,
total_files,
@@ -1821,14 +1817,13 @@ def _run_preview(
)
else:
logger.info(
"data process preview completion ignored for inactive run task_id=%s "
"preview_run_id=%s",
"数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
except Exception as exc:
logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
"数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id,
preview_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -1842,8 +1837,7 @@ def _run_preview(
)
except Exception:
logger.exception(
"failed to persist data process preview failure task_id=%s "
"preview_run_id=%s",
"数据处理预览失败持久化异常 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -2053,6 +2047,7 @@ def stop(
) -> dict[str, Any]:
with api_errors():
store.stop_task(task_id)
biz_logger.info("用户停止数据处理任务成功", taskId=task_id)
return ok(store.progress(task_id), "data process task stopped")
@@ -2094,7 +2089,9 @@ def confirm_results(
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
with api_errors():
return ok(store.confirm_results(task_id), "data process results confirmed")
result = store.confirm_results(task_id)
biz_logger.info("用户确认数据处理结果成功", taskId=task_id)
return ok(result, "data process results confirmed")
@router.put("/{task_id}/results/{result_id}")
@@ -2443,8 +2440,7 @@ def regenerate_results_batch(
}))
logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s "
"requested=%s prepared=%s concurrency=%s",
"数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s",
batch_id,
task_id,
len(payload.items),
@@ -2512,8 +2508,7 @@ def regenerate_results_batch(
except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error"
logger.exception(
"data process result batch regeneration crashed "
"batch_id=%s task_id=%s result_id=%s",
"数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s",
batch_id,
task_id,
result_id,
@@ -2524,8 +2519,7 @@ def regenerate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch item finished batch_id=%s task_id=%s "
"result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2545,8 +2539,7 @@ def regenerate_results_batch(
)
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
"数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2593,8 +2586,7 @@ def evaluate_results_batch(
evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError:
logger.warning(
"data process evaluation model unavailable, judge layer "
"skipped task_id=%s model_id=%s",
"数据处理评测模型不可用，跳过评测层 task_id=%s model_id=%s",
task_id,
model_id,
)
@@ -2642,8 +2634,7 @@ def evaluate_results_batch(
}))
logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s "
"requested=%s prepared=%s judge_enabled=%s",
"数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s",
batch_id,
task_id,
len(payload.items),
@@ -2660,8 +2651,7 @@ def evaluate_results_batch(
semantic_embedding_model()
except Exception:
logger.warning(
"data process semantic embedding unavailable, semantic layer "
"will be skipped batch_id=%s",
"数据处理语义嵌入模型不可用，语义层将跳过 batch_id=%s",
batch_id,
)
request_timeout = _result_regeneration_timeout(config)
@@ -2714,8 +2704,7 @@ def evaluate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation item finished "
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2727,8 +2716,7 @@ def evaluate_results_batch(
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s duration_ms=%.2f",
"数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2784,8 +2772,7 @@ def regenerate_result(
)
except _ResultRegenerationFailed as exc:
logger.warning(
"data process result regeneration failed task_id=%s result_id=%s "
"duration_ms=%.2f reason=%s",
"数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2793,7 +2780,7 @@ def regenerate_result(
)
raise
logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
"数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2809,5 +2796,6 @@ def publish(
) -> dict[str, Any]:
with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json"))
biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", ""))
message = "dataset published" if result["created"] else "dataset already published"
return ok(result, message)