YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新日志的中文说明

Browse Source
This commit is contained in:
wangjiming
2026-08-21 09:42:03 +08:00
parent 8455d99d49
commit 29d8a506cd
12 changed files with 1923 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
backend/app/api/v1/endpoints/data_process.py
View File

@@ -271,7 +271,7 @@ def _commit_source_batch(
except Exception:
# 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。
logger.exception(
"failed to roll back data process source object task_id=%s",
"数据处理源对象回滚失败 task_id=%s",
task_id,
)
raise
@@ -641,7 +641,7 @@ def _run_generation(
) -> None:
started_at = time.perf_counter()
logger.info(
"data process generation worker started task_id=%s generation_run_id=%s",
"数据处理生成任务开始 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -649,8 +649,7 @@ def _run_generation(
task = store.get_task(task_id)
if not store.generation_is_running(task_id, generation_run_id):
logger.info(
"data process generation worker skipped inactive run task_id=%s "
"generation_run_id=%s",
"数据处理生成任务跳过（非活跃运行） task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -740,8 +739,7 @@ def _run_generation(
len(preview_items),
):
logger.info(
"data process generation stopped before completion task_id=%s "
"generation_run_id=%s",
"数据处理生成任务被中止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -832,9 +830,7 @@ def _run_generation(
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
"duration_ms=%.2f",
"数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f",
task_id,
generation_run_id,
completed.get("output_count", len(accepted)),
@@ -845,14 +841,13 @@ def _run_generation(
)
else:
logger.info(
"data process generation stopped before result persistence task_id=%s "
"generation_run_id=%s",
"数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
except Exception as exc:
logger.exception(
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
"数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f",
task_id,
generation_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -866,8 +861,7 @@ def _run_generation(
)
except Exception:
logger.exception(
"failed to persist data process generation failure task_id=%s "
"generation_run_id=%s",
"数据处理生成失败持久化异常 task_id=%s generation_run_id=%s",
task_id,
generation_run_id,
)
@@ -916,7 +910,7 @@ def create_task(
) -> dict[str, Any]:
with api_errors():
task = store.create_task(payload.model_dump(mode="json"))
biz_logger.info("data-process:create success", taskId=task["id"], processType=task.get("process_type", ""))
biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", ""))
return ok(task, "data process task created")
@@ -943,7 +937,7 @@ def update_task(
) -> dict[str, Any]:
with api_errors():
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
biz_logger.info("data-process:update success", taskId=task_id)
biz_logger.info("用户更新数据处理任务成功", taskId=task_id)
return ok(result, "data process task updated")
@@ -1033,7 +1027,7 @@ def _remove_repeated_storage_objects(
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
"数据处理源对象重复回滚失败 task_id=%s",
task_id,
)
@@ -1109,7 +1103,7 @@ def delete_task(
) -> dict[str, Any]:
with api_errors():
store.delete_task(task_id)
biz_logger.info("data-process:delete success", taskId=task_id)
biz_logger.info("用户删除数据处理任务成功", taskId=task_id)
return ok({"deleted": task_id}, "data process task deleted")
@@ -1424,10 +1418,10 @@ def delete_source_file(
except Exception:
cleanup_pending = True
logger.exception(
"failed to remove data process source object after soft deletion",
"数据处理源对象软删除后存储清理失败",
extra={"task_id": task_id, "source_file_id": file_id},
)
biz_logger.info("data-process:delete-source-file success", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
return ok(
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
"source file removed",
@@ -1700,7 +1694,7 @@ def _prepare_preview_items(
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
logger.warning(
"skip PDF document noise detection because stored offsets differ for %s",
"跳过PDF文档噪声检测（存储偏移量不一致） source_id=%s",
source["id"],
)
continue
@@ -1723,7 +1717,7 @@ def _run_preview(
started_at = time.perf_counter()
logger.info(
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
"数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s",
task_id,
preview_run_id,
len(source_file_ids),
@@ -1732,7 +1726,7 @@ def _run_preview(
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
"数据处理预览跳过（非活跃运行） task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -1742,8 +1736,7 @@ def _run_preview(
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
if not store.preview_is_running(task_id, preview_run_id):
logger.info(
"data process preview cancelled task_id=%s preview_run_id=%s "
"completed_files=%s total_files=%s",
"数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files - 1,
@@ -1774,8 +1767,7 @@ def _run_preview(
total_files,
):
logger.info(
"data process preview stopped before progress update task_id=%s "
"preview_run_id=%s completed_files=%s total_files=%s",
"数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
task_id,
preview_run_id,
completed_files,
@@ -1784,8 +1776,7 @@ def _run_preview(
return
if store.complete_preview(task_id, preview_run_id):
logger.info(
"data process preview completed task_id=%s preview_run_id=%s "
"total_files=%s total_items=%s duration_ms=%.2f",
"数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f",
task_id,
preview_run_id,
total_files,
@@ -1794,14 +1785,13 @@ def _run_preview(
)
else:
logger.info(
"data process preview completion ignored for inactive run task_id=%s "
"preview_run_id=%s",
"数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
except Exception as exc:
logger.exception(
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
"数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f",
task_id,
preview_run_id,
(time.perf_counter() - started_at) * 1000,
@@ -1815,8 +1805,7 @@ def _run_preview(
)
except Exception:
logger.exception(
"failed to persist data process preview failure task_id=%s "
"preview_run_id=%s",
"数据处理预览失败持久化异常 task_id=%s preview_run_id=%s",
task_id,
preview_run_id,
)
@@ -2026,7 +2015,7 @@ def stop(
) -> dict[str, Any]:
with api_errors():
store.stop_task(task_id)
biz_logger.info("data-process:stop success", taskId=task_id)
biz_logger.info("用户停止数据处理任务成功", taskId=task_id)
return ok(store.progress(task_id), "data process task stopped")
@@ -2069,7 +2058,7 @@ def confirm_results(
) -> dict[str, Any]:
with api_errors():
result = store.confirm_results(task_id)
biz_logger.info("data-process:confirm-results success", taskId=task_id)
biz_logger.info("用户确认数据处理结果成功", taskId=task_id)
return ok(result, "data process results confirmed")
@@ -2419,8 +2408,7 @@ def regenerate_results_batch(
}))
logger.info(
"data process result batch regeneration started batch_id=%s task_id=%s "
"requested=%s prepared=%s concurrency=%s",
"数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s",
batch_id,
task_id,
len(payload.items),
@@ -2488,8 +2476,7 @@ def regenerate_results_batch(
except Exception as exc: # pragma: no cover - defensive boundary
outcome = "internal_error"
logger.exception(
"data process result batch regeneration crashed "
"batch_id=%s task_id=%s result_id=%s",
"数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s",
batch_id,
task_id,
result_id,
@@ -2500,8 +2487,7 @@ def regenerate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch item finished batch_id=%s task_id=%s "
"result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2521,8 +2507,7 @@ def regenerate_results_batch(
)
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch regeneration completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
"数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2569,8 +2554,7 @@ def evaluate_results_batch(
evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError:
logger.warning(
"data process evaluation model unavailable, judge layer "
"skipped task_id=%s model_id=%s",
"数据处理评测模型不可用，跳过评测层 task_id=%s model_id=%s",
task_id,
model_id,
)
@@ -2618,8 +2602,7 @@ def evaluate_results_batch(
}))
logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s "
"requested=%s prepared=%s judge_enabled=%s",
"数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s",
batch_id,
task_id,
len(payload.items),
@@ -2636,8 +2619,7 @@ def evaluate_results_batch(
semantic_embedding_model()
except Exception:
logger.warning(
"data process semantic embedding unavailable, semantic layer "
"will be skipped batch_id=%s",
"数据处理语义嵌入模型不可用，语义层将跳过 batch_id=%s",
batch_id,
)
request_timeout = _result_regeneration_timeout(config)
@@ -2690,8 +2672,7 @@ def evaluate_results_batch(
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation item finished "
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
"数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
@@ -2703,8 +2684,7 @@ def evaluate_results_batch(
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s duration_ms=%.2f",
"数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
@@ -2760,8 +2740,7 @@ def regenerate_result(
)
except _ResultRegenerationFailed as exc:
logger.warning(
"data process result regeneration failed task_id=%s result_id=%s "
"duration_ms=%.2f reason=%s",
"数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2769,7 +2748,7 @@ def regenerate_result(
)
raise
logger.info(
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
"数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f",
task_id,
result_id,
(time.perf_counter() - started_at) * 1000,
@@ -2785,6 +2764,6 @@ def publish(
) -> dict[str, Any]:
with api_errors():
result = store.publish(task_id, payload.model_dump(mode="json"))
biz_logger.info("data-process:publish success", taskId=task_id, datasetId=result.get("dataset_id", ""))
biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", ""))
message = "dataset published" if result["created"] else "dataset already published"
return ok(result, message)