feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,8 @@ from fastapi.responses import StreamingResponse
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.data_process.algorithms import (
|
||||
ParsedText,
|
||||
canonical_record_json,
|
||||
@@ -85,6 +87,8 @@ from app.modules.data_process.store import (
|
||||
new_id,
|
||||
repeat_task_id,
|
||||
)
|
||||
from app.modules.storage.minio_store import get_object_storage
|
||||
from app.modules.storage.policy import should_store_in_minio
|
||||
from app.schemas.data_process import (
|
||||
DataProcessRegenerateRequest,
|
||||
DataProcessRepeatRequest,
|
||||
@@ -224,9 +228,38 @@ def _commit_source_batch(
|
||||
staged: list[StagedSourceObject],
|
||||
) -> list[dict[str, Any]]:
|
||||
storage.publish(staged)
|
||||
storage_object_ids: list[str] = []
|
||||
try:
|
||||
# The source reference remains in the task schema for compatibility,
|
||||
# while storage_objects provides the authoritative MinIO index.
|
||||
if get_settings().minio_enabled:
|
||||
for item in prepared:
|
||||
reference = str(item.get("storage_object_id") or "")
|
||||
if not reference.startswith("minio://"):
|
||||
continue
|
||||
object_key = storage.object_key(reference)
|
||||
metadata = get_object_storage().stat(object_key)
|
||||
object_row = get_platform_store().create_storage_object({
|
||||
"resource_type": "data_process_source",
|
||||
"resource_id": task_id,
|
||||
"version_id": str(item.get("id") or new_id("dpsf")),
|
||||
"bucket": get_object_storage().bucket,
|
||||
"object_key": object_key,
|
||||
"file_name": item.get("name"),
|
||||
"content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"),
|
||||
"byte_size": metadata.get("byte_size") or item.get("raw_size") or 0,
|
||||
"checksum_sha256": item.get("checksum_sha256"),
|
||||
"status": "available",
|
||||
"created_by": item.get("created_by"),
|
||||
})
|
||||
storage_object_ids.append(str(object_row["id"]))
|
||||
return store.add_source_files(task_id, prepared)
|
||||
except Exception:
|
||||
for object_id in storage_object_ids:
|
||||
try:
|
||||
get_platform_store().update_storage_object(object_id, {"status": "deleted"})
|
||||
except Exception:
|
||||
pass
|
||||
for item in staged:
|
||||
try:
|
||||
storage.delete(item.reference)
|
||||
@@ -774,6 +807,25 @@ def _run_generation(
|
||||
duplicate_count=duplicate_count,
|
||||
error_count=error_count,
|
||||
)
|
||||
result_bytes = "".join(
|
||||
structured_json_dumps(item) + "\n" for item in accepted
|
||||
).encode("utf-8")
|
||||
if should_store_in_minio(len(result_bytes)):
|
||||
result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl"
|
||||
uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl")
|
||||
get_platform_store().create_storage_object({
|
||||
"resource_type": "data_process_result",
|
||||
"resource_id": task_id,
|
||||
"version_id": generation_run_id,
|
||||
"bucket": uploaded["bucket"],
|
||||
"object_key": result_key,
|
||||
"file_name": f"{generation_run_id}.jsonl",
|
||||
"content_type": "application/jsonl",
|
||||
"byte_size": len(result_bytes),
|
||||
"checksum_sha256": hashlib.sha256(result_bytes).hexdigest(),
|
||||
"status": "available",
|
||||
"created_by": (store.get_task(task_id) or {}).get("created_by"),
|
||||
})
|
||||
logger.info(
|
||||
"data process generation completed task_id=%s generation_run_id=%s "
|
||||
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
|
||||
@@ -933,7 +985,7 @@ def _repeat_file_copies(
|
||||
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
||||
new_file_id = new_id("dpsf")
|
||||
old_reference = str(source.get("storage_object_id") or "")
|
||||
if old_reference.startswith("local://data-process/"):
|
||||
if old_reference.startswith(("local://data-process/", "minio://data-process/")):
|
||||
staged_object = storage.stage_copy(
|
||||
batch_id=batch_id,
|
||||
source_reference=old_reference,
|
||||
|
||||
Reference in New Issue
Block a user