YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与对象存储增强，审批中心与运行日志整合

Browse Source 
- 新增 storage/policy.py 落盘策略：按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects，支持失败回滚
- 算力节点训练产物按版本归档到 MinIO，登记 model_artifacts
- 数据转换任务输入输出对象化，支持从 MinIO 读写
- 新增审批中心（申请/我的/策略）、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
backend/app/modules/data_process/storage.py
View File

@@ -1,4 +1,4 @@
"""数据处理原始源文件的受控本地对象存储。"""
"""数据处理源文件的受控暂存与分层对象存储。"""
from __future__ import annotations
@@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath
from typing import Iterable, Iterator
from urllib.parse import quote, unquote, urlsplit
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
class DataProcessStorageError(ValueError):
"""本地对象引用或文件系统状态不安全。"""
@@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str:
class LocalDataProcessStorage:
"""只允许访问配置根目录下的版本化原始文件。"""
"""Stage locally, but publish and read authoritative source files from MinIO."""
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
configured = Path(root) if root is not None else _configured_storage_root()
@@ -132,10 +135,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -168,6 +168,18 @@ class LocalDataProcessStorage:
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
if self._is_minio_reference(source_reference):
content = self.read(source_reference)
if content is None:
raise DataProcessStorageError("original source object is not available")
return self.stage_bytes(
batch_id=batch_id,
task_id=task_id,
source_file_id=source_file_id,
version=version,
name=basename,
content=content,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
@@ -193,10 +205,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -212,14 +221,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("duplicate staged source object")
seen_temporary_paths.add(item._temporary_path)
for item in staged:
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
if self._is_minio_reference(item.reference):
content = item._temporary_path.read_bytes()
get_object_storage().put_bytes(
self.object_key(item.reference),
content,
"application/octet-stream",
)
elif not item.reference.startswith("db://data-process/"):
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
self._fsync_directory(final_path.parent)
published.append(item)
item._temporary_path.unlink()
self._fsync_directory(final_path.parent)
except Exception:
for item in reversed(published):
try:
@@ -258,7 +275,13 @@ class LocalDataProcessStorage:
raise first_error
def read(self, reference: str) -> bytes | None:
"""读取 local 引用；旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。"""
"""Read a MinIO object or legacy local reference."""
if self._is_minio_reference(reference):
try:
return get_object_storage().get_bytes(self.object_key(reference))
except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
@@ -276,6 +299,11 @@ class LocalDataProcessStorage:
) -> int | None:
"""返回受控 local 对象大小；旧 ``db://`` 对象没有原始文件。"""
if self._is_minio_reference(reference):
try:
return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0)
except Exception as exc: # noqa: BLE001
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return None
@@ -301,6 +329,15 @@ class LocalDataProcessStorage:
) -> Iterator[bytes]:
"""按范围流式读取原始文件，避免 PDF 预览把大文件整体载入内存。"""
if self._is_minio_reference(reference):
content = self.read(reference) or b""
if start < 0 or expected_size != len(content) or start > expected_size:
raise DataProcessStorageError("source object size does not match metadata")
remaining = expected_size - start if length is None else length
if remaining < 0 or start + remaining > expected_size:
raise DataProcessStorageError("invalid source byte range")
yield content[start : start + remaining]
return
relative_path = self._relative_from_reference(reference)
if relative_path is None:
raise DataProcessStorageError("original source object is not available")
@@ -337,6 +374,12 @@ class LocalDataProcessStorage:
) -> bool:
"""校验 local 引用归属；旧 ``db://`` 引用无需文件系统处理。"""
if self._is_minio_reference(reference):
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
return True
if str(reference or "").startswith("db://data-process/"):
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return True
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -382,6 +425,19 @@ class LocalDataProcessStorage:
) -> bool:
"""删除受控 local 对象；旧 ``db://`` 引用保持不变。"""
if self._is_minio_reference(reference):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
get_object_storage().delete(self.object_key(reference))
return True
if str(reference or "").startswith("db://data-process/"):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return False
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -426,7 +482,7 @@ class LocalDataProcessStorage:
if reference.startswith("db://"):
return None
parsed = urlsplit(reference)
if parsed.scheme != "local" or parsed.netloc != "data-process":
if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process":
raise DataProcessStorageError("unsupported source storage reference")
if parsed.query or parsed.fragment or "\\" in parsed.path:
raise DataProcessStorageError("unsafe source storage reference")
@@ -460,6 +516,39 @@ class LocalDataProcessStorage:
basename = _safe_basename(decoded[3])
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
@staticmethod
def _is_minio_reference(reference: str) -> bool:
return str(reference or "").startswith("minio://data-process/")
@staticmethod
def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str:
scheme = "minio" if get_settings().minio_enabled else "local"
return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
@staticmethod
def object_key(reference: str) -> str:
parsed = urlsplit(reference)
if parsed.scheme != "minio" or parsed.netloc != "data-process":
raise DataProcessStorageError("reference is not a MinIO source object")
return "data-process/" + parsed.path.lstrip("/")
def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None:
relative = self._relative_from_reference(reference)
if relative is None:
raise DataProcessStorageError("invalid MinIO source reference")
self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id)
@staticmethod
def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None:
parsed = urlsplit(reference)
parts = parsed.path.lstrip("/").split("/")
if parsed.netloc != "data-process" or len(parts) != 3:
raise DataProcessStorageError("invalid database source reference")
expected_task_id = _safe_component(task_id, "expected task id")
expected_source_file_id = _safe_component(source_file_id, "expected source file id")
if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1":
raise DataProcessStorageError("source storage object owner mismatch")
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
if relative_path.is_absolute() or any(
part in {"", ".", ".."} for part in relative_path.parts
@@ -479,9 +568,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("invalid staged source object")
if self._issued_staged_objects.get(item._temporary_path) is not item:
raise DataProcessStorageError("staged source object was not issued by this storage")
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
if item.reference.startswith("db://data-process/"):
parsed = urlsplit(item.reference)
parts = parsed.path.lstrip("/").split("/")
expected = item._relative_path.parts[:3]
if (
parsed.netloc != "data-process"
or parsed.query
or parsed.fragment
or len(parts) != 3
or tuple(parts) != expected
):
raise DataProcessStorageError("staged source object reference mismatch")
else:
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
staging_root = self._root / ".staging"
try:
relative_temporary = item._temporary_path.relative_to(staging_root)

7
backend/app/modules/data_process/store/base.py
View File

@@ -247,14 +247,19 @@ def _source_storage_descriptor(
or f"db://data-process/{task_id}/{file_id}/v1"
)
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
expected_local_prefix
):
storage_backend = "local"
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
expected_minio_prefix
):
storage_backend = "minio"
elif storage_object_id == expected_database_reference:
storage_backend = "database"
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
raise DataProcessStoreError("source storage object owner mismatch")
else:
raise DataProcessStoreError("unsupported source storage object reference")

24
backend/app/modules/data_process/store/datasets.py
View File

@@ -11,6 +11,7 @@ import psycopg
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from .base import (
StoreBase,
@@ -168,7 +169,11 @@ class DatasetsMixin:
split_name: assignments.count(split_name) for split_name in split_order
}
split_specs: list[dict[str, Any]] = []
use_minio = bool(get_settings().minio_enabled)
# Explicit local is retained for old callers/tests that request the
# legacy backend; all normal platform requests default to MinIO.
allow_minio = bool(get_settings().minio_enabled) and str(
payload.get("storage_type") or "minio"
).lower() != "local"
for split_name in split_order:
split_records = [
(source_row, record)
@@ -193,12 +198,15 @@ class DatasetsMixin:
"storage_object_id": (
f"db://data-process/{task_id}/{file_id}/v1"
),
"store_in_minio": allow_minio and should_store_in_minio(
len(raw), content_type="application/jsonl", file_format="jsonl"
),
}
)
source_result_ids = [row["id"] for row in rows]
common_metadata = {
"source": "data_process",
"storage_backend": "minio" if use_minio else "database",
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
"source_task_id": task_id,
"output_type": _task_output_type(task),
"reasoning_detail": _task_reasoning_detail(task),
@@ -273,7 +281,7 @@ class DatasetsMixin:
split_name = str(spec["split"])
dataset_id = dataset_ids[split_name]
storage_object_id = str(spec["storage_object_id"])
if use_minio:
if spec["store_in_minio"]:
file_name = f"{base_dataset_name}.{split_name}.jsonl"
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
uploaded = get_object_storage().put_bytes(
@@ -356,7 +364,7 @@ class DatasetsMixin:
(
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
f"{len(spec['raw'])} B",
len(spec["raw"]),
len(spec["records"]),
@@ -386,7 +394,7 @@ class DatasetsMixin:
dataset_id,
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
task_id,
task_id,
f"{len(spec['raw'])} B",
@@ -405,7 +413,11 @@ class DatasetsMixin:
),
).fetchone()
file_metadata = {**dataset_metadata, "file_split": split_name}
file_metadata = {
**dataset_metadata,
"file_split": split_name,
"storage_backend": "minio" if spec["store_in_minio"] else "database",
}
version = {
"id": spec["version_id"],
"version_no": 1,

14
backend/app/modules/data_process/store/source_files.py
View File

@@ -137,7 +137,7 @@ class SourceFilesMixin:
payload["record_count"],
payload["file_format"],
payload["checksum_sha256"],
payload["content"],
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
str(payload["content"])[:2000],
json_dumps(metadata_payload),
task.get("tenant_id"),
@@ -223,7 +223,17 @@ class SourceFilesMixin:
).fetchone()
if not row:
raise NotFoundError("source file not found")
return _decode_row(row) or {}
decoded = _decode_row(row) or {}
# New source files keep only a preview in PostgreSQL. Load the
# authoritative body from MinIO on demand for existing processing code.
reference = str(decoded.get("storage_object_id") or "")
if include_content and not decoded.get("content") and reference.startswith("minio://"):
from app.modules.data_process.storage import get_data_process_storage
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
"utf-8", errors="replace"
)
return decoded
def source_content_window(
self, task_id: str, file_id: str, offset: int, limit: int