YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与对象存储增强，审批中心与运行日志整合

Browse Source 
- 新增 storage/policy.py 落盘策略：按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects，支持失败回滚
- 算力节点训练产物按版本归档到 MinIO，登记 model_artifacts
- 数据转换任务输入输出对象化，支持从 MinIO 读写
- 新增审批中心（申请/我的/策略）、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

311
backend/app/modules/data_convert/router.py
View File

@@ -1,17 +1,21 @@
from __future__ import annotations
import json
import hashlib
import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
@@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path:
return _task_dir(task_id) / "output"
def _minio_enabled() -> bool:
return bool(get_settings().minio_enabled)
def _input_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/input/{Path(name).name}"
def _output_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/output/{Path(name).name}"
def _task_objects(task_id: str) -> list[dict[str, Any]]:
return get_platform_store().storage_objects_for_resource("data_convert", task_id)
def _register_object(
task_id: str,
*,
version_id: str,
object_key: str,
file_name: str,
content_type: str,
content: bytes,
created_by: str | None,
) -> dict[str, Any]:
storage = get_object_storage()
uploaded = storage.put_bytes(object_key, content, content_type)
return get_platform_store().create_storage_object(
{
"resource_type": "data_convert",
"resource_id": task_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": file_name,
"content_type": content_type,
"byte_size": len(content),
"checksum_sha256": hashlib.sha256(content).hexdigest(),
"status": "available",
"created_by": created_by,
}
)
def _input_objects(task_id: str) -> list[dict[str, Any]]:
prefix = f"data-convert/{task_id}/input/"
return sorted(
[item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)],
key=lambda item: str(item.get("file_name") or item.get("object_key") or ""),
)
def _output_object(task: dict[str, Any]) -> dict[str, Any] | None:
key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename")))
return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None)
def _read_output(task: dict[str, Any]) -> bytes | None:
if _minio_enabled():
item = _output_object(task)
if item:
return get_object_storage().get_bytes(item["object_key"])
inline = task.get("output_content")
return str(inline).encode("utf-8") if inline is not None else None
path = _task_output_path(task)
return path.read_bytes() if path.exists() else None
def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]:
output_name = _safe_output_filename(task.get("output_filename"))
output_lines: list[str] = []
input_count = 0
output_count = 0
for item in _input_objects(task["id"]):
input_count += 1
raw = get_object_storage().get_bytes(item["object_key"])
data = json.loads(raw.decode("utf-8"))
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {item.get('file_name')}")
for record in records:
output_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
output = "".join(output_lines).encode("utf-8")
store = get_platform_store()
with store.connect() as conn:
if should_store_in_minio(len(output)):
output_object = _register_object(
task["id"],
version_id="output",
object_key=_output_object_key(task["id"], output_name),
file_name=output_name,
content_type="application/jsonl",
content=output,
created_by=created_by,
)
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s",
(output_object["id"], task["id"]),
)
else:
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s",
(output.decode("utf-8"), task["id"]),
)
return input_count, output_count, output
def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]:
input_dir = _input_dir(task["id"])
output_dir = _output_dir(task["id"])
input_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
output_path.unlink(missing_ok=True)
input_count = 0
output_count = 0
with output_path.open("w", encoding="utf-8") as output_file:
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
data = json.loads(json_file.read_text(encoding="utf-8"))
records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None
if records is None:
raise ValueError(f"JSON must be object or array: {json_file.name}")
for record in records:
output_file.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
return input_count, output_count, output_path.read_bytes()
@router.get("")
def list_tasks(
page: int = 1,
@@ -109,9 +249,10 @@ def create_task(
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
)
# 创建目录
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
return ok(_get_task(task_id))
@@ -123,13 +264,19 @@ def get_task(
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
# 附加输入文件列表
input_dir = _input_dir(task_id)
# 附加输入文件列表；旧任务没有对象记录时继续读取本地兼容目录。
files = []
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
if _minio_enabled():
files = [
{"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0}
for item in _input_objects(task_id)
]
else:
input_dir = _input_dir(task_id)
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
task["input_files"] = files
return ok(task)
@@ -146,16 +293,26 @@ async def upload_source_files(
raise fail(404, "task not found")
if task["status"] not in ("pending", "uploaded"):
raise fail(400, "task is not editable")
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
staged = []
for upload in files:
name = Path(upload.filename or "input.json").name
if not name.lower().endswith(".json"):
raise fail(415, f"only JSON files are supported: {name}")
target = input_dir / name
content = await upload.read()
target.write_bytes(content)
if _minio_enabled():
_register_object(
task_id,
version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}",
object_key=_input_object_key(task_id, name),
file_name=name,
content_type=upload.content_type or "application/json",
content=content,
created_by=task.get("created_by") or current_user.get("id"),
)
else:
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
(input_dir / name).write_bytes(content)
staged.append({"name": name, "size": len(content)})
store = get_platform_store()
# 标记上传完成
@@ -166,30 +323,12 @@ async def upload_source_files(
)
# 自动转换并导入数据集
try:
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
# 清空旧输出（如果重新上传）
if output_path.exists():
output_path.unlink()
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, output = _convert_from_minio(
task, task.get("created_by") or current_user.get("id")
)
else:
input_count, output_count, output = _convert_from_local(task)
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
@@ -197,12 +336,12 @@ async def upload_source_files(
(input_count, output_count, task_id),
)
# 自动导入数据集
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
size_bytes = len(content.encode("utf-8"))
dataset = store.create_dataset({
"name": task["name"],
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -212,7 +351,20 @@ async def upload_source_files(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or current_user.get("id"),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({
"staged_files": staged,
"auto_converted": True,
@@ -248,28 +400,10 @@ def run_convert(
(task_id,),
)
try:
input_dir = _input_dir(task_id)
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id"))
else:
input_count, output_count, _ = _convert_from_local(task)
# 更新任务状态
with store.connect() as conn:
conn.execute(
@@ -297,11 +431,15 @@ def download_result(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
if _minio_enabled():
return Response(content=output, media_type="application/octet-stream", headers={
"Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}"
})
return FileResponse(
str(output_path),
str(_task_output_path(task)),
media_type="application/octet-stream",
filename=_safe_output_filename(task.get("output_filename")),
)
@@ -319,10 +457,10 @@ def import_as_dataset(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
dataset_name = str(payload.get("name") or task["name"]).strip()
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
size_bytes = len(content.encode("utf-8"))
@@ -331,7 +469,7 @@ def import_as_dataset(
dataset = store.create_dataset({
"name": dataset_name,
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -341,7 +479,20 @@ def import_as_dataset(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({"dataset_id": dataset_id, "name": dataset_name})
@@ -360,11 +511,19 @@ def delete_task(
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
(task_id,),
)
# 清理文件
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
if _minio_enabled():
for item in _task_objects(task_id):
try:
get_object_storage().delete(item["object_key"])
store.update_storage_object(item["id"], {"status": "deleted"})
except Exception:
pass
else:
# 旧兼容数据仍清理本地目录。
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
return ok({"deleted": task_id})