YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与对象存储增强，审批中心与运行日志整合

Browse Source 
- 新增 storage/policy.py 落盘策略：按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects，支持失败回滚
- 算力节点训练产物按版本归档到 MinIO，登记 model_artifacts
- 数据转换任务输入输出对象化，支持从 MinIO 读写
- 新增审批中心（申请/我的/策略）、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
backend/app/db/platform_store.py
View File

@@ -551,6 +551,16 @@ class PlatformStore:
"last_error": "TEXT",
},
)
self._ensure_columns(
conn,
"storage_objects",
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
)
self._ensure_columns(
conn,
"model_artifacts",
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
)
schema_dir = Path(__file__).with_name("sql")
for extra in (
"002_governance.sql",
@@ -562,7 +572,17 @@ class PlatformStore:
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
# data_convert_tasks 表补充 created_by 字段（用于数据隔离）
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
self._ensure_columns(
conn,
"data_convert_tasks",
{
"created_by": "TEXT",
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
"output_storage_object_id": "TEXT",
"output_content": "TEXT",
},
)
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
# 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录
try:
conn.execute("""
@@ -1260,6 +1280,13 @@ class PlatformStore:
raise KeyError(artifact_id)
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
with self.connect() as conn:
conn.execute(
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
(storage_object_id, artifact_id),
)
def model_lineage(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
parents = conn.execute(
@@ -2227,7 +2254,7 @@ class PlatformStore:
(str(validation_dataset_id),),
).fetchall(),
]
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
"dataset_split"
@@ -3034,17 +3061,18 @@ class PlatformStore:
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
json_dumps(payload.get("metadata") or {}),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
@@ -3083,6 +3111,13 @@ class PlatformStore:
).fetchall()
return [dict(row) for row in rows]
def storage_object(self, object_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
@@ -3758,6 +3793,8 @@ class PlatformStore:
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
keyword: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
@@ -3780,6 +3817,13 @@ class PlatformStore:
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if target_id:
clauses.append("target_id=?")
params.append(target_id)
if keyword:
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
pattern = f"%{keyword}%"
params.extend([pattern, pattern])
if start_time:
clauses.append("time>=?")
params.append(start_time)

11
backend/app/db/sql/000_full_init.sql
View File

@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
path TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum_sha256 TEXT,
storage_object_id TEXT,
storage_backend TEXT NOT NULL DEFAULT 'minio',
metadata TEXT NOT NULL,
compute_job_id TEXT,
create_time TEXT NOT NULL
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
metadata TEXT NOT NULL DEFAULT '{}',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
input_count INTEGER NOT NULL DEFAULT 0,
output_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
output_content TEXT,
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员