feat: 平台治理与对象存储增强,审批中心与运行日志整合
- 新增 storage/policy.py 落盘策略:按大小/类型决定文件存 MinIO 或内联数据库 - 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects,支持失败回滚 - 算力节点训练产物按版本归档到 MinIO,登记 model_artifacts - 数据转换任务输入输出对象化,支持从 MinIO 读写 - 新增审批中心(申请/我的/策略)、组织与权限、运行日志整合页面 - schema 与 docker 配置、前端路由侧边栏、治理文档同步更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -551,6 +551,16 @@ class PlatformStore:
|
||||
"last_error": "TEXT",
|
||||
},
|
||||
)
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"storage_objects",
|
||||
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
|
||||
)
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"model_artifacts",
|
||||
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
|
||||
)
|
||||
schema_dir = Path(__file__).with_name("sql")
|
||||
for extra in (
|
||||
"002_governance.sql",
|
||||
@@ -562,7 +572,17 @@ class PlatformStore:
|
||||
if extra_path.exists():
|
||||
conn.executescript(extra_path.read_text(encoding="utf-8"))
|
||||
# data_convert_tasks 表补充 created_by 字段(用于数据隔离)
|
||||
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"data_convert_tasks",
|
||||
{
|
||||
"created_by": "TEXT",
|
||||
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
|
||||
"output_storage_object_id": "TEXT",
|
||||
"output_content": "TEXT",
|
||||
},
|
||||
)
|
||||
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
|
||||
# 修复历史数据:将 data_convert_tasks.created_by 回填到关联的 datasets 记录
|
||||
try:
|
||||
conn.execute("""
|
||||
@@ -1260,6 +1280,13 @@ class PlatformStore:
|
||||
raise KeyError(artifact_id)
|
||||
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
|
||||
|
||||
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
|
||||
(storage_object_id, artifact_id),
|
||||
)
|
||||
|
||||
def model_lineage(self, model_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
parents = conn.execute(
|
||||
@@ -2227,7 +2254,7 @@ class PlatformStore:
|
||||
(str(validation_dataset_id),),
|
||||
).fetchall(),
|
||||
]
|
||||
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
|
||||
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
|
||||
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
|
||||
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
|
||||
"dataset_split"
|
||||
@@ -3034,17 +3061,18 @@ class PlatformStore:
|
||||
"""
|
||||
INSERT INTO storage_objects
|
||||
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
|
||||
content_type, checksum_sha256, byte_size, status, created_by, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (resource_type, resource_id, version_id, object_key)
|
||||
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
|
||||
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
|
||||
status=EXCLUDED.status, created_by=EXCLUDED.created_by
|
||||
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
|
||||
""",
|
||||
(
|
||||
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
|
||||
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
|
||||
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
|
||||
json_dumps(payload.get("metadata") or {}),
|
||||
payload.get("created_by"), payload.get("create_time") or utcnow(),
|
||||
),
|
||||
)
|
||||
@@ -3083,6 +3111,13 @@ class PlatformStore:
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def storage_object(self, object_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(object_id)
|
||||
return dict(row)
|
||||
|
||||
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
|
||||
fields = {key: value for key, value in payload.items() if key in allowed}
|
||||
@@ -3758,6 +3793,8 @@ class PlatformStore:
|
||||
actor_id: str | None = None,
|
||||
action: str | None = None,
|
||||
target_type: str | None = None,
|
||||
target_id: str | None = None,
|
||||
keyword: str | None = None,
|
||||
start_time: str | None = None,
|
||||
end_time: str | None = None,
|
||||
limit: int = 50,
|
||||
@@ -3780,6 +3817,13 @@ class PlatformStore:
|
||||
if target_type:
|
||||
clauses.append("target_type=?")
|
||||
params.append(target_type)
|
||||
if target_id:
|
||||
clauses.append("target_id=?")
|
||||
params.append(target_id)
|
||||
if keyword:
|
||||
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
|
||||
pattern = f"%{keyword}%"
|
||||
params.extend([pattern, pattern])
|
||||
if start_time:
|
||||
clauses.append("time>=?")
|
||||
params.append(start_time)
|
||||
|
||||
Reference in New Issue
Block a user