YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与对象存储增强，审批中心与运行日志整合

Browse Source 
- 新增 storage/policy.py 落盘策略：按大小/类型决定文件存 MinIO 或内联数据库
- 数据处理源文件与生成结果写入 MinIO 并登记 storage_objects，支持失败回滚
- 算力节点训练产物按版本归档到 MinIO，登记 model_artifacts
- 数据转换任务输入输出对象化，支持从 MinIO 读写
- 新增审批中心（申请/我的/策略）、组织与权限、运行日志整合页面
- schema 与 docker 配置、前端路由侧边栏、治理文档同步更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-19 16:10:02 +08:00
parent 81c2f85c3a
commit 78e3baa9ba
30 changed files with 1994 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
backend/app/api/v1/endpoints/data_process.py
View File

@@ -35,6 +35,8 @@ from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -85,6 +87,8 @@ from app.modules.data_process.store import (
new_id,
repeat_task_id,
)
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
@@ -224,9 +228,38 @@ def _commit_source_batch(
staged: list[StagedSourceObject],
) -> list[dict[str, Any]]:
storage.publish(staged)
storage_object_ids: list[str] = []
try:
# The source reference remains in the task schema for compatibility,
# while storage_objects provides the authoritative MinIO index.
if get_settings().minio_enabled:
for item in prepared:
reference = str(item.get("storage_object_id") or "")
if not reference.startswith("minio://"):
continue
object_key = storage.object_key(reference)
metadata = get_object_storage().stat(object_key)
object_row = get_platform_store().create_storage_object({
"resource_type": "data_process_source",
"resource_id": task_id,
"version_id": str(item.get("id") or new_id("dpsf")),
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": item.get("name"),
"content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"),
"byte_size": metadata.get("byte_size") or item.get("raw_size") or 0,
"checksum_sha256": item.get("checksum_sha256"),
"status": "available",
"created_by": item.get("created_by"),
})
storage_object_ids.append(str(object_row["id"]))
return store.add_source_files(task_id, prepared)
except Exception:
for object_id in storage_object_ids:
try:
get_platform_store().update_storage_object(object_id, {"status": "deleted"})
except Exception:
pass
for item in staged:
try:
storage.delete(item.reference)
@@ -774,6 +807,25 @@ def _run_generation(
duplicate_count=duplicate_count,
error_count=error_count,
)
result_bytes = "".join(
structured_json_dumps(item) + "\n" for item in accepted
).encode("utf-8")
if should_store_in_minio(len(result_bytes)):
result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl"
uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl")
get_platform_store().create_storage_object({
"resource_type": "data_process_result",
"resource_id": task_id,
"version_id": generation_run_id,
"bucket": uploaded["bucket"],
"object_key": result_key,
"file_name": f"{generation_run_id}.jsonl",
"content_type": "application/jsonl",
"byte_size": len(result_bytes),
"checksum_sha256": hashlib.sha256(result_bytes).hexdigest(),
"status": "available",
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
@@ -933,7 +985,7 @@ def _repeat_file_copies(
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
if old_reference.startswith(("local://data-process/", "minio://data-process/")):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,

321
backend/app/api/v1/endpoints/platform.py
View File

@@ -23,8 +23,9 @@ from app.core.audit import audit_log, AuditActions
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
@@ -100,6 +101,106 @@ async def _wait_for_object_storage() -> None:
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
def _store_json_snapshot(
store: Any,
resource_type: str,
resource_id: str,
version_id: str,
object_key: str,
payload: dict[str, Any],
created_by: str | None = None,
) -> dict[str, Any]:
"""Persist non-secret task/model parameters as an auditable MinIO snapshot."""
sanitized = {
key: value
for key, value in payload.items()
if key not in {"api_key", "secret_key", "password", "token", "access_token"}
}
raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
# Task/evaluation payloads are already persisted in PostgreSQL. Avoid an
# extra MinIO round trip for small non-secret parameter snapshots.
if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"):
return {}
uploaded = get_object_storage().put_bytes(object_key, raw, "application/json")
return store.create_storage_object({
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": Path(object_key).name,
"content_type": "application/json",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
"created_by": created_by,
})
def _dataset_file_bytes(store: Any, file_id: str) -> bytes:
"""Return the canonical dataset bytes, lazily indexing legacy DB content."""
row = store.dataset_file(file_id)
if not get_settings().minio_enabled:
return str(row.get("content") or "").encode("utf-8")
storage_object = None
object_id = str(row.get("storage_object_id") or "")
if object_id:
try:
storage_object = store.storage_object(object_id)
except KeyError:
storage_object = None
if storage_object and storage_object.get("status") == "available":
try:
return get_object_storage().get_bytes(storage_object["object_key"])
except Exception as exc: # noqa: BLE001 - expose storage outage to callers
raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc
# Compatibility migration for files created before MinIO was enabled.
raw = str(row.get("content") or "").encode("utf-8")
if not raw:
raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}")
# Small files intentionally remain database-backed. They can still be
# copied to a compute node directly when a task needs them.
if not should_store_in_minio(len(raw), file_format=row.get("file_format")):
return raw
object_key = (
f"datasets/{row['dataset_id']}/versions/"
f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}"
)
uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream")
created = store.create_storage_object({
"resource_type": "dataset",
"resource_id": str(row["dataset_id"]),
"version_id": str(row.get("active_version_id") or row["id"]),
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": row.get("name"),
"content_type": "application/octet-stream",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(row["id"]), created["id"])
return raw
def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes:
row = store.dataset_file(file_id)
try:
version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id)
except StopIteration as exc:
raise KeyError(version_id) from exc
object_id = str(version.get("storage_object_id") or "")
if get_settings().minio_enabled and object_id:
try:
obj = store.storage_object(object_id)
return get_object_storage().get_bytes(obj["object_key"])
except KeyError:
pass
return _dataset_file_bytes(store, file_id)
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job.
@@ -130,18 +231,63 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id:
if not get_settings().minio_enabled or not resource_id:
return None
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects and resource_type in {"model", "trained_model"}:
resource = None
if resource_type == "model":
try:
resource = store.model(resource_id)
except KeyError:
try:
resource = store.model_by_name(resource_id)
except KeyError:
resource = next((item for item in store.models() if item.get("path") == resource_id), None)
else:
resource = next(
(item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id),
None,
)
source_path = str(
(resource or {}).get("path")
or (resource or {}).get("merged_path")
or (resource or {}).get("artifact_dir")
or ""
)
resolved_id = str((resource or {}).get("id") or resource_id)
if source_path and resolved_id:
client = ComputeNodeClient(node["api_base_url"], timeout=900)
await _archive_node_directory(
store,
client,
node,
source_path,
resource_type,
resolved_id,
"legacy-import",
f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}",
)
resource_id = resolved_id
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
return None
client = ComputeNodeClient(node["api_base_url"], timeout=900)
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
for obj in objects:
object_key = str(obj["object_key"])
marker = f"{root_name}/{resource_id}/versions/"
relative_name = Path(str(obj.get("file_name") or object_key)).name
if marker in object_key:
suffix = object_key.split(marker, 1)[1]
if "/" in suffix:
suffix = suffix.split("/", 1)[1]
if suffix:
relative_name = suffix
await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": get_object_storage().presigned_get(obj["object_key"]),
"download_url": get_object_storage().presigned_get(object_key),
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
"relative_path": f"{root_name}/{resource_id}/{relative_name}",
})
return f"/data/yg-ft/{root_name}/{resource_id}"
@@ -397,8 +543,23 @@ async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[st
if not preflight["valid"]:
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
raise RuntimeError(f"preflight failed: {errors}")
payload = {**payload, "compute_node_id": preflight["node"]["id"]}
prepared_job_payload = preflight.get("job_payload") or {}
payload = {
**payload,
"compute_node_id": preflight["node"]["id"],
"prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"),
}
task = store.start_task(payload)
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"fine_tune",
str(task["id"]),
str(task["id"]),
f"training/{task['id']}/versions/{task['id']}/training-config.json",
task,
task.get("created_by"),
)
if get_settings().compute_mode == "simulator":
return task
node, job_payload = store.build_compute_job_payload(task["id"])
@@ -454,6 +615,20 @@ async def _fine_tune_preflight_with_job_payload(
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "")
base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "")
base_model = None
if base_model_id:
try:
base_model = store.model(base_model_id)
except KeyError:
base_model = None
if base_model is None:
base_model = next((item for item in store.models() if item.get("path") == base_model_path), None)
if base_model:
prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node)
if prepared_model:
job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model}
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
sync_errors.append(f"shared storage health check failed: {exc}")
if get_settings().compute_mode == "simulator":
@@ -1091,8 +1266,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict =
@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str) -> dict[str, Any]:
try:
row = get_platform_store().dataset_file(file_id)
return ok({"content": row["content"]})
store = get_platform_store()
content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace")
return ok({"content": content})
except KeyError:
raise fail(404, "dataset file not found")
@@ -1121,7 +1297,8 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An
version = next((item for item in versions if item["id"] == version_id), None)
if not version:
raise KeyError(version_id)
return ok({"version": version, "content": row["content"]})
content = _dataset_version_bytes(get_platform_store(), file_id, version_id)
return ok({"version": version, "content": content.decode("utf-8", errors="replace")})
except KeyError:
raise fail(404, "dataset version not found")
@@ -1210,7 +1387,7 @@ async def _sync_training_dataset_to_compute_node(
) -> list[dict[str, Any]]:
if get_settings().minio_enabled:
files = store.training_dataset_files(dataset_id)
object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {}
object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {}
resource_ids = {str(dataset_id)} | {
str(item.get("dataset_id"))
for item in files
@@ -1219,35 +1396,58 @@ async def _sync_training_dataset_to_compute_node(
for resource_id in resource_ids:
for obj in store.storage_objects_for_resource("dataset", resource_id):
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
object_by_resource_name[(resource_id, file_name)] = obj
object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj
results: list[dict[str, Any]] = []
client = ComputeNodeClient(node["api_base_url"])
for item in files:
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
item_dataset_id = str(item.get("dataset_id") or dataset_id)
obj = object_by_resource_name.get((item_dataset_id, target_name))
version_id = str(item.get("active_version_id") or item["id"])
obj = object_by_resource_name.get((item_dataset_id, target_name, version_id))
if not obj and item.get("content"):
# 兼容 MinIO 接入前已经发布的数据处理数据集：
# 预检时用数据库正文补建对象，避免要求用户重新处理数据集。
# 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建
# MinIO 对象，小文件直接从数据库正文同步到目标节点。
raw = str(item.get("content") or "").encode("utf-8")
version_id = str(item.get("active_version_id") or item["id"])
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
obj = store.create_storage_object({
"resource_type": "dataset",
"resource_id": item_dataset_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": target_name,
"content_type": "application/jsonl",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
if should_store_in_minio(len(raw)):
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
obj = store.create_storage_object({
"resource_type": "dataset",
"resource_id": item_dataset_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": target_name,
"content_type": "application/jsonl",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
else:
result = await client.upload_file(
target_name,
raw,
f"datasets/{dataset_id}/{target_name}",
resource_type="dataset",
resource_id=dataset_id,
)
store.upsert_resource_replica(
node["id"], "dataset", dataset_id, str(result.get("local_path") or "")
)
results.append({
"node_id": node["id"],
"node_code": node.get("code"),
"file_id": item.get("id"),
"name": target_name,
"local_path": result.get("local_path"),
"byte_size": result.get("byte_size"),
"checksum_sha256": result.get("checksum_sha256"),
"storage_backend": "database",
})
continue
if not obj:
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
raise RuntimeError(f"dataset file is not available: {target_name}")
url = get_object_storage().presigned_get(obj["object_key"])
result = await client.prepare_cache({
"resource_id": dataset_id,
@@ -1263,7 +1463,13 @@ async def _sync_training_dataset_to_compute_node(
dataset_id,
str(result.get("local_path") or ""),
)
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
results.append({
**result,
"file_id": item.get("id"),
"name": target_name,
"node_id": node["id"],
"storage_backend": "minio",
})
return results
if not dataset_id:
raise RuntimeError("train_dataset_id is required")
@@ -1328,7 +1534,11 @@ async def upload_dataset_files(
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file)
pending_sync.append((created_file["id"], created_file["name"], raw))
if get_settings().minio_enabled:
if should_store_in_minio(
len(raw),
content_type=file.content_type,
file_format=Path(created_file["name"]).suffix,
):
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
storage_object = get_platform_store().create_storage_object({
@@ -1372,7 +1582,10 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
full_file = store.dataset_file(str(item["id"]))
except KeyError:
continue
files.append({**item, "content": full_file.get("content") or ""})
files.append({
**item,
"content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"),
})
if not files:
raise fail(404, "dataset has no downloadable files")
@@ -1413,8 +1626,12 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
row = get_platform_store().dataset_file(file_id)
return PlainTextResponse(row["content"], media_type="text/plain")
store = get_platform_store()
row = store.dataset_file(file_id)
if str(row.get("dataset_id")) != str(dataset_id):
raise fail(404, "dataset file not found")
content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id)
return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain")
@router.get("/dataset-manage")
@@ -1880,12 +2097,26 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
# 1. Create eval task record
payload.setdefault("created_by", current_user.get("id"))
task = store.create_eval_task({**payload, "status": "pending"})
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"eval",
str(task["id"]),
str(task["id"]),
f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json",
payload,
current_user.get("id"),
)
# 2. Resolve model path (supports both regular models and trained models)
model_id = str(payload.get("model_id", ""))
model_path = ""
adapter_path = payload.get("adapter_path", "")
model_node_id = ""
ds_files: list[dict[str, Any]] = []
model_resource_type = "model"
model_resource_id = model_id
adapter_resource_id = ""
try:
db_model = store.model(model_id)
model_path = db_model.get("path", "")
@@ -1894,6 +2125,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
# Try trained_models table (IDs prefixed with tm_)
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
if trained:
model_resource_type = "trained_model" if trained.get("merged") else "model"
model_resource_id = trained.get("id") or model_id
adapter_resource_id = trained.get("id") or ""
model_node_id = trained.get("compute_node_id") or ""
merged_path = trained.get("merged_path", "")
base_path = trained.get("base_model_path", "")
@@ -1907,6 +2141,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
adapter_path = merged_path
else:
model_path = merged_path or base_path
if model_resource_type == "model":
base_model = next((item for item in store.models() if item.get("path") == base_path), None)
model_resource_id = str((base_model or {}).get("id") or base_path)
if not model_path:
store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
@@ -1979,6 +2216,22 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node)
if prepared_model:
model_path = prepared_model
if adapter_resource_id and model_resource_type == "model":
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node)
if prepared_adapter:
adapter_path = prepared_adapter
dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id)
if dataset_sync:
dataset_path = str(dataset_sync[0].get("local_path") or dataset_path)
except Exception as exc:
store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"})
return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})
node_gpus = {
int(item.get("id", item.get("gpu_index", -1))): item
for item in store.gpus()

13
backend/app/core/audit.py
View File

@@ -61,6 +61,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -87,6 +88,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -136,6 +138,7 @@ def _build_detail(template: str, kwargs: dict) -> str:
def _record_audit(
action: str,
actor_id: Optional[str],
target_type: str,
target_id: Optional[str],
detail: str,
@@ -149,6 +152,7 @@ def _record_audit(
store = get_platform_store()
store.record_audit(
action=action,
actor_id=actor_id,
target_type=target_type or None,
target_id=target_id,
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
@@ -157,6 +161,15 @@ def _record_audit(
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
def _extract_actor_id(kwargs: dict) -> Optional[str]:
"""从 FastAPI 注入的当前用户中提取操作人 ID。"""
for key in ("current_user", "user"):
value = kwargs.get(key)
if isinstance(value, dict) and value.get("id"):
return str(value["id"])
return None
# ==================== 预定义的审计操作常量 ====================
class AuditActions:

3
backend/app/core/config.py
View File

@@ -68,6 +68,9 @@ class Settings:
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
minio_secure: bool = _bool_env("MINIO_SECURE", False)
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
# MinIO round trips. Larger files remain the shared canonical objects.
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")

54
backend/app/db/platform_store.py
View File

@@ -551,6 +551,16 @@ class PlatformStore:
"last_error": "TEXT",
},
)
self._ensure_columns(
conn,
"storage_objects",
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
)
self._ensure_columns(
conn,
"model_artifacts",
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
)
schema_dir = Path(__file__).with_name("sql")
for extra in (
"002_governance.sql",
@@ -562,7 +572,17 @@ class PlatformStore:
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
# data_convert_tasks 表补充 created_by 字段（用于数据隔离）
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
self._ensure_columns(
conn,
"data_convert_tasks",
{
"created_by": "TEXT",
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
"output_storage_object_id": "TEXT",
"output_content": "TEXT",
},
)
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
# 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录
try:
conn.execute("""
@@ -1260,6 +1280,13 @@ class PlatformStore:
raise KeyError(artifact_id)
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
with self.connect() as conn:
conn.execute(
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
(storage_object_id, artifact_id),
)
def model_lineage(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
parents = conn.execute(
@@ -2227,7 +2254,7 @@ class PlatformStore:
(str(validation_dataset_id),),
).fetchall(),
]
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
"dataset_split"
@@ -3034,17 +3061,18 @@ class PlatformStore:
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
json_dumps(payload.get("metadata") or {}),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
@@ -3083,6 +3111,13 @@ class PlatformStore:
).fetchall()
return [dict(row) for row in rows]
def storage_object(self, object_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
@@ -3758,6 +3793,8 @@ class PlatformStore:
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
keyword: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
@@ -3780,6 +3817,13 @@ class PlatformStore:
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if target_id:
clauses.append("target_id=?")
params.append(target_id)
if keyword:
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
pattern = f"%{keyword}%"
params.extend([pattern, pattern])
if start_time:
clauses.append("time>=?")
params.append(start_time)

11
backend/app/db/sql/000_full_init.sql
View File

@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
path TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum_sha256 TEXT,
storage_object_id TEXT,
storage_backend TEXT NOT NULL DEFAULT 'minio',
metadata TEXT NOT NULL,
compute_job_id TEXT,
create_time TEXT NOT NULL
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
metadata TEXT NOT NULL DEFAULT '{}',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
input_count INTEGER NOT NULL DEFAULT 0,
output_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
output_content TEXT,
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员

128
backend/app/modules/compute_gateway/sync.py
View File

@@ -2,15 +2,75 @@ from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.storage.minio_store import get_object_storage
# starting 状态允许的最大轮询次数（约 40 * 3s ≈ 2 分钟），超过即判定节点不可达
MAX_STARTING_ATTEMPTS = 40
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
node: dict[str, Any],
source_path: str,
resource_type: str,
resource_id: str,
version_id: str,
object_prefix: str,
) -> list[dict[str, Any]]:
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
for item in listing.get("items") or []:
item_relative = str(item.get("relative_path") or "")
if item.get("type") == "directory":
queue.append(item_relative)
continue
path = str(item.get("path") or "")
if not path:
continue
try:
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
archived.append(
store.create_storage_object(
{
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": relative_file,
"content_type": "application/octet-stream",
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
"checksum_sha256": result.get("checksum_sha256") or "",
"status": "available",
}
)
)
return archived
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -139,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
updated_task = store.apply_compute_job(task["id"], job)
if (
get_settings().minio_enabled
and job.get("status") == "completed"
and job.get("output_dir")
):
trained_model = next(
(
item
for item in store.trained_models()
if item.get("name")
== (task.get("output_model_name") or f"{task.get('name')}-lora")
),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
@@ -151,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
try:
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
payload = (store.compute_job(record["id"]).get("payload") or {})
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
if trained_model_id:
trained_model = next(
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
ComputeNodeClient(node["api_base_url"], timeout=900),
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or record["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
@@ -175,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"eval",
str(eval_task["id"]),
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
# 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001

311
backend/app/modules/data_convert/router.py
View File

@@ -1,17 +1,21 @@
from __future__ import annotations
import json
import hashlib
import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
@@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path:
return _task_dir(task_id) / "output"
def _minio_enabled() -> bool:
return bool(get_settings().minio_enabled)
def _input_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/input/{Path(name).name}"
def _output_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/output/{Path(name).name}"
def _task_objects(task_id: str) -> list[dict[str, Any]]:
return get_platform_store().storage_objects_for_resource("data_convert", task_id)
def _register_object(
task_id: str,
*,
version_id: str,
object_key: str,
file_name: str,
content_type: str,
content: bytes,
created_by: str | None,
) -> dict[str, Any]:
storage = get_object_storage()
uploaded = storage.put_bytes(object_key, content, content_type)
return get_platform_store().create_storage_object(
{
"resource_type": "data_convert",
"resource_id": task_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": file_name,
"content_type": content_type,
"byte_size": len(content),
"checksum_sha256": hashlib.sha256(content).hexdigest(),
"status": "available",
"created_by": created_by,
}
)
def _input_objects(task_id: str) -> list[dict[str, Any]]:
prefix = f"data-convert/{task_id}/input/"
return sorted(
[item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)],
key=lambda item: str(item.get("file_name") or item.get("object_key") or ""),
)
def _output_object(task: dict[str, Any]) -> dict[str, Any] | None:
key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename")))
return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None)
def _read_output(task: dict[str, Any]) -> bytes | None:
if _minio_enabled():
item = _output_object(task)
if item:
return get_object_storage().get_bytes(item["object_key"])
inline = task.get("output_content")
return str(inline).encode("utf-8") if inline is not None else None
path = _task_output_path(task)
return path.read_bytes() if path.exists() else None
def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]:
output_name = _safe_output_filename(task.get("output_filename"))
output_lines: list[str] = []
input_count = 0
output_count = 0
for item in _input_objects(task["id"]):
input_count += 1
raw = get_object_storage().get_bytes(item["object_key"])
data = json.loads(raw.decode("utf-8"))
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {item.get('file_name')}")
for record in records:
output_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
output = "".join(output_lines).encode("utf-8")
store = get_platform_store()
with store.connect() as conn:
if should_store_in_minio(len(output)):
output_object = _register_object(
task["id"],
version_id="output",
object_key=_output_object_key(task["id"], output_name),
file_name=output_name,
content_type="application/jsonl",
content=output,
created_by=created_by,
)
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s",
(output_object["id"], task["id"]),
)
else:
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s",
(output.decode("utf-8"), task["id"]),
)
return input_count, output_count, output
def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]:
input_dir = _input_dir(task["id"])
output_dir = _output_dir(task["id"])
input_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
output_path.unlink(missing_ok=True)
input_count = 0
output_count = 0
with output_path.open("w", encoding="utf-8") as output_file:
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
data = json.loads(json_file.read_text(encoding="utf-8"))
records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None
if records is None:
raise ValueError(f"JSON must be object or array: {json_file.name}")
for record in records:
output_file.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
return input_count, output_count, output_path.read_bytes()
@router.get("")
def list_tasks(
page: int = 1,
@@ -109,9 +249,10 @@ def create_task(
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
)
# 创建目录
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
return ok(_get_task(task_id))
@@ -123,13 +264,19 @@ def get_task(
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
# 附加输入文件列表
input_dir = _input_dir(task_id)
# 附加输入文件列表；旧任务没有对象记录时继续读取本地兼容目录。
files = []
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
if _minio_enabled():
files = [
{"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0}
for item in _input_objects(task_id)
]
else:
input_dir = _input_dir(task_id)
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
task["input_files"] = files
return ok(task)
@@ -146,16 +293,26 @@ async def upload_source_files(
raise fail(404, "task not found")
if task["status"] not in ("pending", "uploaded"):
raise fail(400, "task is not editable")
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
staged = []
for upload in files:
name = Path(upload.filename or "input.json").name
if not name.lower().endswith(".json"):
raise fail(415, f"only JSON files are supported: {name}")
target = input_dir / name
content = await upload.read()
target.write_bytes(content)
if _minio_enabled():
_register_object(
task_id,
version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}",
object_key=_input_object_key(task_id, name),
file_name=name,
content_type=upload.content_type or "application/json",
content=content,
created_by=task.get("created_by") or current_user.get("id"),
)
else:
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
(input_dir / name).write_bytes(content)
staged.append({"name": name, "size": len(content)})
store = get_platform_store()
# 标记上传完成
@@ -166,30 +323,12 @@ async def upload_source_files(
)
# 自动转换并导入数据集
try:
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
# 清空旧输出（如果重新上传）
if output_path.exists():
output_path.unlink()
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, output = _convert_from_minio(
task, task.get("created_by") or current_user.get("id")
)
else:
input_count, output_count, output = _convert_from_local(task)
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
@@ -197,12 +336,12 @@ async def upload_source_files(
(input_count, output_count, task_id),
)
# 自动导入数据集
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
size_bytes = len(content.encode("utf-8"))
dataset = store.create_dataset({
"name": task["name"],
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -212,7 +351,20 @@ async def upload_source_files(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or current_user.get("id"),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({
"staged_files": staged,
"auto_converted": True,
@@ -248,28 +400,10 @@ def run_convert(
(task_id,),
)
try:
input_dir = _input_dir(task_id)
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id"))
else:
input_count, output_count, _ = _convert_from_local(task)
# 更新任务状态
with store.connect() as conn:
conn.execute(
@@ -297,11 +431,15 @@ def download_result(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
if _minio_enabled():
return Response(content=output, media_type="application/octet-stream", headers={
"Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}"
})
return FileResponse(
str(output_path),
str(_task_output_path(task)),
media_type="application/octet-stream",
filename=_safe_output_filename(task.get("output_filename")),
)
@@ -319,10 +457,10 @@ def import_as_dataset(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
dataset_name = str(payload.get("name") or task["name"]).strip()
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
size_bytes = len(content.encode("utf-8"))
@@ -331,7 +469,7 @@ def import_as_dataset(
dataset = store.create_dataset({
"name": dataset_name,
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -341,7 +479,20 @@ def import_as_dataset(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({"dataset_id": dataset_id, "name": dataset_name})
@@ -360,11 +511,19 @@ def delete_task(
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
(task_id,),
)
# 清理文件
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
if _minio_enabled():
for item in _task_objects(task_id):
try:
get_object_storage().delete(item["object_key"])
store.update_storage_object(item["id"], {"status": "deleted"})
except Exception:
pass
else:
# 旧兼容数据仍清理本地目录。
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
return ok({"deleted": task_id})

144
backend/app/modules/data_process/storage.py
View File

@@ -1,4 +1,4 @@
"""数据处理原始源文件的受控本地对象存储。"""
"""数据处理源文件的受控暂存与分层对象存储。"""
from __future__ import annotations
@@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath
from typing import Iterable, Iterator
from urllib.parse import quote, unquote, urlsplit
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
class DataProcessStorageError(ValueError):
"""本地对象引用或文件系统状态不安全。"""
@@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str:
class LocalDataProcessStorage:
"""只允许访问配置根目录下的版本化原始文件。"""
"""Stage locally, but publish and read authoritative source files from MinIO."""
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
configured = Path(root) if root is not None else _configured_storage_root()
@@ -132,10 +135,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -168,6 +168,18 @@ class LocalDataProcessStorage:
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
if self._is_minio_reference(source_reference):
content = self.read(source_reference)
if content is None:
raise DataProcessStorageError("original source object is not available")
return self.stage_bytes(
batch_id=batch_id,
task_id=task_id,
source_file_id=source_file_id,
version=version,
name=basename,
content=content,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
@@ -193,10 +205,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -212,14 +221,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("duplicate staged source object")
seen_temporary_paths.add(item._temporary_path)
for item in staged:
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
if self._is_minio_reference(item.reference):
content = item._temporary_path.read_bytes()
get_object_storage().put_bytes(
self.object_key(item.reference),
content,
"application/octet-stream",
)
elif not item.reference.startswith("db://data-process/"):
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
self._fsync_directory(final_path.parent)
published.append(item)
item._temporary_path.unlink()
self._fsync_directory(final_path.parent)
except Exception:
for item in reversed(published):
try:
@@ -258,7 +275,13 @@ class LocalDataProcessStorage:
raise first_error
def read(self, reference: str) -> bytes | None:
"""读取 local 引用；旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。"""
"""Read a MinIO object or legacy local reference."""
if self._is_minio_reference(reference):
try:
return get_object_storage().get_bytes(self.object_key(reference))
except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
@@ -276,6 +299,11 @@ class LocalDataProcessStorage:
) -> int | None:
"""返回受控 local 对象大小；旧 ``db://`` 对象没有原始文件。"""
if self._is_minio_reference(reference):
try:
return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0)
except Exception as exc: # noqa: BLE001
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return None
@@ -301,6 +329,15 @@ class LocalDataProcessStorage:
) -> Iterator[bytes]:
"""按范围流式读取原始文件，避免 PDF 预览把大文件整体载入内存。"""
if self._is_minio_reference(reference):
content = self.read(reference) or b""
if start < 0 or expected_size != len(content) or start > expected_size:
raise DataProcessStorageError("source object size does not match metadata")
remaining = expected_size - start if length is None else length
if remaining < 0 or start + remaining > expected_size:
raise DataProcessStorageError("invalid source byte range")
yield content[start : start + remaining]
return
relative_path = self._relative_from_reference(reference)
if relative_path is None:
raise DataProcessStorageError("original source object is not available")
@@ -337,6 +374,12 @@ class LocalDataProcessStorage:
) -> bool:
"""校验 local 引用归属；旧 ``db://`` 引用无需文件系统处理。"""
if self._is_minio_reference(reference):
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
return True
if str(reference or "").startswith("db://data-process/"):
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return True
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -382,6 +425,19 @@ class LocalDataProcessStorage:
) -> bool:
"""删除受控 local 对象；旧 ``db://`` 引用保持不变。"""
if self._is_minio_reference(reference):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
get_object_storage().delete(self.object_key(reference))
return True
if str(reference or "").startswith("db://data-process/"):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return False
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -426,7 +482,7 @@ class LocalDataProcessStorage:
if reference.startswith("db://"):
return None
parsed = urlsplit(reference)
if parsed.scheme != "local" or parsed.netloc != "data-process":
if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process":
raise DataProcessStorageError("unsupported source storage reference")
if parsed.query or parsed.fragment or "\\" in parsed.path:
raise DataProcessStorageError("unsafe source storage reference")
@@ -460,6 +516,39 @@ class LocalDataProcessStorage:
basename = _safe_basename(decoded[3])
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
@staticmethod
def _is_minio_reference(reference: str) -> bool:
return str(reference or "").startswith("minio://data-process/")
@staticmethod
def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str:
scheme = "minio" if get_settings().minio_enabled else "local"
return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
@staticmethod
def object_key(reference: str) -> str:
parsed = urlsplit(reference)
if parsed.scheme != "minio" or parsed.netloc != "data-process":
raise DataProcessStorageError("reference is not a MinIO source object")
return "data-process/" + parsed.path.lstrip("/")
def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None:
relative = self._relative_from_reference(reference)
if relative is None:
raise DataProcessStorageError("invalid MinIO source reference")
self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id)
@staticmethod
def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None:
parsed = urlsplit(reference)
parts = parsed.path.lstrip("/").split("/")
if parsed.netloc != "data-process" or len(parts) != 3:
raise DataProcessStorageError("invalid database source reference")
expected_task_id = _safe_component(task_id, "expected task id")
expected_source_file_id = _safe_component(source_file_id, "expected source file id")
if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1":
raise DataProcessStorageError("source storage object owner mismatch")
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
if relative_path.is_absolute() or any(
part in {"", ".", ".."} for part in relative_path.parts
@@ -479,9 +568,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("invalid staged source object")
if self._issued_staged_objects.get(item._temporary_path) is not item:
raise DataProcessStorageError("staged source object was not issued by this storage")
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
if item.reference.startswith("db://data-process/"):
parsed = urlsplit(item.reference)
parts = parsed.path.lstrip("/").split("/")
expected = item._relative_path.parts[:3]
if (
parsed.netloc != "data-process"
or parsed.query
or parsed.fragment
or len(parts) != 3
or tuple(parts) != expected
):
raise DataProcessStorageError("staged source object reference mismatch")
else:
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
staging_root = self._root / ".staging"
try:
relative_temporary = item._temporary_path.relative_to(staging_root)

7
backend/app/modules/data_process/store/base.py
View File

@@ -247,14 +247,19 @@ def _source_storage_descriptor(
or f"db://data-process/{task_id}/{file_id}/v1"
)
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
expected_local_prefix
):
storage_backend = "local"
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
expected_minio_prefix
):
storage_backend = "minio"
elif storage_object_id == expected_database_reference:
storage_backend = "database"
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
raise DataProcessStoreError("source storage object owner mismatch")
else:
raise DataProcessStoreError("unsupported source storage object reference")

24
backend/app/modules/data_process/store/datasets.py
View File

@@ -11,6 +11,7 @@ import psycopg
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from .base import (
StoreBase,
@@ -168,7 +169,11 @@ class DatasetsMixin:
split_name: assignments.count(split_name) for split_name in split_order
}
split_specs: list[dict[str, Any]] = []
use_minio = bool(get_settings().minio_enabled)
# Explicit local is retained for old callers/tests that request the
# legacy backend; all normal platform requests default to MinIO.
allow_minio = bool(get_settings().minio_enabled) and str(
payload.get("storage_type") or "minio"
).lower() != "local"
for split_name in split_order:
split_records = [
(source_row, record)
@@ -193,12 +198,15 @@ class DatasetsMixin:
"storage_object_id": (
f"db://data-process/{task_id}/{file_id}/v1"
),
"store_in_minio": allow_minio and should_store_in_minio(
len(raw), content_type="application/jsonl", file_format="jsonl"
),
}
)
source_result_ids = [row["id"] for row in rows]
common_metadata = {
"source": "data_process",
"storage_backend": "minio" if use_minio else "database",
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
"source_task_id": task_id,
"output_type": _task_output_type(task),
"reasoning_detail": _task_reasoning_detail(task),
@@ -273,7 +281,7 @@ class DatasetsMixin:
split_name = str(spec["split"])
dataset_id = dataset_ids[split_name]
storage_object_id = str(spec["storage_object_id"])
if use_minio:
if spec["store_in_minio"]:
file_name = f"{base_dataset_name}.{split_name}.jsonl"
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
uploaded = get_object_storage().put_bytes(
@@ -356,7 +364,7 @@ class DatasetsMixin:
(
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
f"{len(spec['raw'])} B",
len(spec["raw"]),
len(spec["records"]),
@@ -386,7 +394,7 @@ class DatasetsMixin:
dataset_id,
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
task_id,
task_id,
f"{len(spec['raw'])} B",
@@ -405,7 +413,11 @@ class DatasetsMixin:
),
).fetchone()
file_metadata = {**dataset_metadata, "file_split": split_name}
file_metadata = {
**dataset_metadata,
"file_split": split_name,
"storage_backend": "minio" if spec["store_in_minio"] else "database",
}
version = {
"id": spec["version_id"],
"version_no": 1,

14
backend/app/modules/data_process/store/source_files.py
View File

@@ -137,7 +137,7 @@ class SourceFilesMixin:
payload["record_count"],
payload["file_format"],
payload["checksum_sha256"],
payload["content"],
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
str(payload["content"])[:2000],
json_dumps(metadata_payload),
task.get("tenant_id"),
@@ -223,7 +223,17 @@ class SourceFilesMixin:
).fetchone()
if not row:
raise NotFoundError("source file not found")
return _decode_row(row) or {}
decoded = _decode_row(row) or {}
# New source files keep only a preview in PostgreSQL. Load the
# authoritative body from MinIO on demand for existing processing code.
reference = str(decoded.get("storage_object_id") or "")
if include_content and not decoded.get("content") and reference.startswith("minio://"):
from app.modules.data_process.storage import get_data_process_storage
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
"utf-8", errors="replace"
)
return decoded
def source_content_window(
self, task_id: str, file_id: str, offset: int, limit: int

59
backend/app/modules/storage/minio_store.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import timedelta
from functools import lru_cache
from io import BytesIO
from collections.abc import Iterator
from typing import Any
from minio import Minio
@@ -53,6 +54,64 @@ class MinioObjectStorage:
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def get_bytes(self, object_key: str) -> bytes:
"""Read an object through the backend for small API responses and workers."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
return response.read()
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
"""Stream an object without loading the complete file into memory."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
while True:
chunk = response.read(chunk_size)
if not chunk:
break
yield chunk
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
self._ensure_enabled()
self.ensure_bucket()
try:
return [
{
"object_key": item.object_name,
"byte_size": item.size or 0,
"etag": item.etag,
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
}
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
]
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def delete(self, object_key: str) -> None:
self._ensure_enabled()
self.ensure_bucket()
try:
self.client.remove_object(self.bucket, object_key)
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()

54
backend/app/modules/storage/policy.py Normal file
View File

@@ -0,0 +1,54 @@
"""Storage placement rules shared by dataset and data-processing flows."""
from __future__ import annotations
from app.core.config import get_settings
_INLINE_TEXT_FORMATS = {
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
"yaml", "yml", "xml", "html", "text/plain", "application/json",
"application/jsonl", "text/csv",
}
def should_store_in_minio(
size_bytes: int | None,
*,
content_type: str | None = None,
file_format: str | None = None,
) -> bool:
"""Return whether a file is large enough to use the shared object store.
Small files remain inline in PostgreSQL so page previews and metadata reads
do not pay an object-storage round trip. MinIO is still mandatory for
large files when it is enabled.
"""
if not get_settings().minio_enabled:
return False
try:
size = max(0, int(size_bytes or 0))
except (TypeError, ValueError):
size = 0
if size > get_settings().minio_inline_max_bytes:
return True
# Binary office/document files remain in MinIO even when small because
# their original bytes cannot be safely represented by a text DB column.
normalized_format = str(file_format or "").strip().lower().lstrip(".")
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
if normalized_format or normalized_type:
return not (
normalized_format in _INLINE_TEXT_FORMATS
or normalized_type in _INLINE_TEXT_FORMATS
or normalized_type.startswith("text/")
)
return False
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
"""Return ``minio`` or ``database`` for a managed file."""
if str(requested or "").strip().lower() == "local":
return "database"
return "minio" if should_store_in_minio(size_bytes) else "database"

10
backend/app/modules/system/router.py
View File

@@ -56,13 +56,15 @@ def audit_logs(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""审计日志查询：按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
"""审计日志查询：按组织、操作人、动作、资源、关键字和时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
@@ -73,6 +75,8 @@ def audit_logs(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=limit,
@@ -88,6 +92,8 @@ def audit_logs_export(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
@@ -103,6 +109,8 @@ def audit_logs_export(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=10000,