YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'ft_wyt' of http://www.caoxiaozhu.com:13001/YG-Soft/YG_FT into ft_wyt

Browse Source 
# Conflicts:
#	backend/app/api/v1/endpoints/platform.py
#	compute/requirements.txt
This commit is contained in:
wuyongtao
2026-08-03 09:42:49 +08:00
parent 46d343fb63 15c4223f2c
commit 24c77a990a
67 changed files with 7775 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
backend/app/api/v1/endpoints/data_process.py
View File

@@ -8,12 +8,14 @@ import os
import re
import socket
import time
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
from copy import deepcopy
from dataclasses import asdict
from pathlib import Path
from threading import BoundedSemaphore, Lock
from typing import Any, Iterator, Literal
from typing import Any, Literal
from urllib.parse import quote, urlsplit
import httpx
@@ -32,6 +34,7 @@ from fastapi import (
from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -45,9 +48,10 @@ from app.modules.data_process.algorithms import (
is_near_duplicate,
near_duplicate_fingerprint,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
remove_document_noise,
score_quality,
structured_json_dumps,
)
from app.modules.data_process.document_chunking import (
DocumentChunk,
@@ -75,9 +79,11 @@ from app.modules.data_process.store import (
NotFoundError,
get_data_process_store,
new_id,
repeat_task_id,
)
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
DataProcessStatus,
DataProcessTaskCreate,
DataProcessTaskUpdate,
@@ -261,7 +267,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
content = str(source.get("content") or "")
file_format = str(source.get("file_format") or "").lower()
if file_format == "xlsx":
# XLSX 上传阶段已安全解析为 JSONL 后入库。
raw_content = source.get("raw_content")
if isinstance(raw_content, bytes):
return parse_text_content(
raw_content,
filename=str(source.get("name") or "source.xlsx"),
file_format="xlsx",
)
# 兼容原始对象已缺失的历史文件：退化为上传阶段生成的 JSONL。
return parse_text_content(content, file_format="jsonl")
if file_format in {"pdf", "docx", "pptx"}:
# 文档上传阶段已抽取文本，预览阶段只需要对正文切片。
@@ -381,11 +394,12 @@ def _build_preview_items(
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
items: list[dict[str, Any]] = []
def append_item(item: dict[str, Any]) -> None:
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
content = str(item.get("edited_content") or "").strip()
if should_clean_invalid and not content:
return
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
# 去重必须基于脱敏前内容，否则不同原文可能在替换 PII 后被错误合并。
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
if should_deduplicate and content_hash in seen_content_hashes:
return
seen_content_hashes.add(content_hash)
@@ -447,6 +461,7 @@ def _build_preview_items(
continue
for key in band_keys:
seen_near_duplicate_bands.setdefault(key, []).append(content)
dedup_content = content
pii_counts: dict[str, int] = {}
if should_desensitize:
content, pii_counts = desensitize_pii(content)
@@ -476,7 +491,8 @@ def _build_preview_items(
else "original"
),
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
continue
@@ -488,48 +504,76 @@ def _build_preview_items(
"filter_anomaly",
}
source_records = list(parsed.records)
processed_records = preprocess_structured_records(
processed_records = preprocess_structured_records_with_lineage(
source_records,
structured_options,
)
if not processed_records and parsed.text and not source_records:
processed_records = [{"value": parsed.text}]
same_cardinality = len(processed_records) == len(source_records)
for index, record in enumerate(processed_records):
original_record = source_records[index] if same_cardinality else record
original_content = json.dumps(
original_record,
ensure_ascii=False,
separators=(",", ":"),
for processed in processed_records:
source_index = processed.source_index
record = processed.record
original_record = (
source_records[source_index]
if source_index < len(source_records)
else record
)
source_locator = (
deepcopy(parsed.record_locators[source_index])
if source_index < len(parsed.record_locators)
else None
)
original_content = structured_json_dumps(original_record)
pii_counts: dict[str, int] = {}
edited_record = record
dedup_content = (
canonical_record_json(record)
if "normalize_format" in preprocess_options
else structured_json_dumps(record)
)
if should_desensitize:
edited_record, pii_counts = desensitize_structured_record(record)
content = (
canonical_record_json(edited_record)
if "normalize_format" in preprocess_options
else json.dumps(
edited_record,
ensure_ascii=False,
separators=(",", ":"),
)
else structured_json_dumps(edited_record)
)
quality = _preview_quality(content, config)
quality["pii_replacements"] = pii_counts
if source_locator is not None:
quality["source_locator"] = source_locator
source_start = (
source_locator.get("source_start")
if source_locator is not None
else None
)
source_end = (
source_locator.get("source_end")
if source_locator is not None
else None
)
source_start_line = (
source_locator.get("start_line")
if source_locator is not None
else None
)
source_end_line = (
source_locator.get("end_line")
if source_locator is not None
else None
)
append_item(
{
"source_file_id": source["id"],
"original_content": original_content,
"edited_content": content,
"source_start": None,
"source_end": None,
"source_start_line": None,
"source_end_line": None,
"source_start": source_start,
"source_end": source_end,
"source_start_line": source_start_line,
"source_end_line": source_end_line,
"token_count": estimate_token_count(content),
"status": "modified" if content != original_content else "original",
"quality_score": quality,
}
},
dedup_content=dedup_content,
)
return items
@@ -772,18 +816,33 @@ def list_tasks(
keyword: str | None = Query(default=None),
status: DataProcessStatus | None = Query(default=None),
process_type: ProcessType | None = Query(default=None),
tenant_id: str | None = Query(default=None),
project_id: str | None = Query(default=None),
store: DataProcessStore = Depends(get_data_process_store),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
with api_errors():
return ok(
store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
)
tasks = store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
tenant_id=tenant_id,
project_id=project_id,
)
# #4 资源 ACL 过滤：admin 放行，普通用户只看到自己被授权的数据处理任务
items = tasks.get("items", [])
if not is_admin(current_user) and items:
accessible_ids = set(
filter_accessible_resource_ids(
"data-process", [t["id"] for t in items], current_user
)
)
items = [t for t in items if t["id"] in accessible_ids]
tasks["items"] = items
tasks["total"] = len(items)
return ok(tasks)
@router.post("")
@@ -850,6 +909,135 @@ def prepare_regeneration(
)
def _repeat_file_copies(
store: DataProcessStore,
storage: LocalDataProcessStorage,
source_task_id: str,
request_id: str,
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
"""为新任务创建独立的源文件引用，避免删除任一任务时互相影响。"""
target_task_id = repeat_task_id(source_task_id, request_id)
copies: dict[str, dict[str, str]] = {}
staged: list[StagedSourceObject] = []
batch_id = storage.new_batch_id()
for summary in store.list_source_files(source_task_id):
old_file_id = str(summary["id"])
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,
expected_source_task_id=source_task_id,
expected_source_file_id=old_file_id,
task_id=target_task_id,
source_file_id=new_file_id,
version=1,
name=str(source["name"]),
)
staged.append(staged_object)
new_reference = staged_object.reference
elif old_reference.startswith("db://data-process/") or not old_reference:
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
else:
raise ValueError("源任务包含不受支持的文件存储引用")
copies[old_file_id] = {
"id": new_file_id,
"storage_object_id": new_reference,
}
return copies, staged
def _remove_repeated_storage_objects(
storage: LocalDataProcessStorage,
task_id: str,
staged: list[StagedSourceObject],
copies: dict[str, dict[str, str]],
) -> None:
source_file_ids = {
str(copy["storage_object_id"]): str(copy["id"])
for copy in copies.values()
}
for item in staged:
try:
storage.delete(
item.reference,
expected_task_id=task_id,
expected_source_file_id=source_file_ids[item.reference],
)
except Exception:
logger.exception(
"failed to roll back repeated data process source object task_id=%s",
task_id,
)
@router.post("/{task_id}/repeat", status_code=202)
def repeat_generation(
task_id: str,
payload: DataProcessRepeatRequest,
background_tasks: BackgroundTasks,
store: DataProcessStore = Depends(get_data_process_store),
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
) -> dict[str, Any]:
"""按原任务快照创建独立任务，并立即在后台开始新一批生成。"""
with api_errors():
repeated = store.find_repeated_task(task_id, payload.request_id)
staged: list[StagedSourceObject] = []
target_task_id = repeat_task_id(task_id, payload.request_id)
if repeated is None:
copies, staged = _repeat_file_copies(
store,
storage,
task_id,
payload.request_id,
)
storage.publish(staged)
try:
repeated = store.repeat_task(
task_id,
expected_updated_at=payload.expected_updated_at,
request_id=payload.request_id,
file_copies=copies,
)
except Exception:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
raise
if not repeated["created"]:
_remove_repeated_storage_objects(
storage,
target_task_id,
staged,
copies,
)
repeated_task = repeated["task"]
if repeated_task.get("status") == "pending":
try:
started = store.start_generation(target_task_id, replace_existing=True)
background_tasks.add_task(
_run_generation,
store,
target_task_id,
str(started["generation_run_id"]),
)
except ConflictError:
latest = store.get_task(target_task_id)
if latest.get("status") != "running":
raise
repeated["task"] = store.get_task(target_task_id)
repeated["progress"] = store.progress(target_task_id)
return ok(repeated, "已按原配置创建新任务并开始后台生成")
@router.delete("/{task_id}")
def delete_task(
task_id: str,
@@ -919,7 +1107,7 @@ async def upload_source_files(
f"{suffix} is not supported for {process_type} data processing",
)
parsed = parse_text_content(raw, filename=name)
if not parsed.text:
if not parsed.text.strip():
raise fail(400, f"source file is empty: {name}")
batch_size += len(raw)
if batch_size > MAX_SOURCE_BATCH_BYTES:
@@ -934,7 +1122,11 @@ async def upload_source_files(
content=raw,
)
staged.append(staged_object)
record_count = len(parsed.records) or (1 if parsed.text else 0)
record_count = (
len(parsed.records)
if process_type == "structured"
else (1 if parsed.text else 0)
)
prepared.append(
{
"id": source_file_id,
@@ -1362,15 +1554,28 @@ def _prepare_preview_items(
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
)
is_unstructured = task.get("process_type") == "unstructured"
if is_unstructured and (
needs_unstructured_raw = is_unstructured and (
chunk_method == "layout_hybrid"
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
):
)
has_structured_xlsx = not is_unstructured and any(
str(source.get("file_format") or "").lower() == "xlsx"
for source in sources
)
if needs_unstructured_raw or has_structured_xlsx:
for index, source in enumerate(sources):
if (
chunk_method != "layout_hybrid"
and str(source.get("file_format") or "").lower() != "pdf"
):
source_format = str(source.get("file_format") or "").lower()
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
needs_pdf_noise = (
is_unstructured
and not needs_layout_raw
and source_format == "pdf"
and bool(
preprocess_options & {"clean_invalid", "clean_invalid_content"}
)
)
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
continue
storage_object_id = str(source.get("storage_object_id") or "")
actual_size = storage.file_size(
@@ -1379,7 +1584,7 @@ def _prepare_preview_items(
expected_source_file_id=str(source["id"]),
)
if actual_size is None:
if chunk_method == "layout_hybrid":
if needs_layout_raw:
raise InvalidStateError(
"版面结构混合切分无法读取原始文件，请重新上传后再处理"
)
@@ -1396,12 +1601,10 @@ def _prepare_preview_items(
)
)
enriched = dict(source)
if chunk_method == "layout_hybrid":
if needs_structured_xlsx or needs_layout_raw:
enriched["raw_content"] = raw
sources[index] = enriched
continue
if str(source.get("file_format") or "").lower() != "pdf":
continue
pages = extract_pdf_page_texts(raw)
extracted_text = "\n\n".join(page.text for page in pages if page.text)
if extracted_text != str(source.get("content") or ""):
@@ -1413,7 +1616,7 @@ def _prepare_preview_items(
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
sources[index] = enriched
items = _build_preview_items(task, sources)
if not items and source_file_ids is None:
if not items and source_file_ids is None and is_unstructured:
raise InvalidStateError("source files did not produce preview items")
return items
@@ -1435,6 +1638,7 @@ def _run_preview(
len(source_file_ids),
)
try:
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
if not store.mark_preview_running(task_id, preview_run_id):
logger.info(
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
@@ -1461,7 +1665,7 @@ def _run_preview(
storage,
[source_file_id],
)
if not items:
if not items and is_unstructured:
raise InvalidStateError(
f"source file did not produce preview items: {source_file_id}"
)
@@ -1657,8 +1861,13 @@ def update_preview_item(
) -> dict[str, Any]:
with api_errors():
task = store.get_task(task_id)
existing = store.get_preview_item(task_id, preview_id)
update = payload.model_dump(exclude_unset=True, mode="json")
update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {})
quality = _preview_quality(payload.edited_content, task.get("config") or {})
source_locator = (existing.get("quality_score") or {}).get("source_locator")
if isinstance(source_locator, Mapping):
quality["source_locator"] = deepcopy(dict(source_locator))
update["quality_score"] = quality
item = store.update_preview_item(
task_id,
preview_id,
@@ -2089,13 +2298,11 @@ def regenerate_results_batch(
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
# httpx.Client 支持跨线程复用，批次内共享连接池可减少重复建连开销。
with (
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor,
):
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor:
futures = {
executor.submit(
_regenerate_result_in_place,

320
backend/app/api/v1/endpoints/platform.py
View File

@@ -2,14 +2,16 @@
import json
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Body, File, HTTPException, Query, UploadFile
from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import PlainTextResponse, StreamingResponse
import httpx
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
@@ -91,6 +93,35 @@ def fail(status_code: int, message: str) -> HTTPException:
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
def _require_approval_or_admin(
resource_type: str,
resource_id: str,
current_user: dict[str, Any],
action_desc: str = "",
) -> dict[str, Any] | None:
"""
高风险操作审批旁路：
- admin 用户直接放行（返回 None）
- 普通用户创建审批实例，返回审批待定响应（code=202，非 None）
code=202 使前端响应拦截器走业务错误分支，弹提示并 reject，
避免前端误认为删除成功。
"""
if is_admin(current_user):
return None
store = get_platform_store()
instance = store.create_approval_instance({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"template_id": None,
})
return {
"code": 202,
"message": f"操作已提交审批，等待管理员批准：{action_desc}",
"data": {"approval_required": True, "approval_id": instance["id"]},
}
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -287,8 +318,18 @@ async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/me")
async def me() -> dict[str, Any]:
return ok(get_platform_store().users()[0])
async def me(request: Request) -> dict[str, Any]:
"""根据 Authorization header 中的 token 返回当前登录用户信息"""
store = get_platform_store()
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
# token 格式: platform-token-{user_id}
if token.startswith("platform-token-"):
user_id = token[len("platform-token-"):]
for u in store.users():
if u.get("id") == user_id:
return ok(u)
raise fail(401, "invalid or missing token")
@router.get("/dashboard/overview")
@@ -307,6 +348,183 @@ async def dashboard_overview() -> dict[str, Any]:
)
@router.get("/dashboard/stats")
async def dashboard_stats() -> dict[str, Any]:
"""看板聚合数据：基于平台真实数据；缺项做合理近似。"""
store = get_platform_store()
tasks = store.tasks()
users = store.users()
nodes = store.compute_nodes()
datasets = store.datasets()
eval_tasks = store.eval_tasks()
# 数据处理任务总数（来自 data_process 模块）
try:
from app.modules.data_process.store import get_data_process_store
dp_store = get_data_process_store()
dp_result = dp_store.list_tasks(page=1, page_size=1)
dp_count = int(dp_result.get("total", 0))
except Exception:
dp_count = 0
running_statuses = {"syncing", "queued", "running"}
running_ft = [t for t in tasks if t.get("status") in running_statuses]
failed_ft = [t for t in tasks if t.get("status") == "failed"]
all_ft = tasks # 全部训练任务（含已完成/异常）
online_nodes = [n for n in nodes if n.get("scheduler_status") == "online"]
# 近 7 天训练统计（按创建日期分桶）
now = datetime.now(timezone.utc)
train_by_day: dict[str, int] = {}
for t in tasks:
ct = t.get("create_time")
if ct:
train_by_day[ct[:10]] = train_by_day.get(ct[:10], 0) + 1
training_7d = []
for i in range(6, -1, -1):
day = (now - timedelta(days=i)).strftime("%Y-%m-%d")
training_7d.append(
{
"date": day[5:],
"train": train_by_day.get(day, 0),
"gpu": sum(len(t.get("gpus") or []) for t in running_ft),
"accuracy": None,
}
)
# 服务状态 —— 与界面实际数据对齐
service_status = [
{
"type": "模型推理",
"status": "error" if (nodes and not online_nodes) else ("busy" if (nodes and len(online_nodes) < len(nodes)) else "normal"),
"count": len(online_nodes),
},
{
"type": "模型训练",
"status": "error" if failed_ft else ("busy" if running_ft else "normal"),
"count": len(all_ft),
},
{
"type": "模型评测",
"status": "normal",
"count": len(eval_tasks),
},
{
"type": "数据处理",
"status": "normal" if not failed_ft else "busy",
"count": dp_count,
},
]
# 训练任务状态归一化
status_map = {
"syncing": "running",
"queued": "running",
"running": "running",
"pending": "pending",
"paused": "pending",
"completed": "completed",
"failed": "failed",
"error": "failed",
"cancelled": "failed",
"stopped": "failed",
}
training_tasks = [
{
"id": t.get("id"),
"name": t.get("name"),
"status": status_map.get(t.get("status"), "pending"),
"train_type": t.get("train_type") or t.get("trainType") or "",
"train_method": t.get("train_method") or t.get("trainMethod") or "",
"base_model": t.get("base_model") or t.get("baseModel") or "",
"progress": t.get("progress", 0),
"accuracy": t.get("accuracy"),
"started_at": (t.get("create_time") or "")[:16],
}
for t in tasks[:8]
]
# 用户操作分布：统计平台全部操作（含治理模块）
MODULE_LABELS = [
("data-process", "数据处理"),
("data_process", "数据处理"),
("dataset", "数据集管理"),
("fine-tune", "模型训练"),
("fine_tune", "模型训练"),
("model-eval", "模型评测"),
("eval", "模型评测"),
("model-inference", "模型推理"),
("inference", "模型推理"),
("model-manage", "模型管理"),
("model", "模型管理"),
("trained", "模型管理"),
# 治理模块操作
("tenant", "租户与项目"),
("project", "租户与项目"),
("approval", "租户与项目"),
("acl", "租户与项目"),
("user", "用户管理"),
("role", "用户管理"),
]
OP_ORDER = [
"数据集管理",
"数据处理",
"模型训练",
"模型评测",
"模型推理",
"模型管理",
"租户与项目",
"用户管理",
]
def _op_module(action: str) -> str | None:
a = (action or "").lower()
for prefix, label in MODULE_LABELS:
if a.startswith(prefix):
return label
return None
audit = store.audit_logs(limit=1000)
op_counter: dict[str, int] = {label: 0 for label in OP_ORDER}
for log in audit.get("items", []):
label = _op_module(log.get("action") or "")
if label:
op_counter[label] += 1
operation_distribution = [{"name": k, "value": v} for k, v in op_counter.items()]
# 最近登录用户
recent = sorted(
[u for u in users if u.get("last_login")],
key=lambda u: u["last_login"],
reverse=True,
)[:5]
recent_login_users = [
{
"user": u.get("display_name") or u.get("username"),
"role": u.get("role"),
"last_login": (u.get("last_login") or "")[:16],
}
for u in recent
]
# 登录时长排行（本月）
login_duration_rank = store.login_duration_rank()
return ok(
{
"online_services": sum(s["count"] for s in service_status),
"running_tasks": len(running_ft),
"pending_alerts": 0,
"training_7d": training_7d,
"service_status": service_status,
"training_tasks": training_tasks,
"operation_distribution": operation_distribution,
"login_duration_rank": login_duration_rank,
"recent_login_users": recent_login_users,
}
)
@router.get("/system-info")
async def system_info() -> dict[str, Any]:
return ok(get_platform_store().system_info())
@@ -341,6 +559,21 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
raise fail(400, str(exc))
@router.post("/users/{user_id}/reset-password")
async def reset_user_password(
user_id: str,
payload: dict[str, Any] = Body(default={}),
) -> dict[str, Any]:
new_password = payload.get("password") or "Platform@123"
try:
get_platform_store().reset_password(user_id, new_password)
return ok({"reset": user_id})
except KeyError:
raise fail(404, "user not found")
except ValueError as exc:
raise fail(400, str(exc))
@router.get("/model-manage/local-models")
async def local_models() -> dict[str, Any]:
store = get_platform_store()
@@ -404,8 +637,13 @@ async def model_by_name(name: str) -> dict[str, Any]:
@router.get("/model-manage")
async def model_list() -> dict[str, Any]:
return ok(get_platform_store().models())
async def model_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
models = get_platform_store().models()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(models)
# 普通用户只返回有 ACL 授权的模型
accessible = set(filter_accessible_resource_ids("model", [m["id"] for m in models], current_user))
return ok([m for m in models if m["id"] in accessible])
@router.post("/model-manage")
@@ -421,11 +659,14 @@ async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/model-manage/{model_id}")
async def model_detail(model_id: str) -> dict[str, Any]:
async def model_detail(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().model(model_id))
model = get_platform_store().model(model_id)
except KeyError:
raise fail(404, "model not found")
if not has_resource_access("model", model_id, current_user, "read"):
raise fail(403, "no permission to access this model")
return ok(model)
@router.put("/model-manage/{model_id}")
@@ -445,7 +686,12 @@ async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...
@router.delete("/model-manage/{model_id}")
async def delete_model(model_id: str) -> dict[str, Any]:
async def delete_model(model_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("model", model_id, current_user, "delete"):
raise fail(403, "no permission to delete this model")
pending = _require_approval_or_admin("model", model_id, current_user, f"删除模型 {model_id}")
if pending:
return pending
get_platform_store().delete_model(model_id)
return ok({"deleted": model_id})
@@ -707,8 +953,12 @@ async def download_dataset_file(dataset_id: str, file_id: str, version_id: str |
@router.get("/dataset-manage")
async def dataset_list() -> dict[str, Any]:
return ok(get_platform_store().datasets())
async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
datasets = get_platform_store().datasets()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(datasets)
accessible = set(filter_accessible_resource_ids("dataset", [d["id"] for d in datasets], current_user))
return ok([d for d in datasets if d["id"] in accessible])
@router.post("/dataset-manage")
@@ -718,11 +968,14 @@ async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/dataset-manage/{dataset_id}")
async def dataset_detail(dataset_id: str) -> dict[str, Any]:
async def dataset_detail(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
return ok(get_platform_store().dataset(dataset_id))
dataset = get_platform_store().dataset(dataset_id)
except KeyError:
raise fail(404, "dataset not found")
if not has_resource_access("dataset", dataset_id, current_user, "read"):
raise fail(403, "no permission to access this dataset")
return ok(dataset)
@router.put("/dataset-manage/{dataset_id}")
@@ -734,7 +987,12 @@ async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -
@router.delete("/dataset-manage/{dataset_id}")
async def delete_dataset(dataset_id: str) -> dict[str, Any]:
async def delete_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("dataset", dataset_id, current_user, "delete"):
raise fail(403, "no permission to delete this dataset")
pending = _require_approval_or_admin("dataset", dataset_id, current_user, f"删除数据集 {dataset_id}")
if pending:
return pending
get_platform_store().delete_dataset(dataset_id)
return ok({"deleted": dataset_id})
@@ -759,8 +1017,12 @@ async def tensorboard_start() -> dict[str, Any]:
@router.get("/fine-tune")
async def fine_tune_list() -> dict[str, Any]:
return ok(get_platform_store().tasks())
async def fine_tune_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
tasks = get_platform_store().tasks()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(tasks)
accessible = set(filter_accessible_resource_ids("fine-tune", [t["id"] for t in tasks], current_user))
return ok([t for t in tasks if t["id"] in accessible])
@router.post("/fine-tune")
@@ -953,7 +1215,12 @@ async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(de
@router.delete("/fine-tune/{task_id}")
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
async def delete_fine_tune(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("fine-tune", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this task")
pending = _require_approval_or_admin("fine-tune", task_id, current_user, f"删除训练任务 {task_id}")
if pending:
return pending
get_platform_store().delete_task(task_id)
return ok({"deleted": task_id})
@@ -993,12 +1260,16 @@ async def fine_tune_metrics(task_id: str) -> dict[str, Any]:
@router.get("/model-eval")
async def model_eval_list() -> dict[str, Any]:
return ok(get_platform_store().eval_tasks())
async def model_eval_list(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
tasks = get_platform_store().eval_tasks()
if current_user.get("role") == "admin" or current_user.get("protected"):
return ok(tasks)
accessible = set(filter_accessible_resource_ids("eval", [t["id"] for t in tasks], current_user))
return ok([t for t in tasks if t["id"] in accessible])
@router.get("/model-eval/{task_id}")
async def model_eval_detail(task_id: str) -> dict[str, Any]:
async def model_eval_detail(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
try:
store = get_platform_store()
task = store.eval_task(task_id)
@@ -1019,9 +1290,11 @@ async def model_eval_detail(task_id: str) -> dict[str, Any]:
break
except Exception:
pass
return ok(task)
except KeyError:
raise fail(404, "eval task not found")
if not has_resource_access("eval", task_id, current_user, "read"):
raise fail(403, "no permission to access this eval task")
return ok(task)
@router.post("/model-eval/start")
@@ -1172,7 +1445,12 @@ async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any
@router.delete("/model-eval/{task_id}")
async def model_eval_delete(task_id: str) -> dict[str, Any]:
async def model_eval_delete(task_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("eval", task_id, current_user, "delete"):
raise fail(403, "no permission to delete this eval task")
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
if pending:
return pending
get_platform_store().delete_eval_task(task_id)
return ok({"deleted": task_id})

12
backend/app/api/v1/router.py
View File

@@ -3,8 +3,20 @@
from app.api.v1.endpoints.data_process import router as data_process_router
from app.api.v1.endpoints.platform import router as platform_router
from app.api.v1.endpoints.health import router as health_router
from app.modules.tenant.router import router as tenant_router
from app.modules.project.router import router as project_router
from app.modules.approval.router import router as approval_router
from app.modules.system.router import router as system_router
from app.modules.retention.router import router as retention_router
from app.modules.resource.router import router as resource_router
api_router = APIRouter()
api_router.include_router(health_router, tags=["health"])
api_router.include_router(data_process_router, tags=["data-process"])
api_router.include_router(platform_router, tags=["platform"])
api_router.include_router(system_router, tags=["system"])
api_router.include_router(tenant_router, tags=["tenant"])
api_router.include_router(project_router, tags=["project"])
api_router.include_router(approval_router, tags=["approval"])
api_router.include_router(retention_router, tags=["retention"])
api_router.include_router(resource_router, tags=["resource"])

138
backend/app/core/auth.py Normal file
View File

@@ -0,0 +1,138 @@
"""鉴权依赖：从 Authorization header 解析当前用户，提供权限校验。"""
from __future__ import annotations
from typing import Any
from fastapi import Depends, HTTPException, Query, Request, status
from app.db.platform_store import get_platform_store
# 无需鉴权的路径前缀（健康检查、登录等）
PUBLIC_PATHS = ("/health", "/login", "/system-info")
def _extract_token(request: Request) -> str | None:
"""从 Authorization header 提取 token（格式: Bearer platform-token-{user_id}）。"""
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
if token.startswith("platform-token-"):
return token[len("platform-token-"):]
return None
def get_current_user(request: Request) -> dict[str, Any]:
"""
FastAPI 依赖：解析当前登录用户。
- 公开路径（/health, /login 等）直接放行，返回匿名用户。
- 无 token 或 token 无效时抛 401。
- admin 用户标记为超级管理员，拥有全部权限。
"""
path = request.url.path
# 去掉路由前缀后判断
for prefix in PUBLIC_PATHS:
if path.endswith(prefix):
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
user_id = _extract_token(request)
if not user_id:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
store = get_platform_store()
for u in store.users():
if u.get("id") == user_id:
return u
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
"""FastAPI 依赖：要求当前用户是管理员（role=admin 或 protected）。"""
if current_user.get("role") == "admin" or current_user.get("protected"):
return current_user
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
def is_admin(user: dict[str, Any]) -> bool:
"""判断用户是否为管理员（admin 角色或 protected 标记）。"""
return user.get("role") == "admin" or user.get("protected", False)
def has_resource_access(
resource_type: str,
resource_id: str,
user: dict[str, Any],
permission: str = "read",
) -> bool:
"""
检查用户对某资源是否有指定权限。
- admin/protected 用户直接放行（旁路）。
- 其他用户检查 acls 表中是否有对应授权。
"""
if user.get("role") == "admin" or user.get("protected"):
return True
store = get_platform_store()
acls = store.get_acl(resource_type, resource_id)
user_id = user.get("id")
user_role = user.get("role")
for entry in acls:
# 按 user 授权
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
if _permission_covers(entry.get("permission"), permission):
return True
# 按 role 授权
if entry.get("principal_type") == "role" and entry.get("principal_id") == user_role:
if _permission_covers(entry.get("permission"), permission):
return True
return False
def _permission_covers(granted: str | None, required: str) -> bool:
"""权限覆盖判断：write/execute 覆盖 read；admin 覆盖一切。"""
if not granted:
return False
if granted == "admin":
return True
if granted == required:
return True
# write 覆盖 read
if required == "read" and granted in ("write", "execute"):
return True
return False
def filter_accessible_resource_ids(
resource_type: str,
all_ids: list[str],
user: dict[str, Any],
) -> list[str]:
"""
从全部资源 ID 中过滤出当前用户可访问的 ID 列表。
- admin 直接返回全部。
- 普通用户查 acls 表取交集。
"""
if user.get("role") == "admin" or user.get("protected"):
return all_ids
if not all_ids:
return []
store = get_platform_store()
user_id = user.get("id")
user_role = user.get("role")
# 查询该用户在该资源类型下有 read 权限的所有 resource_id
with store.connect() as conn:
rows = conn.execute(
"""
SELECT DISTINCT resource_id FROM acls
WHERE resource_type=? AND (
(principal_type='user' AND principal_id=?)
OR (principal_type='role' AND principal_id=?)
)
""",
(resource_type, user_id, user_role),
).fetchall()
accessible = {r["resource_id"] for r in rows}
return [rid for rid in all_ids if rid in accessible]

12
backend/app/core/config.py
View File

@@ -2,6 +2,18 @@
from functools import lru_cache
import os
try:
from pathlib import Path as _Path
from dotenv import load_dotenv
# 显式指定 backend 目录下的 .env，并强制覆盖已有环境变量，
# 确保远程数据库配置生效，不被本地默认值或残留环境变量影响。
_env_path = _Path(__file__).resolve().parent.parent.parent / ".env"
load_dotenv(dotenv_path=_env_path, override=True)
except ImportError:
pass
def _int_env(name: str, default: int) -> int:
raw = os.getenv(name)

746
backend/app/db/platform_store.py
View File

@@ -14,6 +14,7 @@ from pathlib import Path
from typing import Any, Iterator
import psycopg
from psycopg_pool import ConnectionPool
from app.core.config import get_settings
@@ -291,6 +292,32 @@ class PlatformStore:
def __init__(self, database_url: str | None = None) -> None:
settings = get_settings()
self.database_url = _psycopg_url(database_url or settings.database_url)
# Reuse connections via a pool to avoid the TCP+auth handshake on every
# request (notably expensive against the remote PostgreSQL instance).
# TCP keepalive 让操作系统持续保活连接，抵抗远程库空闲静默断连。
pool_kwargs = {
"keepalives": 1,
"keepalives_idle": 30,
"keepalives_interval": 10,
"keepalives_count": 5,
}
self._pool = ConnectionPool(
conninfo=self.database_url,
kwargs=pool_kwargs,
min_size=2,
max_size=10,
# 借出前校验连接可用性，避免执行 SQL 时才发现 [BAD] 再重建。
check=ConnectionPool.check_connection,
# 不主动回收空闲连接（远程库约 10s 断，由 keepalive 维持），
# 减少无谓的重建握手。
max_idle=0,
# 请求最多排队等待 5s，避免雪崩时无限堆积。
max_waiting=16,
open=False,
)
# 注意：不要在此调用 pool.wait()，它会阻塞等待 min_size 个连接就绪，
# 在远程库响应慢/超时时会卡死 uvicorn worker 进程，导致所有请求无响应。
self._pool.open()
self.ensure_schema()
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
@@ -309,16 +336,23 @@ class PlatformStore:
@contextmanager
def connect(self) -> Iterator["PgConnection"]:
raw_conn = psycopg.connect(self.database_url)
conn = PgConnection(raw_conn)
with self._pool.connection() as raw_conn:
conn = PgConnection(raw_conn)
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def close_pool(self) -> None:
"""Release pooled connections. Safe to call multiple times."""
try:
yield conn
conn.commit()
self._pool.close()
except Exception:
conn.rollback()
raise
finally:
conn.close()
pass
def ensure_schema(self) -> None:
schema_path = Path(__file__).with_name("sql") / "001_platform_runtime.sql"
@@ -348,6 +382,11 @@ class PlatformStore:
"last_error": "TEXT",
},
)
schema_dir = Path(__file__).with_name("sql")
for extra in ("002_governance.sql", "003_tenant_quota.sql"):
extra_path = schema_dir / extra
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
def _column_names(self, conn: PgConnection, table_name: str) -> set[str]:
columns = conn.execute(
@@ -1108,6 +1147,18 @@ class PlatformStore:
raise ValueError("protected user cannot be deleted")
conn.execute("DELETE FROM users WHERE id=?", (user_id,))
def reset_password(self, user_id: str, new_password: str) -> None:
with self.connect() as conn:
row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone()
if not row:
raise KeyError(user_id)
if row["protected"]:
raise ValueError("protected user cannot reset password")
conn.execute(
"UPDATE users SET password_hash=? WHERE id=?",
(hash_password(new_password), user_id),
)
def _user(self, row: PgRow) -> dict[str, Any]:
return {
"id": row["id"],
@@ -2954,6 +3005,672 @@ class PlatformStore:
]
return {"file": file_name, "content": "\n".join(lines), "size": "1 KB"}
# ===================== 平台治理：角色 =====================
def roles(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM roles ORDER BY name").fetchall()
return [dict(r) for r in rows]
# ===================== 平台治理：审计日志 =====================
def audit_logs(
self,
*,
tenant_id: str | None = None,
project_id: str | None = None,
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
offset: int = 0,
) -> dict[str, Any]:
clauses: list[str] = []
params: list[Any] = []
if tenant_id:
clauses.append("tenant_id=?")
params.append(tenant_id)
if project_id:
clauses.append("project_id=?")
params.append(project_id)
if actor_id:
clauses.append("actor_id=?")
params.append(actor_id)
if action:
clauses.append("action=?")
params.append(action)
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if start_time:
clauses.append("time>=?")
params.append(start_time)
if end_time:
clauses.append("time<=?")
params.append(end_time)
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
with self.connect() as conn:
total = conn.execute(f"SELECT COUNT(*) AS c FROM audit_logs{where}", tuple(params)).fetchone()["c"]
params_paged = list(params) + [limit, offset]
rows = conn.execute(
f"SELECT * FROM audit_logs{where} ORDER BY time DESC LIMIT ? OFFSET ?",
tuple(params_paged),
).fetchall()
return {"total": total, "items": [dict(r) for r in rows]}
def record_audit(
self,
*,
action: str,
actor_id: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
tenant_id: str | None = None,
project_id: str | None = None,
detail: str | None = None,
ip: str | None = None,
) -> None:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO audit_logs
(id, tenant_id, project_id, actor_id, action, target_type, target_id, detail, client_ip, time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
new_id("log"),
tenant_id,
project_id,
actor_id,
action,
target_type,
target_id,
detail,
ip,
utcnow(),
),
)
# ===================== 平台治理：会话 =====================
def create_session(self, user_id: str, *, ip: str | None = None) -> dict[str, Any]:
sid = new_id("sess")
login_at = utcnow()
with self.connect() as conn:
conn.execute(
"INSERT INTO sessions (id, user_id, username, login_at, create_time) "
"VALUES (%s, %s, (SELECT username FROM users WHERE id=%s), %s, %s)",
(sid, user_id, user_id, login_at, login_at),
)
return {"session_id": sid, "user_id": user_id, "login_at": login_at}
def finish_session(self, session_id: str) -> None:
"""登出时记录 logout_at 与时长（秒）。"""
logout_at = utcnow()
with self.connect() as conn:
conn.execute(
"UPDATE sessions SET logout_at=%s, "
"duration_seconds=EXTRACT(EPOCH FROM (%s::timestamptz - login_at::timestamptz))::int "
"WHERE id=%s AND logout_at IS NULL",
(logout_at, logout_at, session_id),
)
def active_sessions(self, user_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM sessions WHERE user_id=%s AND logout_at IS NULL "
"ORDER BY login_at DESC",
(user_id,),
).fetchall()
return [dict(r) for r in rows]
def destroy_session(self, session_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM sessions WHERE id=%s", (session_id,))
def extend_session(self, session_id: str, *, expires_in_seconds: int = 3600 * 8) -> dict[str, Any] | None:
# 兼容旧调用，仅更新 login_at 之后延长的含义在此简化为 no-op 返回现有记录。
with self.connect() as conn:
row = conn.execute("SELECT * FROM sessions WHERE id=%s", (session_id,)).fetchone()
if not row:
return None
return {
"session_id": session_id,
"user_id": row["user_id"],
"login_at": row["login_at"],
}
def set_session_user(self, session_id: str, user_id: str) -> None:
with self.connect() as conn:
conn.execute("UPDATE sessions SET user_id=%s WHERE id=%s", (user_id, session_id))
def login_duration_rank(self, limit: int = 8, days: int = 30) -> list[dict[str, Any]]:
"""登录时长排行：按用户聚合近 N 天的会话时长（小时）。
sessions 表列：login_at(TEXT), logout_at(TEXT), duration_seconds(INT)。
优先用 duration_seconds；为空时回退计算 now-login_at（未登出）或 logout_at-login_at。
"""
with self.connect() as conn:
rows = conn.execute(
"SELECT s.user_id, s.login_at, s.logout_at, s.duration_seconds, "
"u.username, u.display_name, u.role "
"FROM sessions s LEFT JOIN users u ON s.user_id = u.id "
"WHERE s.login_at::timestamptz >= NOW() - make_interval(days => %s)",
(days,),
).fetchall()
now = datetime.now(timezone.utc)
agg: dict[str, dict[str, Any]] = {}
for r in rows:
uid = r["user_id"] or ""
bucket = agg.setdefault(
uid,
{
"user": r["display_name"] or r["username"] or uid,
"role": r["role"] or "",
"total": 0.0,
},
)
dur = r["duration_seconds"]
if dur is not None:
bucket["total"] += float(dur)
continue
start = parse_time(r["login_at"])
end = parse_time(r["logout_at"]) if r["logout_at"] else None
if start and end:
bucket["total"] += max(0, (end - start).total_seconds())
elif start:
bucket["total"] += max(0, (now - start).total_seconds())
result = [
{"user": b["user"], "role": b["role"], "duration": round(b["total"] / 3600, 1)}
for b in agg.values()
]
result.sort(key=lambda x: x["duration"], reverse=True)
return result[:limit]
# ===================== 平台治理：审批 =====================
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
tid = new_id("tpl")
conn.execute(
"INSERT INTO approval_templates (id, name, steps, create_time) VALUES (?, ?, ?, ?)",
(tid, payload["name"], json_dumps(payload.get("steps", [])), utcnow()),
)
return self.approval_template(tid)
def approval_templates(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM approval_templates ORDER BY create_time DESC").fetchall()
return [dict(r) for r in rows]
def approval_template(self, template_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM approval_templates WHERE id=?", (template_id,)).fetchone()
if not row:
raise KeyError(template_id)
return dict(row)
def update_approval_template(self, template_id: str, payload: dict[str, Any]) -> dict[str, Any]:
fields = {k: v for k, v in payload.items() if k in ("name", "steps")}
if "steps" in fields:
fields["steps"] = json_dumps(fields["steps"])
if not fields:
return self.approval_template(template_id)
set_clause = ", ".join(f"{k}=?" for k in fields)
params = list(fields.values()) + [template_id]
with self.connect() as conn:
conn.execute(f"UPDATE approval_templates SET {set_clause} WHERE id=?", tuple(params))
return self.approval_template(template_id)
def delete_approval_template(self, template_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM approval_templates WHERE id=?", (template_id,)).fetchone()
if not row:
raise KeyError(template_id)
conn.execute("DELETE FROM approval_templates WHERE id=?", (template_id,))
return dict(row)
def create_approval_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
template_id = payload.get("template_id")
steps = []
if template_id:
tpl = self.approval_template(template_id)
steps = json_loads(tpl["steps"]) if tpl.get("steps") else []
with self.connect() as conn:
iid = new_id("appr")
conn.execute(
"""
INSERT INTO approval_instances
(id, template_id, resource_type, resource_id, applicant_id, status, current_step, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
iid,
template_id,
payload["resource_type"],
payload["resource_id"],
payload["applicant_id"],
"pending",
0,
utcnow(),
),
)
for idx, step in enumerate(steps):
conn.execute(
"INSERT INTO approval_steps (id, instance_id, step_index, approver_id, status, time) VALUES (?, ?, ?, ?, ?, ?)",
(new_id("step"), iid, idx, step.get("approver_id"), "pending", None),
)
return self.approval_instance(iid)
def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]:
with self.connect() as conn:
if status:
rows = conn.execute(
"SELECT * FROM approval_instances WHERE status=? ORDER BY create_time DESC", (status,)
).fetchall()
else:
rows = conn.execute("SELECT * FROM approval_instances ORDER BY create_time DESC").fetchall()
return [dict(r) for r in rows]
def approval_instance(self, instance_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM approval_instances WHERE id=?", (instance_id,)).fetchone()
if not row:
raise KeyError(instance_id)
steps = conn.execute(
"SELECT * FROM approval_steps WHERE instance_id=? ORDER BY step_index", (instance_id,)
).fetchall()
result = dict(row)
result["steps"] = [dict(s) for s in steps]
return result
def decide_approval_step(self, instance_id: str, step_index: int, *, approver_id: str, approved: bool, comment: str | None = None) -> dict[str, Any]:
with self.connect() as conn:
inst = conn.execute("SELECT * FROM approval_instances WHERE id=?", (instance_id,)).fetchone()
if not inst:
raise KeyError(instance_id)
if inst["status"] != "pending":
raise ValueError("instance not pending")
step = conn.execute(
"SELECT * FROM approval_steps WHERE instance_id=? AND step_index=?",
(instance_id, step_index),
).fetchone()
if not step:
raise KeyError("step not found")
if step["status"] != "pending":
raise ValueError("step already decided")
new_status = "approved" if approved else "rejected"
conn.execute(
"UPDATE approval_steps SET status=?, comment=?, time=? WHERE id=?",
(new_status, comment, utcnow(), step["id"]),
)
if approved:
conn.execute(
"UPDATE approval_instances SET current_step=? WHERE id=?",
(step_index + 1, instance_id),
)
step_rows = conn.execute(
"SELECT * FROM approval_steps WHERE instance_id=? ORDER BY step_index", (instance_id,)
).fetchall()
if all(s["status"] == "approved" for s in step_rows):
conn.execute("UPDATE approval_instances SET status='approved' WHERE id=?", (instance_id,))
else:
conn.execute("UPDATE approval_instances SET status='rejected' WHERE id=?", (instance_id,))
return self.approval_instance(instance_id)
# ===================== 平台治理：租户 =====================
def tenants(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM tenants ORDER BY create_time DESC").fetchall()
return [dict(r) for r in rows]
def tenant(self, tenant_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM tenants WHERE id=?", (tenant_id,)).fetchone()
if not row:
raise KeyError(tenant_id)
return dict(row)
def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
tid = new_id("tnt")
conn.execute(
"""
INSERT INTO tenants (id, name, code, status, owner_user_id, quota, retention_policy_id, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
tid,
payload["name"],
payload.get("code"),
"active",
payload.get("owner_user_id"),
json_dumps(payload.get("quota", {})),
payload.get("retention_policy_id"),
utcnow(),
),
)
return self.tenant(tid)
def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]:
fields = {k: v for k, v in payload.items() if k in ("name", "code", "status", "owner_user_id", "quota", "retention_policy_id")}
if "quota" in fields:
fields["quota"] = json_dumps(fields["quota"])
if not fields:
return self.tenant(tenant_id)
set_clause = ", ".join(f"{k}=?" for k in fields)
params = list(fields.values()) + [tenant_id]
with self.connect() as conn:
conn.execute(f"UPDATE tenants SET {set_clause} WHERE id=?", tuple(params))
return self.tenant(tenant_id)
def set_tenant_quota(self, tenant_id: str, quota: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
conn.execute("UPDATE tenants SET quota=? WHERE id=?", (json_dumps(quota), tenant_id))
return self.tenant(tenant_id)
def set_tenant_retention(self, tenant_id: str, retention_policy_id: str | None) -> dict[str, Any]:
with self.connect() as conn:
conn.execute("UPDATE tenants SET retention_policy_id=? WHERE id=?", (retention_policy_id, tenant_id))
return self.tenant(tenant_id)
def delete_tenant(self, tenant_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM tenants WHERE id=?", (tenant_id,)).fetchone()
if not row:
raise KeyError(tenant_id)
conn.execute("DELETE FROM tenants WHERE id=?", (tenant_id,))
return dict(row)
def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM acls WHERE resource_type=? AND resource_id=?",
(resource_type, resource_id),
).fetchall()
return [dict(r) for r in rows]
def set_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
with self.connect() as conn:
conn.execute(
"DELETE FROM acls WHERE resource_type=? AND resource_id=?",
(resource_type, resource_id),
)
for e in entries:
conn.execute(
"""
INSERT INTO acls (id, resource_type, resource_id, principal_type, principal_id, permission, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(
new_id("acl"),
resource_type,
resource_id,
e.get("principal_type"),
e.get("principal_id"),
e.get("permission"),
utcnow(),
),
)
rows = conn.execute(
"SELECT * FROM acls WHERE resource_type=? AND resource_id=?",
(resource_type, resource_id),
).fetchall()
return [dict(r) for r in rows]
# ===================== 平台治理：项目空间 =====================
def projects(self, *, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]:
clauses = ["tenant_id=?"]
params: list[Any] = [tenant_id]
if status:
clauses.append("status=?")
params.append(status)
if keyword:
clauses.append("(name LIKE ? OR code LIKE ?)")
params.extend([f"%{keyword}%", f"%{keyword}%"])
with self.connect() as conn:
rows = conn.execute(
f"SELECT * FROM projects WHERE {' AND '.join(clauses)} ORDER BY create_time DESC",
tuple(params),
).fetchall()
return [dict(r) for r in rows]
def project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM projects WHERE id=?", (project_id,)).fetchone()
if not row:
raise KeyError(project_id)
return dict(row)
def create_project(self, payload: dict[str, Any]) -> dict[str, Any]:
with self.connect() as conn:
pid = new_id("prj")
conn.execute(
"""
INSERT INTO projects (id, tenant_id, name, code, description, quota, status, create_time, create_by, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
pid,
payload.get("tenant_id", "default"),
payload["name"],
payload["code"],
payload.get("description"),
json_dumps(payload.get("quota", {})),
"active",
utcnow(),
payload.get("create_by"),
utcnow(),
),
)
return self.project(pid)
def update_project(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
fields = {k: v for k, v in payload.items() if k in ("name", "code", "description", "quota", "status")}
if "quota" in fields:
fields["quota"] = json_dumps(fields["quota"])
if not fields:
return self.project(project_id)
set_clause = ", ".join(f"{k}=?" for k in fields)
params = list(fields.values()) + [project_id]
with self.connect() as conn:
conn.execute(f"UPDATE projects SET {set_clause} WHERE id=?", tuple(params))
return self.project(project_id)
def archive_project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute("UPDATE projects SET status='archived' WHERE id=?", (project_id,))
return self.project(project_id)
def activate_project(self, project_id: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute("UPDATE projects SET status='active' WHERE id=?", (project_id,))
return self.project(project_id)
def delete_project(self, project_id: str) -> None:
with self.connect() as conn:
conn.execute("DELETE FROM projects WHERE id=?", (project_id,))
def project_members(self, project_id: str) -> list[dict[str, Any]]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM projects WHERE id=?", (project_id,)).fetchone()
if not row:
raise KeyError(project_id)
rows = conn.execute(
"""
SELECT pm.*, u.username, u.display_name
FROM project_members pm JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=?
""",
(project_id,),
).fetchall()
return [dict(r) for r in rows]
def add_project_member(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
user_id = payload["user_id"]
role = payload.get("role", "member")
with self.connect() as conn:
conn.execute(
"INSERT INTO project_members (project_id, user_id, role, create_time) VALUES (?, ?, ?, ?)",
(project_id, user_id, role, utcnow()),
)
row = conn.execute(
"""
SELECT pm.*, u.username, u.display_name
FROM project_members pm JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=? AND pm.user_id=?
""",
(project_id, user_id),
).fetchone()
return {
"project_id": row["project_id"],
"user_id": row["user_id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"create_time": row["create_time"],
}
def update_project_member_role(self, project_id: str, user_id: str, role: str) -> dict[str, Any]:
with self.connect() as conn:
conn.execute(
"UPDATE project_members SET role=? WHERE project_id=? AND user_id=?",
(role, project_id, user_id),
)
row = conn.execute(
"""
SELECT pm.*, u.username, u.display_name
FROM project_members pm JOIN users u ON u.id = pm.user_id
WHERE pm.project_id=? AND pm.user_id=?
""",
(project_id, user_id),
).fetchone()
if not row:
raise KeyError(user_id)
return {
"project_id": row["project_id"],
"user_id": row["user_id"],
"username": row["username"],
"display_name": row["display_name"],
"role": row["role"],
"create_time": row["create_time"],
}
def remove_project_member(self, project_id: str, user_id: str) -> None:
with self.connect() as conn:
conn.execute(
"DELETE FROM project_members WHERE project_id=? AND user_id=?",
(project_id, user_id),
)
# ===================== 平台治理：资源 ACL =====================
def resource_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
"""返回资源 ACL，按主体分组，permissions 为数组。"""
rows = self.get_acl(resource_type, resource_id)
grouped: dict[str, dict[str, Any]] = {}
for r in rows:
key = f"{r.get('principal_type')}:{r.get('principal_id')}"
bucket = grouped.setdefault(
key,
{
"subject_type": r.get("principal_type"),
"subject_id": r.get("principal_id"),
"permissions": [],
},
)
perm = r.get("permission")
if perm and perm not in bucket["permissions"]:
bucket["permissions"].append(perm)
return list(grouped.values())
def set_resource_acl(
self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]
) -> list[dict[str, Any]]:
"""按前端格式设置资源 ACL：entries 为 [{subject_type, subject_id, permissions: []}]。"""
flat: list[dict[str, Any]] = []
for e in entries:
for perm in e.get("permissions") or []:
flat.append(
{
"principal_type": e.get("subject_type"),
"principal_id": e.get("subject_id"),
"permission": perm,
}
)
self.set_acl(resource_type, resource_id, flat)
return self.resource_acl(resource_type, resource_id)
# ===================== 平台治理：留存策略 =====================
def retention_policies(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute(
"SELECT * FROM retention_policies ORDER BY create_time DESC"
).fetchall()
return [dict(r) for r in rows]
def retention_policy(self, policy_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute(
"SELECT * FROM retention_policies WHERE id=?", (policy_id,)
).fetchone()
if not row:
raise KeyError(policy_id)
return dict(row)
def create_retention_policy(self, payload: dict[str, Any]) -> dict[str, Any]:
pid = payload.get("id") or new_id("rpol")
with self.connect() as conn:
conn.execute(
"""
INSERT INTO retention_policies
(id, name, scope, rule, status, create_time, create_by, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
pid,
payload["name"],
payload.get("scope"),
payload.get("rule"),
payload.get("status", "active"),
utcnow(),
payload.get("create_by"),
utcnow(),
),
)
return self.retention_policy(pid)
def update_retention_policy(
self, policy_id: str, payload: dict[str, Any]
) -> dict[str, Any]:
fields = {
k: v
for k, v in payload.items()
if k in ("name", "scope", "rule", "status")
}
if not fields:
return self.retention_policy(policy_id)
fields["updated_at"] = utcnow()
set_clause = ", ".join(f"{k}=?" for k in fields)
params = list(fields.values()) + [policy_id]
with self.connect() as conn:
conn.execute(
f"UPDATE retention_policies SET {set_clause} WHERE id=?",
tuple(params),
)
return self.retention_policy(policy_id)
def delete_retention_policy(self, policy_id: str) -> None:
with self.connect() as conn:
conn.execute(
"DELETE FROM retention_policies WHERE id=?", (policy_id,)
)
_store: PlatformStore | None = None
@@ -2963,3 +3680,16 @@ def get_platform_store() -> PlatformStore:
if _store is None:
_store = PlatformStore()
return _store
import atexit as _atexit
def _close_store_pool() -> None:
global _store
if _store is not None:
_store.close_pool()
_store = None
_atexit.register(_close_store_pool)

48
backend/app/db/sql/001_platform_runtime.sql
View File

@@ -282,3 +282,51 @@ CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(targe
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
-- ===================== Project / Tenant =====================
CREATE TABLE IF NOT EXISTS projects (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL DEFAULT 'default',
name TEXT NOT NULL,
code TEXT NOT NULL,
description TEXT,
quota TEXT,
status TEXT NOT NULL DEFAULT 'active',
create_time TEXT NOT NULL,
create_by TEXT,
updated_at TEXT
);
CREATE TABLE IF NOT EXISTS project_members (
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
role TEXT NOT NULL DEFAULT 'member',
create_time TEXT NOT NULL,
PRIMARY KEY (project_id, user_id)
);
CREATE TABLE IF NOT EXISTS roles (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
permissions TEXT NOT NULL DEFAULT '[]',
create_time TEXT
);
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
issued_at TEXT NOT NULL,
expires_at TEXT NOT NULL,
ip TEXT
);
CREATE TABLE IF NOT EXISTS acls (
id TEXT PRIMARY KEY,
resource_type TEXT NOT NULL,
resource_id TEXT NOT NULL,
principal_type TEXT NOT NULL,
principal_id TEXT NOT NULL,
permission TEXT NOT NULL,
create_time TEXT
);

68
backend/app/db/sql/002_governance.sql Normal file
View File

@@ -0,0 +1,68 @@
-- 平台治理：租户 / 审批 / 审计（字段以 platform_store 实际写入为准）
CREATE TABLE IF NOT EXISTS tenants (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
code TEXT,
status TEXT DEFAULT 'active',
owner_user_id TEXT,
quota TEXT,
retention_policy_id TEXT,
create_time TEXT
);
CREATE TABLE IF NOT EXISTS approval_templates (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
steps TEXT,
create_time TEXT
);
CREATE TABLE IF NOT EXISTS approval_instances (
id TEXT PRIMARY KEY,
template_id TEXT,
resource_type TEXT,
resource_id TEXT,
applicant_id TEXT,
status TEXT DEFAULT 'pending',
current_step INTEGER DEFAULT 0,
create_time TEXT
);
CREATE TABLE IF NOT EXISTS approval_steps (
id TEXT PRIMARY KEY,
instance_id TEXT,
step_index INTEGER,
approver_id TEXT,
status TEXT DEFAULT 'pending',
comment TEXT,
time TEXT
);
CREATE TABLE IF NOT EXISTS audit_logs (
id TEXT PRIMARY KEY,
tenant_id TEXT,
project_id TEXT,
actor_id TEXT,
action TEXT,
target_type TEXT,
target_id TEXT,
detail TEXT,
client_ip TEXT,
time TEXT
);
CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
CREATE TABLE IF NOT EXISTS retention_policies (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
scope TEXT,
rule TEXT,
status TEXT DEFAULT 'active',
create_time TEXT,
create_by TEXT,
updated_at TEXT
);

3
backend/app/db/sql/003_tenant_quota.sql Normal file
View File

@@ -0,0 +1,3 @@
-- 租户配额与保留策略扩展（如后续治理表需补列，可在此追加）
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS gpu_quota TEXT;
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;

91
backend/app/modules/approval/router.py Normal file
View File

@@ -0,0 +1,91 @@
from __future__ import annotations
from fastapi import APIRouter, Body
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/approvals", tags=["approval"])
@router.get("/templates")
def list_templates() -> dict[str, Any]:
return ok(get_platform_store().approval_templates())
@router.post("/templates")
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
if not payload.get("name"):
raise fail(400, "name 必填")
return ok(get_platform_store().create_approval_template(payload))
@router.get("/templates/{template_id}")
def get_template(template_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().approval_template(template_id))
except KeyError:
raise fail(404, "template not found")
@router.put("/templates/{template_id}")
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
try:
return ok(get_platform_store().update_approval_template(template_id, payload))
except KeyError:
raise fail(404, "template not found")
@router.delete("/templates/{template_id}")
def delete_template(template_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().delete_approval_template(template_id))
except KeyError:
raise fail(404, "template not found")
@router.get("")
def list_instances(status: str | None = None) -> dict[str, Any]:
return ok(get_platform_store().approval_instances(status=status))
@router.post("")
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
for field in ("resource_type", "resource_id", "applicant_id"):
if not payload.get(field):
raise fail(400, f"{field} 必填")
try:
return ok(get_platform_store().create_approval_instance(payload))
except KeyError:
raise fail(404, "template not found")
@router.get("/{instance_id}")
def get_instance(instance_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().approval_instance(instance_id))
except KeyError:
raise fail(404, "instance not found")
@router.post("/{instance_id}/steps/{step_index}/decision")
def decide(
instance_id: str,
step_index: int,
payload: dict[str, Any] = Body(...),
) -> dict[str, Any]:
if not payload.get("approver_id"):
raise fail(400, "approver_id 必填")
try:
return ok(
get_platform_store().decide_approval_step(
instance_id,
step_index,
approver_id=payload["approver_id"],
approved=bool(payload.get("approved", False)),
comment=payload.get("comment"),
)
)
except (KeyError, ValueError) as e:
raise fail(400, str(e))

845
backend/app/modules/data_process/algorithms.py
View File

File diff suppressed because it is too large Load Diff

61
backend/app/modules/data_process/storage.py
View File

@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
self._issued_staged_objects[temporary_path] = staged
return staged
def stage_copy(
self,
*,
batch_id: str,
source_reference: str,
expected_source_task_id: str,
expected_source_file_id: str,
task_id: str,
source_file_id: str,
version: int,
name: str,
) -> StagedSourceObject:
"""为不可变源对象创建独立目录项，不把大文件重新读入内存。"""
batch_id = _safe_component(batch_id, "batch id")
task_id = _safe_component(task_id, "task id")
source_file_id = _safe_component(source_file_id, "source file id")
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
raise DataProcessStorageError("invalid source file version")
basename = _safe_basename(name)
source_relative = self._relative_from_reference(source_reference)
if source_relative is None:
raise DataProcessStorageError("original source object is not available")
self._assert_expected_owner(
source_relative,
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
source_path = self._path_for_relative(source_relative)
try:
os.link(source_path, temporary_path, follow_symlinks=False)
copy_info = temporary_path.lstat()
if (
not stat.S_ISREG(copy_info.st_mode)
or source_info.st_dev != copy_info.st_dev
or source_info.st_ino != copy_info.st_ino
):
raise DataProcessStorageError("source storage object changed while copying")
except Exception:
temporary_path.unlink(missing_ok=True)
raise
relative_path = PurePosixPath(
task_id,
source_file_id,
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
staged = list(objects)
published: list[StagedSourceObject] = []

321
backend/app/modules/data_process/store.py
View File

@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
"preserve_lists": True,
}
_REGENERATION_MARKER_KEY = "_regeneration_prepared"
_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
_REPEAT_REQUEST_KEY = "_repeat_request_id"
_INTERNAL_CONFIG_KEYS = {
_REGENERATION_MARKER_KEY,
_REPEAT_SOURCE_TASK_KEY,
_REPEAT_REQUEST_KEY,
}
class DataProcessStoreError(RuntimeError):
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:20]}"
def repeat_task_id(source_task_id: str, request_id: str) -> str:
"""按源任务和请求幂等键生成稳定的新任务 ID。"""
digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
return f"dpt_{digest[:20]}"
def json_dumps(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
return _regeneration_marker(task) is not None
def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
"""过滤只供服务端维护的工作流标记。"""
return {
key: value
for key, value in (config or {}).items()
if key not in _INTERNAL_CONFIG_KEYS
}
def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
"""从 API 任务快照中移除服务端内部重新生成标记。"""
"""从 API 任务快照中移除服务端内部工作流标记。"""
if item is None:
return None
public = dict(item)
config = public.get("config")
if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
public["config"] = {
key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
}
if isinstance(config, dict):
public["config"] = _business_config(config)
return public
@@ -353,13 +375,7 @@ class DataProcessStore:
payload.get("description") or "",
payload["process_type"],
payload.get("source_dataset_id"),
json_dumps(
{
key: value
for key, value in (payload.get("config") or {}).items()
if key != _REGENERATION_MARKER_KEY
}
),
json_dumps(_business_config(payload.get("config"))),
payload.get("tenant_id"),
payload.get("project_id"),
payload.get("owner_id"),
@@ -373,6 +389,268 @@ class DataProcessStore:
raise ConflictError("data process task name already exists") from exc
return _public_task(_decode_row(row)) or {}
@staticmethod
def _repeat_response(
conn: psycopg.Connection[dict[str, Any]],
row: dict[str, Any],
*,
source_task_id: str,
created: bool,
) -> dict[str, Any]:
task_id = str(row["id"])
counts = conn.execute(
"""
SELECT
(SELECT COUNT(*) FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
(SELECT COUNT(*) FROM data_process_preview_items
WHERE task_id=%s) AS preview_count
""",
(task_id, task_id),
).fetchone() or {}
task = _public_task(_decode_row(row)) or {}
task["source_file_count"] = int(counts.get("source_file_count") or 0)
task["preview_count"] = int(counts.get("preview_count") or 0)
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": task["source_file_count"],
"copied_preview_count": task["preview_count"],
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
"""查找同一幂等请求已创建的新任务。"""
task_id = repeat_task_id(source_task_id, request_id)
with self.connect() as conn:
row = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s",
(task_id,),
).fetchone()
if row is None:
return None
decoded = _decode_row(row) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
return self._repeat_response(
conn,
row,
source_task_id=source_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
"""复制已确认任务的配置、源文件和预览，结果与发布数据保持独立。"""
task_id = repeat_task_id(source_task_id, request_id)
now = utcnow()
try:
with self.connect() as conn:
existing = conn.execute(
"SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
(task_id,),
).fetchone()
if existing is not None:
decoded = _decode_row(existing) or {}
config = decoded.get("config") or {}
if (
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
or config.get(_REPEAT_REQUEST_KEY) != request_id
):
raise ConflictError("再次生成请求与现有任务冲突")
if decoded.get("deleted_at"):
raise ConflictError("此次再次生成创建的任务已被删除，请重新发起")
return self._repeat_response(
conn,
existing,
source_task_id=source_task_id,
created=False,
)
source_task = self._task_in_connection(
conn,
source_task_id,
for_update=True,
)
if (
source_task.get("status") != "completed"
or source_task.get("results_confirmed") is False
):
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
raise ConflictError("源任务仍在处理切分，暂时不能再次生成")
if expected_updated_at != _serialize_value(source_task.get("updated_at")):
raise ConflictError("源任务已被其他操作修改，请刷新后重试")
source_files = conn.execute(
"""
SELECT * FROM data_process_source_files
WHERE task_id=%s AND deleted_at IS NULL
ORDER BY created_at, id
""",
(source_task_id,),
).fetchall()
source_file_ids = {str(row["id"]) for row in source_files}
if source_file_ids != set(file_copies):
raise ConflictError("源文件快照已变化，请刷新后重试")
previews = conn.execute(
"""
SELECT * FROM data_process_preview_items
WHERE task_id=%s
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
created_at, id
""",
(source_task_id,),
).fetchall()
if not previews:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
suffix = f"（再次生成-{task_id[-6:]}）"
base_name = str(source_task.get("name") or "数据处理任务")
repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
repeated_config = _business_config(source_task.get("config") or {})
repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
repeated_config[_REPEAT_REQUEST_KEY] = request_id
input_count = sum(int(row.get("record_count") or 0) for row in source_files)
task_row = conn.execute(
"""
INSERT INTO data_process_tasks
(id, name, description, status, process_type, source_dataset_id,
output_dataset_id, config, progress, input_count, output_count,
filtered_count, duplicate_count, error_count, failure_reason,
generation_run_id, results_confirmed, workflow_step,
preview_status, preview_progress, preview_run_id,
preview_failure_reason, preview_total_files,
preview_completed_files, tenant_id, project_id, owner_id,
approval_status, created_by, updated_by, created_at, updated_at)
VALUES
(%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
task_id,
repeated_name,
source_task.get("description") or "",
source_task["process_type"],
source_task.get("source_dataset_id"),
json_dumps(repeated_config),
input_count,
len(source_files),
len(source_files),
source_task.get("tenant_id"),
source_task.get("project_id"),
source_task.get("owner_id"),
source_task.get("approval_status") or "not_required",
source_task.get("created_by"),
source_task.get("created_by"),
now,
now,
),
).fetchone()
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
new_file_id = str(copy["id"])
storage_object_id, metadata = _source_storage_descriptor(
{
"storage_object_id": copy["storage_object_id"],
"metadata": _json_value(source.get("metadata"), {}),
},
task_id,
new_file_id,
)
file_id_map[old_file_id] = new_file_id
conn.execute(
"""
INSERT INTO data_process_source_files
(id, task_id, storage_object_id, name, size_bytes, record_count,
file_format, checksum_sha256, version_no, content,
content_preview, metadata, tenant_id, project_id, created_by,
created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
%s, %s, %s, %s, %s)
""",
(
new_file_id,
task_id,
storage_object_id,
source["name"],
source.get("size_bytes") or 0,
source.get("record_count") or 0,
source.get("file_format"),
source["checksum_sha256"],
source.get("content") or "",
source.get("content_preview"),
json_dumps(metadata),
source_task.get("tenant_id"),
source_task.get("project_id"),
source.get("created_by") or source_task.get("created_by"),
now,
now,
),
)
for preview in previews:
old_source_file_id = preview.get("source_file_id")
conn.execute(
"""
INSERT INTO data_process_preview_items
(id, task_id, source_file_id, original_content, edited_content,
source_start, source_end, source_start_line, source_end_line,
token_count, status, quality_score, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s)
""",
(
new_id("dpp"),
task_id,
file_id_map.get(str(old_source_file_id))
if old_source_file_id
else None,
preview.get("original_content") or "",
preview.get("edited_content") or "",
preview.get("source_start"),
preview.get("source_end"),
preview.get("source_start_line"),
preview.get("source_end_line"),
max(0, int(preview.get("token_count") or 0)),
preview.get("status") or "original",
json_dumps(_json_value(preview.get("quality_score"), {})),
now,
now,
),
)
return self._repeat_response(
conn,
task_row or {},
source_task_id=source_task_id,
created=True,
)
except psycopg.errors.UniqueViolation as exc:
raise ConflictError("再次生成任务名称或请求发生冲突，请重试") from exc
def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
lock = " FOR UPDATE" if for_update else ""
with self.connect() as conn:
@@ -654,14 +932,11 @@ class DataProcessStore:
"process type and source dataset cannot change during regeneration"
)
if payload.get("config") is not None:
next_config = {
key: value
for key, value in payload["config"].items()
if key != _REGENERATION_MARKER_KEY
}
current_marker = _regeneration_marker(task)
if current_marker:
next_config[_REGENERATION_MARKER_KEY] = current_marker
next_config = _business_config(payload["config"])
current_config = dict(task.get("config") or {})
for key in _INTERNAL_CONFIG_KEYS:
if key in current_config:
next_config[key] = current_config[key]
values["config"] = json_dumps(next_config)
invalidates_results = (
("config" in payload and payload.get("config") != task.get("config"))
@@ -753,8 +1028,10 @@ class DataProcessStore:
raise InvalidStateError("process_type cannot be changed during regeneration")
current_config = dict(task.get("config") or {})
next_config = dict(payload.get("config") or {})
next_config.pop(_REGENERATION_MARKER_KEY, None)
next_config = _business_config(payload.get("config"))
for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
if key in current_config:
next_config[key] = current_config[key]
preview_invalidated = _preview_config_changed(
process_type,
current_config,

244
backend/app/modules/project/router.py Normal file
View File

@@ -0,0 +1,244 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Depends, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/projects", tags=["project"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
def _require_no_pending_approval(resource_type: str, resource_id: str) -> None:
"""第 4 周：写操作审批拦截——存在待审批实例时拒绝执行。"""
store = get_platform_store()
pending = [
i for i in store.approval_instances(status="pending")
if i["resource_type"] == resource_type and i["resource_id"] == resource_id
]
if pending:
raise fail(409, "存在待审批的变更，请先完成审批")
def _require_approval_or_admin(
resource_type: str,
resource_id: str,
current_user: dict[str, Any],
action_desc: str = "",
) -> dict[str, Any] | None:
"""高风险操作审批旁路：admin 直接放行，普通用户创建审批实例（code=202）。"""
if is_admin(current_user):
return None
store = get_platform_store()
instance = store.create_approval_instance({
"resource_type": resource_type,
"resource_id": resource_id,
"applicant_id": current_user.get("id"),
"template_id": None,
})
return {
"code": 202,
"message": f"操作已提交审批，等待管理员批准：{action_desc}",
"data": {"approval_required": True, "approval_id": instance["id"]},
}
@router.get("")
def list_projects(
tenant_id: str = "default",
status: str | None = None,
keyword: str | None = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
store = get_platform_store()
projects = store.projects(tenant_id=tenant_id, status=status, keyword=keyword)
# #1 ACL 过滤：admin 直接放行，普通用户只能看到自己被授权的项目
accessible_ids = set(
filter_accessible_resource_ids("project", [p["id"] for p in projects], current_user)
)
filtered = [p for p in projects if p["id"] in accessible_ids]
return ok(filtered)
@router.post("")
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
proj = store.create_project(payload)
store.record_audit(
action="project.create",
actor_id=_actor(request) if request else None,
target_type="project",
target_id=proj["id"],
tenant_id=proj.get("tenant_id"),
detail=f"name={proj.get('name')}",
)
return ok(proj)
@router.get("/{project_id}")
def get_project(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
# #2 访问控制：普通用户无 read 权限则拒绝
if not has_resource_access("project", project_id, current_user, "read"):
raise fail(403, "no permission to access this project")
try:
return ok(get_platform_store().project(project_id))
except KeyError:
raise fail(404, "project not found")
@router.put("/{project_id}")
def update_project(
project_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access("project", project_id, current_user, "write"):
raise fail(403, "no permission to update this project")
store = get_platform_store()
try:
proj = store.update_project(project_id, payload)
except KeyError:
raise fail(404, "project not found")
store.record_audit(
action="project.update",
actor_id=_actor(request) if request else None,
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
detail=f"fields={','.join(payload.keys())}",
)
return ok(proj)
@router.post("/{project_id}/archive")
def archive_project(
project_id: str,
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
if pending:
return pending
store = get_platform_store()
try:
proj = store.archive_project(project_id)
except KeyError:
raise fail(404, "project not found")
store.record_audit(
action="project.archive",
actor_id=_actor(request) if request else None,
target_type="project",
target_id=project_id,
tenant_id=proj.get("tenant_id"),
)
return ok(proj)
@router.delete("/{project_id}")
def delete_project(
project_id: str,
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
_require_no_pending_approval("project", project_id)
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
if pending:
return pending
store = get_platform_store()
store.delete_project(project_id)
store.record_audit(
action="project.delete",
actor_id=_actor(request) if request else None,
target_type="project",
target_id=project_id,
)
return ok(None)
@router.get("/{project_id}/members")
def list_members(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
if not has_resource_access("project", project_id, current_user, "read"):
raise fail(403, "no permission to access this project")
try:
return ok(get_platform_store().project_members(project_id))
except KeyError:
raise fail(404, "project not found")
@router.post("/{project_id}/members")
def add_member(
project_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access("project", project_id, current_user, "write"):
raise fail(403, "no permission to manage members of this project")
store = get_platform_store()
try:
member = store.add_project_member(project_id, payload)
except KeyError:
raise fail(404, "project not found")
store.record_audit(
action="project.member.add",
actor_id=_actor(request) if request else None,
target_type="project.member",
target_id=project_id,
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
)
return ok(member)
@router.put("/{project_id}/members/{user_id}")
def update_member(
project_id: str,
user_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access("project", project_id, current_user, "write"):
raise fail(403, "no permission to manage members of this project")
store = get_platform_store()
try:
member = store.update_project_member_role(project_id, user_id, payload)
except KeyError:
raise fail(404, "project or member not found")
store.record_audit(
action="project.member.update",
actor_id=_actor(request) if request else None,
target_type="project.member",
target_id=project_id,
detail=f"user_id={user_id},role={payload.get('role')}",
)
return ok(member)
@router.delete("/{project_id}/members/{user_id}")
def remove_member(
project_id: str,
user_id: str,
request: Request = None,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
if not has_resource_access("project", project_id, current_user, "write"):
raise fail(403, "no permission to manage members of this project")
store = get_platform_store()
store.remove_project_member(project_id, user_id)
store.record_audit(
action="project.member.remove",
actor_id=_actor(request) if request else None,
target_type="project.member",
target_id=project_id,
detail=f"user_id={user_id}",
)
return ok(None)

1
backend/app/modules/resource/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Resource access control list (ACL) module."""

41
backend/app/modules/resource/router.py Normal file
View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/resources", tags=["resource"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
@router.get("/{resource_type}/{resource_id}/acl")
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]:
"""查询资源 ACL，返回按主体分组的权限列表。"""
return ok(get_platform_store().resource_acl(resource_type, resource_id))
@router.put("/{resource_type}/{resource_id}/acl")
def set_acl(
resource_type: str,
resource_id: str,
payload: dict[str, Any] = Body(...),
request: Request = None,
) -> dict[str, Any]:
"""设置资源 ACL，body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
entries = payload.get("entries") or []
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
get_platform_store().record_audit(
action="resource.acl.set",
actor_id=_actor(request) if request else None,
target_type=resource_type,
target_id=resource_id,
detail=f"entries={len(entries)}",
)
return ok(result)

75
backend/app/modules/retention/router.py Normal file
View File

@@ -0,0 +1,75 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/retention-policies", tags=["retention"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
@router.get("")
def list_policies() -> dict[str, Any]:
return ok(get_platform_store().retention_policies())
@router.post("")
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
if not payload.get("name"):
raise fail(400, "name 必填")
policy = get_platform_store().create_retention_policy(payload)
get_platform_store().record_audit(
action="retention.create",
actor_id=_actor(request) if request else None,
target_type="retention_policy",
target_id=policy["id"],
detail=f"name={policy.get('name')}",
)
return ok(policy)
@router.get("/{policy_id}")
def get_policy(policy_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().retention_policy(policy_id))
except KeyError:
raise fail(404, "retention policy not found")
@router.put("/{policy_id}")
def update_policy(
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
) -> dict[str, Any]:
store = get_platform_store()
try:
policy = store.update_retention_policy(policy_id, payload)
except KeyError:
raise fail(404, "retention policy not found")
store.record_audit(
action="retention.update",
actor_id=_actor(request) if request else None,
target_type="retention_policy",
target_id=policy_id,
detail=f"fields={','.join(payload.keys())}",
)
return ok(policy)
@router.delete("/{policy_id}")
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
store = get_platform_store()
store.delete_retention_policy(policy_id)
store.record_audit(
action="retention.delete",
actor_id=_actor(request) if request else None,
target_type="retention_policy",
target_id=policy_id,
)
return ok({"deleted": policy_id})

93
backend/app/modules/system/router.py Normal file
View File

@@ -0,0 +1,93 @@
from __future__ import annotations
from fastapi import APIRouter, Query
from fastapi.responses import StreamingResponse
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
router = APIRouter(prefix="/system", tags=["system"])
@router.get("/permissions/codes")
def permission_codes() -> dict:
"""返回平台权限码清单（权限码接口）。"""
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
@router.get("/permissions")
def permissions_overview() -> dict:
"""返回权限码清单与角色定义。"""
store = get_platform_store()
return {
"code": 0,
"message": "ok",
"data": {"codes": ALL_PERMISSIONS, "roles": store.roles()},
}
@router.get("/audit-logs")
def audit_logs(
tenant_id: str | None = Query(default=None, description="租户 ID"),
project_id: str | None = Query(default=None, description="项目 ID"),
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> dict:
"""审计日志查询：按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
store = get_platform_store()
result = store.audit_logs(
tenant_id=tenant_id,
project_id=project_id,
actor_id=actor_id,
action=action,
target_type=target_type,
start_time=start_time,
end_time=end_time,
limit=limit,
offset=offset,
)
return {"code": 0, "message": "ok", "data": result}
@router.get("/audit-logs/export")
def audit_logs_export(
tenant_id: str | None = Query(default=None, description="租户 ID"),
project_id: str | None = Query(default=None, description="项目 ID"),
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
) -> StreamingResponse:
"""审计日志导出：返回 CSV 流，与应用查询相同的过滤条件。"""
store = get_platform_store()
result = store.audit_logs(
tenant_id=tenant_id,
project_id=project_id,
actor_id=actor_id,
action=action,
target_type=target_type,
start_time=start_time,
end_time=end_time,
limit=10000,
offset=0,
)
items = result["items"]
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
header = ",".join(columns) + "\n"
def iter_rows():
yield header
for row in items:
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
return StreamingResponse(
iter_rows(),
media_type="text/csv",
headers={"Content-Disposition": "attachment; filename=audit_logs.csv"},
)

116
backend/app/modules/tenant/router.py Normal file
View File

@@ -0,0 +1,116 @@
from __future__ import annotations
from fastapi import APIRouter, Body, Request
from typing import Any
from app.api.v1.endpoints.platform import ok, fail
from app.db.platform_store import get_platform_store
router = APIRouter(prefix="/tenants", tags=["tenant"])
def _actor(request: Request) -> str | None:
auth = request.headers.get("Authorization", "")
token = auth.replace("Bearer ", "").strip()
return token or None
@router.get("")
def list_tenants() -> dict[str, Any]:
return ok(get_platform_store().tenants())
@router.post("")
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.create_tenant(payload)
except KeyError as e:
raise fail(400, f"missing field: {e}")
store.record_audit(
action="tenant.create",
actor_id=_actor(request) if request else None,
target_type="tenant",
target_id=tenant["id"],
tenant_id=tenant["id"],
detail=f"name={tenant.get('name')}",
)
return ok(tenant)
@router.get("/{tenant_id}")
def get_tenant(tenant_id: str) -> dict[str, Any]:
try:
return ok(get_platform_store().tenant(tenant_id))
except KeyError:
raise fail(404, "tenant not found")
@router.put("/{tenant_id}")
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.update_tenant(tenant_id, payload)
except KeyError:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.update",
actor_id=_actor(request) if request else None,
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"fields={','.join(payload.keys())}",
)
return ok(tenant)
@router.put("/{tenant_id}/quota")
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_quota(tenant_id, payload.get("quota", {}))
except KeyError:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.quota.set",
actor_id=_actor(request) if request else None,
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
)
return ok(tenant)
@router.put("/{tenant_id}/retention-policy")
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
except KeyError:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.retention.set",
actor_id=_actor(request) if request else None,
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
)
return ok(tenant)
@router.delete("/{tenant_id}")
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
store = get_platform_store()
try:
tenant = store.delete_tenant(tenant_id)
except KeyError:
raise fail(404, "tenant not found")
store.record_audit(
action="tenant.delete",
actor_id=_actor(request) if request else None,
target_type="tenant",
target_id=tenant_id,
tenant_id=tenant_id,
detail=f"name={tenant.get('name')}",
)
return ok(tenant)

13
backend/app/schemas/data_process.py
View File

@@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
return self
class DataProcessRepeatRequest(BaseModel):
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
model_config = ConfigDict(extra="forbid")
expected_updated_at: str = Field(min_length=1)
request_id: str = Field(
min_length=8,
max_length=80,
pattern=r"^[A-Za-z0-9_-]+$",
)
class PreviewBuildRequest(BaseModel):
model_config = ConfigDict(extra="forbid")

303
backend/tests/test_data_process_algorithms.py
View File

@@ -5,6 +5,7 @@ import json
import xml.etree.ElementTree as ET
import zipfile
from datetime import datetime
from decimal import Decimal
import pytest
from docx import Document
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
normalize_text,
parse_text_content,
preprocess_structured_records,
preprocess_structured_records_with_lineage,
record_fingerprint,
remove_document_noise,
score_quality,
stable_split,
stable_split_assignments,
structured_json_dumps,
)
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
assert parsed_txt.text == "普通文本"
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
root_json = parse_text_content('{"id":1}', filename="root.json")
assert root_json.record_locators == (
{
"kind": "json",
"record_index": 1,
"json_pointer": "",
"source_start": 0,
"source_end": 8,
"start_line": 1,
"end_line": 1,
},
)
wrapped_json = parse_text_content(
'{"records":[{"id":1},{"id":1}]}',
filename="wrapped.json",
)
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
"/records/0",
"/records/1",
]
parsed_jsonl = parse_text_content(
'{"id":1}\r\n\r\n{"id":1}',
filename="records.jsonl",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_jsonl.record_locators
] == [(1, 1, 1), (2, 3, 3)]
assert [
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_jsonl.record_locators
] == ['{"id":1}', '{"id":1}']
parsed_csv = parse_text_content(
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
filename="records.csv",
)
assert [
(locator["record_index"], locator["start_line"], locator["end_line"])
for locator in parsed_csv.record_locators
] == [(1, 2, 3), (2, 5, 5)]
assert [
parsed_csv.text[locator["source_start"] : locator["source_end"]]
for locator in parsed_csv.record_locators
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
processed = preprocess_structured_records_with_lineage(
[
{"id": "A", "value": "first", "empty": ""},
{"id": "", "value": "invalid", "empty": ""},
{"id": "A", "value": "duplicate identity", "empty": ""},
{"id": "B", "value": "second", "empty": ""},
],
["clean_invalid", "deduplicate"],
)
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
assert [entry.record for entry in processed] == [
{"id": "A", "value": "first"},
{"id": "", "value": "invalid"},
{"id": "A", "value": "duplicate identity"},
{"id": "B", "value": "second"},
]
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
assert parsed_pdf.format == "pdf"
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
)
assert parsed_xlsx.record_locators == (
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "数据",
"row_number": 3,
"sheet_record_index": 1,
},
)
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active
first.title = "甲表"
first.append(["说明"])
first.append([])
first.append(["id", "value"])
first.append([1, "same"])
first.append([1, "same"])
second = workbook.create_sheet("乙表")
second.append(["id", "value"])
second.append([1, "same"])
output = io.BytesIO()
workbook.save(output)
workbook.close()
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
assert parsed.records == (
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
{"id": 1, "value": "same"},
)
assert [
(
locator["record_index"],
locator["sheet_index"],
locator["sheet_name"],
locator["row_number"],
locator["sheet_record_index"],
)
for locator in parsed.record_locators
] == [
(1, 0, "甲表", 4, 0),
(2, 0, "甲表", 5, 1),
(3, 1, "乙表", 2, 0),
]
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
pages = _pdf_page_texts(
"""
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
"json",
)
assert result == [{"text": "内容"}]
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
assert extract_structured_records(
'{"items":[{"text":" 内容 "}],"total":1}',
"json",
) == [{"text": " 内容 "}]
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
source = '{"code":"００１","text":" 内容 ","quote":"＂"}'
parsed = parse_text_content(source, filename="records.json")
assert parsed.text == source
assert parsed.records == (
{"code": "００１", "text": " 内容 ", "quote": "＂"},
)
invalid_values = (
'{"id":1,"id":2}',
'{"nested":{"id":1,"id":2}}',
'{"value":NaN}',
'{"value":Infinity}',
'{"value":-Infinity}',
'{"value":"bad\x00control"}',
)
for invalid in invalid_values:
with pytest.raises(ValueError):
parse_text_content(invalid, filename="invalid.json")
with pytest.raises(ValueError):
parse_text_content("｛\"id\":1｝", filename="invalid.json")
with pytest.raises(ValueError, match="nesting exceeds"):
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
source = (
' {"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}\r\n\r\n'
'{"id":2}\r\n'
)
parsed = parse_text_content(source, filename="records.jsonl")
assert parsed.text == source
assert parsed.records[0] == {
"code": "００１",
"text": " 内容 ",
"value": Decimal("0.123456789012345678901234567890"),
}
assert [
source[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == [
(
'{"code":"００１","text":" 内容 ",'
'"value":0.123456789012345678901234567890}'
),
'{"id":2}',
]
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
parse_text_content(invalid, filename="invalid.jsonl")
def test_json_record_contract_avoids_business_field_collisions() -> None:
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
{"id": 1},
{"id": 2},
]
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
{"id": 1, "data": [{"id": 2}]}
]
assert extract_structured_records(
'{"records":[{"id":1}],"data":[{"id":2}]}',
"json",
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
assert extract_structured_records(
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
"json",
) == [{"id": 1}]
assert extract_structured_records(
'{"payload":{"data":[{"id":2}],"total":1}}',
"json",
) == [{"id": 2}]
assert extract_structured_records('{"records":[],"total":0}', "json") == []
# 包装数组中的非对象不是记录集合，整体按一条业务对象保留。
assert extract_structured_records('{"data":[1,2]}', "json") == [
{"data": [1, 2]}
]
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
pretty = (
'{\n "records": [\n {"id": 1},\n'
' {\n "id": 2\n }\n ],\n "total": 2\n}'
)
parsed = parse_text_content(pretty, filename="pretty.json")
assert [
pretty[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id": 1}', '{\n "id": 2\n }']
assert [
(locator["start_line"], locator["end_line"])
for locator in parsed.record_locators
] == [(3, 3), (4, 6)]
minified = '[{"id":1},{"id":2}]'
parsed = parse_text_content(minified, filename="minified.json")
assert [
minified[locator["source_start"] : locator["source_end"]]
for locator in parsed.record_locators
] == ['{"id":1}', '{"id":2}']
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
parsed = parse_text_content(source, filename="precise.json")
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
assert parsed.records[1]["value"] == Decimal("1e400")
assert structured_json_dumps(parsed.records[0]) == (
'{"value":0.123456789012345678901234567890}'
)
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
assert isinstance(parsed.records[0]["value"], Decimal)
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
assert preprocess_structured_records(clean_source, []) == clean_source
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
{"id": "1", "name": "有效"},
{"id": "", "name": "缺少关键字段"},
{"id": "2", "name": "有效"},
]
hierarchy = [
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
{"id": "", "parent_id": "", "name": "", "empty": ""},
]
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
{"id": "1", "parent_id": None, "name": "根节点"},
{"id": "2", "parent_id": "1", "name": "子节点"},
]
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
assert "profile" in preprocess_structured_records(nested, [])[0]
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
duplicates = [
{"customer_id": "C-1", "value": "first"},
{"customer_id": "C-1", "value": "updated"},
{"customer_id": "C-1", "value": "first"},
{"customer_id": "", "value": "blank-one"},
{"customer_id": "", "value": "blank-two"},
]
assert len(preprocess_structured_records(duplicates, [])) == 4
assert len(preprocess_structured_records(duplicates, [])) == 5
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
assert [record["value"] for record in deduplicated] == [
"first",
"updated",
"blank-one",
"blank-two",
]
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
)
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
masked, counts = desensitize_structured_record(
{
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "张三",
"contact_name": "李四",
"姓名": "王五",
"profile.name": "赵六",
}
)
assert masked == {
"table_name": "customer_profile",
"chinese_name": "zh_CN",
"english_name": "en_US",
"product_name": "智能助手",
"metadata.table_name": "customer_archive",
"name": "[NAME]",
"contact_name": "[NAME]",
"姓名": "[NAME]",
"profile.name": "[NAME]",
}
assert counts == {
"email": 0,
"phone": 0,
"id_card": 0,
"name": 4,
"total": 4,
}
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
valid = {
"instruction": "如何修改收货地址？",

564
backend/tests/test_data_process_api.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from copy import deepcopy
from io import BytesIO
from pathlib import Path
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
LocalDataProcessStorage,
get_data_process_storage,
)
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
from app.modules.data_process.store import (
InvalidStateError,
NotFoundError,
get_data_process_store,
repeat_task_id,
)
class FakeDataProcessStore:
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
self.datasets: dict[str, dict[str, Any]] = {}
self.models: dict[str, dict[str, Any]] = {}
self.regeneration_prepared: set[str] = set()
self.repeat_requests: dict[tuple[str, str], str] = {}
self.sequence = 0
def _id(self, prefix: str) -> str:
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
"published_outputs_preserved": published_outputs_preserved,
}
def _repeat_response(
self,
source_task_id: str,
repeated_task_id: str,
*,
created: bool,
) -> dict[str, Any]:
task = self.get_task(repeated_task_id)
task["source_file_count"] = len(self.sources[repeated_task_id])
task["preview_count"] = len(self.previews[repeated_task_id])
return {
"task": task,
"source_task_id": source_task_id,
"created": created,
"copied_source_file_count": len(self.sources[repeated_task_id]),
"copied_preview_count": len(self.previews[repeated_task_id]),
}
def find_repeated_task(
self,
source_task_id: str,
request_id: str,
) -> dict[str, Any] | None:
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
if repeated_task_id is None:
return None
return self._repeat_response(
source_task_id,
repeated_task_id,
created=False,
)
def repeat_task(
self,
source_task_id: str,
*,
expected_updated_at: str,
request_id: str,
file_copies: dict[str, dict[str, str]],
) -> dict[str, Any]:
existing = self.find_repeated_task(source_task_id, request_id)
if existing is not None:
return existing
source_task = self.get_task(source_task_id)
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
if source_task.get("updated_at") != expected_updated_at:
raise InvalidStateError("源任务已被其他操作修改，请刷新后重试")
source_files = self.sources[source_task_id]
if set(file_copies) != {str(item["id"]) for item in source_files}:
raise InvalidStateError("源文件快照已变化，请刷新后重试")
if not self.previews[source_task_id]:
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
repeated_task_id = repeat_task_id(source_task_id, request_id)
suffix = f"（再次生成-{repeated_task_id[-6:]}）"
task = {
**deepcopy(source_task),
"id": repeated_task_id,
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
"status": "pending",
"progress": 20,
"output_dataset_id": None,
"output_datasets": [],
"output_count": 0,
"filtered_count": 0,
"duplicate_count": 0,
"error_count": 0,
"failure_reason": None,
"generation_run_id": None,
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_run_id": None,
"preview_failure_reason": None,
"preview_total_files": len(source_files),
"preview_completed_files": len(source_files),
"started_at": None,
"completed_at": None,
}
self.tasks[repeated_task_id] = task
self.sources[repeated_task_id] = []
file_id_map: dict[str, str] = {}
for source in source_files:
old_file_id = str(source["id"])
copy = file_copies[old_file_id]
file_id_map[old_file_id] = copy["id"]
self.sources[repeated_task_id].append(
{
**deepcopy(source),
"id": copy["id"],
"task_id": repeated_task_id,
"storage_object_id": copy["storage_object_id"],
}
)
self.previews[repeated_task_id] = [
{
**deepcopy(item),
"id": self._id("dpp"),
"task_id": repeated_task_id,
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
if item.get("source_file_id")
else None,
}
for item in self.previews[source_task_id]
]
self.results[repeated_task_id] = []
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
return self._repeat_response(
source_task_id,
repeated_task_id,
created=True,
)
def delete_task(self, task_id: str, **_: Any) -> None:
self.get_task(task_id)
del self.tasks[task_id]
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
assert listed_preview.json()["data"]["total"] == 2
preview_item = listed_preview.json()["data"]["items"][0]
source_locator = preview_item["quality_score"]["source_locator"]
assert source_locator == {
"kind": "jsonl",
"record_index": 1,
"start_line": 1,
"end_line": 1,
"source_start": 0,
"source_end": len(preview_item["original_content"]),
}
updated_preview = client.put(
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
json={
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
},
)
assert "quality_score" in updated_preview.json()["data"]
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
assert generated.status_code == 200
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
def test_completed_task_can_repeat_into_an_independent_background_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, storage = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "原始生成任务",
"process_type": "structured",
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
)
assert uploaded.status_code == 200
source_id = uploaded.json()["data"]["files"][0]["id"]
built = client.post(
f"/modelTF/data-process/{task_id}/preview/build",
json={"replace_existing": True},
)
assert built.status_code == 200
store.tasks[task_id].update(
status="completed",
progress=100,
results_confirmed=True,
workflow_step="results",
output_count=1,
output_dataset_id="dataset-original",
updated_at="2026-07-28T12:00:00Z",
)
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
store.datasets["dataset-original"] = {
"id": "dataset-original",
"name": "原数据集",
"type": "train",
"source_task_id": task_id,
"deleted_at": None,
}
original_task = deepcopy(store.tasks[task_id])
original_sources = deepcopy(store.sources[task_id])
original_previews = deepcopy(store.previews[task_id])
original_results = deepcopy(store.results[task_id])
original_datasets = deepcopy(store.datasets)
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
payload = {
"expected_updated_at": "2026-07-28T12:00:00Z",
"request_id": "repeat-request-0001",
}
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert response.status_code == 202
repeated = response.json()["data"]
repeated_task_id = repeated["task"]["id"]
assert repeated["created"] is True
assert repeated_task_id != task_id
assert repeated["task"]["status"] == "running"
assert repeated["task"]["workflow_step"] == "generate"
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == len(original_previews)
assert store.tasks[task_id] == original_task
assert store.sources[task_id] == original_sources
assert store.previews[task_id] == original_previews
assert store.results[task_id] == original_results
assert store.datasets == original_datasets
repeated_source = store.sources[repeated_task_id][0]
repeated_preview = store.previews[repeated_task_id][0]
assert repeated_source["id"] != source_id
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
assert repeated_preview["id"] != original_previews[0]["id"]
assert repeated_preview["source_file_id"] == repeated_source["id"]
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
assert replay.status_code == 202
assert replay.json()["data"]["created"] is False
assert replay.json()["data"]["task"]["id"] == repeated_task_id
assert len(store.tasks) == 2
assert len(store.sources[repeated_task_id]) == 1
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "源任务", "process_type": "structured", "config": {}},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
results_confirmed=True,
updated_at="2026-07-28T12:00:00Z",
)
before = deepcopy(store.tasks)
response = client.post(
f"/modelTF/data-process/{task_id}/repeat",
json={
"expected_updated_at": "2026-07-28T11:59:59Z",
"request_id": "repeat-request-stale",
},
)
assert response.status_code == 409
assert store.tasks == before
def test_published_split_datasets_remain_in_detail_after_regeneration(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "先去重再脱敏",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={
"files": (
"names.jsonl",
(
'{"name":"张三","role":"开发"}\n'
'{"name":"李四","role":"开发"}\n'
),
"application/jsonl",
)
},
)
assert uploaded.status_code == 200
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert len(items) == 2
assert len({item["original_content"] for item in items}) == 2
assert {item["edited_content"] for item in items} == {
'{"name":"[NAME]","role":"开发"}'
}
def test_structured_deduplication_removes_identical_rows_across_sources(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={
"name": "跨源原文去重",
"process_type": "structured",
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
(
"files",
(
"first.jsonl",
'{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
(
"files",
(
"second.jsonl",
'\n{"name":"张三","role":"开发"}\n',
"application/jsonl",
),
),
],
)
assert uploaded.status_code == 200
first_source, second_source = uploaded.json()["data"]["files"]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
data = preview.json()["data"]
assert data["total"] == 1
assert data["file_counts"] == {
first_source["id"]: 1,
second_source["id"]: 0,
}
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
store = FakeDataProcessStore()
task = store.create_task(
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
)
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["total"] == 2
preview_items = preview.json()["data"]["items"]
assert len(preview_items) == 2
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
{
"kind": "xlsx",
"record_index": 1,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 2,
"sheet_record_index": 0,
},
{
"kind": "xlsx",
"record_index": 2,
"sheet_index": 0,
"sheet_name": "Sheet",
"row_number": 3,
"sheet_record_index": 1,
},
]
def test_docx_preview_preserves_document_block_order_and_source_offsets(
@@ -2757,6 +3105,218 @@ def _preview_task(
)
def _structured_preview_task(
content: str,
*,
file_format: str,
options: list[str] | None = None,
) -> list[dict[str, Any]]:
return data_process_endpoint._build_preview_items(
{
"process_type": "structured",
"config": {"preprocess_options": options or []},
},
[
{
"id": "structured-source",
"name": f"records.{file_format}",
"file_format": file_format,
"content": content,
}
],
)
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
json_source = '{"records":[{"id":1},{"id":2}]}'
json_items = _structured_preview_task(
json_source,
file_format="json",
)
assert [
item["quality_score"]["source_locator"]["json_pointer"]
for item in json_items
] == ["/records/0", "/records/1"]
assert [
json_source[item["source_start"] : item["source_end"]]
for item in json_items
] == ['{"id":1}', '{"id":2}']
assert [item["source_start_line"] for item in json_items] == [1, 1]
jsonl_source = '{"id":1}\n\n{"id":2}'
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in jsonl_items
] == [1, 2]
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
assert [
jsonl_source[item["source_start"] : item["source_end"]]
for item in jsonl_items
] == ['{"id":1}', '{"id":2}']
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
csv_items = _structured_preview_task(csv_source, file_format="csv")
assert [
(item["source_start_line"], item["source_end_line"])
for item in csv_items
] == [(2, 3), (5, 5)]
assert [
csv_source[item["source_start"] : item["source_end"]]
for item in csv_items
] == ['1,"hello\nworld"', "2,plain"]
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "空 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("empty-array.json", "[]", "application/json")),
(
"files",
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
assert preview.json()["data"]["items"] == []
assert preview.json()["data"]["total"] == 0
assert set(preview.json()["data"]["file_counts"].values()) == {0}
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
invalid_sources = (
("duplicate.json", '{"id":1,"id":2}'),
("duplicate.jsonl", '{"id":1,"id":2}\n'),
("nan.json", '{"value":NaN}'),
("infinity.json", '{"value":Infinity}'),
("control.json", '{"value":"bad\x00control"}'),
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
)
for filename, content in invalid_sources:
response = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files={"files": (filename, content, "application/json")},
)
assert response.status_code == 400, (filename, response.text)
def test_structured_json_preview_preserves_precision_and_business_data_field(
tmp_path: Path,
) -> None:
client, _, _ = make_client(tmp_path)
task_id = client.post(
"/modelTF/data-process",
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
).json()["data"]["id"]
precise = '{"value":0.123456789012345678901234567890}'
business = '{"id":7,"data":[{"id":8}]}'
uploaded = client.post(
f"/modelTF/data-process/{task_id}/source-files",
files=[
("files", ("precise.json", precise, "application/json")),
("files", ("business.json", business, "application/json")),
],
)
assert uploaded.status_code == 200
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
assert preview.status_code == 200
items = preview.json()["data"]["items"]
assert [item["original_content"] for item in items] == [precise, business]
assert [
item["quality_score"]["source_locator"]["json_pointer"] for item in items
] == ["", ""]
assert [item["source_start"] for item in items] == [0, 0]
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
source_records = [
{"id": "A", "amount": 10, "empty": ""},
{"id": "A", "amount": 10, "empty": ""},
{"id": "", "amount": 11, "empty": ""},
{"id": "B", "amount": 11, "empty": ""},
{"id": "C", "amount": 12, "empty": ""},
{"id": "D", "amount": 12, "empty": ""},
{"id": "E", "amount": 13, "empty": ""},
{"id": "F", "amount": 13, "empty": ""},
{"id": "G", "amount": 14, "empty": ""},
{"id": "H", "amount": 1000, "empty": ""},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate", "filter_anomaly"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
assert [json.loads(item["original_content"])["id"] for item in items] == [
"A",
"",
"B",
"C",
"D",
"E",
"F",
"G",
]
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
source_records = [
{"customer_id": "C-1", "status": "old"},
{"customer_id": "C-1", "status": "new"},
{"status": "old", "customer_id": "C-1"},
]
source = "\n".join(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
for record in source_records
)
items = _structured_preview_task(
source,
file_format="jsonl",
options=["clean_invalid", "deduplicate"],
)
assert [
item["quality_score"]["source_locator"]["record_index"]
for item in items
] == [1, 2]
assert [item["source_start_line"] for item in items] == [1, 2]
assert [json.loads(item["original_content"])["status"] for item in items] == [
"old",
"new",
]
def test_fixed_preview_preserves_source_offsets() -> None:
content = (
"# 第一章\n"

34
backend/tests/test_data_process_storage.py
View File

@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
_assert_staging_empty(storage)
def test_stage_copy_creates_an_independently_deletable_source_object(
tmp_path: Path,
) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")
original = _stage(storage, content=b"immutable source")
storage.publish([original])
copied = storage.stage_copy(
batch_id="batch-copy",
source_reference=original.reference,
expected_source_task_id="task-1",
expected_source_file_id="source-1",
task_id="task-2",
source_file_id="source-2",
version=1,
name="source.txt",
)
storage.publish([copied])
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
original.reference,
expected_task_id="task-1",
expected_source_file_id="source-1",
) is True
assert storage.read(copied.reference) == b"immutable source"
assert storage.delete(
copied.reference,
expected_task_id="task-2",
expected_source_file_id="source-2",
) is True
_assert_staging_empty(storage)
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
storage = LocalDataProcessStorage(tmp_path / "storage")

176
backend/tests/test_data_process_store.py
View File

@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
_preview_config_changed,
_reasoning_output_is_valid,
_source_storage_descriptor,
repeat_task_id,
)
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
yield self._conn
class _RepeatConnection:
def __init__(self) -> None:
self.source_files = [
{
"id": "source-old",
"name": "source.jsonl",
"size_bytes": 12,
"record_count": 1,
"file_format": "jsonl",
"checksum_sha256": "a" * 64,
"content": '{"id":1}\n',
"content_preview": '{"id":1}',
"metadata": {"storage_backend": "local"},
"created_by": "user-1",
}
]
self.source_previews = [
{
"id": "preview-old",
"source_file_id": "source-old",
"original_content": '{"id":1}',
"edited_content": '{"id":1,"checked":true}',
"source_start": 0,
"source_end": 8,
"source_start_line": 1,
"source_end_line": 1,
"token_count": 5,
"status": "modified",
"quality_score": {"overall": 90},
}
]
self.created_task: dict[str, Any] | None = None
self.created_files: list[dict[str, Any]] = []
self.created_previews: list[dict[str, Any]] = []
def execute(self, sql: str, params: Any = None) -> _Result:
normalized = " ".join(sql.split())
if params is not None:
assert normalized.count("%s") == len(params)
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
return _Result(row=None)
if normalized.startswith("SELECT * FROM data_process_source_files"):
return _Result(rows=[dict(item) for item in self.source_files])
if normalized.startswith("SELECT * FROM data_process_preview_items"):
return _Result(rows=[dict(item) for item in self.source_previews])
if normalized.startswith("INSERT INTO data_process_tasks"):
self.created_task = {
"id": params[0],
"name": params[1],
"description": params[2],
"status": "pending",
"process_type": params[3],
"source_dataset_id": params[4],
"config": params[5],
"progress": 20,
"input_count": params[6],
"results_confirmed": False,
"workflow_step": "preview",
"preview_status": "completed",
"preview_progress": 100,
"preview_total_files": params[7],
"preview_completed_files": params[8],
"created_at": params[15],
"updated_at": params[16],
}
return _Result(row=dict(self.created_task))
if normalized.startswith("INSERT INTO data_process_source_files"):
self.created_files.append(
{
"id": params[0],
"task_id": params[1],
"storage_object_id": params[2],
"content": params[8],
}
)
return _Result()
if normalized.startswith("INSERT INTO data_process_preview_items"):
self.created_previews.append(
{
"id": params[0],
"task_id": params[1],
"source_file_id": params[2],
"edited_content": params[4],
}
)
return _Result()
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
return _Result(
row={
"source_file_count": len(self.created_files),
"preview_count": len(self.created_previews),
}
)
raise AssertionError(f"unexpected SQL: {normalized}")
class _RepeatStore(DataProcessStore):
def __init__(self, conn: _RepeatConnection) -> None:
self._conn = conn
@contextmanager
def connect(self) -> Iterator[_RepeatConnection]:
yield self._conn
def _task_in_connection(
self,
conn: Any,
task_id: str,
*,
for_update: bool = False,
) -> dict[str, Any]:
assert task_id == "task-source"
assert for_update is True
return {
"id": task_id,
"name": "原任务",
"description": "原描述",
"status": "completed",
"process_type": "structured",
"source_dataset_id": None,
"config": {
"temperature": 0.3,
"_regeneration_prepared": {"prepared": True},
},
"results_confirmed": True,
"preview_status": "completed",
"tenant_id": "tenant-1",
"project_id": "project-1",
"owner_id": "owner-1",
"created_by": "user-1",
"updated_at": "2026-07-28T12:00:00Z",
}
class _TaskListConnection:
def __init__(self) -> None:
self.task = {
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
conn = _RepeatConnection()
store = _RepeatStore(conn)
request_id = "repeat-request-0001"
target_task_id = repeat_task_id("task-source", request_id)
repeated = store.repeat_task(
"task-source",
expected_updated_at="2026-07-28T12:00:00Z",
request_id=request_id,
file_copies={
"source-old": {
"id": "source-new",
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
}
},
)
assert repeated["created"] is True
assert repeated["task"]["id"] == target_task_id
assert repeated["task"]["config"] == {"temperature": 0.3}
assert repeated["task"]["results_confirmed"] is False
assert repeated["copied_source_file_count"] == 1
assert repeated["copied_preview_count"] == 1
assert conn.created_files == [
{
"id": "source-new",
"task_id": target_task_id,
"storage_object_id": (
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
),
"content": '{"id":1}\n',
}
]
assert conn.created_previews[0]["task_id"] == target_task_id
assert conn.created_previews[0]["source_file_id"] == "source-new"
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
decoded = _decode_row(
{

774
backend/tests/test_governance.py Normal file
View File

@@ -0,0 +1,774 @@
"""
平台治理功能集成测试 —— 覆盖第 1-4 周交付内容。
测试策略：
- 在导入 app 模块前 mock psycopg / psycopg_pool，避免依赖真实数据库驱动
- 使用 FastAPI TestClient 对真实路由栈发起请求
- 通过 mock.get_platform_store 替换为内存 FakeStore
- 每周交付内容对应一组 test class，方便分阶段验收
覆盖范围：
第 1 周 — 登录、当前用户、用户列表、权限码、日志查询
第 2 周 — 租户、项目、项目成员、资源 ACL
第 3 周 — 审批实例、审批模板、审计日志查询和导出
第 4 周 — 写操作审计、审批拦截、权限校验
"""
from __future__ import annotations
import json
import sys
import types
from contextlib import contextmanager
from typing import Any, Iterator
from unittest.mock import MagicMock, patch
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
# ============================================================
# 在导入 app 之前 mock psycopg / psycopg_pool
# ============================================================
_psycopg_mock = types.ModuleType("psycopg")
_psycopg_mock.PgConn = type("PgConn", (), {})
_psycopg_mock.PostgresConnectionPool = MagicMock()
_psycopg_mock.connection = MagicMock()
sys.modules.setdefault("psycopg", _psycopg_mock)
_psycopg_pool_mock = types.ModuleType("psycopg_pool")
_psycopg_pool_mock.ConnectionPool = MagicMock()
sys.modules.setdefault("psycopg_pool", _psycopg_pool_mock)
# 现在安全导入 app 模块
from app.api.v1.endpoints.platform import ok, fail # noqa: E402
from app.modules.tenant.router import router as tenant_router # noqa: E402
from app.modules.project.router import router as project_router # noqa: E402
from app.modules.approval.router import router as approval_router # noqa: E402
from app.modules.system.router import router as system_router # noqa: E402
from app.modules.retention.router import router as retention_router # noqa: E402
from app.modules.resource.router import router as resource_router # noqa: E402
from app.api.v1.endpoints.platform import router as platform_router # noqa: E402
PREFIX = "/modelTF"
ADMIN_TOKEN = "platform-token-u_admin"
OP_TOKEN = "platform-token-u_op"
# ============================================================
# FakePlatformStore —— 内存实现，模拟 PlatformStore 全部治理接口
# ============================================================
class FakePlatformStore:
"""平台治理测试专用内存 store，确保测试不连接真实数据库。"""
def __init__(self) -> None:
self._users: list[dict[str, Any]] = [
{
"id": "u_admin",
"username": "admin",
"display_name": "Admin",
"role": "admin",
"status": "active",
"permissions": [
"dashboard", "fine-tune", "model-eval", "model-inference",
"model-manage", "dataset", "data-process", "data-convert",
"compute", "hardware", "logs", "user-settings",
],
"last_login": "2026-08-01T10:00:00Z",
"protected": True,
},
{
"id": "u_op",
"username": "operator",
"display_name": "Operator",
"role": "operator",
"status": "active",
"permissions": ["dashboard", "fine-tune"],
"last_login": "2026-08-01T11:00:00Z",
"protected": False,
},
]
self._tenants: dict[str, dict[str, Any]] = {}
self._projects: dict[str, dict[str, Any]] = {}
self._members: dict[str, list[dict[str, Any]]] = {}
self._acl: dict[str, list[dict[str, Any]]] = {}
self._audit_logs: list[dict[str, Any]] = []
self._approval_templates: dict[str, dict[str, Any]] = {}
self._approval_instances: dict[str, dict[str, Any]] = {}
self._retention_policies: dict[str, dict[str, Any]] = {}
self._models: list[dict[str, Any]] = []
self._datasets: list[dict[str, Any]] = []
self._tasks: list[dict[str, Any]] = []
self._compute_nodes: list[dict[str, Any]] = []
self._gpus: list[dict[str, Any]] = []
self._sessions: list[dict[str, Any]] = []
self._seq = 0
@contextmanager
def connect(self) -> Iterator[Any]:
class FakeConn:
def execute(self, *a, **kw):
return []
def commit(self):
pass
def rollback(self):
pass
def close(self):
pass
yield FakeConn()
# ---- helpers ----
def _next_id(self, prefix: str) -> str:
self._seq += 1
return f"{prefix}_{self._seq}"
# ==================== 第1周：登录 / 用户 / 权限码 / 日志 ====================
def login(self, username: str, password: str) -> dict[str, Any] | None:
for u in self._users:
if u["username"] == username and u["status"] == "active":
if password in ("admin123", "operator123", "test123"):
return dict(u)
return None
def users(self) -> list[dict[str, Any]]:
return [dict(u) for u in self._users]
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
u = {"id": self._next_id("u"), "protected": False, **payload}
self._users.append(u)
return u
def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]:
for u in self._users:
if u["id"] == user_id:
u.update(payload)
return u
raise KeyError(user_id)
def delete_user(self, user_id: str) -> None:
self._users = [u for u in self._users if u["id"] != user_id]
def roles(self) -> list[dict[str, Any]]:
return [
{"name": "admin", "display_name": "管理员"},
{"name": "operator", "display_name": "操作员"},
{"name": "viewer", "display_name": "访客"},
]
def log_files(self, date: str | None = None) -> list[dict[str, Any]]:
return [{"name": "backend-2026-08-01.log", "size": "1 KB", "date": "2026-08-01"}]
def log_content(self, file: str) -> dict[str, Any]:
return {"file": file, "content": "[INFO] test line", "size": "1 KB"}
def training_log_files(self) -> list[dict[str, Any]]:
return [{"task_id": "ft_001", "name": "ft_001.log", "size": "2 KB"}]
def training_log_content(self, file: str) -> dict[str, Any]:
return {"file": file, "content": "epoch 0 loss 1.0", "size": "2 KB"}
# ==================== 第2周：租户 / 项目 / 成员 / ACL ====================
def tenants(self) -> list[dict[str, Any]]:
return list(self._tenants.values())
def tenant(self, tenant_id: str) -> dict[str, Any]:
if tenant_id not in self._tenants:
raise KeyError(tenant_id)
return dict(self._tenants[tenant_id])
def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]:
tid = self._next_id("tnt")
t = {"id": tid, "status": "active", "quota": "{}", "retention_policy_id": None,
"create_time": "2026-08-01T00:00:00Z", **payload}
self._tenants[tid] = t
return dict(t)
def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]:
self._tenants[tenant_id].update(payload)
return dict(self._tenants[tenant_id])
def set_tenant_quota(self, tenant_id: str, quota: dict[str, Any]) -> dict[str, Any]:
self._tenants[tenant_id]["quota"] = json.dumps(quota)
return dict(self._tenants[tenant_id])
def set_tenant_retention(self, tenant_id: str, retention_policy_id: str | None) -> dict[str, Any]:
self._tenants[tenant_id]["retention_policy_id"] = retention_policy_id
return dict(self._tenants[tenant_id])
def projects(self, *, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]:
result = []
for p in self._projects.values():
if p.get("tenant_id") != tenant_id:
continue
if status and p.get("status") != status:
continue
if keyword and keyword.lower() not in p.get("name", "").lower():
continue
result.append(dict(p))
return result
def project(self, project_id: str) -> dict[str, Any]:
if project_id not in self._projects:
raise KeyError(project_id)
return dict(self._projects[project_id])
def create_project(self, payload: dict[str, Any]) -> dict[str, Any]:
pid = self._next_id("prj")
p = {"id": pid, "status": "active", "quota": "{}", "create_time": "2026-08-01T00:00:00Z", **payload}
self._projects[pid] = p
self._members[pid] = []
return dict(p)
def update_project(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
self._projects[project_id].update(payload)
return dict(self._projects[project_id])
def archive_project(self, project_id: str) -> dict[str, Any]:
self._projects[project_id]["status"] = "archived"
return dict(self._projects[project_id])
def delete_project(self, project_id: str) -> None:
self._projects.pop(project_id, None)
self._members.pop(project_id, None)
def project_members(self, project_id: str) -> list[dict[str, Any]]:
return [dict(m) for m in self._members.get(project_id, [])]
def add_project_member(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
m = {"joined_at": "2026-08-01T00:00:00Z", **payload}
self._members.setdefault(project_id, []).append(m)
return m
def update_project_member_role(self, project_id: str, user_id: str, role: str) -> dict[str, Any]:
for m in self._members.get(project_id, []):
if m["user_id"] == user_id:
m["role"] = role
return m
raise KeyError(user_id)
def remove_project_member(self, project_id: str, user_id: str) -> None:
self._members[project_id] = [m for m in self._members.get(project_id, []) if m["user_id"] != user_id]
# ---- ACL ----
def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
key = f"{resource_type}:{resource_id}"
return [dict(a) for a in self._acl.get(key, [])]
def set_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
key = f"{resource_type}:{resource_id}"
self._acl[key] = [dict(e) for e in entries]
return self.get_acl(resource_type, resource_id)
def resource_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
rows = self.get_acl(resource_type, resource_id)
grouped: dict[str, dict[str, Any]] = {}
for r in rows:
k = f"{r.get('principal_type')}:{r.get('principal_id')}"
bucket = grouped.setdefault(k, {
"subject_type": r.get("principal_type"),
"subject_id": r.get("principal_id"),
"permissions": [],
})
perm = r.get("permission")
if perm and perm not in bucket["permissions"]:
bucket["permissions"].append(perm)
return list(grouped.values())
def set_resource_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
flat: list[dict[str, Any]] = []
for e in entries:
for perm in e.get("permissions") or []:
flat.append({
"principal_type": e.get("subject_type"),
"principal_id": e.get("subject_id"),
"permission": perm,
})
self.set_acl(resource_type, resource_id, flat)
return self.resource_acl(resource_type, resource_id)
# ==================== 第3周：审批 / 审计 / 留存 ====================
def approval_templates(self) -> list[dict[str, Any]]:
return list(self._approval_templates.values())
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
tid = payload.get("id") or self._next_id("tpl")
t = {"id": tid, "steps": [], "create_time": "2026-08-01T00:00:00Z", **payload}
self._approval_templates[tid] = t
return dict(t)
def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]:
result = []
for i in self._approval_instances.values():
if status and i.get("status") != status:
continue
result.append(dict(i))
return result
def approval_instance(self, instance_id: str) -> dict[str, Any]:
if instance_id not in self._approval_instances:
raise KeyError(instance_id)
return dict(self._approval_instances[instance_id])
def create_approval_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
iid = self._next_id("appr")
inst = {
"id": iid,
"status": "pending",
"current_step": 0,
"steps": [],
"create_time": "2026-08-01T00:00:00Z",
**payload,
}
self._approval_instances[iid] = inst
return dict(inst)
def decide_approval_step(self, instance_id: str, step_index: int, *, approver_id: str, approved: bool, comment: str | None = None) -> dict[str, Any]:
inst = self._approval_instances[instance_id]
inst["status"] = "approved" if approved else "rejected"
inst["current_step"] = step_index + 1
return dict(inst)
def audit_logs(self, **kw) -> dict[str, Any]:
items = [dict(l) for l in self._audit_logs]
for filter_key in ("tenant_id", "project_id", "actor_id", "action", "target_type"):
val = kw.get(filter_key)
if val:
items = [l for l in items if l.get(filter_key) == val]
limit = kw.get("limit", 50)
offset = kw.get("offset", 0)
total = len(items)
items = items[offset:offset + limit]
return {"items": items, "total": total}
def record_audit(self, **kw) -> None:
log = {"id": self._next_id("log"), "time": "2026-08-01T12:00:00Z", **kw}
self._audit_logs.append(log)
# ---- 留存策略 ----
def retention_policies(self) -> list[dict[str, Any]]:
return list(self._retention_policies.values())
def retention_policy(self, policy_id: str) -> dict[str, Any]:
if policy_id not in self._retention_policies:
raise KeyError(policy_id)
return dict(self._retention_policies[policy_id])
def create_retention_policy(self, payload: dict[str, Any]) -> dict[str, Any]:
pid = payload.get("id") or self._next_id("rpol")
p = {"id": pid, "status": "active", "create_time": "2026-08-01T00:00:00Z", **payload}
self._retention_policies[pid] = p
return dict(p)
def update_retention_policy(self, policy_id: str, payload: dict[str, Any]) -> dict[str, Any]:
self._retention_policies[policy_id].update(payload)
return dict(self._retention_policies[policy_id])
def delete_retention_policy(self, policy_id: str) -> None:
self._retention_policies.pop(policy_id, None)
# ---- dashboard & other stubs ----
def login_duration_rank(self, limit: int = 8, days: int = 30) -> list[dict[str, Any]]:
return [{"user": "admin", "role": "admin", "duration": 10.0}]
def models(self) -> list[dict[str, Any]]:
return self._models
def datasets(self) -> list[dict[str, Any]]:
return self._datasets
def tasks(self) -> list[dict[str, Any]]:
return self._tasks
def compute_nodes(self) -> list[dict[str, Any]]:
return self._compute_nodes
def gpus(self) -> list[dict[str, Any]]:
return self._gpus
def system_info(self) -> dict[str, Any]:
return {"cpu": {}, "memory": {}}
# ============================================================
# 测试 fixtures
# ============================================================
@pytest.fixture(scope="module")
def fake_store() -> FakePlatformStore:
return FakePlatformStore()
def _build_client(store: FakePlatformStore) -> TestClient:
"""构建 TestClient，patch 所有治理模块的 get_platform_store。"""
app = FastAPI()
app.include_router(platform_router, prefix=PREFIX)
app.include_router(system_router, prefix=PREFIX)
app.include_router(tenant_router, prefix=PREFIX)
app.include_router(project_router, prefix=PREFIX)
app.include_router(approval_router, prefix=PREFIX)
app.include_router(retention_router, prefix=PREFIX)
app.include_router(resource_router, prefix=PREFIX)
patches = [
patch("app.db.platform_store.get_platform_store", return_value=store),
patch("app.core.auth.get_platform_store", return_value=store),
patch("app.api.v1.endpoints.platform.get_platform_store", return_value=store),
patch("app.modules.system.router.get_platform_store", return_value=store),
patch("app.modules.tenant.router.get_platform_store", return_value=store),
patch("app.modules.project.router.get_platform_store", return_value=store),
patch("app.modules.approval.router.get_platform_store", return_value=store),
patch("app.modules.retention.router.get_platform_store", return_value=store),
patch("app.modules.resource.router.get_platform_store", return_value=store),
]
for p in patches:
p.start()
client = TestClient(app, raise_server_exceptions=False)
client._fake_store = store # type: ignore[attr-defined]
return client
@pytest.fixture(scope="module")
def client(fake_store: FakePlatformStore) -> TestClient:
c = _build_client(fake_store)
yield c
def _admin_headers() -> dict[str, str]:
return {"Authorization": f"Bearer {ADMIN_TOKEN}"}
def _op_headers() -> dict[str, str]:
return {"Authorization": f"Bearer {OP_TOKEN}"}
# ============================================================
# 第 1 周测试：登录、当前用户、用户列表、权限码、日志查询
# ============================================================
class TestWeek1AuthUserPermissionsLogs:
"""第 1 周：登录、当前用户、用户列表、权限码、日志查询接口。"""
def test_login_success(self, client: TestClient):
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "admin123"})
assert resp.status_code == 200
data = resp.json()["data"]
assert data["token"] == ADMIN_TOKEN
assert data["user"]["username"] == "admin"
def test_login_invalid(self, client: TestClient):
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "wrong"})
assert resp.status_code == 401
def test_me_with_valid_token(self, client: TestClient):
resp = client.get(f"{PREFIX}/me", headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["username"] == "admin"
def test_me_without_token(self, client: TestClient):
resp = client.get(f"{PREFIX}/me")
assert resp.status_code == 401
def test_users_list(self, client: TestClient):
resp = client.get(f"{PREFIX}/users", headers=_admin_headers())
assert resp.status_code == 200
users = resp.json()["data"]
assert len(users) >= 2
assert any(u["username"] == "admin" for u in users)
def test_create_user(self, client: TestClient):
resp = client.post(
f"{PREFIX}/users",
json={"username": "tester", "display_name": "Tester", "role": "viewer", "password": "test123"},
headers=_admin_headers(),
)
assert resp.status_code == 200
assert resp.json()["data"]["username"] == "tester"
def test_permission_codes(self, client: TestClient):
resp = client.get(f"{PREFIX}/system/permissions/codes")
assert resp.status_code == 200
codes = resp.json()["data"]["codes"]
assert "dashboard" in codes
assert "user-settings" in codes
def test_permissions_overview(self, client: TestClient):
resp = client.get(f"{PREFIX}/system/permissions")
assert resp.status_code == 200
data = resp.json()["data"]
assert "codes" in data
assert "roles" in data
def test_log_files(self, client: TestClient):
resp = client.get(f"{PREFIX}/log-files", headers=_admin_headers())
assert resp.status_code == 200
files = resp.json()["data"]
assert len(files) >= 1
def test_log_content(self, client: TestClient):
resp = client.get(f"{PREFIX}/log-content", params={"file": "backend.log"}, headers=_admin_headers())
assert resp.status_code == 200
assert "content" in resp.json()["data"]
def test_training_log_files(self, client: TestClient):
resp = client.get(f"{PREFIX}/training-log-files", headers=_admin_headers())
assert resp.status_code == 200
assert len(resp.json()["data"]) >= 1
def test_training_log_content(self, client: TestClient):
resp = client.get(f"{PREFIX}/training-log-content", params={"file": "ft_001.log"}, headers=_admin_headers())
assert resp.status_code == 200
assert "content" in resp.json()["data"]
# ============================================================
# 第 2 周测试：租户、项目、项目成员、资源 ACL
# ============================================================
class TestWeek2TenantProjectACL:
"""第 2 周：租户、项目、项目成员、资源 ACL。"""
def test_tenant_crud(self, client: TestClient):
# 创建
resp = client.post(f"{PREFIX}/tenants", json={"name": "Tenant-A", "code": "ta"}, headers=_admin_headers())
assert resp.status_code == 200
tid = resp.json()["data"]["id"]
# 查列表
resp = client.get(f"{PREFIX}/tenants", headers=_admin_headers())
assert resp.status_code == 200
assert any(t["id"] == tid for t in resp.json()["data"])
# 查详情
resp = client.get(f"{PREFIX}/tenants/{tid}", headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["name"] == "Tenant-A"
# 更新
resp = client.put(f"{PREFIX}/tenants/{tid}", json={"name": "Tenant-A2"}, headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["name"] == "Tenant-A2"
def test_tenant_quota(self, client: TestClient):
resp = client.post(f"{PREFIX}/tenants", json={"name": "Q-Tenant", "code": "qt"}, headers=_admin_headers())
tid = resp.json()["data"]["id"]
resp = client.put(f"{PREFIX}/tenants/{tid}/quota", json={"quota": {"gpu": 4}}, headers=_admin_headers())
assert resp.status_code == 200
def test_tenant_retention(self, client: TestClient):
resp = client.post(f"{PREFIX}/tenants", json={"name": "R-Tenant", "code": "rt"}, headers=_admin_headers())
tid = resp.json()["data"]["id"]
resp = client.put(f"{PREFIX}/tenants/{tid}/retention-policy", json={"retention_policy_id": "rpol_1"}, headers=_admin_headers())
assert resp.status_code == 200
def test_project_crud(self, client: TestClient):
# 创建项目
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-1", "code": "p1", "tenant_id": "default"}, headers=_admin_headers())
assert resp.status_code == 200
pid = resp.json()["data"]["id"]
# 查列表
resp = client.get(f"{PREFIX}/projects", params={"tenant_id": "default"}, headers=_admin_headers())
assert resp.status_code == 200
assert any(p["id"] == pid for p in resp.json()["data"])
# 查详情
resp = client.get(f"{PREFIX}/projects/{pid}", headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["name"] == "Proj-1"
# 更新
resp = client.put(f"{PREFIX}/projects/{pid}", json={"description": "updated"}, headers=_admin_headers())
assert resp.status_code == 200
# 归档
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["status"] == "archived"
def test_project_members(self, client: TestClient):
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-M", "code": "pm", "tenant_id": "default"}, headers=_admin_headers())
pid = resp.json()["data"]["id"]
# 加成员
resp = client.post(f"{PREFIX}/projects/{pid}/members", json={"user_id": "u_op", "role": "developer"}, headers=_admin_headers())
assert resp.status_code == 200
# 列成员
resp = client.get(f"{PREFIX}/projects/{pid}/members", headers=_admin_headers())
assert resp.status_code == 200
assert len(resp.json()["data"]) >= 1
# 改角色
resp = client.put(f"{PREFIX}/projects/{pid}/members/u_op", json={"role": "maintainer"}, headers=_admin_headers())
assert resp.status_code == 200
# 删成员
resp = client.delete(f"{PREFIX}/projects/{pid}/members/u_op", headers=_admin_headers())
assert resp.status_code == 200
def test_resource_acl(self, client: TestClient):
# 设置 ACL
resp = client.put(
f"{PREFIX}/resources/model/m001/acl",
json={"entries": [{"subject_type": "user", "subject_id": "u_op", "permissions": ["read", "write"]}]},
headers=_admin_headers(),
)
assert resp.status_code == 200
result = resp.json()["data"]
assert len(result) == 1
assert set(result[0]["permissions"]) == {"read", "write"}
# 查询 ACL
resp = client.get(f"{PREFIX}/resources/model/m001/acl", headers=_admin_headers())
assert resp.status_code == 200
assert len(resp.json()["data"]) == 1
# ============================================================
# 第 3 周测试：审批实例、审批模板、审计日志查询和导出
# ============================================================
class TestWeek3ApprovalAudit:
"""第 3 周：审批实例、审批模板、审计日志查询和导出。"""
def test_approval_template_crud(self, client: TestClient):
# 创建模板
resp = client.post(f"{PREFIX}/approvals/templates", json={"name": "delete-approval", "steps": [{"approver_id": "u_admin", "status": "pending"}]}, headers=_admin_headers())
assert resp.status_code == 200
tpl_id = resp.json()["data"]["id"]
# 查列表
resp = client.get(f"{PREFIX}/approvals/templates", headers=_admin_headers())
assert resp.status_code == 200
assert any(t["id"] == tpl_id for t in resp.json()["data"])
def test_approval_instance_flow(self, client: TestClient):
# 创建审批实例
resp = client.post(f"{PREFIX}/approvals", json={
"resource_type": "dataset", "resource_id": "ds_001",
"applicant_id": "u_op",
}, headers=_admin_headers())
assert resp.status_code == 200
iid = resp.json()["data"]["id"]
# 查详情
resp = client.get(f"{PREFIX}/approvals/{iid}", headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["status"] == "pending"
# 审批决策
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
"approver_id": "u_admin", "approved": True, "comment": "ok",
}, headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["status"] == "approved"
def test_approval_instance_reject(self, client: TestClient):
resp = client.post(f"{PREFIX}/approvals", json={
"resource_type": "model", "resource_id": "m_002",
"applicant_id": "u_op",
}, headers=_admin_headers())
iid = resp.json()["data"]["id"]
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
"approver_id": "u_admin", "approved": False, "comment": "no",
}, headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["status"] == "rejected"
def test_approval_missing_field(self, client: TestClient):
resp = client.post(f"{PREFIX}/approvals", json={"resource_type": "dataset"}, headers=_admin_headers())
assert resp.status_code == 400
def test_audit_logs_query(self, client: TestClient):
# 通过 API 写操作触发审计
client.post(f"{PREFIX}/tenants", json={"name": "Audit-Tenant", "code": "at"}, headers=_admin_headers())
# 查询
resp = client.get(f"{PREFIX}/system/audit-logs", params={"limit": 50}, headers=_admin_headers())
assert resp.status_code == 200
data = resp.json()["data"]
assert "items" in data
assert "total" in data
assert data["total"] >= 1
def test_audit_logs_filter_by_action(self, client: TestClient):
resp = client.get(f"{PREFIX}/system/audit-logs", params={"action": "tenant.create"}, headers=_admin_headers())
assert resp.status_code == 200
items = resp.json()["data"]["items"]
assert all(i.get("action") == "tenant.create" for i in items)
def test_audit_logs_export_csv(self, client: TestClient):
resp = client.get(f"{PREFIX}/system/audit-logs/export", headers=_admin_headers())
assert resp.status_code == 200
assert "text/csv" in resp.headers.get("content-type", "")
# CSV 首行是表头
lines = resp.text.strip().split("\n")
assert "time" in lines[0]
# ============================================================
# 第 4 周测试：写操作审计、审批拦截、权限校验
# ============================================================
class TestWeek4AuditInterceptPermission:
"""第 4 周：写操作审计、审批拦截、权限校验。"""
def test_write_operation_produces_audit(self, client: TestClient, fake_store: FakePlatformStore):
# 清空审计日志便于断言
fake_store._audit_logs.clear()
# 创建租户 → 应产生 tenant.create 审计
client.post(f"{PREFIX}/tenants", json={"name": "W-Tenant", "code": "wt"}, headers=_admin_headers())
assert any(l["action"] == "tenant.create" for l in fake_store._audit_logs)
# 创建项目 → 应产生 project.create 审计
client.post(f"{PREFIX}/projects", json={"name": "W-Proj", "code": "wp", "tenant_id": "default"}, headers=_admin_headers())
assert any(l["action"] == "project.create" for l in fake_store._audit_logs)
# 设置 ACL → 应产生 resource.acl.set 审计
client.put(f"{PREFIX}/resources/model/w001/acl", json={"entries": []}, headers=_admin_headers())
assert any(l["action"] == "resource.acl.set" for l in fake_store._audit_logs)
def test_approval_intercept_on_project_archive(self, client: TestClient, fake_store: FakePlatformStore):
# 创建项目
resp = client.post(f"{PREFIX}/projects", json={"name": "I-Proj", "code": "ip", "tenant_id": "default"}, headers=_admin_headers())
pid = resp.json()["data"]["id"]
# 无待审批 → 可归档
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
assert resp.status_code == 200
def test_approval_intercept_blocks_when_pending(self, client: TestClient, fake_store: FakePlatformStore):
# 创建项目
resp = client.post(f"{PREFIX}/projects", json={"name": "B-Proj", "code": "bp", "tenant_id": "default"}, headers=_admin_headers())
pid = resp.json()["data"]["id"]
# 注入一条待审批实例
fake_store.create_approval_instance({
"resource_type": "project",
"resource_id": pid,
"applicant_id": "u_op",
})
# 有待审批 → 归档应被拒绝
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
assert resp.status_code == 409
def test_retention_policy_crud_with_audit(self, client: TestClient, fake_store: FakePlatformStore):
fake_store._audit_logs.clear()
# 创建
resp = client.post(f"{PREFIX}/retention-policies", json={"name": "30d-keep", "scope": "tenant"}, headers=_admin_headers())
assert resp.status_code == 200
rpid = resp.json()["data"]["id"]
assert any(l["action"] == "retention.create" for l in fake_store._audit_logs)
# 查列表
resp = client.get(f"{PREFIX}/retention-policies", headers=_admin_headers())
assert resp.status_code == 200
assert any(p["id"] == rpid for p in resp.json()["data"])
# 更新
resp = client.put(f"{PREFIX}/retention-policies/{rpid}", json={"status": "inactive"}, headers=_admin_headers())
assert resp.status_code == 200
assert resp.json()["data"]["status"] == "inactive"
# 删除
resp = client.delete(f"{PREFIX}/retention-policies/{rpid}", headers=_admin_headers())
assert resp.status_code == 200
def test_login_duration_rank_in_dashboard(self, client: TestClient):
resp = client.get(f"{PREFIX}/dashboard/stats", headers=_admin_headers())
assert resp.status_code == 200
data = resp.json()["data"]
assert "login_duration_rank" in data
assert "recent_login_users" in data
assert "service_status" in data
assert "training_7d" in data