Compare commits
6 Commits
8455d99d49
...
ft_wyt
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0b59c1ebee | ||
|
|
0233755859 | ||
|
|
03254f8196 | ||
|
|
3b9361c237 | ||
|
|
6f0e82f351 | ||
|
|
29d8a506cd |
0
.cache/.gitkeep
Normal file
0
.cache/.gitkeep
Normal file
0
.cache/huggingface/.gitkeep
Normal file
0
.cache/huggingface/.gitkeep
Normal file
0
.cache/tiktoken/.gitkeep
Normal file
0
.cache/tiktoken/.gitkeep
Normal file
35
.github/workflows/ci.yml
vendored
Normal file
35
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
name: yg-ft-ci
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, master, develop]
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
frontend:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: frontend
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: frontend/package-lock.json
|
||||||
|
- run: npm ci
|
||||||
|
- run: npm run build
|
||||||
|
|
||||||
|
backend:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
- run: python -m pip install -r backend/requirements.txt
|
||||||
|
- run: python -m compileall -q backend/app compute
|
||||||
|
- run: python -m pytest -q backend/tests/test_storage_security.py
|
||||||
|
env:
|
||||||
|
PYTHONPATH: backend
|
||||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -55,7 +55,6 @@ htmlcov/
|
|||||||
.nox/
|
.nox/
|
||||||
.coverage
|
.coverage
|
||||||
.coverage.*
|
.coverage.*
|
||||||
.cache
|
|
||||||
nosetests.xml
|
nosetests.xml
|
||||||
coverage.xml
|
coverage.xml
|
||||||
*.cover
|
*.cover
|
||||||
@@ -203,6 +202,7 @@ docker/llamafactory-latest.tar.gz
|
|||||||
docker/compute/data/yg-ft/datasets/*
|
docker/compute/data/yg-ft/datasets/*
|
||||||
docker/compute/data/yg-ft/models/*
|
docker/compute/data/yg-ft/models/*
|
||||||
docker/compute/data/yg-ft/outputs/*
|
docker/compute/data/yg-ft/outputs/*
|
||||||
|
docker/compute/data/yg-ft/trained_models/*
|
||||||
docker/compute/data/yg-ft/logs/**
|
docker/compute/data/yg-ft/logs/**
|
||||||
!docker/compute/data/yg-ft/logs/compute/
|
!docker/compute/data/yg-ft/logs/compute/
|
||||||
!docker/compute/data/yg-ft/logs/training/
|
!docker/compute/data/yg-ft/logs/training/
|
||||||
@@ -215,3 +215,14 @@ docker/offline/
|
|||||||
# MinIO object storage data - 对象存储运行时数据,勿提交,保留目录结构
|
# MinIO object storage data - 对象存储运行时数据,勿提交,保留目录结构
|
||||||
docker/minio/data/*
|
docker/minio/data/*
|
||||||
!docker/minio/data/.gitkeep
|
!docker/minio/data/.gitkeep
|
||||||
|
|
||||||
|
# nlp-eval-demo - 独立演示项目,不进版本库
|
||||||
|
nlp-eval-demo/
|
||||||
|
nlp-eval-demo.zip
|
||||||
|
|
||||||
|
# 项目本地缓存(HuggingFace / tiktoken 等大文件),保留目录结构与占位文件
|
||||||
|
.cache/*
|
||||||
|
!.cache/.gitkeep
|
||||||
|
!.cache/tiktoken/
|
||||||
|
!.cache/huggingface/
|
||||||
|
!.cache/**/.gitkeep
|
||||||
|
|||||||
@@ -29,12 +29,18 @@ from fastapi import (
|
|||||||
Header,
|
Header,
|
||||||
HTTPException,
|
HTTPException,
|
||||||
Query,
|
Query,
|
||||||
|
Request,
|
||||||
UploadFile,
|
UploadFile,
|
||||||
)
|
)
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from psycopg.rows import dict_row
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
from app.core.auth import (
|
||||||
|
filter_accessible_resource_ids,
|
||||||
|
get_current_user,
|
||||||
|
has_resource_access,
|
||||||
|
is_admin,
|
||||||
|
)
|
||||||
|
|
||||||
from app.core.logging import get_structured_logger
|
from app.core.logging import get_structured_logger
|
||||||
|
|
||||||
@@ -114,7 +120,6 @@ from app.schemas.data_process import (
|
|||||||
ResultUpdate,
|
ResultUpdate,
|
||||||
)
|
)
|
||||||
|
|
||||||
router = APIRouter(prefix="/data-process")
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
biz_logger = get_structured_logger("app.biz.data_process")
|
biz_logger = get_structured_logger("app.biz.data_process")
|
||||||
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||||
@@ -168,6 +173,27 @@ def fail(status_code: int, message: str) -> HTTPException:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _authorize_data_process_request(
|
||||||
|
request: Request,
|
||||||
|
task_id: str | None = None,
|
||||||
|
current_user: dict[str, Any] = Depends(get_current_user),
|
||||||
|
) -> None:
|
||||||
|
"""Apply resource ACL and tenant checks to every task-scoped endpoint."""
|
||||||
|
if not task_id or is_admin(current_user):
|
||||||
|
return
|
||||||
|
permission = "read" if request.method in {"GET", "HEAD"} else "write"
|
||||||
|
if any(marker in request.url.path for marker in ("/start", "/generate", "/regenerate", "/repeat", "/publish", "/external/")):
|
||||||
|
permission = "execute"
|
||||||
|
if not has_resource_access("data_process", task_id, current_user, permission):
|
||||||
|
raise fail(403, "no permission to access this data process task")
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(
|
||||||
|
prefix="/data-process",
|
||||||
|
dependencies=[Depends(_authorize_data_process_request)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def api_errors() -> Iterator[None]:
|
def api_errors() -> Iterator[None]:
|
||||||
try:
|
try:
|
||||||
@@ -271,7 +297,7 @@ def _commit_source_batch(
|
|||||||
except Exception:
|
except Exception:
|
||||||
# 文件系统回滚失败不能覆盖数据库抛出的根因,并继续清理其余对象。
|
# 文件系统回滚失败不能覆盖数据库抛出的根因,并继续清理其余对象。
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"failed to roll back data process source object task_id=%s",
|
"数据处理源对象回滚失败 task_id=%s",
|
||||||
task_id,
|
task_id,
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
@@ -641,7 +667,7 @@ def _run_generation(
|
|||||||
) -> None:
|
) -> None:
|
||||||
started_at = time.perf_counter()
|
started_at = time.perf_counter()
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation worker started task_id=%s generation_run_id=%s",
|
"数据处理生成任务开始 task_id=%s generation_run_id=%s",
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
)
|
)
|
||||||
@@ -649,8 +675,7 @@ def _run_generation(
|
|||||||
task = store.get_task(task_id)
|
task = store.get_task(task_id)
|
||||||
if not store.generation_is_running(task_id, generation_run_id):
|
if not store.generation_is_running(task_id, generation_run_id):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation worker skipped inactive run task_id=%s "
|
"数据处理生成任务跳过(非活跃运行) task_id=%s generation_run_id=%s",
|
||||||
"generation_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
)
|
)
|
||||||
@@ -740,8 +765,7 @@ def _run_generation(
|
|||||||
len(preview_items),
|
len(preview_items),
|
||||||
):
|
):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation stopped before completion task_id=%s "
|
"数据处理生成任务被中止 task_id=%s generation_run_id=%s",
|
||||||
"generation_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
)
|
)
|
||||||
@@ -832,9 +856,7 @@ def _run_generation(
|
|||||||
"created_by": (store.get_task(task_id) or {}).get("created_by"),
|
"created_by": (store.get_task(task_id) or {}).get("created_by"),
|
||||||
})
|
})
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation completed task_id=%s generation_run_id=%s "
|
"数据处理生成完成 task_id=%s generation_run_id=%s output_count=%s filtered_count=%s duplicate_count=%s error_count=%s duration_ms=%.2f",
|
||||||
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
|
|
||||||
"duration_ms=%.2f",
|
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
completed.get("output_count", len(accepted)),
|
completed.get("output_count", len(accepted)),
|
||||||
@@ -845,14 +867,13 @@ def _run_generation(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation stopped before result persistence task_id=%s "
|
"数据处理生成任务在持久化前被停止 task_id=%s generation_run_id=%s",
|
||||||
"generation_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
|
"数据处理生成失败 task_id=%s generation_run_id=%s duration_ms=%.2f",
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
(time.perf_counter() - started_at) * 1000,
|
(time.perf_counter() - started_at) * 1000,
|
||||||
@@ -866,8 +887,7 @@ def _run_generation(
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"failed to persist data process generation failure task_id=%s "
|
"数据处理生成失败持久化异常 task_id=%s generation_run_id=%s",
|
||||||
"generation_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
generation_run_id,
|
generation_run_id,
|
||||||
)
|
)
|
||||||
@@ -892,7 +912,7 @@ def list_tasks(
|
|||||||
keyword=keyword,
|
keyword=keyword,
|
||||||
status=status,
|
status=status,
|
||||||
process_type=process_type,
|
process_type=process_type,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id if is_admin(current_user) else (current_user.get("tenant_id") or "default"),
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
)
|
)
|
||||||
# #4 资源 ACL 过滤:admin 放行,普通用户只看到自己被授权的数据处理任务
|
# #4 资源 ACL 过滤:admin 放行,普通用户只看到自己被授权的数据处理任务
|
||||||
@@ -913,10 +933,16 @@ def list_tasks(
|
|||||||
def create_task(
|
def create_task(
|
||||||
payload: DataProcessTaskCreate,
|
payload: DataProcessTaskCreate,
|
||||||
store: DataProcessStore = Depends(get_data_process_store),
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
task = store.create_task(payload.model_dump(mode="json"))
|
values = payload.model_dump(mode="json")
|
||||||
biz_logger.info("data-process:create success", taskId=task["id"], processType=task.get("process_type", ""))
|
values["created_by"] = current_user.get("id")
|
||||||
|
values["owner_id"] = current_user.get("id")
|
||||||
|
values["tenant_id"] = current_user.get("tenant_id") or "default"
|
||||||
|
get_platform_store().assert_active_tenant(values["tenant_id"])
|
||||||
|
task = store.create_task(values)
|
||||||
|
biz_logger.info("用户创建数据处理任务成功", taskId=task["id"], processType=task.get("process_type", ""))
|
||||||
return ok(task, "data process task created")
|
return ok(task, "data process task created")
|
||||||
|
|
||||||
|
|
||||||
@@ -943,7 +969,7 @@ def update_task(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
|
result = store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json"))
|
||||||
biz_logger.info("data-process:update success", taskId=task_id)
|
biz_logger.info("用户更新数据处理任务成功", taskId=task_id)
|
||||||
return ok(result, "data process task updated")
|
return ok(result, "data process task updated")
|
||||||
|
|
||||||
|
|
||||||
@@ -1033,7 +1059,7 @@ def _remove_repeated_storage_objects(
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"failed to roll back repeated data process source object task_id=%s",
|
"数据处理源对象重复回滚失败 task_id=%s",
|
||||||
task_id,
|
task_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1109,7 +1135,7 @@ def delete_task(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
store.delete_task(task_id)
|
store.delete_task(task_id)
|
||||||
biz_logger.info("data-process:delete success", taskId=task_id)
|
biz_logger.info("用户删除数据处理任务成功", taskId=task_id)
|
||||||
return ok({"deleted": task_id}, "data process task deleted")
|
return ok({"deleted": task_id}, "data process task deleted")
|
||||||
|
|
||||||
|
|
||||||
@@ -1424,10 +1450,10 @@ def delete_source_file(
|
|||||||
except Exception:
|
except Exception:
|
||||||
cleanup_pending = True
|
cleanup_pending = True
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"failed to remove data process source object after soft deletion",
|
"数据处理源对象软删除后存储清理失败",
|
||||||
extra={"task_id": task_id, "source_file_id": file_id},
|
extra={"task_id": task_id, "source_file_id": file_id},
|
||||||
)
|
)
|
||||||
biz_logger.info("data-process:delete-source-file success", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
|
biz_logger.info("用户删除数据处理源文件成功", taskId=task_id, fileId=file_id, storageCleanupPending=cleanup_pending)
|
||||||
return ok(
|
return ok(
|
||||||
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
|
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
|
||||||
"source file removed",
|
"source file removed",
|
||||||
@@ -1660,7 +1686,6 @@ def _prepare_preview_items(
|
|||||||
)
|
)
|
||||||
needs_pdf_noise = (
|
needs_pdf_noise = (
|
||||||
is_unstructured
|
is_unstructured
|
||||||
and not needs_layout_raw
|
|
||||||
and source_format == "pdf"
|
and source_format == "pdf"
|
||||||
and bool(
|
and bool(
|
||||||
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||||
@@ -1694,16 +1719,17 @@ def _prepare_preview_items(
|
|||||||
enriched = dict(source)
|
enriched = dict(source)
|
||||||
if needs_structured_xlsx or needs_layout_raw:
|
if needs_structured_xlsx or needs_layout_raw:
|
||||||
enriched["raw_content"] = raw
|
enriched["raw_content"] = raw
|
||||||
sources[index] = enriched
|
if needs_pdf_noise:
|
||||||
continue
|
# layout_hybrid 路径下也跑 PDF 文本规则噪声检测,
|
||||||
|
# 弥补 docling layout 模型对中文 PDF 页眉/页脚识别率低的问题。
|
||||||
pages = extract_pdf_page_texts(raw)
|
pages = extract_pdf_page_texts(raw)
|
||||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||||
if extracted_text != str(source.get("content") or ""):
|
if extracted_text != str(source.get("content") or ""):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"skip PDF document noise detection because stored offsets differ for %s",
|
"跳过PDF文档噪声检测(存储偏移量不一致) source_id=%s",
|
||||||
source["id"],
|
source["id"],
|
||||||
)
|
)
|
||||||
continue
|
else:
|
||||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||||
sources[index] = enriched
|
sources[index] = enriched
|
||||||
items = _build_preview_items(task, sources)
|
items = _build_preview_items(task, sources)
|
||||||
@@ -1723,7 +1749,7 @@ def _run_preview(
|
|||||||
|
|
||||||
started_at = time.perf_counter()
|
started_at = time.perf_counter()
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
|
"数据处理预览开始 task_id=%s preview_run_id=%s total_files=%s",
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
len(source_file_ids),
|
len(source_file_ids),
|
||||||
@@ -1732,7 +1758,7 @@ def _run_preview(
|
|||||||
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
|
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
|
||||||
if not store.mark_preview_running(task_id, preview_run_id):
|
if not store.mark_preview_running(task_id, preview_run_id):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
|
"数据处理预览跳过(非活跃运行) task_id=%s preview_run_id=%s",
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
)
|
)
|
||||||
@@ -1742,8 +1768,7 @@ def _run_preview(
|
|||||||
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
|
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
|
||||||
if not store.preview_is_running(task_id, preview_run_id):
|
if not store.preview_is_running(task_id, preview_run_id):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview cancelled task_id=%s preview_run_id=%s "
|
"数据处理预览被取消 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
|
||||||
"completed_files=%s total_files=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
completed_files - 1,
|
completed_files - 1,
|
||||||
@@ -1774,8 +1799,7 @@ def _run_preview(
|
|||||||
total_files,
|
total_files,
|
||||||
):
|
):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview stopped before progress update task_id=%s "
|
"数据处理预览在进度更新前被停止 task_id=%s preview_run_id=%s completed_files=%s total_files=%s",
|
||||||
"preview_run_id=%s completed_files=%s total_files=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
completed_files,
|
completed_files,
|
||||||
@@ -1784,8 +1808,7 @@ def _run_preview(
|
|||||||
return
|
return
|
||||||
if store.complete_preview(task_id, preview_run_id):
|
if store.complete_preview(task_id, preview_run_id):
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview completed task_id=%s preview_run_id=%s "
|
"数据处理预览完成 task_id=%s preview_run_id=%s total_files=%s total_items=%s duration_ms=%.2f",
|
||||||
"total_files=%s total_items=%s duration_ms=%.2f",
|
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
total_files,
|
total_files,
|
||||||
@@ -1794,14 +1817,13 @@ def _run_preview(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process preview completion ignored for inactive run task_id=%s "
|
"数据处理预览完成但运行已失效 task_id=%s preview_run_id=%s",
|
||||||
"preview_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
|
"数据处理预览失败 task_id=%s preview_run_id=%s duration_ms=%.2f",
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
(time.perf_counter() - started_at) * 1000,
|
(time.perf_counter() - started_at) * 1000,
|
||||||
@@ -1815,8 +1837,7 @@ def _run_preview(
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"failed to persist data process preview failure task_id=%s "
|
"数据处理预览失败持久化异常 task_id=%s preview_run_id=%s",
|
||||||
"preview_run_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
preview_run_id,
|
preview_run_id,
|
||||||
)
|
)
|
||||||
@@ -2026,7 +2047,7 @@ def stop(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
store.stop_task(task_id)
|
store.stop_task(task_id)
|
||||||
biz_logger.info("data-process:stop success", taskId=task_id)
|
biz_logger.info("用户停止数据处理任务成功", taskId=task_id)
|
||||||
return ok(store.progress(task_id), "data process task stopped")
|
return ok(store.progress(task_id), "data process task stopped")
|
||||||
|
|
||||||
|
|
||||||
@@ -2069,7 +2090,7 @@ def confirm_results(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
result = store.confirm_results(task_id)
|
result = store.confirm_results(task_id)
|
||||||
biz_logger.info("data-process:confirm-results success", taskId=task_id)
|
biz_logger.info("用户确认数据处理结果成功", taskId=task_id)
|
||||||
return ok(result, "data process results confirmed")
|
return ok(result, "data process results confirmed")
|
||||||
|
|
||||||
|
|
||||||
@@ -2419,8 +2440,7 @@ def regenerate_results_batch(
|
|||||||
}))
|
}))
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch regeneration started batch_id=%s task_id=%s "
|
"数据处理结果批量重新生成开始 batch_id=%s task_id=%s requested=%s prepared=%s concurrency=%s",
|
||||||
"requested=%s prepared=%s concurrency=%s",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
len(payload.items),
|
len(payload.items),
|
||||||
@@ -2488,8 +2508,7 @@ def regenerate_results_batch(
|
|||||||
except Exception as exc: # pragma: no cover - defensive boundary
|
except Exception as exc: # pragma: no cover - defensive boundary
|
||||||
outcome = "internal_error"
|
outcome = "internal_error"
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"data process result batch regeneration crashed "
|
"数据处理结果批量重新生成崩溃 batch_id=%s task_id=%s result_id=%s",
|
||||||
"batch_id=%s task_id=%s result_id=%s",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -2500,8 +2519,7 @@ def regenerate_results_batch(
|
|||||||
"message": _safe_regeneration_error(exc),
|
"message": _safe_regeneration_error(exc),
|
||||||
}))
|
}))
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch item finished batch_id=%s task_id=%s "
|
"数据处理结果批量项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
|
||||||
"result_id=%s outcome=%s duration_ms=%.2f",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -2521,8 +2539,7 @@ def regenerate_results_batch(
|
|||||||
)
|
)
|
||||||
duration_ms = (time.perf_counter() - started_at) * 1000
|
duration_ms = (time.perf_counter() - started_at) * 1000
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch regeneration completed batch_id=%s task_id=%s "
|
"数据处理结果批量重新生成完成 batch_id=%s task_id=%s succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
|
||||||
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
len(success_items),
|
len(success_items),
|
||||||
@@ -2569,8 +2586,7 @@ def evaluate_results_batch(
|
|||||||
evaluation_model = store.get_generation_model(str(model_id))
|
evaluation_model = store.get_generation_model(str(model_id))
|
||||||
except NotFoundError:
|
except NotFoundError:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"data process evaluation model unavailable, judge layer "
|
"数据处理评测模型不可用,跳过评测层 task_id=%s model_id=%s",
|
||||||
"skipped task_id=%s model_id=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
model_id,
|
model_id,
|
||||||
)
|
)
|
||||||
@@ -2618,8 +2634,7 @@ def evaluate_results_batch(
|
|||||||
}))
|
}))
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch evaluation started batch_id=%s task_id=%s "
|
"数据处理结果批量评测开始 batch_id=%s task_id=%s requested=%s prepared=%s judge_enabled=%s",
|
||||||
"requested=%s prepared=%s judge_enabled=%s",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
len(payload.items),
|
len(payload.items),
|
||||||
@@ -2636,8 +2651,7 @@ def evaluate_results_batch(
|
|||||||
semantic_embedding_model()
|
semantic_embedding_model()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"data process semantic embedding unavailable, semantic layer "
|
"数据处理语义嵌入模型不可用,语义层将跳过 batch_id=%s",
|
||||||
"will be skipped batch_id=%s",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
)
|
)
|
||||||
request_timeout = _result_regeneration_timeout(config)
|
request_timeout = _result_regeneration_timeout(config)
|
||||||
@@ -2690,8 +2704,7 @@ def evaluate_results_batch(
|
|||||||
"message": _safe_regeneration_error(exc),
|
"message": _safe_regeneration_error(exc),
|
||||||
}))
|
}))
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch evaluation item finished "
|
"数据处理结果批量评测项完成 batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
|
||||||
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -2703,8 +2716,7 @@ def evaluate_results_batch(
|
|||||||
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
|
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
|
||||||
duration_ms = (time.perf_counter() - started_at) * 1000
|
duration_ms = (time.perf_counter() - started_at) * 1000
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result batch evaluation completed batch_id=%s task_id=%s "
|
"数据处理结果批量评测完成 batch_id=%s task_id=%s succeeded=%s failed=%s duration_ms=%.2f",
|
||||||
"succeeded=%s failed=%s duration_ms=%.2f",
|
|
||||||
batch_id,
|
batch_id,
|
||||||
task_id,
|
task_id,
|
||||||
len(success_items),
|
len(success_items),
|
||||||
@@ -2760,8 +2772,7 @@ def regenerate_result(
|
|||||||
)
|
)
|
||||||
except _ResultRegenerationFailed as exc:
|
except _ResultRegenerationFailed as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"data process result regeneration failed task_id=%s result_id=%s "
|
"数据处理结果重新生成失败 task_id=%s result_id=%s duration_ms=%.2f reason=%s",
|
||||||
"duration_ms=%.2f reason=%s",
|
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
(time.perf_counter() - started_at) * 1000,
|
(time.perf_counter() - started_at) * 1000,
|
||||||
@@ -2769,7 +2780,7 @@ def regenerate_result(
|
|||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
|
"数据处理结果重新生成完成 task_id=%s result_id=%s duration_ms=%.2f",
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
(time.perf_counter() - started_at) * 1000,
|
(time.perf_counter() - started_at) * 1000,
|
||||||
@@ -2785,6 +2796,6 @@ def publish(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
with api_errors():
|
with api_errors():
|
||||||
result = store.publish(task_id, payload.model_dump(mode="json"))
|
result = store.publish(task_id, payload.model_dump(mode="json"))
|
||||||
biz_logger.info("data-process:publish success", taskId=task_id, datasetId=result.get("dataset_id", ""))
|
biz_logger.info("用户发布数据处理任务成功", taskId=task_id, datasetId=result.get("dataset_id", ""))
|
||||||
message = "dataset published" if result["created"] else "dataset already published"
|
message = "dataset published" if result["created"] else "dataset already published"
|
||||||
return ok(result, message)
|
return ok(result, message)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ from typing import Any, Callable, Optional, TypeVar
|
|||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
|
|
||||||
from app.core.logging import get_logger, request_id_var
|
from app.core.logging import get_client_ip, get_logger, mask_sensitive_string, request_id_var
|
||||||
|
|
||||||
logger = get_logger("app.audit")
|
logger = get_logger("app.audit")
|
||||||
|
|
||||||
@@ -67,12 +67,25 @@ def audit_log(
|
|||||||
detail=detail,
|
detail=detail,
|
||||||
trace_id=trace_id,
|
trace_id=trace_id,
|
||||||
duration_ms=elapsed_ms,
|
duration_ms=elapsed_ms,
|
||||||
|
kwargs=kwargs,
|
||||||
|
args=args,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
logger.error(
|
_record_audit(
|
||||||
"审计日志记录失败 action=%s", action, exc_info=True
|
action=action,
|
||||||
|
actor_id=_extract_actor_id(kwargs),
|
||||||
|
target_type=target_type,
|
||||||
|
target_id=_extract_target_id(None, kwargs, extract_target_id),
|
||||||
|
detail=_build_detail(detail_template, kwargs),
|
||||||
|
trace_id=trace_id,
|
||||||
|
duration_ms=(time.perf_counter() - started_at) * 1000,
|
||||||
|
result="failure",
|
||||||
|
reason=_safe_exception_reason(exc),
|
||||||
|
kwargs=kwargs,
|
||||||
|
args=args,
|
||||||
)
|
)
|
||||||
|
logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
return async_wrapper # type: ignore
|
return async_wrapper # type: ignore
|
||||||
@@ -94,12 +107,25 @@ def audit_log(
|
|||||||
detail=detail,
|
detail=detail,
|
||||||
trace_id=trace_id,
|
trace_id=trace_id,
|
||||||
duration_ms=elapsed_ms,
|
duration_ms=elapsed_ms,
|
||||||
|
kwargs=kwargs,
|
||||||
|
args=args,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
logger.error(
|
_record_audit(
|
||||||
"审计日志记录失败 action=%s", action, exc_info=True
|
action=action,
|
||||||
|
actor_id=_extract_actor_id(kwargs),
|
||||||
|
target_type=target_type,
|
||||||
|
target_id=_extract_target_id(None, kwargs, extract_target_id),
|
||||||
|
detail=_build_detail(detail_template, kwargs),
|
||||||
|
trace_id=trace_id,
|
||||||
|
duration_ms=(time.perf_counter() - started_at) * 1000,
|
||||||
|
result="failure",
|
||||||
|
reason=_safe_exception_reason(exc),
|
||||||
|
kwargs=kwargs,
|
||||||
|
args=args,
|
||||||
)
|
)
|
||||||
|
logger.warning("业务操作失败 action=%s reason=%s", action, _safe_exception_reason(exc))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
return sync_wrapper # type: ignore
|
return sync_wrapper # type: ignore
|
||||||
@@ -144,18 +170,36 @@ def _record_audit(
|
|||||||
detail: str,
|
detail: str,
|
||||||
trace_id: str,
|
trace_id: str,
|
||||||
duration_ms: float,
|
duration_ms: float,
|
||||||
|
*,
|
||||||
|
kwargs: dict[str, Any] | None = None,
|
||||||
|
args: tuple[Any, ...] = (),
|
||||||
|
result: str = "success",
|
||||||
|
reason: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""通过已有的 record_audit 方法写入审计日志"""
|
"""通过已有的 record_audit 方法写入审计日志"""
|
||||||
try:
|
try:
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
kwargs = kwargs or {}
|
||||||
|
request = _extract_request(args, kwargs)
|
||||||
|
current_user = kwargs.get("current_user") or kwargs.get("user") or {}
|
||||||
|
request_id = request.headers.get("X-Request-ID") if request else None
|
||||||
|
request_id = request_id or trace_id
|
||||||
|
client_ip = get_client_ip(request) or None
|
||||||
|
detail_text = f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}"
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action=action,
|
action=action,
|
||||||
actor_id=actor_id,
|
actor_id=actor_id,
|
||||||
target_type=target_type or None,
|
target_type=target_type or None,
|
||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
|
tenant_id=str(current_user.get("tenant_id") or "") or None,
|
||||||
|
detail=mask_sensitive_string(detail_text),
|
||||||
|
result=result,
|
||||||
|
reason=mask_sensitive_string(reason or "") or None,
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=str(current_user.get("session_id") or "") or None,
|
||||||
|
ip=client_ip,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
|
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
|
||||||
@@ -170,6 +214,19 @@ def _extract_actor_id(kwargs: dict) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_request(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Request | None:
|
||||||
|
for value in tuple(kwargs.values()) + tuple(args):
|
||||||
|
if isinstance(value, Request):
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_exception_reason(exc: Exception) -> str:
|
||||||
|
"""Keep audit failures useful without recording credentials or tokens."""
|
||||||
|
value = getattr(exc, "detail", None) or str(exc) or exc.__class__.__name__
|
||||||
|
return mask_sensitive_string(str(value))[:500]
|
||||||
|
|
||||||
|
|
||||||
# ==================== 预定义的审计操作常量 ====================
|
# ==================== 预定义的审计操作常量 ====================
|
||||||
|
|
||||||
class AuditActions:
|
class AuditActions:
|
||||||
|
|||||||
@@ -2,20 +2,76 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
import json
|
||||||
|
|
||||||
from fastapi import Depends, HTTPException, Query, Request, status
|
from fastapi import Depends, HTTPException, Query, Request, status
|
||||||
|
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.core.logging import get_client_ip
|
||||||
|
|
||||||
# 无需鉴权的路径前缀(健康检查、登录等)
|
# 无需鉴权的路径前缀(健康检查、登录等)
|
||||||
PUBLIC_PATHS = ("/health", "/login", "/system-info")
|
PUBLIC_PATHS = ("/health", "/login", "/system-info")
|
||||||
OWNER_TABLES = {
|
OWNER_TABLES = {
|
||||||
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
|
"dataset": ("datasets", "created_by"), "model": ("models", "created_by"),
|
||||||
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
|
"trained_model": ("trained_models", "created_by"), "eval": ("eval_tasks", "created_by"),
|
||||||
"fine-tune": ("fine_tune_tasks", "payload"), "fine_tune_task": ("fine_tune_tasks", "payload"),
|
"fine-tune": ("fine_tune_tasks", "created_by"), "fine_tune_task": ("fine_tune_tasks", "created_by"),
|
||||||
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
|
"compare": ("compare_tasks", "payload"), "inference": ("compare_tasks", "payload"),
|
||||||
"project": ("projects", "created_by"), "data_process": ("data_process_tasks", "created_by"),
|
"project": ("projects", "create_by"), "data_process": ("data_process_tasks", "created_by"),
|
||||||
|
"data_convert": ("data_convert_tasks", "created_by"),
|
||||||
}
|
}
|
||||||
|
RESOURCE_TABLES = {
|
||||||
|
"dataset": "datasets",
|
||||||
|
"model": "models",
|
||||||
|
"trained_model": "trained_models",
|
||||||
|
"eval": "eval_tasks",
|
||||||
|
"fine-tune": "fine_tune_tasks",
|
||||||
|
"fine_tune_task": "fine_tune_tasks",
|
||||||
|
"compare": "compare_tasks",
|
||||||
|
"inference": "compare_tasks",
|
||||||
|
"project": "projects",
|
||||||
|
"data_process": "data_process_tasks",
|
||||||
|
"data_convert": "data_convert_tasks",
|
||||||
|
}
|
||||||
|
MODULE_PERMISSIONS = {
|
||||||
|
"dashboard": "dashboard",
|
||||||
|
"fine-tune": "fine-tune",
|
||||||
|
"model-eval": "model-eval",
|
||||||
|
"model-compare": "model-inference",
|
||||||
|
"model-inference": "model-inference",
|
||||||
|
"model-chat": "model-inference",
|
||||||
|
"model-manage": "model-manage",
|
||||||
|
"dataset-manage": "dataset",
|
||||||
|
"data-process": "data-process",
|
||||||
|
"data-convert": "data-convert",
|
||||||
|
"compute": "compute",
|
||||||
|
"hardware": "hardware",
|
||||||
|
"users": "user-settings",
|
||||||
|
}
|
||||||
|
|
||||||
|
# These endpoints are the user-facing compute view used by training,
|
||||||
|
# inference, and evaluation forms. They only return the current user's
|
||||||
|
# assigned nodes/GPUs in the endpoint implementation, so they must remain
|
||||||
|
# available after an approval without granting access to the admin compute
|
||||||
|
# management page.
|
||||||
|
SELF_SERVICE_COMPUTE_PATHS = {
|
||||||
|
"/compute/nodes",
|
||||||
|
"/compute/gpus",
|
||||||
|
"/compute/my-gpus",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Resource actions are deliberately kept separate from module permissions.
|
||||||
|
# A user may be allowed to open a module while still lacking the action on a
|
||||||
|
# specific resource (for example, download or delete).
|
||||||
|
RESOURCE_ACTIONS = frozenset({
|
||||||
|
"read",
|
||||||
|
"write",
|
||||||
|
"execute",
|
||||||
|
"download",
|
||||||
|
"export",
|
||||||
|
"delete",
|
||||||
|
"admin",
|
||||||
|
})
|
||||||
|
RESOURCE_ACTION_ALIASES = {"export": "download"}
|
||||||
|
|
||||||
|
|
||||||
def _extract_token(request: Request) -> str | None:
|
def _extract_token(request: Request) -> str | None:
|
||||||
@@ -30,6 +86,26 @@ def _session_token(user_id: str, session_id: str) -> str:
|
|||||||
return f"platform-token-{user_id}.{session_id}"
|
return f"platform-token-{user_id}.{session_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def _record_auth_event(actor_id: str | None, action: str, reason: str, request: Request) -> None:
|
||||||
|
"""Best-effort security audit for authentication and permission denials."""
|
||||||
|
try:
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action=action,
|
||||||
|
actor_id=actor_id,
|
||||||
|
target_type="auth",
|
||||||
|
target_id=request.url.path,
|
||||||
|
detail=reason,
|
||||||
|
result="denied",
|
||||||
|
reason=reason,
|
||||||
|
request_id=request.headers.get("X-Request-ID"),
|
||||||
|
ip=get_client_ip(request) or None,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# An audit failure must never turn an authentication decision into an
|
||||||
|
# accidental allow or an unrelated 500 response.
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def get_current_user(request: Request) -> dict[str, Any]:
|
def get_current_user(request: Request) -> dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
FastAPI 依赖:解析当前登录用户。
|
FastAPI 依赖:解析当前登录用户。
|
||||||
@@ -55,31 +131,219 @@ def get_current_user(request: Request) -> dict[str, Any]:
|
|||||||
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
|
"SELECT user_id, logout_at, expires_at FROM sessions WHERE id=?", (session_id,)
|
||||||
).fetchone()
|
).fetchone()
|
||||||
if not session or session["user_id"] != user_id or session["logout_at"]:
|
if not session or session["user_id"] != user_id or session["logout_at"]:
|
||||||
|
_record_auth_event(user_id, "auth.session.denied", "session expired or logged out", request)
|
||||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
||||||
if session["expires_at"]:
|
if session["expires_at"]:
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
try:
|
try:
|
||||||
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
if datetime.fromisoformat(str(session["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||||
|
_record_auth_event(user_id, "auth.session.denied", "session expired", request)
|
||||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="session expired")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
|
user_row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()
|
||||||
if user_row:
|
if user_row:
|
||||||
return store._user(user_row)
|
user = store._user(user_row)
|
||||||
|
if user.get("status") != "active":
|
||||||
|
_record_auth_event(user_id, "auth.user.disabled", "user is not active", request)
|
||||||
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user disabled")
|
||||||
|
requested_tenant = request.headers.get("X-Tenant-ID", "").strip()
|
||||||
|
if requested_tenant and (is_admin(user) or requested_tenant in user_tenant_ids(user)):
|
||||||
|
user["tenant_id"] = requested_tenant
|
||||||
|
# Keep the session identifier in request context so business audit
|
||||||
|
# records can be traced back to the exact login session.
|
||||||
|
if session_id:
|
||||||
|
user["session_id"] = session_id
|
||||||
|
path_parts = path.strip("/").split("/")
|
||||||
|
# The API may be mounted directly at /modelTF or behind /api/v1/modelTF.
|
||||||
|
# Locate the first known module segment instead of relying on a fixed index.
|
||||||
|
segment = next((part for part in path_parts if part in MODULE_PERMISSIONS), "")
|
||||||
|
required = MODULE_PERMISSIONS.get(segment)
|
||||||
|
relative_path = "/" + "/".join(path_parts[path_parts.index(segment):]) if segment else path
|
||||||
|
self_service_compute = relative_path.rstrip("/") in SELF_SERVICE_COMPUTE_PATHS
|
||||||
|
if (
|
||||||
|
required
|
||||||
|
and not is_admin(user)
|
||||||
|
and required not in (user.get("permissions") or [])
|
||||||
|
and not self_service_compute
|
||||||
|
):
|
||||||
|
_record_auth_event(user_id, "auth.permission.denied", f"missing permission: {required}", request)
|
||||||
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"missing permission: {required}")
|
||||||
|
return user
|
||||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
|
||||||
|
|
||||||
|
|
||||||
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
|
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
"""FastAPI 依赖:要求当前用户是管理员(role=admin 或 protected)。"""
|
"""FastAPI 依赖:要求当前用户是平台管理员。"""
|
||||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
if is_admin(current_user):
|
||||||
return current_user
|
return current_user
|
||||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
|
||||||
|
|
||||||
|
|
||||||
|
def require_tenant_admin(
|
||||||
|
tenant_id: str,
|
||||||
|
current_user: dict[str, Any] = Depends(get_current_user),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Allow platform admins and active owner/admin tenant members."""
|
||||||
|
if is_admin(current_user) or is_tenant_admin(current_user, tenant_id):
|
||||||
|
return current_user
|
||||||
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant admin permission required")
|
||||||
|
|
||||||
|
|
||||||
def is_admin(user: dict[str, Any]) -> bool:
|
def is_admin(user: dict[str, Any]) -> bool:
|
||||||
"""判断用户是否为管理员(admin 角色或 protected 标记)。"""
|
"""判断用户是否为平台管理员;兼容历史 role=admin/protected 数据。"""
|
||||||
return user.get("role") == "admin" or user.get("protected", False)
|
return (
|
||||||
|
user.get("platform_role") == "platform_admin"
|
||||||
|
or user.get("role") == "admin"
|
||||||
|
or user.get("protected", False)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def user_tenant_ids(user: dict[str, Any]) -> set[str]:
|
||||||
|
"""Return the tenant scope of a user.
|
||||||
|
|
||||||
|
Existing installations keep the primary tenant on ``users.tenant_id``.
|
||||||
|
``tenant_members`` is optional during the migration and adds memberships
|
||||||
|
when the permission v2 schema is available.
|
||||||
|
"""
|
||||||
|
if is_admin(user):
|
||||||
|
return {"*"}
|
||||||
|
primary_tenant = str(user.get("tenant_id") or "default")
|
||||||
|
result: set[str] = set()
|
||||||
|
user_id = user.get("id")
|
||||||
|
if not user_id:
|
||||||
|
return result
|
||||||
|
try:
|
||||||
|
store = get_platform_store()
|
||||||
|
with store.connect() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT tenant_id FROM tenant_members "
|
||||||
|
"WHERE user_id=? AND status='active' "
|
||||||
|
"AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)",
|
||||||
|
(user_id,),
|
||||||
|
).fetchall()
|
||||||
|
result.update(str(row["tenant_id"]) for row in rows if row.get("tenant_id"))
|
||||||
|
if not result:
|
||||||
|
active_tenant = conn.execute(
|
||||||
|
"SELECT id FROM tenants WHERE id=? "
|
||||||
|
"AND COALESCE(status, 'active')='active' "
|
||||||
|
"AND COALESCE(deleted_at, '')=''",
|
||||||
|
(primary_tenant,),
|
||||||
|
).fetchone()
|
||||||
|
if active_tenant:
|
||||||
|
result.add(primary_tenant)
|
||||||
|
except Exception:
|
||||||
|
# Older databases are upgraded lazily. The primary users.tenant_id
|
||||||
|
# remains a valid fallback until the additive table is available.
|
||||||
|
result.add(primary_tenant)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def tenant_membership(user: dict[str, Any], tenant_id: str | None = None) -> dict[str, Any] | None:
|
||||||
|
"""Return the active membership for the selected tenant, if any."""
|
||||||
|
if is_admin(user):
|
||||||
|
return {"tenant_id": tenant_id or user.get("tenant_id") or "default", "role": "owner", "status": "active"}
|
||||||
|
target = str(tenant_id or user.get("tenant_id") or "default")
|
||||||
|
try:
|
||||||
|
with get_platform_store().connect() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT tenant_id, role, status, expires_at FROM tenant_members "
|
||||||
|
"WHERE tenant_id=? AND user_id=? AND status='active'",
|
||||||
|
(target, user.get("id")),
|
||||||
|
).fetchone()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
if row.get("expires_at"):
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
try:
|
||||||
|
if datetime.fromisoformat(str(row["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||||
|
return None
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return dict(row)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_tenant_admin(user: dict[str, Any], tenant_id: str | None = None) -> bool:
|
||||||
|
membership = tenant_membership(user, tenant_id)
|
||||||
|
return bool(membership and membership.get("role") in {"owner", "admin"})
|
||||||
|
|
||||||
|
|
||||||
|
def is_tenant_admin_for_resource(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool:
|
||||||
|
if is_admin(user) or resource_type == "model":
|
||||||
|
return False
|
||||||
|
tenant_id = resource_tenant_id(resource_type, resource)
|
||||||
|
return bool(tenant_id and is_tenant_admin(user, tenant_id))
|
||||||
|
|
||||||
|
|
||||||
|
def bind_active_tenant(payload: dict[str, Any], user: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Bind a new resource to the authenticated tenant context.
|
||||||
|
|
||||||
|
Platform administrators may explicitly create a resource in another
|
||||||
|
active tenant. Ordinary users can only use the tenant selected by the
|
||||||
|
authenticated session/X-Tenant-ID header, never a client-supplied tenant
|
||||||
|
id alone.
|
||||||
|
"""
|
||||||
|
requested = str(payload.get("tenant_id") or user.get("tenant_id") or "default")
|
||||||
|
if not is_admin(user) and requested not in user_tenant_ids(user):
|
||||||
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="tenant access denied")
|
||||||
|
payload["tenant_id"] = requested if is_admin(user) else str(user.get("tenant_id") or requested)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def resource_record(resource_type: str, resource_id: str) -> dict[str, Any] | None:
|
||||||
|
"""Load a resource row for authorization without exposing storage details."""
|
||||||
|
table = RESOURCE_TABLES.get(resource_type)
|
||||||
|
if not table or not resource_id:
|
||||||
|
return None
|
||||||
|
store = get_platform_store()
|
||||||
|
try:
|
||||||
|
with store.connect() as conn:
|
||||||
|
row = conn.execute(f"SELECT * FROM {table} WHERE id=?", (resource_id,)).fetchone()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
result = dict(row)
|
||||||
|
if result.get("deleted_at"):
|
||||||
|
return None
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def resource_tenant_id(resource_type: str, resource: dict[str, Any]) -> str | None:
|
||||||
|
"""Resolve tenant ownership, including legacy JSON-backed task rows."""
|
||||||
|
# Administrator-created base models are platform shared. Online models
|
||||||
|
# created by ordinary users retain tenant scope, so their visibility can
|
||||||
|
# be limited to the owner and members of that tenant.
|
||||||
|
if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}:
|
||||||
|
return None
|
||||||
|
tenant_id = resource.get("tenant_id")
|
||||||
|
if tenant_id:
|
||||||
|
return str(tenant_id)
|
||||||
|
payload = resource.get("payload")
|
||||||
|
if payload:
|
||||||
|
try:
|
||||||
|
data = json.loads(payload) if isinstance(payload, str) else payload
|
||||||
|
if isinstance(data, dict) and data.get("tenant_id"):
|
||||||
|
return str(data["tenant_id"])
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def resource_in_user_tenant(resource_type: str, resource: dict[str, Any], user: dict[str, Any]) -> bool:
|
||||||
|
if is_admin(user):
|
||||||
|
return True
|
||||||
|
if resource_type == "model" and str(resource.get("model_source") or "").lower() not in {"api", "online"}:
|
||||||
|
return True
|
||||||
|
tenant_id = resource_tenant_id(resource_type, resource)
|
||||||
|
if not tenant_id:
|
||||||
|
# A missing tenant is not an implicit shared scope. The migration must
|
||||||
|
# assign historical rows before ordinary users can access them.
|
||||||
|
return False
|
||||||
|
return "*" in user_tenant_ids(user) or tenant_id in user_tenant_ids(user)
|
||||||
|
|
||||||
|
|
||||||
def has_resource_access(
|
def has_resource_access(
|
||||||
@@ -93,16 +357,57 @@ def has_resource_access(
|
|||||||
- admin/protected 用户直接放行(旁路)。
|
- admin/protected 用户直接放行(旁路)。
|
||||||
- 其他用户检查 acls 表中是否有对应授权。
|
- 其他用户检查 acls 表中是否有对应授权。
|
||||||
"""
|
"""
|
||||||
if user.get("role") == "admin" or user.get("protected"):
|
if permission not in RESOURCE_ACTIONS:
|
||||||
|
return False
|
||||||
|
permission = RESOURCE_ACTION_ALIASES.get(permission, permission)
|
||||||
|
if is_admin(user):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
resource = resource_record(resource_type, resource_id)
|
||||||
|
if not resource:
|
||||||
|
return False
|
||||||
|
# Some historical datasets were created before tenant_id was introduced.
|
||||||
|
# They are not shared by default, but an explicit user ACL granted by an
|
||||||
|
# administrator is still a valid compatibility path for that legacy row.
|
||||||
|
legacy_unscoped = resource_type != "model" and not resource_tenant_id(resource_type, resource)
|
||||||
|
if not resource_in_user_tenant(resource_type, resource, user) and not legacy_unscoped:
|
||||||
|
return False
|
||||||
|
if resource_type == "model":
|
||||||
|
model_source = str(resource.get("model_source") or "").lower()
|
||||||
|
if permission in {"read", "execute"}:
|
||||||
|
# Local/registered base models are platform resources. For online
|
||||||
|
# models, only administrator-created records are global; a normal
|
||||||
|
# user's online model is shared with the active tenant below.
|
||||||
|
if model_source not in {"api", "online"}:
|
||||||
|
return True
|
||||||
|
creator_id = str(resource.get("created_by") or "")
|
||||||
|
if creator_id:
|
||||||
|
with store.connect() as conn:
|
||||||
|
creator = conn.execute(
|
||||||
|
"SELECT platform_role, role, protected FROM users WHERE id=?",
|
||||||
|
(creator_id,),
|
||||||
|
).fetchone()
|
||||||
|
if creator and (
|
||||||
|
creator.get("platform_role") == "platform_admin"
|
||||||
|
or creator.get("role") == "admin"
|
||||||
|
or bool(creator.get("protected"))
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
# The tenant check above has already rejected models outside the
|
||||||
|
# user's active tenant. Same-tenant online models are usable.
|
||||||
|
if resource_tenant_id(resource_type, resource) in user_tenant_ids(user):
|
||||||
|
return True
|
||||||
|
if is_tenant_admin_for_resource(resource_type, resource, user):
|
||||||
|
return True
|
||||||
acls = store.get_acl(resource_type, resource_id)
|
acls = store.get_acl(resource_type, resource_id)
|
||||||
user_id = user.get("id")
|
user_id = user.get("id")
|
||||||
user_role = user.get("role")
|
# ACL role principals are tenant roles, not platform roles. Falling back
|
||||||
|
# to the platform role keeps compatibility with old ACL records.
|
||||||
|
membership = tenant_membership(user, resource_tenant_id(resource_type, resource))
|
||||||
|
user_role = (membership or {}).get("role") or user.get("role")
|
||||||
|
|
||||||
owner_tables = OWNER_TABLES
|
table_info = OWNER_TABLES.get(resource_type)
|
||||||
table_info = owner_tables.get(resource_type)
|
|
||||||
if table_info and user_id:
|
if table_info and user_id:
|
||||||
table, column = table_info
|
table, column = table_info
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
@@ -111,7 +416,6 @@ def has_resource_access(
|
|||||||
owner = row[column]
|
owner = row[column]
|
||||||
if column == "payload":
|
if column == "payload":
|
||||||
try:
|
try:
|
||||||
import json
|
|
||||||
owner = json.loads(owner or "{}").get("created_by")
|
owner = json.loads(owner or "{}").get("created_by")
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
owner = None
|
owner = None
|
||||||
@@ -119,6 +423,15 @@ def has_resource_access(
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
for entry in acls:
|
for entry in acls:
|
||||||
|
if entry.get("revoked_at"):
|
||||||
|
continue
|
||||||
|
if entry.get("expires_at"):
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
try:
|
||||||
|
if datetime.fromisoformat(str(entry["expires_at"]).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
# 按 user 授权
|
# 按 user 授权
|
||||||
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
|
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
|
||||||
if _permission_covers(entry.get("permission"), permission):
|
if _permission_covers(entry.get("permission"), permission):
|
||||||
@@ -154,48 +467,13 @@ def filter_accessible_resource_ids(
|
|||||||
- admin 直接返回全部。
|
- admin 直接返回全部。
|
||||||
- 普通用户查 acls 表取交集。
|
- 普通用户查 acls 表取交集。
|
||||||
"""
|
"""
|
||||||
if user.get("role") == "admin" or user.get("protected"):
|
if is_admin(user):
|
||||||
return all_ids
|
return all_ids
|
||||||
|
|
||||||
if not all_ids:
|
if not all_ids:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
store = get_platform_store()
|
accessible = filter_accessible_resource_ids_batch(resource_type, all_ids, user)
|
||||||
user_id = user.get("id")
|
|
||||||
user_role = user.get("role")
|
|
||||||
|
|
||||||
# 查询该用户在该资源类型下有 read 权限的所有 resource_id
|
|
||||||
with store.connect() as conn:
|
|
||||||
rows = conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT DISTINCT resource_id FROM acls
|
|
||||||
WHERE resource_type=? AND (
|
|
||||||
(principal_type='user' AND principal_id=?)
|
|
||||||
OR (principal_type='role' AND principal_id=?)
|
|
||||||
)
|
|
||||||
""",
|
|
||||||
(resource_type, user_id, user_role),
|
|
||||||
).fetchall()
|
|
||||||
|
|
||||||
accessible = {r["resource_id"] for r in rows}
|
|
||||||
if resource_type in OWNER_TABLES:
|
|
||||||
table, column = OWNER_TABLES[resource_type]
|
|
||||||
if column == "payload":
|
|
||||||
# payload 是 JSON 字符串,需要查出后解析 created_by
|
|
||||||
with store.connect() as conn:
|
|
||||||
owned = conn.execute(f"SELECT id, {column} FROM {table}").fetchall()
|
|
||||||
for row in owned:
|
|
||||||
try:
|
|
||||||
import json
|
|
||||||
payload = json.loads(row[column] or "{}")
|
|
||||||
if payload.get("created_by") == user_id:
|
|
||||||
accessible.add(row["id"])
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
with store.connect() as conn:
|
|
||||||
owned = conn.execute(f"SELECT id FROM {table} WHERE {column}=?", (user_id,)).fetchall()
|
|
||||||
accessible.update(row["id"] for row in owned)
|
|
||||||
return [rid for rid in all_ids if rid in accessible]
|
return [rid for rid in all_ids if rid in accessible]
|
||||||
|
|
||||||
|
|
||||||
@@ -204,37 +482,134 @@ def filter_accessible_resource_ids_batch(
|
|||||||
resource_ids: list[str],
|
resource_ids: list[str],
|
||||||
user: dict[str, Any],
|
user: dict[str, Any],
|
||||||
) -> set[str]:
|
) -> set[str]:
|
||||||
"""Filter a list endpoint with one ACL query instead of one query per row."""
|
"""Filter a list endpoint with a bounded set of SQL queries.
|
||||||
|
|
||||||
|
The previous implementation called ``has_resource_access`` once per
|
||||||
|
resource. Each call loaded the resource, tenant membership and ACL again,
|
||||||
|
which made ordinary-user list pages slow on a remote PostgreSQL server.
|
||||||
|
"""
|
||||||
if is_admin(user):
|
if is_admin(user):
|
||||||
return set(resource_ids)
|
return set(resource_ids)
|
||||||
if not resource_ids:
|
ids = list(dict.fromkeys(str(item) for item in resource_ids if item))
|
||||||
|
if not ids:
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
|
table = RESOURCE_TABLES.get(resource_type)
|
||||||
|
if not table:
|
||||||
|
return set()
|
||||||
|
placeholders = ",".join("?" for _ in ids)
|
||||||
|
primary_tenant = str(user.get("tenant_id") or "default")
|
||||||
|
tenant_ids = {primary_tenant}
|
||||||
|
tenant_roles: dict[str, str] = {primary_tenant: str(user.get("role") or "")}
|
||||||
|
user_id = str(user.get("id") or "")
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
placeholders = ",".join("?" for _ in resource_ids)
|
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
|
memberships = conn.execute(
|
||||||
|
"SELECT tenant_id, role FROM tenant_members "
|
||||||
|
"WHERE user_id=? AND status='active' "
|
||||||
|
"AND (expires_at IS NULL OR expires_at='' OR expires_at > NOW()::text)",
|
||||||
|
(user_id,),
|
||||||
|
).fetchall()
|
||||||
|
for membership in memberships:
|
||||||
|
tenant_id = str(membership["tenant_id"] or "")
|
||||||
|
if tenant_id:
|
||||||
|
tenant_ids.add(tenant_id)
|
||||||
|
tenant_roles[tenant_id] = str(membership["role"] or "")
|
||||||
|
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
f"SELECT DISTINCT resource_id FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
|
f"SELECT * FROM {table} WHERE id IN ({placeholders})", tuple(ids)
|
||||||
"AND ((principal_type='user' AND principal_id=?) OR (principal_type='role' AND principal_id=?))",
|
|
||||||
(resource_type, *resource_ids, user.get("id"), user.get("role")),
|
|
||||||
).fetchall()
|
).fetchall()
|
||||||
accessible = {row["resource_id"] for row in rows}
|
row_by_id = {str(row["id"]): dict(row) for row in rows}
|
||||||
table_info = OWNER_TABLES.get(resource_type)
|
acl_rows = conn.execute(
|
||||||
if table_info and user.get("id"):
|
"SELECT resource_id, principal_type, principal_id, permission, expires_at "
|
||||||
table, column = table_info
|
f"FROM acls WHERE resource_type=? AND resource_id IN ({placeholders}) "
|
||||||
|
"AND (revoked_at IS NULL OR revoked_at='')",
|
||||||
|
(resource_type, *ids),
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
acl_by_resource: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for row in acl_rows:
|
||||||
|
acl_by_resource.setdefault(str(row["resource_id"]), []).append(dict(row))
|
||||||
|
|
||||||
|
allowed: set[str] = set()
|
||||||
|
for resource_id in ids:
|
||||||
|
resource = row_by_id.get(resource_id)
|
||||||
|
if not resource or resource.get("deleted_at"):
|
||||||
|
continue
|
||||||
|
if resource_type == "model":
|
||||||
|
model_source = str(resource.get("model_source") or "").lower()
|
||||||
|
if model_source not in {"api", "online"}:
|
||||||
|
allowed.add(resource_id)
|
||||||
|
continue
|
||||||
|
creator_id = str(resource.get("created_by") or "")
|
||||||
|
if creator_id:
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
owned = conn.execute(
|
creator = conn.execute(
|
||||||
f"SELECT id, {column} FROM {table} WHERE id IN ({placeholders})",
|
"SELECT platform_role, role, protected FROM users WHERE id=?",
|
||||||
(*resource_ids,),
|
(creator_id,),
|
||||||
).fetchall()
|
).fetchone()
|
||||||
for row in owned:
|
if creator and (
|
||||||
owner = row[column]
|
creator.get("platform_role") == "platform_admin"
|
||||||
# 如果列是 payload(JSON),需要解析后提取 created_by
|
or creator.get("role") == "admin"
|
||||||
if column == "payload":
|
or bool(creator.get("protected"))
|
||||||
|
):
|
||||||
|
allowed.add(resource_id)
|
||||||
|
continue
|
||||||
|
tenant_id = resource_tenant_id(resource_type, resource)
|
||||||
|
if tenant_id and tenant_id in tenant_ids:
|
||||||
|
allowed.add(resource_id)
|
||||||
|
continue
|
||||||
|
tenant_id = resource_tenant_id(resource_type, resource)
|
||||||
|
if not tenant_id:
|
||||||
|
# Legacy rows without a tenant are never implicitly visible. Only
|
||||||
|
# a direct user ACL can expose one to its explicitly named user;
|
||||||
|
# role ACLs remain blocked until the row is tenant-migrated.
|
||||||
|
for entry in acl_by_resource.get(resource_id, []):
|
||||||
|
expires_at = entry.get("expires_at")
|
||||||
|
if expires_at:
|
||||||
try:
|
try:
|
||||||
import json
|
from datetime import datetime, timezone
|
||||||
owner = json.loads(owner or "{}").get("created_by")
|
if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||||
except (TypeError, ValueError):
|
continue
|
||||||
owner = None
|
except ValueError:
|
||||||
if owner == user["id"]:
|
continue
|
||||||
accessible.add(row["id"])
|
if (
|
||||||
return accessible
|
entry.get("principal_type") == "user"
|
||||||
|
and entry.get("principal_id") == user_id
|
||||||
|
and _permission_covers(entry.get("permission"), "read")
|
||||||
|
):
|
||||||
|
allowed.add(resource_id)
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
if tenant_id not in tenant_ids:
|
||||||
|
continue
|
||||||
|
if tenant_roles.get(tenant_id) in {"owner", "admin"}:
|
||||||
|
allowed.add(resource_id)
|
||||||
|
continue
|
||||||
|
owner = resource.get("created_by")
|
||||||
|
if resource_type in {"eval", "fine-tune", "fine_tune_task", "compare", "inference"} and resource.get("payload"):
|
||||||
|
try:
|
||||||
|
payload = json.loads(resource["payload"]) if isinstance(resource["payload"], str) else resource["payload"]
|
||||||
|
if isinstance(payload, dict) and payload.get("created_by"):
|
||||||
|
owner = payload["created_by"]
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
pass
|
||||||
|
if owner == user_id:
|
||||||
|
allowed.add(resource_id)
|
||||||
|
continue
|
||||||
|
role = tenant_roles.get(tenant_id) or str(user.get("role") or "")
|
||||||
|
for entry in acl_by_resource.get(resource_id, []):
|
||||||
|
expires_at = entry.get("expires_at")
|
||||||
|
if expires_at:
|
||||||
|
try:
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
if datetime.fromisoformat(str(expires_at).replace("Z", "+00:00")) <= datetime.now(timezone.utc):
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
user_match = entry.get("principal_type") == "user" and entry.get("principal_id") == user_id
|
||||||
|
role_match = entry.get("principal_type") == "role" and entry.get("principal_id") == role
|
||||||
|
if (user_match or role_match) and _permission_covers(entry.get("permission"), "read"):
|
||||||
|
allowed.add(resource_id)
|
||||||
|
break
|
||||||
|
return allowed
|
||||||
|
|||||||
46
backend/app/core/cache_paths.py
Normal file
46
backend/app/core/cache_paths.py
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
"""集中管理项目本地缓存目录(HuggingFace / tiktoken)。
|
||||||
|
|
||||||
|
所有 Python 库通过环境变量引用 ``<repo_root>/.cache/{huggingface,tiktoken}``,
|
||||||
|
避免写入用户家目录,也避免不同部署路径(本地 / Docker)下缓存位置不一致。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def repo_root() -> Path:
|
||||||
|
# backend/app/core/cache_paths.py -> backend -> 仓库根目录
|
||||||
|
return Path(__file__).resolve().parents[3]
|
||||||
|
|
||||||
|
|
||||||
|
def cache_root() -> Path:
|
||||||
|
return repo_root() / ".cache"
|
||||||
|
|
||||||
|
|
||||||
|
def huggingface_cache_dir() -> Path:
|
||||||
|
return cache_root() / "huggingface"
|
||||||
|
|
||||||
|
|
||||||
|
def tiktoken_cache_dir() -> Path:
|
||||||
|
return cache_root() / "tiktoken"
|
||||||
|
|
||||||
|
|
||||||
|
def setup_local_caches() -> None:
|
||||||
|
"""进程启动时统一设置 HF / tiktoken 缓存环境变量,并确保目录存在。
|
||||||
|
|
||||||
|
必须在 import ``docling`` / ``tiktoken`` 等依赖之前调用,否则首次使用会
|
||||||
|
仍然走到默认 ``~/.cache`` 路径。
|
||||||
|
"""
|
||||||
|
|
||||||
|
hf_dir = huggingface_cache_dir()
|
||||||
|
tiktoken_dir = tiktoken_cache_dir()
|
||||||
|
hf_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(hf_dir / "hub").mkdir(parents=True, exist_ok=True)
|
||||||
|
tiktoken_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
os.environ["HF_HOME"] = str(hf_dir)
|
||||||
|
os.environ["HUGGINGFACE_HUB_CACHE"] = str(hf_dir / "hub")
|
||||||
|
os.environ["HF_HUB_CACHE"] = str(hf_dir / "hub")
|
||||||
|
os.environ["TIKTOKEN_CACHE_DIR"] = str(tiktoken_dir)
|
||||||
@@ -71,6 +71,7 @@ class Settings:
|
|||||||
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
|
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
|
||||||
# MinIO round trips. Larger files remain the shared canonical objects.
|
# MinIO round trips. Larger files remain the shared canonical objects.
|
||||||
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
|
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
|
||||||
|
minio_presign_max_bytes: int = _int_env("MINIO_PRESIGN_MAX_BYTES", 1024 * 1024 * 1024 * 1024)
|
||||||
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
|
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
|
||||||
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
|
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
|
||||||
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||||
|
|||||||
@@ -23,6 +23,17 @@ request_id_var: ContextVar[str] = ContextVar("request_id", default="-")
|
|||||||
user_id_var: ContextVar[str] = ContextVar("user_id", default="")
|
user_id_var: ContextVar[str] = ContextVar("user_id", default="")
|
||||||
client_ip_var: ContextVar[str] = ContextVar("client_ip", default="")
|
client_ip_var: ContextVar[str] = ContextVar("client_ip", default="")
|
||||||
|
|
||||||
|
|
||||||
|
def get_client_ip(request: Request | None) -> str:
|
||||||
|
"""获取客户端地址,兼容前置反向代理传递的真实地址。"""
|
||||||
|
if request is None:
|
||||||
|
return ""
|
||||||
|
for header in ("X-Real-IP", "X-Forwarded-For"):
|
||||||
|
value = request.headers.get(header, "")
|
||||||
|
if value:
|
||||||
|
return value.split(",", 1)[0].strip()
|
||||||
|
return request.client.host if request.client else ""
|
||||||
|
|
||||||
# ==================== 敏感数据脱敏 ====================
|
# ==================== 敏感数据脱敏 ====================
|
||||||
|
|
||||||
SENSITIVE_KEYS: set[str] = {
|
SENSITIVE_KEYS: set[str] = {
|
||||||
@@ -91,6 +102,18 @@ def mask_sensitive_string(text: str) -> str:
|
|||||||
"""从文本中脱敏常见敏感信息。"""
|
"""从文本中脱敏常见敏感信息。"""
|
||||||
if not text:
|
if not text:
|
||||||
return text
|
return text
|
||||||
|
# Mask the value as well as the key. Replacing only ``api_key=`` would
|
||||||
|
# still leak the credential in audit messages and exception text.
|
||||||
|
assignment_pattern = (
|
||||||
|
r"(Bearer\s+|(?:api[-_]?key|access[_-]?token|refresh[_-]?token|"
|
||||||
|
r"secret[_-]?key|password|private[_-]?key|token)\s*[:=]\s*)"
|
||||||
|
r"(\"[^\"]*\"|'[^']*'|[^\s,;]+)"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
text = re.sub(assignment_pattern, r"\1***", text, flags=re.IGNORECASE)
|
||||||
|
except re.error:
|
||||||
|
pass
|
||||||
|
|
||||||
patterns: list[tuple[str, str]] = [
|
patterns: list[tuple[str, str]] = [
|
||||||
(r"Bearer\s+[A-Za-z0-9\-._]+", "Bearer ***"),
|
(r"Bearer\s+[A-Za-z0-9\-._]+", "Bearer ***"),
|
||||||
(r"(?i)token\s*[:=]\s*\S+", "token=***"),
|
(r"(?i)token\s*[:=]\s*\S+", "token=***"),
|
||||||
@@ -431,6 +454,7 @@ def setup_request_logging(app: FastAPI) -> None:
|
|||||||
# 入口生成 traceId(优先使用前端传入的 X-Trace-Id)
|
# 入口生成 traceId(优先使用前端传入的 X-Trace-Id)
|
||||||
trace_id = request.headers.get("X-Trace-Id") or request.headers.get("X-Request-ID") or str(uuid4())
|
trace_id = request.headers.get("X-Trace-Id") or request.headers.get("X-Request-ID") or str(uuid4())
|
||||||
token = request_id_var.set(trace_id)
|
token = request_id_var.set(trace_id)
|
||||||
|
ip_token = client_ip_var.set(get_client_ip(request))
|
||||||
started_at = time.perf_counter()
|
started_at = time.perf_counter()
|
||||||
|
|
||||||
# 提取客户端 IP
|
# 提取客户端 IP
|
||||||
@@ -457,9 +481,9 @@ def setup_request_logging(app: FastAPI) -> None:
|
|||||||
elif response.status_code >= 400:
|
elif response.status_code >= 400:
|
||||||
log_method = logger.warning
|
log_method = logger.warning
|
||||||
|
|
||||||
# 结构化访问日志
|
# 结构化访问日志(中文 message,方便直接阅读)
|
||||||
log_method(
|
log_method(
|
||||||
"request completed",
|
f"HTTP请求 {request.method} {request.url.path} → {response.status_code}(耗时{round(elapsed_ms, 2)}ms)",
|
||||||
extra={
|
extra={
|
||||||
"request_method": request.method,
|
"request_method": request.method,
|
||||||
"request_path": request.url.path,
|
"request_path": request.url.path,
|
||||||
@@ -497,7 +521,7 @@ def setup_request_logging(app: FastAPI) -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
elapsed_ms = (time.perf_counter() - started_at) * 1000
|
||||||
logger.error(
|
logger.error(
|
||||||
"request failed",
|
f"HTTP请求异常 {request.method} {request.url.path}(耗时{round(elapsed_ms, 2)}ms)— 服务内部错误",
|
||||||
extra={
|
extra={
|
||||||
"request_method": request.method,
|
"request_method": request.method,
|
||||||
"request_path": request.url.path,
|
"request_path": request.url.path,
|
||||||
@@ -531,6 +555,7 @@ def setup_request_logging(app: FastAPI) -> None:
|
|||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
request_id_var.reset(token)
|
request_id_var.reset(token)
|
||||||
|
client_ip_var.reset(ip_token)
|
||||||
|
|
||||||
|
|
||||||
# ==================== 配置函数 ====================
|
# ==================== 配置函数 ====================
|
||||||
|
|||||||
@@ -77,6 +77,92 @@ class OpStatus:
|
|||||||
FAILURE = "failure"
|
FAILURE = "failure"
|
||||||
|
|
||||||
|
|
||||||
|
# ==================== 中文映射表(让日志 message 直接可读)====================
|
||||||
|
|
||||||
|
MODULE_CN: dict[str, str] = {
|
||||||
|
"fine-tune": "模型训练",
|
||||||
|
"model-eval": "模型评测",
|
||||||
|
"model-inference": "模型推理",
|
||||||
|
"model-manage": "模型管理",
|
||||||
|
"dataset": "数据集",
|
||||||
|
"data-process": "数据处理",
|
||||||
|
"data-convert": "数据转换",
|
||||||
|
"compute": "算力节点",
|
||||||
|
"system": "系统",
|
||||||
|
}
|
||||||
|
|
||||||
|
ACTION_CN: dict[str, str] = {
|
||||||
|
"create": "创建",
|
||||||
|
"update": "更新",
|
||||||
|
"delete": "删除",
|
||||||
|
"start": "启动",
|
||||||
|
"stop": "停止",
|
||||||
|
"upload": "上传",
|
||||||
|
"download": "下载",
|
||||||
|
"convert": "转换",
|
||||||
|
"merge": "合并",
|
||||||
|
"import": "导入",
|
||||||
|
"login": "登录",
|
||||||
|
"logout": "退出登录",
|
||||||
|
"publish": "发布",
|
||||||
|
"retry": "重试",
|
||||||
|
"request": "请求",
|
||||||
|
}
|
||||||
|
|
||||||
|
TARGET_TYPE_CN: dict[str, str] = {
|
||||||
|
"fine_tune": "训练任务",
|
||||||
|
"eval": "评测任务",
|
||||||
|
"inference": "推理任务",
|
||||||
|
"model": "模型",
|
||||||
|
"trained_model": "训练产出模型",
|
||||||
|
"dataset": "数据集",
|
||||||
|
"dataset_version": "数据集版本",
|
||||||
|
"data_process_task": "数据处理任务",
|
||||||
|
"user": "用户",
|
||||||
|
"api": "接口",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cn_message(
|
||||||
|
module: str,
|
||||||
|
action: str,
|
||||||
|
target_type: str,
|
||||||
|
target_name: str | None,
|
||||||
|
target_id: str | None,
|
||||||
|
status: str,
|
||||||
|
username: str | None,
|
||||||
|
error_type: str,
|
||||||
|
error_message: str,
|
||||||
|
) -> str:
|
||||||
|
"""构建中文人类可读的日志消息,格式:[用户] 对 [模块] 执行了 [动作],结果:成功/失败"""
|
||||||
|
user_part = f"用户[{username}]" if username else "系统"
|
||||||
|
module_cn = MODULE_CN.get(module, module)
|
||||||
|
action_cn = ACTION_CN.get(action, action)
|
||||||
|
target_cn = TARGET_TYPE_CN.get(target_type, target_type or "")
|
||||||
|
target_label = target_name or target_id or ""
|
||||||
|
|
||||||
|
# 拼接操作对象描述
|
||||||
|
if target_cn and target_label:
|
||||||
|
target_part = f"{target_cn}「{target_label}」"
|
||||||
|
elif target_cn:
|
||||||
|
target_part = target_cn
|
||||||
|
elif target_label:
|
||||||
|
target_part = f"「{target_label}」"
|
||||||
|
else:
|
||||||
|
target_part = ""
|
||||||
|
|
||||||
|
if status == OpStatus.SUCCESS:
|
||||||
|
result = "成功"
|
||||||
|
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:成功"
|
||||||
|
else:
|
||||||
|
result = "失败"
|
||||||
|
err_brief = error_message[:120] if error_message else ""
|
||||||
|
err_part = f"({error_type}: {err_brief})" if error_type and err_brief else f"({error_type})" if error_type else ""
|
||||||
|
msg = f"{user_part} {action_cn}{module_cn}{target_part},结果:失败{err_part}"
|
||||||
|
|
||||||
|
return msg
|
||||||
|
|
||||||
|
|
||||||
def op_log(
|
def op_log(
|
||||||
module: str,
|
module: str,
|
||||||
action: str,
|
action: str,
|
||||||
@@ -354,6 +440,12 @@ def _write_log(
|
|||||||
req_path = request.url.path
|
req_path = request.url.path
|
||||||
|
|
||||||
# ---- 写文件日志(app-biz)----
|
# ---- 写文件日志(app-biz)----
|
||||||
|
cn_msg = _build_cn_message(
|
||||||
|
module=module, action=action, target_type=target_type,
|
||||||
|
target_name=target_name, target_id=target_id, status=status,
|
||||||
|
username=username, error_type=error_type, error_message=error_message,
|
||||||
|
)
|
||||||
|
|
||||||
log_fields = {
|
log_fields = {
|
||||||
"bizModule": module,
|
"bizModule": module,
|
||||||
"action": action,
|
"action": action,
|
||||||
@@ -364,6 +456,10 @@ def _write_log(
|
|||||||
"durationMs": round(duration_ms, 2),
|
"durationMs": round(duration_ms, 2),
|
||||||
"username": username or "",
|
"username": username or "",
|
||||||
}
|
}
|
||||||
|
if req_method:
|
||||||
|
log_fields["requestMethod"] = req_method
|
||||||
|
if req_path:
|
||||||
|
log_fields["requestPath"] = req_path
|
||||||
if detail:
|
if detail:
|
||||||
log_fields["detail"] = detail[:500]
|
log_fields["detail"] = detail[:500]
|
||||||
if error_message:
|
if error_message:
|
||||||
@@ -372,9 +468,9 @@ def _write_log(
|
|||||||
log_fields["errorType"] = error_type
|
log_fields["errorType"] = error_type
|
||||||
|
|
||||||
if status == OpStatus.SUCCESS:
|
if status == OpStatus.SUCCESS:
|
||||||
biz_logger.info(f"{module}:{action} {status} - {target_name or target_id or ''}", **log_fields)
|
biz_logger.info(cn_msg, **log_fields)
|
||||||
elif status == OpStatus.FAILURE:
|
elif status == OpStatus.FAILURE:
|
||||||
biz_logger.error(f"{module}:{action} {status} - {target_name or target_id or ''} - {error_type}: {error_message[:200]}", **log_fields)
|
biz_logger.error(cn_msg, **log_fields)
|
||||||
|
|
||||||
# ---- 写数据库 ----
|
# ---- 写数据库 ----
|
||||||
try:
|
try:
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@
|
|||||||
-- 用途:切换到新的 PG 数据集时,一次性创建平台运行所需的全部数据库对象与
|
-- 用途:切换到新的 PG 数据集时,一次性创建平台运行所需的全部数据库对象与
|
||||||
-- 基础种子数据(幂等,可重复执行)。
|
-- 基础种子数据(幂等,可重复执行)。
|
||||||
--
|
--
|
||||||
-- 覆盖范围(与运行时代码实际使用的表一致):
|
-- 覆盖范围(与运行时代码实际使用的表一致,离线新库只需执行本文件):
|
||||||
-- 001_platform_runtime.sql 平台核心表
|
-- 001_platform_runtime.sql 平台核心表
|
||||||
-- 002_governance.sql 治理表(租户 / 审批 / 审计 / 留存)
|
-- 002_governance.sql 治理表(租户 / 审批 / 审计 / 留存)
|
||||||
-- 003_tenant_quota.sql 租户配额列
|
-- 003_tenant_quota.sql 租户配额列
|
||||||
@@ -16,10 +16,9 @@
|
|||||||
-- 说明:
|
-- 说明:
|
||||||
-- * 本脚本通过 psql 执行,包含 DO $$ ... $$ 块与事务,不能用应用的
|
-- * 本脚本通过 psql 执行,包含 DO $$ ... $$ 块与事务,不能用应用的
|
||||||
-- executescript()(按分号切分)执行。
|
-- executescript()(按分号切分)执行。
|
||||||
-- * 应用启动时 PlatformStore.ensure_schema() 只会自动执行
|
-- * 本文件已经合并平台运行、治理、数据处理、权限、MinIO、GPU 预留、
|
||||||
-- 001 / 002_governance / 003_tenant_quota;数据处理表需另跑
|
-- 缓存治理和数据转换等全部初始化对象;全新数据库无需再执行其他 SQL。
|
||||||
-- 002_data_process.sql(本脚本已包含)。应用首次启动还会自动补充
|
-- 应用启动时的 ensure_schema() 仅作为兼容兜底,不是离线初始化前置条件。
|
||||||
-- admin/operator 种子用户(本脚本已包含,二选一即可)。
|
|
||||||
-- * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
|
-- * 脚本内所有 DDL 均使用 IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
|
||||||
-- 可在已初始化的库上安全重复执行。
|
-- 可在已初始化的库上安全重复执行。
|
||||||
--
|
--
|
||||||
@@ -45,12 +44,20 @@ CREATE TABLE IF NOT EXISTS users (
|
|||||||
password_hash TEXT NOT NULL,
|
password_hash TEXT NOT NULL,
|
||||||
display_name TEXT NOT NULL,
|
display_name TEXT NOT NULL,
|
||||||
role TEXT NOT NULL,
|
role TEXT NOT NULL,
|
||||||
|
platform_role TEXT NOT NULL DEFAULT 'platform_user',
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
permissions TEXT NOT NULL,
|
permissions TEXT NOT NULL,
|
||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
last_login TEXT,
|
last_login TEXT,
|
||||||
protected INTEGER NOT NULL DEFAULT 0
|
protected INTEGER NOT NULL DEFAULT 0,
|
||||||
|
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
deleted_at TEXT,
|
||||||
|
deleted_by TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS models (
|
CREATE TABLE IF NOT EXISTS models (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -68,6 +75,9 @@ CREATE TABLE IF NOT EXISTS models (
|
|||||||
created_by TEXT,
|
created_by TEXT,
|
||||||
tenant_id TEXT,
|
tenant_id TEXT,
|
||||||
project_id TEXT,
|
project_id TEXT,
|
||||||
|
storage_status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
storage_error TEXT,
|
||||||
|
storage_version_id TEXT,
|
||||||
deleted_at TEXT,
|
deleted_at TEXT,
|
||||||
deleted_by TEXT
|
deleted_by TEXT
|
||||||
);
|
);
|
||||||
@@ -131,6 +141,10 @@ CREATE TABLE IF NOT EXISTS model_export_jobs (
|
|||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
completed_at TEXT
|
completed_at TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending';
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS datasets (
|
CREATE TABLE IF NOT EXISTS datasets (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -209,8 +223,16 @@ CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
|||||||
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
||||||
gpus TEXT NOT NULL,
|
gpus TEXT NOT NULL,
|
||||||
sync_job_id TEXT,
|
sync_job_id TEXT,
|
||||||
compute_job_id TEXT
|
compute_job_id TEXT,
|
||||||
|
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
created_by TEXT,
|
||||||
|
deleted_at TEXT,
|
||||||
|
deleted_by TEXT
|
||||||
);
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active
|
||||||
|
ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator
|
||||||
|
ON fine_tune_tasks(created_by, deleted_at, create_time DESC);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
|
CREATE TABLE IF NOT EXISTS fine_tune_metrics (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -273,6 +295,8 @@ CREATE TABLE IF NOT EXISTS resource_replicas (
|
|||||||
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||||
resource_type TEXT NOT NULL,
|
resource_type TEXT NOT NULL,
|
||||||
resource_id TEXT NOT NULL,
|
resource_id TEXT NOT NULL,
|
||||||
|
version_id TEXT,
|
||||||
|
storage_object_id TEXT,
|
||||||
local_path TEXT NOT NULL,
|
local_path TEXT NOT NULL,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
sync_status TEXT NOT NULL,
|
sync_status TEXT NOT NULL,
|
||||||
@@ -310,6 +334,11 @@ CREATE TABLE IF NOT EXISTS storage_objects (
|
|||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
UNIQUE (resource_type, resource_id, version_id, object_key)
|
UNIQUE (resource_type, resource_id, version_id, object_key)
|
||||||
);
|
);
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
|
CREATE TABLE IF NOT EXISTS storage_cache_jobs (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -323,6 +352,23 @@ CREATE TABLE IF NOT EXISTS storage_cache_jobs (
|
|||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
completed_at TEXT
|
completed_at TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS storage_cleanup_jobs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
|
||||||
|
action TEXT NOT NULL DEFAULT 'delete',
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
error TEXT,
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
completed_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
|
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource ON storage_objects(resource_type, resource_id, version_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
|
CREATE INDEX IF NOT EXISTS idx_storage_cache_jobs_node_status ON storage_cache_jobs(node_id, status);
|
||||||
@@ -332,8 +378,16 @@ CREATE TABLE IF NOT EXISTS eval_tasks (
|
|||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
payload TEXT NOT NULL,
|
payload TEXT NOT NULL,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
create_time TEXT NOT NULL
|
create_time TEXT NOT NULL,
|
||||||
|
created_by TEXT,
|
||||||
|
tenant_id TEXT,
|
||||||
|
deleted_at TEXT,
|
||||||
|
deleted_by TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -349,8 +403,17 @@ CREATE TABLE IF NOT EXISTS compare_tasks (
|
|||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
payload TEXT NOT NULL,
|
payload TEXT NOT NULL,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
create_time TEXT NOT NULL
|
create_time TEXT NOT NULL,
|
||||||
|
created_by TEXT,
|
||||||
|
tenant_id TEXT,
|
||||||
|
deleted_at TEXT,
|
||||||
|
deleted_by TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
|
||||||
@@ -368,6 +431,24 @@ CREATE INDEX IF NOT EXISTS idx_compute_jobs_task ON compute_jobs(task_id);
|
|||||||
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status);
|
CREATE INDEX IF NOT EXISTS idx_compute_jobs_node_status ON compute_jobs(node_id, status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
|
CREATE INDEX IF NOT EXISTS idx_gpu_allocations_node_status ON gpu_allocations(node_id, status);
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running');
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_allocations_active ON gpu_allocations(node_id, gpu_index) WHERE status IN ('allocated','running');
|
||||||
|
|
||||||
|
-- 评测/推理等非训练任务的统一 GPU 原子预留。训练任务继续使用 gpu_allocations。
|
||||||
|
CREATE TABLE IF NOT EXISTS gpu_reservations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||||
|
gpu_index INTEGER NOT NULL,
|
||||||
|
owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')),
|
||||||
|
owner_id TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')),
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
released_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner
|
||||||
|
ON gpu_reservations(owner_type, owner_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status
|
||||||
|
ON gpu_reservations(node_id, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active
|
||||||
|
ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved';
|
||||||
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
|
CREATE INDEX IF NOT EXISTS idx_scheduler_locks_expires ON scheduler_locks(expires_at);
|
||||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
||||||
@@ -393,6 +474,9 @@ CREATE TABLE IF NOT EXISTS projects (
|
|||||||
create_by TEXT,
|
create_by TEXT,
|
||||||
updated_at TEXT
|
updated_at TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS project_members (
|
CREATE TABLE IF NOT EXISTS project_members (
|
||||||
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||||
@@ -433,6 +517,15 @@ CREATE TABLE IF NOT EXISTS acls (
|
|||||||
);
|
);
|
||||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_status TEXT NOT NULL DEFAULT 'pending';
|
||||||
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_error TEXT;
|
||||||
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS storage_version_id TEXT;
|
||||||
|
UPDATE models SET storage_status = CASE
|
||||||
|
WHEN model_source IN ('api', 'online') THEN 'not_applicable'
|
||||||
|
WHEN storage_status IS NULL OR storage_status = '' THEN 'pending'
|
||||||
|
ELSE storage_status
|
||||||
|
END
|
||||||
|
WHERE storage_status IS NULL OR storage_status = '' OR model_source IN ('api', 'online');
|
||||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
@@ -462,6 +555,7 @@ ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
|||||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
|
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||||
|
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- 二、治理表(来源:002_governance.sql)
|
-- 二、治理表(来源:002_governance.sql)
|
||||||
@@ -477,6 +571,15 @@ CREATE TABLE IF NOT EXISTS tenants (
|
|||||||
retention_policy_id TEXT,
|
retention_policy_id TEXT,
|
||||||
create_time TEXT
|
create_time TEXT
|
||||||
);
|
);
|
||||||
|
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at);
|
||||||
|
INSERT INTO tenants (id, name, code, status, quota, create_time)
|
||||||
|
VALUES ('default', '默认租户', 'default', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
|
||||||
|
ON CONFLICT (id) DO NOTHING;
|
||||||
|
INSERT INTO tenants (id, name, code, status, quota, create_time)
|
||||||
|
VALUES ('admin', '管理员租户', 'admin', 'active', '{}', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
|
||||||
|
ON CONFLICT (id) DO UPDATE SET name='管理员租户', code='admin', status='active', deleted_at=NULL, deleted_by=NULL;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS approval_templates (
|
CREATE TABLE IF NOT EXISTS approval_templates (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -534,6 +637,34 @@ CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
|
|||||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||||
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
||||||
|
|
||||||
|
-- ---- 操作日志(接口操作审计) ----
|
||||||
|
CREATE TABLE IF NOT EXISTS operation_logs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT,
|
||||||
|
username TEXT,
|
||||||
|
module TEXT,
|
||||||
|
action TEXT,
|
||||||
|
target_type TEXT,
|
||||||
|
target_id TEXT,
|
||||||
|
target_name TEXT,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
error_message TEXT,
|
||||||
|
error_type TEXT,
|
||||||
|
error_traceback TEXT,
|
||||||
|
func_name TEXT,
|
||||||
|
detail TEXT,
|
||||||
|
client_ip TEXT,
|
||||||
|
request_method TEXT,
|
||||||
|
request_path TEXT,
|
||||||
|
trace_id TEXT,
|
||||||
|
duration_ms REAL,
|
||||||
|
create_time TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_operation_logs_user_time ON operation_logs(user_id, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_operation_logs_module_time ON operation_logs(module, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_operation_logs_status ON operation_logs(status, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_operation_logs_create_time ON operation_logs(create_time DESC);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
@@ -876,29 +1007,201 @@ CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_ta
|
|||||||
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
||||||
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
|
||||||
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_by TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS processed_at TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at);
|
||||||
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
|
||||||
|
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT;
|
||||||
|
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_002 ON resource_replicas(storage_object_id, node_id);
|
||||||
|
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- 七、种子数据:初始管理员 / 操作员
|
-- 六、权限 2.0:租户成员、资源申请、ACL 生命周期和审批动作
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tenant_members (
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
role TEXT NOT NULL DEFAULT 'member',
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
invited_by TEXT,
|
||||||
|
joined_at TEXT,
|
||||||
|
expires_at TEXT,
|
||||||
|
PRIMARY KEY (tenant_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
|
||||||
|
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
|
||||||
|
SELECT COALESCE(u.tenant_id, 'default'), u.id,
|
||||||
|
CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END,
|
||||||
|
'active', COALESCE(u.create_time, NOW()::text)
|
||||||
|
FROM users u
|
||||||
|
ON CONFLICT (tenant_id, user_id) DO NOTHING;
|
||||||
|
|
||||||
|
-- GPU/存储任务使用的租户配额原子预留账本。任务提交时预留,终态或故障回收。
|
||||||
|
CREATE TABLE IF NOT EXISTS tenant_quota_reservations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
owner_type TEXT NOT NULL,
|
||||||
|
owner_id TEXT NOT NULL,
|
||||||
|
gpu_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
storage_bytes BIGINT NOT NULL DEFAULT 0,
|
||||||
|
status TEXT NOT NULL DEFAULT 'reserved',
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
released_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant
|
||||||
|
ON tenant_quota_reservations(tenant_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner
|
||||||
|
ON tenant_quota_reservations(owner_type, owner_id, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner
|
||||||
|
ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS resource_access_requests (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
resource_type TEXT NOT NULL,
|
||||||
|
resource_id TEXT NOT NULL,
|
||||||
|
applicant_id TEXT NOT NULL,
|
||||||
|
principal_type TEXT NOT NULL DEFAULT 'user',
|
||||||
|
principal_id TEXT NOT NULL,
|
||||||
|
requested_permissions TEXT NOT NULL DEFAULT '[]',
|
||||||
|
reason TEXT,
|
||||||
|
approval_id TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
expires_at TEXT,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
decided_at TEXT,
|
||||||
|
decided_by TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource
|
||||||
|
ON resource_access_requests(resource_type, resource_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant
|
||||||
|
ON resource_access_requests(applicant_id, status);
|
||||||
|
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at);
|
||||||
|
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
|
||||||
|
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- 七、权限 2.0 完整闭环:审批策略、执行状态、结构化审计与历史归属
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant';
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active';
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_approval_templates_match
|
||||||
|
ON approval_templates(tenant_id, action, resource_type, status);
|
||||||
|
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending';
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_approval_instances_execution
|
||||||
|
ON approval_instances(status, execution_status, action, resource_type, resource_id);
|
||||||
|
|
||||||
|
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT;
|
||||||
|
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry
|
||||||
|
ON resource_access_requests(status, expires_at);
|
||||||
|
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success';
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time);
|
||||||
|
|
||||||
|
UPDATE datasets d
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = '');
|
||||||
|
UPDATE models m
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
|
||||||
|
UPDATE trained_models m
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
|
||||||
|
UPDATE eval_tasks e
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = '');
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- 八、种子数据:初始管理员 / 操作员
|
||||||
-- 应用首次启动(ensure_seed_data)也会自动创建;此处提供以便脱离应用直接初始化。
|
-- 应用首次启动(ensure_seed_data)也会自动创建;此处提供以便脱离应用直接初始化。
|
||||||
-- 密码:admin / admin123,operator / operator123(上线前请改密)。
|
-- 密码:admin / admin123,operator / operator123(上线前请改密)。
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
|
|
||||||
INSERT INTO users
|
INSERT INTO users
|
||||||
(id, username, password_hash, display_name, role, status, permissions, create_time, protected)
|
(id, username, password_hash, display_name, role, platform_role, status, permissions, create_time, protected, tenant_id)
|
||||||
VALUES
|
VALUES
|
||||||
(
|
(
|
||||||
'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd',
|
'u_admin', 'admin', 'pbkdf2_sha256$390000$ygft_init_salt_admin$2b6f31f22968c4f5a30bcf0acf066b7a0f58d4773d15c5ab898ba715ea87b5bd',
|
||||||
'Platform Admin', 'admin', 'active',
|
'Admin', 'admin', 'platform_admin', 'active',
|
||||||
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]',
|
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs","user-settings"]',
|
||||||
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1
|
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 1, 'admin'
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125',
|
'u_operator', 'operator', 'pbkdf2_sha256$390000$ygft_init_salt_op$525bf35d02ed26f37952cbd6862b0ae358b9d1a7fa0cbbf0217aa2b5dd544125',
|
||||||
'Platform Operator', 'operator', 'active',
|
'Platform Operator', 'operator', 'platform_user', 'active',
|
||||||
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","compute","hardware","logs"]',
|
'["dashboard","fine-tune","model-eval","model-inference","model-manage","dataset","data-process","data-convert","hardware","logs"]',
|
||||||
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0
|
to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 0, 'default'
|
||||||
)
|
)
|
||||||
ON CONFLICT (username) DO NOTHING;
|
ON CONFLICT (username) DO NOTHING;
|
||||||
|
UPDATE users SET display_name='Admin', tenant_id='admin' WHERE username='admin';
|
||||||
|
|
||||||
|
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
|
||||||
|
VALUES
|
||||||
|
('admin', 'u_admin', 'owner', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||||
|
('default', 'u_operator', 'member', 'active', to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
|
||||||
|
ON CONFLICT (tenant_id, user_id) DO NOTHING;
|
||||||
|
UPDATE tenant_members SET status='disabled' WHERE tenant_id='default' AND user_id='u_admin';
|
||||||
|
|
||||||
|
-- Canonical hierarchy fields for new and historical rows. Project columns are
|
||||||
|
-- intentionally retained only as compatibility data and are not used here.
|
||||||
|
UPDATE users
|
||||||
|
SET platform_role = CASE WHEN role = 'admin' OR COALESCE(protected, 0) = 1
|
||||||
|
THEN 'platform_admin' ELSE 'platform_user' END
|
||||||
|
WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user');
|
||||||
|
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
|
||||||
|
SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text)
|
||||||
|
FROM tenants t
|
||||||
|
JOIN users u ON u.id = t.owner_user_id
|
||||||
|
WHERE t.owner_user_id IS NOT NULL AND COALESCE(t.status, 'active') = 'active'
|
||||||
|
ON CONFLICT (tenant_id, user_id) DO UPDATE SET role='owner', status='active';
|
||||||
|
UPDATE tenant_members tm
|
||||||
|
SET role='member'
|
||||||
|
FROM users u
|
||||||
|
WHERE tm.user_id=u.id AND tm.role='owner'
|
||||||
|
AND (u.role <> 'admin' AND COALESCE(u.protected, 0)=0)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM tenants t WHERE t.id=tm.tenant_id AND t.owner_user_id=tm.user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC);
|
||||||
|
|
||||||
COMMIT;
|
COMMIT;
|
||||||
|
|||||||
32
backend/app/db/sql/005_storage_progress_migration.sql
Normal file
32
backend/app/db/sql/005_storage_progress_migration.sql
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
-- YG Fine-Tune Platform additive migration: MinIO completion and cache manifest.
|
||||||
|
-- Execute with psql against an existing database after taking a schema backup.
|
||||||
|
-- The statements are idempotent and are also included in 000_full_init.sql.
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS version_id TEXT;
|
||||||
|
ALTER TABLE resource_replicas ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||||
|
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
||||||
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_replicas_object_005
|
||||||
|
ON resource_replicas(storage_object_id, node_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_storage_objects_resource_005
|
||||||
|
ON storage_objects(resource_type, resource_id, version_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_tasks_active_005
|
||||||
|
ON eval_tasks(create_time DESC) WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
21
backend/app/db/sql/006_gpu_reservations.sql
Normal file
21
backend/app/db/sql/006_gpu_reservations.sql
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
-- 原子 GPU 预留:评测和推理与训练统一纳入调度占用模型。
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS gpu_reservations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
node_id TEXT NOT NULL REFERENCES compute_nodes(id) ON DELETE CASCADE,
|
||||||
|
gpu_index INTEGER NOT NULL,
|
||||||
|
owner_type TEXT NOT NULL CHECK (owner_type IN ('eval', 'inference')),
|
||||||
|
owner_id TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'reserved' CHECK (status IN ('reserved', 'released')),
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
released_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_owner
|
||||||
|
ON gpu_reservations(owner_type, owner_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_gpu_reservations_node_status
|
||||||
|
ON gpu_reservations(node_id, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpu_reservations_active
|
||||||
|
ON gpu_reservations(node_id, gpu_index) WHERE status = 'reserved';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
33
backend/app/db/sql/007_platform_completion.sql
Normal file
33
backend/app/db/sql/007_platform_completion.sql
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
-- 平台可靠性闭环:导出权限、对象清理、缓存 manifest 元数据。
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_status TEXT NOT NULL DEFAULT 'pending';
|
||||||
|
ALTER TABLE model_export_jobs ADD COLUMN IF NOT EXISTS archive_error TEXT;
|
||||||
|
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
|
||||||
|
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS version_id TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS checksum_sha256 TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS byte_size BIGINT NOT NULL DEFAULT 0;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS last_accessed_at TEXT;
|
||||||
|
ALTER TABLE storage_cache_jobs ADD COLUMN IF NOT EXISTS protected_until TEXT;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS storage_cleanup_jobs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
storage_object_id TEXT NOT NULL REFERENCES storage_objects(id) ON DELETE CASCADE,
|
||||||
|
action TEXT NOT NULL DEFAULT 'delete',
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
error TEXT,
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
completed_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_storage_cleanup_status ON storage_cleanup_jobs(status, create_time);
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
53
backend/app/db/sql/008_permission_v2.sql
Normal file
53
backend/app/db/sql/008_permission_v2.sql
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
-- 权限 2.0:租户成员、资源申请、ACL 生命周期和审批动作
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tenant_members (
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
role TEXT NOT NULL DEFAULT 'member',
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
invited_by TEXT,
|
||||||
|
joined_at TEXT,
|
||||||
|
expires_at TEXT,
|
||||||
|
PRIMARY KEY (tenant_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS resource_access_requests (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
resource_type TEXT NOT NULL,
|
||||||
|
resource_id TEXT NOT NULL,
|
||||||
|
applicant_id TEXT NOT NULL,
|
||||||
|
principal_type TEXT NOT NULL DEFAULT 'user',
|
||||||
|
principal_id TEXT NOT NULL,
|
||||||
|
requested_permissions TEXT NOT NULL DEFAULT '[]',
|
||||||
|
reason TEXT,
|
||||||
|
approval_id TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
expires_at TEXT,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
decided_at TEXT,
|
||||||
|
decided_by TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_resource
|
||||||
|
ON resource_access_requests(resource_type, resource_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_applicant
|
||||||
|
ON resource_access_requests(applicant_id, status);
|
||||||
|
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_acls_tenant_active ON acls(tenant_id, revoked_at, expires_at);
|
||||||
|
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
|
||||||
|
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';
|
||||||
|
|
||||||
116
backend/app/db/sql/009_permission_completion.sql
Normal file
116
backend/app/db/sql/009_permission_completion.sql
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
-- 权限 2.0 完整闭环:审批策略、执行状态、结构化审计与历史租户归属
|
||||||
|
|
||||||
|
-- 该迁移可独立执行:兼容仅执行过 000_full_init.sql、或未执行 008 的旧数据库。
|
||||||
|
CREATE TABLE IF NOT EXISTS tenant_members (
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
role TEXT NOT NULL DEFAULT 'member',
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
invited_by TEXT,
|
||||||
|
joined_at TEXT,
|
||||||
|
expires_at TEXT,
|
||||||
|
PRIMARY KEY (tenant_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_members_tenant ON tenant_members(tenant_id, status);
|
||||||
|
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
|
||||||
|
SELECT COALESCE(u.tenant_id, 'default'), u.id,
|
||||||
|
CASE WHEN u.role='admin' OR COALESCE(u.protected, 0)=1 THEN 'owner' ELSE 'member' END,
|
||||||
|
'active', COALESCE(u.create_time, NOW()::text)
|
||||||
|
FROM users u
|
||||||
|
ON CONFLICT (tenant_id, user_id) DO NOTHING;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS resource_access_requests (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
resource_type TEXT NOT NULL,
|
||||||
|
resource_id TEXT NOT NULL,
|
||||||
|
applicant_id TEXT NOT NULL,
|
||||||
|
principal_type TEXT NOT NULL DEFAULT 'user',
|
||||||
|
principal_id TEXT NOT NULL,
|
||||||
|
requested_permissions TEXT NOT NULL DEFAULT '[]',
|
||||||
|
reason TEXT,
|
||||||
|
approval_id TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
expires_at TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
|
||||||
|
decided_at TEXT,
|
||||||
|
decided_by TEXT
|
||||||
|
);
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS action TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS requested_permissions TEXT NOT NULL DEFAULT '[]';
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS reason TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_by TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS decided_at TEXT;
|
||||||
|
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS approver_type TEXT NOT NULL DEFAULT 'user';
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS granted_by TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS source_request_id TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS expires_at TEXT;
|
||||||
|
ALTER TABLE acls ADD COLUMN IF NOT EXISTS revoked_at TEXT;
|
||||||
|
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE models ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS action TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS resource_type TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'tenant';
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active';
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE approval_templates ADD COLUMN IF NOT EXISTS updated_at TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_approval_templates_match
|
||||||
|
ON approval_templates(tenant_id, action, resource_type, status);
|
||||||
|
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_status TEXT NOT NULL DEFAULT 'pending';
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS execution_error TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_at TEXT;
|
||||||
|
ALTER TABLE approval_instances ADD COLUMN IF NOT EXISTS executed_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_approval_instances_execution
|
||||||
|
ON approval_instances(status, execution_status, action, resource_type, resource_id);
|
||||||
|
|
||||||
|
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_at TEXT;
|
||||||
|
ALTER TABLE resource_access_requests ADD COLUMN IF NOT EXISTS cancelled_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_resource_access_requests_expiry
|
||||||
|
ON resource_access_requests(status, expires_at);
|
||||||
|
|
||||||
|
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_projects_active_tenant ON projects(tenant_id, status, deleted_at);
|
||||||
|
|
||||||
|
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenants_active ON tenants(status, deleted_at);
|
||||||
|
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_tenant ON data_convert_tasks(tenant_id, deleted_at);
|
||||||
|
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS result TEXT NOT NULL DEFAULT 'success';
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS reason TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS request_id TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS session_id TEXT;
|
||||||
|
ALTER TABLE audit_logs ADD COLUMN IF NOT EXISTS metadata TEXT;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_audit_request ON audit_logs(request_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_audit_result ON audit_logs(result, time);
|
||||||
|
|
||||||
|
-- 历史资源按创建者租户补齐归属。无法识别的资源保留 default,后续由管理员复核。
|
||||||
|
UPDATE datasets d
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE d.created_by = u.id AND (d.tenant_id IS NULL OR d.tenant_id = '');
|
||||||
|
UPDATE models m
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
|
||||||
|
UPDATE trained_models m
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE m.created_by = u.id AND (m.tenant_id IS NULL OR m.tenant_id = '');
|
||||||
|
UPDATE eval_tasks e
|
||||||
|
SET tenant_id = COALESCE(u.tenant_id, 'default')
|
||||||
|
FROM users u
|
||||||
|
WHERE e.created_by = u.id AND (e.tenant_id IS NULL OR e.tenant_id = '');
|
||||||
21
backend/app/db/sql/010_permission_quota_membership.sql
Normal file
21
backend/app/db/sql/010_permission_quota_membership.sql
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
-- Permission 2.0: atomic tenant quota reservations for GPU-backed tasks.
|
||||||
|
-- This migration is additive and safe to run repeatedly.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tenant_quota_reservations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
owner_type TEXT NOT NULL,
|
||||||
|
owner_id TEXT NOT NULL,
|
||||||
|
gpu_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
storage_bytes BIGINT NOT NULL DEFAULT 0,
|
||||||
|
status TEXT NOT NULL DEFAULT 'reserved',
|
||||||
|
create_time TEXT NOT NULL,
|
||||||
|
released_at TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_tenant
|
||||||
|
ON tenant_quota_reservations(tenant_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tenant_quota_reservations_owner
|
||||||
|
ON tenant_quota_reservations(owner_type, owner_id, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_tenant_quota_reservations_active_owner
|
||||||
|
ON tenant_quota_reservations(owner_type, owner_id) WHERE status = 'reserved';
|
||||||
20
backend/app/db/sql/011_permission_lifecycle.sql
Normal file
20
backend/app/db/sql/011_permission_lifecycle.sql
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
-- Permission 2.0: user and inference task lifecycle tombstones.
|
||||||
|
-- Additive migration, safe to execute repeatedly.
|
||||||
|
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE compare_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS cleanup_attempts INTEGER NOT NULL DEFAULT 0;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_cleanup_error TEXT;
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS last_verified_at TEXT;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_users_active ON users(status, deleted_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_compare_tasks_active ON compare_tasks(status, deleted_at);
|
||||||
63
backend/app/db/sql/012_tenant_user_hierarchy.sql
Normal file
63
backend/app/db/sql/012_tenant_user_hierarchy.sql
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
-- Tenant/user hierarchy alignment. Additive and safe to run repeatedly.
|
||||||
|
-- New business flows use tenant_members, platform_role, tenant_id and
|
||||||
|
-- created_by. Legacy role/project fields remain for compatibility only.
|
||||||
|
|
||||||
|
ALTER TABLE users ADD COLUMN IF NOT EXISTS platform_role TEXT NOT NULL DEFAULT 'platform_user';
|
||||||
|
UPDATE users
|
||||||
|
SET platform_role = CASE
|
||||||
|
WHEN role = 'admin' OR COALESCE(protected, 0) = 1 THEN 'platform_admin'
|
||||||
|
ELSE 'platform_user'
|
||||||
|
END
|
||||||
|
WHERE platform_role IS NULL OR platform_role NOT IN ('platform_admin', 'platform_user');
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_users_platform_role ON users(platform_role, status, deleted_at);
|
||||||
|
|
||||||
|
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT 'default';
|
||||||
|
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||||
|
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_at TEXT;
|
||||||
|
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS deleted_by TEXT;
|
||||||
|
UPDATE fine_tune_tasks
|
||||||
|
SET tenant_id = COALESCE(NULLIF(tenant_id, ''), payload::json->>'tenant_id', 'default'),
|
||||||
|
created_by = COALESCE(NULLIF(created_by, ''), payload::json->>'created_by')
|
||||||
|
WHERE tenant_id IS NULL OR tenant_id = '' OR created_by IS NULL OR created_by = '';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_tenant_active
|
||||||
|
ON fine_tune_tasks(tenant_id, status, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_creator
|
||||||
|
ON fine_tune_tasks(created_by, deleted_at, create_time DESC);
|
||||||
|
|
||||||
|
INSERT INTO tenants (id, name, code, status, quota, create_time)
|
||||||
|
VALUES ('default', '默认租户', 'default', 'active', '{}', NOW()::text)
|
||||||
|
ON CONFLICT (id) DO UPDATE
|
||||||
|
SET status = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN 'active' ELSE tenants.status END,
|
||||||
|
deleted_at = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_at END,
|
||||||
|
deleted_by = CASE WHEN COALESCE(tenants.status, 'active') = 'deleted' THEN NULL ELSE tenants.deleted_by END;
|
||||||
|
|
||||||
|
-- Make the tenant owner relationship explicit and repair older tenant rows.
|
||||||
|
INSERT INTO tenant_members (tenant_id, user_id, role, status, joined_at)
|
||||||
|
SELECT t.id, t.owner_user_id, 'owner', 'active', COALESCE(t.create_time, NOW()::text)
|
||||||
|
FROM tenants t
|
||||||
|
JOIN users u ON u.id = t.owner_user_id
|
||||||
|
WHERE t.owner_user_id IS NOT NULL
|
||||||
|
AND COALESCE(t.status, 'active') = 'active'
|
||||||
|
ON CONFLICT (tenant_id, user_id) DO UPDATE
|
||||||
|
SET role = 'owner', status = 'active';
|
||||||
|
|
||||||
|
-- Existing account creation used admin as a tenant owner. Platform role and
|
||||||
|
-- tenant role are separate, so keep only explicit tenant owners as owners.
|
||||||
|
UPDATE tenant_members tm
|
||||||
|
SET role = 'member'
|
||||||
|
FROM users u
|
||||||
|
WHERE tm.user_id = u.id
|
||||||
|
AND tm.role = 'owner'
|
||||||
|
AND (u.role <> 'admin' AND COALESCE(u.protected, 0) = 0)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM tenants t
|
||||||
|
WHERE t.id = tm.tenant_id AND t.owner_user_id = tm.user_id
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Project is no longer a business isolation boundary. Keep historical columns
|
||||||
|
-- readable, but make tenant-scoped queries the only supported new path.
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_datasets_tenant_active ON datasets(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_models_tenant_active ON models(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_trained_models_tenant_active ON trained_models(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_tasks_tenant_active ON eval_tasks(tenant_id, deleted_at, create_time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_compare_tasks_tenant_active ON compare_tasks(tenant_id, deleted_at, create_time DESC);
|
||||||
@@ -4,10 +4,15 @@ from contextlib import suppress
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
from app.api.v1.router import api_router
|
from app.core.cache_paths import setup_local_caches
|
||||||
from app.core.config import docs_kwargs, get_settings
|
|
||||||
from app.core.logging import configure_logging, setup_request_logging
|
# 在任何 docling / tiktoken 模块被实例化之前设置缓存路径,避免首调用走到 ~/.cache。
|
||||||
from app.workers.compute_poller import run_compute_poller
|
setup_local_caches()
|
||||||
|
|
||||||
|
from app.api.v1.router import api_router # noqa: E402
|
||||||
|
from app.core.config import docs_kwargs, get_settings # noqa: E402
|
||||||
|
from app.core.logging import configure_logging, setup_request_logging # noqa: E402
|
||||||
|
from app.workers.compute_poller import run_compute_poller # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
|
|||||||
@@ -1,17 +1,105 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from fastapi import APIRouter, Body, Depends
|
from fastapi import APIRouter, Body, Depends
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
from app.core.auth import get_current_user, is_admin
|
from app.core.auth import (
|
||||||
|
get_current_user,
|
||||||
|
has_resource_access,
|
||||||
|
is_tenant_admin,
|
||||||
|
is_admin,
|
||||||
|
resource_in_user_tenant,
|
||||||
|
resource_record,
|
||||||
|
)
|
||||||
|
|
||||||
router = APIRouter(prefix="/approvals", tags=["approval"])
|
router = APIRouter(prefix="/approvals", tags=["approval"])
|
||||||
|
|
||||||
|
# Approval actions are deliberately finite. A caller must not be able to
|
||||||
|
# create an arbitrary approval that no executor or audit policy understands.
|
||||||
|
ALLOWED_APPROVAL_ACTIONS = {
|
||||||
|
"resource.access",
|
||||||
|
"gpu.assign",
|
||||||
|
"tenant.quota.update",
|
||||||
|
"tenant.member.add",
|
||||||
|
"tenant.member.remove",
|
||||||
|
"model.use",
|
||||||
|
"dataset.use",
|
||||||
|
"dataset.delete",
|
||||||
|
"trained_model.merge",
|
||||||
|
"trained_model.export",
|
||||||
|
"trained_model.delete",
|
||||||
|
"fine_tune.stop",
|
||||||
|
"fine_tune.delete",
|
||||||
|
"eval.delete",
|
||||||
|
"inference.delete",
|
||||||
|
"project.archive",
|
||||||
|
"project.delete",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _available_gpu_options() -> list[dict[str, Any]]:
|
||||||
|
"""Return only online nodes and currently unassigned, idle GPUs."""
|
||||||
|
store = get_platform_store()
|
||||||
|
nodes = store.compute_nodes()
|
||||||
|
gpus = store.gpus()
|
||||||
|
assigned = {(str(item.get("node_id")), int(item.get("gpu_index"))) for item in store.gpu_assignments()}
|
||||||
|
by_node: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for gpu in gpus:
|
||||||
|
node_id = str(gpu.get("node_id") or "")
|
||||||
|
index = int(gpu.get("id") or 0)
|
||||||
|
if gpu.get("status") != "idle" or (node_id, index) in assigned:
|
||||||
|
continue
|
||||||
|
by_node.setdefault(node_id, []).append({
|
||||||
|
"index": index,
|
||||||
|
"name": gpu.get("name") or "GPU",
|
||||||
|
"memory_total_gb": float(gpu.get("memory_total_gb") or 0),
|
||||||
|
})
|
||||||
|
result = []
|
||||||
|
for node in nodes:
|
||||||
|
node_id = str(node.get("id") or "")
|
||||||
|
if not node.get("enabled") or node.get("scheduler_status") != "online" or not by_node.get(node_id):
|
||||||
|
continue
|
||||||
|
result.append({
|
||||||
|
"id": node_id,
|
||||||
|
"code": node.get("code") or node_id,
|
||||||
|
"name": node.get("name") or node.get("code") or node_id,
|
||||||
|
"gpus": sorted(by_node[node_id], key=lambda item: item["index"]),
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_gpu_request(assignments: Any) -> list[dict[str, Any]]:
|
||||||
|
if not isinstance(assignments, list) or not assignments:
|
||||||
|
raise fail(400, "assignments 不能为空")
|
||||||
|
available = {
|
||||||
|
(node["id"], gpu["index"])
|
||||||
|
for node in _available_gpu_options()
|
||||||
|
for gpu in node["gpus"]
|
||||||
|
}
|
||||||
|
normalized = []
|
||||||
|
seen: set[tuple[str, int]] = set()
|
||||||
|
for item in assignments:
|
||||||
|
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
|
||||||
|
raise fail(400, "每项必须包含 node_id 和 gpu_index")
|
||||||
|
try:
|
||||||
|
key = (str(item["node_id"]), int(item["gpu_index"]))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raise fail(400, "gpu_index 必须是整数")
|
||||||
|
if key not in available:
|
||||||
|
raise fail(409, f"GPU {key[0]}:{key[1]} 当前不可申请")
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
normalized.append({"node_id": key[0], "gpu_index": key[1]})
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
@router.get("/templates")
|
@router.get("/templates")
|
||||||
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def list_templates(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
if not is_admin(current_user):
|
||||||
|
raise fail(403, "admin permission required")
|
||||||
return ok(get_platform_store().approval_templates())
|
return ok(get_platform_store().approval_templates())
|
||||||
|
|
||||||
|
|
||||||
@@ -20,11 +108,31 @@ def create_template(payload: dict[str, Any] = Body(...), current_user: dict = De
|
|||||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||||
if not payload.get("name"):
|
if not payload.get("name"):
|
||||||
raise fail(400, "name 必填")
|
raise fail(400, "name 必填")
|
||||||
return ok(get_platform_store().create_approval_template(payload))
|
steps = payload.get("steps") or []
|
||||||
|
if not isinstance(steps, list) or any(not isinstance(step, dict) for step in steps):
|
||||||
|
raise fail(400, "steps 格式无效")
|
||||||
|
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
|
||||||
|
raise fail(400, "不支持的审批动作")
|
||||||
|
payload = {
|
||||||
|
**payload,
|
||||||
|
"created_by": current_user.get("id"),
|
||||||
|
"tenant_id": payload.get("tenant_id") or current_user.get("tenant_id") or "default",
|
||||||
|
"scope": payload.get("scope") or "tenant",
|
||||||
|
"status": payload.get("status") or "active",
|
||||||
|
}
|
||||||
|
template = get_platform_store().create_approval_template(payload)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="approval.template.create", actor_id=current_user.get("id"),
|
||||||
|
target_type="approval_template", target_id=template["id"],
|
||||||
|
tenant_id=template.get("tenant_id"),
|
||||||
|
)
|
||||||
|
return ok(template)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/templates/{template_id}")
|
@router.get("/templates/{template_id}")
|
||||||
def get_template(template_id: str) -> dict[str, Any]:
|
def get_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
if not is_admin(current_user):
|
||||||
|
raise fail(403, "admin permission required")
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().approval_template(template_id))
|
return ok(get_platform_store().approval_template(template_id))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -34,8 +142,16 @@ def get_template(template_id: str) -> dict[str, Any]:
|
|||||||
@router.put("/templates/{template_id}")
|
@router.put("/templates/{template_id}")
|
||||||
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def update_template(template_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||||
|
if payload.get("action") and payload["action"] not in ALLOWED_APPROVAL_ACTIONS:
|
||||||
|
raise fail(400, "不支持的审批动作")
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().update_approval_template(template_id, payload))
|
template = get_platform_store().update_approval_template(template_id, payload)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="approval.template.update", actor_id=current_user.get("id"),
|
||||||
|
target_type="approval_template", target_id=template_id,
|
||||||
|
tenant_id=template.get("tenant_id"), detail=f"fields={','.join(payload.keys())}",
|
||||||
|
)
|
||||||
|
return ok(template)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "template not found")
|
raise fail(404, "template not found")
|
||||||
|
|
||||||
@@ -44,15 +160,77 @@ def update_template(template_id: str, payload: dict[str, Any] = Body(...), curre
|
|||||||
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def delete_template(template_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
if not is_admin(current_user): raise fail(403, "admin permission required")
|
if not is_admin(current_user): raise fail(403, "admin permission required")
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().delete_approval_template(template_id))
|
template = get_platform_store().delete_approval_template(template_id)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="approval.template.delete", actor_id=current_user.get("id"),
|
||||||
|
target_type="approval_template", target_id=template_id,
|
||||||
|
tenant_id=template.get("tenant_id"),
|
||||||
|
)
|
||||||
|
return ok(template)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "template not found")
|
raise fail(404, "template not found")
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_instances(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def list_instances(
|
||||||
items = get_platform_store().approval_instances(status=status)
|
status: str | None = None,
|
||||||
return ok(items if is_admin(current_user) else [item for item in items if item.get("applicant_id") == current_user.get("id")])
|
mine: bool = False,
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
user_id = current_user.get("id")
|
||||||
|
items = get_platform_store().approval_instances(
|
||||||
|
status=status,
|
||||||
|
applicant_id=user_id if mine and not is_admin(current_user) else None,
|
||||||
|
)
|
||||||
|
if is_admin(current_user):
|
||||||
|
return ok(items)
|
||||||
|
if mine:
|
||||||
|
return ok(items)
|
||||||
|
visible = []
|
||||||
|
for item in items:
|
||||||
|
if item.get("applicant_id") == user_id:
|
||||||
|
visible.append(item)
|
||||||
|
continue
|
||||||
|
if any(step.get("approver_id") == user_id and step.get("status") == "pending" for step in item.get("steps", [])):
|
||||||
|
visible.append(item)
|
||||||
|
continue
|
||||||
|
if is_tenant_admin(current_user, item.get("tenant_id")) and item.get("status") == "pending":
|
||||||
|
visible.append(item)
|
||||||
|
return ok(visible)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/gpu-options")
|
||||||
|
def gpu_request_options(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
"""Self-service GPU options; does not expose the admin compute page."""
|
||||||
|
return ok({"nodes": _available_gpu_options()})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/gpu-requests")
|
||||||
|
def create_gpu_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
assignments = _validate_gpu_request(payload.get("assignments"))
|
||||||
|
user_id = str(current_user.get("id") or "")
|
||||||
|
if is_admin(current_user):
|
||||||
|
try:
|
||||||
|
return ok({"approval_required": False, "assignments": get_platform_store().assign_gpus(
|
||||||
|
[{**item, "user_id": user_id} for item in assignments], assigned_by=user_id,
|
||||||
|
)})
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
|
normalized = [{**item, "user_id": user_id} for item in assignments]
|
||||||
|
instance = get_platform_store().create_approval_instance({
|
||||||
|
"resource_type": "gpu",
|
||||||
|
"resource_id": f"batch:{user_id}",
|
||||||
|
"applicant_id": user_id,
|
||||||
|
"action": "gpu.assign",
|
||||||
|
"tenant_id": current_user.get("tenant_id") or "default",
|
||||||
|
"reason": json.dumps({"assignments": normalized, "reason": payload.get("reason")}, ensure_ascii=False),
|
||||||
|
})
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
|
||||||
|
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
|
||||||
|
detail=f"count={len(normalized)}",
|
||||||
|
)
|
||||||
|
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
@@ -61,6 +239,24 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
|
|||||||
for field in ("resource_type", "resource_id"):
|
for field in ("resource_type", "resource_id"):
|
||||||
if not payload.get(field):
|
if not payload.get(field):
|
||||||
raise fail(400, f"{field} 必填")
|
raise fail(400, f"{field} 必填")
|
||||||
|
resource = resource_record(str(payload["resource_type"]), str(payload["resource_id"]))
|
||||||
|
if not resource and not is_admin(current_user):
|
||||||
|
raise fail(404, "resource not found")
|
||||||
|
action = str(payload.get("action") or "")
|
||||||
|
if not action or action not in ALLOWED_APPROVAL_ACTIONS:
|
||||||
|
raise fail(400, "不支持的审批动作")
|
||||||
|
if action != "resource.access" and not is_admin(current_user) and not has_resource_access(
|
||||||
|
str(payload["resource_type"]), str(payload["resource_id"]), current_user, "read"
|
||||||
|
):
|
||||||
|
raise fail(403, "no permission to request approval for this resource")
|
||||||
|
if resource and not is_admin(current_user) and not resource_in_user_tenant(str(payload["resource_type"]), resource, current_user):
|
||||||
|
raise fail(403, "resource belongs to another tenant")
|
||||||
|
requested = payload.get("requested_permissions") or []
|
||||||
|
allowed = {"read", "write", "execute", "download"}
|
||||||
|
if any(permission not in allowed for permission in requested):
|
||||||
|
raise fail(400, "invalid requested permission")
|
||||||
|
payload["tenant_id"] = current_user.get("tenant_id") or "default"
|
||||||
|
payload["requested_permissions"] = requested
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().create_approval_instance(payload))
|
return ok(get_platform_store().create_approval_instance(payload))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -71,7 +267,7 @@ def create_instance(payload: dict[str, Any] = Body(...), current_user: dict = De
|
|||||||
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def get_instance(instance_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
item = get_platform_store().approval_instance(instance_id)
|
item = get_platform_store().approval_instance(instance_id)
|
||||||
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id"):
|
if not is_admin(current_user) and item.get("applicant_id") != current_user.get("id") and not is_tenant_admin(current_user, item.get("tenant_id")):
|
||||||
raise fail(403, "no permission to access approval")
|
raise fail(403, "no permission to access approval")
|
||||||
return ok(item)
|
return ok(item)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -83,18 +279,118 @@ def decide(
|
|||||||
instance_id: str,
|
instance_id: str,
|
||||||
step_index: int,
|
step_index: int,
|
||||||
payload: dict[str, Any] = Body(...),
|
payload: dict[str, Any] = Body(...),
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if not payload.get("approver_id"):
|
|
||||||
raise fail(400, "approver_id 必填")
|
|
||||||
try:
|
try:
|
||||||
return ok(
|
instance = get_platform_store().approval_instance(instance_id)
|
||||||
get_platform_store().decide_approval_step(
|
if instance.get("applicant_id") == current_user.get("id"):
|
||||||
|
raise fail(403, "applicant cannot approve own request")
|
||||||
|
step = next((item for item in instance.get("steps", []) if int(item.get("step_index", -1)) == step_index), None)
|
||||||
|
if not step and is_admin(current_user) and not instance.get("steps"):
|
||||||
|
step = {"approver_id": None, "status": "pending"}
|
||||||
|
if not step:
|
||||||
|
raise fail(404, "approval step not found")
|
||||||
|
designated = step.get("approver_id")
|
||||||
|
if not is_admin(current_user) and designated != current_user.get("id") and not (
|
||||||
|
step.get("approver_type") == "admin" and is_tenant_admin(current_user, instance.get("tenant_id"))
|
||||||
|
):
|
||||||
|
raise fail(403, "current user is not the designated approver")
|
||||||
|
if instance.get("action") == "tenant.quota.update" and not is_admin(current_user):
|
||||||
|
raise fail(403, "only platform administrator can approve tenant quota changes")
|
||||||
|
submitted_approver = payload.get("approver_id")
|
||||||
|
if submitted_approver and submitted_approver != current_user.get("id"):
|
||||||
|
raise fail(403, "approver_id must match current session")
|
||||||
|
store = get_platform_store()
|
||||||
|
result = store.decide_approval_step(
|
||||||
instance_id,
|
instance_id,
|
||||||
step_index,
|
step_index,
|
||||||
approver_id=payload["approver_id"],
|
approver_id=str(current_user.get("id")),
|
||||||
approved=bool(payload.get("approved", False)),
|
approved=bool(payload.get("approved", False)),
|
||||||
comment=payload.get("comment"),
|
comment=payload.get("comment"),
|
||||||
)
|
)
|
||||||
|
if result.get("status") == "approved" and result.get("execution_status") == "ready":
|
||||||
|
try:
|
||||||
|
effect = store.apply_approval_effect(result, str(current_user.get("id") or ""))
|
||||||
|
if effect is not None:
|
||||||
|
result = store.approval_instance(instance_id)
|
||||||
|
except Exception as exc:
|
||||||
|
with store.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE approval_instances SET execution_status='failed', execution_error=? WHERE id=?",
|
||||||
|
(str(exc), instance_id),
|
||||||
)
|
)
|
||||||
|
raise fail(409, f"审批已通过,但执行失败:{exc}")
|
||||||
|
store.record_audit(
|
||||||
|
action="approval.decision", actor_id=current_user.get("id"),
|
||||||
|
target_type="approval_instance", target_id=instance_id,
|
||||||
|
tenant_id=result.get("tenant_id"), result="success" if payload.get("approved") else "rejected",
|
||||||
|
detail=f"step={step_index}", reason=payload.get("comment"),
|
||||||
|
)
|
||||||
|
return ok(result)
|
||||||
except (KeyError, ValueError) as e:
|
except (KeyError, ValueError) as e:
|
||||||
raise fail(400, str(e))
|
raise fail(400, str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/resource-access/requests")
|
||||||
|
def create_resource_access_request(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
resource_type = str(payload.get("resource_type") or "")
|
||||||
|
resource_id = str(payload.get("resource_id") or "")
|
||||||
|
resource = resource_record(resource_type, resource_id)
|
||||||
|
if not resource:
|
||||||
|
raise fail(404, "resource not found")
|
||||||
|
if not is_admin(current_user) and not resource_in_user_tenant(resource_type, resource, current_user):
|
||||||
|
raise fail(403, "resource belongs to another tenant")
|
||||||
|
permissions = payload.get("requested_permissions") or ["read"]
|
||||||
|
allowed = {"read", "write", "execute", "download"}
|
||||||
|
if not permissions or any(permission not in allowed for permission in permissions):
|
||||||
|
raise fail(400, "invalid requested permissions")
|
||||||
|
result = get_platform_store().create_resource_access_request({
|
||||||
|
"resource_type": resource_type,
|
||||||
|
"resource_id": resource_id,
|
||||||
|
"applicant_id": current_user.get("id"),
|
||||||
|
"principal_type": "user",
|
||||||
|
"principal_id": current_user.get("id"),
|
||||||
|
"requested_permissions": permissions,
|
||||||
|
"reason": payload.get("reason"),
|
||||||
|
"template_id": payload.get("template_id"),
|
||||||
|
"tenant_id": current_user.get("tenant_id") or "default",
|
||||||
|
"expires_at": payload.get("expires_at"),
|
||||||
|
})
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="resource.access.request", actor_id=current_user.get("id"),
|
||||||
|
target_type=resource_type, target_id=resource_id,
|
||||||
|
tenant_id=current_user.get("tenant_id") or "default",
|
||||||
|
detail=f"permissions={','.join(permissions)}",
|
||||||
|
)
|
||||||
|
return ok(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/resource-access/requests")
|
||||||
|
def list_resource_access_requests(status: str | None = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
items = get_platform_store().resource_access_requests(
|
||||||
|
user_id=None if is_admin(current_user) else current_user.get("id"),
|
||||||
|
status=status,
|
||||||
|
)
|
||||||
|
return ok(items)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/resource-access/requests/{request_id}/cancel")
|
||||||
|
def cancel_resource_access_request(request_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
item = get_platform_store().cancel_resource_access_request(
|
||||||
|
request_id,
|
||||||
|
str(current_user.get("id") or ""),
|
||||||
|
is_admin_actor=is_admin(current_user),
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "access request not found")
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise fail(403, str(exc))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="resource.access.cancel", actor_id=current_user.get("id"),
|
||||||
|
target_type="resource_access_request", target_id=request_id,
|
||||||
|
tenant_id=item.get("tenant_id"),
|
||||||
|
)
|
||||||
|
return ok(item)
|
||||||
|
|||||||
@@ -158,7 +158,12 @@ class ComputeNodeClient:
|
|||||||
return _unwrap_dict(response.json())
|
return _unwrap_dict(response.json())
|
||||||
|
|
||||||
async def stop_job(self, job_id: str) -> dict[str, Any]:
|
async def stop_job(self, job_id: str) -> dict[str, Any]:
|
||||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
# Compute API waits for the child process to exit (up to 10 seconds)
|
||||||
|
# before returning. The normal polling timeout is intentionally short,
|
||||||
|
# but is too aggressive for a stop request and used to surface as a
|
||||||
|
# platform 500 even when the node eventually stopped the job.
|
||||||
|
stop_timeout = max(float(self.timeout), 30.0)
|
||||||
|
async with httpx.AsyncClient(timeout=stop_timeout, headers=self.headers()) as client:
|
||||||
response = await client.post(_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/{job_id}/stop"))
|
response = await client.post(_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/{job_id}/stop"))
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return _unwrap_dict(response.json())
|
return _unwrap_dict(response.json())
|
||||||
|
|||||||
@@ -14,6 +14,18 @@ from app.modules.storage.minio_store import get_object_storage
|
|||||||
MAX_STARTING_ATTEMPTS = 40
|
MAX_STARTING_ATTEMPTS = 40
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_job_failure_reason(log_text: str, limit: int = 2000) -> str:
|
||||||
|
"""Return a concise actionable reason from a failed Compute job log."""
|
||||||
|
lines = [line.strip() for line in str(log_text or "").splitlines() if line.strip()]
|
||||||
|
if not lines:
|
||||||
|
return ""
|
||||||
|
markers = ("[eval] FAILED", "Traceback", "RuntimeError", "Error:", "ERROR")
|
||||||
|
for index in range(len(lines) - 1, -1, -1):
|
||||||
|
if any(marker in lines[index] for marker in markers):
|
||||||
|
return "\n".join(lines[index : index + 8])[-limit:]
|
||||||
|
return "\n".join(lines[-8:])[-limit:]
|
||||||
|
|
||||||
|
|
||||||
async def _archive_node_directory(
|
async def _archive_node_directory(
|
||||||
store: Any,
|
store: Any,
|
||||||
client: ComputeNodeClient,
|
client: ComputeNodeClient,
|
||||||
@@ -27,12 +39,15 @@ async def _archive_node_directory(
|
|||||||
"""Archive a completed node directory to MinIO, preserving subdirectories."""
|
"""Archive a completed node directory to MinIO, preserving subdirectories."""
|
||||||
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
|
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
|
||||||
source = Path(source_path).resolve()
|
source = Path(source_path).resolve()
|
||||||
|
if source == data_root:
|
||||||
|
raise RuntimeError("refuse to archive compute data root; output_dir must be a task subdirectory")
|
||||||
try:
|
try:
|
||||||
relative_root = source.relative_to(data_root).as_posix()
|
relative_root = source.relative_to(data_root).as_posix()
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
|
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
|
||||||
queue = [relative_root]
|
queue = [relative_root]
|
||||||
archived: list[dict[str, Any]] = []
|
archived: list[dict[str, Any]] = []
|
||||||
|
max_files = 10000
|
||||||
while queue:
|
while queue:
|
||||||
relative = queue.pop(0)
|
relative = queue.pop(0)
|
||||||
listing = await client.list_files(root="data", relative_path=relative)
|
listing = await client.list_files(root="data", relative_path=relative)
|
||||||
@@ -49,6 +64,8 @@ async def _archive_node_directory(
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
relative_file = Path(str(item.get("name") or Path(path).name)).name
|
relative_file = Path(str(item.get("name") or Path(path).name)).name
|
||||||
object_key = f"{object_prefix}/{version_id}/{relative_file}"
|
object_key = f"{object_prefix}/{version_id}/{relative_file}"
|
||||||
|
if len(archived) >= max_files:
|
||||||
|
raise RuntimeError(f"archive file count exceeds limit {max_files}")
|
||||||
upload_url = get_object_storage().presigned_put(object_key)
|
upload_url = get_object_storage().presigned_put(object_key)
|
||||||
result = await client.upload_file_to_url(path, upload_url, object_key)
|
result = await client.upload_file_to_url(path, upload_url, object_key)
|
||||||
metadata = get_object_storage().stat(object_key)
|
metadata = get_object_storage().stat(object_key)
|
||||||
@@ -115,11 +132,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
|||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = "compute node deleted"
|
item["error"] = "compute node deleted"
|
||||||
store.mark_inference_unloaded(item.get("node_id") or "")
|
store.mark_inference_unloaded(item.get("node_id") or "")
|
||||||
|
store.release_external_gpus("inference", str(task["id"]), item.get("node_id"))
|
||||||
continue
|
continue
|
||||||
if not node.get("enabled") or node.get("scheduler_status") != "online":
|
if not node.get("enabled") or node.get("scheduler_status") != "online":
|
||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = "compute node offline"
|
item["error"] = "compute node offline"
|
||||||
store.mark_inference_unloaded(node["id"])
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
store.release_external_gpus("inference", str(task["id"]), node["id"])
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
|
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
|
||||||
@@ -128,6 +147,7 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
|||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = f"compute node unreachable: {exc}"
|
item["error"] = f"compute node unreachable: {exc}"
|
||||||
store.mark_inference_unloaded(node["id"])
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
store.release_external_gpus("inference", str(task["id"]), node["id"])
|
||||||
continue
|
continue
|
||||||
node_status = status.get("status")
|
node_status = status.get("status")
|
||||||
if node_status == "ready":
|
if node_status == "ready":
|
||||||
@@ -139,11 +159,13 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
|||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = status.get("error") or "model load failed on compute node"
|
item["error"] = status.get("error") or "model load failed on compute node"
|
||||||
store.mark_inference_unloaded(node["id"])
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
store.release_external_gpus("inference", str(task["id"]), node["id"])
|
||||||
elif node_status == "idle":
|
elif node_status == "idle":
|
||||||
# 节点重启导致已加载模型丢失
|
# 节点重启导致已加载模型丢失
|
||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = "model disappeared from compute node (node may have restarted)"
|
item["error"] = "model disappeared from compute node (node may have restarted)"
|
||||||
store.mark_inference_unloaded(node["id"])
|
store.mark_inference_unloaded(node["id"])
|
||||||
|
store.release_external_gpus("inference", str(task["id"]), node["id"])
|
||||||
# node_status == "loading" -> 保持 starting,下轮再查
|
# node_status == "loading" -> 保持 starting,下轮再查
|
||||||
if dirty:
|
if dirty:
|
||||||
if any(i.get("status") in {"ready", "running"} for i in items):
|
if any(i.get("status") in {"ready", "running"} for i in items):
|
||||||
@@ -157,11 +179,16 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
|||||||
return reconciled
|
return reconciled
|
||||||
|
|
||||||
|
|
||||||
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
async def fetch_eval_result_content(
|
||||||
|
client: ComputeNodeClient,
|
||||||
|
node: dict[str, Any],
|
||||||
|
job: dict[str, Any],
|
||||||
|
file_name: str = "eval_results.json",
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
output_dir = job.get("output_dir")
|
output_dir = job.get("output_dir")
|
||||||
if not output_dir:
|
if not output_dir:
|
||||||
return None
|
return None
|
||||||
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
|
full_path = f"{str(output_dir).rstrip('/')}/{file_name}"
|
||||||
data_root = "/data/yg-ft/"
|
data_root = "/data/yg-ft/"
|
||||||
if full_path.startswith(data_root):
|
if full_path.startswith(data_root):
|
||||||
full_path = full_path[len(data_root):]
|
full_path = full_path[len(data_root):]
|
||||||
@@ -175,11 +202,36 @@ async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, A
|
|||||||
return payload if isinstance(payload, dict) else None
|
return payload if isinstance(payload, dict) else None
|
||||||
|
|
||||||
|
|
||||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
async def fetch_eval_progress_content(
|
||||||
store = get_platform_store()
|
client: ComputeNodeClient,
|
||||||
|
node: dict[str, Any],
|
||||||
|
job: dict[str, Any],
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
return await fetch_eval_result_content(client, node, job, "eval_progress.json")
|
||||||
|
|
||||||
|
|
||||||
|
async def poll_compute_jobs_once(store: Any | None = None) -> dict[str, Any]:
|
||||||
|
store = store or get_platform_store()
|
||||||
synced: list[dict[str, Any]] = []
|
synced: list[dict[str, Any]] = []
|
||||||
failed: list[dict[str, str]] = []
|
failed: list[dict[str, str]] = []
|
||||||
for task in store.running_compute_tasks():
|
online_nodes = {
|
||||||
|
str(node.get("id"))
|
||||||
|
for node in store.compute_nodes()
|
||||||
|
if node.get("enabled") and node.get("scheduler_status") in {"online", "draining"}
|
||||||
|
}
|
||||||
|
training_tasks = {str(task["id"]): task for task in store.running_compute_tasks()}
|
||||||
|
# Completed tasks whose MinIO archive was interrupted remain eligible for
|
||||||
|
# reconciliation after a Backend restart or a transient node failure.
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
for task in store.tasks():
|
||||||
|
if task.get("status") != "completed" or not task.get("compute_job_id"):
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
str(task.get("archive_status") or "") != "completed"
|
||||||
|
and str(task.get("compute_node_id")) in online_nodes
|
||||||
|
):
|
||||||
|
training_tasks.setdefault(str(task["id"]), task)
|
||||||
|
for task in training_tasks.values():
|
||||||
node = _node_for_task(task)
|
node = _node_for_task(task)
|
||||||
if not node:
|
if not node:
|
||||||
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
||||||
@@ -215,6 +267,7 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
if trained_model:
|
if trained_model:
|
||||||
|
try:
|
||||||
archived = await _archive_node_directory(
|
archived = await _archive_node_directory(
|
||||||
store,
|
store,
|
||||||
client,
|
client,
|
||||||
@@ -230,11 +283,31 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
store.link_model_artifact_storage_object(
|
store.link_model_artifact_storage_object(
|
||||||
str(artifacts[0]["id"]), str(archived[0]["id"])
|
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||||
)
|
)
|
||||||
|
store.update_task(task["id"], {
|
||||||
|
"archive_status": "completed",
|
||||||
|
"archive_object_ids": [str(item["id"]) for item in archived],
|
||||||
|
"archive_error": "",
|
||||||
|
})
|
||||||
|
except Exception as archive_exc:
|
||||||
|
store.update_task(task["id"], {
|
||||||
|
"archive_status": "pending",
|
||||||
|
"archive_error": str(archive_exc)[:2000],
|
||||||
|
})
|
||||||
|
raise
|
||||||
synced.append(updated_task)
|
synced.append(updated_task)
|
||||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||||
standalone_synced: list[dict[str, Any]] = []
|
standalone_synced: list[dict[str, Any]] = []
|
||||||
for record in store.active_standalone_compute_jobs():
|
standalone_jobs = {
|
||||||
|
str(record["id"]): record
|
||||||
|
for record in store.active_standalone_compute_jobs()
|
||||||
|
if str(record.get("node_id")) in online_nodes
|
||||||
|
}
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
for record in store.standalone_compute_jobs_pending_archive():
|
||||||
|
if str(record.get("node_id")) in online_nodes:
|
||||||
|
standalone_jobs.setdefault(str(record["id"]), record)
|
||||||
|
for record in standalone_jobs.values():
|
||||||
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
node = next((item for item in store.compute_nodes() if item["id"] == record.get("node_id")), None)
|
||||||
if not node:
|
if not node:
|
||||||
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
failed.append({"job_id": record["id"], "error": "compute node not found"})
|
||||||
@@ -266,12 +339,32 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
store.link_model_artifact_storage_object(
|
store.link_model_artifact_storage_object(
|
||||||
str(artifacts[0]["id"]), str(archived[0]["id"])
|
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||||
)
|
)
|
||||||
|
store.update_compute_job_archive(record["id"], "completed", [str(item["id"]) for item in archived])
|
||||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
|
try:
|
||||||
|
store.update_compute_job_archive(record["id"], "pending", [], str(exc)[:2000])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||||
|
|
||||||
# ── Eval job sync ────────────────────────────────────────────────
|
# ── Eval job sync ────────────────────────────────────────────────
|
||||||
eval_synced = 0
|
eval_synced = 0
|
||||||
for eval_task in store.running_eval_tasks():
|
eval_tasks = {str(task["id"]): task for task in store.running_eval_tasks()}
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
# A completed evaluation can win the race with the poller: its status
|
||||||
|
# is persisted before the report archive finishes. Keep such tasks in
|
||||||
|
# the reconciliation set until the report object is available.
|
||||||
|
for task in store.eval_tasks():
|
||||||
|
if (
|
||||||
|
task.get("status") == "completed"
|
||||||
|
and task.get("compute_job_id")
|
||||||
|
and str(task.get("archive_status") or "") != "completed"
|
||||||
|
and str(task.get("compute_node_id")) in online_nodes
|
||||||
|
):
|
||||||
|
eval_tasks.setdefault(str(task["id"]), task)
|
||||||
|
for eval_task in eval_tasks.values():
|
||||||
|
if str(eval_task.get("compute_node_id")) not in online_nodes:
|
||||||
|
continue
|
||||||
node = next(
|
node = next(
|
||||||
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
|
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
|
||||||
None,
|
None,
|
||||||
@@ -283,15 +376,39 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
job = await client.get_job(eval_task["compute_job_id"])
|
job = await client.get_job(eval_task["compute_job_id"])
|
||||||
result_content = None
|
result_content = None
|
||||||
# Try to read eval_results.json from the job output directory
|
# Read live progress and partial results while the evaluator is running.
|
||||||
|
if job.get("status") in {"queued", "running"} and job.get("output_dir"):
|
||||||
|
try:
|
||||||
|
progress_content = await fetch_eval_progress_content(client, node, job)
|
||||||
|
if progress_content:
|
||||||
|
store.update_eval_task(
|
||||||
|
eval_task["id"],
|
||||||
|
{
|
||||||
|
"progress_detail": progress_content,
|
||||||
|
"progress": progress_content.get("percentage", eval_task.get("progress", 0)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
result_content = await fetch_eval_result_content(client, node, job)
|
||||||
|
except Exception:
|
||||||
|
result_content = None
|
||||||
|
# Try to read eval_results.json from the job output directory on completion.
|
||||||
if job.get("status") == "completed" and job.get("output_dir"):
|
if job.get("status") == "completed" and job.get("output_dir"):
|
||||||
try:
|
try:
|
||||||
result_content = await fetch_eval_result_content(client, node, job)
|
result_content = await fetch_eval_result_content(client, node, job)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
if job.get("status") in {"failed", "stopped"} and not job.get("error"):
|
||||||
|
try:
|
||||||
|
failure_logs = await client.job_logs(eval_task["compute_job_id"], tail_lines=120)
|
||||||
|
job["error"] = _extract_job_failure_reason(str(failure_logs.get("content") or ""))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||||
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
||||||
await _archive_node_directory(
|
archived = await _archive_node_directory(
|
||||||
store,
|
store,
|
||||||
client,
|
client,
|
||||||
node,
|
node,
|
||||||
@@ -301,9 +418,28 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
|
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
|
||||||
f"evaluations/{eval_task['id']}",
|
f"evaluations/{eval_task['id']}",
|
||||||
)
|
)
|
||||||
|
report_object = next(
|
||||||
|
(item for item in archived if Path(str(item.get("file_name") or "")).name == "eval_results.json"),
|
||||||
|
archived[0] if archived else None,
|
||||||
|
)
|
||||||
|
store.update_eval_task(eval_task["id"], {
|
||||||
|
"report_storage_object_id": str(report_object["id"]) if report_object else "",
|
||||||
|
"archive_status": "completed",
|
||||||
|
"archive_object_ids": [str(item["id"]) for item in archived],
|
||||||
|
"archive_error": "",
|
||||||
|
})
|
||||||
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
||||||
eval_synced += 1
|
eval_synced += 1
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc: # noqa: BLE001
|
||||||
|
try:
|
||||||
|
current_eval = store.eval_task(eval_task["id"])
|
||||||
|
except Exception:
|
||||||
|
current_eval = eval_task
|
||||||
|
if current_eval.get("status") == "completed":
|
||||||
|
try:
|
||||||
|
store.update_eval_task(eval_task["id"], {"archive_status": "pending", "archive_error": str(exc)[:2000]})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
||||||
|
|
||||||
# ── Inference load reconciliation ─────────────────────────────────────
|
# ── Inference load reconciliation ─────────────────────────────────────
|
||||||
|
|||||||
@@ -6,11 +6,11 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Depends, File, UploadFile
|
from fastapi import APIRouter, Body, Depends, File, Request, UploadFile
|
||||||
from fastapi.responses import FileResponse, Response
|
from fastapi.responses import FileResponse, Response
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.core.auth import get_current_user, is_admin
|
from app.core.auth import get_current_user, has_resource_access, is_admin
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.core.op_log import op_log, OpModule, OpAction
|
from app.core.op_log import op_log, OpModule, OpAction
|
||||||
from app.db.platform_store import get_platform_store, new_id
|
from app.db.platform_store import get_platform_store, new_id
|
||||||
@@ -18,7 +18,26 @@ from app.modules.storage.minio_store import get_object_storage
|
|||||||
from app.modules.storage.policy import should_store_in_minio
|
from app.modules.storage.policy import should_store_in_minio
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
|
def _authorize_data_convert_request(
|
||||||
|
request: Request,
|
||||||
|
task_id: str | None = None,
|
||||||
|
current_user: dict[str, Any] = Depends(get_current_user),
|
||||||
|
) -> None:
|
||||||
|
"""Protect every task-scoped conversion endpoint with resource ACL."""
|
||||||
|
if not task_id or is_admin(current_user):
|
||||||
|
return
|
||||||
|
permission = "read" if request.method in {"GET", "HEAD"} else "write"
|
||||||
|
if request.url.path.endswith("/run") or request.url.path.endswith("/import-as-dataset"):
|
||||||
|
permission = "execute"
|
||||||
|
if not has_resource_access("data_convert", task_id, current_user, permission):
|
||||||
|
raise fail(403, "no permission to access this data convert task")
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(
|
||||||
|
prefix="/data-convert",
|
||||||
|
tags=["data-convert"],
|
||||||
|
dependencies=[Depends(_authorize_data_convert_request)],
|
||||||
|
)
|
||||||
|
|
||||||
# 存储根目录
|
# 存储根目录
|
||||||
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
|
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
|
||||||
@@ -207,24 +226,32 @@ def list_tasks(
|
|||||||
if is_admin(current_user):
|
if is_admin(current_user):
|
||||||
# 管理员可见全部
|
# 管理员可见全部
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL "
|
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
|
||||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
"FROM data_convert_tasks task "
|
||||||
|
"LEFT JOIN users creator ON creator.id=task.created_by "
|
||||||
|
"LEFT JOIN users processor ON processor.id=task.processed_by "
|
||||||
|
"WHERE task.deleted_at IS NULL "
|
||||||
|
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
|
||||||
(page_size, (page - 1) * page_size),
|
(page_size, (page - 1) * page_size),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
total = conn.execute(
|
total = conn.execute(
|
||||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
|
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
else:
|
else:
|
||||||
# 普通用户只能看到自己创建的
|
# 普通用户只能看到本租户且由自己创建的任务;跨租户 ACL 通过任务级依赖访问。
|
||||||
user_id = current_user.get("id")
|
user_id = current_user.get("id")
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT * FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s "
|
"SELECT task.*, creator.display_name AS creator_name, processor.display_name AS processor_name "
|
||||||
"ORDER BY create_time DESC LIMIT %s OFFSET %s",
|
"FROM data_convert_tasks task "
|
||||||
(user_id, page_size, (page - 1) * page_size),
|
"LEFT JOIN users creator ON creator.id=task.created_by "
|
||||||
|
"LEFT JOIN users processor ON processor.id=task.processed_by "
|
||||||
|
"WHERE task.deleted_at IS NULL AND task.tenant_id=%s AND task.created_by=%s "
|
||||||
|
"ORDER BY task.create_time DESC LIMIT %s OFFSET %s",
|
||||||
|
(current_user.get("tenant_id") or "default", user_id, page_size, (page - 1) * page_size),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
total = conn.execute(
|
total = conn.execute(
|
||||||
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND created_by=%s",
|
"SELECT COUNT(*) FROM data_convert_tasks WHERE deleted_at IS NULL AND tenant_id=%s AND created_by=%s",
|
||||||
(user_id,)
|
(current_user.get("tenant_id") or "default", user_id,)
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
return ok({"items": [dict(r) for r in rows], "total": total})
|
return ok({"items": [dict(r) for r in rows], "total": total})
|
||||||
|
|
||||||
@@ -243,11 +270,16 @@ def create_task(
|
|||||||
description = str(payload.get("description") or "").strip()
|
description = str(payload.get("description") or "").strip()
|
||||||
user_id = current_user.get("id")
|
user_id = current_user.get("id")
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
tenant_id = current_user.get("tenant_id") or "default"
|
||||||
|
try:
|
||||||
|
store.assert_active_tenant(tenant_id)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by) "
|
"INSERT INTO data_convert_tasks (id, name, description, output_filename, created_by, tenant_id) "
|
||||||
"VALUES (%s, %s, %s, %s, %s)",
|
"VALUES (%s, %s, %s, %s, %s, %s)",
|
||||||
(task_id, name, description, output_filename, user_id),
|
(task_id, name, description, output_filename, user_id, tenant_id),
|
||||||
)
|
)
|
||||||
# MinIO 是正式存储;本地目录只在关闭 MinIO 的旧兼容模式下创建。
|
# MinIO 是正式存储;本地目录只在关闭 MinIO 的旧兼容模式下创建。
|
||||||
if not _minio_enabled():
|
if not _minio_enabled():
|
||||||
@@ -318,8 +350,8 @@ async def upload_source_files(
|
|||||||
# 标记上传完成
|
# 标记上传完成
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='uploaded', update_time=NOW() WHERE id=%s",
|
"UPDATE data_convert_tasks SET status='uploaded', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(task_id,),
|
(current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
# 自动转换并导入数据集
|
# 自动转换并导入数据集
|
||||||
try:
|
try:
|
||||||
@@ -332,8 +364,8 @@ async def upload_source_files(
|
|||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='completed', "
|
"UPDATE data_convert_tasks SET status='completed', "
|
||||||
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
|
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(input_count, output_count, task_id),
|
(input_count, output_count, current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
# 自动导入数据集
|
# 自动导入数据集
|
||||||
content = output.decode("utf-8")
|
content = output.decode("utf-8")
|
||||||
@@ -348,6 +380,7 @@ async def upload_source_files(
|
|||||||
"count": output_count,
|
"count": output_count,
|
||||||
"description": f"由数据类型转换任务 {task_id} 自动导入",
|
"description": f"由数据类型转换任务 {task_id} 自动导入",
|
||||||
"created_by": task.get("created_by") or current_user.get("id"),
|
"created_by": task.get("created_by") or current_user.get("id"),
|
||||||
|
"tenant_id": task.get("tenant_id") or current_user.get("tenant_id") or "default",
|
||||||
})
|
})
|
||||||
dataset_id = dataset["id"]
|
dataset_id = dataset["id"]
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
@@ -375,8 +408,8 @@ async def upload_source_files(
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
|
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(str(exc)[:500], task_id),
|
(str(exc)[:500], current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]})
|
return ok({"staged_files": staged, "auto_converted": False, "error": str(exc)[:500]})
|
||||||
|
|
||||||
@@ -396,8 +429,8 @@ def run_convert(
|
|||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='running', error_message='', update_time=NOW() WHERE id=%s",
|
"UPDATE data_convert_tasks SET status='running', error_message='', processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(task_id,),
|
(current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
if _minio_enabled():
|
if _minio_enabled():
|
||||||
@@ -408,14 +441,14 @@ def run_convert(
|
|||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='completed', "
|
"UPDATE data_convert_tasks SET status='completed', "
|
||||||
"input_count=%s, output_count=%s, update_time=NOW() WHERE id=%s",
|
"input_count=%s, output_count=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(input_count, output_count, task_id),
|
(input_count, output_count, current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='failed', error_message=%s, update_time=NOW() WHERE id=%s",
|
"UPDATE data_convert_tasks SET status='failed', error_message=%s, processed_by=%s, processed_at=NOW(), update_time=NOW() WHERE id=%s",
|
||||||
(str(exc)[:500], task_id),
|
(str(exc)[:500], current_user.get("id"), task_id),
|
||||||
)
|
)
|
||||||
raise fail(500, f"convert failed: {exc}")
|
raise fail(500, f"convert failed: {exc}")
|
||||||
return ok(_get_task(task_id))
|
return ok(_get_task(task_id))
|
||||||
|
|||||||
@@ -49,13 +49,16 @@ from .text_utils import (
|
|||||||
|
|
||||||
# Layer 1: 解析器(依赖 types 与 text_utils)
|
# Layer 1: 解析器(依赖 types 与 text_utils)
|
||||||
from .parsers import (
|
from .parsers import (
|
||||||
|
LayoutRepeatedBlock,
|
||||||
_infer_xlsx_header_region,
|
_infer_xlsx_header_region,
|
||||||
_rewrite_xlsx_workbook_relationships,
|
_rewrite_xlsx_workbook_relationships,
|
||||||
_validate_office_archive,
|
_validate_office_archive,
|
||||||
_xlsx_sheet_merge_ranges,
|
_xlsx_sheet_merge_ranges,
|
||||||
|
detect_layout_repeated_blocks,
|
||||||
detect_pdf_document_noise,
|
detect_pdf_document_noise,
|
||||||
extract_pdf_page_texts,
|
extract_pdf_page_texts,
|
||||||
remove_document_noise,
|
remove_document_noise,
|
||||||
|
remove_layout_repeated_blocks,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Layer 3: 数据转换与质量评分
|
# Layer 3: 数据转换与质量评分
|
||||||
@@ -115,6 +118,7 @@ __all__ = [
|
|||||||
"desensitize_pii",
|
"desensitize_pii",
|
||||||
"desensitize_structured_record",
|
"desensitize_structured_record",
|
||||||
"detect_document_structure",
|
"detect_document_structure",
|
||||||
|
"detect_layout_repeated_blocks",
|
||||||
"detect_pdf_document_noise",
|
"detect_pdf_document_noise",
|
||||||
"detect_text_format",
|
"detect_text_format",
|
||||||
"estimate_token_count",
|
"estimate_token_count",
|
||||||
@@ -137,7 +141,9 @@ __all__ = [
|
|||||||
"preprocess_structured_records_with_lineage",
|
"preprocess_structured_records_with_lineage",
|
||||||
"protected_context_ranges",
|
"protected_context_ranges",
|
||||||
"record_fingerprint",
|
"record_fingerprint",
|
||||||
|
"LayoutRepeatedBlock",
|
||||||
"remove_document_noise",
|
"remove_document_noise",
|
||||||
|
"remove_layout_repeated_blocks",
|
||||||
"score_quality",
|
"score_quality",
|
||||||
"stable_split",
|
"stable_split",
|
||||||
"stable_split_assignments",
|
"stable_split_assignments",
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
"""文档解析器模块。"""
|
"""文档解析器模块。"""
|
||||||
|
|
||||||
|
from .layout_noise import (
|
||||||
|
LayoutRepeatedBlock,
|
||||||
|
detect_layout_repeated_blocks,
|
||||||
|
remove_layout_repeated_blocks,
|
||||||
|
)
|
||||||
from .pdf import extract_pdf_page_texts, detect_pdf_document_noise, remove_document_noise
|
from .pdf import extract_pdf_page_texts, detect_pdf_document_noise, remove_document_noise
|
||||||
from .office import (
|
from .office import (
|
||||||
_validate_office_archive,
|
_validate_office_archive,
|
||||||
@@ -12,6 +17,9 @@ __all__ = [
|
|||||||
'extract_pdf_page_texts',
|
'extract_pdf_page_texts',
|
||||||
'detect_pdf_document_noise',
|
'detect_pdf_document_noise',
|
||||||
'remove_document_noise',
|
'remove_document_noise',
|
||||||
|
'LayoutRepeatedBlock',
|
||||||
|
'detect_layout_repeated_blocks',
|
||||||
|
'remove_layout_repeated_blocks',
|
||||||
'_validate_office_archive',
|
'_validate_office_archive',
|
||||||
'_rewrite_xlsx_workbook_relationships',
|
'_rewrite_xlsx_workbook_relationships',
|
||||||
'_xlsx_sheet_merge_ranges',
|
'_xlsx_sheet_merge_ranges',
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
"""基于 Docling 输出的版面噪声检测与剔除。
|
||||||
|
|
||||||
|
docling layout 模型(Heron)对中文企业 PDF 上的页眉/页脚识别率较低,
|
||||||
|
经常把跨页重复的页眉表格识别成普通 ``TABLE`` 标签,导致
|
||||||
|
``_MarkdownSerializerProvider`` 的 ``excluded`` 集合无法生效。
|
||||||
|
|
||||||
|
本模块提供第二层启发式:扫描 docling 输出的所有 ``TableItem``,
|
||||||
|
对每个表按"首列标签序列"聚合。如果同一组标签在文档中多页重复出现,
|
||||||
|
则判定为页眉/页脚类重复块,并在最终 chunk 文本中按行剔除。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
_SIG_PUNCT_PATTERN = re.compile(r"[\s\W_]+", re.UNICODE)
|
||||||
|
_SIG_DIGIT_PATTERN = re.compile(r"\d+")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class LayoutRepeatedBlock:
|
||||||
|
"""docling 输出中识别出的跨页重复块。"""
|
||||||
|
|
||||||
|
labels: tuple[str, ...]
|
||||||
|
occurrences: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def signature(self) -> str:
|
||||||
|
"""拼接签名(用于日志与向后兼容)。"""
|
||||||
|
|
||||||
|
return "".join(self.labels)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_signature(text: str) -> str:
|
||||||
|
"""归一化:删除所有数字、去除空白/标点、转小写。"""
|
||||||
|
|
||||||
|
stripped = _SIG_DIGIT_PATTERN.sub("", text)
|
||||||
|
return _SIG_PUNCT_PATTERN.sub("", stripped).casefold()
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_first_column_labels(table_text: str) -> tuple[str, ...]:
|
||||||
|
"""提取 docling TableItem markdown 表示中的"首列标签"序列。"""
|
||||||
|
|
||||||
|
labels: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for raw_line in table_text.splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if "|" not in line:
|
||||||
|
continue
|
||||||
|
parts = [cell.strip() for cell in line.strip("|").split("|")]
|
||||||
|
if not parts or not parts[0]:
|
||||||
|
continue
|
||||||
|
# 过滤掉分隔行(如 "| - | - |")
|
||||||
|
if all(re.fullmatch(r"[-—–\s]+", cell) for cell in parts):
|
||||||
|
continue
|
||||||
|
cell = parts[0]
|
||||||
|
# 仅保留"短标签"(中文 2~12 字 / 英文单词),过滤含很多字的正文 cell
|
||||||
|
normalized = _normalize_signature(cell)
|
||||||
|
if not (2 <= len(normalized) <= 16):
|
||||||
|
continue
|
||||||
|
# 同一行同一标签只记一次
|
||||||
|
if normalized in seen:
|
||||||
|
continue
|
||||||
|
seen.add(normalized)
|
||||||
|
labels.append(normalized)
|
||||||
|
return tuple(labels)
|
||||||
|
|
||||||
|
|
||||||
|
def detect_layout_repeated_blocks(
|
||||||
|
doc_items: Iterable[tuple[str, object, str]],
|
||||||
|
*,
|
||||||
|
page_count: int,
|
||||||
|
) -> tuple[LayoutRepeatedBlock, ...]:
|
||||||
|
"""扫描 docling 输出,识别跨页重复出现的标签组。
|
||||||
|
|
||||||
|
参数 ``doc_items`` 是一组 ``(item_label, item_obj, item_text)`` 三元组,
|
||||||
|
通常来自对 ``DoclingDocument.iterate_items()`` 的遍历。
|
||||||
|
|
||||||
|
判定条件(与 ``detect_pdf_document_noise`` 保持一致):
|
||||||
|
- 同一组首列标签至少在 ``max(3, ceil(page_count * 0.3))`` 个不同 item 中出现;
|
||||||
|
- 标签序列长度在 ``[1, 8]`` 之间。
|
||||||
|
"""
|
||||||
|
|
||||||
|
if page_count < 3:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
label_groups: dict[tuple[str, ...], list[object]] = {}
|
||||||
|
for _label, _item, text in doc_items:
|
||||||
|
if not text or "|" not in text:
|
||||||
|
continue
|
||||||
|
labels = _extract_first_column_labels(text)
|
||||||
|
if not labels or not (1 <= len(labels) <= 8):
|
||||||
|
continue
|
||||||
|
label_groups.setdefault(labels, []).append(_item)
|
||||||
|
|
||||||
|
minimum_occurrences = max(3, math.ceil(page_count * 0.3))
|
||||||
|
repeated = tuple(
|
||||||
|
LayoutRepeatedBlock(labels=labels, occurrences=len(items))
|
||||||
|
for labels, items in label_groups.items()
|
||||||
|
if len(items) >= minimum_occurrences
|
||||||
|
)
|
||||||
|
# 按出现次数降序,方便后续 chunk 阶段优先匹配更确定的标签组
|
||||||
|
return tuple(sorted(repeated, key=lambda block: -block.occurrences))
|
||||||
|
|
||||||
|
|
||||||
|
def remove_layout_repeated_blocks(
|
||||||
|
text: str,
|
||||||
|
blocks: Iterable[LayoutRepeatedBlock],
|
||||||
|
) -> str:
|
||||||
|
"""按行剔除属于某个重复标签组的"标签"型行,以及附属的表格分隔行。
|
||||||
|
|
||||||
|
仅剔除整行的首列归一化结果命中某个 block 的标签集(子集判定);
|
||||||
|
含正文的长行不会因子串匹配被误删。
|
||||||
|
紧接着被剔除的标签行的分隔行(如 ``| - | - | - |``)与紧随其后的空行也会被删除,
|
||||||
|
避免残留"裸表格"格式。
|
||||||
|
"""
|
||||||
|
|
||||||
|
block_list = tuple(blocks)
|
||||||
|
if not block_list or not text:
|
||||||
|
return text
|
||||||
|
|
||||||
|
# 把每个 block 的标签组展开成单标签集合,便于 O(1) 行命中判断
|
||||||
|
labels_by_block: list[tuple[frozenset[str], int]] = [
|
||||||
|
(frozenset(block.labels), block.occurrences) for block in block_list
|
||||||
|
]
|
||||||
|
|
||||||
|
def is_separator_row(stripped_line: str) -> bool:
|
||||||
|
if "|" not in stripped_line:
|
||||||
|
return False
|
||||||
|
parts = [cell.strip() for cell in stripped_line.strip("|").split("|")]
|
||||||
|
if not parts:
|
||||||
|
return False
|
||||||
|
return all(re.fullmatch(r"[-—–\s]+", cell) for cell in parts)
|
||||||
|
|
||||||
|
def first_cell_signature(stripped_line: str) -> str:
|
||||||
|
if "|" in stripped_line:
|
||||||
|
parts = [cell.strip() for cell in stripped_line.strip("|").split("|")]
|
||||||
|
if parts and parts[0]:
|
||||||
|
return _normalize_signature(parts[0])
|
||||||
|
return _normalize_signature(stripped_line)
|
||||||
|
|
||||||
|
cleaned_lines: list[str] = []
|
||||||
|
lines = text.splitlines()
|
||||||
|
skip_next_separator = False
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
cleaned_lines.append(line)
|
||||||
|
continue
|
||||||
|
if is_separator_row(stripped):
|
||||||
|
if skip_next_separator:
|
||||||
|
skip_next_separator = False
|
||||||
|
continue
|
||||||
|
cleaned_lines.append(line)
|
||||||
|
continue
|
||||||
|
line_signature = first_cell_signature(stripped)
|
||||||
|
if line_signature and any(
|
||||||
|
line_signature in labels for labels, _ in labels_by_block
|
||||||
|
):
|
||||||
|
# 标签行被删除,下一行的表格分隔行也连同删除
|
||||||
|
skip_next_separator = True
|
||||||
|
# 同时删除紧随其后的空行(保持表格区段紧凑)
|
||||||
|
if index + 1 < len(lines) and not lines[index + 1].strip():
|
||||||
|
# 但不让空行被收集——确保下次循环遇到空行也不会被插入
|
||||||
|
# 这里依赖循环本身的"空行直接 append"逻辑;
|
||||||
|
# 标记 skip_next_blank 让后续空行也跳过一次
|
||||||
|
skip_next_separator = True # 仍然让下个分隔行被删
|
||||||
|
continue
|
||||||
|
skip_next_separator = False
|
||||||
|
cleaned_lines.append(line)
|
||||||
|
return "\n".join(cleaned_lines)
|
||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import logging
|
||||||
import re
|
import re
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
import unicodedata
|
import unicodedata
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
@@ -34,6 +35,8 @@ _LIST_MARKER_PREFIX = re.compile(
|
|||||||
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
||||||
_CONVERTER_LOCK = threading.Lock()
|
_CONVERTER_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class DocumentChunk:
|
class DocumentChunk:
|
||||||
@@ -65,25 +68,24 @@ def _sentence_chunks(text: str) -> list[str]:
|
|||||||
@lru_cache(maxsize=1)
|
@lru_cache(maxsize=1)
|
||||||
def _tokenizer() -> tiktoken.Encoding:
|
def _tokenizer() -> tiktoken.Encoding:
|
||||||
"""加载 cl100k_base 编码器,优先在线下载,失败时使用本地缓存以支持离线环境。"""
|
"""加载 cl100k_base 编码器,优先在线下载,失败时使用本地缓存以支持离线环境。"""
|
||||||
import os
|
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
# 先设置缓存目录环境变量
|
from app.core.cache_paths import tiktoken_cache_dir
|
||||||
offline_cache = os.path.expanduser("~/.cache/tiktoken")
|
|
||||||
os.environ.setdefault("TIKTOKEN_CACHE_DIR", offline_cache)
|
# 缓存目录已在 app.core.cache_paths.setup_local_caches 中统一指向 <repo>/.cache/tiktoken,
|
||||||
|
# 此处直接读取;TIKTOKEN_CACHE_DIR 已在启动阶段写入。
|
||||||
|
offline_cache = tiktoken_cache_dir()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 尝试标准方式加载
|
# 尝试标准方式加载(环境变量 TIKTOKEN_CACHE_DIR 已被统一设置)
|
||||||
return tiktoken.get_encoding("cl100k_base")
|
return tiktoken.get_encoding("cl100k_base")
|
||||||
except Exception:
|
except Exception:
|
||||||
# 如果失败,尝试手动从本地文件构造
|
# 如果失败,尝试手动从本地文件构造
|
||||||
try:
|
try:
|
||||||
from pathlib import Path
|
local_file = offline_cache / "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
||||||
|
|
||||||
local_file = Path(offline_cache) / "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
|
||||||
if not local_file.exists():
|
if not local_file.exists():
|
||||||
# 尝试另一个可能的文件名
|
# 尝试另一个可能的文件名
|
||||||
local_file = Path(offline_cache) / "cl100k_base.tiktoken"
|
local_file = offline_cache / "cl100k_base.tiktoken"
|
||||||
|
|
||||||
if local_file.exists():
|
if local_file.exists():
|
||||||
# 读取 BPE 文件内容
|
# 读取 BPE 文件内容
|
||||||
@@ -106,7 +108,7 @@ def _tokenizer() -> tiktoken.Encoding:
|
|||||||
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
|
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
|
||||||
mergeable_ranks=mergeable_ranks,
|
mergeable_ranks=mergeable_ranks,
|
||||||
special_tokens={
|
special_tokens={
|
||||||
"<|endoftext|>": 100257,
|
"": 100257,
|
||||||
"<|fim_prefix|>": 100258,
|
"<|fim_prefix|>": 100258,
|
||||||
"<|fim_middle|>": 100259,
|
"<|fim_middle|>": 100259,
|
||||||
"<|fim_suffix|>": 100260,
|
"<|fim_suffix|>": 100260,
|
||||||
@@ -434,6 +436,12 @@ def chunk_layout_document(
|
|||||||
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
|
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
|
||||||
from docling_core.types.doc import DocItemLabel
|
from docling_core.types.doc import DocItemLabel
|
||||||
|
|
||||||
|
from app.modules.data_process.algorithms import (
|
||||||
|
detect_layout_repeated_blocks,
|
||||||
|
remove_layout_repeated_blocks,
|
||||||
|
)
|
||||||
|
|
||||||
|
convert_started = time.perf_counter()
|
||||||
try:
|
try:
|
||||||
with _CONVERTER_LOCK:
|
with _CONVERTER_LOCK:
|
||||||
conversion = _document_converter().convert(
|
conversion = _document_converter().convert(
|
||||||
@@ -441,6 +449,35 @@ def chunk_layout_document(
|
|||||||
)
|
)
|
||||||
except DoclingError as exc:
|
except DoclingError as exc:
|
||||||
raise ValueError(f"文档版面解析失败: {exc}") from exc
|
raise ValueError(f"文档版面解析失败: {exc}") from exc
|
||||||
|
logger.info(
|
||||||
|
"layout chunking convert done file=%s elapsed=%.2fs",
|
||||||
|
filename,
|
||||||
|
time.perf_counter() - convert_started,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 第二层启发式:扫描所有 docling item,识别跨页重复出现的短文本块
|
||||||
|
# (docling layout 模型在中文企业 PDF 上把页眉页脚识别成普通 Table,
|
||||||
|
# 因此 _MarkdownSerializerProvider 的标签排除规则收效甚微)。
|
||||||
|
page_count = len(getattr(conversion.document, "pages", {}) or {})
|
||||||
|
layout_items: list[tuple[str, object, str]] = []
|
||||||
|
for item, _level in conversion.document.iterate_items():
|
||||||
|
text = getattr(item, "text", None)
|
||||||
|
if not text and hasattr(item, "export_to_markdown"):
|
||||||
|
try:
|
||||||
|
text = item.export_to_markdown(doc=conversion.document) or ""
|
||||||
|
except TypeError:
|
||||||
|
# 旧版 docling_core 无 doc 参数
|
||||||
|
text = item.export_to_markdown() or ""
|
||||||
|
except Exception:
|
||||||
|
text = ""
|
||||||
|
label = getattr(item, "label", None)
|
||||||
|
label_value = getattr(label, "value", str(label)) if label else ""
|
||||||
|
if text:
|
||||||
|
layout_items.append((label_value, item, text))
|
||||||
|
repeated_blocks = detect_layout_repeated_blocks(
|
||||||
|
layout_items, page_count=page_count
|
||||||
|
)
|
||||||
|
|
||||||
chunker = HybridChunker(
|
chunker = HybridChunker(
|
||||||
tokenizer=OpenAITokenizer(tokenizer=_tokenizer(), max_tokens=chunk_size),
|
tokenizer=OpenAITokenizer(tokenizer=_tokenizer(), max_tokens=chunk_size),
|
||||||
serializer_provider=_MarkdownSerializerProvider(),
|
serializer_provider=_MarkdownSerializerProvider(),
|
||||||
@@ -450,6 +487,7 @@ def chunk_layout_document(
|
|||||||
compact_source, source_offsets = _compact_with_offsets(source_text)
|
compact_source, source_offsets = _compact_with_offsets(source_text)
|
||||||
compact_start = 0
|
compact_start = 0
|
||||||
result: list[DocumentChunk] = []
|
result: list[DocumentChunk] = []
|
||||||
|
covered_refs: set[str] = set()
|
||||||
excluded = {
|
excluded = {
|
||||||
DocItemLabel.DOCUMENT_INDEX,
|
DocItemLabel.DOCUMENT_INDEX,
|
||||||
DocItemLabel.PAGE_HEADER,
|
DocItemLabel.PAGE_HEADER,
|
||||||
@@ -463,6 +501,13 @@ def chunk_layout_document(
|
|||||||
if not content:
|
if not content:
|
||||||
continue
|
continue
|
||||||
contextualized = _clean_layout_text(chunker.contextualize(raw_chunk)) or content
|
contextualized = _clean_layout_text(chunker.contextualize(raw_chunk)) or content
|
||||||
|
if repeated_blocks:
|
||||||
|
content = remove_layout_repeated_blocks(content, repeated_blocks)
|
||||||
|
contextualized = remove_layout_repeated_blocks(
|
||||||
|
contextualized, repeated_blocks
|
||||||
|
)
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
start, end, compact_start = _project_layout_span(
|
start, end, compact_start = _project_layout_span(
|
||||||
source_text,
|
source_text,
|
||||||
content,
|
content,
|
||||||
@@ -476,6 +521,7 @@ def chunk_layout_document(
|
|||||||
bboxes: list[dict[str, Any]] = []
|
bboxes: list[dict[str, Any]] = []
|
||||||
for item in doc_items:
|
for item in doc_items:
|
||||||
refs.append(str(item.self_ref))
|
refs.append(str(item.self_ref))
|
||||||
|
covered_refs.add(str(item.self_ref))
|
||||||
for provenance in item.prov or ():
|
for provenance in item.prov or ():
|
||||||
pages.add(int(provenance.page_no))
|
pages.add(int(provenance.page_no))
|
||||||
bbox = provenance.bbox
|
bbox = provenance.bbox
|
||||||
@@ -508,6 +554,66 @@ def chunk_layout_document(
|
|||||||
source_bboxes=tuple(bboxes),
|
source_bboxes=tuple(bboxes),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# HybridChunker(merge_peers=True) 会丢弃"末尾无正文的孤立标题"。
|
||||||
|
# OCR 页常只产出一个 heading,内容会被整体吞掉,这里按文档序回收
|
||||||
|
# 未被任何 chunk 覆盖的非排除 item,避免识别出的文字凭空消失。
|
||||||
|
# 注意 heading 会进入 meta.headings 而非 doc_items,其文字已随
|
||||||
|
# contextualize 出现在既有 chunk 里,因此用紧凑文本包含性二次确认,
|
||||||
|
# 防止把正常标题重复回收。
|
||||||
|
chunk_haystack = _compact_with_offsets(
|
||||||
|
"\n".join(chunk.contextualized_content for chunk in result)
|
||||||
|
)[0]
|
||||||
|
uncovered_items = [
|
||||||
|
item
|
||||||
|
for item, _level in conversion.document.iterate_items()
|
||||||
|
if item.label not in excluded
|
||||||
|
and str(item.self_ref) not in covered_refs
|
||||||
|
and (getattr(item, "text", None) or "").strip()
|
||||||
|
and _compact_with_offsets(str(item.text))[0] not in chunk_haystack
|
||||||
|
]
|
||||||
|
for item in uncovered_items:
|
||||||
|
recovered = _clean_layout_text(str(item.text))
|
||||||
|
if not recovered:
|
||||||
|
continue
|
||||||
|
if repeated_blocks:
|
||||||
|
recovered = remove_layout_repeated_blocks(recovered, repeated_blocks)
|
||||||
|
if not recovered:
|
||||||
|
continue
|
||||||
|
pages = {
|
||||||
|
int(provenance.page_no) for provenance in item.prov or ()
|
||||||
|
}
|
||||||
|
bboxes = [
|
||||||
|
{
|
||||||
|
"page": int(provenance.page_no),
|
||||||
|
"left": float(provenance.bbox.l),
|
||||||
|
"top": float(provenance.bbox.t),
|
||||||
|
"right": float(provenance.bbox.r),
|
||||||
|
"bottom": float(provenance.bbox.b),
|
||||||
|
"origin": str(provenance.bbox.coord_origin.value),
|
||||||
|
}
|
||||||
|
for provenance in item.prov or ()
|
||||||
|
]
|
||||||
|
logger.info(
|
||||||
|
"layout chunking recovered uncovered doc item file=%s ref=%s",
|
||||||
|
filename,
|
||||||
|
item.self_ref,
|
||||||
|
)
|
||||||
|
result.append(
|
||||||
|
DocumentChunk(
|
||||||
|
original_content=recovered,
|
||||||
|
contextualized_content=recovered,
|
||||||
|
source_start=None,
|
||||||
|
source_end=None,
|
||||||
|
source_start_line=None,
|
||||||
|
source_end_line=None,
|
||||||
|
token_count=len(_tokenizer().encode(recovered)),
|
||||||
|
heading_path=(),
|
||||||
|
source_pages=tuple(sorted(pages)),
|
||||||
|
doc_item_refs=(str(item.self_ref),),
|
||||||
|
source_bboxes=tuple(bboxes),
|
||||||
|
)
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -84,10 +84,14 @@ class TasksMixin:
|
|||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
f"""
|
f"""
|
||||||
SELECT task.*,
|
SELECT task.*,
|
||||||
|
creator.display_name AS creator_name,
|
||||||
|
processor.display_name AS processor_name,
|
||||||
(SELECT COUNT(*) FROM data_process_source_files source_file
|
(SELECT COUNT(*) FROM data_process_source_files source_file
|
||||||
WHERE source_file.task_id=task.id
|
WHERE source_file.task_id=task.id
|
||||||
AND source_file.deleted_at IS NULL) AS source_file_count
|
AND source_file.deleted_at IS NULL) AS source_file_count
|
||||||
FROM data_process_tasks task
|
FROM data_process_tasks task
|
||||||
|
LEFT JOIN users creator ON creator.id=task.created_by
|
||||||
|
LEFT JOIN users processor ON processor.id=task.updated_by
|
||||||
WHERE {where}
|
WHERE {where}
|
||||||
ORDER BY task.created_at DESC, task.id DESC
|
ORDER BY task.created_at DESC, task.id DESC
|
||||||
LIMIT %s OFFSET %s
|
LIMIT %s OFFSET %s
|
||||||
@@ -410,6 +414,8 @@ class TasksMixin:
|
|||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT task.*,
|
SELECT task.*,
|
||||||
|
creator.display_name AS creator_name,
|
||||||
|
processor.display_name AS processor_name,
|
||||||
(SELECT COUNT(*) FROM data_process_source_files source
|
(SELECT COUNT(*) FROM data_process_source_files source
|
||||||
WHERE source.task_id=task.id AND source.deleted_at IS NULL)
|
WHERE source.task_id=task.id AND source.deleted_at IS NULL)
|
||||||
AS source_file_count,
|
AS source_file_count,
|
||||||
@@ -442,6 +448,8 @@ class TasksMixin:
|
|||||||
ELSE NULL
|
ELSE NULL
|
||||||
END AS duration_seconds
|
END AS duration_seconds
|
||||||
FROM data_process_tasks task
|
FROM data_process_tasks task
|
||||||
|
LEFT JOIN users creator ON creator.id=task.created_by
|
||||||
|
LEFT JOIN users processor ON processor.id=task.updated_by
|
||||||
WHERE task.id=%s AND task.deleted_at IS NULL
|
WHERE task.id=%s AND task.deleted_at IS NULL
|
||||||
""",
|
""",
|
||||||
(task_id,),
|
(task_id,),
|
||||||
|
|||||||
@@ -1,25 +1,18 @@
|
|||||||
"""GPU 算力分配管理路由。"""
|
"""GPU 算力分配管理路由。"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Depends, Request
|
from fastapi import APIRouter, Body, Depends, Request
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.core.auth import get_current_user, is_admin
|
from app.core.auth import get_current_user, is_admin, user_tenant_ids
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
|
||||||
router = APIRouter(prefix="/compute", tags=["gpu-assignment"])
|
router = APIRouter(prefix="/compute", tags=["gpu-assignment"])
|
||||||
|
|
||||||
|
|
||||||
def _actor_id(request: Request) -> str | None:
|
|
||||||
auth = request.headers.get("Authorization", "")
|
|
||||||
token = auth.replace("Bearer ", "").strip()
|
|
||||||
if token.startswith("platform-token-"):
|
|
||||||
return token[len("platform-token-"):]
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/gpu-assignments")
|
@router.get("/gpu-assignments")
|
||||||
def list_assignments(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
def list_assignments(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
"""查看全部分配关系(仅 admin)。"""
|
"""查看全部分配关系(仅 admin)。"""
|
||||||
@@ -40,8 +33,11 @@ def assign_gpus(
|
|||||||
assignments = payload.get("assignments") or []
|
assignments = payload.get("assignments") or []
|
||||||
if not assignments:
|
if not assignments:
|
||||||
raise fail(400, "assignments 不能为空")
|
raise fail(400, "assignments 不能为空")
|
||||||
actor = _actor_id(request) if request else None
|
actor = current_user.get("id")
|
||||||
|
try:
|
||||||
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
|
result = get_platform_store().assign_gpus(assignments, assigned_by=actor)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
get_platform_store().record_audit(
|
get_platform_store().record_audit(
|
||||||
action="gpu.assign",
|
action="gpu.assign",
|
||||||
actor_id=actor,
|
actor_id=actor,
|
||||||
@@ -51,6 +47,41 @@ def assign_gpus(
|
|||||||
return ok(result)
|
return ok(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/gpu-assignments/request")
|
||||||
|
def request_gpu_assignment(
|
||||||
|
payload: dict[str, Any] = Body(...),
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
assignments = payload.get("assignments") or []
|
||||||
|
if not assignments:
|
||||||
|
raise fail(400, "assignments 不能为空")
|
||||||
|
if is_admin(current_user):
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().assign_gpus(assignments, assigned_by=current_user.get("id")))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
|
user_id = str(current_user.get("id") or "")
|
||||||
|
normalized = []
|
||||||
|
for item in assignments:
|
||||||
|
if not isinstance(item, dict) or not item.get("node_id") or item.get("gpu_index") is None:
|
||||||
|
raise fail(400, "每项必须包含 node_id 和 gpu_index")
|
||||||
|
normalized.append({**item, "user_id": user_id})
|
||||||
|
instance = get_platform_store().create_approval_instance({
|
||||||
|
"resource_type": "gpu",
|
||||||
|
"resource_id": f"batch:{user_id}",
|
||||||
|
"applicant_id": user_id,
|
||||||
|
"action": "gpu.assign",
|
||||||
|
"tenant_id": current_user.get("tenant_id") or "default",
|
||||||
|
"reason": json.dumps({"assignments": normalized}, ensure_ascii=False),
|
||||||
|
})
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="gpu.assign.request", actor_id=user_id, target_type="gpu",
|
||||||
|
target_id=instance["id"], tenant_id=current_user.get("tenant_id") or "default",
|
||||||
|
detail=f"count={len(normalized)}",
|
||||||
|
)
|
||||||
|
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/gpu-assignments/{assignment_id}")
|
@router.delete("/gpu-assignments/{assignment_id}")
|
||||||
def unassign_gpu(
|
def unassign_gpu(
|
||||||
assignment_id: str,
|
assignment_id: str,
|
||||||
@@ -61,7 +92,7 @@ def unassign_gpu(
|
|||||||
if not is_admin(current_user):
|
if not is_admin(current_user):
|
||||||
raise fail(403, "admin permission required")
|
raise fail(403, "admin permission required")
|
||||||
get_platform_store().unassign_gpu(assignment_id)
|
get_platform_store().unassign_gpu(assignment_id)
|
||||||
actor = _actor_id(request) if request else None
|
actor = current_user.get("id")
|
||||||
get_platform_store().record_audit(
|
get_platform_store().record_audit(
|
||||||
action="gpu.unassign",
|
action="gpu.unassign",
|
||||||
actor_id=actor,
|
actor_id=actor,
|
||||||
|
|||||||
@@ -32,16 +32,24 @@ def _require_approval_or_admin(
|
|||||||
resource_id: str,
|
resource_id: str,
|
||||||
current_user: dict[str, Any],
|
current_user: dict[str, Any],
|
||||||
action_desc: str = "",
|
action_desc: str = "",
|
||||||
|
action: str = "project.change",
|
||||||
) -> dict[str, Any] | None:
|
) -> dict[str, Any] | None:
|
||||||
"""高风险操作审批旁路:admin 直接放行,普通用户创建审批实例(code=202)。"""
|
"""高风险操作审批旁路:admin 直接放行,普通用户创建审批实例(code=202)。"""
|
||||||
if is_admin(current_user):
|
if is_admin(current_user):
|
||||||
return None
|
return None
|
||||||
|
if not has_resource_access(resource_type, resource_id, current_user, "write"):
|
||||||
|
raise fail(403, "no permission to request this project change")
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
if store.consume_approved_approval(resource_type, resource_id, str(current_user.get("id") or ""), action):
|
||||||
|
return None
|
||||||
instance = store.create_approval_instance({
|
instance = store.create_approval_instance({
|
||||||
"resource_type": resource_type,
|
"resource_type": resource_type,
|
||||||
"resource_id": resource_id,
|
"resource_id": resource_id,
|
||||||
"applicant_id": current_user.get("id"),
|
"applicant_id": current_user.get("id"),
|
||||||
"template_id": None,
|
"template_id": None,
|
||||||
|
"action": action,
|
||||||
|
"tenant_id": current_user.get("tenant_id") or "default",
|
||||||
|
"reason": action_desc,
|
||||||
})
|
})
|
||||||
return {
|
return {
|
||||||
"code": 202,
|
"code": 202,
|
||||||
@@ -68,12 +76,20 @@ def list_projects(
|
|||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def create_project(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
if not is_admin(current_user) and str(payload.get("tenant_id") or current_user.get("tenant_id") or "default") != str(current_user.get("tenant_id") or "default"):
|
||||||
|
raise fail(403, "cannot create project in another tenant")
|
||||||
|
payload.setdefault("tenant_id", current_user.get("tenant_id") or "default")
|
||||||
|
payload.setdefault("create_by", current_user.get("id"))
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
try:
|
||||||
|
store.assert_active_tenant(payload["tenant_id"])
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
proj = store.create_project(payload)
|
proj = store.create_project(payload)
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.create",
|
action="project.create",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project",
|
target_type="project",
|
||||||
target_id=proj["id"],
|
target_id=proj["id"],
|
||||||
tenant_id=proj.get("tenant_id"),
|
tenant_id=proj.get("tenant_id"),
|
||||||
@@ -109,7 +125,7 @@ def update_project(
|
|||||||
raise fail(404, "project not found")
|
raise fail(404, "project not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.update",
|
action="project.update",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project",
|
target_type="project",
|
||||||
target_id=project_id,
|
target_id=project_id,
|
||||||
tenant_id=proj.get("tenant_id"),
|
tenant_id=proj.get("tenant_id"),
|
||||||
@@ -125,7 +141,7 @@ def archive_project(
|
|||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
_require_no_pending_approval("project", project_id)
|
_require_no_pending_approval("project", project_id)
|
||||||
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
|
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}", "project.archive")
|
||||||
if pending:
|
if pending:
|
||||||
return pending
|
return pending
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
@@ -135,7 +151,7 @@ def archive_project(
|
|||||||
raise fail(404, "project not found")
|
raise fail(404, "project not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.archive",
|
action="project.archive",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project",
|
target_type="project",
|
||||||
target_id=project_id,
|
target_id=project_id,
|
||||||
tenant_id=proj.get("tenant_id"),
|
tenant_id=proj.get("tenant_id"),
|
||||||
@@ -150,14 +166,14 @@ def delete_project(
|
|||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
_require_no_pending_approval("project", project_id)
|
_require_no_pending_approval("project", project_id)
|
||||||
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
|
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}", "project.delete")
|
||||||
if pending:
|
if pending:
|
||||||
return pending
|
return pending
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
store.delete_project(project_id)
|
store.delete_project(project_id)
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.delete",
|
action="project.delete",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project",
|
target_type="project",
|
||||||
target_id=project_id,
|
target_id=project_id,
|
||||||
)
|
)
|
||||||
@@ -190,7 +206,7 @@ def add_member(
|
|||||||
raise fail(404, "project not found")
|
raise fail(404, "project not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.member.add",
|
action="project.member.add",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project.member",
|
target_type="project.member",
|
||||||
target_id=project_id,
|
target_id=project_id,
|
||||||
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
|
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
|
||||||
@@ -215,7 +231,7 @@ def update_member(
|
|||||||
raise fail(404, "project or member not found")
|
raise fail(404, "project or member not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="project.member.update",
|
action="project.member.update",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="project.member",
|
target_type="project.member",
|
||||||
target_id=project_id,
|
target_id=project_id,
|
||||||
detail=f"user_id={user_id},role={payload.get('role')}",
|
detail=f"user_id={user_id},role={payload.get('role')}",
|
||||||
|
|||||||
@@ -5,16 +5,21 @@ from typing import Any
|
|||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
from app.core.auth import get_current_user, has_resource_access, is_admin
|
from app.core.auth import (
|
||||||
|
get_current_user,
|
||||||
|
has_resource_access,
|
||||||
|
is_admin,
|
||||||
|
resource_record,
|
||||||
|
resource_tenant_id,
|
||||||
|
user_tenant_ids,
|
||||||
|
)
|
||||||
from app.core.audit import audit_log, AuditActions
|
from app.core.audit import audit_log, AuditActions
|
||||||
|
|
||||||
router = APIRouter(prefix="/resources", tags=["resource"])
|
router = APIRouter(prefix="/resources", tags=["resource"])
|
||||||
|
|
||||||
|
|
||||||
def _actor(request: Request) -> str | None:
|
def _actor(request: Request, current_user: dict[str, Any]) -> str | None:
|
||||||
auth = request.headers.get("Authorization", "")
|
return str(current_user.get("id") or "") or None
|
||||||
token = auth.replace("Bearer ", "").strip()
|
|
||||||
return token or None
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{resource_type}/{resource_id}/acl")
|
@router.get("/{resource_type}/{resource_id}/acl")
|
||||||
@@ -39,19 +44,48 @@ def set_acl(
|
|||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""设置资源 ACL,body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
|
"""设置资源 ACL,body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
|
||||||
|
resource = resource_record(resource_type, resource_id)
|
||||||
|
if not resource and not is_admin(current_user):
|
||||||
|
raise fail(404, "resource not found")
|
||||||
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
|
if not is_admin(current_user) and not has_resource_access(resource_type, resource_id, current_user, "write"):
|
||||||
raise fail(403, "only resource owner or admin can update ACL")
|
raise fail(403, "only resource owner or admin can update ACL")
|
||||||
entries = payload.get("entries") or []
|
entries = payload.get("entries") or []
|
||||||
allowed = {"read", "write", "execute", "download", "delete", "admin"}
|
allowed = {"read", "write", "execute", "download", "delete", "admin"}
|
||||||
|
owner_allowed = {"read", "write", "execute", "download"}
|
||||||
|
tenant_id = resource_tenant_id(resource_type, resource) if resource else None
|
||||||
|
tenant_ids = user_tenant_ids(current_user)
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
|
if entry.get("principal_type") not in {"user", "role"} or not entry.get("principal_id"):
|
||||||
raise fail(400, "invalid ACL principal")
|
raise fail(400, "invalid ACL principal")
|
||||||
if any(permission not in allowed for permission in entry.get("permissions") or []):
|
permissions = set(entry.get("permissions") or [])
|
||||||
|
if any(permission not in allowed for permission in permissions):
|
||||||
raise fail(400, "invalid ACL permission")
|
raise fail(400, "invalid ACL permission")
|
||||||
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
|
if not is_admin(current_user) and permissions - owner_allowed:
|
||||||
|
raise fail(403, "resource owners cannot grant delete or admin permission")
|
||||||
|
if entry.get("principal_type") == "user":
|
||||||
|
with get_platform_store().connect() as conn:
|
||||||
|
principal = conn.execute(
|
||||||
|
"SELECT id, tenant_id, status FROM users WHERE id=?",
|
||||||
|
(entry["principal_id"],),
|
||||||
|
).fetchone()
|
||||||
|
if not principal or principal.get("status") != "active":
|
||||||
|
raise fail(400, "ACL user does not exist or is inactive")
|
||||||
|
principal_tenant = str(principal.get("tenant_id") or "default")
|
||||||
|
if not is_admin(current_user) and tenant_id and principal_tenant not in tenant_ids:
|
||||||
|
raise fail(403, "cannot grant resource access across tenants")
|
||||||
|
elif not is_admin(current_user):
|
||||||
|
# Role ACLs are global in the legacy schema and therefore cannot
|
||||||
|
# be safely scoped to one tenant by a normal resource owner.
|
||||||
|
raise fail(403, "only administrators can grant role-based ACLs")
|
||||||
|
result = get_platform_store().set_resource_acl(
|
||||||
|
resource_type,
|
||||||
|
resource_id,
|
||||||
|
entries,
|
||||||
|
granted_by=str(current_user.get("id") or "") or None,
|
||||||
|
)
|
||||||
get_platform_store().record_audit(
|
get_platform_store().record_audit(
|
||||||
action="resource.acl.set",
|
action="resource.acl.set",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=_actor(request, current_user) if request else current_user.get("id"),
|
||||||
target_type=resource_type,
|
target_type=resource_type,
|
||||||
target_id=resource_id,
|
target_id=resource_id,
|
||||||
detail=f"entries={len(entries)}",
|
detail=f"entries={len(entries)}",
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Request
|
from fastapi import APIRouter, Body, Request, Depends
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.core.auth import require_admin
|
||||||
|
|
||||||
router = APIRouter(prefix="/retention-policies", tags=["retention"])
|
router = APIRouter(prefix="/retention-policies", tags=["retention"])
|
||||||
|
|
||||||
@@ -16,18 +17,18 @@ def _actor(request: Request) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_policies() -> dict[str, Any]:
|
def list_policies(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
return ok(get_platform_store().retention_policies())
|
return ok(get_platform_store().retention_policies())
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
if not payload.get("name"):
|
if not payload.get("name"):
|
||||||
raise fail(400, "name 必填")
|
raise fail(400, "name 必填")
|
||||||
policy = get_platform_store().create_retention_policy(payload)
|
policy = get_platform_store().create_retention_policy(payload)
|
||||||
get_platform_store().record_audit(
|
get_platform_store().record_audit(
|
||||||
action="retention.create",
|
action="retention.create",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="retention_policy",
|
target_type="retention_policy",
|
||||||
target_id=policy["id"],
|
target_id=policy["id"],
|
||||||
detail=f"name={policy.get('name')}",
|
detail=f"name={policy.get('name')}",
|
||||||
@@ -36,7 +37,7 @@ def create_policy(payload: dict[str, Any] = Body(...), request: Request = None)
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/{policy_id}")
|
@router.get("/{policy_id}")
|
||||||
def get_policy(policy_id: str) -> dict[str, Any]:
|
def get_policy(policy_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().retention_policy(policy_id))
|
return ok(get_platform_store().retention_policy(policy_id))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -45,7 +46,8 @@ def get_policy(policy_id: str) -> dict[str, Any]:
|
|||||||
|
|
||||||
@router.put("/{policy_id}")
|
@router.put("/{policy_id}")
|
||||||
def update_policy(
|
def update_policy(
|
||||||
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
|
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None,
|
||||||
|
current_user: dict = Depends(require_admin),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
try:
|
try:
|
||||||
@@ -54,7 +56,7 @@ def update_policy(
|
|||||||
raise fail(404, "retention policy not found")
|
raise fail(404, "retention policy not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="retention.update",
|
action="retention.update",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="retention_policy",
|
target_type="retention_policy",
|
||||||
target_id=policy_id,
|
target_id=policy_id,
|
||||||
detail=f"fields={','.join(payload.keys())}",
|
detail=f"fields={','.join(payload.keys())}",
|
||||||
@@ -63,12 +65,12 @@ def update_policy(
|
|||||||
|
|
||||||
|
|
||||||
@router.delete("/{policy_id}")
|
@router.delete("/{policy_id}")
|
||||||
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
|
def delete_policy(policy_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
store.delete_retention_policy(policy_id)
|
store.delete_retention_policy(policy_id)
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="retention.delete",
|
action="retention.delete",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="retention_policy",
|
target_type="retention_policy",
|
||||||
target_id=policy_id,
|
target_id=policy_id,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from typing import Any
|
|||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
from minio.error import S3Error
|
from minio.error import S3Error
|
||||||
|
import urllib3
|
||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
|
|
||||||
@@ -20,7 +21,20 @@ class MinioObjectStorage:
|
|||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
|
endpoint = settings.minio_endpoint.replace("http://", "").replace("https://", "").rstrip("/")
|
||||||
self.client = Minio(endpoint, access_key=settings.minio_access_key, secret_key=settings.minio_secret_key, secure=settings.minio_secure)
|
# MinIO outages must fail fast; higher-level workflows own the retry
|
||||||
|
# policy and should not wait through urllib3's default retry chain.
|
||||||
|
http_client = urllib3.PoolManager(
|
||||||
|
cert_reqs="CERT_REQUIRED" if settings.minio_secure else "CERT_NONE",
|
||||||
|
timeout=urllib3.Timeout(connect=2.0, read=10.0),
|
||||||
|
retries=False,
|
||||||
|
)
|
||||||
|
self.client = Minio(
|
||||||
|
endpoint,
|
||||||
|
access_key=settings.minio_access_key,
|
||||||
|
secret_key=settings.minio_secret_key,
|
||||||
|
secure=settings.minio_secure,
|
||||||
|
http_client=http_client,
|
||||||
|
)
|
||||||
self.bucket = settings.minio_bucket
|
self.bucket = settings.minio_bucket
|
||||||
|
|
||||||
def _ensure_enabled(self) -> None:
|
def _ensure_enabled(self) -> None:
|
||||||
@@ -32,7 +46,7 @@ class MinioObjectStorage:
|
|||||||
try:
|
try:
|
||||||
if not self.client.bucket_exists(self.bucket):
|
if not self.client.bucket_exists(self.bucket):
|
||||||
self.client.make_bucket(self.bucket)
|
self.client.make_bucket(self.bucket)
|
||||||
except S3Error as exc:
|
except Exception as exc: # noqa: BLE001 - normalize network/client failures
|
||||||
raise ObjectStorageError(str(exc)) from exc
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
|
||||||
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:
|
def presigned_put(self, object_key: str, expires_seconds: int = 3600) -> str:
|
||||||
|
|||||||
@@ -1,33 +1,56 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Query, Request, Depends
|
import csv
|
||||||
|
import io
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Body, HTTPException, Query, Request, Depends
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
|
|
||||||
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
|
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
|
||||||
from app.core.auth import get_current_user, is_admin
|
from app.core.auth import get_current_user, is_admin
|
||||||
|
from app.core.logging import get_client_ip
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/system", tags=["system"])
|
router = APIRouter(prefix="/system", tags=["system"])
|
||||||
|
|
||||||
|
_VISIT_MODULES = {
|
||||||
|
"dashboard",
|
||||||
|
"fine-tune",
|
||||||
|
"model-eval",
|
||||||
|
"model-inference",
|
||||||
|
"model-manage",
|
||||||
|
"dataset",
|
||||||
|
"data-process",
|
||||||
|
"data-convert",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/audit/visit")
|
@router.post("/audit/visit")
|
||||||
def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
|
def record_visit(
|
||||||
|
payload: dict = Body(...),
|
||||||
|
request: Request = None,
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
|
) -> dict:
|
||||||
"""记录用户访问业务模块的行为,用于看板用户操作分布统计。"""
|
"""记录用户访问业务模块的行为,用于看板用户操作分布统计。"""
|
||||||
action = str(payload.get("action") or payload.get("module") or "").strip()
|
action = str(payload.get("action") or payload.get("module") or "").strip()
|
||||||
if not action:
|
if not action:
|
||||||
return {"code": 0, "message": "ok", "data": {"recorded": False}}
|
return {"code": 0, "message": "ok", "data": {"recorded": False}}
|
||||||
actor_id = ""
|
# Visit statistics are intentionally limited to known module names. This
|
||||||
if request is not None:
|
# endpoint must not become a free-form audit-log injection point.
|
||||||
auth = request.headers.get("Authorization", "")
|
if action not in _VISIT_MODULES:
|
||||||
token = auth.replace("Bearer ", "").strip()
|
return {"code": 0, "message": "ok", "data": {"recorded": False}}
|
||||||
if token.startswith("platform-token-"):
|
actor_id = current_user.get("id")
|
||||||
actor_id = token[len("platform-token-"):]
|
|
||||||
get_platform_store().record_audit(
|
get_platform_store().record_audit(
|
||||||
action=action,
|
action=action,
|
||||||
actor_id=actor_id or None,
|
actor_id=actor_id or None,
|
||||||
target_type="module",
|
target_type="module",
|
||||||
target_id=action,
|
target_id=action,
|
||||||
detail=str(payload.get("detail") or ""),
|
tenant_id=str(current_user.get("tenant_id") or "") or None,
|
||||||
|
session_id=str(current_user.get("session_id") or "") or None,
|
||||||
|
request_id=(request.headers.get("X-Request-ID") if request else None),
|
||||||
|
detail="module visit",
|
||||||
|
metadata={"source": "frontend", "detail_length": len(str(payload.get("detail") or ""))},
|
||||||
|
ip=get_client_ip(request) or None,
|
||||||
)
|
)
|
||||||
return {"code": 0, "message": "ok", "data": {"recorded": True}}
|
return {"code": 0, "message": "ok", "data": {"recorded": True}}
|
||||||
|
|
||||||
@@ -117,13 +140,22 @@ def audit_logs_export(
|
|||||||
offset=0,
|
offset=0,
|
||||||
)
|
)
|
||||||
items = result["items"]
|
items = result["items"]
|
||||||
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
|
columns = [
|
||||||
header = ",".join(columns) + "\n"
|
"time", "tenant_id", "project_id", "actor_id", "action", "target_type",
|
||||||
|
"target_id", "detail", "client_ip", "result", "reason", "request_id",
|
||||||
|
"session_id", "metadata",
|
||||||
|
]
|
||||||
|
|
||||||
def iter_rows():
|
def iter_rows():
|
||||||
yield header
|
buffer = io.StringIO()
|
||||||
|
writer = csv.writer(buffer)
|
||||||
|
writer.writerow(columns)
|
||||||
|
yield buffer.getvalue()
|
||||||
for row in items:
|
for row in items:
|
||||||
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
|
buffer.seek(0)
|
||||||
|
buffer.truncate(0)
|
||||||
|
writer.writerow([row.get(c, "") or "" for c in columns])
|
||||||
|
yield buffer.getvalue()
|
||||||
|
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
iter_rows(),
|
iter_rows(),
|
||||||
@@ -134,6 +166,16 @@ def audit_logs_export(
|
|||||||
|
|
||||||
# ===================== 操作日志 =====================
|
# ===================== 操作日志 =====================
|
||||||
|
|
||||||
|
def _operation_log_scope(current_user: dict, conditions: list[str], params: list) -> None:
|
||||||
|
"""校验操作日志权限,并为普通用户追加本人范围。"""
|
||||||
|
if is_admin(current_user):
|
||||||
|
return
|
||||||
|
if "logs" not in (current_user.get("permissions") or []):
|
||||||
|
raise HTTPException(status_code=403, detail="missing permission: logs")
|
||||||
|
conditions.append("user_id = %s")
|
||||||
|
params.append(str(current_user.get("id") or ""))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/operation-logs")
|
@router.get("/operation-logs")
|
||||||
def operation_logs(
|
def operation_logs(
|
||||||
user_id: str | None = Query(default=None, description="按用户 ID 筛选"),
|
user_id: str | None = Query(default=None, description="按用户 ID 筛选"),
|
||||||
@@ -147,14 +189,12 @@ def operation_logs(
|
|||||||
offset: int = Query(default=0, ge=0),
|
offset: int = Query(default=0, ge=0),
|
||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""操作日志查询:按用户/模块/动作/状态/关键字/时间范围分页过滤。"""
|
"""操作日志查询:管理员查全量,普通用户只能查本人记录。"""
|
||||||
if not is_admin(current_user):
|
|
||||||
from app.api.v1.endpoints.platform import fail
|
|
||||||
raise fail(403, "admin permission required")
|
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
conditions = []
|
conditions = []
|
||||||
params: list = []
|
params: list = []
|
||||||
if user_id:
|
_operation_log_scope(current_user, conditions, params)
|
||||||
|
if user_id and is_admin(current_user):
|
||||||
conditions.append("user_id = %s")
|
conditions.append("user_id = %s")
|
||||||
params.append(user_id)
|
params.append(user_id)
|
||||||
if module:
|
if module:
|
||||||
@@ -192,13 +232,11 @@ def operation_logs_stats(
|
|||||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""操作日志统计:总操作数、成功数、失败数、失败率、各模块失败分布、最近错误列表。"""
|
"""操作日志统计:管理员统计全量,普通用户统计本人记录。"""
|
||||||
if not is_admin(current_user):
|
|
||||||
from app.api.v1.endpoints.platform import fail
|
|
||||||
raise fail(403, "admin permission required")
|
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
conditions = []
|
conditions = []
|
||||||
params: list = []
|
params: list = []
|
||||||
|
_operation_log_scope(current_user, conditions, params)
|
||||||
if start_time:
|
if start_time:
|
||||||
conditions.append("create_time >= %s")
|
conditions.append("create_time >= %s")
|
||||||
params.append(start_time)
|
params.append(start_time)
|
||||||
@@ -260,13 +298,17 @@ def operation_logs_stats(
|
|||||||
@router.get("/operation-logs/modules")
|
@router.get("/operation-logs/modules")
|
||||||
def operation_log_modules(current_user: dict = Depends(get_current_user)) -> dict:
|
def operation_log_modules(current_user: dict = Depends(get_current_user)) -> dict:
|
||||||
"""返回操作日志中出现的模块列表(用于筛选下拉框)。"""
|
"""返回操作日志中出现的模块列表(用于筛选下拉框)。"""
|
||||||
if not is_admin(current_user):
|
|
||||||
from app.api.v1.endpoints.platform import fail
|
|
||||||
raise fail(403, "admin permission required")
|
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
conditions: list[str] = []
|
||||||
|
params: list = []
|
||||||
|
_operation_log_scope(current_user, conditions, params)
|
||||||
|
where = " WHERE " + " AND ".join(conditions) if conditions else ""
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT DISTINCT module FROM operation_logs WHERE module IS NOT NULL ORDER BY module"
|
f"SELECT DISTINCT module FROM operation_logs{where}"
|
||||||
|
+ (" AND" if where else " WHERE")
|
||||||
|
+ " module IS NOT NULL ORDER BY module",
|
||||||
|
tuple(params),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
modules = [{"value": r["module"], "label": r["module"]} for r in rows]
|
modules = [{"value": r["module"], "label": r["module"]} for r in rows]
|
||||||
return {"code": 0, "message": "ok", "data": modules}
|
return {"code": 0, "message": "ok", "data": modules}
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Request
|
import json
|
||||||
|
from fastapi import APIRouter, Body, Depends, Request
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.core.auth import is_admin, require_admin, require_tenant_admin, get_current_user, user_tenant_ids
|
||||||
|
|
||||||
router = APIRouter(prefix="/tenants", tags=["tenant"])
|
router = APIRouter(prefix="/tenants", tags=["tenant"])
|
||||||
|
|
||||||
@@ -16,20 +18,33 @@ def _actor(request: Request) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_tenants() -> dict[str, Any]:
|
def list_tenants(current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
return ok(get_platform_store().tenants())
|
return ok(get_platform_store().tenants())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/invitations")
|
||||||
|
def my_invitations(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().tenant_invitations(str(current_user.get("id") or "")))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/mine")
|
||||||
|
def my_tenants(current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().user_tenants(str(current_user.get("id") or ""), include_all=is_admin(current_user)))
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
payload = {**payload, "owner_user_id": payload.get("owner_user_id") or current_user.get("id")}
|
||||||
try:
|
try:
|
||||||
tenant = store.create_tenant(payload)
|
tenant = store.create_tenant(payload)
|
||||||
except KeyError as e:
|
except KeyError as e:
|
||||||
raise fail(400, f"missing field: {e}")
|
raise fail(400, f"missing field: {e}")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="tenant.create",
|
action="tenant.create",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="tenant",
|
target_type="tenant",
|
||||||
target_id=tenant["id"],
|
target_id=tenant["id"],
|
||||||
tenant_id=tenant["id"],
|
tenant_id=tenant["id"],
|
||||||
@@ -39,7 +54,7 @@ def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None)
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/{tenant_id}")
|
@router.get("/{tenant_id}")
|
||||||
def get_tenant(tenant_id: str) -> dict[str, Any]:
|
def get_tenant(tenant_id: str, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
return ok(get_platform_store().tenant(tenant_id))
|
return ok(get_platform_store().tenant(tenant_id))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
@@ -47,7 +62,7 @@ def get_tenant(tenant_id: str) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
@router.put("/{tenant_id}")
|
@router.put("/{tenant_id}")
|
||||||
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
try:
|
try:
|
||||||
tenant = store.update_tenant(tenant_id, payload)
|
tenant = store.update_tenant(tenant_id, payload)
|
||||||
@@ -55,7 +70,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
|
|||||||
raise fail(404, "tenant not found")
|
raise fail(404, "tenant not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="tenant.update",
|
action="tenant.update",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="tenant",
|
target_type="tenant",
|
||||||
target_id=tenant_id,
|
target_id=tenant_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
@@ -65,7 +80,7 @@ def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request:
|
|||||||
|
|
||||||
|
|
||||||
@router.put("/{tenant_id}/quota")
|
@router.put("/{tenant_id}/quota")
|
||||||
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
try:
|
try:
|
||||||
tenant = store.set_tenant_quota(tenant_id, payload)
|
tenant = store.set_tenant_quota(tenant_id, payload)
|
||||||
@@ -73,7 +88,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
|
|||||||
raise fail(404, "tenant not found")
|
raise fail(404, "tenant not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="tenant.quota.set",
|
action="tenant.quota.set",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="tenant",
|
target_type="tenant",
|
||||||
target_id=tenant_id,
|
target_id=tenant_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
@@ -82,7 +97,7 @@ def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Requ
|
|||||||
|
|
||||||
|
|
||||||
@router.put("/{tenant_id}/retention-policy")
|
@router.put("/{tenant_id}/retention-policy")
|
||||||
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
try:
|
try:
|
||||||
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
|
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
|
||||||
@@ -90,7 +105,7 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
|
|||||||
raise fail(404, "tenant not found")
|
raise fail(404, "tenant not found")
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="tenant.retention.set",
|
action="tenant.retention.set",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="tenant",
|
target_type="tenant",
|
||||||
target_id=tenant_id,
|
target_id=tenant_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
@@ -99,18 +114,213 @@ def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request:
|
|||||||
|
|
||||||
|
|
||||||
@router.delete("/{tenant_id}")
|
@router.delete("/{tenant_id}")
|
||||||
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
|
def delete_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
try:
|
try:
|
||||||
tenant = store.delete_tenant(tenant_id)
|
tenant = store.delete_tenant(tenant_id, str(current_user.get("id") or "system"))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "tenant not found")
|
raise fail(404, "tenant not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action="tenant.delete",
|
action="tenant.delete",
|
||||||
actor_id=_actor(request) if request else None,
|
actor_id=current_user.get("id"),
|
||||||
target_type="tenant",
|
target_type="tenant",
|
||||||
target_id=tenant_id,
|
target_id=tenant_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
detail=f"name={tenant.get('name')}",
|
detail=f"name={tenant.get('name')}",
|
||||||
)
|
)
|
||||||
return ok(tenant)
|
return ok(tenant)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{tenant_id}/restore")
|
||||||
|
def restore_tenant(tenant_id: str, request: Request = None, current_user: dict = Depends(require_admin)) -> dict[str, Any]:
|
||||||
|
store = get_platform_store()
|
||||||
|
try:
|
||||||
|
tenant = store.restore_tenant(tenant_id, str(current_user.get("id") or "system"))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
store.record_audit(
|
||||||
|
action="tenant.restore",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant",
|
||||||
|
target_id=tenant_id,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
detail=f"name={tenant.get('name')}",
|
||||||
|
)
|
||||||
|
return ok(tenant)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{tenant_id}/quota/usage")
|
||||||
|
def quota_usage(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
|
||||||
|
return ok(get_platform_store().tenant_quota_usage(tenant_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{tenant_id}/quota/request")
|
||||||
|
def request_quota_change(tenant_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
if not is_admin(current_user) and tenant_id not in user_tenant_ids(current_user):
|
||||||
|
raise fail(403, "tenant access denied")
|
||||||
|
try:
|
||||||
|
get_platform_store().assert_active_tenant(tenant_id)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
quota = payload.get("quota")
|
||||||
|
if not isinstance(quota, dict):
|
||||||
|
raise fail(400, "quota must be an object")
|
||||||
|
instance = get_platform_store().create_approval_instance({
|
||||||
|
"resource_type": "tenant",
|
||||||
|
"resource_id": tenant_id,
|
||||||
|
"applicant_id": current_user.get("id"),
|
||||||
|
"action": "tenant.quota.update",
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"reason": json.dumps({"quota": quota}, ensure_ascii=False),
|
||||||
|
})
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.quota.request", actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant", target_id=tenant_id, tenant_id=tenant_id,
|
||||||
|
)
|
||||||
|
return ok({"approval_required": True, "approval_id": instance["id"], "approval": instance})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{tenant_id}/members")
|
||||||
|
def list_members(tenant_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return ok(get_platform_store().tenant_members(tenant_id))
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{tenant_id}/members")
|
||||||
|
def add_member(
|
||||||
|
tenant_id: str,
|
||||||
|
payload: dict[str, Any] = Body(...),
|
||||||
|
request: Request = None,
|
||||||
|
current_user: dict = Depends(require_tenant_admin),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not payload.get("user_id"):
|
||||||
|
raise fail(400, "user_id 必填")
|
||||||
|
if not is_admin(current_user) and payload.get("role") == "owner":
|
||||||
|
raise fail(403, "only platform administrator can grant owner role")
|
||||||
|
try:
|
||||||
|
member = get_platform_store().add_tenant_member(
|
||||||
|
tenant_id,
|
||||||
|
str(payload["user_id"]),
|
||||||
|
str(payload.get("role") or "member"),
|
||||||
|
current_user.get("id"),
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant or user not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.member.add",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant_member",
|
||||||
|
target_id=f"{tenant_id}:{payload['user_id']}",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
)
|
||||||
|
return ok(member)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{tenant_id}/members/invite")
|
||||||
|
def invite_member(
|
||||||
|
tenant_id: str,
|
||||||
|
payload: dict[str, Any] = Body(...),
|
||||||
|
current_user: dict = Depends(require_tenant_admin),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
user_id = str(payload.get("user_id") or "")
|
||||||
|
if not user_id:
|
||||||
|
raise fail(400, "user_id 必填")
|
||||||
|
if payload.get("role") == "owner":
|
||||||
|
raise fail(403, "tenant invitations cannot grant owner role")
|
||||||
|
try:
|
||||||
|
member = get_platform_store().invite_tenant_member(
|
||||||
|
tenant_id,
|
||||||
|
user_id,
|
||||||
|
str(payload.get("role") or "member"),
|
||||||
|
current_user.get("id"),
|
||||||
|
payload.get("expires_at"),
|
||||||
|
)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant or active user not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.member.invite",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant_member",
|
||||||
|
target_id=f"{tenant_id}:{user_id}",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
detail=f"role={member.get('role')};expires_at={member.get('expires_at')}",
|
||||||
|
)
|
||||||
|
return ok(member)
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{tenant_id}/members/{user_id}")
|
||||||
|
def update_member(
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
payload: dict[str, Any] = Body(...),
|
||||||
|
current_user: dict = Depends(require_tenant_admin),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
if not is_admin(current_user) and payload.get("role") == "owner":
|
||||||
|
raise fail(403, "only platform administrator can grant owner role")
|
||||||
|
member = get_platform_store().update_tenant_member(tenant_id, user_id, payload)
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.member.update",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant_member",
|
||||||
|
target_id=f"{tenant_id}:{user_id}",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
detail=f"fields={','.join(payload.keys())}",
|
||||||
|
)
|
||||||
|
return ok(member)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant member not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{tenant_id}/members/{user_id}")
|
||||||
|
def remove_member(tenant_id: str, user_id: str, current_user: dict = Depends(require_tenant_admin)) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
get_platform_store().remove_tenant_member(tenant_id, user_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant member not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.member.remove",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant_member",
|
||||||
|
target_id=f"{tenant_id}:{user_id}",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
)
|
||||||
|
return ok({"tenant_id": tenant_id, "user_id": user_id, "removed": True})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{tenant_id}/members/{user_id}/accept")
|
||||||
|
def accept_invitation(
|
||||||
|
tenant_id: str,
|
||||||
|
user_id: str,
|
||||||
|
current_user: dict = Depends(get_current_user),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not is_admin(current_user) and str(current_user.get("id") or "") != user_id:
|
||||||
|
raise fail(403, "only the invited user can accept this invitation")
|
||||||
|
try:
|
||||||
|
member = get_platform_store().accept_tenant_invitation(tenant_id, user_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "tenant invitation not found")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
|
get_platform_store().record_audit(
|
||||||
|
action="tenant.member.accept",
|
||||||
|
actor_id=current_user.get("id"),
|
||||||
|
target_type="tenant_member",
|
||||||
|
target_id=f"{tenant_id}:{user_id}",
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
)
|
||||||
|
return ok(member)
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.core.logging import get_logger
|
from app.core.logging import get_logger
|
||||||
|
from app.db.platform_store import get_platform_store
|
||||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||||
|
|
||||||
|
|
||||||
@@ -13,21 +15,45 @@ logger = get_logger(__name__)
|
|||||||
async def run_compute_poller() -> None:
|
async def run_compute_poller() -> None:
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
|
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
|
||||||
logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode})
|
logger.info("计算轮询已禁用", extra={"compute_mode": settings.compute_mode})
|
||||||
return
|
return
|
||||||
|
|
||||||
interval = max(3, settings.compute_poll_interval_seconds)
|
interval = max(3, settings.compute_poll_interval_seconds)
|
||||||
logger.info("compute poller started", extra={"interval_seconds": interval})
|
logger.info("计算轮询已启动", extra={"interval_seconds": interval})
|
||||||
|
# PlatformStore may run additive schema checks against a remote PostgreSQL
|
||||||
|
# server on first use. Keep that startup work off the Uvicorn event loop so
|
||||||
|
# health checks and normal API requests can still respond while the DB is
|
||||||
|
# unavailable or slow.
|
||||||
|
store = None
|
||||||
|
last_failure_signature = ""
|
||||||
|
last_failure_logged_at = 0.0
|
||||||
|
await asyncio.sleep(1)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
result = await poll_compute_jobs_once()
|
if store is None:
|
||||||
|
store = await asyncio.to_thread(get_platform_store)
|
||||||
|
result = await poll_compute_jobs_once(store)
|
||||||
if result["failed"]:
|
if result["failed"]:
|
||||||
logger.warning("compute polling reported failures", extra={"result": result})
|
signature = "|".join(sorted({
|
||||||
|
str(item.get("error") or "")[:120]
|
||||||
|
for item in result["failed"]
|
||||||
|
}))
|
||||||
|
now = time.monotonic()
|
||||||
|
if signature != last_failure_signature or now - last_failure_logged_at >= 300:
|
||||||
|
logger.warning(
|
||||||
|
"计算轮询报告失败任务 count=%d first_error=%s",
|
||||||
|
len(result["failed"]),
|
||||||
|
signature[:500],
|
||||||
|
)
|
||||||
|
last_failure_signature = signature
|
||||||
|
last_failure_logged_at = now
|
||||||
elif result["synced"]:
|
elif result["synced"]:
|
||||||
logger.debug("compute jobs synchronized", extra={"result": result})
|
logger.debug("计算任务状态已同步", extra={"result": result})
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
logger.info("compute poller stopped")
|
logger.info("计算轮询已停止")
|
||||||
raise
|
raise
|
||||||
except Exception as exc: # noqa: BLE001 - keep background polling alive
|
except Exception as exc: # noqa: BLE001 - keep background polling alive
|
||||||
logger.exception("compute poller failed", extra={"error": str(exc)})
|
logger.exception("计算轮询执行失败", extra={"error": str(exc)})
|
||||||
|
if "store" in locals() and isinstance(exc, (ConnectionError, TimeoutError)):
|
||||||
|
store = None
|
||||||
await asyncio.sleep(interval)
|
await asyncio.sleep(interval)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ alembic>=1.13.1
|
|||||||
redis>=5.0.4
|
redis>=5.0.4
|
||||||
httpx>=0.27.0
|
httpx>=0.27.0
|
||||||
minio>=7.2.7
|
minio>=7.2.7
|
||||||
|
urllib3>=2.0.7
|
||||||
PyJWT>=2.8.0
|
PyJWT>=2.8.0
|
||||||
passlib[bcrypt]>=1.7.4
|
passlib[bcrypt]>=1.7.4
|
||||||
python-dotenv>=1.0.1
|
python-dotenv>=1.0.1
|
||||||
|
|||||||
266
backend/test_results.json
Normal file
266
backend/test_results.json
Normal file
@@ -0,0 +1,266 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "health",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/health",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(4 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "system-info",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/system-info",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(7 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "me",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/me",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(9 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "dashboard/overview",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dashboard/overview",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(6 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "dashboard/stats",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dashboard/stats",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(9 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "users-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "9 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "users-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(9 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "users-change-password",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/users/me/password",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-manage-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "5 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-manage-local",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage/local-models",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-manage-trained",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage/trained-models",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-manage-export-jobs",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage/export-jobs",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "11 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-manage-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/model-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "dataset-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dataset-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "29 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "dataset-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/dataset-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "fine-tune-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/fine-tune",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "9 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "fine-tune-check-name",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/fine-tune/check-name?name=test_task",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "fine-tune-preflight",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/fine-tune/preflight",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(4 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-eval-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-eval",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "4 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "dimension-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dimension",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "18 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-compare-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-compare",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "6 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-chat-local-status",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-chat/local/status",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(8 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "data-process-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/data-process",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(4 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-nodes",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/nodes",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "2 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-gpus",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/gpus",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "2 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-queue",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/queue",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "0 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "log-files",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/log-files",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "2 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "training-log-files",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/training-log-files",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "9 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "web-log",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/web-log",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(3 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "data-convert-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/data-convert",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(2 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-404",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/nonexistent-endpoint",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-unauthorized",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "10 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "viewer-login",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/login",
|
||||||
|
"status": "SKIP",
|
||||||
|
"message": "viewer user not found",
|
||||||
|
"data_desc": "-"
|
||||||
|
}
|
||||||
|
]
|
||||||
258
backend/test_results_advanced.json
Normal file
258
backend/test_results_advanced.json
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "crud-model-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/model-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-model-get-by-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage/m_dcebe627d644",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-model-update",
|
||||||
|
"method": "PUT",
|
||||||
|
"url": "/model-manage/m_dcebe627d644",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-model-purpose",
|
||||||
|
"method": "PUT",
|
||||||
|
"url": "/model-manage/m_dcebe627d644/purpose",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-model-delete",
|
||||||
|
"method": "DELETE",
|
||||||
|
"url": "/model-manage/m_dcebe627d644",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dataset-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/dataset-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dataset-get-by-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dataset-manage/ds_b8dd915d5e09",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(28 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dataset-update",
|
||||||
|
"method": "PUT",
|
||||||
|
"url": "/dataset-manage/ds_b8dd915d5e09",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(27 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dataset-delete",
|
||||||
|
"method": "DELETE",
|
||||||
|
"url": "/dataset-manage/ds_b8dd915d5e09",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-user-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(9 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-user-list",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "11 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-user-update",
|
||||||
|
"method": "PUT",
|
||||||
|
"url": "/users/u_eb94ee60769e",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(9 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-user-reset-pwd",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/users/u_eb94ee60769e/reset-password",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-user-delete",
|
||||||
|
"method": "DELETE",
|
||||||
|
"url": "/users/u_eb94ee60769e",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(2 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-invalid-model-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-manage/nonexistent_id_12345",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-invalid-dataset-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dataset-manage/nonexistent_id_12345",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-invalid-finetune-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/fine-tune/nonexistent_id_12345",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-invalid-eval-id",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/model-eval/nonexistent_id_12345",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-duplicate-login",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/login",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "error-missing-fields",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/model-manage",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(17 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "auth-no-token-users",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "10 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "auth-no-token-finetune",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/fine-tune",
|
||||||
|
"status": "FAIL(code=-1)",
|
||||||
|
"message": "",
|
||||||
|
"data_desc": "null"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "auth-invalid-token",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "10 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "auth-empty-token",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/users",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "10 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dimension-create",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/dimension",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(6 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dimension-get",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/dimension/dim_12124ed44bbe",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(6 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dimension-update",
|
||||||
|
"method": "PUT",
|
||||||
|
"url": "/dimension/dim_12124ed44bbe",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(6 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crud-dimension-delete",
|
||||||
|
"method": "DELETE",
|
||||||
|
"url": "/dimension/dim_12124ed44bbe",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(1 keys)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-nodes-detail",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/nodes",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "2 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-nodes-list2",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/nodes",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "2 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-node-replicas",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/nodes/node_1499a71b4871/replicas",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "9 items"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "compute-node-engines",
|
||||||
|
"method": "GET",
|
||||||
|
"url": "/compute/nodes/node_1499a71b4871/engines",
|
||||||
|
"status": "PASS",
|
||||||
|
"message": "ok",
|
||||||
|
"data_desc": "obj(2 keys)"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -18,10 +18,12 @@ from pypdf import PdfWriter
|
|||||||
|
|
||||||
from app.modules.data_process.algorithms import (
|
from app.modules.data_process.algorithms import (
|
||||||
PdfPageText,
|
PdfPageText,
|
||||||
|
LayoutRepeatedBlock,
|
||||||
content_quality_flags,
|
content_quality_flags,
|
||||||
desensitize_pii,
|
desensitize_pii,
|
||||||
desensitize_structured_record,
|
desensitize_structured_record,
|
||||||
detect_document_structure,
|
detect_document_structure,
|
||||||
|
detect_layout_repeated_blocks,
|
||||||
detect_pdf_document_noise,
|
detect_pdf_document_noise,
|
||||||
detect_text_format,
|
detect_text_format,
|
||||||
extract_pdf_page_texts,
|
extract_pdf_page_texts,
|
||||||
@@ -35,6 +37,7 @@ from app.modules.data_process.algorithms import (
|
|||||||
preprocess_structured_records_with_lineage,
|
preprocess_structured_records_with_lineage,
|
||||||
record_fingerprint,
|
record_fingerprint,
|
||||||
remove_document_noise,
|
remove_document_noise,
|
||||||
|
remove_layout_repeated_blocks,
|
||||||
score_quality,
|
score_quality,
|
||||||
stable_split,
|
stable_split,
|
||||||
stable_split_assignments,
|
stable_split_assignments,
|
||||||
@@ -1143,3 +1146,95 @@ def test_generate_standard_records_rejects_out_of_range_count(
|
|||||||
) -> None:
|
) -> None:
|
||||||
with pytest.raises(ValueError, match=r"\[1, 50\]"):
|
with pytest.raises(ValueError, match=r"\[1, 50\]"):
|
||||||
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)
|
generate_standard_records([], qa_pairs_per_item=qa_pairs_per_item)
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_repeated_blocks_detects_repeating_header_table() -> None:
|
||||||
|
"""跨页重复的页眉表格应被识别为重复块(出现 ≥ max(3, ceil(pages*0.3)) 次)。"""
|
||||||
|
|
||||||
|
header_table = (
|
||||||
|
"| 文件编码 | 2024 |\n"
|
||||||
|
"| - | - |\n"
|
||||||
|
"| 秘密等级 | 商密【中】 |\n"
|
||||||
|
"| 现行版本 | 1.0 |\n"
|
||||||
|
"| 页次 | 第1页 共47页 |\n"
|
||||||
|
)
|
||||||
|
body_table = (
|
||||||
|
"| 支出项目 | 税务票据要求 |\n"
|
||||||
|
"| - | - |\n"
|
||||||
|
"| 工资奖金 | 无 |\n"
|
||||||
|
"| 交通费 | 车票 |\n"
|
||||||
|
)
|
||||||
|
doc_items: list[tuple[str, object, str]] = []
|
||||||
|
for index in range(20):
|
||||||
|
# 20 个页面里 18 个有页眉表,2 个有正文表
|
||||||
|
text = header_table if index < 18 else body_table
|
||||||
|
doc_items.append(("table", index, text))
|
||||||
|
|
||||||
|
blocks = detect_layout_repeated_blocks(doc_items, page_count=20)
|
||||||
|
|
||||||
|
# 仅页眉表对应的标签序列应被识别
|
||||||
|
assert len(blocks) == 1
|
||||||
|
assert "文件编码" in blocks[0].labels
|
||||||
|
assert "秘密等级" in blocks[0].labels
|
||||||
|
assert blocks[0].occurrences == 18
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_repeated_blocks_short_documents_skip() -> None:
|
||||||
|
"""短文档(< 3 页)不推断重复块。"""
|
||||||
|
|
||||||
|
doc_items: list[tuple[str, object, str]] = [
|
||||||
|
("table", 0, "| 文件编码 | 2024 |\n| - | - |\n"),
|
||||||
|
("table", 1, "| 文件编码 | 2024 |\n| - | - |\n"),
|
||||||
|
]
|
||||||
|
assert detect_layout_repeated_blocks(doc_items, page_count=2) == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_remove_layout_repeated_blocks_strips_label_rows_and_separators() -> None:
|
||||||
|
"""剔除首列命中重复标签集的行,及其后的表格分隔行。"""
|
||||||
|
|
||||||
|
blocks = [
|
||||||
|
LayoutRepeatedBlock(
|
||||||
|
labels=("文件编码", "秘密等级", "现行版本", "页次"),
|
||||||
|
occurrences=18,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
chunk = (
|
||||||
|
"报销指引\n"
|
||||||
|
"| 文件编码 | 2024 |\n"
|
||||||
|
"| - | - |\n"
|
||||||
|
"| 秘密等级 | 商密【中】 |\n"
|
||||||
|
"| 现行版本 | 1.0 |\n"
|
||||||
|
"| 页次 | 第3页 共47页 |\n"
|
||||||
|
"正文第一段\n"
|
||||||
|
"| 支出项目 | 税务票据要求 |\n"
|
||||||
|
"| - | - |\n"
|
||||||
|
"| 工资奖金 | 无 |\n"
|
||||||
|
)
|
||||||
|
cleaned = remove_layout_repeated_blocks(chunk, blocks)
|
||||||
|
|
||||||
|
# 重复标签行 + 紧随其后的表格分隔行被剔除;正文与内容表格保留
|
||||||
|
assert "文件编码" not in cleaned
|
||||||
|
assert "秘密等级" not in cleaned
|
||||||
|
assert "现行版本" not in cleaned
|
||||||
|
assert "页次" not in cleaned
|
||||||
|
# 第一组表格的 | - | - | 在 文件编码 行之后被一并删除
|
||||||
|
# (但 cleaned 中可能还有第二个表格的分隔行)
|
||||||
|
assert cleaned.count("| - | - |") == 1
|
||||||
|
assert "报销指引" in cleaned
|
||||||
|
assert "正文第一段" in cleaned
|
||||||
|
assert "支出项目" in cleaned
|
||||||
|
assert "工资奖金" in cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def test_remove_layout_repeated_blocks_returns_text_unchanged_when_no_blocks() -> None:
|
||||||
|
"""无重复块时直接返回原文。"""
|
||||||
|
|
||||||
|
chunk = "| 文件编码 | 2024 |\n| 正文 |\n"
|
||||||
|
assert remove_layout_repeated_blocks(chunk, []) == chunk
|
||||||
|
assert (
|
||||||
|
remove_layout_repeated_blocks(
|
||||||
|
"",
|
||||||
|
[LayoutRepeatedBlock(labels=("x",), occurrences=5)],
|
||||||
|
)
|
||||||
|
== ""
|
||||||
|
)
|
||||||
|
|||||||
30
backend/tests/test_permission_security.py
Normal file
30
backend/tests/test_permission_security.py
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.api.v1.endpoints.platform import _public_model
|
||||||
|
from app.core.audit import _safe_exception_reason
|
||||||
|
from app.core.auth import RESOURCE_ACTIONS, RESOURCE_ACTION_ALIASES
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_model_never_exposes_provider_credentials() -> None:
|
||||||
|
result = _public_model({
|
||||||
|
"id": "m_1",
|
||||||
|
"name": "online",
|
||||||
|
"api_url": "https://example.invalid/v1",
|
||||||
|
"api_key": "secret-value",
|
||||||
|
})
|
||||||
|
|
||||||
|
assert "api_key" not in result
|
||||||
|
assert result["api_key_configured"] is True
|
||||||
|
assert result["name"] == "online"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resource_action_registry_keeps_export_separate_from_module_permissions() -> None:
|
||||||
|
assert "download" in RESOURCE_ACTIONS
|
||||||
|
assert "execute" in RESOURCE_ACTIONS
|
||||||
|
assert RESOURCE_ACTION_ALIASES["export"] == "download"
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_exception_reason_masks_credentials() -> None:
|
||||||
|
reason = _safe_exception_reason(ValueError("api_key=secret-value"))
|
||||||
|
assert "secret-value" not in reason
|
||||||
|
assert "***" in reason
|
||||||
26
backend/tests/test_storage_security.py
Normal file
26
backend/tests/test_storage_security.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.api.v1.endpoints.platform import _validate_storage_key
|
||||||
|
|
||||||
|
|
||||||
|
def test_storage_key_is_scoped_to_resource_version() -> None:
|
||||||
|
assert (
|
||||||
|
_validate_storage_key("dataset", "ds_123", "v1", None, "train.jsonl")
|
||||||
|
== "datasets/ds_123/versions/v1/train.jsonl"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"object_key",
|
||||||
|
[
|
||||||
|
"models/other/versions/v1/model.bin",
|
||||||
|
"datasets/ds_123/versions/v1/../secret.bin",
|
||||||
|
"datasets/ds_123/versions/v1/../../secret.bin",
|
||||||
|
"/datasets/ds_123/versions/v1/model.bin",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_storage_key_rejects_escape_or_cross_resource_paths(object_key: str) -> None:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
_validate_storage_key("dataset", "ds_123", "v1", object_key, None)
|
||||||
@@ -9,6 +9,7 @@ import shutil
|
|||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -70,6 +71,76 @@ def create_app() -> FastAPI:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def cache_max_bytes() -> int:
|
||||||
|
return max(0, _int_env("COMPUTE_CACHE_MAX_BYTES", 0))
|
||||||
|
|
||||||
|
def cache_ttl_seconds() -> int:
|
||||||
|
return max(0, _int_env("COMPUTE_CACHE_TTL_SECONDS", 0))
|
||||||
|
|
||||||
|
def cache_meta_path(target: Path) -> Path:
|
||||||
|
return target.with_name(f".{target.name}.cache-meta.json")
|
||||||
|
|
||||||
|
def cache_protected_until(target: Path) -> float:
|
||||||
|
try:
|
||||||
|
value = json.loads(cache_meta_path(target).read_text(encoding="utf-8")).get("protected_until")
|
||||||
|
return float(value or 0)
|
||||||
|
except (OSError, TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
def write_cache_meta(target: Path, resource_id: str, version_id: str, protected_until: float) -> None:
|
||||||
|
meta = cache_meta_path(target)
|
||||||
|
meta.write_text(json.dumps({
|
||||||
|
"resource_id": resource_id,
|
||||||
|
"version_id": version_id,
|
||||||
|
"protected_until": protected_until,
|
||||||
|
"last_accessed_at": now(),
|
||||||
|
}), encoding="utf-8")
|
||||||
|
|
||||||
|
def cache_usage(cache_root: Path) -> int:
|
||||||
|
total = 0
|
||||||
|
if not cache_root.exists():
|
||||||
|
return 0
|
||||||
|
for item in cache_root.rglob("*"):
|
||||||
|
try:
|
||||||
|
if item.is_file() and not item.name.endswith(".part"):
|
||||||
|
total += item.stat().st_size
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return total
|
||||||
|
|
||||||
|
def ensure_cache_capacity(cache_root: Path, required_bytes: int, protected: Path) -> None:
|
||||||
|
limit = cache_max_bytes()
|
||||||
|
if not limit or required_bytes <= 0:
|
||||||
|
return
|
||||||
|
usage = cache_usage(cache_root)
|
||||||
|
if usage + required_bytes <= limit:
|
||||||
|
return
|
||||||
|
candidates: list[tuple[float, int, Path]] = []
|
||||||
|
for item in cache_root.rglob("*"):
|
||||||
|
try:
|
||||||
|
if (
|
||||||
|
item.is_file()
|
||||||
|
and not item.name.endswith(".part")
|
||||||
|
and not item.name.endswith(".cache-meta.json")
|
||||||
|
and item.resolve() != protected.resolve()
|
||||||
|
and cache_protected_until(item) <= now()
|
||||||
|
):
|
||||||
|
stat = item.stat()
|
||||||
|
candidates.append((stat.st_atime, stat.st_size, item))
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
candidates.sort(key=lambda value: value[0])
|
||||||
|
for _, size, item in candidates:
|
||||||
|
try:
|
||||||
|
item.unlink(missing_ok=True)
|
||||||
|
usage -= size
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if usage + required_bytes <= limit:
|
||||||
|
break
|
||||||
|
if usage + required_bytes > limit:
|
||||||
|
raise HTTPException(status_code=507, detail="compute cache capacity is insufficient")
|
||||||
|
|
||||||
def _llama_factory_version() -> str:
|
def _llama_factory_version() -> str:
|
||||||
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
|
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
|
||||||
try:
|
try:
|
||||||
@@ -662,13 +733,26 @@ def create_app() -> FastAPI:
|
|||||||
raise HTTPException(status_code=400, detail="upload_url is required")
|
raise HTTPException(status_code=400, detail="upload_url is required")
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
byte_size = 0
|
byte_size = 0
|
||||||
|
content_length = source.stat().st_size
|
||||||
|
|
||||||
|
async def file_chunks():
|
||||||
|
nonlocal byte_size
|
||||||
|
with source.open("rb") as handle:
|
||||||
|
while chunk := handle.read(1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
byte_size += len(chunk)
|
||||||
|
yield chunk
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
|
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
|
||||||
with source.open("rb") as handle:
|
response = await client.put(
|
||||||
content = handle.read()
|
upload_url,
|
||||||
digest.update(content)
|
content=file_chunks(),
|
||||||
byte_size = len(content)
|
headers={
|
||||||
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
|
"Content-Type": str(payload.get("content_type") or "application/octet-stream"),
|
||||||
|
"Content-Length": str(content_length),
|
||||||
|
},
|
||||||
|
)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
|
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
|
||||||
@@ -916,6 +1000,14 @@ def create_app() -> FastAPI:
|
|||||||
temp_target = target.with_name(f".{target.name}.part")
|
temp_target = target.with_name(f".{target.name}.part")
|
||||||
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
|
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
|
||||||
expected_size = int(payload.get("byte_size") or 0)
|
expected_size = int(payload.get("byte_size") or 0)
|
||||||
|
requested_protected_until = str(payload.get("protected_until") or "")
|
||||||
|
try:
|
||||||
|
protected_until = float(requested_protected_until)
|
||||||
|
except ValueError:
|
||||||
|
try:
|
||||||
|
protected_until = datetime.fromisoformat(requested_protected_until.replace("Z", "+00:00")).timestamp()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
protected_until = now() + cache_ttl_seconds() if cache_ttl_seconds() else 0.0
|
||||||
lock = cache_locks.setdefault(str(target), asyncio.Lock())
|
lock = cache_locks.setdefault(str(target), asyncio.Lock())
|
||||||
async with lock:
|
async with lock:
|
||||||
if target.is_file() and expected_checksum:
|
if target.is_file() and expected_checksum:
|
||||||
@@ -924,7 +1016,10 @@ def create_app() -> FastAPI:
|
|||||||
while chunk := existing.read(1024 * 1024):
|
while chunk := existing.read(1024 * 1024):
|
||||||
existing_digest.update(chunk)
|
existing_digest.update(chunk)
|
||||||
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
|
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
|
||||||
|
os.utime(target, None)
|
||||||
|
write_cache_meta(target, resource_id, version_id, protected_until)
|
||||||
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
|
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
|
||||||
|
ensure_cache_capacity(cache_root / "resources", expected_size, target)
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
byte_size = 0
|
byte_size = 0
|
||||||
try:
|
try:
|
||||||
@@ -956,6 +1051,7 @@ def create_app() -> FastAPI:
|
|||||||
temp_target.unlink(missing_ok=True)
|
temp_target.unlink(missing_ok=True)
|
||||||
raise HTTPException(status_code=502, detail="cache byte size mismatch")
|
raise HTTPException(status_code=502, detail="cache byte size mismatch")
|
||||||
temp_target.replace(target)
|
temp_target.replace(target)
|
||||||
|
write_cache_meta(target, resource_id, version_id, protected_until)
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -975,12 +1071,20 @@ def create_app() -> FastAPI:
|
|||||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||||
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
|
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
|
||||||
target = cache_root / "resources" / resource_id / version_id / "resource"
|
target = cache_root / "resources" / resource_id / version_id / "resource"
|
||||||
|
ttl = cache_ttl_seconds()
|
||||||
|
if target.is_file() and ttl and target.stat().st_atime + ttl < now() and cache_protected_until(target) <= now():
|
||||||
|
target.unlink(missing_ok=True)
|
||||||
|
cache_meta_path(target).unlink(missing_ok=True)
|
||||||
return {
|
return {
|
||||||
"resource_id": resource_id,
|
"resource_id": resource_id,
|
||||||
"version_id": version_id,
|
"version_id": version_id,
|
||||||
"status": "ready" if target.is_file() else "missing",
|
"status": "ready" if target.is_file() else "missing",
|
||||||
"local_path": str(target),
|
"local_path": str(target),
|
||||||
"byte_size": target.stat().st_size if target.is_file() else 0,
|
"byte_size": target.stat().st_size if target.is_file() else 0,
|
||||||
|
"cache_usage_bytes": cache_usage(cache_root / "resources"),
|
||||||
|
"cache_max_bytes": cache_max_bytes(),
|
||||||
|
"cache_ttl_seconds": ttl,
|
||||||
|
"protected_until": cache_protected_until(target) if target.is_file() else 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
@app.delete(f"{route_prefix}/compute/cache")
|
@app.delete(f"{route_prefix}/compute/cache")
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
|||||||
"--save_steps",
|
"--save_steps",
|
||||||
str(config.get("save_steps", 50)),
|
str(config.get("save_steps", 50)),
|
||||||
"--logging_steps",
|
"--logging_steps",
|
||||||
str(config.get("logging_steps", 10)),
|
str(max(1, int(config.get("logging_steps", 1) or 1))),
|
||||||
"--overwrite_output_dir",
|
"--overwrite_output_dir",
|
||||||
"true",
|
"true",
|
||||||
"--plot_loss",
|
"--plot_loss",
|
||||||
@@ -336,6 +336,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
|||||||
_optional_arg(config, command, "--val_size", "val_size")
|
_optional_arg(config, command, "--val_size", "val_size")
|
||||||
_optional_arg(config, command, "--max_samples", "max_samples")
|
_optional_arg(config, command, "--max_samples", "max_samples")
|
||||||
_optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers")
|
_optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers")
|
||||||
|
_optional_arg(config, command, "--resume_from_checkpoint", "resume_from_checkpoint")
|
||||||
_optional_bool_arg(config, command, "--fp16", "fp16")
|
_optional_bool_arg(config, command, "--fp16", "fp16")
|
||||||
_optional_bool_arg(config, command, "--bf16", "bf16")
|
_optional_bool_arg(config, command, "--bf16", "bf16")
|
||||||
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import math
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
from difflib import SequenceMatcher
|
from difflib import SequenceMatcher
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -94,15 +95,13 @@ def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4)
|
|||||||
try:
|
try:
|
||||||
from sacrebleu.metrics import BLEU
|
from sacrebleu.metrics import BLEU
|
||||||
except ImportError:
|
except ImportError:
|
||||||
return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
|
return _metric_record(None, len(predictions), "sacrebleu 未安装", available=False)
|
||||||
|
if not predictions:
|
||||||
|
return _metric_record(0, 0)
|
||||||
bleu = BLEU(max_ngram_order=ngram)
|
bleu = BLEU(max_ngram_order=ngram)
|
||||||
# sacrebleu expects list-of-strings; we have one reference per prediction
|
# sacrebleu expects list-of-strings; we have one reference per prediction
|
||||||
score = bleu.corpus_score(predictions, [references])
|
score = bleu.corpus_score(predictions, [references])
|
||||||
return {
|
return _metric_record(score.score, len(predictions), bleu=round(score.score, 2))
|
||||||
"enabled": True,
|
|
||||||
"score": round(score.score, 2),
|
|
||||||
"bleu": round(score.score, 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
|
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
|
||||||
@@ -110,20 +109,27 @@ def _compute_rouge(references: list[str], predictions: list[str], methods: list[
|
|||||||
try:
|
try:
|
||||||
from rouge_score import rouge_scorer
|
from rouge_score import rouge_scorer
|
||||||
except ImportError:
|
except ImportError:
|
||||||
return {"enabled": False, "error": "rouge-score not installed", "score": 0}
|
rouge_scorer = None
|
||||||
methods = methods or ["rouge1", "rouge2", "rougeL"]
|
methods = methods or ["rouge1", "rouge2", "rougeL"]
|
||||||
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
|
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
|
||||||
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
|
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
|
||||||
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
|
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
|
||||||
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
|
if rouge_scorer is None:
|
||||||
|
values = [_pair_rouge(ref, pred) for ref, pred in zip(references, predictions)]
|
||||||
|
avg = {key: round(sum(item.get(key, 0.0) for item in values) / max(len(values), 1), 4) for key in methods}
|
||||||
|
else:
|
||||||
|
class _Tokenizer:
|
||||||
|
def tokenize(self, value: str) -> list[str]:
|
||||||
|
return value.split()
|
||||||
|
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=False, tokenizer=_Tokenizer())
|
||||||
totals: dict[str, float] = {}
|
totals: dict[str, float] = {}
|
||||||
n = max(len(predictions), 1)
|
n = max(len(predictions), 1)
|
||||||
for ref, pred in zip(references, predictions):
|
for ref, pred in zip(references, predictions):
|
||||||
result = scorer.score(ref, pred)
|
result = scorer.score(_rouge_tokens(ref), _rouge_tokens(pred))
|
||||||
for key in methods:
|
for key in methods:
|
||||||
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
||||||
avg = {k: round(v / n, 4) for k, v in totals.items()}
|
avg = {key: round(value / n, 4) for key, value in totals.items()}
|
||||||
return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
|
return _metric_record(avg.get("rougeL", avg.get("rouge1", 0)) * 100, len(predictions), **avg)
|
||||||
|
|
||||||
|
|
||||||
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||||
@@ -132,7 +138,9 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
|||||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||||
from sklearn.metrics.pairwise import cosine_similarity
|
from sklearn.metrics.pairwise import cosine_similarity
|
||||||
except ImportError:
|
except ImportError:
|
||||||
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
|
return _metric_record(None, len(predictions), "scikit-learn 未安装", available=False)
|
||||||
|
if not predictions:
|
||||||
|
return _metric_record(0, 0)
|
||||||
try:
|
try:
|
||||||
vectorizer = TfidfVectorizer()
|
vectorizer = TfidfVectorizer()
|
||||||
tfidf = vectorizer.fit_transform(references + predictions)
|
tfidf = vectorizer.fit_transform(references + predictions)
|
||||||
@@ -140,41 +148,169 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
|||||||
ref_vec = tfidf[:n]
|
ref_vec = tfidf[:n]
|
||||||
pred_vec = tfidf[n:]
|
pred_vec = tfidf[n:]
|
||||||
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
|
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
|
||||||
return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
|
return _metric_record(float(sims.mean()) * 100, n)
|
||||||
except ValueError:
|
except ValueError as exc:
|
||||||
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
|
return _metric_record(0, len(predictions), str(exc))
|
||||||
|
|
||||||
|
|
||||||
def _normalize_text(value: str) -> str:
|
def _normalize_text(value: str) -> str:
|
||||||
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
||||||
|
|
||||||
|
|
||||||
|
def _metric_record(score: float | None, sample_count: int, error: str = "", available: bool = True, **extra: Any) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"available": available,
|
||||||
|
"score": None if score is None else round(max(0.0, min(100.0, float(score))), 2),
|
||||||
|
"max_score": 100,
|
||||||
|
"unit": "percent",
|
||||||
|
"sample_count": sample_count,
|
||||||
|
"error": error,
|
||||||
|
**extra,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _metric_dimension_summary(metrics: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
labels = {
|
||||||
|
"bleu": "BLEU",
|
||||||
|
"rouge": "ROUGE-L",
|
||||||
|
"cosine": "Cosine 相似度",
|
||||||
|
"exact_match": "精确匹配",
|
||||||
|
"text_similarity": "文本相似度",
|
||||||
|
}
|
||||||
|
result: list[dict[str, Any]] = []
|
||||||
|
for name, item in metrics.items():
|
||||||
|
if not isinstance(item, dict) or item.get("score") is None:
|
||||||
|
continue
|
||||||
|
result.append({
|
||||||
|
"name": labels.get(name, name),
|
||||||
|
"score": float(item.get("score") or 0),
|
||||||
|
"max_score": float(item.get("max_score") or 100),
|
||||||
|
"pass_rate": float(item.get("score") or 0),
|
||||||
|
"sample_count": int(item.get("sample_count") or 0),
|
||||||
|
"available": item.get("available", True),
|
||||||
|
"error": item.get("error", ""),
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _rouge_tokens(text: str) -> str:
|
||||||
|
text = str(text or "").strip().lower()
|
||||||
|
tokens: list[str] = []
|
||||||
|
for segment in re.findall(r"[一-鿿]+|[^\s一-鿿]+", text):
|
||||||
|
tokens.extend(segment if "一" <= segment[0] <= "鿿" else [segment])
|
||||||
|
return " ".join(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
def _pair_rouge(reference: str, prediction: str) -> dict[str, float]:
|
||||||
|
try:
|
||||||
|
from rouge_score import rouge_scorer
|
||||||
|
except ImportError:
|
||||||
|
reference_tokens = _rouge_tokens(reference).split()
|
||||||
|
prediction_tokens = _rouge_tokens(prediction).split()
|
||||||
|
if not reference_tokens or not prediction_tokens:
|
||||||
|
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
|
||||||
|
|
||||||
|
def f1(overlap: int, reference_count: int, prediction_count: int) -> float:
|
||||||
|
if not reference_count or not prediction_count or not overlap:
|
||||||
|
return 0.0
|
||||||
|
precision = overlap / prediction_count
|
||||||
|
recall = overlap / reference_count
|
||||||
|
return 2 * precision * recall / (precision + recall)
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
reference_unigrams = Counter(reference_tokens)
|
||||||
|
prediction_unigrams = Counter(prediction_tokens)
|
||||||
|
unigram_overlap = sum((reference_unigrams & prediction_unigrams).values())
|
||||||
|
reference_bigrams = Counter(zip(reference_tokens, reference_tokens[1:]))
|
||||||
|
prediction_bigrams = Counter(zip(prediction_tokens, prediction_tokens[1:]))
|
||||||
|
bigram_overlap = sum((reference_bigrams & prediction_bigrams).values())
|
||||||
|
matrix = [[0] * (len(prediction_tokens) + 1) for _ in range(len(reference_tokens) + 1)]
|
||||||
|
for row, reference_token in enumerate(reference_tokens, start=1):
|
||||||
|
for column, prediction_token in enumerate(prediction_tokens, start=1):
|
||||||
|
matrix[row][column] = matrix[row - 1][column - 1] + 1 if reference_token == prediction_token else max(matrix[row - 1][column], matrix[row][column - 1])
|
||||||
|
return {
|
||||||
|
"rouge1": f1(unigram_overlap, len(reference_tokens), len(prediction_tokens)),
|
||||||
|
"rouge2": f1(bigram_overlap, max(len(reference_tokens) - 1, 0), max(len(prediction_tokens) - 1, 0)),
|
||||||
|
"rougeL": f1(matrix[-1][-1], len(reference_tokens), len(prediction_tokens)),
|
||||||
|
}
|
||||||
|
class _Tokenizer:
|
||||||
|
def tokenize(self, value: str) -> list[str]:
|
||||||
|
return value.split()
|
||||||
|
if not reference.strip() or not prediction.strip():
|
||||||
|
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
|
||||||
|
scorer = rouge_scorer.RougeScorer(("rouge1", "rouge2", "rougeL"), use_stemmer=False, tokenizer=_Tokenizer())
|
||||||
|
scores = scorer.score(_rouge_tokens(reference), _rouge_tokens(prediction))
|
||||||
|
return {key: float(value.fmeasure) for key, value in scores.items()}
|
||||||
|
|
||||||
|
|
||||||
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||||
total = len(predictions)
|
total = len(predictions)
|
||||||
if not total:
|
if not total:
|
||||||
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
|
return _metric_record(0, 0, matched=0, total=0)
|
||||||
matched = sum(
|
matched = sum(
|
||||||
1
|
1
|
||||||
for ref, pred in zip(references, predictions)
|
for ref, pred in zip(references, predictions)
|
||||||
if _normalize_text(ref) == _normalize_text(pred)
|
if _normalize_text(ref) == _normalize_text(pred)
|
||||||
)
|
)
|
||||||
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
|
return _metric_record(matched / total * 100, total, matched=matched, total=total)
|
||||||
|
|
||||||
|
|
||||||
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||||
if not predictions:
|
if not predictions:
|
||||||
return {"enabled": True, "score": 0}
|
return _metric_record(0, 0)
|
||||||
scores = [
|
scores = [
|
||||||
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
||||||
for ref, pred in zip(references, predictions)
|
for ref, pred in zip(references, predictions)
|
||||||
]
|
]
|
||||||
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
|
return _metric_record(sum(scores) / max(len(scores), 1) * 100, len(predictions))
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# LLM Judge
|
# LLM Judge
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _normalise_api_url(value: str) -> str:
|
||||||
|
url = str(value or "").strip().rstrip("/")
|
||||||
|
return url + "/chat/completions" if url.endswith("/v1") else url + "/v1/chat/completions"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_judge_reply(reply: str, score_min: float, score_max: float) -> tuple[float | None, dict[str, Any], str]:
|
||||||
|
payload: dict[str, Any] = {}
|
||||||
|
match = re.search(r"\{[\s\S]*\}", str(reply or ""))
|
||||||
|
if match:
|
||||||
|
try:
|
||||||
|
value = json.loads(match.group(0))
|
||||||
|
if isinstance(value, dict):
|
||||||
|
payload = value
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
reason = str(payload.get("综合评价") or payload.get("reason") or reply or "")
|
||||||
|
raw = payload.get("score", payload.get("overall_score"))
|
||||||
|
if raw is None:
|
||||||
|
matches = re.findall(r"(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)", reason, re.IGNORECASE)
|
||||||
|
raw = matches[-1] if matches else None
|
||||||
|
try:
|
||||||
|
numeric = float(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
dimensions = payload.get("dimensions") if isinstance(payload.get("dimensions"), dict) else {}
|
||||||
|
if not dimensions:
|
||||||
|
dimensions = {
|
||||||
|
key: value
|
||||||
|
for key, value in payload.items()
|
||||||
|
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||||
|
}
|
||||||
|
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
|
||||||
|
numeric = sum(values) / len(values) if values else None
|
||||||
|
if numeric is None:
|
||||||
|
return None, payload, reason
|
||||||
|
# The judge prompt uses the configured score range. Do not treat a
|
||||||
|
# decimal such as 0.5/5 as a 0.5/1 score, otherwise low scores are
|
||||||
|
# incorrectly inflated (0.5/5 would become 50 instead of 10).
|
||||||
|
normalized = (numeric - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||||
|
return max(0.0, min(100.0, normalized)), payload, reason
|
||||||
|
|
||||||
|
|
||||||
def _judge_sample(
|
def _judge_sample(
|
||||||
question: str,
|
question: str,
|
||||||
reference: str,
|
reference: str,
|
||||||
@@ -198,7 +334,7 @@ def _judge_sample(
|
|||||||
pass_threshold = float(config.get("pass_threshold", 3))
|
pass_threshold = float(config.get("pass_threshold", 3))
|
||||||
|
|
||||||
if not api_url or not eval_model:
|
if not api_url or not eval_model:
|
||||||
return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
|
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "未配置",
|
||||||
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
|
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
|
||||||
|
|
||||||
system_msg = (
|
system_msg = (
|
||||||
@@ -227,7 +363,7 @@ def _judge_sample(
|
|||||||
}).encode("utf-8")
|
}).encode("utf-8")
|
||||||
|
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
f"{api_url}/v1/chat/completions",
|
_normalise_api_url(api_url),
|
||||||
data=body,
|
data=body,
|
||||||
headers={
|
headers={
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
@@ -238,39 +374,54 @@ def _judge_sample(
|
|||||||
data = json.loads(resp.read().decode("utf-8"))
|
data = json.loads(resp.read().decode("utf-8"))
|
||||||
reply = data["choices"][0]["message"]["content"]
|
reply = data["choices"][0]["message"]["content"]
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"score": 0, "max_score": score_max, "passed": False,
|
return {"available": False, "score": None, "max_score": 100, "passed": False,
|
||||||
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
|
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
|
||||||
"error_type": "其他"}
|
"error_type": "其他"}
|
||||||
|
|
||||||
# Parse score from reply — look for patterns like "4分" or "Score: 4"
|
parsed: dict[str, Any] = {}
|
||||||
score = 0
|
match = re.search(r"\{[\s\S]*\}", reply)
|
||||||
import re
|
if match:
|
||||||
score_patterns = [
|
|
||||||
r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
|
|
||||||
r'(\d+(?:\.\d+)?)\s*分',
|
|
||||||
r'(\d+(?:\.\d+)?)\s*/\s*\d+',
|
|
||||||
]
|
|
||||||
for pat in score_patterns:
|
|
||||||
m = re.search(pat, reply, re.IGNORECASE)
|
|
||||||
if m:
|
|
||||||
try:
|
try:
|
||||||
score = float(m.group(1))
|
value = json.loads(match.group(0))
|
||||||
except ValueError:
|
if isinstance(value, dict):
|
||||||
continue
|
parsed = value
|
||||||
break
|
except json.JSONDecodeError:
|
||||||
score = max(score_min, min(score_max, score))
|
pass
|
||||||
passed = score >= pass_threshold
|
raw_score = parsed.get("score", parsed.get("overall_score"))
|
||||||
|
reason = str(parsed.get("综合评价") or parsed.get("reason") or reply)
|
||||||
|
if raw_score is None:
|
||||||
|
matches = re.findall(r'(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)', reason, re.IGNORECASE)
|
||||||
|
raw_score = matches[-1] if matches else None
|
||||||
|
try:
|
||||||
|
numeric_score = float(raw_score)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {}
|
||||||
|
if not dimensions:
|
||||||
|
dimensions = {
|
||||||
|
key: value
|
||||||
|
for key, value in parsed.items()
|
||||||
|
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||||
|
}
|
||||||
|
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
|
||||||
|
numeric_score = sum(values) / len(values) if values else None
|
||||||
|
if numeric_score is None:
|
||||||
|
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "错误",
|
||||||
|
"evaluation_reason": "评测模型未返回可解析分数:" + reason[:1800], "error_type": "其他"}
|
||||||
|
normalized_score = (numeric_score - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||||
|
normalized_score = max(0, min(100, normalized_score))
|
||||||
|
normalized_threshold = (pass_threshold - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||||
|
passed = normalized_score >= normalized_threshold
|
||||||
|
|
||||||
# Determine judgement label
|
# Determine judgement label
|
||||||
if score >= pass_threshold + 1:
|
if normalized_score >= min(100, normalized_threshold + 20):
|
||||||
judgement = "正确"
|
judgement = "正确"
|
||||||
elif score >= pass_threshold:
|
elif passed:
|
||||||
judgement = "部分正确"
|
judgement = "部分正确"
|
||||||
else:
|
else:
|
||||||
judgement = "错误"
|
judgement = "错误"
|
||||||
|
|
||||||
# Guess error type from reply
|
# Guess error type from reply
|
||||||
reply_lower = reply.lower()
|
reply_lower = (reply + " " + reason).lower()
|
||||||
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
|
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
|
||||||
error_type = "幻觉"
|
error_type = "幻觉"
|
||||||
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
|
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
|
||||||
@@ -282,14 +433,82 @@ def _judge_sample(
|
|||||||
else:
|
else:
|
||||||
error_type = "其他"
|
error_type = "其他"
|
||||||
|
|
||||||
|
parsed_dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {
|
||||||
|
key: value
|
||||||
|
for key, value in parsed.items()
|
||||||
|
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
"score": score,
|
"available": True,
|
||||||
"max_score": score_max,
|
"score": round(normalized_score, 2),
|
||||||
|
"max_score": 100,
|
||||||
|
"raw_score": numeric_score,
|
||||||
|
"raw_max_score": score_max,
|
||||||
"passed": passed,
|
"passed": passed,
|
||||||
"judgement": judgement,
|
"judgement": judgement,
|
||||||
"evaluation_reason": reply[:2000],
|
"evaluation_reason": reason[:2000],
|
||||||
"error_type": error_type,
|
"error_type": error_type,
|
||||||
|
"dimension_scores": [
|
||||||
|
{
|
||||||
|
"name": str(name),
|
||||||
|
"score": round(
|
||||||
|
max(
|
||||||
|
0,
|
||||||
|
min(
|
||||||
|
100,
|
||||||
|
(float(value) - score_min)
|
||||||
|
/ max(score_max - score_min, 1e-9)
|
||||||
|
* 100,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
"max_score": 100,
|
||||||
}
|
}
|
||||||
|
for name, value in parsed_dimensions.items()
|
||||||
|
if isinstance(value, (int, float))
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||||
|
temporary = path.with_suffix(path.suffix + ".part")
|
||||||
|
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_eval_progress(
|
||||||
|
output_dir: Path,
|
||||||
|
status: str,
|
||||||
|
stage: str,
|
||||||
|
total: int,
|
||||||
|
completed: int,
|
||||||
|
message: str = "",
|
||||||
|
current_index: int = 0,
|
||||||
|
) -> None:
|
||||||
|
_write_json(
|
||||||
|
output_dir / "eval_progress.json",
|
||||||
|
{
|
||||||
|
"status": status,
|
||||||
|
"stage": stage,
|
||||||
|
"total": total,
|
||||||
|
"completed": completed,
|
||||||
|
"percentage": round(completed / total * 100, 1) if total else 100,
|
||||||
|
"current_index": current_index,
|
||||||
|
"message": message,
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _deterministic_sample_score(reference: str, prediction: str) -> float:
|
||||||
|
values = [SequenceMatcher(None, _normalize_text(reference), _normalize_text(prediction)).ratio()]
|
||||||
|
if _normalize_text(reference) == _normalize_text(prediction):
|
||||||
|
values.append(1.0)
|
||||||
|
rouge = _pair_rouge(reference, prediction)
|
||||||
|
if rouge.get("rougeL") is not None:
|
||||||
|
values.append(float(rouge["rougeL"]))
|
||||||
|
return round(sum(values) / len(values) * 100, 2)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -312,11 +531,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
print(f"[eval] loading dataset: {dataset_path}")
|
print(f"[eval] loading dataset: {dataset_path}")
|
||||||
raw_samples = _load_dataset(dataset_path)
|
raw_samples = _load_dataset(dataset_path)
|
||||||
print(f"[eval] loaded {len(raw_samples)} samples")
|
print(f"[eval] loaded {len(raw_samples)} samples")
|
||||||
|
_write_eval_progress(output_dir, "running", "dataset", len(raw_samples), 0, f"已加载 {len(raw_samples)} 条样本")
|
||||||
|
|
||||||
# ---- 2. Load model ----
|
# ---- 2. Load model ----
|
||||||
print(f"[eval] loading model: {model_path}")
|
print(f"[eval] loading model: {model_path}")
|
||||||
from compute.engines.llama_factory.inference import InferenceSession
|
from compute.engines.llama_factory.inference import InferenceSession
|
||||||
session = InferenceSession()
|
session = InferenceSession()
|
||||||
|
_write_eval_progress(output_dir, "running", "model_loading", len(raw_samples), 0, "正在加载评测模型")
|
||||||
session.load(
|
session.load(
|
||||||
model_name_or_path=model_path,
|
model_name_or_path=model_path,
|
||||||
adapter_name_or_path=adapter_path,
|
adapter_name_or_path=adapter_path,
|
||||||
@@ -329,6 +550,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
if not load_result.get("loaded"):
|
if not load_result.get("loaded"):
|
||||||
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
|
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
|
||||||
print(f"[eval] model loaded OK")
|
print(f"[eval] model loaded OK")
|
||||||
|
_write_eval_progress(output_dir, "running", "inference", len(raw_samples), 0, "开始生成模型回答")
|
||||||
|
|
||||||
# ---- 3. Run inference on each sample ----
|
# ---- 3. Run inference on each sample ----
|
||||||
samples: list[dict[str, Any]] = []
|
samples: list[dict[str, Any]] = []
|
||||||
@@ -384,12 +606,45 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
] if judge_result else [],
|
] if judge_result else [],
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
})
|
})
|
||||||
|
if not judge_enabled:
|
||||||
|
deterministic_score = _deterministic_sample_score(reference, prediction)
|
||||||
|
samples[-1].update({
|
||||||
|
"score": deterministic_score,
|
||||||
|
"max_score": 100,
|
||||||
|
"raw_score": deterministic_score,
|
||||||
|
"raw_max_score": 100,
|
||||||
|
"passed": deterministic_score >= 60,
|
||||||
|
"judgement": "正确" if deterministic_score >= 80 else "部分正确" if deterministic_score >= 60 else "错误",
|
||||||
|
"evaluation_reason": "基于精确匹配、文本相似度和 ROUGE-L 的确定性评分",
|
||||||
|
})
|
||||||
|
_write_json(
|
||||||
|
output_dir / "eval_results.json",
|
||||||
|
{
|
||||||
|
"status": "running",
|
||||||
|
"overall_score": round(
|
||||||
|
sum(float(item["score"]) for item in samples if item.get("score") is not None)
|
||||||
|
/ max(len([item for item in samples if item.get("score") is not None]), 1),
|
||||||
|
output_precision,
|
||||||
|
),
|
||||||
|
"overall_score_max": 100,
|
||||||
|
"overall_evaluation": f"已完成 {len(samples)}/{total} 条样本",
|
||||||
|
"dimension_summary": [],
|
||||||
|
"samples": samples,
|
||||||
|
"sample_count": total,
|
||||||
|
"completed_count": len(samples),
|
||||||
|
"passed_count": sum(bool(item.get("passed")) for item in samples),
|
||||||
|
"basic_metrics": {},
|
||||||
|
"metric_summary_version": 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
progress_pct = int(idx / max(total, 1) * 100)
|
progress_pct = int(idx / max(total, 1) * 100)
|
||||||
|
_write_eval_progress(output_dir, "running", "inference", total, len(samples), f"已完成第 {idx} 条样本", idx)
|
||||||
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
|
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
|
||||||
|
|
||||||
# ---- 4. Compute basic metrics ----
|
# ---- 4. Compute basic metrics ----
|
||||||
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
|
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
|
||||||
|
_write_eval_progress(output_dir, "running", "metrics", total, len(samples), "正在计算评测指标", total)
|
||||||
metrics_result: dict[str, Any] = {}
|
metrics_result: dict[str, Any] = {}
|
||||||
|
|
||||||
bleu_cfg = basic_cfg.get("bleu", {})
|
bleu_cfg = basic_cfg.get("bleu", {})
|
||||||
@@ -397,11 +652,11 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
|
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
|
||||||
|
|
||||||
rouge_cfg = basic_cfg.get("rouge", {})
|
rouge_cfg = basic_cfg.get("rouge", {})
|
||||||
if rouge_cfg.get("enabled"):
|
if rouge_cfg.get("enabled") or not judge_enabled:
|
||||||
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
|
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
|
||||||
|
|
||||||
cosine_cfg = basic_cfg.get("cosine", {})
|
cosine_cfg = basic_cfg.get("cosine", {})
|
||||||
if cosine_cfg.get("enabled"):
|
if cosine_cfg.get("enabled") or not judge_enabled:
|
||||||
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
||||||
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
||||||
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
||||||
@@ -412,16 +667,18 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
scored = [s for s in samples if s.get("score") is not None]
|
scored = [s for s in samples if s.get("score") is not None]
|
||||||
passed_count = len([s for s in scored if s.get("passed")])
|
passed_count = len([s for s in scored if s.get("passed")])
|
||||||
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
|
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
|
||||||
max_score = dimension_cfg.get("score_max", 5)
|
overall_score = avg_score
|
||||||
overall_score = round(avg_score / max_score * 100, output_precision)
|
|
||||||
overall_score_max = 100
|
overall_score_max = 100
|
||||||
dimension_summary = [{
|
dimension_summary = [{
|
||||||
"name": "综合评分",
|
"name": "LLM Judge",
|
||||||
"score": overall_score,
|
"score": overall_score,
|
||||||
"max_score": 100,
|
"max_score": 100,
|
||||||
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
|
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
|
||||||
}]
|
"sample_count": completed,
|
||||||
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/{max_score} 分"
|
"available": bool(scored),
|
||||||
|
"error": "部分样本未返回可解析评分" if len(scored) < completed else "",
|
||||||
|
}] + _metric_dimension_summary(metrics_result)
|
||||||
|
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/100 分"
|
||||||
else:
|
else:
|
||||||
passed_count = 0
|
passed_count = 0
|
||||||
enabled_scores = [
|
enabled_scores = [
|
||||||
@@ -431,19 +688,11 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
]
|
]
|
||||||
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
|
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
|
||||||
overall_score_max = 100
|
overall_score_max = 100
|
||||||
dimension_summary = [
|
dimension_summary = _metric_dimension_summary(metrics_result)
|
||||||
{
|
|
||||||
"name": name,
|
|
||||||
"score": float(item.get("score") or 0),
|
|
||||||
"max_score": 100,
|
|
||||||
"pass_rate": float(item.get("score") or 0),
|
|
||||||
}
|
|
||||||
for name, item in metrics_result.items()
|
|
||||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
|
||||||
]
|
|
||||||
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
|
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
|
"status": "completed",
|
||||||
"overall_score": overall_score,
|
"overall_score": overall_score,
|
||||||
"overall_score_max": overall_score_max,
|
"overall_score_max": overall_score_max,
|
||||||
"overall_evaluation": overall_evaluation,
|
"overall_evaluation": overall_evaluation,
|
||||||
@@ -454,11 +703,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"completed_count": completed,
|
"completed_count": completed,
|
||||||
"passed_count": passed_count,
|
"passed_count": passed_count,
|
||||||
"basic_metrics": metrics_result,
|
"basic_metrics": metrics_result,
|
||||||
|
"metric_summary_version": 2,
|
||||||
}
|
}
|
||||||
|
|
||||||
# ---- 6. Write results ----
|
# ---- 6. Write results ----
|
||||||
result_path = output_dir / "eval_results.json"
|
result_path = output_dir / "eval_results.json"
|
||||||
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
_write_eval_progress(output_dir, "completed", "completed", total, completed, "评测完成", total)
|
||||||
print(f"[eval] results written to {result_path}")
|
print(f"[eval] results written to {result_path}")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,28 @@ import uuid
|
|||||||
from typing import Any, Iterator
|
from typing import Any, Iterator
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_peft_transformers_compat() -> None:
|
||||||
|
"""Bridge a removed PEFT helper used by the bundled Transformers build.
|
||||||
|
|
||||||
|
The offline Compute image currently contains Transformers 5.8.0 and PEFT
|
||||||
|
0.18.1. Transformers imports this private helper when a model directory
|
||||||
|
contains PEFT metadata, but PEFT 0.18.1 does not expose it. Evaluation
|
||||||
|
runs in single-process HuggingFace mode, so tensor-parallel sharding is
|
||||||
|
not applicable and a no-op compatibility hook is the correct behavior.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from peft.utils import save_and_load
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
if hasattr(save_and_load, "_maybe_shard_state_dict_for_tp"):
|
||||||
|
return
|
||||||
|
|
||||||
|
def _maybe_shard_state_dict_for_tp(_model: Any, _state_dict: dict[str, Any], _adapter_name: str) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
save_and_load._maybe_shard_state_dict_for_tp = _maybe_shard_state_dict_for_tp
|
||||||
|
|
||||||
|
|
||||||
class InferenceSession:
|
class InferenceSession:
|
||||||
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
||||||
|
|
||||||
@@ -143,6 +165,7 @@ class InferenceSession:
|
|||||||
|
|
||||||
args = dict(self._load_args)
|
args = dict(self._load_args)
|
||||||
infer_result = get_infer_args(args)
|
infer_result = get_infer_args(args)
|
||||||
|
_ensure_peft_transformers_compat()
|
||||||
model = ChatModel(args)
|
model = ChatModel(args)
|
||||||
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
||||||
generating_args = infer_result[-1]
|
generating_args = infer_result[-1]
|
||||||
@@ -182,6 +205,7 @@ class InferenceSession:
|
|||||||
self._loaded_at = time.time()
|
self._loaded_at = time.time()
|
||||||
self._status = "ready"
|
self._status = "ready"
|
||||||
|
|
||||||
|
|
||||||
def _release_model(self) -> None:
|
def _release_model(self) -> None:
|
||||||
with self._chat_lock:
|
with self._chat_lock:
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
|
|||||||
@@ -2,7 +2,14 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from compute.engines.llama_factory.eval_runner import _load_dataset
|
from compute.engines.llama_factory.eval_runner import (
|
||||||
|
_compute_exact_match,
|
||||||
|
_compute_rouge,
|
||||||
|
_compute_text_similarity,
|
||||||
|
_normalise_api_url,
|
||||||
|
_parse_judge_reply,
|
||||||
|
_load_dataset,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _write(tmp_path, name: str, text: str) -> str:
|
def _write(tmp_path, name: str, text: str) -> str:
|
||||||
@@ -40,3 +47,48 @@ def test_load_jsonl_with_bom_and_embedded_array(tmp_path) -> None:
|
|||||||
"" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]),
|
"" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]),
|
||||||
)
|
)
|
||||||
assert len(_load_dataset(path)) == 2
|
assert len(_load_dataset(path)) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_deterministic_metrics_use_percent_scale() -> None:
|
||||||
|
references = ["北京是中国的首都"]
|
||||||
|
predictions = ["北京是中国的首都"]
|
||||||
|
assert _compute_exact_match(references, predictions)["score"] == 100
|
||||||
|
assert _compute_text_similarity(references, predictions)["score"] == 100
|
||||||
|
|
||||||
|
|
||||||
|
def test_rouge_supports_chinese_character_tokenization() -> None:
|
||||||
|
import pytest
|
||||||
|
pytest.importorskip("rouge_score")
|
||||||
|
result = _compute_rouge(["北京是中国的首都"], ["北京是中国的首都"])
|
||||||
|
assert result["available"] is True
|
||||||
|
assert result["score"] == 100
|
||||||
|
|
||||||
|
|
||||||
|
def test_judge_reply_accepts_json_and_normalises_score() -> None:
|
||||||
|
score, payload, reason = _parse_judge_reply(
|
||||||
|
'{"score": 4, "dimensions": {"正确性": 4}, "reason": "内容正确"}',
|
||||||
|
0,
|
||||||
|
5,
|
||||||
|
)
|
||||||
|
assert score == 80
|
||||||
|
assert payload["dimensions"]["正确性"] == 4
|
||||||
|
assert reason == "内容正确"
|
||||||
|
|
||||||
|
|
||||||
|
def test_judge_reply_accepts_nlp_demo_dimension_format() -> None:
|
||||||
|
score, _, _ = _parse_judge_reply(
|
||||||
|
'{"语义一致性": 4, "信息完整性": 3, "事实准确性": 5, "语言流畅性": 4, "综合评价": "整体良好,0.8"}',
|
||||||
|
0,
|
||||||
|
5,
|
||||||
|
)
|
||||||
|
assert score == 80
|
||||||
|
|
||||||
|
|
||||||
|
def test_judge_reply_keeps_decimal_scores_in_configured_range() -> None:
|
||||||
|
score, _, _ = _parse_judge_reply('{"score": 0.5}', 0, 5)
|
||||||
|
assert score == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_openai_url_does_not_duplicate_v1() -> None:
|
||||||
|
assert _normalise_api_url("https://example.test/v1") == "https://example.test/v1/chat/completions"
|
||||||
|
assert _normalise_api_url("https://example.test") == "https://example.test/v1/chat/completions"
|
||||||
|
|||||||
@@ -25,6 +25,22 @@ def test_build_command_uses_explicit_validation_dataset_without_resplitting() ->
|
|||||||
assert "--val_size" not in result.command
|
assert "--val_size" not in result.command
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_command_resumes_from_checkpoint() -> None:
|
||||||
|
result = build_command(
|
||||||
|
{
|
||||||
|
"base_model": "/models/qwen",
|
||||||
|
"dataset": "ygft_dataset_train",
|
||||||
|
"dataset_dir": "/datasets/example",
|
||||||
|
"output_dir": "/outputs/example",
|
||||||
|
"resume_from_checkpoint": "/data/yg-ft/fine-tunes/ft_1/resume/checkpoint-50",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.command[result.command.index("--resume_from_checkpoint") + 1] == (
|
||||||
|
"/data/yg-ft/fine-tunes/ft_1/resume/checkpoint-50"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _write(tmp_path, name: str, lines: list[dict]) -> object:
|
def _write(tmp_path, name: str, lines: list[dict]) -> object:
|
||||||
path = tmp_path / name
|
path = tmp_path / name
|
||||||
path.write_text(
|
path.write_text(
|
||||||
|
|||||||
@@ -61,5 +61,6 @@ MINIO_SECRET_KEY=change_me_minio_secret
|
|||||||
MINIO_BUCKET=yg-ft-resources
|
MINIO_BUCKET=yg-ft-resources
|
||||||
MINIO_SECURE=false
|
MINIO_SECURE=false
|
||||||
MINIO_INLINE_MAX_BYTES=262144
|
MINIO_INLINE_MAX_BYTES=262144
|
||||||
|
MINIO_PRESIGN_MAX_BYTES=1099511627776
|
||||||
STORAGE_WAIT_SECONDS=300
|
STORAGE_WAIT_SECONDS=300
|
||||||
STORAGE_CHECK_INTERVAL_SECONDS=10
|
STORAGE_CHECK_INTERVAL_SECONDS=10
|
||||||
|
|||||||
@@ -14,11 +14,12 @@ RUN pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple \
|
|||||||
|
|
||||||
RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, minio, alembic; print('backend dependency check ok')"
|
RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, minio, alembic; print('backend dependency check ok')"
|
||||||
|
|
||||||
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \
|
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft /opt/tiktoken_cache \
|
||||||
&& chmod -R 0775 /opt/yg-ft /data/yg-ft
|
&& chmod -R 0775 /opt/yg-ft /data/yg-ft /opt/tiktoken_cache
|
||||||
|
|
||||||
# 离线打包 tiktoken cl100k_base 词表,避免无网环境下运行时联网下载
|
# 离线打包 tiktoken cl100k_base 词表(与运行时 TIKTOKEN_CACHE_DIR 对齐),
|
||||||
COPY docker/app/tiktoken /opt/tiktoken_cache
|
# 文件名采用官方 SHA,避免代码走 fallback 重建 Encoding 的分支。
|
||||||
|
COPY docker/app/tiktoken/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 /opt/tiktoken_cache/
|
||||||
|
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ services:
|
|||||||
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
|
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
|
||||||
MINIO_SECURE: ${MINIO_SECURE:-false}
|
MINIO_SECURE: ${MINIO_SECURE:-false}
|
||||||
MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144}
|
MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144}
|
||||||
|
MINIO_PRESIGN_MAX_BYTES: ${MINIO_PRESIGN_MAX_BYTES:-1099511627776}
|
||||||
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
||||||
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
||||||
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}
|
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ YG_FT_DATASET_ROOT=/data/yg-ft/datasets
|
|||||||
YG_FT_DATASET_ROOT_HOST=./data/yg-ft/datasets
|
YG_FT_DATASET_ROOT_HOST=./data/yg-ft/datasets
|
||||||
YG_FT_OUTPUT_ROOT=/data/yg-ft/outputs
|
YG_FT_OUTPUT_ROOT=/data/yg-ft/outputs
|
||||||
YG_FT_OUTPUT_ROOT_HOST=./data/yg-ft/outputs
|
YG_FT_OUTPUT_ROOT_HOST=./data/yg-ft/outputs
|
||||||
|
COMPUTE_CACHE_MAX_BYTES=0
|
||||||
|
COMPUTE_CACHE_TTL_SECONDS=0
|
||||||
TRAINING_LOG_ROOT=/opt/yg-ft/logs/training
|
TRAINING_LOG_ROOT=/opt/yg-ft/logs/training
|
||||||
TRAINING_LOG_ROOT_HOST=./data/yg-ft/logs/training
|
TRAINING_LOG_ROOT_HOST=./data/yg-ft/logs/training
|
||||||
COMPUTE_LOG_ROOT_HOST=./data/yg-ft/logs/compute
|
COMPUTE_LOG_ROOT_HOST=./data/yg-ft/logs/compute
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ services:
|
|||||||
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
||||||
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
||||||
YG_FT_CACHE_ROOT: ${YG_FT_CACHE_ROOT:-/data/yg-ft}
|
YG_FT_CACHE_ROOT: ${YG_FT_CACHE_ROOT:-/data/yg-ft}
|
||||||
|
COMPUTE_CACHE_MAX_BYTES: ${COMPUTE_CACHE_MAX_BYTES:-0}
|
||||||
|
COMPUTE_CACHE_TTL_SECONDS: ${COMPUTE_CACHE_TTL_SECONDS:-0}
|
||||||
PYTHONPATH: /app
|
PYTHONPATH: /app
|
||||||
volumes:
|
volumes:
|
||||||
- ../../compute:/app/compute:ro
|
- ../../compute:/app/compute:ro
|
||||||
|
|||||||
38
docs/20260812/database-migration.md
Normal file
38
docs/20260812/database-migration.md
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# 数据库迁移说明
|
||||||
|
|
||||||
|
## 当前迁移文件
|
||||||
|
|
||||||
|
- 新库初始化:`backend/app/db/sql/000_full_init.sql`
|
||||||
|
- 已有数据库增量迁移:`backend/app/db/sql/005_storage_progress_migration.sql`
|
||||||
|
- GPU 预留迁移:`backend/app/db/sql/006_gpu_reservations.sql`
|
||||||
|
- 平台闭环迁移:`backend/app/db/sql/007_platform_completion.sql`
|
||||||
|
- 离线部署使用:`docker/offline/src/backend/app/db/sql/000_full_init.sql`
|
||||||
|
|
||||||
|
## 执行方式
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for migration in 005_storage_progress_migration.sql 006_gpu_reservations.sql 007_platform_completion.sql; do
|
||||||
|
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "backend/app/db/sql/$migration"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
迁移前应完成数据库备份,并保存以下结构快照:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT table_name, column_name, data_type
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
ORDER BY table_name, ordinal_position;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 本次迁移内容
|
||||||
|
|
||||||
|
- 为算力节点资源副本增加 `version_id` 和 `storage_object_id`,用于记录 MinIO 版本与本地缓存对应关系。
|
||||||
|
- 补齐 `storage_objects.metadata`、模型产物、数据集文件和数据转换任务的 MinIO 字段。
|
||||||
|
- 补齐评测任务软删除、创建者、租户和报告对象字段。
|
||||||
|
- 增加活动评测任务、资源副本和 MinIO 对象索引。
|
||||||
|
- 增加 GPU 预留表,统一训练、推理、评测的卡级占用约束。
|
||||||
|
- 增加模型导出创建者/租户/归档状态字段。
|
||||||
|
- 增加 MinIO 对象软删除清理、缓存版本校验和访问保护字段及清理任务表。
|
||||||
|
|
||||||
|
`000_full_init.sql` 已包含相同的幂等变更,新增数据库直接执行初始化脚本即可;已有数据库不要依赖容器重启自动完成迁移。
|
||||||
383
docs/20260812/当前项目开发进度.md
Normal file
383
docs/20260812/当前项目开发进度.md
Normal file
@@ -0,0 +1,383 @@
|
|||||||
|
# 当前项目开发进度
|
||||||
|
|
||||||
|
> 评估基线:2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。
|
||||||
|
>
|
||||||
|
> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容,统一标记为“部分完成”或“未完成”。
|
||||||
|
|
||||||
|
## 一、项目定位与总体结论
|
||||||
|
|
||||||
|
当前项目是一个面向多用户、多算力节点的模型训练与推理平台,主要链路为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Vue 前端
|
||||||
|
|
|
||||||
|
FastAPI Backend API
|
||||||
|
|-- PostgreSQL:业务元数据、权限、任务状态、小型内容和预览数据
|
||||||
|
|-- Redis:会话、限流、短期缓存和任务辅助状态
|
||||||
|
|-- MinIO:模型、数据集、报告和大文件的统一对象存储
|
||||||
|
|-- Compute API / Agent:训练、推理、评测、模型合并和 GPU 执行
|
||||||
|
|
|
||||||
|
多台算力节点
|
||||||
|
```
|
||||||
|
|
||||||
|
整体判断:
|
||||||
|
|
||||||
|
| 范围 | 当前状态 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 |
|
||||||
|
| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 |
|
||||||
|
| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入,跨节点一致性和失败恢复仍需加强 |
|
||||||
|
| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现,但完整的租户/项目隔离尚未闭环 |
|
||||||
|
| MinIO 统一存储 | 部分完成 | 大文件和模型已接入,仍存在兼容性的本地路径和部分数据双写/回退路径 |
|
||||||
|
| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 |
|
||||||
|
| 前端体验 | 基本可用,需优化 | 构建问题已持续修复,但页面响应等待、首屏体积和部分错误提示仍需优化 |
|
||||||
|
|
||||||
|
## 二、已完成的功能
|
||||||
|
|
||||||
|
### 2.1 平台基础与部署
|
||||||
|
|
||||||
|
- 已建立 Vue 3 + TypeScript + Vite 前端工程。
|
||||||
|
- 已建立 FastAPI 后端服务,提供登录、平台管理和模型业务接口。
|
||||||
|
- 已建立 Compute API / Agent,用于连接算力节点并执行训练、推理、评测和模型处理任务。
|
||||||
|
- 已使用 PostgreSQL 保存核心业务数据,Redis 提供会话、限流和缓存能力。
|
||||||
|
- 已增加 MinIO 服务及 Backend 的 MinIO 配置,支持和 Compute Agent 分离部署。
|
||||||
|
- 已提供 `docker/app`、`docker/compute`、`docker/minio` 和 `docker/offline` 部署目录。
|
||||||
|
- 已考虑后端、算力服务、MinIO 分布在不同服务器时使用独立网络;Compute 节点访问 MinIO 需要配置所有节点都能访问的固定 IP 或 DNS。
|
||||||
|
- 离线部署目录已经同步后端、算力相关源码和初始化 SQL 的主要改造内容。
|
||||||
|
|
||||||
|
### 2.2 登录、用户和权限基础
|
||||||
|
|
||||||
|
- 用户登录、退出、当前用户信息和密码修改接口已经存在。
|
||||||
|
- 已有 Token 会话、Redis 会话记录和登录限流逻辑。
|
||||||
|
- 已建立用户、角色、权限码和角色权限关系。
|
||||||
|
- 已实现管理员、普通用户等基础角色分层。
|
||||||
|
- 已实现页面路由守卫、菜单过滤和前端按钮级权限的基础能力。
|
||||||
|
- 已建立资源 ACL 管理页面和相关接口,可对用户或角色授予资源级权限。
|
||||||
|
- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。
|
||||||
|
- 已建立运行日志、审计日志查询页面及审计记录写入机制。
|
||||||
|
- 已加入软删除相关字段和部分删除逻辑,避免直接物理删除业务资源。
|
||||||
|
|
||||||
|
### 2.3 算力节点与 GPU 资源
|
||||||
|
|
||||||
|
- 已实现算力节点的新增、编辑、启用、禁用、维护/删除、连通性测试和健康检查。
|
||||||
|
- 已实现节点列表、节点详情、节点副本/同步状态和 Compute Agent 连接。
|
||||||
|
- 已实现 GPU 信息发现、GPU 状态查询和队列查询。
|
||||||
|
- 已建立 `gpu_allocations`、`gpu_assignments`、调度锁等资源分配表。
|
||||||
|
- 训练任务已经支持选择调度节点和一张或多张 GPU,并在预检阶段校验资源可用性。
|
||||||
|
- Compute Agent 已支持训练、评测、推理和缓存准备等任务接口。
|
||||||
|
- 已存在资源副本和同步任务模型,用于记录节点侧资源同步状态。
|
||||||
|
|
||||||
|
### 2.4 数据集管理
|
||||||
|
|
||||||
|
- 已实现数据集创建、列表、详情、编辑、删除和文件上传。
|
||||||
|
- 已实现数据集文件下载、预览、记录列表和数据记录编辑入口。
|
||||||
|
- 已处理 JSON 与 JSONL 的记录数差异:JSON 数组按元素计数,JSONL 按有效行计数,避免把整个 JSON 文件误按行数统计。
|
||||||
|
- 已提供数据集版本列表、版本详情、创建版本、切换当前激活版本和删除版本接口。
|
||||||
|
- 已增加数据集文件、版本、数据记录等初始化表结构。
|
||||||
|
- 已支持数据集文件在数据库小内容和 MinIO 大文件之间按策略存储。
|
||||||
|
- 已在训练预检中检查数据集文件是否存在、是否可从 MinIO 获取以及是否能准备到目标算力节点。
|
||||||
|
|
||||||
|
### 2.5 数据处理与数据转换
|
||||||
|
|
||||||
|
- 已提供结构化数据、非结构化数据、外部数据源的处理创建流程。
|
||||||
|
- 已实现源文件上传、预览、分片/切分、生成、质量检查、去重和结果管理等数据处理流程。
|
||||||
|
- 已支持处理结果生成数据集或导入数据集版本。
|
||||||
|
- 已建立数据处理任务、源文件、预览项、结果等数据表。
|
||||||
|
- 已提供 JSON、JSONL 等数据格式转换页面和后端任务接口。
|
||||||
|
- 已将数据转换输出接入 MinIO/数据库分层存储:小型文本结果可存数据库,大文件存 MinIO。
|
||||||
|
- 已处理输出文件下载和转换结果元数据保存问题。
|
||||||
|
|
||||||
|
### 2.6 模型训练
|
||||||
|
|
||||||
|
- 已实现训练任务创建、配置预检、命令预览、启动、停止、重试和删除。
|
||||||
|
- 已接入 LLaMA-Factory 等训练适配逻辑。
|
||||||
|
- 已支持选择训练数据、基座模型、算力节点和 GPU。
|
||||||
|
- 已实现训练日志获取、训练任务概览、诊断信息、检查点和训练指标查询。
|
||||||
|
- 前端训练详情已经具备训练曲线解析和展示逻辑,日志轮询间隔已调整为 3 秒。
|
||||||
|
- 已增加 GPU 详情展示入口,包括显存和利用率等 Compute Agent 上报信息。
|
||||||
|
- 已支持训练任务的 MinIO 数据准备和目标算力节点缓存准备。
|
||||||
|
|
||||||
|
### 2.7 模型管理与权重合并
|
||||||
|
|
||||||
|
- 已实现在线模型/基座模型和训练模型的列表、创建、详情、用途修改和删除。
|
||||||
|
- 已建立模型、训练模型、模型血缘、模型产物和导出任务相关表。
|
||||||
|
- 已提供权重合并入口,能够根据训练任务准备基座模型和 Adapter,并提交 Compute Agent 执行合并。
|
||||||
|
- 已增加模型产物和 MinIO 对象关联字段。
|
||||||
|
- 合并结果能够在任务完成后归档到 MinIO 的设计和主要代码路径已经建立。
|
||||||
|
|
||||||
|
### 2.8 模型推理与模型对比
|
||||||
|
|
||||||
|
- 已实现推理模型列表、创建、详情和删除入口。
|
||||||
|
- 已实现模型加载、卸载、服务启动、服务状态查询和对话调用。
|
||||||
|
- 已实现模型对比任务及多模型聊天相关接口。
|
||||||
|
- 已增加推理失败重试、停止和资源释放的处理路径。
|
||||||
|
- 已支持根据页面选择的算力节点准备模型缓存,兼容训练时所选节点优先的业务要求。
|
||||||
|
- Compute Agent 已提供本地推理会话和模型缓存状态接口。
|
||||||
|
|
||||||
|
### 2.9 模型评测
|
||||||
|
|
||||||
|
- 已实现评测任务列表、创建、详情和删除。
|
||||||
|
- 已实现评测维度管理和评测规则配置页面。
|
||||||
|
- 已建立评测任务、评测维度、对比任务等数据库表。
|
||||||
|
- 已支持选择模型、数据集、评测维度、算力节点和 GPU 的基础流程。
|
||||||
|
- 已接入 Compute Agent 执行评测任务,并保存评测结果和报告相关元数据。
|
||||||
|
|
||||||
|
### 2.10 数据存储策略
|
||||||
|
|
||||||
|
- 已建立 `storage_objects`、`storage_cache_jobs` 等 MinIO 元数据和缓存任务表。
|
||||||
|
- 已建立 MinIO 对象上传、下载、预签名 URL 和节点缓存准备的主要接口。
|
||||||
|
- 已采用分层策略:
|
||||||
|
- 小型 JSON、JSONL、CSV、任务参数快照、预览数据保留在数据库,降低频繁预览的 MinIO 延迟。
|
||||||
|
- 模型权重、训练产物、评测报告和大文件使用 MinIO。
|
||||||
|
- 小型 PDF、DOCX、XLSX 仍优先存 MinIO,以保留原始二进制文件内容。
|
||||||
|
- 已增加 `data_convert_tasks.output_content`,用于保存小型转换结果,避免所有小结果都依赖 MinIO。
|
||||||
|
- Backend 和离线包中的 `000_full_init.sql` 已同步,当前两份初始化脚本内容一致。
|
||||||
|
|
||||||
|
## 三、部分完成、仍需完善的功能
|
||||||
|
|
||||||
|
### 3.1 MinIO 统一数据源尚未完全闭环
|
||||||
|
|
||||||
|
当前 MinIO 已成为模型、大文件和跨节点资源的主存储方向,但仍保留以下兼容路径:
|
||||||
|
|
||||||
|
- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。
|
||||||
|
- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段,这是当前已确认的小文件性能策略,不是错误,但必须统一记录来源、大小、校验值和版本。
|
||||||
|
- Compute Agent 节点侧的大文件上传已改为流式读取;Backend 端部分历史上传/下载路径仍未完成统一的分片传输。
|
||||||
|
- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联,数据库没有直接外键,删除和数据一致性需要应用层保证。
|
||||||
|
- 删除业务资源后,对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。
|
||||||
|
|
||||||
|
### 3.2 MinIO 预签名接口的权限边界
|
||||||
|
|
||||||
|
资源写权限、服务端对象 Key、上传完成确认和对象大小校验已经完成;仍需补充 Content-Type 白名单、对象过期回收和完整审计事件。
|
||||||
|
|
||||||
|
### 3.3 激活版本和跨节点资源版本仍需加强
|
||||||
|
|
||||||
|
数据集已经有 `active_version_id` 和版本表,但以下场景仍需补充:
|
||||||
|
|
||||||
|
- 训练、评测、模型对比和权重合并已在任务创建时固化资源版本 ID;推理服务启动和导出任务仍需统一补齐。
|
||||||
|
- 同一文件名的不同版本不能只依靠文件名同步,应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。
|
||||||
|
- 已创建任务在后续切换激活版本后,不能被意外切换到新版本。
|
||||||
|
- 已为资源副本保存版本、对象 ID 和本地路径;对象 ETag/校验值及多文件 manifest 仍需补齐。
|
||||||
|
- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。
|
||||||
|
|
||||||
|
### 3.4 权限 2.0 尚未完全落地
|
||||||
|
|
||||||
|
已有用户、角色、权限码、ACL、审批和审计基础,但仍存在以下差距:
|
||||||
|
|
||||||
|
- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。
|
||||||
|
- 项目空间设计已经讨论过取消,但数据库中仍保留 `projects`、`project_members` 等历史结构,需要明确兼容策略和最终迁移方式。
|
||||||
|
- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。
|
||||||
|
- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。
|
||||||
|
- 前端按钮权限已经有基础实现,但不能替代后端鉴权;仍需要对所有关键动作进行后端默认拒绝校验。
|
||||||
|
- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。
|
||||||
|
|
||||||
|
### 3.5 模型合并、导出和评测报告闭环不足
|
||||||
|
|
||||||
|
- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立,但失败时的清理、重试和幂等性仍需加强。
|
||||||
|
- 合并结果归档已增加失败状态和服务重启后的补偿轮询;仍需加入独立归档队列和幂等对象清单,降低重复扫描成本。
|
||||||
|
- 模型导出任务目前有查询模型和表结构,但完整的创建、执行、进度、失败重试和下载闭环尚未完成。
|
||||||
|
- 评测结果和报告字段已经存在,但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。
|
||||||
|
- 评测指标配置和执行器返回指标之间仍需要强类型映射,避免前端显示为通用的 `custom`。
|
||||||
|
|
||||||
|
### 3.6 GPU 资源分配需要统一到所有任务类型
|
||||||
|
|
||||||
|
- 训练已经有较完整的节点/GPU 选择和预检流程。
|
||||||
|
- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码,但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。
|
||||||
|
- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。
|
||||||
|
- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。
|
||||||
|
- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报,仍需要校验采样时间、单位、空值和任务对应关系。
|
||||||
|
|
||||||
|
## 四、尚未完成的功能
|
||||||
|
|
||||||
|
以下功能在当前代码中没有形成可验收的完整闭环,或仍处于设计/基础代码阶段:
|
||||||
|
|
||||||
|
1. **完整的租户隔离和资源继承模型**:核心列表、详情、下载、缓存、训练、推理、评测和导出接口已补齐基础租户/ACL校验;历史 NULL 租户归属仍需专项迁移。
|
||||||
|
2. **项目取消后的正式数据迁移方案**:当前保留项目表作为兼容结构,正式删除项目设计前仍需为历史数据生成归属迁移和回滚脚本。
|
||||||
|
3. **预签名上传策略的完整约束**:已完成 Content-Type、大小、过期时间、Key 白名单、资源写权限、对象存在性和真实 SHA-256 校验;定时清理过期未完成对象仍需接入运维调度。
|
||||||
|
4. **MinIO 对象生命周期管理**:已提供软删除清理、失败记录/重试、孤儿扫描和管理员清理入口;自动定时执行策略需由部署环境接入。
|
||||||
|
5. **Compute Agent 缓存治理**:已完成容量上限、LRU、TTL、保护窗口、版本/校验清单和状态查询;运行中任务动态保护和磁盘告警仍需结合生产指标系统。
|
||||||
|
6. **统一的资源归档编排器**:训练、合并、导出、评测已纳入重启补偿轮询和 MinIO 归档;独立消息队列和大规模断点分片仍属于生产增强项。
|
||||||
|
7. **模型导出完整流程**:已完成后端创建、节点准备、CPU 导出、量化参数、任务记录、制品血缘、归档轮询、权限和前端按钮;真实大模型多格式压力测试待专用环境执行。
|
||||||
|
8. **流式和分片文件传输**:数据集单文件下载、Compute Agent 上传/下载已采用流式处理;多文件 ZIP 和超大对象的分片上传仍需继续增强。
|
||||||
|
9. **生产级 MinIO 安全和高可用**:开发环境接入和故障快速失败已完成;默认密钥替换、TLS、网络隔离、Console 隔离、容量监控、备份恢复需在生产部署阶段实施。
|
||||||
|
10. **完整的端到端测试和持续集成**:已新增前端构建、后端编译和存储安全测试 CI 基线;跨租户、多节点、多 GPU 并行压力和故障注入仍需扩展执行矩阵。
|
||||||
|
11. **统一数据库迁移体系**:已形成 005/006/007 幂等增量迁移并加入运行时兼容执行;后续可再替换为 Alembic 等正式迁移工具以满足生产审计要求。
|
||||||
|
|
||||||
|
## 五、需要优化的功能
|
||||||
|
|
||||||
|
### 5.1 后端响应性能
|
||||||
|
|
||||||
|
- 页面列表接口需要避免每条记录重复查询用户、资源、MinIO 元数据和 Compute 节点状态。
|
||||||
|
- MinIO 的 Bucket 检查、对象 Head 和预签名生成应使用连接复用、短期缓存和批量查询。
|
||||||
|
- 训练、推理、评测页面不应通过过短间隔轮询大量详情接口,应按任务状态动态退避,并在完成后停止轮询。
|
||||||
|
- 对 dashboard、节点健康、GPU 状态等高频数据应区分实时数据和缓存数据。
|
||||||
|
- 后端日志轮询和健康检查日志需要继续降噪,仅在状态变化、失败或达到较长周期时输出。
|
||||||
|
|
||||||
|
### 5.2 前端加载和交互
|
||||||
|
|
||||||
|
- 列表页面应区分首屏 loading、刷新 loading、操作 loading,避免整页长时间无反馈。
|
||||||
|
- 推理、评测、训练详情应使用统一的任务状态刷新策略和超时提示。
|
||||||
|
- 前端仍有 FontAwesome 在线资源解析警告,应清理对外部网络文件的依赖,保证离线环境打开速度。
|
||||||
|
- 应继续拆分首屏大体积 chunk,并减少一次性加载不相关页面组件。
|
||||||
|
- GPU 选择组件需要明确显示空闲、占用、不可达、预留和已分配状态。
|
||||||
|
- 错误提示应携带资源名称、节点名称、版本和下一步处理建议,减少只显示 500/404 的情况。
|
||||||
|
|
||||||
|
### 5.3 训练、推理和评测可靠性
|
||||||
|
|
||||||
|
- 所有任务创建前应执行同一套资源权限、版本存在性、MinIO 可用性和 GPU 原子分配校验。
|
||||||
|
- 任务创建接口应支持幂等键,避免前端重复点击造成重复任务。
|
||||||
|
- 节点不可达时应快速失败或进入可见的等待状态,不能让页面长时间无反馈。
|
||||||
|
- 失败重试应区分网络瞬时失败、资源不足、模型文件缺失、参数错误和执行器失败。
|
||||||
|
- 任务停止后必须释放 GPU 分配、推理端口、缓存锁和临时目录。
|
||||||
|
|
||||||
|
### 5.4 数据和模型一致性
|
||||||
|
|
||||||
|
- 每个对象都应保存大小、校验值、版本 ID、来源、创建者、租户和引用状态。
|
||||||
|
- 数据库中的小文件内容和 MinIO 对象不能同时被当作可独立修改的主副本;需要明确唯一写入入口。
|
||||||
|
- 数据集激活版本变更需要留下审计记录,并影响后续任务创建但不改变已创建任务。
|
||||||
|
- 模型权重、Adapter、合并结果和导出结果需要形成完整血缘关系。
|
||||||
|
|
||||||
|
## 六、数据库和初始化脚本状态
|
||||||
|
|
||||||
|
当前 `backend/app/db/sql/000_full_init.sql` 已包含以下主要类别:
|
||||||
|
|
||||||
|
- 用户、模型、训练模型、模型血缘、模型产物、模型导出任务。
|
||||||
|
- 数据集、数据集文件、数据集版本、数据集记录。
|
||||||
|
- 算力节点、GPU、GPU 分配、调度锁、Compute Job。
|
||||||
|
- 资源副本、资源同步任务、MinIO 对象、缓存任务。
|
||||||
|
- 评测任务、评测维度、模型对比任务。
|
||||||
|
- 租户、项目兼容表、项目成员、角色、会话、ACL。
|
||||||
|
- 审批模板、审批实例、审批步骤、审计日志、留存策略。
|
||||||
|
- 数据处理任务、源文件、预览项、处理结果、数据转换任务。
|
||||||
|
|
||||||
|
已确认的近期字段包括:
|
||||||
|
|
||||||
|
- `model_artifacts.storage_object_id`
|
||||||
|
- `model_artifacts.storage_backend`
|
||||||
|
- `dataset_files.storage_object_id`
|
||||||
|
- `data_convert_tasks.output_content`
|
||||||
|
- `data_convert_tasks.output_storage_object_id`
|
||||||
|
- `data_convert_tasks.storage_backend`
|
||||||
|
- `eval_tasks.report_storage_object_id`
|
||||||
|
|
||||||
|
离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意:
|
||||||
|
|
||||||
|
- 初始化 SQL 主要用于新数据库或新数据卷;已有数据库不能仅靠重启容器自动获得全部新字段。
|
||||||
|
- 生产/测试数据库需要执行可追踪的迁移脚本,并在迁移前备份或生成结构快照。
|
||||||
|
- `ensure_schema` 类运行时补字段逻辑只能作为兼容兜底,不能替代正式迁移。
|
||||||
|
- 后续如果正式移除项目设计,需要先完成数据归属迁移,再决定是否删除历史表,不能直接从初始化 SQL 中删除表。
|
||||||
|
|
||||||
|
## 七、当前验证结果
|
||||||
|
|
||||||
|
### 7.1 2026-08-19 本轮按计划落地内容
|
||||||
|
|
||||||
|
- P0 MinIO 预签名上传已增加资源存在性、资源写权限、管理员基座模型写权限、资源版本和对象 Key 前缀校验;新增上传完成确认接口,会校验 MinIO 对象存在、大小和状态后再标记为可用。
|
||||||
|
- 训练、评测、模型对比和权重合并任务会保存创建时的租户信息与资源版本快照,后续切换数据集激活版本不会改变已创建任务的输入版本。
|
||||||
|
- 数据集、评测任务、训练任务和模型对比列表增加租户范围过滤;用户表、模型/数据集创建入口补齐租户归属;数据转换资源补充所有者权限映射。
|
||||||
|
- Compute Agent 准备缓存后会回写资源副本的版本、MinIO 对象和本地路径;缓存支持可选容量上限、按访问时间淘汰非临时文件,并提供当前占用量和上限状态。
|
||||||
|
- 训练产物、权重合并结果和评测报告增加 MinIO 归档、失败状态记录以及 Backend 重启后的补偿轮询;离线部署源码已同步对应逻辑。
|
||||||
|
- Compute Agent 大文件上传已改为流式读取,避免将整个文件一次性读入内存;离线 Compute Agent 同步完成。
|
||||||
|
- 增加 `005_storage_progress_migration.sql` 增量迁移脚本,并修复 `000_full_init.sql` 中旧数据库执行时索引早于字段补齐的问题;主工程和离线初始化脚本已同步。
|
||||||
|
- 增加 `MINIO_PRESIGN_MAX_BYTES` 配置和上传 Content-Type 白名单;首次数据库 schema 检查移出 Compute Poller 的 Uvicorn 事件循环,避免远程 PostgreSQL 慢连接拖垮健康检查;轮询相同失败改为 300 秒限频记录,并跳过已标记 offline 节点。
|
||||||
|
- 前端 `npm run build` 已通过;容器内 MinIO Key 越权校验专项测试为 `5 passed`,Backend 和 Compute Agent 重启后均为 healthy。
|
||||||
|
|
||||||
|
已完成的静态和局部验证:
|
||||||
|
|
||||||
|
- Backend 和离线 Backend 源码 `compileall` 检查通过。
|
||||||
|
- MinIO 分层策略冒烟验证通过:小型 JSON/JSONL 可落数据库,小型二进制和超过阈值的内容进入 MinIO。
|
||||||
|
- 主工程和离线包初始化 SQL 已做同步检查,内容一致。
|
||||||
|
- 前端此前已完成 `npm run build` 类型错误修复,构建剩余问题主要是非阻断的资源/分包警告。
|
||||||
|
- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。
|
||||||
|
- 当前 WSL 运行验证中 Backend、Compute Agent、MinIO 均为 healthy;`gpu-node-02`(`172.25.179.69:19100`)连接、GPU 0 分配和任务执行均正常。`gpu-node-01` 的历史地址不可达,已通过健康检查标记为 offline,轮询器不再重复轮询其历史任务。
|
||||||
|
|
||||||
|
### 7.2 2026-08-19 gpu-node-02 真实流程验证
|
||||||
|
|
||||||
|
- 训练:使用 `qwen3.5-0.8B` + `test_data_0817`,任务 `ft_172a2da65140` 完成,GPU 0 使用期间可看到显存、利用率、温度和进程,结束后恢复 idle;24 个训练产物已归档 MinIO。
|
||||||
|
- 资源快照:任务 `ft_d93b0fdae1c9` 创建时已保存数据集 `ds_8f6b8c5714c7` 的活动版本 `file_d550c2128722_v1`。
|
||||||
|
- 训练曲线:任务 `ft_d93b0fdae1c9` 通过 `logging_steps=1` 生成 3 个 loss/epoch/learning-rate/grad-norm 数据点,并生成 `training_loss.png`;后端解析器已兼容带引号数字格式。
|
||||||
|
- 权重合并:任务 `merge_1dc15a290810` 完成,基座模型路径、合并路径已回写,8 个合并模型文件归档 MinIO。
|
||||||
|
- 推理:任务 `cmp_abdf0762869d` 在 GPU 0 加载 `qwen3.5-0.8B` 成功,对话接口返回正常;卸载后 GPU 恢复 idle。
|
||||||
|
- 评测:任务 `eval_3c3f84263e23` 完成 2 条样本评测,报告、样本明细和基础指标已落库;GPU 恢复 idle。评审模型 HTTP 400 导致评审分数为 0,属于评审模型接口配置问题,算力评测链路本身已跑通。
|
||||||
|
|
||||||
|
### 7.3 多 GPU、MinIO 故障和跨用户权限专项验证
|
||||||
|
|
||||||
|
- 多 GPU 调度开发:新增 `gpu_reservations` 表,评测和推理在远程提交/加载前与训练统一纳入数据库原子预留;失败、停止、删除、节点不可达和卸载路径自动释放预留。当前 `gpu-node-02` 只有 GPU 0,已通过同卡“推理预留后评测抢占”测试,评测被明确拒绝,释放后 GPU 恢复 idle。多节点、多卡的真实并行测试待增加第二个可达节点和多卡节点后执行。
|
||||||
|
- 调度锁优化:调度锁改为事务内持有并在提交前释放,避免一次调度成功后后续请求等待 30 秒 TTL;schema 初始化增加 PostgreSQL advisory lock,避免轮询器与首个请求并发初始化造成死锁。
|
||||||
|
- MinIO 故障注入:停止 `yg-ft-minio` 后,`GET /modelTF/health` 返回 HTTP 200 且 `storage.status=unavailable`;恢复容器后返回 `storage.status=ready`。MinIO 客户端关闭默认重试链,故障健康探测耗时从约 6 秒降至约 0.3 秒。
|
||||||
|
- 跨用户权限:创建临时用户 `qa_user_0819`,未授权访问管理员数据集返回 403;授予资源 `read/download` ACL 后列表和详情可访问;撤销 ACL 后再次返回 403;非管理员创建用户返回 403。测试用户已删除,未遗留测试 ACL 或 GPU 预留。
|
||||||
|
- 用户管理接口已补齐管理员依赖,创建、列表、修改、删除和重置密码不再允许普通用户调用。
|
||||||
|
|
||||||
|
当前不能据此宣称“全量功能测试通过”:
|
||||||
|
|
||||||
|
- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设,需要按当前分层存储策略更新。
|
||||||
|
- 本轮已完成当前 WSL Docker 容器健康检查、`gpu-node-02` 单节点真实流程、单卡冲突、MinIO 故障恢复和跨用户 ACL 专项验证;多节点、多卡的真实并行测试仍需要第二个可达节点和多卡节点。
|
||||||
|
|
||||||
|
- 本轮专项安全测试为 `5 passed`;全量 pytest 仍未执行完毕,需要按测试类别拆分并设置超时。
|
||||||
|
|
||||||
|
### 7.4 本轮 11 项未验证能力的补齐结果
|
||||||
|
|
||||||
|
- 租户管理接口已统一要求管理员身份;模型制品、模型血缘、导出任务、评测报告和 MinIO 资源清单均增加资源级访问校验。
|
||||||
|
- 新增 `POST /modelTF/model-manage/export`,导出沿用节点选择、MinIO 缓存准备、GPU/调度约束和归档轮询;导出结果记录到 `model_export_jobs`,并建立导出制品与模型血缘。
|
||||||
|
- 新增 `GET /modelTF/model-eval/{task_id}/report`,优先流式返回 MinIO 报告,历史任务无归档对象时返回数据库报告兼容结果。
|
||||||
|
- 新增 `GET /modelTF/storage/resources/{resource_type}/{resource_id}/manifest`、管理员对象清理和孤儿扫描接口;预签名上传增加过期时间和真实 SHA-256 内容校验。
|
||||||
|
- Compute Agent 增加缓存 TTL、缓存保护窗口和元数据清单;超过 TTL 的非保护缓存会在状态检查时清理,容量淘汰会跳过保护中的资源。
|
||||||
|
- 数据集单文件下载改为 MinIO 流式传输,避免 Backend 一次性载入完整大文件;训练、合并、导出、评测仍由统一轮询器负责重启后的归档补偿。
|
||||||
|
- 新增 `007_platform_completion.sql`,并已加入运行时兼容迁移;主工程和 `docker/offline/src` 的源码及初始化 SQL 已同步。
|
||||||
|
- 以上为代码闭环和单节点验证;生产级 MinIO TLS/HA、第二节点/多卡并行压力测试、全量 CI 和历史项目数据正式迁移仍属于上线前专项工作。
|
||||||
|
|
||||||
|
## 八、下一阶段开发计划
|
||||||
|
|
||||||
|
### P0:安全与数据正确性
|
||||||
|
|
||||||
|
1. 为预签名上传补充 Content-Type、大小上限、过期对象清理和上传完成审计;当前已完成 Key、资源写权限、对象存在性和大小校验。
|
||||||
|
2. 将资源版本快照继续接入推理服务启动、模型导出和缓存准备的所有入口,并增加版本切换回归测试。
|
||||||
|
3. 完成模型导出接口的后端权限、租户范围和审计闭环。
|
||||||
|
4. 补充历史数据租户归属迁移;当前新建资源和主要任务列表已完成租户过滤,历史 NULL 租户仍按兼容策略处理。
|
||||||
|
|
||||||
|
### P1:跨节点可靠性
|
||||||
|
|
||||||
|
1. 将当前资源副本字段升级为完整 manifest,记录每个文件的校验值、大小、目标节点路径和准备时间。
|
||||||
|
2. 完善推理模型缓存的重启对账、失效版本清理和服务级重试;训练、合并、评测归档已具备补偿轮询基础。
|
||||||
|
3. GPU 原子分配、异常回收和任务释放已完成基础闭环;下一步补充多节点重连对账及多卡并行压力测试。
|
||||||
|
4. 增加缓存 TTL、运行任务保护、磁盘占用告警和可视化清理入口;当前已实现可选容量上限和按访问时间淘汰。
|
||||||
|
5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。
|
||||||
|
|
||||||
|
### P2:性能与用户体验
|
||||||
|
|
||||||
|
1. 优化页面列表接口和高频轮询,采用批量查询、短期缓存和动态退避。
|
||||||
|
2. 将大文件上传/下载/复制改为流式或分片传输。
|
||||||
|
3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。
|
||||||
|
4. 处理前端离线资源警告,继续拆分首屏 chunk。
|
||||||
|
5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。
|
||||||
|
|
||||||
|
### P3:工程化和上线准备
|
||||||
|
|
||||||
|
1. 建立正式数据库版本迁移机制和离线升级脚本。
|
||||||
|
2. 增加 CI:前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。
|
||||||
|
3. 增加第二个可达节点/多卡节点后执行多节点端到端并行测试;MinIO 故障注入基础测试已完成。
|
||||||
|
4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。
|
||||||
|
5. 建立生产运行手册,包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。
|
||||||
|
|
||||||
|
## 九、阶段验收标准
|
||||||
|
|
||||||
|
完成下一阶段后,至少应满足:
|
||||||
|
|
||||||
|
- 用户只能看到和操作其所属租户授权的模型、数据集、训练任务、推理服务和评测任务。
|
||||||
|
- 任何任务创建都能明确记录用户、租户、资源版本、算力节点、GPU 列表和 MinIO 对象版本。
|
||||||
|
- 同一个节点的同一张 GPU 不能被两个活动任务同时分配。
|
||||||
|
- MinIO 临时不可用时,任务进入可解释的等待/失败状态,并能按策略重试,页面不会无限等待。
|
||||||
|
- Backend 或 Compute Agent 重启后,任务、缓存、GPU 分配和归档状态可以对账恢复。
|
||||||
|
- 训练、合并、评测和推理产物都能在 MinIO 中找到,并且可以通过权限校验后的接口下载或使用。
|
||||||
|
- 删除资源后不会继续出现在普通列表中,关联对象能够按引用状态延迟清理并留下审计记录。
|
||||||
|
- 新数据库初始化和已有数据库迁移后,所有业务接口不再因为缺表或缺字段启动失败。
|
||||||
|
- 离线部署不依赖外部字体、图标或 CDN,前端首屏和核心业务操作在无网络环境下可用。
|
||||||
|
|
||||||
|
## 十、相关文件索引
|
||||||
|
|
||||||
|
- 平台架构:[platform-architecture-requirements.md](./platform-architecture-requirements.md)
|
||||||
|
- 权限设计:[permissions-design.md](./permissions-design.md)
|
||||||
|
- MinIO 与 Compute 缓存方案:[minio-compute-cache-plan.md](./minio-compute-cache-plan.md)
|
||||||
|
- 平台治理菜单设计:[platform-governance-menu-design.md](./platform-governance-menu-design.md)
|
||||||
|
- 数据处理设计:[data-process-design.md](./data-process-design.md)
|
||||||
|
- 数据库初始化脚本:[../backend/app/db/sql/000_full_init.sql](../backend/app/db/sql/000_full_init.sql)
|
||||||
|
- 离线部署目录:[../docker/offline](../docker/offline)
|
||||||
|
|
||||||
447
docs/20260812/权限开发进度.md
Normal file
447
docs/20260812/权限开发进度.md
Normal file
@@ -0,0 +1,447 @@
|
|||||||
|
# 权限开发进度
|
||||||
|
|
||||||
|
> 更新时间:2026-08-20
|
||||||
|
> 适用范围:YG_FT 平台当前主工程、Backend、Frontend、Compute Agent、MinIO 资源访问及 `docker/offline/src` 离线源码。
|
||||||
|
> 当前结论:权限 2.0 的基础能力已形成闭环;本轮继续完成 GPU/租户配额原子预留、租户成员邀请与接受、审批动作白名单/幂等/过期处理、GPU 节点与卡冲突校验,并补充租户详情页成员管理。在线数据库迁移已执行并核验,前端构建、后端静态检查、权限用例和基础容器健康检查通过。由于当前只有一个可达算力节点,跨租户双用户、第二节点/多卡并发、节点故障回收仍需后续专项验证,暂不能标记为“全部完成”。按工作包估算,核心权限开发约完成 94%,上线验收约完成 76%。
|
||||||
|
|
||||||
|
## 本轮 11 项完成情况(2026-08-19)
|
||||||
|
|
||||||
|
| 编号 | 权限能力 | 完成内容 | 状态 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | 审批决策安全 | 审批人从当前会话获取,校验指定审批人、租户管理员、禁止申请人自审,记录审批审计 | 已完成 |
|
||||||
|
| 2 | 资源访问申请 | 新增资源访问申请记录、申请权限/期限/理由、审批后自动写入 ACL、支持撤回和过期 | 已完成 |
|
||||||
|
| 3 | 审批策略执行 | 按租户、动作、资源类型匹配策略;高风险操作审批通过后可一次性重试执行 | 已完成 |
|
||||||
|
| 4 | 租户与项目隔离 | `tenant_members`、活跃租户校验、历史资源归属补齐、跨租户 ACL 主体校验、项目软删除 | 已完成 |
|
||||||
|
| 5 | 用户与角色边界 | 禁用用户立即注销会话;租户成员支持 owner/admin/member/viewer,保护最后一个 owner | 已完成 |
|
||||||
|
| 6 | 数据集权限入口 | 预览、来源、版本查询/创建/激活/删除、上传/编辑/下载统一接入资源动作权限 | 已完成 |
|
||||||
|
| 7 | 模型和推理权限 | 训练模型合并/导出分别校验 execute/download;聊天、预加载、批量、卸载绑定授权模型或推理任务 | 已完成 |
|
||||||
|
| 8 | GPU 分配申请 | 普通用户可提交 GPU 分配申请,审批通过后自动分配;管理员可直接分配 | 已完成 |
|
||||||
|
| 9 | 租户配额申请 | 租户成员可提交配额变更申请,平台管理员审批后自动更新配额 | 已完成 |
|
||||||
|
| 10 | 软删除与安全状态 | 资源删除撤销 ACL、取消待处理申请和审批;租户/项目采用软删除状态 | 已完成 |
|
||||||
|
| 11 | 前端按钮与审计 | 审批决策、ACL 编辑、访问申请/撤回按权限显示;下载、导出、审批、GPU 等敏感操作补充结构化审计 | 已完成 |
|
||||||
|
|
||||||
|
### 数据库迁移结果
|
||||||
|
|
||||||
|
- 新增增量迁移:`backend/app/db/sql/009_permission_completion.sql`,可独立兼容旧库执行,并已在当前远程 PostgreSQL 执行成功。
|
||||||
|
- 当前远程库已核验包含:`tenant_members`、`resource_access_requests`、审批策略/执行状态字段、审计结果字段、ACL 生命周期字段、项目/租户软删除字段、数据转换任务租户字段。
|
||||||
|
- 已有用户已补齐到 `tenant_members`,当前迁移后共生成 5 条租户成员关系。
|
||||||
|
- `backend/app/db/sql/000_full_init.sql` 已合并完整结构;离线目录继续只保留该完整脚本。
|
||||||
|
|
||||||
|
### 验证结果
|
||||||
|
|
||||||
|
- 后端相关模块 `python -m py_compile`:通过。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 路径提示和大 chunk 警告,无 TypeScript 错误。
|
||||||
|
- `docker/offline/src` 的后端权限代码、Compute Agent、前端源码和完整初始化 SQL 已与主工程关键文件哈希一致。
|
||||||
|
- 运行中的 Backend、Frontend、Compute API 容器已重启,健康检查通过;当前容器采用源码挂载方式,无需重建镜像即可加载本轮代码。已验证健康接口可用,未登录访问受保护接口返回 401;完整双用户、跨租户和多 GPU 回归仍待执行。
|
||||||
|
|
||||||
|
## 一、检查依据
|
||||||
|
|
||||||
|
本次按代码和初始化脚本逐项核对,主要依据如下:
|
||||||
|
|
||||||
|
- `docs/permissions-design.md`
|
||||||
|
- `backend/app/core/auth.py`
|
||||||
|
- `backend/app/api/v1/endpoints/platform.py`
|
||||||
|
- `backend/app/modules/resource/router.py`
|
||||||
|
- `backend/app/modules/approval/router.py`
|
||||||
|
- `backend/app/modules/tenant/router.py`
|
||||||
|
- `backend/app/modules/system/router.py`
|
||||||
|
- `backend/app/db/platform_store.py`
|
||||||
|
- `backend/app/db/sql/000_full_init.sql`
|
||||||
|
- `frontend/src/stores/auth.ts`
|
||||||
|
- `frontend/src/router/index.ts`
|
||||||
|
- `frontend/src/views/governance/ResourceAclView.vue`
|
||||||
|
|
||||||
|
检查口径不是只判断“是否有接口”,还检查了接口是否验证当前用户、是否校验资源所有权和 ACL、是否校验租户边界、审批结果是否真正改变授权、前端按钮是否与后端动作一致。
|
||||||
|
|
||||||
|
## 二、改造前总体进度(历史基线)
|
||||||
|
|
||||||
|
| 能力模块 | 当前状态 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| 登录、密码、Token 会话 | 已实现基础能力 | 有会话过期、注销、状态校验和登录限流基础,仍需统一续期和失败审计 |
|
||||||
|
| 平台角色与权限码 | 部分实现 | `admin/operator/viewer` 和业务权限码存在,但后端部分业务只校验登录,未统一校验权限码 |
|
||||||
|
| 用户创建与管理 | 部分实现 | 管理员可创建、修改、删除和重置密码;租户范围、角色边界、邀请/申请流程未完成 |
|
||||||
|
| 租户与配额 | 部分实现 | 租户 CRUD、配额和留存策略接口为管理员专属;缺少租户成员、租户管理员和统一隔离 |
|
||||||
|
| 资源 ACL | 已实现基础能力 | 支持用户/角色的 `read/write/execute/download/delete/admin`,授权边界和跨租户限制不足 |
|
||||||
|
| 资源申请 | 未完整实现 | 没有独立的资源访问申请对象,现有审批实例不能可靠地落地 ACL 或配额变更 |
|
||||||
|
| 资源审批 | 部分实现且存在安全缺口 | 模板、实例、步骤和部分高风险操作存在;审批决策接口必须先修复越权问题 |
|
||||||
|
| 基座模型权限 | 平台共享已实现 | 登录用户可查看和使用,上传/修改/删除限制管理员;缺少按租户/用途/配额的精细控制 |
|
||||||
|
| 训练模型权限 | 部分实现 | 所有者、ACL、删除、合并、推理加载已有校验;导出下载动作和派生权限继承还不完整 |
|
||||||
|
| 数据集权限 | 部分实现且入口不一致 | 列表、详情、删除、部分下载和训练/评测使用有校验;上传、编辑、预览、版本接口仍有缺口 |
|
||||||
|
| 训练/评测/推理权限 | 部分实现 | 资源执行权和 GPU 分配已有基础校验;部分聊天、状态和历史入口没有统一鉴权 |
|
||||||
|
| GPU 与节点权限 | 基础能力已实现 | 管理员分配、用户可用 GPU 过滤、原子预留和释放已存在;配额审批和跨节点回收对账仍需完善 |
|
||||||
|
| 前端按钮级权限 | 部分实现 | 页面和菜单有基础控制,资源动作没有集中式权限决策,很多按钮依赖后端报错 |
|
||||||
|
| 审计与软删除 | 部分实现 | 主要写操作有审计,资源软删除已覆盖若干表;actor、下载、拒绝和跨租户查询仍需统一 |
|
||||||
|
|
||||||
|
## 三、已经实现的内容
|
||||||
|
|
||||||
|
### 3.1 认证、角色与用户
|
||||||
|
|
||||||
|
- `get_current_user` 会校验 Bearer Token、用户存在性、用户状态和会话有效期;`sessions` 支持注销和过期时间。
|
||||||
|
- `require_admin` 对管理员专属接口提供后端保护,受保护用户也具备管理员旁路能力。
|
||||||
|
- 用户列表、创建、修改、删除、重置密码均已增加管理员依赖;用户可修改自己的密码。
|
||||||
|
- 用户记录包含 `tenant_id`、`role`、`permissions`、`protected` 等字段,角色权限在初始化脚本中有基础种子数据。
|
||||||
|
- 登录失败限流和旧明文密码升级已经具备基础实现。
|
||||||
|
|
||||||
|
### 3.2 租户、资源和 ACL
|
||||||
|
|
||||||
|
- `tenants` 表和租户 CRUD、配额、留存策略接口已经存在,当前接口统一要求管理员。
|
||||||
|
- 数据集、模型、训练模型、评测任务等核心资源已经具备 `created_by`、`tenant_id` 或任务 payload 中的归属信息。
|
||||||
|
- `acls` 表支持用户和角色两类授权主体,权限包括 `read`、`write`、`execute`、`download`、`delete`、`admin`。
|
||||||
|
- 资源 ACL 查询和全量替换接口已经存在,资源所有者或管理员可以管理 ACL。
|
||||||
|
- 列表接口对数据集、评测、训练和推理任务已增加“本人资源 + ACL 授权资源”的过滤逻辑。
|
||||||
|
- MinIO 预签名、资源清单、对象列表、模型制品、模型血缘和评测报告等新入口已增加登录和资源访问校验。
|
||||||
|
|
||||||
|
### 3.3 训练、评测、推理和 GPU
|
||||||
|
|
||||||
|
- 训练创建会检查训练数据集的 `execute` 权限,并对用户选择的 GPU 执行分配校验。
|
||||||
|
- 评测创建会检查训练模型、数据集的 `execute` 权限,并校验算力节点和 GPU。
|
||||||
|
- 推理任务加载会检查任务及训练模型的执行权,同时使用 `gpu_reservations` 防止同一张 GPU 被并发占用。
|
||||||
|
- GPU 分配接口和用户可用 GPU 查询已经存在,失败、停止、删除、节点异常等路径具备基础释放逻辑。
|
||||||
|
- 训练模型删除、训练任务停止/删除、评测任务删除和推理任务删除已接入部分审批拦截。
|
||||||
|
- 权重合并、导出、缓存准备和 MinIO 归档已经能记录模型、节点、租户和部分血缘信息。
|
||||||
|
|
||||||
|
### 3.4 审计和前端
|
||||||
|
|
||||||
|
- `audit_logs`、`operation_logs` 表及管理员查询/导出页面已经存在。
|
||||||
|
- 资源 ACL 变更、租户管理、模型/数据集/任务等主要写操作已经接入审计或操作日志装饰器。
|
||||||
|
- 前端路由和侧边栏对治理、组织、资源授权、审批、日志、算力节点等页面做了管理员限制。
|
||||||
|
- `ResourceAclView` 已支持资源类型、用户/角色主体和多权限编辑。
|
||||||
|
|
||||||
|
## 四、改造前未实现或存在明显安全缺口(历史基线)
|
||||||
|
|
||||||
|
> 本节保留本轮改造前的检查快照,不代表当前状态。当前剩余问题以“十、当前仍需开发的功能”和“十一、下一步开发计划”为准。
|
||||||
|
|
||||||
|
以下项目属于必须继续开发的内容,优先级高于页面样式和性能优化。
|
||||||
|
|
||||||
|
### 4.1 审批决策不能直接信任请求参数
|
||||||
|
|
||||||
|
`backend/app/modules/approval/router.py` 的审批决策接口当前没有 `get_current_user` 依赖,并且从请求体读取 `approver_id`。调用方可以伪造审批人 ID,属于高风险越权问题。
|
||||||
|
|
||||||
|
必须改为:服务端从当前会话取得审批人;校验其是否为当前步骤指定审批人、租户管理员或平台管理员;校验当前步骤、实例状态和申请人不能自审;审批通过后再执行对应动作或写入 ACL。
|
||||||
|
|
||||||
|
### 4.2 资源申请流程尚未形成
|
||||||
|
|
||||||
|
当前有 `approval_templates`、`approval_instances`、`approval_steps`,但没有独立的资源访问申请记录,也没有统一的“申请资源 -> 审批 -> 授权/配额变更”事务流程:
|
||||||
|
|
||||||
|
- 创建审批实例时未统一验证资源是否存在、申请人是否属于资源租户、申请动作是否合法。
|
||||||
|
- 审批实例批准后不会自动创建 ACL、GPU 分配或租户配额变更。
|
||||||
|
- 不能表达申请权限、申请期限、申请原因、审批后的 ACL 权限和撤销时间。
|
||||||
|
- 审批模板查询和详情接口没有完整的租户/角色范围控制。
|
||||||
|
|
||||||
|
### 4.3 租户隔离不是全量强制
|
||||||
|
|
||||||
|
- 当前用户只有单一 `tenant_id`,没有 `tenant_members` 或租户角色关系;无法支持租户管理员、跨租户平台管理员和成员邀请的清晰边界。
|
||||||
|
- `filter_accessible_resource_ids` 和批量版本主要按 ACL/所有者过滤,不在统一函数中校验资源租户。
|
||||||
|
- 数据集、评测等查询仍对 `tenant_id IS NULL` 做兼容放行;历史数据未完成归属迁移。
|
||||||
|
- `trained_models()` 没有统一 tenant 参数,资源过滤依赖上层二次处理。
|
||||||
|
- ACL 设置接口没有校验授权主体和资源是否属于同一租户,存在跨租户授权风险。
|
||||||
|
- 新建模型、数据集、任务虽然多数会补默认租户,但缺少“租户必须存在且处于 active”的统一校验。
|
||||||
|
|
||||||
|
### 4.4 数据集权限入口不一致
|
||||||
|
|
||||||
|
核心列表、详情、删除、部分下载已校验,但以下文件/版本接口当前未统一接入当前用户和资源权限:
|
||||||
|
|
||||||
|
- 文件预览和记录来源查询。
|
||||||
|
- 文件版本列表、版本内容查询。
|
||||||
|
- 创建、激活、删除数据集文件版本。
|
||||||
|
- 数据集上传接口没有统一的当前用户、写权限和租户校验。
|
||||||
|
- 数据集编辑接口没有统一的当前用户和写权限校验。
|
||||||
|
|
||||||
|
此外,数据集下载和单文件下载目前检查的是 `read`,没有严格使用设计中的独立 `download` 权限。
|
||||||
|
|
||||||
|
### 4.5 模型使用、导出和推理入口不一致
|
||||||
|
|
||||||
|
- 基座模型按平台共享资源处理,登录用户可以查看和使用;但模型名称查询、本地模型列表和部分本地聊天/状态/卸载入口缺少统一鉴权。
|
||||||
|
- 训练模型列表、详情、合并和加载已有 ACL 校验,但导出接口实际属于下载/外发动作,不能只检查 `execute`。
|
||||||
|
- 评测报告下载当前检查 `read`,应单独检查 `download` 并记录下载审计。
|
||||||
|
- 训练模型由训练任务生成时,尚未完整实现“基座模型 + 数据集授权关系”向派生模型权限和血缘策略的统一继承。
|
||||||
|
- 推理聊天接口没有始终绑定到已授权的推理任务、模型资源和指定算力节点,存在绕过模型使用流程的风险。
|
||||||
|
|
||||||
|
### 4.6 用户创建和角色边界不完整
|
||||||
|
|
||||||
|
- 当前只有平台管理员创建用户,普通用户不能申请加入租户,也没有邀请、审批、禁用后会话清理的完整流程。
|
||||||
|
- 用户只能直接挂在一个 `tenant_id` 上,不能表达一个用户属于多个租户或在不同租户中拥有不同角色。
|
||||||
|
- 后端 `create_user` 对非管理员角色会归一为普通用户,前端出现的 `operator` 角色与后端实际行为可能不一致。
|
||||||
|
- 创建用户时缺少租户存在性、租户状态、租户用户配额和角色白名单校验。
|
||||||
|
- 管理员可以创建管理员角色,尚未区分平台管理员和租户管理员的授予权限。
|
||||||
|
|
||||||
|
### 4.7 前端按钮级权限没有闭环
|
||||||
|
|
||||||
|
- 前端已有菜单和路由权限基础,但 `hasPermission` 没有覆盖所有业务路由动作,部分业务页对已登录用户直接开放。
|
||||||
|
- 资源级按钮主要靠资源对象中的所有者字段判断,没有统一使用后端返回的 ACL 决策。
|
||||||
|
- 下载、执行、删除、授权、导出、审批等动作没有统一的 `can(resource, action)` 机制。
|
||||||
|
- 即使按钮隐藏,前端 API 模块仍可能直接调用敏感接口;必须以后端鉴权为最终边界。
|
||||||
|
|
||||||
|
### 4.8 审计、软删除和权限拒绝记录不完整
|
||||||
|
|
||||||
|
- 部分模块的 `_actor` 直接保存 Authorization Token,而不是规范的用户 ID,导致审计主体不一致。
|
||||||
|
- 访问、下载、导出、ACL 授权、审批拒绝、GPU 分配和权限拒绝没有全部统一记录租户、资源和结果。
|
||||||
|
- `record_visit` 公开接口容易被伪造;如果继续保留匿名访问,应明确它不是安全审计日志。
|
||||||
|
- 资源软删除已覆盖模型、训练模型、数据集、评测任务等主要表,但关联 MinIO 对象、ACL、审批和血缘的延迟清理策略还不完整。
|
||||||
|
|
||||||
|
## 五、需要优化的现有流程
|
||||||
|
|
||||||
|
### 5.1 统一权限模型
|
||||||
|
|
||||||
|
将当前分散的 `is_admin`、所有者判断、ACL 查询、租户判断、GPU 判断收敛为统一服务:
|
||||||
|
|
||||||
|
```text
|
||||||
|
认证 -> 平台/租户角色 -> 租户边界 -> 资源所有权/ACL -> 动作权限 -> GPU/配额 -> 审批 -> 审计
|
||||||
|
```
|
||||||
|
|
||||||
|
每个敏感接口都应明确动作,例如 `dataset.read`、`dataset.download`、`dataset.execute`、`trained_model.export`、`inference.load`、`gpu.reserve`,禁止只使用“已登录”作为业务权限。
|
||||||
|
|
||||||
|
### 5.2 重新设计资源申请和审批
|
||||||
|
|
||||||
|
推荐流程:
|
||||||
|
|
||||||
|
```text
|
||||||
|
申请人选择资源和动作
|
||||||
|
-> 校验申请人所属租户和基础权限
|
||||||
|
-> 创建 resource_access_requests
|
||||||
|
-> 根据租户/资源类型匹配审批策略
|
||||||
|
-> 指定审批人完成审批
|
||||||
|
-> 事务内写入 ACL/配额/GPU 预留
|
||||||
|
-> 记录授权有效期、来源和审计
|
||||||
|
```
|
||||||
|
|
||||||
|
审批拒绝、撤回、过期和资源删除都应撤销或冻结对应授权,不能仅修改审批实例状态。
|
||||||
|
|
||||||
|
### 5.3 模型、数据集、训练任务联合授权
|
||||||
|
|
||||||
|
训练、评测、推理分别使用以下最小权限:
|
||||||
|
|
||||||
|
| 场景 | 必须具备的权限 |
|
||||||
|
|---|---|
|
||||||
|
| 查看基座模型 | `model.read`;基座模型默认平台共享 |
|
||||||
|
| 使用基座模型训练 | `model.execute` 或平台共享策略 + 数据集 `execute` |
|
||||||
|
| 下载基座模型 | 单独的 `model.download`,默认不授予 |
|
||||||
|
| 使用训练模型推理 | `trained_model.execute` |
|
||||||
|
| 下载/导出训练模型 | `trained_model.download` 或 `trained_model.export` |
|
||||||
|
| 使用数据集训练/评测 | `dataset.execute` |
|
||||||
|
| 查看数据集 | `dataset.read` |
|
||||||
|
| 下载数据集文件 | `dataset.download` |
|
||||||
|
| 创建训练任务 | 上述资源权限 + 指定节点/GPU 使用权 + 租户配额 |
|
||||||
|
|
||||||
|
训练模型应保留创建者、租户、基座模型、数据集、训练任务和资源版本快照。派生模型默认只对创建者和同租户授权,不应因为基座模型共享而自动公开训练产物。
|
||||||
|
|
||||||
|
## 六、建议的数据库改造
|
||||||
|
|
||||||
|
当前 `000_full_init.sql` 已包含用户、角色、会话、ACL、租户、审批、审计、GPU 分配和 GPU 预留表,但要完成权限 2.0,建议增加或扩展以下结构。实施时必须同步主工程和 `docker/offline/src/backend/app/db/sql/000_full_init.sql`。
|
||||||
|
|
||||||
|
1. `tenant_members`:用户、租户、租户角色、状态、加入来源和有效期,解决一个用户多租户和租户管理员问题。
|
||||||
|
2. `resource_access_requests`:申请人、租户、资源、动作、申请原因、申请权限、有效期、审批状态和最终授权记录。
|
||||||
|
3. `acls` 增加 `tenant_id`、`granted_by`、`source_request_id`、`expires_at`、`revoked_at`,保留授权来源和自动过期能力。
|
||||||
|
4. `approval_templates` 增加 `tenant_id`、`action`、`resource_type`、`scope`、`status`;审批步骤增加主体类型、主体 ID 和租户范围。
|
||||||
|
5. 资源表统一补齐非空租户策略、归属用户、软删除字段和必要索引;历史 NULL 数据通过一次性迁移处理。
|
||||||
|
6. `audit_logs` 增加结果、拒绝原因、request_id、认证会话和结构化 detail,避免把 Token 当作 actor。
|
||||||
|
|
||||||
|
不建议把完整 ACL 和审批状态继续塞入模型、数据集或任务 JSON 字段;JSON 可保留兼容信息,但权限判断应以结构化表为准。
|
||||||
|
|
||||||
|
## 七、历史开发计划(已执行)
|
||||||
|
|
||||||
|
### 第一阶段:P0 安全修复
|
||||||
|
|
||||||
|
- 修复审批决策鉴权和审批人伪造问题。
|
||||||
|
- 补齐数据集文件/版本/上传/编辑接口权限。
|
||||||
|
- 补齐本地模型聊天、状态、卸载、模型名称查询等历史入口鉴权。
|
||||||
|
- 统一 `download`、`execute`、`write`、`delete` 动作检查。
|
||||||
|
- 限制 ACL 授权范围,校验主体存在、同租户和资源归属。
|
||||||
|
- 增加至少 20 个后端权限回归用例,覆盖未登录、本人、同租户他人、跨租户、管理员和已撤销 ACL。
|
||||||
|
|
||||||
|
### 第二阶段:P1 租户和资源申请
|
||||||
|
|
||||||
|
- 引入租户成员和租户角色,明确平台管理员、租户管理员、成员、只读成员边界。
|
||||||
|
- 开发资源访问申请接口和前端申请页面。
|
||||||
|
- 重做审批模板匹配、审批人校验、审批结果落地 ACL、有效期和撤销流程。
|
||||||
|
- 将配额申请、GPU 分配申请、模型导出和跨用户删除纳入审批策略。
|
||||||
|
- 新资源强制租户归属并完成历史数据迁移。
|
||||||
|
|
||||||
|
### 第三阶段:P1 联合资源权限
|
||||||
|
|
||||||
|
- 建立模型/数据集/训练任务/评测任务/推理任务的统一资源授权服务。
|
||||||
|
- 训练前一次性校验基座模型、数据集、节点、GPU 和租户配额。
|
||||||
|
- 训练模型生成时写入血缘和权限来源;推理、评测、合并、导出使用同一套动作权限。
|
||||||
|
- 版本快照、MinIO 对象、算力节点本地缓存沿用同一资源授权结果。
|
||||||
|
|
||||||
|
### 第四阶段:P2 前端和审计
|
||||||
|
|
||||||
|
- 增加统一 `can(resource, action)` 和按钮权限组件。
|
||||||
|
- 授权和审批界面使用用户/租户/资源中文名称,展示授权来源、有效期和审批状态。
|
||||||
|
- 规范审计 actor、租户、请求 ID、资源、动作、结果和失败原因。
|
||||||
|
- 将权限不足、审批中、资源过期、MinIO 不可用分别展示,避免全部显示为通用 500。
|
||||||
|
|
||||||
|
### 第五阶段:P3 测试与上线
|
||||||
|
|
||||||
|
- 新库初始化、增量迁移、离线目录同步和前端构建全部纳入 CI。
|
||||||
|
- 执行跨租户、跨用户、ACL 撤销、审批越权、软删除、MinIO 故障和 GPU 冲突专项测试。
|
||||||
|
- 在第二个可达节点或多卡节点到位后,执行多节点、多 GPU 的权限和并发测试。
|
||||||
|
- 补充权限运维手册:新建租户、创建用户、授权模型/数据集、审批、撤销权限、审计追踪和故障恢复。
|
||||||
|
|
||||||
|
## 八、验收标准
|
||||||
|
|
||||||
|
- 未登录用户不能访问任何模型、数据集、任务、聊天、版本、下载和审批接口。
|
||||||
|
- 用户只能访问所属租户中本人拥有或被明确授权的资源;跨租户 ACL 默认禁止。
|
||||||
|
- 查看、下载、执行、编辑、删除和授权是独立动作,不能用 `read` 替代所有动作。
|
||||||
|
- 普通用户不能伪造审批人、审批其他租户资源或审批自己的申请。
|
||||||
|
- 训练、评测和推理创建必须同时满足资源权限、GPU 权限、节点权限和租户配额。
|
||||||
|
- 审批通过后才产生授权,审批拒绝、撤回、过期和资源删除后授权不会继续生效。
|
||||||
|
- 基座模型共享不等于训练产物共享;训练后的模型和数据集必须按租户、所有者和 ACL 控制。
|
||||||
|
- 前端隐藏按钮不能替代后端校验,直接调用 API 也必须返回明确的 401/403。
|
||||||
|
- 所有敏感操作可通过用户、租户、资源、动作和请求 ID追溯到审计记录。
|
||||||
|
|
||||||
|
## 九、历史实施记录(已完成)
|
||||||
|
|
||||||
|
本轮已完成权限闭环的代码实现:
|
||||||
|
|
||||||
|
1. 统一校验当前会话、会话过期、退出状态和可用租户范围。
|
||||||
|
2. 补齐模型、数据集、训练、评测、推理、数据处理、数据转换、算力、存储和审计入口的登录及模块权限。
|
||||||
|
3. 新增 `tenant_members`,新建用户、项目、数据集、训练任务和数据处理任务写入当前租户与创建者。
|
||||||
|
4. ACL 写入校验主体存在性、状态、租户边界、有效期和撤销状态;普通所有者不能授予 `delete/admin` 或跨租户权限。
|
||||||
|
5. 新增 `resource_access_requests`;审批人由当前会话确定,禁止伪造审批人和申请人自审;审批通过后才落 ACL。
|
||||||
|
6. 训练、评测和推理统一校验数据集、基座模型、训练模型、任务、节点和 GPU 使用权限;下载和导出使用独立的 `download` 权限。
|
||||||
|
7. 前端认证 store 新增 `can(resource, action)`,模型管理和审批页面按权限显示操作按钮,后端 401/403 仍是最终防线。
|
||||||
|
8. 新增 `009_permission_completion.sql`;主工程与离线目录的完整初始化 SQL 已同步且 SHA-256 一致,离线目录只保留完整初始化脚本。
|
||||||
|
|
||||||
|
### 本轮验证
|
||||||
|
|
||||||
|
- 后端权限相关文件 `py_compile` 通过。
|
||||||
|
- 前端 `npm run build` 通过;仅保留已有字体路径和 chunk size 警告。
|
||||||
|
- 当前 WSL 容器已确认 Backend、Frontend、Compute、Redis、MinIO 均运行;基础健康和未登录拦截已验证,历史治理测试夹具与当前鉴权/数据库行为不完全兼容,不能替代新的权限专项回归。
|
||||||
|
|
||||||
|
### 上线前验证
|
||||||
|
|
||||||
|
- 第二个可达节点或多卡节点到位后的多租户、多 GPU 并行压力测试。
|
||||||
|
- 对已执行的 `009_permission_completion.sql` 进行旧数据租户归属、ACL、审批和软删除记录对账。
|
||||||
|
- 更新 Backend/Frontend 容器后重新执行治理测试和权限专项测试。
|
||||||
|
|
||||||
|
## 十、当前复核结论(2026-08-19)
|
||||||
|
|
||||||
|
### 10.1 已完成的基础能力
|
||||||
|
|
||||||
|
本轮 11 项权限改造已经形成基础闭环:会话和用户状态校验、租户成员关系、资源 ACL、资源访问申请、审批决策安全、模型/数据集/训练/评测/推理入口权限、GPU 分配申请、配额申请、软删除和前端基础按钮控制均已落地;数据库迁移和完整初始化 SQL 已同步到主工程及离线目录。
|
||||||
|
|
||||||
|
这些能力可以作为后续完善的基础,但“接口存在”不等于“所有资源和所有异常路径均已达到上线标准”。尤其是密钥暴露、配额实际拦截、资源列表一致性和专项测试仍需要继续处理。
|
||||||
|
|
||||||
|
### 10.2 仍需开发的功能
|
||||||
|
|
||||||
|
| 优先级 | 功能 | 当前缺口 | 处理结论 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| P0 | 模型密钥和敏感信息保护 | 模型列表、详情、名称查询、创建和更新响应已移除真实 `api_key`,仍需继续排查任务详情、日志和其他敏感配置输出 | 基础闭环已完成;继续做全量敏感字段扫描和受控密钥管理 |
|
||||||
|
| P0 | 统一资源动作策略 | 已增加资源动作白名单和 `export -> download` 归一化,但各业务入口仍需逐步迁移到统一授权服务 | 第一阶段已完成;继续完成全量入口收敛 |
|
||||||
|
| P0 | 数据转换、数据处理和存储权限一致性 | 部分列表、文件、预览、版本、缓存和 MinIO 对象入口仍需逐接口核对“租户 + 所有者/ACL + 动作” | 必须完成全量入口审计,尤其是 `read/download/execute/write/delete` 的区分 |
|
||||||
|
| P0 | 审计失败链路 | 审计装饰器已记录失败结果、原因、请求 ID、会话 ID 和租户;手工审计入口仍需继续统一 | 基础闭环已完成;继续补齐所有权限拒绝和异常入口 |
|
||||||
|
| P1 | 租户成员生命周期 | 已有成员表和角色,但缺少邀请、加入申请、审批、过期、移除和前端租户切换的完整流程 | 需要开发,明确平台管理员与租户管理员的授予边界 |
|
||||||
|
| P1 | 配额强制执行 | 配额申请和审批已实现,但训练、评测、推理、存储等资源消耗尚未全部进行原子配额检查和扣减 | 需要开发配额使用量/预留量/释放量账本,不能只保存配额配置 |
|
||||||
|
| P1 | GPU 分配强校验 | 申请链路已实现,但分配前仍需统一校验节点状态、GPU 存在性、冲突、租户配额和释放对账 | 需要开发原子预留、超时回收和节点故障对账 |
|
||||||
|
| P1 | 审批策略完整性 | 需要严格限制动作白名单、资源类型、模板作用域和重复申请;过期处理不应只依赖查询触发 | 需要补充定时过期、幂等键、执行失败重试和策略管理边界 |
|
||||||
|
| P1 | 派生模型和数据血缘授权 | 基座模型、数据集、训练模型之间已有部分血缘,但权限来源、版本快照和派生资源默认授权规则还不够统一 | 需要固化“创建者 + 租户 + 显式 ACL”,禁止共享基座模型自动公开训练产物 |
|
||||||
|
| P1 | 用户软删除和对象清理 | 用户及部分关联关系仍有物理删除风险;MinIO 对象、ACL、审批、缓存的异步清理缺少统一编排 | 需要补齐软删除、撤销、延迟清理和失败重试 |
|
||||||
|
| P2 | 前端资源级权限体验 | 已有菜单/按钮级基础控制,但缺少统一 `can(resource, action)`、授权来源、有效期、审批中和 403 状态展示 | 需要开发统一权限组件和错误状态处理,后端校验仍是最终边界 |
|
||||||
|
| P2 | 权限专项测试与上线检查 | 基础构建、静态检查、健康接口已验证,尚未完成双用户、跨租户、撤销、过期、MinIO 故障、GPU 冲突的全流程矩阵 | 必须补充自动化测试、迁移回归和离线部署验收 |
|
||||||
|
|
||||||
|
### 10.3 需要优化的设计
|
||||||
|
|
||||||
|
1. **从“角色判断”改为“动作授权”**:统一使用 `resource_type + resource_id + action + tenant_id + actor` 判断,平台管理员只作为明确的管理范围,不再作为各业务模块的隐式旁路。
|
||||||
|
2. **区分平台角色和租户角色**:`users.role` 只表达平台级角色,`tenant_members.role` 表达租户内角色;禁止通过租户成员关系授予平台管理员权限。
|
||||||
|
3. **区分查看、下载、执行、编辑、删除、授权**:`read` 不能替代 `download`,`execute` 不能替代 `export`,高风险动作必须绑定审批和审计。
|
||||||
|
4. **统一资源列表和详情规则**:列表、详情、文件、版本、预览、缓存、下载和导出必须调用同一授权服务,避免“列表看不到但接口可访问”或“列表能看到但操作必然 403”。
|
||||||
|
5. **权限与配额采用预留模型**:任务提交时原子预留 GPU、显存、并发数和存储额度,任务完成、失败、取消和节点失联时统一释放或对账。
|
||||||
|
6. **审批采用可执行状态机**:申请、审批、拒绝、撤回、过期、执行中、执行失败、已执行状态分离;审批结果应有幂等执行记录,避免重复授权或重复分配。
|
||||||
|
7. **敏感字段默认拒绝返回**:API key、对象内部凭据、节点访问凭据不能随普通资源详情返回;日志、审计和错误信息也不能泄露 Token、密码或完整连接串。
|
||||||
|
8. **安全审计与访问统计分离**:权限成功、拒绝、下载、导出、授权、审批和异常进入不可篡改审计;页面访问统计单独存储,避免污染安全审计记录。
|
||||||
|
|
||||||
|
## 十一、下一步开发计划
|
||||||
|
|
||||||
|
### 阶段一:P0 安全封堵与统一策略
|
||||||
|
|
||||||
|
1. 对模型列表、详情、名称查询及相关 DTO 做 API key/凭据脱敏,增加“仅后端内部读取”的封装。(基础闭环已完成,继续扫描任务和日志输出)
|
||||||
|
2. 建立统一 `authorize(resource, action)` 服务,定义动作白名单和平台管理员、租户管理员、所有者、ACL 的判定顺序。(已完成动作白名单,继续迁移全部入口)
|
||||||
|
3. 逐项审计平台、数据处理、数据转换、存储、MinIO、模型、数据集、训练、评测、推理的列表、详情、预览、下载、导出、缓存和删除入口。
|
||||||
|
4. 统一记录权限成功、拒绝和异常审计,补齐 `tenant_id`、`resource_id`、`action`、`request_id`、`session_id`、`reason` 和 `result`。(装饰器基础闭环已完成,继续覆盖手工记录入口)
|
||||||
|
|
||||||
|
### 阶段二:租户成员、审批和配额闭环
|
||||||
|
|
||||||
|
1. 开发租户邀请/加入申请/审批/移除/过期流程及前端租户切换。
|
||||||
|
2. 为审批模板增加动作和资源类型白名单、租户作用域、幂等键、过期任务和执行失败重试。
|
||||||
|
3. 建立配额预留账本,训练、评测、推理、存储和 GPU 任务提交前统一原子检查;任务终态和故障恢复统一释放。
|
||||||
|
4. 完善 GPU 节点、GPU 卡、租户、用户和任务的占用关系校验,覆盖冲突、超时、节点不可达和服务重启恢复。
|
||||||
|
|
||||||
|
### 阶段三:资源血缘和生命周期
|
||||||
|
|
||||||
|
1. 固化基座模型、数据集、数据处理结果、训练任务、训练模型、评测和推理任务的资源血缘及版本快照。
|
||||||
|
2. 明确派生模型默认授权规则,训练产物不因基座模型共享而自动对外公开。
|
||||||
|
3. 补齐用户、租户、资源、ACL、审批、MinIO 对象和本地缓存的软删除、撤销、延迟清理和失败重试。
|
||||||
|
|
||||||
|
### 阶段四:前端权限体验与测试
|
||||||
|
|
||||||
|
1. 开发统一资源级权限组件,按动作隐藏/禁用按钮,并展示授权来源、有效期、审批状态和明确的 401/403 原因。
|
||||||
|
2. 将审批、ACL、审计列表中的用户、租户、资源 ID 映射为可读名称,同时保留 ID 作为详情字段。
|
||||||
|
3. 编写并执行权限专项测试矩阵:未登录、同租户成员、资源所有者、ACL 授权、跨租户、禁用用户、会话注销、审批撤回/过期、软删除、MinIO 故障、GPU 冲突和配额不足。
|
||||||
|
4. 在第二个可达节点或多卡节点准备后执行多节点、多 GPU 并发及故障恢复测试;完成主工程与 `docker/offline/src` 的源码、初始化 SQL、迁移和部署文档一致性检查。
|
||||||
|
|
||||||
|
## 十二、下一阶段完成标准
|
||||||
|
|
||||||
|
- 普通资源接口不再返回 API key、密码、Token 或节点访问凭据。
|
||||||
|
- 所有资源入口都经过统一的租户边界和动作授权,跨租户访问返回明确 403。
|
||||||
|
- 训练、评测、推理创建同时满足资源权限、GPU 预留和租户配额,失败时不会留下孤儿占用。
|
||||||
|
- 审批只能由合法审批人执行,申请人不能自审;重复回调不会重复授权、分配或扣减配额。
|
||||||
|
- 权限拒绝、下载、导出、授权、审批和异常均能按用户、租户、资源和请求 ID追溯。
|
||||||
|
- 主工程和离线目录初始化新库均无缺表、缺字段;迁移可重复执行且不破坏既有数据。
|
||||||
|
- 权限专项自动化测试通过,且完成至少一次双用户、跨租户和 MinIO 故障场景的真实容器验证。
|
||||||
|
|
||||||
|
## 十三、上一阶段权限闭环开发结果(2026-08-20)
|
||||||
|
|
||||||
|
本轮围绕 P0 安全封堵完成了一个可验证的权限闭环:
|
||||||
|
|
||||||
|
1. **模型凭据不出接口**:模型列表、详情、按名称查询、创建、更新和用途更新响应统一经过公开 DTO 脱敏,移除 `api_key`、密码、Token 等字段,仅返回 `api_key_configured` 布尔状态;后端内部评测、数据生成仍使用数据库中的真实配置。
|
||||||
|
2. **编辑密钥不被意外清空**:前端编辑在线模型时不回填真实密钥,留空表示保留已有配置,只有管理员主动输入新值时才提交替换。
|
||||||
|
3. **资源动作统一入口**:增加资源动作白名单 `read/write/execute/download/export/delete/admin`,并将 `export` 统一归一到下载权限,非法动作默认拒绝,避免把模块权限误当成资源权限。
|
||||||
|
4. **失败审计闭环**:审计装饰器对成功和异常分别记录,失败记录包含结果、失败原因、租户、请求 ID、会话 ID和耗时,并对异常中的常见凭据进行脱敏。
|
||||||
|
5. **访问统计受控**:模块访问统计只接受平台已登记的模块名,不能通过请求参数向安全审计表写入任意动作;审计 CSV 导出改为标准 CSV 编码,并补充结果、原因、请求和会话字段。
|
||||||
|
6. **离线版本同步**:上述后端权限代码、前端模型编辑代码和权限安全回归用例已同步到 `docker/offline/src`;本轮未新增数据库字段,因此 `000_full_init.sql` 无需变更。
|
||||||
|
|
||||||
|
### 上一阶段验证结果
|
||||||
|
|
||||||
|
- 后端权限相关模块 `python -m py_compile`:通过。
|
||||||
|
- WSL Backend 容器执行权限安全用例:8 passed;仅有 pytest 缓存目录只读警告。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 路径和 chunk size 警告。
|
||||||
|
- 容器接口验证:未登录访问模型接口返回 401;管理员访问模型列表/详情时响应不包含 `api_key`,仅返回 `api_key_configured`。
|
||||||
|
- Backend 容器已重启并加载当前源码;数据库无需迁移。
|
||||||
|
|
||||||
|
### 上一阶段未覆盖的范围(已在第十四节继续处理)
|
||||||
|
|
||||||
|
本轮没有声称完成配额账本、GPU 原子预留、租户邀请/加入申请、审批定时过期、所有资源入口的逐接口审计,以及第二节点/多 GPU 的真实并发测试;这些仍按“十一、下一步开发计划”执行。
|
||||||
|
|
||||||
|
## 十四、本轮继续开发结果(2026-08-20)
|
||||||
|
|
||||||
|
### 已完成
|
||||||
|
|
||||||
|
1. 新增 `tenant_quota_reservations` 配额预留账本,支持按租户统计 GPU 预留量、原子限制 GPU 配额,以及任务完成、失败、停止、删除和节点故障时释放预留。
|
||||||
|
2. 训练、评测和推理统一接入租户 GPU 配额预留;评测/推理使用的 `gpu_reservations` 与配额账本在同一事务内创建或释放,避免只释放算力卡而遗留配额占用。
|
||||||
|
3. 租户成员支持邀请、接受、过期、角色更新和移除;邀请不允许授予 `owner`,成员状态和租户有效性由后端校验,租户详情页补充成员管理和 GPU 配额使用量展示。
|
||||||
|
4. 审批动作增加白名单,非法动作拒绝创建;相同申请人在同一资源上的待审批申请幂等复用;审批实例读取时自动处理过期状态,资源访问申请也避免重复创建。
|
||||||
|
5. GPU 分配增加节点启用状态、GPU 卡存在性、用户 active 状态和同卡跨用户冲突校验;冲突返回 409,不再静默覆盖或产生重复授权。
|
||||||
|
6. 完整初始化 SQL 增加配额预留表和索引,新增 `010_permission_quota_membership.sql` 增量迁移;主工程相关源码、前端租户 API/页面和 SQL 已同步到离线源码目录。
|
||||||
|
|
||||||
|
### 本轮验证
|
||||||
|
|
||||||
|
- WSL Backend 容器重启后,`tenant_quota_reservations` 可正常查询,默认租户配额使用量返回正常。
|
||||||
|
- WSL Backend 容器执行 `test_permission_security.py`:3 passed。
|
||||||
|
- 后端相关文件 `python -m py_compile`:通过。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 资源路径和大 chunk 警告。
|
||||||
|
|
||||||
|
### 必须后续验证的场景
|
||||||
|
|
||||||
|
1. 创建第二个 active 用户和第二个租户,验证邀请接受、租户切换、跨租户资源访问均按预期返回 401/403。
|
||||||
|
2. 将租户 GPU 配额设为 1,同时提交两个需要 GPU 的训练/评测/推理任务,确认第二个任务被拒绝;第一个任务终态后确认配额可再次使用。
|
||||||
|
3. 在同一多卡节点上让两个用户申请同一张卡,确认审批执行阶段冲突返回 409,另一张空闲卡仍可正常分配。
|
||||||
|
4. 构造过期审批、重复提交、审批拒绝/撤回,确认不会重复创建 ACL、GPU 分配或配额预留。
|
||||||
|
5. 准备第二个可达节点或多卡节点,执行训练、评测、推理的多节点/多 GPU 并发和节点失联回收测试。
|
||||||
|
6. 注入 MinIO 故障,确认任务失败后 GPU 与租户配额均能释放,恢复 MinIO 后可以重新提交任务。
|
||||||
|
|
||||||
|
### 下一步计划
|
||||||
|
|
||||||
|
- 补齐数据处理、数据转换、MinIO 对象、缓存、版本和报告下载等剩余资源入口的逐接口动作授权审计。
|
||||||
|
- 将用户物理删除改为统一软删除,并编排成员关系、ACL、审批、MinIO 对象和本地缓存的撤销与延迟清理。
|
||||||
|
- 将上述后续验证场景加入自动化测试矩阵和离线部署验收脚本;在多节点条件满足后更新本文件的上线验收进度。
|
||||||
38
docs/database-migration.md
Normal file
38
docs/database-migration.md
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# 数据库迁移说明
|
||||||
|
|
||||||
|
## 当前迁移文件
|
||||||
|
|
||||||
|
- 新库初始化:`backend/app/db/sql/000_full_init.sql`
|
||||||
|
- 已有数据库增量迁移:`backend/app/db/sql/005_storage_progress_migration.sql`
|
||||||
|
- GPU 预留迁移:`backend/app/db/sql/006_gpu_reservations.sql`
|
||||||
|
- 平台闭环迁移:`backend/app/db/sql/007_platform_completion.sql`
|
||||||
|
- 离线部署使用:`docker/offline/src/backend/app/db/sql/000_full_init.sql`
|
||||||
|
|
||||||
|
## 执行方式
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for migration in 005_storage_progress_migration.sql 006_gpu_reservations.sql 007_platform_completion.sql; do
|
||||||
|
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "backend/app/db/sql/$migration"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
迁移前应完成数据库备份,并保存以下结构快照:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT table_name, column_name, data_type
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
ORDER BY table_name, ordinal_position;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 本次迁移内容
|
||||||
|
|
||||||
|
- 为算力节点资源副本增加 `version_id` 和 `storage_object_id`,用于记录 MinIO 版本与本地缓存对应关系。
|
||||||
|
- 补齐 `storage_objects.metadata`、模型产物、数据集文件和数据转换任务的 MinIO 字段。
|
||||||
|
- 补齐评测任务软删除、创建者、租户和报告对象字段。
|
||||||
|
- 增加活动评测任务、资源副本和 MinIO 对象索引。
|
||||||
|
- 增加 GPU 预留表,统一训练、推理、评测的卡级占用约束。
|
||||||
|
- 增加模型导出创建者/租户/归档状态字段。
|
||||||
|
- 增加 MinIO 对象软删除清理、缓存版本校验和访问保护字段及清理任务表。
|
||||||
|
|
||||||
|
`000_full_init.sql` 已包含相同的幂等变更,新增数据库直接执行初始化脚本即可;已有数据库不要依赖容器重启自动完成迁移。
|
||||||
@@ -111,7 +111,7 @@
|
|||||||
| 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` |
|
| 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` |
|
||||||
| 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` |
|
| 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` |
|
||||||
| 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` |
|
| 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` |
|
||||||
| 系统设置 - 运行日志 | 系统/训练日志;管理员可查看审计记录、操作诊断 | `/logs` |
|
| 系统设置 - 运行日志 | 普通用户查看本人操作记录;管理员可查看系统/训练日志、审计记录和全量操作诊断 | `/logs` |
|
||||||
| 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` |
|
| 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` |
|
||||||
|
|
||||||
### 2.4 重置用户密码
|
### 2.4 重置用户密码
|
||||||
|
|||||||
@@ -131,7 +131,9 @@
|
|||||||
| `compute` | `/compute` | 算力节点 |
|
| `compute` | `/compute` | 算力节点 |
|
||||||
| `hardware` | `/hardware` | 平台性能 |
|
| `hardware` | `/hardware` | 平台性能 |
|
||||||
| `logs` | `/logs`, `/training-log/:id` | 查看日志 |
|
| `logs` | `/logs`, `/training-log/:id` | 查看日志 |
|
||||||
| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances`, `/logs` | 平台治理和运行日志(管理员) |
|
| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances` | 平台治理管理功能(管理员) |
|
||||||
|
|
||||||
|
运行日志采用分层访问模型:拥有 `logs` 权限的普通用户可以进入 `/logs`,页面只展示其本人产生的操作记录,后端按当前用户 ID 强制过滤,忽略普通用户传入的跨用户筛选条件。管理员在同一入口保留系统日志、训练日志、审计记录和全量操作诊断能力。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
191
docs/当前项目开发进度.md
191
docs/当前项目开发进度.md
@@ -1,6 +1,6 @@
|
|||||||
# 当前项目开发进度
|
# 当前项目开发进度
|
||||||
|
|
||||||
> 评估基线:2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。
|
> 评估基线:2026-08-20 当前工作区代码、数据库初始化脚本、增量迁移脚本、Docker 部署文件、前端页面和 WSL 容器验证结果。
|
||||||
>
|
>
|
||||||
> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容,统一标记为“部分完成”或“未完成”。
|
> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容,统一标记为“部分完成”或“未完成”。
|
||||||
|
|
||||||
@@ -24,14 +24,16 @@ FastAPI Backend API
|
|||||||
|
|
||||||
| 范围 | 当前状态 | 结论 |
|
| 范围 | 当前状态 | 结论 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 |
|
| 平台基础架构 | 已完成开发基线 | 前端、Backend、PostgreSQL、Redis、MinIO、Compute Agent 和离线 Docker 编排均已具备 |
|
||||||
| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 |
|
| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测主要流程已闭环 |
|
||||||
| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入,跨节点一致性和失败恢复仍需加强 |
|
| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入,跨节点一致性和失败恢复仍需加强 |
|
||||||
| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现,但完整的租户/项目隔离尚未闭环 |
|
| 权限治理 | 功能闭环基本完成 | 登录、角色、权限码、ACL、审批、资源申请、GPU 申请和审计已实现,历史数据迁移与全量回归待完成 |
|
||||||
| MinIO 统一存储 | 部分完成 | 大文件和模型已接入,仍存在兼容性的本地路径和部分数据双写/回退路径 |
|
| MinIO 统一存储 | 基本完成 | 大文件、模型、产物、报告和节点缓存已接入;小型内容保留数据库属于明确的性能策略 |
|
||||||
| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 |
|
| 生产可靠性 | 部分完成 | 重试、补偿、缓存治理和故障快速失败已实现,生命周期运维、高可用、备份和大规模压力测试未完成 |
|
||||||
| 前端体验 | 基本可用,需优化 | 构建问题已持续修复,但页面响应等待、首屏体积和部分错误提示仍需优化 |
|
| 前端体验 | 基本可用,需优化 | 构建问题已持续修复,但页面响应等待、首屏体积和部分错误提示仍需优化 |
|
||||||
|
|
||||||
|
当前结论:项目已经进入“核心功能闭环后的专项验证和上线前加固阶段”,不再是单纯的功能骨架开发。尚不能宣称生产可上线,主要风险集中在历史数据迁移、跨节点多 GPU 并发、评测指标质量、MinIO 运维和全量回归测试。
|
||||||
|
|
||||||
## 二、已完成的功能
|
## 二、已完成的功能
|
||||||
|
|
||||||
### 2.1 平台基础与部署
|
### 2.1 平台基础与部署
|
||||||
@@ -55,6 +57,7 @@ FastAPI Backend API
|
|||||||
- 已建立资源 ACL 管理页面和相关接口,可对用户或角色授予资源级权限。
|
- 已建立资源 ACL 管理页面和相关接口,可对用户或角色授予资源级权限。
|
||||||
- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。
|
- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。
|
||||||
- 已建立运行日志、审计日志查询页面及审计记录写入机制。
|
- 已建立运行日志、审计日志查询页面及审计记录写入机制。
|
||||||
|
- “运行日志”已按分层模型实现:普通用户只查看本人操作日志,管理员查看系统日志、训练日志、审计记录和全量操作诊断。
|
||||||
- 已加入软删除相关字段和部分删除逻辑,避免直接物理删除业务资源。
|
- 已加入软删除相关字段和部分删除逻辑,避免直接物理删除业务资源。
|
||||||
|
|
||||||
### 2.3 算力节点与 GPU 资源
|
### 2.3 算力节点与 GPU 资源
|
||||||
@@ -141,72 +144,61 @@ FastAPI Backend API
|
|||||||
|
|
||||||
- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。
|
- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。
|
||||||
- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段,这是当前已确认的小文件性能策略,不是错误,但必须统一记录来源、大小、校验值和版本。
|
- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段,这是当前已确认的小文件性能策略,不是错误,但必须统一记录来源、大小、校验值和版本。
|
||||||
- 大文件在部分代码路径中仍通过 `read()` 或 `put_bytes()` 一次性读入内存,未完成流式或分片上传。
|
- Compute Agent 节点侧的大文件上传已改为流式读取;Backend 端部分历史上传/下载路径仍未完成统一的分片传输。
|
||||||
- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联,数据库没有直接外键,删除和数据一致性需要应用层保证。
|
- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联,数据库没有直接外键,删除和数据一致性需要应用层保证。
|
||||||
- 删除业务资源后,对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。
|
- 删除业务资源后,对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。
|
||||||
|
|
||||||
### 3.2 MinIO 预签名接口的权限边界需要加强
|
### 3.2 MinIO 预签名接口的权限边界
|
||||||
|
|
||||||
当前预签名接口已经存在,但 PUT 上传场景仍需要重点补强:
|
资源写权限、服务端对象 Key、上传完成确认和对象大小校验已经完成;仍需补充 Content-Type 白名单、对象过期回收和完整审计事件。
|
||||||
|
|
||||||
- 需要根据资源类型和资源 ID 校验当前用户的写权限,而不应只校验读取权限。
|
|
||||||
- 需要服务端生成并校验对象 Key,避免客户端任意写入其他用户或其他资源的对象路径。
|
|
||||||
- 需要增加上传完成确认接口,校验对象实际存在、大小和校验值后再写入业务表。
|
|
||||||
- 需要限制允许的 Bucket、Content-Type、大小和有效期。
|
|
||||||
- 需要记录预签名创建、上传完成、失败和过期事件,便于审计。
|
|
||||||
|
|
||||||
### 3.3 激活版本和跨节点资源版本仍需加强
|
### 3.3 激活版本和跨节点资源版本仍需加强
|
||||||
|
|
||||||
数据集已经有 `active_version_id` 和版本表,但以下场景仍需补充:
|
数据集已经有 `active_version_id` 和版本表,但以下场景仍需补充:
|
||||||
|
|
||||||
- 训练、推理和评测必须只使用资源当前激活版本,并在任务创建时固化版本 ID。
|
- 训练、评测、模型对比和权重合并已在任务创建时固化资源版本 ID;推理服务启动和导出任务仍需统一补齐。
|
||||||
- 同一文件名的不同版本不能只依靠文件名同步,应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。
|
- 同一文件名的不同版本不能只依靠文件名同步,应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。
|
||||||
- 已创建任务在后续切换激活版本后,不能被意外切换到新版本。
|
- 已创建任务在后续切换激活版本后,不能被意外切换到新版本。
|
||||||
- 需要为每个准备到算力节点的资源保存版本、对象 ETag/校验值和本地路径清单。
|
- 已为资源副本保存版本、对象 ID 和本地路径;对象 ETag/校验值及多文件 manifest 仍需补齐。
|
||||||
- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。
|
- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。
|
||||||
|
|
||||||
### 3.4 权限 2.0 尚未完全落地
|
### 3.4 权限 2.0 已基本闭环,仍需历史迁移和全量回归
|
||||||
|
|
||||||
已有用户、角色、权限码、ACL、审批和审计基础,但仍存在以下差距:
|
已有用户、角色、权限码、ACL、审批、资源申请、GPU 申请、审计和普通用户自助日志能力,仍存在以下差距:
|
||||||
|
|
||||||
- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。
|
- 历史数据中的 NULL 租户归属仍需盘点和迁移,之后再增加更严格的数据库约束。
|
||||||
- 项目空间设计已经讨论过取消,但数据库中仍保留 `projects`、`project_members` 等历史结构,需要明确兼容策略和最终迁移方式。
|
- 项目空间已从前端和新业务流程移除,但 `projects`、`project_members` 等历史表仍需兼容迁移和下线方案。
|
||||||
- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。
|
- 训练、推理、评测、模型合并、导出、缓存准备和下载接口仍需执行全量跨用户/跨租户回归。
|
||||||
- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。
|
- 资源动作鉴权已经分散接入,仍需继续统一授权函数和后端默认拒绝策略。
|
||||||
- 前端按钮权限已经有基础实现,但不能替代后端鉴权;仍需要对所有关键动作进行后端默认拒绝校验。
|
- 审批策略、配额、GPU 预留和资源 ACL 的组合规则需要继续用自动化矩阵验证。
|
||||||
- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。
|
|
||||||
|
|
||||||
### 3.5 模型合并、导出和评测报告闭环不足
|
### 3.5 模型合并、导出和评测报告已基本闭环
|
||||||
|
|
||||||
- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立,但失败时的清理、重试和幂等性仍需加强。
|
- 权重合并前自动准备 Base Model 和 Adapter、失败清理、重试和 MinIO 归档路径已经接入,仍需进行更多幂等和长任务测试。
|
||||||
- 合并结果归档到 MinIO 的逻辑主要依赖任务完成轮询,服务重启或轮询中断时可能需要补偿扫描。
|
- 模型导出已经具备创建、节点准备、CPU 执行、制品血缘、归档轮询、权限和前端入口,真实大模型多格式压力测试待完成。
|
||||||
- 模型导出任务目前有查询模型和表结构,但完整的创建、执行、进度、失败重试和下载闭环尚未完成。
|
- 评测报告接口已支持 MinIO 流式返回和历史数据库兼容结果,但报告为空、报告版本和下载流程仍需回归。
|
||||||
- 评测结果和报告字段已经存在,但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。
|
- 评测指标配置和执行器返回指标之间仍需强类型映射,避免前端显示为通用的 `custom`。
|
||||||
- 评测指标配置和执行器返回指标之间仍需要强类型映射,避免前端显示为通用的 `custom`。
|
|
||||||
|
|
||||||
### 3.6 GPU 资源分配需要统一到所有任务类型
|
### 3.6 GPU 资源分配已统一接入,待多节点多卡验证
|
||||||
|
|
||||||
- 训练已经有较完整的节点/GPU 选择和预检流程。
|
- 训练、推理和评测已经接入节点/GPU 选择、原子预留和释放流程。
|
||||||
- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码,但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。
|
- 单节点同卡冲突已经验证;多节点、多 GPU 并行和长时间压力仍需真实环境验证。
|
||||||
- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。
|
- 需要继续处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。
|
||||||
- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。
|
- 训练详情中的显存使用量、GPU 使用率等指标需要继续校验采样时间、单位、空值和任务对应关系。
|
||||||
- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报,仍需要校验采样时间、单位、空值和任务对应关系。
|
|
||||||
|
|
||||||
## 四、尚未完成的功能
|
## 四、待完成或待专项验证的功能
|
||||||
|
|
||||||
以下功能在当前代码中没有形成可验收的完整闭环,或仍处于设计/基础代码阶段:
|
以下内容已经有代码基础或单节点验证,但尚未达到上线验收条件:
|
||||||
|
|
||||||
1. **完整的租户隔离和资源继承模型**:所有列表、详情、下载、缓存、训练、推理、评测和导出接口都需要统一的租户范围过滤。
|
1. **历史租户和项目数据迁移**:新数据已补齐租户/ACL 基础校验,历史 NULL 租户和项目兼容表仍需迁移、校验和下线方案。
|
||||||
2. **项目取消后的正式数据迁移方案**:需要决定历史项目数据如何归属到用户或租户,并提供一次性迁移脚本和回滚方案。
|
2. **MinIO 生命周期运维**:预签名安全、软删除、失败记录、重试、孤儿扫描和管理员清理接口已存在,自动定时清理和运行监控仍需接入。
|
||||||
3. **预签名上传完成确认和对象校验**:包括 Key 白名单、ACL、大小限制、哈希/ETag 和状态回写。
|
3. **Compute Agent 缓存生产治理**:容量上限、LRU、TTL、保护窗口、版本/校验清单已实现,运行中任务动态保护、磁盘告警和大规模缓存压力待验证。
|
||||||
4. **MinIO 对象生命周期管理**:软删除后的延迟删除、失败重试、孤儿对象扫描、对象引用检查和管理员清理入口。
|
4. **多节点、多 GPU 并行**:节点/GPU 选择、原子预留、冲突拒绝和释放已实现,真实多节点、多卡并发尚未执行。
|
||||||
5. **Compute Agent 缓存治理**:容量上限、LRU/TTL、运行任务保护、磁盘占用监控、缓存清单和版本校验。
|
5. **模型导出和评测报告质量**:创建、执行、归档、权限和下载接口已建立,真实大模型多格式导出、报告版本和指标质量仍需专项验证。
|
||||||
6. **统一的资源归档编排器**:训练、合并、评测和推理相关产物需要支持断点恢复、幂等重试和服务重启补偿。
|
6. **超大文件传输**:单文件流式下载和 Compute Agent 流式上传已完成,多文件 ZIP、分片上传和断点续传未完成。
|
||||||
7. **模型导出完整流程**:导出任务创建、格式/量化参数、进度、失败重试、MinIO 归档和下载权限。
|
7. **生产级 MinIO 安全和高可用**:开发环境接入和故障快速失败已完成,TLS、密钥管理、Console 隔离、容量监控、备份恢复需在生产阶段实施。
|
||||||
8. **流式和分片文件传输**:避免大文件上传、下载和对象复制时将完整内容读入 Backend 或 Compute Agent 内存。
|
8. **全量端到端测试和持续集成**:已有前端构建、Backend 编译和存储安全 CI 基线,权限全矩阵、跨节点并行和故障注入仍需扩展。
|
||||||
9. **生产级 MinIO 安全和高可用**:默认密钥替换、TLS、网络访问控制、管理员 Console 隔离、容量监控、备份和恢复。
|
9. **正式数据库迁移体系**:已形成 `005` 至 `012` 幂等增量迁移和运行时兼容执行,仍需增加版本记录、迁移前检查和离线升级演练。
|
||||||
10. **完整的端到端测试和持续集成**:至少覆盖单节点、多节点、多 GPU、跨用户、跨租户、版本切换、MinIO 不可用和服务重启恢复。
|
|
||||||
11. **统一数据库迁移体系**:当前初始化 SQL 适合新库初始化,但尚未替代正式的版本化迁移工具;已有数据库更新仍需要明确迁移脚本和执行记录。
|
|
||||||
|
|
||||||
## 五、需要优化的功能
|
## 五、需要优化的功能
|
||||||
|
|
||||||
@@ -265,6 +257,8 @@ FastAPI Backend API
|
|||||||
- `data_convert_tasks.storage_backend`
|
- `data_convert_tasks.storage_backend`
|
||||||
- `eval_tasks.report_storage_object_id`
|
- `eval_tasks.report_storage_object_id`
|
||||||
|
|
||||||
|
当前增量迁移脚本为 `005_storage_progress_migration.sql` 至 `012_tenant_user_hierarchy.sql`,已覆盖存储进度、GPU 预留、平台补全、权限 2.0、配额/成员关系、权限生命周期和租户/用户层级等结构。
|
||||||
|
|
||||||
离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意:
|
离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意:
|
||||||
|
|
||||||
- 初始化 SQL 主要用于新数据库或新数据卷;已有数据库不能仅靠重启容器自动获得全部新字段。
|
- 初始化 SQL 主要用于新数据库或新数据卷;已有数据库不能仅靠重启容器自动获得全部新字段。
|
||||||
@@ -274,6 +268,18 @@ FastAPI Backend API
|
|||||||
|
|
||||||
## 七、当前验证结果
|
## 七、当前验证结果
|
||||||
|
|
||||||
|
### 7.1 2026-08-19 本轮按计划落地内容
|
||||||
|
|
||||||
|
- P0 MinIO 预签名上传已增加资源存在性、资源写权限、管理员基座模型写权限、资源版本和对象 Key 前缀校验;新增上传完成确认接口,会校验 MinIO 对象存在、大小和状态后再标记为可用。
|
||||||
|
- 训练、评测、模型对比和权重合并任务会保存创建时的租户信息与资源版本快照,后续切换数据集激活版本不会改变已创建任务的输入版本。
|
||||||
|
- 数据集、评测任务、训练任务和模型对比列表增加租户范围过滤;用户表、模型/数据集创建入口补齐租户归属;数据转换资源补充所有者权限映射。
|
||||||
|
- Compute Agent 准备缓存后会回写资源副本的版本、MinIO 对象和本地路径;缓存支持可选容量上限、按访问时间淘汰非临时文件,并提供当前占用量和上限状态。
|
||||||
|
- 训练产物、权重合并结果和评测报告增加 MinIO 归档、失败状态记录以及 Backend 重启后的补偿轮询;离线部署源码已同步对应逻辑。
|
||||||
|
- Compute Agent 大文件上传已改为流式读取,避免将整个文件一次性读入内存;离线 Compute Agent 同步完成。
|
||||||
|
- 增加 `005_storage_progress_migration.sql` 增量迁移脚本,并修复 `000_full_init.sql` 中旧数据库执行时索引早于字段补齐的问题;主工程和离线初始化脚本已同步。
|
||||||
|
- 增加 `MINIO_PRESIGN_MAX_BYTES` 配置和上传 Content-Type 白名单;首次数据库 schema 检查移出 Compute Poller 的 Uvicorn 事件循环,避免远程 PostgreSQL 慢连接拖垮健康检查;轮询相同失败改为 300 秒限频记录,并跳过已标记 offline 节点。
|
||||||
|
- 前端 `npm run build` 已通过;容器内 MinIO Key 越权校验专项测试为 `5 passed`,Backend 和 Compute Agent 重启后均为 healthy。
|
||||||
|
|
||||||
已完成的静态和局部验证:
|
已完成的静态和局部验证:
|
||||||
|
|
||||||
- Backend 和离线 Backend 源码 `compileall` 检查通过。
|
- Backend 和离线 Backend 源码 `compileall` 检查通过。
|
||||||
@@ -281,45 +287,80 @@ FastAPI Backend API
|
|||||||
- 主工程和离线包初始化 SQL 已做同步检查,内容一致。
|
- 主工程和离线包初始化 SQL 已做同步检查,内容一致。
|
||||||
- 前端此前已完成 `npm run build` 类型错误修复,构建剩余问题主要是非阻断的资源/分包警告。
|
- 前端此前已完成 `npm run build` 类型错误修复,构建剩余问题主要是非阻断的资源/分包警告。
|
||||||
- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。
|
- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。
|
||||||
|
- 当前 WSL 运行验证中 Backend、Compute Agent、MinIO 均为 healthy;`gpu-node-02`(`172.25.179.69:19100`)连接、GPU 0 分配和任务执行均正常。`gpu-node-01` 的历史地址不可达,已通过健康检查标记为 offline,轮询器不再重复轮询其历史任务。
|
||||||
|
|
||||||
|
### 7.2 2026-08-19 gpu-node-02 真实流程验证
|
||||||
|
|
||||||
|
- 训练:使用 `qwen3.5-0.8B` + `test_data_0817`,任务 `ft_172a2da65140` 完成,GPU 0 使用期间可看到显存、利用率、温度和进程,结束后恢复 idle;24 个训练产物已归档 MinIO。
|
||||||
|
- 资源快照:任务 `ft_d93b0fdae1c9` 创建时已保存数据集 `ds_8f6b8c5714c7` 的活动版本 `file_d550c2128722_v1`。
|
||||||
|
- 训练曲线:任务 `ft_d93b0fdae1c9` 通过 `logging_steps=1` 生成 3 个 loss/epoch/learning-rate/grad-norm 数据点,并生成 `training_loss.png`;后端解析器已兼容带引号数字格式。
|
||||||
|
- 权重合并:任务 `merge_1dc15a290810` 完成,基座模型路径、合并路径已回写,8 个合并模型文件归档 MinIO。
|
||||||
|
- 推理:任务 `cmp_abdf0762869d` 在 GPU 0 加载 `qwen3.5-0.8B` 成功,对话接口返回正常;卸载后 GPU 恢复 idle。
|
||||||
|
- 评测:任务 `eval_3c3f84263e23` 完成 2 条样本评测,报告、样本明细和基础指标已落库;GPU 恢复 idle。评审模型 HTTP 400 导致评审分数为 0,属于评审模型接口配置问题,算力评测链路本身已跑通。
|
||||||
|
|
||||||
|
### 7.3 多 GPU、MinIO 故障和跨用户权限专项验证
|
||||||
|
|
||||||
|
- 多 GPU 调度开发:新增 `gpu_reservations` 表,评测和推理在远程提交/加载前与训练统一纳入数据库原子预留;失败、停止、删除、节点不可达和卸载路径自动释放预留。当前 `gpu-node-02` 只有 GPU 0,已通过同卡“推理预留后评测抢占”测试,评测被明确拒绝,释放后 GPU 恢复 idle。多节点、多卡的真实并行测试待增加第二个可达节点和多卡节点后执行。
|
||||||
|
- 调度锁优化:调度锁改为事务内持有并在提交前释放,避免一次调度成功后后续请求等待 30 秒 TTL;schema 初始化增加 PostgreSQL advisory lock,避免轮询器与首个请求并发初始化造成死锁。
|
||||||
|
- MinIO 故障注入:停止 `yg-ft-minio` 后,`GET /modelTF/health` 返回 HTTP 200 且 `storage.status=unavailable`;恢复容器后返回 `storage.status=ready`。MinIO 客户端关闭默认重试链,故障健康探测耗时从约 6 秒降至约 0.3 秒。
|
||||||
|
- 跨用户权限:创建临时用户 `qa_user_0819`,未授权访问管理员数据集返回 403;授予资源 `read/download` ACL 后列表和详情可访问;撤销 ACL 后再次返回 403;非管理员创建用户返回 403。测试用户已删除,未遗留测试 ACL 或 GPU 预留。
|
||||||
|
- 用户管理接口已补齐管理员依赖,创建、列表、修改、删除和重置密码不再允许普通用户调用。
|
||||||
|
|
||||||
当前不能据此宣称“全量功能测试通过”:
|
当前不能据此宣称“全量功能测试通过”:
|
||||||
|
|
||||||
- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设,需要按当前分层存储策略更新。
|
- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设,需要按当前分层存储策略更新。
|
||||||
- WSL Docker 运行时验证受当前环境的 `E_ACCESSDENIED` 影响,不能在本次文档生成时完成全部容器健康、数据库字段和跨节点测试。
|
- 本轮已完成当前 WSL Docker 容器健康检查、`gpu-node-02` 单节点真实流程、单卡冲突、MinIO 故障恢复和跨用户 ACL 专项验证;多节点、多卡的真实并行测试仍需要第二个可达节点和多卡节点。
|
||||||
- 多节点、多 GPU、MinIO 临时不可用、服务重启恢复和跨用户权限测试仍需要在可用运行环境中执行。
|
|
||||||
|
- 本轮专项安全测试为 `5 passed`;全量 pytest 仍未执行完毕,需要按测试类别拆分并设置超时。
|
||||||
|
|
||||||
|
### 7.4 本轮 11 项未验证能力的补齐结果
|
||||||
|
|
||||||
|
- 租户管理接口已统一要求管理员身份;模型制品、模型血缘、导出任务、评测报告和 MinIO 资源清单均增加资源级访问校验。
|
||||||
|
- 新增 `POST /modelTF/model-manage/export`,导出沿用节点选择、MinIO 缓存准备、GPU/调度约束和归档轮询;导出结果记录到 `model_export_jobs`,并建立导出制品与模型血缘。
|
||||||
|
- 新增 `GET /modelTF/model-eval/{task_id}/report`,优先流式返回 MinIO 报告,历史任务无归档对象时返回数据库报告兼容结果。
|
||||||
|
- 新增 `GET /modelTF/storage/resources/{resource_type}/{resource_id}/manifest`、管理员对象清理和孤儿扫描接口;预签名上传增加过期时间和真实 SHA-256 内容校验。
|
||||||
|
- Compute Agent 增加缓存 TTL、缓存保护窗口和元数据清单;超过 TTL 的非保护缓存会在状态检查时清理,容量淘汰会跳过保护中的资源。
|
||||||
|
- 数据集单文件下载改为 MinIO 流式传输,避免 Backend 一次性载入完整大文件;训练、合并、导出、评测仍由统一轮询器负责重启后的归档补偿。
|
||||||
|
- 新增 `007_platform_completion.sql`,并已加入运行时兼容迁移;主工程和 `docker/offline/src` 的源码及初始化 SQL 已同步。
|
||||||
|
- 以上为代码闭环和单节点验证;生产级 MinIO TLS/HA、第二节点/多卡并行压力测试、全量 CI 和历史项目数据正式迁移仍属于上线前专项工作。
|
||||||
|
|
||||||
|
### 7.5 2026-08-20 普通用户运行日志验证
|
||||||
|
|
||||||
|
- 已修复普通用户进入“运行日志”调用管理员专属 `/log-files` 接口导致 403 的问题。
|
||||||
|
- 普通用户现在查看本人操作日志,后端按当前用户 ID 强制过滤,忽略普通用户传入的跨用户 `user_id` 条件。
|
||||||
|
- 管理员仍可查看系统日志、训练日志、审计记录和全量操作诊断。
|
||||||
|
- 临时普通用户创建数据转换任务后验证:操作日志查询和统计均返回 200,返回记录全部属于该用户;测试用户和测试资源已清理。
|
||||||
|
- 前端 `npm run build` 再次通过,离线前端静态资源和源码已同步,离线 Frontend/Backend 容器健康。
|
||||||
|
|
||||||
## 八、下一阶段开发计划
|
## 八、下一阶段开发计划
|
||||||
|
|
||||||
### P0:安全与数据正确性
|
### P0:权限与租户隔离安全闭环
|
||||||
|
|
||||||
1. 完善 MinIO 预签名 PUT 的资源写权限、对象 Key 白名单、大小/类型限制和上传完成确认。
|
1. 对现有数据库执行结构差异检查,盘点所有 `tenant_id`、创建者、ACL、资源版本和历史 NULL 数据。
|
||||||
2. 统一任务创建时的资源版本固化,训练、推理、评测只使用已授权的激活版本快照。
|
2. 编写历史租户归属迁移和项目兼容数据迁移脚本,迁移前生成结构/数据快照,迁移后增加一致性检查。
|
||||||
3. 逐一补齐评测、推理、模型合并、模型导出、缓存准备的后端权限校验和审计记录。
|
3. 建立接口权限矩阵,覆盖数据集下载、模型使用、推理、评测、权重合并、导出、缓存准备、审批和运行日志。
|
||||||
4. 完成租户隔离查询范围,清理或兼容历史项目字段,补充数据迁移脚本。
|
4. 补齐跨用户、跨租户、已撤销 ACL、软删除资源和审批未通过场景的自动化测试。
|
||||||
|
|
||||||
### P1:跨节点可靠性
|
### P1:资源申请、审批和联合权限
|
||||||
|
|
||||||
1. 建立资源清单/manifest,记录 MinIO 对象版本、校验值、目标节点路径和缓存状态。
|
1. 对资源申请、审批通过后的 ACL 写入、配额扣减和 GPU 预留执行事务一致性回归,重点验证重复审批和撤回。
|
||||||
2. 完善训练、合并、评测和推理的准备、执行、归档、失败重试和服务重启补偿。
|
2. 验证租户管理员、普通成员、只读成员和平台管理员在跨租户资源、模型、数据集和算力上的差异。
|
||||||
3. 完善 GPU 原子分配、异常回收、节点重连对账和任务释放。
|
3. 验证训练创建的基座模型、数据集、GPU、训练产物之间的联合授权和血缘快照。
|
||||||
4. 增加缓存容量、TTL/LRU、运行任务保护和磁盘占用监控。
|
4. 统一审批策略、资源 ACL、租户配额和 GPU 预留的错误提示,避免只返回 403/500。
|
||||||
5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。
|
|
||||||
|
|
||||||
### P2:性能与用户体验
|
### P2:前端权限体验与审计完善
|
||||||
|
|
||||||
1. 优化页面列表接口和高频轮询,采用批量查询、短期缓存和动态退避。
|
1. 完善页面级权限与资源动作权限的统一组件,重点覆盖下载、导出、执行、删除、授权和审批按钮。
|
||||||
2. 将大文件上传/下载/复制改为流式或分片传输。
|
2. 运行日志保持普通用户只看本人、管理员看全量的分层模型,并补充权限拒绝和下载/导出审计字段。
|
||||||
3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。
|
3. 优化模型推理、模型评测、训练详情、数据集和运行日志页面的首屏 loading、超时、重试和空状态。
|
||||||
4. 处理前端离线资源警告,继续拆分首屏 chunk。
|
4. 减少列表接口重复查询,继续拆分首屏大 chunk,移除离线环境不需要的外部资源请求。
|
||||||
5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。
|
|
||||||
|
|
||||||
### P3:工程化和上线准备
|
### P3:可靠性、测试和上线准备
|
||||||
|
|
||||||
1. 建立正式数据库版本迁移机制和离线升级脚本。
|
1. 准备第二个可达节点或多卡节点,执行训练、推理、评测并发、节点不可达和服务重启测试。
|
||||||
2. 增加 CI:前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。
|
2. 接入 MinIO 对象生命周期定时清理、容量监控、TLS、密钥管理、备份恢复和故障演练。
|
||||||
3. 增加多节点端到端测试和 MinIO 故障注入测试。
|
3. 建立正式数据库迁移版本记录、迁移前结构检查、离线升级和回滚演练。
|
||||||
4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。
|
4. 将前端构建、Backend 编译、权限安全、存储安全和端到端流程测试分层纳入 CI。
|
||||||
5. 建立生产运行手册,包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。
|
|
||||||
|
|
||||||
## 九、阶段验收标准
|
## 九、阶段验收标准
|
||||||
|
|
||||||
|
|||||||
493
docs/权限开发进度.md
Normal file
493
docs/权限开发进度.md
Normal file
@@ -0,0 +1,493 @@
|
|||||||
|
# 权限开发进度
|
||||||
|
|
||||||
|
> 更新时间:2026-08-20
|
||||||
|
> 适用范围:YG_FT 平台当前主工程、Backend、Frontend、Compute Agent、MinIO 资源访问及 `docker/offline/src` 离线源码。
|
||||||
|
> 当前结论:权限 2.0 的核心功能已形成闭环。本轮继续补齐租户成员角色在 ACL 中的实际判定、有效期租户隔离、用户软删除、资源/MinIO 对象待清理、训练模型预加载资源存在性校验和推理状态接口授权,并已执行在线数据库迁移。自动化权限/存储安全用例、前端构建、后端静态检查、容器健康和用户删除闭环均已验证。由于当前只有一个可达算力节点,跨租户双用户、第二节点/多卡并发、节点故障回收和 MinIO 故障注入仍需专项验证,因此保留为上线前场景。按工作包估算,核心权限开发约完成 98%,上线验收约完成 82%。
|
||||||
|
|
||||||
|
## 本轮 11 项完成情况(2026-08-19)
|
||||||
|
|
||||||
|
| 编号 | 权限能力 | 完成内容 | 状态 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | 审批决策安全 | 审批人从当前会话获取,校验指定审批人、租户管理员、禁止申请人自审,记录审批审计 | 已完成 |
|
||||||
|
| 2 | 资源访问申请 | 新增资源访问申请记录、申请权限/期限/理由、审批后自动写入 ACL、支持撤回和过期 | 已完成 |
|
||||||
|
| 3 | 审批策略执行 | 按租户、动作、资源类型匹配策略;高风险操作审批通过后可一次性重试执行 | 已完成 |
|
||||||
|
| 4 | 租户与项目隔离 | `tenant_members`、活跃租户校验、历史资源归属补齐、跨租户 ACL 主体校验、项目软删除 | 已完成 |
|
||||||
|
| 5 | 用户与角色边界 | 禁用用户立即注销会话;租户成员支持 owner/admin/member/viewer,保护最后一个 owner | 已完成 |
|
||||||
|
| 6 | 数据集权限入口 | 预览、来源、版本查询/创建/激活/删除、上传/编辑/下载统一接入资源动作权限 | 已完成 |
|
||||||
|
| 7 | 模型和推理权限 | 训练模型合并/导出分别校验 execute/download;聊天、预加载、批量、卸载绑定授权模型或推理任务 | 已完成 |
|
||||||
|
| 8 | GPU 分配申请 | 普通用户可提交 GPU 分配申请,审批通过后自动分配;管理员可直接分配 | 已完成 |
|
||||||
|
| 9 | 租户配额申请 | 租户成员可提交配额变更申请,平台管理员审批后自动更新配额 | 已完成 |
|
||||||
|
| 10 | 软删除与安全状态 | 资源删除撤销 ACL、取消待处理申请和审批;租户/项目采用软删除状态 | 已完成 |
|
||||||
|
| 11 | 前端按钮与审计 | 审批决策、ACL 编辑、访问申请/撤回按权限显示;下载、导出、审批、GPU 等敏感操作补充结构化审计 | 已完成 |
|
||||||
|
|
||||||
|
### 数据库迁移结果
|
||||||
|
|
||||||
|
- 新增增量迁移:`backend/app/db/sql/009_permission_completion.sql`,可独立兼容旧库执行,并已在当前远程 PostgreSQL 执行成功。
|
||||||
|
- 当前远程库已核验包含:`tenant_members`、`resource_access_requests`、审批策略/执行状态字段、审计结果字段、ACL 生命周期字段、项目/租户软删除字段、数据转换任务租户字段。
|
||||||
|
- 已有用户已补齐到 `tenant_members`,当前迁移后共生成 5 条租户成员关系。
|
||||||
|
- `backend/app/db/sql/000_full_init.sql` 已合并完整结构;离线目录继续只保留该完整脚本。
|
||||||
|
|
||||||
|
### 验证结果
|
||||||
|
|
||||||
|
- 后端相关模块 `python -m py_compile`:通过。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 路径提示和大 chunk 警告,无 TypeScript 错误。
|
||||||
|
- `docker/offline/src` 的后端权限代码、Compute Agent、前端源码和完整初始化 SQL 已与主工程关键文件哈希一致。
|
||||||
|
- 运行中的 Backend、Frontend、Compute API 容器已重启,健康检查通过;当前容器采用源码挂载方式,无需重建镜像即可加载本轮代码。已验证健康接口可用,未登录访问受保护接口返回 401;完整双用户、跨租户和多 GPU 回归仍待执行。
|
||||||
|
|
||||||
|
## 一、检查依据
|
||||||
|
|
||||||
|
本次按代码和初始化脚本逐项核对,主要依据如下:
|
||||||
|
|
||||||
|
- `docs/permissions-design.md`
|
||||||
|
- `backend/app/core/auth.py`
|
||||||
|
- `backend/app/api/v1/endpoints/platform.py`
|
||||||
|
- `backend/app/modules/resource/router.py`
|
||||||
|
- `backend/app/modules/approval/router.py`
|
||||||
|
- `backend/app/modules/tenant/router.py`
|
||||||
|
- `backend/app/modules/system/router.py`
|
||||||
|
- `backend/app/db/platform_store.py`
|
||||||
|
- `backend/app/db/sql/000_full_init.sql`
|
||||||
|
- `frontend/src/stores/auth.ts`
|
||||||
|
- `frontend/src/router/index.ts`
|
||||||
|
- `frontend/src/views/governance/ResourceAclView.vue`
|
||||||
|
|
||||||
|
检查口径不是只判断“是否有接口”,还检查了接口是否验证当前用户、是否校验资源所有权和 ACL、是否校验租户边界、审批结果是否真正改变授权、前端按钮是否与后端动作一致。
|
||||||
|
|
||||||
|
## 二、改造前总体进度(历史基线)
|
||||||
|
|
||||||
|
| 能力模块 | 当前状态 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| 登录、密码、Token 会话 | 已实现基础能力 | 有会话过期、注销、状态校验和登录限流基础,仍需统一续期和失败审计 |
|
||||||
|
| 平台角色与权限码 | 部分实现 | `admin/operator/viewer` 和业务权限码存在,但后端部分业务只校验登录,未统一校验权限码 |
|
||||||
|
| 用户创建与管理 | 部分实现 | 管理员可创建、修改、删除和重置密码;租户范围、角色边界、邀请/申请流程未完成 |
|
||||||
|
| 租户与配额 | 部分实现 | 租户 CRUD、配额和留存策略接口为管理员专属;缺少租户成员、租户管理员和统一隔离 |
|
||||||
|
| 资源 ACL | 已实现基础能力 | 支持用户/角色的 `read/write/execute/download/delete/admin`,授权边界和跨租户限制不足 |
|
||||||
|
| 资源申请 | 未完整实现 | 没有独立的资源访问申请对象,现有审批实例不能可靠地落地 ACL 或配额变更 |
|
||||||
|
| 资源审批 | 部分实现且存在安全缺口 | 模板、实例、步骤和部分高风险操作存在;审批决策接口必须先修复越权问题 |
|
||||||
|
| 基座模型权限 | 平台共享已实现 | 登录用户可查看和使用,上传/修改/删除限制管理员;缺少按租户/用途/配额的精细控制 |
|
||||||
|
| 训练模型权限 | 部分实现 | 所有者、ACL、删除、合并、推理加载已有校验;导出下载动作和派生权限继承还不完整 |
|
||||||
|
| 数据集权限 | 部分实现且入口不一致 | 列表、详情、删除、部分下载和训练/评测使用有校验;上传、编辑、预览、版本接口仍有缺口 |
|
||||||
|
| 训练/评测/推理权限 | 部分实现 | 资源执行权和 GPU 分配已有基础校验;部分聊天、状态和历史入口没有统一鉴权 |
|
||||||
|
| GPU 与节点权限 | 基础能力已实现 | 管理员分配、用户可用 GPU 过滤、原子预留和释放已存在;配额审批和跨节点回收对账仍需完善 |
|
||||||
|
| 前端按钮级权限 | 部分实现 | 页面和菜单有基础控制,资源动作没有集中式权限决策,很多按钮依赖后端报错 |
|
||||||
|
| 审计与软删除 | 部分实现 | 主要写操作有审计,资源软删除已覆盖若干表;actor、下载、拒绝和跨租户查询仍需统一 |
|
||||||
|
|
||||||
|
## 三、已经实现的内容
|
||||||
|
|
||||||
|
### 3.1 认证、角色与用户
|
||||||
|
|
||||||
|
- `get_current_user` 会校验 Bearer Token、用户存在性、用户状态和会话有效期;`sessions` 支持注销和过期时间。
|
||||||
|
- `require_admin` 对管理员专属接口提供后端保护,受保护用户也具备管理员旁路能力。
|
||||||
|
- 用户列表、创建、修改、删除、重置密码均已增加管理员依赖;用户可修改自己的密码。
|
||||||
|
- 用户记录包含 `tenant_id`、`role`、`permissions`、`protected` 等字段,角色权限在初始化脚本中有基础种子数据。
|
||||||
|
- 登录失败限流和旧明文密码升级已经具备基础实现。
|
||||||
|
|
||||||
|
### 3.2 租户、资源和 ACL
|
||||||
|
|
||||||
|
- `tenants` 表和租户 CRUD、配额、留存策略接口已经存在,当前接口统一要求管理员。
|
||||||
|
- 数据集、模型、训练模型、评测任务等核心资源已经具备 `created_by`、`tenant_id` 或任务 payload 中的归属信息。
|
||||||
|
- `acls` 表支持用户和角色两类授权主体,权限包括 `read`、`write`、`execute`、`download`、`delete`、`admin`。
|
||||||
|
- 资源 ACL 查询和全量替换接口已经存在,资源所有者或管理员可以管理 ACL。
|
||||||
|
- 列表接口对数据集、评测、训练和推理任务已增加“本人资源 + ACL 授权资源”的过滤逻辑。
|
||||||
|
- MinIO 预签名、资源清单、对象列表、模型制品、模型血缘和评测报告等新入口已增加登录和资源访问校验。
|
||||||
|
|
||||||
|
### 3.3 训练、评测、推理和 GPU
|
||||||
|
|
||||||
|
- 训练创建会检查训练数据集的 `execute` 权限,并对用户选择的 GPU 执行分配校验。
|
||||||
|
- 评测创建会检查训练模型、数据集的 `execute` 权限,并校验算力节点和 GPU。
|
||||||
|
- 推理任务加载会检查任务及训练模型的执行权,同时使用 `gpu_reservations` 防止同一张 GPU 被并发占用。
|
||||||
|
- GPU 分配接口和用户可用 GPU 查询已经存在,失败、停止、删除、节点异常等路径具备基础释放逻辑。
|
||||||
|
- 训练模型删除、训练任务停止/删除、评测任务删除和推理任务删除已接入部分审批拦截。
|
||||||
|
- 权重合并、导出、缓存准备和 MinIO 归档已经能记录模型、节点、租户和部分血缘信息。
|
||||||
|
|
||||||
|
### 3.4 审计和前端
|
||||||
|
|
||||||
|
- `audit_logs`、`operation_logs` 表及管理员查询/导出页面已经存在。
|
||||||
|
- 资源 ACL 变更、租户管理、模型/数据集/任务等主要写操作已经接入审计或操作日志装饰器。
|
||||||
|
- 前端路由和侧边栏对治理、组织、资源授权、审批、日志、算力节点等页面做了管理员限制。
|
||||||
|
- `ResourceAclView` 已支持资源类型、用户/角色主体和多权限编辑。
|
||||||
|
|
||||||
|
## 四、改造前未实现或存在明显安全缺口(历史基线)
|
||||||
|
|
||||||
|
> 本节保留本轮改造前的检查快照,不代表当前状态。当前剩余问题以“十、当前仍需开发的功能”和“十一、下一步开发计划”为准。
|
||||||
|
|
||||||
|
以下项目属于必须继续开发的内容,优先级高于页面样式和性能优化。
|
||||||
|
|
||||||
|
### 4.1 审批决策不能直接信任请求参数
|
||||||
|
|
||||||
|
`backend/app/modules/approval/router.py` 的审批决策接口当前没有 `get_current_user` 依赖,并且从请求体读取 `approver_id`。调用方可以伪造审批人 ID,属于高风险越权问题。
|
||||||
|
|
||||||
|
必须改为:服务端从当前会话取得审批人;校验其是否为当前步骤指定审批人、租户管理员或平台管理员;校验当前步骤、实例状态和申请人不能自审;审批通过后再执行对应动作或写入 ACL。
|
||||||
|
|
||||||
|
### 4.2 资源申请流程尚未形成
|
||||||
|
|
||||||
|
当前有 `approval_templates`、`approval_instances`、`approval_steps`,但没有独立的资源访问申请记录,也没有统一的“申请资源 -> 审批 -> 授权/配额变更”事务流程:
|
||||||
|
|
||||||
|
- 创建审批实例时未统一验证资源是否存在、申请人是否属于资源租户、申请动作是否合法。
|
||||||
|
- 审批实例批准后不会自动创建 ACL、GPU 分配或租户配额变更。
|
||||||
|
- 不能表达申请权限、申请期限、申请原因、审批后的 ACL 权限和撤销时间。
|
||||||
|
- 审批模板查询和详情接口没有完整的租户/角色范围控制。
|
||||||
|
|
||||||
|
### 4.3 租户隔离不是全量强制
|
||||||
|
|
||||||
|
- 当前用户只有单一 `tenant_id`,没有 `tenant_members` 或租户角色关系;无法支持租户管理员、跨租户平台管理员和成员邀请的清晰边界。
|
||||||
|
- `filter_accessible_resource_ids` 和批量版本主要按 ACL/所有者过滤,不在统一函数中校验资源租户。
|
||||||
|
- 数据集、评测等查询仍对 `tenant_id IS NULL` 做兼容放行;历史数据未完成归属迁移。
|
||||||
|
- `trained_models()` 没有统一 tenant 参数,资源过滤依赖上层二次处理。
|
||||||
|
- ACL 设置接口没有校验授权主体和资源是否属于同一租户,存在跨租户授权风险。
|
||||||
|
- 新建模型、数据集、任务虽然多数会补默认租户,但缺少“租户必须存在且处于 active”的统一校验。
|
||||||
|
|
||||||
|
### 4.4 数据集权限入口不一致
|
||||||
|
|
||||||
|
核心列表、详情、删除、部分下载已校验,但以下文件/版本接口当前未统一接入当前用户和资源权限:
|
||||||
|
|
||||||
|
- 文件预览和记录来源查询。
|
||||||
|
- 文件版本列表、版本内容查询。
|
||||||
|
- 创建、激活、删除数据集文件版本。
|
||||||
|
- 数据集上传接口没有统一的当前用户、写权限和租户校验。
|
||||||
|
- 数据集编辑接口没有统一的当前用户和写权限校验。
|
||||||
|
|
||||||
|
此外,数据集下载和单文件下载目前检查的是 `read`,没有严格使用设计中的独立 `download` 权限。
|
||||||
|
|
||||||
|
### 4.5 模型使用、导出和推理入口不一致
|
||||||
|
|
||||||
|
- 基座模型按平台共享资源处理,登录用户可以查看和使用;但模型名称查询、本地模型列表和部分本地聊天/状态/卸载入口缺少统一鉴权。
|
||||||
|
- 训练模型列表、详情、合并和加载已有 ACL 校验,但导出接口实际属于下载/外发动作,不能只检查 `execute`。
|
||||||
|
- 评测报告下载当前检查 `read`,应单独检查 `download` 并记录下载审计。
|
||||||
|
- 训练模型由训练任务生成时,尚未完整实现“基座模型 + 数据集授权关系”向派生模型权限和血缘策略的统一继承。
|
||||||
|
- 推理聊天接口没有始终绑定到已授权的推理任务、模型资源和指定算力节点,存在绕过模型使用流程的风险。
|
||||||
|
|
||||||
|
### 4.6 用户创建和角色边界不完整
|
||||||
|
|
||||||
|
- 当前只有平台管理员创建用户,普通用户不能申请加入租户,也没有邀请、审批、禁用后会话清理的完整流程。
|
||||||
|
- 用户只能直接挂在一个 `tenant_id` 上,不能表达一个用户属于多个租户或在不同租户中拥有不同角色。
|
||||||
|
- 后端 `create_user` 对非管理员角色会归一为普通用户,前端出现的 `operator` 角色与后端实际行为可能不一致。
|
||||||
|
- 创建用户时缺少租户存在性、租户状态、租户用户配额和角色白名单校验。
|
||||||
|
- 管理员可以创建管理员角色,尚未区分平台管理员和租户管理员的授予权限。
|
||||||
|
|
||||||
|
### 4.7 前端按钮级权限没有闭环
|
||||||
|
|
||||||
|
- 前端已有菜单和路由权限基础,但 `hasPermission` 没有覆盖所有业务路由动作,部分业务页对已登录用户直接开放。
|
||||||
|
- 资源级按钮主要靠资源对象中的所有者字段判断,没有统一使用后端返回的 ACL 决策。
|
||||||
|
- 下载、执行、删除、授权、导出、审批等动作没有统一的 `can(resource, action)` 机制。
|
||||||
|
- 即使按钮隐藏,前端 API 模块仍可能直接调用敏感接口;必须以后端鉴权为最终边界。
|
||||||
|
|
||||||
|
### 4.8 审计、软删除和权限拒绝记录不完整
|
||||||
|
|
||||||
|
- 部分模块的 `_actor` 直接保存 Authorization Token,而不是规范的用户 ID,导致审计主体不一致。
|
||||||
|
- 访问、下载、导出、ACL 授权、审批拒绝、GPU 分配和权限拒绝没有全部统一记录租户、资源和结果。
|
||||||
|
- `record_visit` 公开接口容易被伪造;如果继续保留匿名访问,应明确它不是安全审计日志。
|
||||||
|
- 资源软删除已覆盖模型、训练模型、数据集、评测任务等主要表,但关联 MinIO 对象、ACL、审批和血缘的延迟清理策略还不完整。
|
||||||
|
|
||||||
|
## 五、需要优化的现有流程
|
||||||
|
|
||||||
|
### 5.1 统一权限模型
|
||||||
|
|
||||||
|
将当前分散的 `is_admin`、所有者判断、ACL 查询、租户判断、GPU 判断收敛为统一服务:
|
||||||
|
|
||||||
|
```text
|
||||||
|
认证 -> 平台/租户角色 -> 租户边界 -> 资源所有权/ACL -> 动作权限 -> GPU/配额 -> 审批 -> 审计
|
||||||
|
```
|
||||||
|
|
||||||
|
每个敏感接口都应明确动作,例如 `dataset.read`、`dataset.download`、`dataset.execute`、`trained_model.export`、`inference.load`、`gpu.reserve`,禁止只使用“已登录”作为业务权限。
|
||||||
|
|
||||||
|
### 5.2 重新设计资源申请和审批
|
||||||
|
|
||||||
|
推荐流程:
|
||||||
|
|
||||||
|
```text
|
||||||
|
申请人选择资源和动作
|
||||||
|
-> 校验申请人所属租户和基础权限
|
||||||
|
-> 创建 resource_access_requests
|
||||||
|
-> 根据租户/资源类型匹配审批策略
|
||||||
|
-> 指定审批人完成审批
|
||||||
|
-> 事务内写入 ACL/配额/GPU 预留
|
||||||
|
-> 记录授权有效期、来源和审计
|
||||||
|
```
|
||||||
|
|
||||||
|
审批拒绝、撤回、过期和资源删除都应撤销或冻结对应授权,不能仅修改审批实例状态。
|
||||||
|
|
||||||
|
### 5.3 模型、数据集、训练任务联合授权
|
||||||
|
|
||||||
|
训练、评测、推理分别使用以下最小权限:
|
||||||
|
|
||||||
|
| 场景 | 必须具备的权限 |
|
||||||
|
|---|---|
|
||||||
|
| 查看基座模型 | `model.read`;基座模型默认平台共享 |
|
||||||
|
| 使用基座模型训练 | `model.execute` 或平台共享策略 + 数据集 `execute` |
|
||||||
|
| 下载基座模型 | 单独的 `model.download`,默认不授予 |
|
||||||
|
| 使用训练模型推理 | `trained_model.execute` |
|
||||||
|
| 下载/导出训练模型 | `trained_model.download` 或 `trained_model.export` |
|
||||||
|
| 使用数据集训练/评测 | `dataset.execute` |
|
||||||
|
| 查看数据集 | `dataset.read` |
|
||||||
|
| 下载数据集文件 | `dataset.download` |
|
||||||
|
| 创建训练任务 | 上述资源权限 + 指定节点/GPU 使用权 + 租户配额 |
|
||||||
|
|
||||||
|
训练模型应保留创建者、租户、基座模型、数据集、训练任务和资源版本快照。派生模型默认只对创建者和同租户授权,不应因为基座模型共享而自动公开训练产物。
|
||||||
|
|
||||||
|
## 六、建议的数据库改造
|
||||||
|
|
||||||
|
当前 `000_full_init.sql` 已包含用户、角色、会话、ACL、租户、审批、审计、GPU 分配和 GPU 预留表,但要完成权限 2.0,建议增加或扩展以下结构。实施时必须同步主工程和 `docker/offline/src/backend/app/db/sql/000_full_init.sql`。
|
||||||
|
|
||||||
|
1. `tenant_members`:用户、租户、租户角色、状态、加入来源和有效期,解决一个用户多租户和租户管理员问题。
|
||||||
|
2. `resource_access_requests`:申请人、租户、资源、动作、申请原因、申请权限、有效期、审批状态和最终授权记录。
|
||||||
|
3. `acls` 增加 `tenant_id`、`granted_by`、`source_request_id`、`expires_at`、`revoked_at`,保留授权来源和自动过期能力。
|
||||||
|
4. `approval_templates` 增加 `tenant_id`、`action`、`resource_type`、`scope`、`status`;审批步骤增加主体类型、主体 ID 和租户范围。
|
||||||
|
5. 资源表统一补齐非空租户策略、归属用户、软删除字段和必要索引;历史 NULL 数据通过一次性迁移处理。
|
||||||
|
6. `audit_logs` 增加结果、拒绝原因、request_id、认证会话和结构化 detail,避免把 Token 当作 actor。
|
||||||
|
|
||||||
|
不建议把完整 ACL 和审批状态继续塞入模型、数据集或任务 JSON 字段;JSON 可保留兼容信息,但权限判断应以结构化表为准。
|
||||||
|
|
||||||
|
## 七、历史开发计划(已执行)
|
||||||
|
|
||||||
|
### 第一阶段:P0 安全修复
|
||||||
|
|
||||||
|
- 修复审批决策鉴权和审批人伪造问题。
|
||||||
|
- 补齐数据集文件/版本/上传/编辑接口权限。
|
||||||
|
- 补齐本地模型聊天、状态、卸载、模型名称查询等历史入口鉴权。
|
||||||
|
- 统一 `download`、`execute`、`write`、`delete` 动作检查。
|
||||||
|
- 限制 ACL 授权范围,校验主体存在、同租户和资源归属。
|
||||||
|
- 增加至少 20 个后端权限回归用例,覆盖未登录、本人、同租户他人、跨租户、管理员和已撤销 ACL。
|
||||||
|
|
||||||
|
### 第二阶段:P1 租户和资源申请
|
||||||
|
|
||||||
|
- 引入租户成员和租户角色,明确平台管理员、租户管理员、成员、只读成员边界。
|
||||||
|
- 开发资源访问申请接口和前端申请页面。
|
||||||
|
- 重做审批模板匹配、审批人校验、审批结果落地 ACL、有效期和撤销流程。
|
||||||
|
- 将配额申请、GPU 分配申请、模型导出和跨用户删除纳入审批策略。
|
||||||
|
- 新资源强制租户归属并完成历史数据迁移。
|
||||||
|
|
||||||
|
### 第三阶段:P1 联合资源权限
|
||||||
|
|
||||||
|
- 建立模型/数据集/训练任务/评测任务/推理任务的统一资源授权服务。
|
||||||
|
- 训练前一次性校验基座模型、数据集、节点、GPU 和租户配额。
|
||||||
|
- 训练模型生成时写入血缘和权限来源;推理、评测、合并、导出使用同一套动作权限。
|
||||||
|
- 版本快照、MinIO 对象、算力节点本地缓存沿用同一资源授权结果。
|
||||||
|
|
||||||
|
### 第四阶段:P2 前端和审计
|
||||||
|
|
||||||
|
- 增加统一 `can(resource, action)` 和按钮权限组件。
|
||||||
|
- 授权和审批界面使用用户/租户/资源中文名称,展示授权来源、有效期和审批状态。
|
||||||
|
- 规范审计 actor、租户、请求 ID、资源、动作、结果和失败原因。
|
||||||
|
- 将权限不足、审批中、资源过期、MinIO 不可用分别展示,避免全部显示为通用 500。
|
||||||
|
|
||||||
|
### 第五阶段:P3 测试与上线
|
||||||
|
|
||||||
|
- 新库初始化、增量迁移、离线目录同步和前端构建全部纳入 CI。
|
||||||
|
- 执行跨租户、跨用户、ACL 撤销、审批越权、软删除、MinIO 故障和 GPU 冲突专项测试。
|
||||||
|
- 在第二个可达节点或多卡节点到位后,执行多节点、多 GPU 的权限和并发测试。
|
||||||
|
- 补充权限运维手册:新建租户、创建用户、授权模型/数据集、审批、撤销权限、审计追踪和故障恢复。
|
||||||
|
|
||||||
|
## 八、验收标准
|
||||||
|
|
||||||
|
- 未登录用户不能访问任何模型、数据集、任务、聊天、版本、下载和审批接口。
|
||||||
|
- 用户只能访问所属租户中本人拥有或被明确授权的资源;跨租户 ACL 默认禁止。
|
||||||
|
- 查看、下载、执行、编辑、删除和授权是独立动作,不能用 `read` 替代所有动作。
|
||||||
|
- 普通用户不能伪造审批人、审批其他租户资源或审批自己的申请。
|
||||||
|
- 训练、评测和推理创建必须同时满足资源权限、GPU 权限、节点权限和租户配额。
|
||||||
|
- 审批通过后才产生授权,审批拒绝、撤回、过期和资源删除后授权不会继续生效。
|
||||||
|
- 基座模型共享不等于训练产物共享;训练后的模型和数据集必须按租户、所有者和 ACL 控制。
|
||||||
|
- 前端隐藏按钮不能替代后端校验,直接调用 API 也必须返回明确的 401/403。
|
||||||
|
- 所有敏感操作可通过用户、租户、资源、动作和请求 ID追溯到审计记录。
|
||||||
|
|
||||||
|
## 九、历史实施记录(已完成)
|
||||||
|
|
||||||
|
本轮已完成权限闭环的代码实现:
|
||||||
|
|
||||||
|
1. 统一校验当前会话、会话过期、退出状态和可用租户范围。
|
||||||
|
2. 补齐模型、数据集、训练、评测、推理、数据处理、数据转换、算力、存储和审计入口的登录及模块权限。
|
||||||
|
3. 新增 `tenant_members`,新建用户、项目、数据集、训练任务和数据处理任务写入当前租户与创建者。
|
||||||
|
4. ACL 写入校验主体存在性、状态、租户边界、有效期和撤销状态;普通所有者不能授予 `delete/admin` 或跨租户权限。
|
||||||
|
5. 新增 `resource_access_requests`;审批人由当前会话确定,禁止伪造审批人和申请人自审;审批通过后才落 ACL。
|
||||||
|
6. 训练、评测和推理统一校验数据集、基座模型、训练模型、任务、节点和 GPU 使用权限;下载和导出使用独立的 `download` 权限。
|
||||||
|
7. 前端认证 store 新增 `can(resource, action)`,模型管理和审批页面按权限显示操作按钮,后端 401/403 仍是最终防线。
|
||||||
|
8. 新增 `009_permission_completion.sql`;主工程与离线目录的完整初始化 SQL 已同步且 SHA-256 一致,离线目录只保留完整初始化脚本。
|
||||||
|
|
||||||
|
### 本轮验证
|
||||||
|
|
||||||
|
- 后端权限相关文件 `py_compile` 通过。
|
||||||
|
- 前端 `npm run build` 通过;仅保留已有字体路径和 chunk size 警告。
|
||||||
|
- 当前 WSL 容器已确认 Backend、Frontend、Compute、Redis、MinIO 均运行;基础健康和未登录拦截已验证,历史治理测试夹具与当前鉴权/数据库行为不完全兼容,不能替代新的权限专项回归。
|
||||||
|
|
||||||
|
### 上线前验证
|
||||||
|
|
||||||
|
- 第二个可达节点或多卡节点到位后的多租户、多 GPU 并行压力测试。
|
||||||
|
- 对已执行的 `009_permission_completion.sql` 进行旧数据租户归属、ACL、审批和软删除记录对账。
|
||||||
|
- 更新 Backend/Frontend 容器后重新执行治理测试和权限专项测试。
|
||||||
|
|
||||||
|
## 十、当前复核结论(2026-08-19)
|
||||||
|
|
||||||
|
### 10.1 已完成的基础能力
|
||||||
|
|
||||||
|
本轮 11 项权限改造已经形成基础闭环:会话和用户状态校验、租户成员关系、资源 ACL、资源访问申请、审批决策安全、模型/数据集/训练/评测/推理入口权限、GPU 分配申请、配额申请、软删除和前端基础按钮控制均已落地;数据库迁移和完整初始化 SQL 已同步到主工程及离线目录。
|
||||||
|
|
||||||
|
这些能力可以作为后续完善的基础,但“接口存在”不等于“所有资源和所有异常路径均已达到上线标准”。尤其是密钥暴露、配额实际拦截、资源列表一致性和专项测试仍需要继续处理。
|
||||||
|
|
||||||
|
### 10.2 仍需开发的功能
|
||||||
|
|
||||||
|
| 优先级 | 功能 | 当前缺口 | 处理结论 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| P0 | 模型密钥和敏感信息保护 | 模型列表、详情、名称查询、创建和更新响应已移除真实 `api_key`,仍需继续排查任务详情、日志和其他敏感配置输出 | 基础闭环已完成;继续做全量敏感字段扫描和受控密钥管理 |
|
||||||
|
| P0 | 统一资源动作策略 | 已增加资源动作白名单和 `export -> download` 归一化,但各业务入口仍需逐步迁移到统一授权服务 | 第一阶段已完成;继续完成全量入口收敛 |
|
||||||
|
| P0 | 数据转换、数据处理和存储权限一致性 | 部分列表、文件、预览、版本、缓存和 MinIO 对象入口仍需逐接口核对“租户 + 所有者/ACL + 动作” | 必须完成全量入口审计,尤其是 `read/download/execute/write/delete` 的区分 |
|
||||||
|
| P0 | 审计失败链路 | 审计装饰器已记录失败结果、原因、请求 ID、会话 ID 和租户;手工审计入口仍需继续统一 | 基础闭环已完成;继续补齐所有权限拒绝和异常入口 |
|
||||||
|
| P1 | 租户成员生命周期 | 已有成员表和角色,但缺少邀请、加入申请、审批、过期、移除和前端租户切换的完整流程 | 需要开发,明确平台管理员与租户管理员的授予边界 |
|
||||||
|
| P1 | 配额强制执行 | 配额申请和审批已实现,但训练、评测、推理、存储等资源消耗尚未全部进行原子配额检查和扣减 | 需要开发配额使用量/预留量/释放量账本,不能只保存配额配置 |
|
||||||
|
| P1 | GPU 分配强校验 | 申请链路已实现,但分配前仍需统一校验节点状态、GPU 存在性、冲突、租户配额和释放对账 | 需要开发原子预留、超时回收和节点故障对账 |
|
||||||
|
| P1 | 审批策略完整性 | 需要严格限制动作白名单、资源类型、模板作用域和重复申请;过期处理不应只依赖查询触发 | 需要补充定时过期、幂等键、执行失败重试和策略管理边界 |
|
||||||
|
| P1 | 派生模型和数据血缘授权 | 基座模型、数据集、训练模型之间已有部分血缘,但权限来源、版本快照和派生资源默认授权规则还不够统一 | 需要固化“创建者 + 租户 + 显式 ACL”,禁止共享基座模型自动公开训练产物 |
|
||||||
|
| P1 | 用户软删除和对象清理 | 用户及部分关联关系仍有物理删除风险;MinIO 对象、ACL、审批、缓存的异步清理缺少统一编排 | 需要补齐软删除、撤销、延迟清理和失败重试 |
|
||||||
|
| P2 | 前端资源级权限体验 | 已有菜单/按钮级基础控制,但缺少统一 `can(resource, action)`、授权来源、有效期、审批中和 403 状态展示 | 需要开发统一权限组件和错误状态处理,后端校验仍是最终边界 |
|
||||||
|
| P2 | 权限专项测试与上线检查 | 基础构建、静态检查、健康接口已验证,尚未完成双用户、跨租户、撤销、过期、MinIO 故障、GPU 冲突的全流程矩阵 | 必须补充自动化测试、迁移回归和离线部署验收 |
|
||||||
|
|
||||||
|
### 10.3 需要优化的设计
|
||||||
|
|
||||||
|
1. **从“角色判断”改为“动作授权”**:统一使用 `resource_type + resource_id + action + tenant_id + actor` 判断,平台管理员只作为明确的管理范围,不再作为各业务模块的隐式旁路。
|
||||||
|
2. **区分平台角色和租户角色**:`users.role` 只表达平台级角色,`tenant_members.role` 表达租户内角色;禁止通过租户成员关系授予平台管理员权限。
|
||||||
|
3. **区分查看、下载、执行、编辑、删除、授权**:`read` 不能替代 `download`,`execute` 不能替代 `export`,高风险动作必须绑定审批和审计。
|
||||||
|
4. **统一资源列表和详情规则**:列表、详情、文件、版本、预览、缓存、下载和导出必须调用同一授权服务,避免“列表看不到但接口可访问”或“列表能看到但操作必然 403”。
|
||||||
|
5. **权限与配额采用预留模型**:任务提交时原子预留 GPU、显存、并发数和存储额度,任务完成、失败、取消和节点失联时统一释放或对账。
|
||||||
|
6. **审批采用可执行状态机**:申请、审批、拒绝、撤回、过期、执行中、执行失败、已执行状态分离;审批结果应有幂等执行记录,避免重复授权或重复分配。
|
||||||
|
7. **敏感字段默认拒绝返回**:API key、对象内部凭据、节点访问凭据不能随普通资源详情返回;日志、审计和错误信息也不能泄露 Token、密码或完整连接串。
|
||||||
|
8. **安全审计与访问统计分离**:权限成功、拒绝、下载、导出、授权、审批和异常进入不可篡改审计;页面访问统计单独存储,避免污染安全审计记录。
|
||||||
|
|
||||||
|
## 十一、下一步开发计划
|
||||||
|
|
||||||
|
### 阶段一:P0 安全封堵与统一策略
|
||||||
|
|
||||||
|
1. 对模型列表、详情、名称查询及相关 DTO 做 API key/凭据脱敏,增加“仅后端内部读取”的封装。(基础闭环已完成,继续扫描任务和日志输出)
|
||||||
|
2. 建立统一 `authorize(resource, action)` 服务,定义动作白名单和平台管理员、租户管理员、所有者、ACL 的判定顺序。(已完成动作白名单,继续迁移全部入口)
|
||||||
|
3. 逐项审计平台、数据处理、数据转换、存储、MinIO、模型、数据集、训练、评测、推理的列表、详情、预览、下载、导出、缓存和删除入口。
|
||||||
|
4. 统一记录权限成功、拒绝和异常审计,补齐 `tenant_id`、`resource_id`、`action`、`request_id`、`session_id`、`reason` 和 `result`。(装饰器基础闭环已完成,继续覆盖手工记录入口)
|
||||||
|
|
||||||
|
### 阶段二:租户成员、审批和配额闭环
|
||||||
|
|
||||||
|
1. 开发租户邀请/加入申请/审批/移除/过期流程及前端租户切换。
|
||||||
|
2. 为审批模板增加动作和资源类型白名单、租户作用域、幂等键、过期任务和执行失败重试。
|
||||||
|
3. 建立配额预留账本,训练、评测、推理、存储和 GPU 任务提交前统一原子检查;任务终态和故障恢复统一释放。
|
||||||
|
4. 完善 GPU 节点、GPU 卡、租户、用户和任务的占用关系校验,覆盖冲突、超时、节点不可达和服务重启恢复。
|
||||||
|
|
||||||
|
### 阶段三:资源血缘和生命周期
|
||||||
|
|
||||||
|
1. 固化基座模型、数据集、数据处理结果、训练任务、训练模型、评测和推理任务的资源血缘及版本快照。
|
||||||
|
2. 明确派生模型默认授权规则,训练产物不因基座模型共享而自动对外公开。
|
||||||
|
3. 补齐用户、租户、资源、ACL、审批、MinIO 对象和本地缓存的软删除、撤销、延迟清理和失败重试。
|
||||||
|
|
||||||
|
### 阶段四:前端权限体验与测试
|
||||||
|
|
||||||
|
1. 开发统一资源级权限组件,按动作隐藏/禁用按钮,并展示授权来源、有效期、审批状态和明确的 401/403 原因。
|
||||||
|
2. 将审批、ACL、审计列表中的用户、租户、资源 ID 映射为可读名称,同时保留 ID 作为详情字段。
|
||||||
|
3. 编写并执行权限专项测试矩阵:未登录、同租户成员、资源所有者、ACL 授权、跨租户、禁用用户、会话注销、审批撤回/过期、软删除、MinIO 故障、GPU 冲突和配额不足。
|
||||||
|
4. 在第二个可达节点或多卡节点准备后执行多节点、多 GPU 并发及故障恢复测试;完成主工程与 `docker/offline/src` 的源码、初始化 SQL、迁移和部署文档一致性检查。
|
||||||
|
|
||||||
|
## 十二、下一阶段完成标准
|
||||||
|
|
||||||
|
- 普通资源接口不再返回 API key、密码、Token 或节点访问凭据。
|
||||||
|
- 所有资源入口都经过统一的租户边界和动作授权,跨租户访问返回明确 403。
|
||||||
|
- 训练、评测、推理创建同时满足资源权限、GPU 预留和租户配额,失败时不会留下孤儿占用。
|
||||||
|
- 审批只能由合法审批人执行,申请人不能自审;重复回调不会重复授权、分配或扣减配额。
|
||||||
|
- 权限拒绝、下载、导出、授权、审批和异常均能按用户、租户、资源和请求 ID追溯。
|
||||||
|
- 主工程和离线目录初始化新库均无缺表、缺字段;迁移可重复执行且不破坏既有数据。
|
||||||
|
- 权限专项自动化测试通过,且完成至少一次双用户、跨租户和 MinIO 故障场景的真实容器验证。
|
||||||
|
|
||||||
|
## 十三、上一阶段权限闭环开发结果(2026-08-20)
|
||||||
|
|
||||||
|
本轮围绕 P0 安全封堵完成了一个可验证的权限闭环:
|
||||||
|
|
||||||
|
1. **模型凭据不出接口**:模型列表、详情、按名称查询、创建、更新和用途更新响应统一经过公开 DTO 脱敏,移除 `api_key`、密码、Token 等字段,仅返回 `api_key_configured` 布尔状态;后端内部评测、数据生成仍使用数据库中的真实配置。
|
||||||
|
2. **编辑密钥不被意外清空**:前端编辑在线模型时不回填真实密钥,留空表示保留已有配置,只有管理员主动输入新值时才提交替换。
|
||||||
|
3. **资源动作统一入口**:增加资源动作白名单 `read/write/execute/download/export/delete/admin`,并将 `export` 统一归一到下载权限,非法动作默认拒绝,避免把模块权限误当成资源权限。
|
||||||
|
4. **失败审计闭环**:审计装饰器对成功和异常分别记录,失败记录包含结果、失败原因、租户、请求 ID、会话 ID和耗时,并对异常中的常见凭据进行脱敏。
|
||||||
|
5. **访问统计受控**:模块访问统计只接受平台已登记的模块名,不能通过请求参数向安全审计表写入任意动作;审计 CSV 导出改为标准 CSV 编码,并补充结果、原因、请求和会话字段。
|
||||||
|
6. **离线版本同步**:上述后端权限代码、前端模型编辑代码和权限安全回归用例已同步到 `docker/offline/src`;本轮未新增数据库字段,因此 `000_full_init.sql` 无需变更。
|
||||||
|
|
||||||
|
### 上一阶段验证结果
|
||||||
|
|
||||||
|
- 后端权限相关模块 `python -m py_compile`:通过。
|
||||||
|
- WSL Backend 容器执行权限安全用例:8 passed;仅有 pytest 缓存目录只读警告。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 路径和 chunk size 警告。
|
||||||
|
- 容器接口验证:未登录访问模型接口返回 401;管理员访问模型列表/详情时响应不包含 `api_key`,仅返回 `api_key_configured`。
|
||||||
|
- Backend 容器已重启并加载当前源码;数据库无需迁移。
|
||||||
|
|
||||||
|
### 上一阶段未覆盖的范围(已在第十四节继续处理)
|
||||||
|
|
||||||
|
本轮没有声称完成配额账本、GPU 原子预留、租户邀请/加入申请、审批定时过期、所有资源入口的逐接口审计,以及第二节点/多 GPU 的真实并发测试;这些仍按“十一、下一步开发计划”执行。
|
||||||
|
|
||||||
|
## 十四、本轮继续开发结果(2026-08-20)
|
||||||
|
|
||||||
|
### 已完成
|
||||||
|
|
||||||
|
1. 新增 `tenant_quota_reservations` 配额预留账本,支持按租户统计 GPU 预留量、原子限制 GPU 配额,以及任务完成、失败、停止、删除和节点故障时释放预留。
|
||||||
|
2. 训练、评测和推理统一接入租户 GPU 配额预留;评测/推理使用的 `gpu_reservations` 与配额账本在同一事务内创建或释放,避免只释放算力卡而遗留配额占用。
|
||||||
|
3. 租户成员支持邀请、接受、过期、角色更新和移除;邀请不允许授予 `owner`,成员状态和租户有效性由后端校验,租户详情页补充成员管理和 GPU 配额使用量展示。
|
||||||
|
4. 审批动作增加白名单,非法动作拒绝创建;相同申请人在同一资源上的待审批申请幂等复用;审批实例读取时自动处理过期状态,资源访问申请也避免重复创建。
|
||||||
|
5. GPU 分配增加节点启用状态、GPU 卡存在性、用户 active 状态和同卡跨用户冲突校验;冲突返回 409,不再静默覆盖或产生重复授权。
|
||||||
|
6. 完整初始化 SQL 增加配额预留表和索引,新增 `010_permission_quota_membership.sql` 增量迁移;主工程相关源码、前端租户 API/页面和 SQL 已同步到离线源码目录。
|
||||||
|
|
||||||
|
### 本轮验证
|
||||||
|
|
||||||
|
- WSL Backend 容器重启后,`tenant_quota_reservations` 可正常查询,默认租户配额使用量返回正常。
|
||||||
|
- WSL Backend 容器执行 `test_permission_security.py`:3 passed。
|
||||||
|
- 后端相关文件 `python -m py_compile`:通过。
|
||||||
|
- 前端 `npm run build`:通过;仅保留既有 Font Awesome 资源路径和大 chunk 警告。
|
||||||
|
|
||||||
|
### 必须后续验证的场景
|
||||||
|
|
||||||
|
1. 创建第二个 active 用户和第二个租户,验证邀请接受、租户切换、跨租户资源访问均按预期返回 401/403。
|
||||||
|
2. 将租户 GPU 配额设为 1,同时提交两个需要 GPU 的训练/评测/推理任务,确认第二个任务被拒绝;第一个任务终态后确认配额可再次使用。
|
||||||
|
3. 在同一多卡节点上让两个用户申请同一张卡,确认审批执行阶段冲突返回 409,另一张空闲卡仍可正常分配。
|
||||||
|
4. 构造过期审批、重复提交、审批拒绝/撤回,确认不会重复创建 ACL、GPU 分配或配额预留。
|
||||||
|
5. 准备第二个可达节点或多卡节点,执行训练、评测、推理的多节点/多 GPU 并发和节点失联回收测试。
|
||||||
|
6. 注入 MinIO 故障,确认任务失败后 GPU 与租户配额均能释放,恢复 MinIO 后可以重新提交任务。
|
||||||
|
|
||||||
|
### 下一步计划
|
||||||
|
|
||||||
|
- 对数据处理、数据转换、MinIO 对象、缓存、版本和报告下载入口继续执行一次接口级回归,重点验证跨租户、过期 ACL 和下载/执行动作分离。
|
||||||
|
- 将双用户、跨租户、配额不足、同卡冲突、审批过期/幂等、MinIO 故障和节点失联场景加入可重复的自动化测试夹具;当前单节点环境先完成无需第二节点的部分。
|
||||||
|
- 在第二个可达节点或多卡节点到位后,执行多节点、多 GPU 并发、任务失败回收和重启恢复测试,完成上线验收闭环。
|
||||||
|
|
||||||
|
## 十五、本轮权限全量收口结果(2026-08-20)
|
||||||
|
|
||||||
|
### 已完成的开发
|
||||||
|
|
||||||
|
1. **租户成员角色真正参与资源授权**:ACL 的 `role` 主体不再只读取平台角色,改为读取资源所属租户中的有效成员角色;成员过期、禁用或不属于资源租户时不能通过角色 ACL 访问。
|
||||||
|
2. **租户有效期隔离**:用户可用租户集合和资源授权均校验成员有效期,避免过期成员继续访问资源。
|
||||||
|
3. **用户生命周期闭环**:用户删除改为软删除,保留用户和审计链路;同时注销会话、禁用租户成员、撤销用户 ACL、取消待审批申请、释放 GPU 分配,并禁止删除租户最后一个 owner。
|
||||||
|
4. **资源和对象清理链路**:被删除用户创建的模型、训练模型、数据集、评测、推理、数据处理和数据转换资源统一标记删除;关联 MinIO `storage_objects` 标记为 `deleted`,进入既有清理/重试队列;资源 ACL 和安全状态同步撤销。
|
||||||
|
5. **推理入口补强**:训练模型预加载必须引用真实且未删除的训练模型资源,普通用户不能通过任意本地路径绕过资源授权;本地推理状态查询要求普通用户提供有 `read` 权限的推理任务。
|
||||||
|
6. **初始化和迁移完整性**:新增 `011_permission_lifecycle.sql`,完整初始化脚本同步补齐用户、评测、推理、数据转换和存储对象生命周期字段;离线目录仍只保留完整的 `000_full_init.sql`。
|
||||||
|
|
||||||
|
### 自动验证结果
|
||||||
|
|
||||||
|
- `python -m py_compile backend/app/core/auth.py backend/app/db/platform_store.py backend/app/api/v1/endpoints/platform.py`:通过。
|
||||||
|
- `git diff --check`:通过。
|
||||||
|
- WSL Backend 容器启动并自动执行迁移后为 `healthy`;已核对 `users`、`eval_tasks`、`compare_tasks`、`data_convert_tasks`、`storage_objects` 的新增字段均存在。
|
||||||
|
- `test_permission_security.py` 与 `test_storage_security.py`:8 passed;仅有容器内 pytest 缓存目录只读警告。
|
||||||
|
- 真实 API 闭环:临时用户删除后状态为 `deleted`、存在 `deleted_at`,使用原凭据登录返回 HTTP 401。
|
||||||
|
- 前端 `npm run build`:需在本轮同步后再次执行,预期仅保留既有资源路径和 chunk size 警告;该项以最终命令结果为准。
|
||||||
|
|
||||||
|
### 仍需现场验证的场景
|
||||||
|
|
||||||
|
1. 第二个 active 用户和第二个租户的邀请、接受、租户切换及跨租户资源访问。
|
||||||
|
2. 租户 GPU 配额为 1 时,训练/评测/推理并发预留、终态释放和服务重启恢复。
|
||||||
|
3. 同一多卡节点的同卡冲突、不同卡并行和节点不可达后的回收。
|
||||||
|
4. 审批过期、重复回调、拒绝、撤回和执行失败重试的真实链路。
|
||||||
|
5. MinIO 故障注入后的任务失败、GPU/配额释放、对象清理重试和恢复后重新提交。
|
||||||
|
|
||||||
|
以上项目属于环境依赖型验收,不再作为代码未开发项;在当前单节点、单用户测试环境中保留为上线前验证项。
|
||||||
|
|
||||||
|
## 十六、前端权限页面收口结果(2026-08-20)
|
||||||
|
|
||||||
|
本轮已完成前端权限页面的主要操作闭环:
|
||||||
|
|
||||||
|
1. `用户权限设置`由占位页改为完整权限矩阵,支持按平台角色、账号状态和页面权限保存;已删除用户进入只读态,管理员角色自动拥有全部页面权限。
|
||||||
|
2. 用户创建页增加页面权限配置,普通用户默认不授予组织管理和算力节点权限。
|
||||||
|
3. 用户管理页增加权限入口、权限数量、软删除状态和危险操作禁用逻辑。
|
||||||
|
4. 资源 ACL 页支持用户/租户角色主体、权限标签、全部撤销二次确认和加载失败提示。
|
||||||
|
5. 审批策略页从 JSON 文本改为可视化配置指定用户、租户管理员和平台管理员审批步骤。
|
||||||
|
6. 审批申请和访问申请页增加中文事项、资源类型、权限和状态展示,保留资源 ID 作为详情信息。
|
||||||
|
7. 普通用户可进入审批中心的“我的申请”“访问申请”“租户邀请”,支持撤回访问申请和接受有效租户邀请。
|
||||||
|
8. 租户成员邀请页增加邀请有效期设置;租户配额和 GPU 预留量继续在租户详情页展示。
|
||||||
|
9. 本轮前端源码已同步至 `docker/offline/src/frontend/src`,初始化 SQL 目录仍只保留 `000_full_init.sql`。
|
||||||
|
|
||||||
|
前端 `npm run build` 已通过。尚需浏览器实际点击验证页面布局、不同账号菜单差异、双用户跨租户流程和多 GPU/MinIO 故障场景;这些属于环境验收,不再是页面缺少开发入口。
|
||||||
227
docs/权限验收测试用例.md
Normal file
227
docs/权限验收测试用例.md
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
# 权限验收测试用例
|
||||||
|
|
||||||
|
> 版本:权限 2.0 前端权限页面收口版
|
||||||
|
> 编写日期:2026-08-20
|
||||||
|
> 适用系统:YG_FT 微调平台主工程、Backend、Frontend、Compute Agent、PostgreSQL、Redis、MinIO
|
||||||
|
> 说明:本文件将自动化验证、单节点页面验证和多用户/多节点专项验证分开记录。
|
||||||
|
|
||||||
|
## 1. 验收目标
|
||||||
|
|
||||||
|
验证平台权限功能是否能够形成以下闭环:
|
||||||
|
|
||||||
|
`登录认证 → 页面权限 → 租户成员 → 资源创建 → ACL/访问申请 → 审批 → 训练/评测/推理执行 → GPU/配额预留 → 任务释放 → 审计 → 软删除和对象清理`
|
||||||
|
|
||||||
|
验收重点:
|
||||||
|
|
||||||
|
- 前端页面是否提供完整的权限管理入口。
|
||||||
|
- 普通用户是否可以提交访问申请、查看自己的申请和接受租户邀请。
|
||||||
|
- 管理员是否可以配置用户页面权限、租户成员、租户配额、资源 ACL 和审批策略。
|
||||||
|
- 页面按钮隐藏只是辅助体验,直接调用接口仍必须由后端返回 401/403。
|
||||||
|
- 用户、租户、资源、ACL、审批、GPU、配额和 MinIO 对象状态是否能够保持一致。
|
||||||
|
|
||||||
|
## 2. 测试环境
|
||||||
|
|
||||||
|
### 2.1 服务
|
||||||
|
|
||||||
|
| 服务 | 地址/容器 | 验收要求 |
|
||||||
|
|---|---|---|
|
||||||
|
| Frontend | `http://127.0.0.1:16801` | 页面可打开,路由切换无白屏 |
|
||||||
|
| Backend | `http://127.0.0.1:17861/modelTF` | 健康接口返回成功 |
|
||||||
|
| Compute API | `http://172.25.179.69:19100` | 当前可达节点可进行单节点验证 |
|
||||||
|
| Redis | WSL Docker `yg-ft-redis` | 容器 healthy |
|
||||||
|
| MinIO | WSL Docker `yg-ft-minio` | 容器 healthy,可进行对象读写 |
|
||||||
|
| PostgreSQL | 当前环境实际远程 PostgreSQL | 初始化字段和增量迁移均存在 |
|
||||||
|
|
||||||
|
### 2.2 测试账号和数据
|
||||||
|
|
||||||
|
准备以下测试对象,禁止使用生产账号和真实敏感数据:
|
||||||
|
|
||||||
|
| 对象 | 建议值 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| 平台管理员 | 当前环境管理员账号 | 执行用户、租户、ACL、审批和配额管理 |
|
||||||
|
| 普通用户 A | 新建测试用户 | 作为资源所有者 |
|
||||||
|
| 普通用户 B | 新建测试用户 | 作为被授权用户和申请人 |
|
||||||
|
| 租户 A | 新建测试租户 | 与租户 B 隔离验证 |
|
||||||
|
| 租户 B | 新建测试租户 | 跨租户访问验证 |
|
||||||
|
| 测试数据集 | `test_data_0817` 或新的小数据集 | 用于 ACL、训练和下载验证 |
|
||||||
|
| 测试模型 | `qwen3.5-0.8B` | 用于模型使用权限和推理验证 |
|
||||||
|
|
||||||
|
## 3. 自动化验证结果
|
||||||
|
|
||||||
|
以下项目已在当前开发环境执行:
|
||||||
|
|
||||||
|
| 编号 | 验证内容 | 结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| AUTO-01 | `npm run build` | 通过;仅有既有 Font Awesome 路径和 chunk size 警告 |
|
||||||
|
| AUTO-02 | `test_permission_security.py` | 通过,8 passed |
|
||||||
|
| AUTO-03 | `test_storage_security.py` | 已包含在 8 passed 中 |
|
||||||
|
| AUTO-04 | 后端权限文件 `py_compile` | 通过 |
|
||||||
|
| AUTO-05 | `git diff --check` | 通过 |
|
||||||
|
| AUTO-06 | Backend、Frontend、Compute、Redis、MinIO 容器状态 | 通过,Backend healthy |
|
||||||
|
| AUTO-07 | 迁移后字段核对 | 通过,users、eval_tasks、compare_tasks、data_convert_tasks、storage_objects 字段存在 |
|
||||||
|
| AUTO-08 | 临时用户软删除闭环 | 通过,状态为 deleted、存在 deleted_at,原凭据登录返回 401 |
|
||||||
|
| AUTO-09 | `docker/offline/src` 同步 | 通过,离线 SQL 目录只保留 `000_full_init.sql` |
|
||||||
|
|
||||||
|
## 4. 前端页面验收
|
||||||
|
|
||||||
|
### 4.1 组织与权限
|
||||||
|
|
||||||
|
页面入口:`/organization`
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| UI-ORG-01 | 管理员打开“组织与权限” | 显示“用户与角色”“租户与配额”两个页签 |
|
||||||
|
| UI-ORG-02 | 点击“创建用户” | 进入用户创建页面,可填写账号、角色、状态和页面权限 |
|
||||||
|
| UI-ORG-03 | 创建普通用户 | 默认授予业务页面权限,不自动授予组织管理和算力节点权限 |
|
||||||
|
| UI-ORG-04 | 用户列表点击“权限” | 进入 `/user-settings/:id/permission`,显示用户角色、所属租户、状态和权限矩阵 |
|
||||||
|
| UI-ORG-05 | 修改用户页面权限并保存 | 页面提示保存成功,刷新后权限保持一致 |
|
||||||
|
| UI-ORG-06 | 将普通用户改为管理员 | 页面显示全部权限;保存时自动补齐全部权限码 |
|
||||||
|
| UI-ORG-07 | 查看已软删除用户 | 显示“已删除”,权限页只读,不能重新激活 |
|
||||||
|
| UI-ORG-08 | 普通用户访问组织与权限 | 路由跳转到无权访问页,菜单不显示管理员治理入口 |
|
||||||
|
|
||||||
|
### 4.2 租户与配额
|
||||||
|
|
||||||
|
页面入口:`/organization?tab=tenants`、`/tenants/:id`
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| UI-TEN-01 | 管理员创建租户并设置 GPU、存储配额 | 租户列表出现新租户,配额显示正确 |
|
||||||
|
| UI-TEN-02 | 打开租户详情 | 显示租户信息、配额设置、GPU 预留量和成员列表 |
|
||||||
|
| UI-TEN-03 | 邀请成员并设置角色、邀请有效期 | 页面提示邀请成功,成员状态为待接受,过期时间正确 |
|
||||||
|
| UI-TEN-04 | 修改成员角色 | 成员列表角色更新,不能通过页面授予非法 owner 权限 |
|
||||||
|
| UI-TEN-05 | 移除成员 | 成员状态/关系更新,后续资源访问按后端权限拒绝 |
|
||||||
|
| UI-TEN-06 | 保存配额 | 配额配置更新,已预留 GPU 数量不超过新配额 |
|
||||||
|
|
||||||
|
### 4.3 我的申请与租户邀请
|
||||||
|
|
||||||
|
页面入口:`/approval-instances?tab=mine`、`/approval-instances?tab=access`、`/approval-instances?tab=invitations`
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| UI-SELF-01 | 普通用户打开审批中心 | 可进入,不再被管理员专属路由拦截 |
|
||||||
|
| UI-SELF-02 | 打开“访问申请” | 可选择可见的数据集/训练模型,资源名称优先显示中文名称,同时保留 ID |
|
||||||
|
| UI-SELF-03 | 提交 read/execute/download 申请 | 申请出现在“我的访问申请”,状态为审批中 |
|
||||||
|
| UI-SELF-04 | 撤回待审批申请 | 二次确认后状态变为已撤回,不能再次撤回 |
|
||||||
|
| UI-SELF-05 | 打开“租户邀请” | 显示租户、角色、邀请人、有效期和状态 |
|
||||||
|
| UI-SELF-06 | 接受有效邀请 | 状态变为已加入,获得对应租户成员关系 |
|
||||||
|
| UI-SELF-07 | 接受过期邀请 | 后端拒绝,页面显示失败原因,成员关系不产生 |
|
||||||
|
|
||||||
|
### 4.4 资源授权
|
||||||
|
|
||||||
|
页面入口:`/resource-acl`
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| UI-ACL-01 | 管理员选择数据集或训练模型 | 显示资源名称和资源 ID,不显示已软删除资源 |
|
||||||
|
| UI-ACL-02 | 查看资源 ACL | 显示用户/租户角色、主体类型和权限标签 |
|
||||||
|
| UI-ACL-03 | 增加用户 ACL | 可选择查看、编辑、使用、下载等权限并保存 |
|
||||||
|
| UI-ACL-04 | 增加角色 ACL | 可选择租户管理员、成员、只读角色;后端校验最终结果 |
|
||||||
|
| UI-ACL-05 | 移除全部 ACL | 页面二次确认后保存空 ACL,授权全部撤销 |
|
||||||
|
| UI-ACL-06 | ACL 查询失败 | 页面显示明确错误提示,不把失败误显示为空授权 |
|
||||||
|
| UI-ACL-07 | 普通用户打开资源授权页面 | 页面入口隐藏,直接访问接口返回 403 |
|
||||||
|
|
||||||
|
### 4.5 审批中心
|
||||||
|
|
||||||
|
页面入口:`/approval-instances`
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| UI-APP-01 | 管理员打开审批申请 | 显示资源类型、申请事项、申请人、状态和当前步骤的中文信息 |
|
||||||
|
| UI-APP-02 | 管理员打开审批策略 | 可配置指定用户、租户管理员、平台管理员审批步骤,不需要输入 JSON |
|
||||||
|
| UI-APP-03 | 指定审批人审批 | 只能由当前会话指定审批人执行,审批结果和意见可见 |
|
||||||
|
| UI-APP-04 | 申请人查看自己的申请 | 只能看到自己的申请,不显示其他租户的审批数据 |
|
||||||
|
| UI-APP-05 | 审批通过 | ACL、GPU 分配或成员关系等业务效果真正落地 |
|
||||||
|
| UI-APP-06 | 审批拒绝/过期 | 状态明确显示,不能产生授权或资源占用 |
|
||||||
|
|
||||||
|
## 5. 后端接口与页面联动验收
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| API-AUTH-01 | 未登录访问任意受保护页面接口 | 返回 401,前端回到登录页 |
|
||||||
|
| API-AUTH-02 | 普通用户直接调用管理员接口 | 返回 403,不能仅依赖前端隐藏按钮 |
|
||||||
|
| API-AUTH-03 | 禁用用户使用旧 Token | 返回 401 或明确无效会话错误 |
|
||||||
|
| API-AUTH-04 | 已删除用户登录 | 返回 401 |
|
||||||
|
| API-TEN-01 | 用户访问其他租户资源详情 | 返回 403 或资源不存在,不泄露资源内容 |
|
||||||
|
| API-TEN-02 | 过期成员访问租户资源 | 返回 403 |
|
||||||
|
| API-ACL-01 | 没有 read 的用户读取资源 ACL | 返回 403 |
|
||||||
|
| API-ACL-02 | 只有 read 的用户执行模型推理 | 返回 403 |
|
||||||
|
| API-ACL-03 | 只有 execute 的用户下载模型 | 返回 403 |
|
||||||
|
| API-ACL-04 | ACL 授权后重新访问 | 按授权动作成功 |
|
||||||
|
| API-APP-01 | 申请人提交自己的审批决策 | 返回 403 |
|
||||||
|
| API-APP-02 | 非指定审批人提交决策 | 返回 403 |
|
||||||
|
| API-APP-03 | 重复提交相同资源访问申请 | 复用或拒绝重复待审批申请,不重复写 ACL |
|
||||||
|
| API-LIFE-01 | 删除用户 | 会话、成员、ACL、GPU 分配、待审批项和用户资源状态同步处理 |
|
||||||
|
| API-LIFE-02 | 执行存储清理 | 已删除 MinIO 对象进入清理,失败任务可重试 |
|
||||||
|
|
||||||
|
## 6. 训练、评测、推理权限验收
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| JOB-01 | 用户使用无权限数据集创建训练 | 预检失败,显示数据集无权限 |
|
||||||
|
| JOB-02 | 用户使用授权数据集创建训练 | 预检通过,GPU 和租户配额原子预留 |
|
||||||
|
| JOB-03 | 用户使用未授权训练模型推理 | 页面提示无权限,接口返回 403 |
|
||||||
|
| JOB-04 | 用户使用已授权训练模型推理 | 可选择有权限的算力节点和 GPU 卡 |
|
||||||
|
| JOB-05 | 用户没有 download 权限导出模型 | 操作按钮不显示,接口返回 403 |
|
||||||
|
| JOB-06 | 评测任务结束 | GPU 预留和租户配额预留释放 |
|
||||||
|
| JOB-07 | 推理任务删除 | 推理服务释放,任务软删除,ACL 状态撤销 |
|
||||||
|
| JOB-08 | 节点不可达或 MinIO 暂时不可用 | 任务进入失败/重试状态,不留下永久 GPU 或配额占用 |
|
||||||
|
|
||||||
|
## 7. GPU、配额和多节点专项验收
|
||||||
|
|
||||||
|
以下用例需要第二个可达节点或一个包含多张可分配 GPU 卡的节点,当前单节点环境暂保留。
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| GPU-01 | 租户 GPU 配额设置为 1,同时提交两个 GPU 任务 | 第二个任务被拒绝,不能超配 |
|
||||||
|
| GPU-02 | 同一节点同一张卡由两个用户申请 | 审批/分配阶段返回 409,不能重复占用 |
|
||||||
|
| GPU-03 | 同一节点选择两张空闲卡 | 任务使用页面指定的两张卡,不使用其他卡 |
|
||||||
|
| GPU-04 | 任务失败或取消 | GPU 分配和配额预留释放 |
|
||||||
|
| GPU-05 | Backend 重启后恢复任务状态 | 已完成/失败任务不重复占用;运行任务保持可对账 |
|
||||||
|
| GPU-06 | 模型训练、推理、评测选择不同节点 | 模型和数据从 MinIO 准备到指定节点,不错误派发到其他节点 |
|
||||||
|
| GPU-07 | 节点失联 | 任务进入失败或待重试,资源占用在超时后回收 |
|
||||||
|
|
||||||
|
## 8. MinIO 故障专项验收
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| MINIO-01 | 暂停 MinIO 容器后上传数据集 | 页面显示明确存储不可用,不显示上传成功 |
|
||||||
|
| MINIO-02 | MinIO 暂停期间预检训练 | 预检提示对象不可用,可重试 |
|
||||||
|
| MINIO-03 | MinIO 恢复后重试任务 | 对象可重新准备,任务继续或重新提交成功 |
|
||||||
|
| MINIO-04 | 删除用户后执行对象清理 | 用户资源对象标记 deleted,清理成功后标记 purged |
|
||||||
|
| MINIO-05 | 清理时对象删除失败 | storage cleanup job 保留 failed 状态和错误原因,后续可重试 |
|
||||||
|
|
||||||
|
## 9. 离线部署验收
|
||||||
|
|
||||||
|
| 用例 | 操作步骤 | 预期结果 |
|
||||||
|
|---|---|---|
|
||||||
|
| OFFLINE-01 | 检查 `docker/offline/src/backend/app/db/sql` | 只存在完整的 `000_full_init.sql` |
|
||||||
|
| OFFLINE-02 | 清空数据库并执行完整初始化 SQL | 不缺权限相关表和字段,Backend 可启动 |
|
||||||
|
| OFFLINE-03 | 离线启动 Frontend | 不请求在线 CDN,页面可正常打开 |
|
||||||
|
| OFFLINE-04 | 离线启动 Backend、Compute、MinIO | 各服务使用独立网络和配置,权限接口可用 |
|
||||||
|
| OFFLINE-05 | 离线新库执行用户/租户/ACL/审批流程 | 与主工程行为一致 |
|
||||||
|
|
||||||
|
## 10. 问题记录规则
|
||||||
|
|
||||||
|
每条失败用例至少记录:
|
||||||
|
|
||||||
|
- 测试编号、执行时间、执行人和账号。
|
||||||
|
- 页面 URL、接口 URL、请求方法和请求 ID。
|
||||||
|
- 当前用户、租户、资源 ID、资源所属租户。
|
||||||
|
- 实际 HTTP 状态码、页面提示、数据库状态和容器日志。
|
||||||
|
- 是否产生了残留 ACL、GPU 分配、配额预留、MinIO 对象或审批记录。
|
||||||
|
- 修复提交、回归结果和是否需要再次现场验证。
|
||||||
|
|
||||||
|
## 11. 当前验收结论
|
||||||
|
|
||||||
|
当前已自动验证:前端构建、后端权限安全测试、存储安全测试、数据库字段迁移、容器健康、用户软删除登录拦截和离线源码 SQL 目录结构。
|
||||||
|
|
||||||
|
当前前端权限页面已完成主要操作闭环:
|
||||||
|
|
||||||
|
- 用户页面权限设置。
|
||||||
|
- 用户创建时的页面权限配置。
|
||||||
|
- 租户成员邀请、角色、有效期和配额展示。
|
||||||
|
- 普通用户访问申请、我的申请和租户邀请接受。
|
||||||
|
- 资源用户/角色 ACL 管理和全部撤销。
|
||||||
|
- 审批申请、审批策略可视化配置和中文状态展示。
|
||||||
|
|
||||||
|
仍需现场验证:双用户跨租户、多 GPU 并发、第二节点调度、节点失联回收、MinIO 故障注入和完整浏览器点击流。这些属于环境依赖型验收,不代表前端页面缺少入口。
|
||||||
161
docs/模型评测优化设计方案.md
Normal file
161
docs/模型评测优化设计方案.md
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
# 模型评测优化设计方案
|
||||||
|
|
||||||
|
## 1. 现状检查
|
||||||
|
|
||||||
|
当前模型评测菜单已经具备以下基础能力:
|
||||||
|
|
||||||
|
- 创建评测任务:选择训练模型、评测数据集、算力节点和 GPU。
|
||||||
|
- 任务调度:后端将模型、适配器、数据集准备到目标算力节点,再提交 Compute API 任务。
|
||||||
|
- 模型推理:Compute Agent 使用 LLaMA-Factory 推理会话逐条生成回答。
|
||||||
|
- 结果落库:任务完成后读取 `eval_results.json`,写入任务详情、样本结果和指标摘要。
|
||||||
|
- 报告归档:评测输出目录可以归档到 MinIO,并通过报告接口下载。
|
||||||
|
- 前端详情:显示综合分、通过率、样本结果、指标摘要,并对运行中的任务进行轮询。
|
||||||
|
|
||||||
|
当前缺陷主要集中在评分和进度链路:
|
||||||
|
|
||||||
|
1. 创建页面的 BLEU、ROUGE、余弦指标默认全部关闭;未配置 LLM 评委时,样本没有确定性评分,容易得到 0 分。
|
||||||
|
2. BLEU、ROUGE、余弦、LLM Judge 的返回范围和含义不统一,百分制、0-1、小量程评分混在一起。
|
||||||
|
3. 评测模型地址直接拼接 `/v1/chat/completions`,当地址已经包含 `/v1` 时会出现 `/v1/v1`。
|
||||||
|
4. LLM Judge 只解析少量文本格式,无法可靠解析 JSON、Markdown JSON 或 0-1 综合评价。
|
||||||
|
5. ROUGE 对中文没有采用 `nlp-eval-demo` 的字符级中文分词策略,中文短文本容易得到失真的结果。
|
||||||
|
6. 后端只在 Compute 任务完成后读取结果文件,运行时没有样本完成数、当前阶段和中间指标。
|
||||||
|
7. 前端没有雷达图,用户无法直观看到 BLEU、ROUGE、语义相似度、精确匹配和 LLM Judge 等维度。
|
||||||
|
|
||||||
|
## 2. 目标
|
||||||
|
|
||||||
|
建立一条可解释、可持续轮询、兼容旧任务的评测闭环:
|
||||||
|
|
||||||
|
```text
|
||||||
|
创建任务 -> 资源准备 -> 模型加载 -> 样本推理 -> 逐样本评分
|
||||||
|
-> 中间进度文件 -> 后端同步 -> 页面进度与雷达图
|
||||||
|
-> 完成报告 -> MinIO 归档 -> 详情与下载
|
||||||
|
```
|
||||||
|
|
||||||
|
目标结果:
|
||||||
|
|
||||||
|
- 所有最终展示分数统一为 0-100,避免不同指标之间直接相加造成误解。
|
||||||
|
- 没有 LLM 评委时,仍然使用确定性指标生成样本得分和综合分,不再因为“未配置评委”自动归零。
|
||||||
|
- 有 LLM 评委时,保留原有自定义评分区间,同时将其归一化到 0-100 展示。
|
||||||
|
- 每个评测任务都能看到阶段、总样本数、已完成样本数、百分比和当前指标状态。
|
||||||
|
- 详情页展示指标雷达图;指标不可用时显示原因,不把“依赖未安装”伪装成 0 分。
|
||||||
|
- 不新增必需数据库表,继续利用 `eval_tasks.payload` 保存评测结果和进度,兼容现有数据库及离线初始化 SQL。
|
||||||
|
|
||||||
|
## 3. 评分设计
|
||||||
|
|
||||||
|
### 3.1 指标契约
|
||||||
|
|
||||||
|
Compute Agent 内部统一使用以下结构:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"available": true,
|
||||||
|
"score": 82.5,
|
||||||
|
"max_score": 100,
|
||||||
|
"unit": "percent",
|
||||||
|
"sample_count": 3,
|
||||||
|
"error": ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`score` 永远是 0-100。指标不可用时 `available=false`,`score` 可以为 null,同时保留 `error`。旧报告中只有 `score` 的结构继续兼容,后端读取时按旧结构补齐默认字段。
|
||||||
|
|
||||||
|
### 3.2 确定性指标
|
||||||
|
|
||||||
|
参考 `nlp-eval-demo` 的实现,支持:
|
||||||
|
|
||||||
|
- BLEU:sacrebleu,结果由 0-100 转换为百分制。
|
||||||
|
- ROUGE-1、ROUGE-2、ROUGE-L:中文按字符切分,英文按词切分,使用 F1 均值并转换为百分制。
|
||||||
|
- 余弦相似度:TF-IDF 余弦相似度,转换为百分制。
|
||||||
|
- 精确匹配:标准化空白和大小写后完全一致,百分制。
|
||||||
|
- 文本相似度:SequenceMatcher,作为无外部模型时的稳定兜底指标。
|
||||||
|
- BERTScore:仅在 `bert_score` 和模型可用时启用;下载/加载失败只标记不可用,不阻断整个评测任务。
|
||||||
|
|
||||||
|
精确匹配和文本相似度始终计算。用户在创建页面选择的 BLEU、ROUGE、余弦作为额外指标;如果没有选择额外指标,也用“文本相似度 + ROUGE-L(可用时)”生成确定性综合分。
|
||||||
|
|
||||||
|
### 3.3 无 LLM 评委时的样本分数
|
||||||
|
|
||||||
|
对每条样本使用可用确定性指标的平均值作为样本百分制得分:
|
||||||
|
|
||||||
|
```text
|
||||||
|
sample_score = mean(exact_match, text_similarity, rougeL, cosine, bleu, bertscore)
|
||||||
|
```
|
||||||
|
|
||||||
|
其中未启用或不可用的指标不参与平均。样本通过阈值默认 60 分;如果旧维度配置的 `pass_threshold <= score_max`,先按旧量程换算为百分制。
|
||||||
|
|
||||||
|
### 3.4 LLM Judge
|
||||||
|
|
||||||
|
- 兼容 OpenAI Chat Completions 接口。
|
||||||
|
- 自动规范化 `api_url`,避免重复 `/v1`。
|
||||||
|
- 优先解析 JSON 的 `score`、`综合评价`、`overall_score`、`dimensions` 字段,再解析 Markdown/自然语言中的评分。
|
||||||
|
- 支持 0-1、0-5、0-100 三种返回量程;最终统一转换为 0-100。
|
||||||
|
- API 调用失败时记录样本失败原因,不把失败当作正常 0 分;如果所有样本都调用失败,任务状态仍可完成但报告会明确提示评委不可用。
|
||||||
|
|
||||||
|
## 4. 进度设计
|
||||||
|
|
||||||
|
Compute Agent 在评测输出目录写入两个文件:
|
||||||
|
|
||||||
|
- `eval_progress.json`:轻量进度文件,每完成一条样本更新一次。
|
||||||
|
- `eval_results.json`:运行中写入部分结果,完成后写入完整报告。
|
||||||
|
|
||||||
|
进度结构:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "running",
|
||||||
|
"stage": "inference",
|
||||||
|
"total": 20,
|
||||||
|
"completed": 7,
|
||||||
|
"percentage": 35,
|
||||||
|
"current_index": 8,
|
||||||
|
"message": "正在生成第 8 条样本",
|
||||||
|
"updated_at": "2026-08-20T00:00:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
后端详情接口每次轮询 Compute Agent:
|
||||||
|
|
||||||
|
1. 任务运行中读取 `eval_progress.json`,写入 `eval_tasks.payload.progress_detail`。
|
||||||
|
2. 读取到部分 `eval_results.json` 时同步已完成样本和基础指标,页面可以边运行边展示。
|
||||||
|
3. Compute 任务完成后读取完整报告,再执行 MinIO 归档。
|
||||||
|
4. Compute Agent 暂时不可达时保留最后一次进度,不因为一次轮询失败把评测任务误判为失败。
|
||||||
|
|
||||||
|
## 5. 前端设计
|
||||||
|
|
||||||
|
- 列表页增加进度列:运行中显示 `已完成/总数` 和百分比,完成后显示最终分数。
|
||||||
|
- 详情页增加当前阶段、进度消息和进度条。
|
||||||
|
- 详情页增加“指标雷达图”,雷达轴来自可用的 `dimension_summary` 指标,最大值统一 100。
|
||||||
|
- 指标卡同时显示分数、通过率、样本数和不可用原因。
|
||||||
|
- 样本结果在运行中支持逐步出现,保留现有搜索、筛选和分页。
|
||||||
|
- 不改变已有任务创建、删除、报告下载和权限校验流程。
|
||||||
|
|
||||||
|
## 6. 数据库与兼容性
|
||||||
|
|
||||||
|
本次不新增数据库表和必需字段。评测结果继续存放在 `eval_tasks.payload` JSON 中,新增键包括:
|
||||||
|
|
||||||
|
- `progress_detail`
|
||||||
|
- `basic_metrics` 的统一指标对象
|
||||||
|
- `metric_summary_version`
|
||||||
|
- 样本的 `raw_score`、`raw_max_score`、`score`、`max_score`
|
||||||
|
|
||||||
|
旧任务兼容规则:缺少进度时由 `progress` 和样本数量推导;旧的量程评分按 `score/max_score` 转换为百分制;没有基础指标时显示“历史任务未保存指标明细”。因此无需修改 `000_full_init.sql`。
|
||||||
|
|
||||||
|
## 7. 实施步骤
|
||||||
|
|
||||||
|
1. 重构 Compute Agent 评测指标、中文 ROUGE、LLM Judge 解析、评分归一化和中间结果写入。
|
||||||
|
2. 增加 Compute Agent 进度文件读取能力,后端同步运行中进度和部分结果。
|
||||||
|
3. 扩展后端评测任务结果兼容、失败信息和进度返回。
|
||||||
|
4. 扩展前端类型、列表进度列、详情进度区和指标雷达图。
|
||||||
|
5. 增加单元测试、前端构建测试和接口/运行时冒烟验证。
|
||||||
|
6. 同步前后端、算力服务到 `docker/offline/src`,确认初始化 SQL 无需变更。
|
||||||
|
|
||||||
|
## 8. 验收标准
|
||||||
|
|
||||||
|
- 使用仅包含 `instruction/output` 的小型 JSONL 数据集,未配置 LLM Judge 时综合分不再固定为 0。
|
||||||
|
- 中文答案能得到可解释的 ROUGE-1/2/L 分数。
|
||||||
|
- LLM Judge 的地址为 `https://host/v1` 时请求路径仍正确。
|
||||||
|
- 评测运行期间能看到 `completed/total` 和百分比变化。
|
||||||
|
- 完成后详情页存在至少 3 个可用指标时显示雷达图,指标少于 3 个时显示明细降级视图。
|
||||||
|
- 失败、依赖缺失、空答案等情况能在报告中区分,不以正常 0 分掩盖原因。
|
||||||
|
- 现有权限、GPU 预约、MinIO 归档、报告下载和旧任务查看不受影响。
|
||||||
|
|
||||||
628
docs/租户与用户权限体系梳理及纠偏流程.md
Normal file
628
docs/租户与用户权限体系梳理及纠偏流程.md
Normal file
@@ -0,0 +1,628 @@
|
|||||||
|
# 租户与用户权限体系梳理及纠偏流程
|
||||||
|
|
||||||
|
> 版本:v1.0
|
||||||
|
> 梳理日期:2026-08-20
|
||||||
|
> 适用范围:YG_FT 平台后端、前端、Compute Agent、MinIO、PostgreSQL 及离线部署目录
|
||||||
|
> 目的:明确租户、用户、角色、权限、资源和审批之间的层级关系,识别当前实现中的概念冲突,并为后续优化提供唯一设计口径。
|
||||||
|
|
||||||
|
## 一、结论摘要
|
||||||
|
|
||||||
|
当前系统已经具备用户登录、租户字段、租户成员、资源 ACL、审批、GPU 分配和资源级动作校验,但概念来源还没有完全收敛,主要表现为:
|
||||||
|
|
||||||
|
1. **用户和租户没有混为一个数据库对象,但业务流程中边界不清**:用户表保存 `tenant_id`,同时又通过 `tenant_members` 表表达租户关系,导致“主租户”和“成员关系”的判定规则并存。
|
||||||
|
2. **平台角色、租户角色、页面权限码、资源动作权限同时存在,但职责没有彻底分离**:`users.role`、`users.permissions`、`tenant_members.role`、`roles.permissions` 可能产生不一致。
|
||||||
|
3. **资源归属字段不统一**:部分资源使用独立的 `created_by`,部分任务把创建者放在 `payload.created_by`,部分资源的 `tenant_id` 允许为空,历史数据因此出现“ACL 已授权但列表不可见”的问题。
|
||||||
|
4. **租户创建、用户创建、成员加入和资源授权是四条不同流程,目前在页面上有交叉**:创建用户时直接写入租户成员,租户的 `owner_user_id` 却没有同步建立 owner 成员关系。
|
||||||
|
5. **项目表和项目字段仍在数据库及部分接口中保留**,但当前产品已经取消项目作为用户可见的业务隔离层,继续把 `project_id` 当权限条件会造成新的歧义。
|
||||||
|
6. **正确的目标模型应当是**:平台管理员管理平台身份和租户;用户通过租户成员关系进入一个或多个租户;资源属于一个租户并有一个所有者;资源使用通过 ACL 动作权限;高风险动作通过审批;GPU 和配额是资源使用前的运行时约束。
|
||||||
|
|
||||||
|
本文件的纠偏原则是:**先统一口径,再做兼容迁移;先保证后端判定一致,再调整前端菜单和数据库约束;历史字段暂不删除,但不再新增新的业务依赖。**
|
||||||
|
|
||||||
|
## 二、目标层级模型
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台 Platform
|
||||||
|
│
|
||||||
|
├── 平台身份与平台角色
|
||||||
|
│ ├── platform_admin:平台管理员,可管理租户、用户、策略和全部资源
|
||||||
|
│ └── platform_user:普通平台用户,不因登录自动获得资源权限
|
||||||
|
│
|
||||||
|
├── 租户 Tenant
|
||||||
|
│ ├── 租户基本信息、状态、配额、留存策略
|
||||||
|
│ ├── 租户成员 TenantMember
|
||||||
|
│ │ ├── owner:租户所有者
|
||||||
|
│ │ ├── admin:租户管理员
|
||||||
|
│ │ ├── member:普通成员
|
||||||
|
│ │ └── viewer:只读成员
|
||||||
|
│ └── 租户内资源
|
||||||
|
│ ├── 数据集
|
||||||
|
│ ├── 基座模型
|
||||||
|
│ ├── 训练任务与训练模型
|
||||||
|
│ ├── 评测任务
|
||||||
|
│ ├── 推理任务
|
||||||
|
│ └── 数据处理/转换任务
|
||||||
|
│
|
||||||
|
├── 资源授权 Resource ACL
|
||||||
|
│ ├── principal:user 或 tenant role
|
||||||
|
│ ├── read:查看
|
||||||
|
│ ├── write:编辑
|
||||||
|
│ ├── execute:训练、推理、评测等使用
|
||||||
|
│ ├── download:下载或导出文件
|
||||||
|
│ ├── delete:删除
|
||||||
|
│ └── admin:资源授权和完整管理
|
||||||
|
│
|
||||||
|
├── 运行时资源
|
||||||
|
│ ├── Compute Node
|
||||||
|
│ ├── GPU
|
||||||
|
│ ├── GPU assignment:用户获批可使用哪些 GPU
|
||||||
|
│ ├── GPU reservation:任务当前占用哪些 GPU
|
||||||
|
│ └── Tenant quota reservation:租户当前预留了多少运行资源
|
||||||
|
│
|
||||||
|
└── 审批与审计
|
||||||
|
├── resource access:申请资源 ACL
|
||||||
|
├── gpu.assign:申请算力卡
|
||||||
|
├── tenant.quota.update:申请租户配额变更
|
||||||
|
├── 高风险删除/导出/合并操作
|
||||||
|
└── 全部结果写入审计日志
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.1 四个必须分开的概念
|
||||||
|
|
||||||
|
| 概念 | 正确含义 | 不应承担的职责 |
|
||||||
|
|---|---|---|
|
||||||
|
| 用户 User | 可登录平台的人或服务账号 | 不代表租户,也不自动代表资源权限 |
|
||||||
|
| 租户 Tenant | 资源、成员、配额和审计的隔离边界 | 不直接作为登录账号 |
|
||||||
|
| 角色 Role | 某个范围内的一组职责 | 不应直接等同于某个用户的全部资源权限 |
|
||||||
|
| 权限 Permission | 对页面或资源的具体操作能力 | 不应只靠前端按钮控制 |
|
||||||
|
|
||||||
|
## 三、当前实现盘点
|
||||||
|
|
||||||
|
### 3.1 用户身份层
|
||||||
|
|
||||||
|
当前主要实现位置:
|
||||||
|
|
||||||
|
- `users` 表:保存账号、密码哈希、平台角色、状态、页面权限 JSON、主租户字段。
|
||||||
|
- `sessions` 表:保存登录会话、过期时间和注销时间。
|
||||||
|
- `backend/app/core/auth.py`:解析 Token、校验用户状态、校验页面权限和租户上下文。
|
||||||
|
- `/users` 接口:当前只有管理员可以创建、修改、禁用和删除用户。
|
||||||
|
|
||||||
|
当前用户对象包含:
|
||||||
|
|
||||||
|
```text
|
||||||
|
id / username / display_name / status
|
||||||
|
role / protected / permissions
|
||||||
|
tenant_id
|
||||||
|
```
|
||||||
|
|
||||||
|
存在的混淆:
|
||||||
|
|
||||||
|
- `role=operator/viewer/user` 是平台用户字段,但 `tenant_members.role` 还有 `owner/admin/member/viewer`,两套角色名称不在同一层级。
|
||||||
|
- `role=user` 在创建接口中被接受,但设计文档主要定义的是 `admin/operator/viewer/guest`,前端又同时显示“普通用户”。
|
||||||
|
- 创建用户时直接生成 `tenant_members` 活跃成员记录;这相当于“创建用户”和“加入租户”一次完成,没有邀请、待确认或成员审批边界。
|
||||||
|
- `users.permissions` 是用户级 JSON,`roles` 表只提供查询接口,当前没有形成“角色变更自动计算权限”的唯一来源。
|
||||||
|
- 非管理员创建/更新用户时会被剥离 `compute` 和 `user-settings`,但初始化 SQL 中的 `u_operator` 种子数据仍包含 `compute`,代码默认值和 SQL 种子不一致。
|
||||||
|
|
||||||
|
### 3.2 租户层
|
||||||
|
|
||||||
|
当前主要实现位置:
|
||||||
|
|
||||||
|
- `tenants` 表:保存租户名称、状态、owner_user_id、配额和留存策略。
|
||||||
|
- `tenant_members` 表:保存租户与用户的多对多关系、租户角色、状态、邀请人和有效期。
|
||||||
|
- `tenant_membership()`、`user_tenant_ids()`:判定当前用户的租户范围。
|
||||||
|
- `/tenants/{tenant_id}/members`:管理员或租户 owner/admin 管理成员。
|
||||||
|
|
||||||
|
当前逻辑:
|
||||||
|
|
||||||
|
```text
|
||||||
|
当前用户租户范围 = users.tenant_id + tenant_members 中 status=active 的 tenant_id
|
||||||
|
当前请求租户 = X-Tenant-ID(仅在用户属于该租户时生效)
|
||||||
|
资源租户 = 资源 tenant_id,部分任务还会从 payload 中读取 tenant_id
|
||||||
|
```
|
||||||
|
|
||||||
|
主要问题:
|
||||||
|
|
||||||
|
1. `users.tenant_id` 和 `tenant_members` 都能表达租户关系,但没有明确谁是主数据、谁是兼容字段。
|
||||||
|
2. `tenants.owner_user_id` 创建租户时写入,但创建租户流程没有自动写入对应的 `tenant_members(role='owner')`。
|
||||||
|
3. 平台管理员创建普通用户时直接加入租户,租户管理员也可以直接添加成员,邀请/加入申请流程虽已具备接口,但不是主流程。
|
||||||
|
4. `tenant_id` 在资源表中有的为 `NOT NULL`,有的允许 `NULL`;历史资源的 `NULL` 租户需要特殊兼容判断。
|
||||||
|
5. 租户删除是状态删除,但成员、资源、ACL、审批、MinIO 对象和本地缓存的后续处理没有统一的生命周期编排入口。
|
||||||
|
|
||||||
|
### 3.3 页面权限层
|
||||||
|
|
||||||
|
页面权限由两部分组成:
|
||||||
|
|
||||||
|
```text
|
||||||
|
前端:路由 meta.permission + 侧边栏过滤 + 按钮显示
|
||||||
|
后端:MODULE_PERMISSIONS + get_current_user / require_admin
|
||||||
|
```
|
||||||
|
|
||||||
|
页面权限码包括:
|
||||||
|
|
||||||
|
```text
|
||||||
|
dashboard, fine-tune, model-eval, model-inference,
|
||||||
|
model-manage, dataset, data-process, data-convert,
|
||||||
|
compute, hardware, logs, user-settings
|
||||||
|
```
|
||||||
|
|
||||||
|
问题:
|
||||||
|
|
||||||
|
- 页面权限码是“能否进入模块”,不是“能否使用某一条数据或 GPU”,但历史实现中一度把 `compute` 页面权限阻断了普通用户读取自己获批 GPU 的接口,造成“有 GPU 分配但训练页面看不到”的问题。
|
||||||
|
- 普通用户的 `compute` 管理菜单应保持隐藏,但训练、推理、评测所需的个人节点/GPU查询应当是独立的自助只读接口。
|
||||||
|
- 前端权限控制和后端权限控制分别维护,按钮隐藏不能作为安全边界。
|
||||||
|
|
||||||
|
### 3.4 资源权限层
|
||||||
|
|
||||||
|
当前资源权限主要由以下条件组合:
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台管理员旁路
|
||||||
|
或
|
||||||
|
资源属于当前租户
|
||||||
|
且(资源所有者 == 当前用户
|
||||||
|
或 ACL 明确授权
|
||||||
|
或当前用户是租户 owner/admin)
|
||||||
|
且满足具体动作 read/write/execute/download/delete/admin
|
||||||
|
```
|
||||||
|
|
||||||
|
已接入资源类型包括:
|
||||||
|
|
||||||
|
```text
|
||||||
|
dataset / model / trained_model / fine-tune / eval / inference
|
||||||
|
data_process / data_convert / compare / project(历史兼容)
|
||||||
|
```
|
||||||
|
|
||||||
|
当前差异:
|
||||||
|
|
||||||
|
| 资源 | 当前所有者来源 | 当前租户来源 | 主要风险 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| datasets | `created_by` | `tenant_id`,部分历史为空 | 列表、详情、训练权限可能不一致 |
|
||||||
|
| models | `created_by` | `tenant_id`,基座模型存在平台共享规则 | 基座模型共享与训练产物共享容易混淆 |
|
||||||
|
| trained_models | `created_by` | `tenant_id` | 血缘来源、默认授权需统一 |
|
||||||
|
| fine_tune_tasks | 多数在 `payload.created_by` | 多数在 payload 或派生字段 | 结构化查询和授权容易漏字段 |
|
||||||
|
| eval_tasks | `created_by` | `tenant_id` | 与数据集/模型执行权限必须同时校验 |
|
||||||
|
| compare_tasks | 表字段和 payload 并存 | 表字段和 payload 并存 | 推理任务与模型使用权限容易分裂 |
|
||||||
|
| data_process_tasks | `created_by` | `tenant_id` | 处理结果派生数据集授权不明确 |
|
||||||
|
| data_convert_tasks | `created_by` | `tenant_id` | 历史表结构和初始化脚本存在演进痕迹 |
|
||||||
|
| projects | `create_by` 等历史字段 | `tenant_id` | 当前已取消项目业务层,但表和接口仍在 |
|
||||||
|
|
||||||
|
### 3.5 GPU 与运行时权限层
|
||||||
|
|
||||||
|
GPU 相关关系实际上有三层:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gpus
|
||||||
|
└── gpu_assignments:管理员批准“用户可以使用哪张卡”
|
||||||
|
└── gpu_allocations / gpu_reservations:任务运行时“当前占用了哪张卡”
|
||||||
|
```
|
||||||
|
|
||||||
|
正确判定顺序应为:
|
||||||
|
|
||||||
|
1. 用户拥有租户成员资格且账号 active。
|
||||||
|
2. 用户已经获得该 GPU 的分配或管理员旁路。
|
||||||
|
3. 节点 online、enabled,GPU 存在且没有被其他任务占用。
|
||||||
|
4. 租户配额可以完成原子预留。
|
||||||
|
5. 训练、推理或评测任务提交到指定节点。
|
||||||
|
|
||||||
|
目前已经有分配申请、审批后写入 `gpu_assignments`、运行时预留和释放逻辑,但仍需把“审批结果、GPU 分配、配额预留、节点故障回收”统一成一个状态机。
|
||||||
|
|
||||||
|
## 四、当前数据库关系与问题
|
||||||
|
|
||||||
|
### 4.1 当前表的职责分组
|
||||||
|
|
||||||
|
```text
|
||||||
|
身份
|
||||||
|
├── users
|
||||||
|
├── sessions
|
||||||
|
└── roles
|
||||||
|
|
||||||
|
租户
|
||||||
|
├── tenants
|
||||||
|
├── tenant_members
|
||||||
|
└── tenant_quota_reservations
|
||||||
|
|
||||||
|
资源权限
|
||||||
|
├── acls
|
||||||
|
├── resource_access_requests
|
||||||
|
├── approval_templates
|
||||||
|
├── approval_instances
|
||||||
|
└── approval_steps
|
||||||
|
|
||||||
|
业务资源
|
||||||
|
├── datasets / dataset_files / dataset_file_versions / dataset_records
|
||||||
|
├── models / trained_models / model_lineage / model_artifacts
|
||||||
|
├── fine_tune_tasks / eval_tasks / compare_tasks
|
||||||
|
├── data_process_tasks / data_convert_tasks
|
||||||
|
└── projects / project_members(历史兼容)
|
||||||
|
|
||||||
|
运行资源
|
||||||
|
├── compute_nodes
|
||||||
|
├── gpus
|
||||||
|
├── gpu_assignments
|
||||||
|
├── gpu_allocations
|
||||||
|
├── gpu_reservations
|
||||||
|
└── resource_replicas / storage_objects / storage_cache_jobs
|
||||||
|
|
||||||
|
审计
|
||||||
|
└── audit_logs
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 数据库设计中的主要问题
|
||||||
|
|
||||||
|
#### 问题 A:租户归属字段可空且没有统一外键
|
||||||
|
|
||||||
|
`models`、`trained_models`、`datasets`、`eval_tasks`、`compare_tasks`、数据处理相关表的 `tenant_id` 演进过程不同,部分字段仍允许空值。多数资源表也没有对 `tenants(id)` 的外键约束。
|
||||||
|
|
||||||
|
后果:
|
||||||
|
|
||||||
|
- 资源可能被创建但没有租户归属。
|
||||||
|
- 列表接口和详情接口的可见性不同。
|
||||||
|
- ACL 授权是否跨租户无法仅靠数据库判断。
|
||||||
|
- 历史数据需要特殊分支,增加权限代码复杂度。
|
||||||
|
|
||||||
|
#### 问题 B:所有者字段不统一
|
||||||
|
|
||||||
|
当前同时存在 `created_by`、`owner_id`、`create_by` 和 `payload.created_by`。这会导致:
|
||||||
|
|
||||||
|
- 列表判断资源所有者时需要按资源类型写特殊逻辑。
|
||||||
|
- 用户删除时无法保证所有资源都能被一致地软删除。
|
||||||
|
- 审计目标和资源所有者可能不是同一个字段。
|
||||||
|
|
||||||
|
#### 问题 C:角色权限没有单一来源
|
||||||
|
|
||||||
|
当前存在以下可能来源:
|
||||||
|
|
||||||
|
```text
|
||||||
|
users.role
|
||||||
|
users.permissions(JSON)
|
||||||
|
tenant_members.role
|
||||||
|
roles.permissions(JSON)
|
||||||
|
前端 PermissionCode
|
||||||
|
后端 ALL_PERMISSIONS / MODULE_PERMISSIONS
|
||||||
|
```
|
||||||
|
|
||||||
|
如果更新了角色但没有同步用户权限 JSON,页面和接口可能出现不同结果;如果只修改前端权限码,后端仍可能拒绝;如果只修改 `tenant_members.role`,并不会自动改变平台页面权限。
|
||||||
|
|
||||||
|
#### 问题 D:资源 ACL 不是所有资源的唯一授权来源
|
||||||
|
|
||||||
|
基座模型在当前设计中对租户用户默认共享 read/execute;资源所有者又有隐式权限;租户管理员又有管理旁路;ACL 只是其中一层。这个设计可以成立,但必须在文档和代码中严格区分:
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台共享规则 ≠ ACL 授权
|
||||||
|
资源所有权 ≠ 租户管理员权限
|
||||||
|
页面权限 ≠ 资源动作权限
|
||||||
|
GPU 分配 ≠ 资源 ACL
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 问题 E:项目字段与当前产品边界不一致
|
||||||
|
|
||||||
|
`projects`、`project_members`、`project_id` 仍在数据库中存在,但当前产品已经决定不把项目作为用户可见的隔离层。它们应被标记为 legacy,不应再参与新资源的权限判定,也不应在新建页面继续要求填写。
|
||||||
|
|
||||||
|
## 五、目标唯一口径
|
||||||
|
|
||||||
|
### 5.1 用户与租户的最终关系
|
||||||
|
|
||||||
|
```text
|
||||||
|
一个用户可以属于多个租户
|
||||||
|
一个租户可以拥有多个用户
|
||||||
|
用户在一次请求中只能选择一个 active_tenant
|
||||||
|
资源必须归属于 active_tenant
|
||||||
|
```
|
||||||
|
|
||||||
|
推荐保留:
|
||||||
|
|
||||||
|
- `users.tenant_id`:暂时保留为兼容字段,表示用户的 primary tenant,不再作为多租户关系的唯一来源。
|
||||||
|
- `tenant_members`:升级为租户成员关系的唯一权威来源。
|
||||||
|
- `X-Tenant-ID`:明确表达当前请求租户,必须来自 active membership。
|
||||||
|
|
||||||
|
最终判定:
|
||||||
|
|
||||||
|
```text
|
||||||
|
有效租户 = tenant_members(status=active, 未过期)
|
||||||
|
primary tenant = users.tenant_id(仅兼容和默认登录上下文)
|
||||||
|
当前租户 = 请求 X-Tenant-ID,否则 primary tenant
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.2 平台角色与租户角色
|
||||||
|
|
||||||
|
推荐收敛为两层:
|
||||||
|
|
||||||
|
| 层级 | 字段/表 | 允许的角色 | 作用 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 平台层 | `users.platform_role`,兼容当前 `users.role` | `platform_admin`、`platform_user` | 管理租户、平台配置、平台用户和全局策略 |
|
||||||
|
| 租户层 | `tenant_members.role` | `owner`、`admin`、`member`、`viewer` | 管理本租户成员、资源和租户级审批 |
|
||||||
|
|
||||||
|
不建议继续把 `operator`、`user`、`member` 混在一个角色字段中。过渡期可以保留旧值,但新增代码只允许通过映射函数转换:
|
||||||
|
|
||||||
|
```text
|
||||||
|
admin/protected -> platform_admin
|
||||||
|
operator/user -> platform_user + tenant_members.member
|
||||||
|
viewer -> platform_user + tenant_members.viewer
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.3 页面权限与资源动作
|
||||||
|
|
||||||
|
页面权限只解决“能否进入模块”:
|
||||||
|
|
||||||
|
```text
|
||||||
|
module permission: dataset / fine-tune / model-inference / ...
|
||||||
|
```
|
||||||
|
|
||||||
|
资源动作解决“能否操作某个资源”:
|
||||||
|
|
||||||
|
```text
|
||||||
|
read / write / execute / download / delete / admin
|
||||||
|
```
|
||||||
|
|
||||||
|
统一授权函数输入应固定为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
authorize(
|
||||||
|
actor_id,
|
||||||
|
active_tenant_id,
|
||||||
|
resource_type,
|
||||||
|
resource_id,
|
||||||
|
action,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
推荐判定顺序:
|
||||||
|
|
||||||
|
1. 会话有效且用户 active。
|
||||||
|
2. 当前租户 active,用户是该租户成员。
|
||||||
|
3. 平台管理员旁路,或资源属于当前租户。
|
||||||
|
4. 资源所有者拥有默认动作集合。
|
||||||
|
5. 租户 owner/admin 按租户范围拥有管理动作。
|
||||||
|
6. ACL 明确授予当前用户或租户角色对应动作。
|
||||||
|
7. 对 download、delete、admin 等高风险动作执行额外审批检查。
|
||||||
|
|
||||||
|
### 5.4 资源标准字段
|
||||||
|
|
||||||
|
所有新资源应统一具备:
|
||||||
|
|
||||||
|
```text
|
||||||
|
id
|
||||||
|
tenant_id NOT NULL
|
||||||
|
created_by NOT NULL
|
||||||
|
created_at
|
||||||
|
updated_at
|
||||||
|
deleted_at
|
||||||
|
deleted_by
|
||||||
|
```
|
||||||
|
|
||||||
|
历史 `owner_id`、`create_by` 和 `payload.created_by` 只在迁移阶段读取,最终写入标准字段。任务配置 JSON 可以保留业务参数,但不再存放权限事实。
|
||||||
|
|
||||||
|
## 六、标准业务流程
|
||||||
|
|
||||||
|
### 6.1 创建租户流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台管理员
|
||||||
|
│
|
||||||
|
├─ 创建租户基本信息
|
||||||
|
├─ 设置租户配额和状态=active
|
||||||
|
├─ 指定或创建租户 owner
|
||||||
|
├─ 写入 tenant_members(tenant_id, owner_user_id, owner)
|
||||||
|
├─ 初始化审批策略
|
||||||
|
└─ 记录 tenant.create / tenant.member.add 审计
|
||||||
|
```
|
||||||
|
|
||||||
|
纠偏要求:`tenants.owner_user_id` 和 `tenant_members(role='owner')` 必须在同一事务中维护;不能只写一个。
|
||||||
|
|
||||||
|
### 6.2 创建用户与加入租户流程
|
||||||
|
|
||||||
|
#### 平台管理员创建用户
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台管理员
|
||||||
|
│
|
||||||
|
├─ 创建 users 记录
|
||||||
|
├─ 选择初始平台角色:platform_user 或 platform_admin
|
||||||
|
├─ 选择 primary tenant
|
||||||
|
├─ 写入 tenant_members
|
||||||
|
│ ├─ owner 仅平台管理员明确指定时允许
|
||||||
|
│ └─ 普通用户默认 member 或 viewer
|
||||||
|
├─ 生成页面权限快照(过渡期)
|
||||||
|
└─ 记录 user.create / tenant.member.add 审计
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 已存在用户加入新租户
|
||||||
|
|
||||||
|
```text
|
||||||
|
租户 owner/admin 或平台管理员
|
||||||
|
│
|
||||||
|
├─ 发出邀请或创建加入申请
|
||||||
|
├─ tenant_members.status = pending
|
||||||
|
├─ 被邀请用户接受
|
||||||
|
├─ tenant_members.status = active
|
||||||
|
└─ 用户选择 X-Tenant-ID 后访问该租户资源
|
||||||
|
```
|
||||||
|
|
||||||
|
纠偏要求:不能把“创建登录用户”和“拥有某租户全部资源”理解为同一件事;新用户默认没有任何资源 ACL,也没有 GPU 分配。
|
||||||
|
|
||||||
|
### 6.3 创建业务资源流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
登录用户
|
||||||
|
│
|
||||||
|
├─ 会话校验
|
||||||
|
├─ 确定 active_tenant
|
||||||
|
├─ 校验租户 active 和用户成员状态
|
||||||
|
├─ 写入资源 tenant_id + created_by
|
||||||
|
├─ 写入资源业务表
|
||||||
|
├─ 写入 MinIO 对象元数据(如有文件)
|
||||||
|
├─ 生成资源血缘/版本快照
|
||||||
|
└─ 记录资源创建审计
|
||||||
|
```
|
||||||
|
|
||||||
|
数据处理结果生成数据集时,必须继承任务的租户和创建者;不能因为是系统生成就把 `tenant_id` 和 `created_by` 留空。
|
||||||
|
|
||||||
|
### 6.4 资源查看和授权流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
用户请求资源列表/详情
|
||||||
|
│
|
||||||
|
├─ 页面权限检查
|
||||||
|
├─ active_tenant 检查
|
||||||
|
├─ 资源 tenant_id 检查
|
||||||
|
├─ 所有者 / 租户角色 / ACL 判定
|
||||||
|
├─ 返回资源元数据
|
||||||
|
└─ 文件、版本、MinIO 对象再次按同一资源动作校验
|
||||||
|
```
|
||||||
|
|
||||||
|
资源授权:
|
||||||
|
|
||||||
|
```text
|
||||||
|
资源所有者或租户管理员/平台管理员
|
||||||
|
│
|
||||||
|
├─ 选择用户或租户角色
|
||||||
|
├─ 选择 read/write/execute/download 等动作
|
||||||
|
├─ 设置有效期
|
||||||
|
├─ 写入 ACL 或创建访问审批
|
||||||
|
└─ 记录授权来源、授权人和审计
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6.5 训练、推理、评测使用资源流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
创建任务
|
||||||
|
│
|
||||||
|
├─ 页面权限:fine-tune / model-inference / model-eval
|
||||||
|
├─ active_tenant 和任务资源归属校验
|
||||||
|
├─ 基座模型:平台共享规则或 model.execute
|
||||||
|
├─ 训练模型:trained_model.execute
|
||||||
|
├─ 数据集:dataset.execute
|
||||||
|
├─ GPU assignment:用户是否获批使用所选卡
|
||||||
|
├─ 节点:GPU 所属节点是否 online/enabled
|
||||||
|
├─ 原子预留 GPU 和租户配额
|
||||||
|
├─ MinIO/节点缓存准备
|
||||||
|
├─ 提交计算任务
|
||||||
|
└─ 成功、失败、取消、超时、节点故障统一释放预留
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6.6 高风险操作审批流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
用户发起高风险操作
|
||||||
|
│
|
||||||
|
├─ 检查当前用户是否有基础资源权限
|
||||||
|
├─ 检查是否为管理员旁路
|
||||||
|
├─ 匹配 tenant + action + resource_type 审批策略
|
||||||
|
├─ 创建 pending 审批实例
|
||||||
|
├─ 审批人从当前会话确定
|
||||||
|
├─ approved -> 幂等执行具体效果
|
||||||
|
│ ├─ ACL 授权
|
||||||
|
│ ├─ GPU 分配
|
||||||
|
│ └─ 配额更新
|
||||||
|
├─ rejected/cancelled/expired -> 不产生资源效果
|
||||||
|
└─ 全流程记录审计
|
||||||
|
```
|
||||||
|
|
||||||
|
## 七、纠偏开发计划
|
||||||
|
|
||||||
|
### 阶段 0:冻结口径
|
||||||
|
|
||||||
|
1. 书面确认:不再把项目作为新业务隔离层。
|
||||||
|
2. 确认平台角色只有平台管理员和平台用户两类;租户职责由 `tenant_members.role` 表达。
|
||||||
|
3. 确认 `tenant_members` 是租户关系权威表,`users.tenant_id` 仅作为 primary tenant 兼容字段。
|
||||||
|
4. 确认所有资源必须有 `tenant_id`、`created_by`、生命周期字段。
|
||||||
|
5. 确认基座模型共享不等于训练模型和数据集共享。
|
||||||
|
|
||||||
|
### 阶段 1:身份和租户纠偏
|
||||||
|
|
||||||
|
1. 为平台角色建立统一映射,停止新增 `role='user'` 的特殊逻辑。
|
||||||
|
2. 用户创建、租户创建、成员邀请、成员接受、成员移除统一走成员服务。
|
||||||
|
3. 创建租户时同步 owner 成员记录。
|
||||||
|
4. 租户成员变更、用户禁用、用户软删除时统一撤销会话、GPU 分配、ACL、待审批流程。
|
||||||
|
5. 新增当前租户查询和切换接口,前端显示“当前租户”,避免仅显示用户主租户。
|
||||||
|
|
||||||
|
### 阶段 2:资源数据纠偏
|
||||||
|
|
||||||
|
1. 对所有资源表补齐并回填 `tenant_id`、`created_by`、`created_at`、`updated_at`。
|
||||||
|
2. 将任务 payload 中的创建者和租户迁移到结构化字段。
|
||||||
|
3. 对历史 `tenant_id IS NULL` 数据按以下顺序处理:
|
||||||
|
- 能从创建者找到租户的,回填创建者所属租户。
|
||||||
|
- 系统生成数据集能从源任务找到租户的,继承源任务租户。
|
||||||
|
- 仍无法确定归属的,进入待治理清单,不自动公开。
|
||||||
|
- 管理员明确授权的历史资源可临时通过用户级 ACL 访问,直到完成归属迁移。
|
||||||
|
4. 为资源和租户建立必要索引;在数据清洗完成后再逐步收紧 `NOT NULL` 和外键。
|
||||||
|
|
||||||
|
### 阶段 3:权限判定收敛
|
||||||
|
|
||||||
|
1. 建立统一 `authorize()` 服务,所有列表、详情、文件、版本、下载、MinIO、缓存和任务接口调用同一套判定。
|
||||||
|
2. 页面权限只用于模块入口;资源动作必须在后端独立判断。
|
||||||
|
3. 统一基座模型、训练模型、数据集和任务的血缘权限来源。
|
||||||
|
4. 统一 GPU assignment、GPU reservation 和租户配额 reservation 的状态流转。
|
||||||
|
5. 审批实例增加幂等键和过期处理任务,避免重复审批和重复执行。
|
||||||
|
|
||||||
|
### 阶段 4:数据库和初始化脚本
|
||||||
|
|
||||||
|
1. 将完整初始化 SQL 与迁移 SQL 的职责分开:新库只执行一份完整脚本,旧库执行版本化迁移。
|
||||||
|
2. 初始化 SQL 中统一种子用户、种子角色、默认租户和租户成员关系,不能出现 operator 仍拥有普通用户不应有的 `compute` 权限而代码又剥离该权限的矛盾。
|
||||||
|
3. 给所有需要的关联增加外键或由应用层统一保证引用完整性。
|
||||||
|
4. 为 `tenant_id + status`、`created_by + deleted_at`、ACL 主体、审批状态和 GPU 状态建立索引。
|
||||||
|
5. 主工程与 `docker/offline/src` 的完整 SQL、迁移、代码和部署文档做 SHA-256 一致性校验。
|
||||||
|
|
||||||
|
### 阶段 5:前端流程纠偏
|
||||||
|
|
||||||
|
1. 用户管理页面分别显示平台角色、所属租户、租户角色、页面权限和资源权限,不再把它们放在一个“权限”字段中。
|
||||||
|
2. 租户详情页显示成员、租户角色、配额使用、审批策略和资源统计。
|
||||||
|
3. 资源详情页显示资源所属租户、创建者、授权主体、动作、有效期和授权来源。
|
||||||
|
4. 训练/推理/评测页面只显示当前用户有 `execute` 权限且已获批 GPU 的资源。
|
||||||
|
5. 对“无权访问”“未授权”“审批中”“资源已删除”“租户已停用”分别展示原因,不统一显示为数据为空或 500。
|
||||||
|
|
||||||
|
## 八、建议的验收矩阵
|
||||||
|
|
||||||
|
| 场景 | 预期结果 |
|
||||||
|
|---|---|
|
||||||
|
| 用户未加入租户访问租户资源 | 403 |
|
||||||
|
| 用户属于租户但没有资源 ACL | 资源列表不可见,详情 403 |
|
||||||
|
| 用户获得 dataset.read | 数据集可见,可预览,但不能训练或上传 |
|
||||||
|
| 用户获得 dataset.execute | 数据集可用于训练/评测,但不能编辑或下载 |
|
||||||
|
| 用户获得 dataset.download | 可以下载文件,但不能训练 |
|
||||||
|
| 用户获得 trained_model.execute | 可以推理/评测,不能导出 |
|
||||||
|
| 用户获得 trained_model.download | 可以导出或下载,不能执行推理 |
|
||||||
|
| 用户获批 GPU 但没有数据集 execute | 不能创建训练任务 |
|
||||||
|
| 用户有数据集 execute 但没有 GPU assignment | 不能创建训练任务 |
|
||||||
|
| 管理员撤销 ACL | 列表、详情、执行、下载均立即重新校验 |
|
||||||
|
| 用户被禁用 | 会话失效,成员和 ACL 不能继续生效 |
|
||||||
|
| 租户被停用 | 租户资源不可创建和执行,运行任务进入待处理/失败策略 |
|
||||||
|
| 历史资源 tenant_id 为空且无 ACL | 普通用户不可见 |
|
||||||
|
| 历史资源 tenant_id 为空且有管理员直接用户 ACL | 仅指定用户在有效期内可见和按动作使用 |
|
||||||
|
| 资源删除 | 资源不可见,ACL、审批、对象清理流程被触发 |
|
||||||
|
| MinIO 不可用 | 资源元数据权限仍可判断,涉及对象的操作返回明确存储故障 |
|
||||||
|
|
||||||
|
## 九、纠偏完成判定
|
||||||
|
|
||||||
|
满足以下条件,才认为租户与用户权限完成统一,而不是“接口能够调用”:
|
||||||
|
|
||||||
|
- `tenant_members` 成为租户关系唯一权威来源,`users.tenant_id` 只作为兼容主租户。
|
||||||
|
- 平台角色和租户角色职责分离,角色权限不再通过多个 JSON/字段重复维护。
|
||||||
|
- 新资源全部具备非空租户和创建者,历史资源有明确归属或进入治理清单。
|
||||||
|
- 所有资源接口、文件接口、MinIO 接口和任务接口使用统一动作授权。
|
||||||
|
- 数据集、基座模型、训练模型和任务之间的派生关系不会自动扩大访问范围。
|
||||||
|
- 用户创建、成员邀请、资源授权、GPU 申请、配额申请和审批都能追溯到用户、租户、资源、动作和请求 ID。
|
||||||
|
- 初始化 SQL、迁移 SQL、在线数据库和离线源码目录的表结构与权限逻辑一致。
|
||||||
|
- 双用户、跨租户、ACL 撤销、权限过期、用户禁用、租户停用、GPU 冲突和 MinIO 故障场景全部通过验证。
|
||||||
|
|
||||||
|
## 十、2026-08-20 本轮改造结果
|
||||||
|
|
||||||
|
本轮已按目标层级模型完成一轮可运行的后端、数据库和前端收敛:
|
||||||
|
|
||||||
|
1. `users.platform_role` 已作为平台角色标准字段,值为 `platform_admin` 或 `platform_user`;历史 `users.role` 继续返回,供旧客户端兼容。
|
||||||
|
2. `tenant_members` 已作为租户成员关系的权威来源。创建租户时会在同一事务中写入 `owner` 成员;创建用户时单独使用 `tenant_role` 写入成员角色,不再因为平台角色是管理员就自动成为租户 owner。
|
||||||
|
3. 用户接口返回 `platform_role`、`tenant_memberships` 和 `tenant_count`,组织页面已区分平台身份与租户成员关系。
|
||||||
|
4. `fine_tune_tasks` 已增加结构化 `tenant_id`、`created_by`、`deleted_at`、`deleted_by`;训练列表、详情和 owner 判定优先使用结构化字段,历史 payload 仅作回退。
|
||||||
|
5. 新建数据集、训练、评测、推理任务会绑定当前认证租户;普通用户不能仅通过请求体伪造其他租户。平台管理员仍可为指定租户创建资源。
|
||||||
|
6. 普通用户的资源列表继续通过租户成员、所有者和 ACL 统一过滤;项目不再出现在前端路由、组织页面和新权限判断中。
|
||||||
|
7. 租户删除已补充 `deleted_at/deleted_by` 软删除,并同步禁用该租户的成员关系。
|
||||||
|
8. `000_full_init.sql` 已包含本轮标准字段、索引、owner 成员修复和无项目业务依赖的种子数据;`docker/offline/src/backend/app/db/sql` 仅保留这一份完整初始化脚本。
|
||||||
|
9. `docker/offline/src` 已同步当前 backend、frontend、compute 源码和构建后的 `frontend-dist`。
|
||||||
|
|
||||||
|
本轮已在线验证:数据库字段存在、普通用户身份返回、用户租户成员关系返回、租户 owner 自动建立、租户软删除、跨租户伪造创建被 403 拒绝;前端 `npm run build` 通过。项目表、项目字段及旧项目 API 仅保留历史兼容读取,不再作为新业务隔离条件。
|
||||||
|
|
||||||
|
## 十一、下一步开发计划
|
||||||
|
|
||||||
|
1. 为资源详情和资源授权页面补充租户、创建者、授权来源、有效期和动作权限的可视化信息。
|
||||||
|
2. 将数据处理、数据转换等派生资源的租户和创建者继承改为统一服务,并清理历史 `tenant_id IS NULL` 待治理清单。
|
||||||
|
3. 将 GPU assignment、GPU reservation、租户配额 reservation 和审批实例整理为同一状态机,补充节点故障回收验收。
|
||||||
|
4. 执行双租户、双用户、ACL 过期/撤销、用户禁用、租户停用、MinIO 故障和多 GPU 冲突的完整回归测试。
|
||||||
@@ -21,17 +21,49 @@ export interface ApprovalInstance {
|
|||||||
status: string
|
status: string
|
||||||
current_step: number
|
current_step: number
|
||||||
create_time?: string
|
create_time?: string
|
||||||
|
action?: string | null
|
||||||
|
tenant_id?: string | null
|
||||||
|
execution_status?: string | null
|
||||||
steps: Array<ApprovalStep & { step_index: number; comment?: string | null; time?: string | null }>
|
steps: Array<ApprovalStep & { step_index: number; comment?: string | null; time?: string | null }>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ResourceAccessRequest {
|
||||||
|
id: string
|
||||||
|
tenant_id: string
|
||||||
|
resource_type: string
|
||||||
|
resource_id: string
|
||||||
|
applicant_id: string
|
||||||
|
principal_type: string
|
||||||
|
principal_id: string
|
||||||
|
requested_permissions: string[]
|
||||||
|
reason?: string | null
|
||||||
|
approval_id?: string | null
|
||||||
|
status: string
|
||||||
|
expires_at?: string | null
|
||||||
|
created_at?: string
|
||||||
|
cancelled_at?: string | null
|
||||||
|
cancelled_by?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GpuRequestOption {
|
||||||
|
id: string
|
||||||
|
code: string
|
||||||
|
name: string
|
||||||
|
gpus: Array<{
|
||||||
|
index: number
|
||||||
|
name: string
|
||||||
|
memory_total_gb: number
|
||||||
|
}>
|
||||||
|
}
|
||||||
|
|
||||||
export const getApprovalTemplates = () =>
|
export const getApprovalTemplates = () =>
|
||||||
get<ApprovalTemplate[]>('/approvals/templates')
|
get<ApprovalTemplate[]>('/approvals/templates')
|
||||||
|
|
||||||
export const createApprovalTemplate = (payload: { name: string; steps: ApprovalStep[] }) =>
|
export const createApprovalTemplate = (payload: { name: string; steps: ApprovalStep[] }) =>
|
||||||
post<ApprovalTemplate>('/approvals/templates', payload)
|
post<ApprovalTemplate>('/approvals/templates', payload)
|
||||||
|
|
||||||
export const getApprovalInstances = (status?: string) =>
|
export const getApprovalInstances = (status?: string, mine = false) =>
|
||||||
get<ApprovalInstance[]>('/approvals', { status })
|
get<ApprovalInstance[]>('/approvals', { status, mine: mine || undefined })
|
||||||
|
|
||||||
export const createApprovalInstance = (payload: {
|
export const createApprovalInstance = (payload: {
|
||||||
template_id?: string
|
template_id?: string
|
||||||
@@ -46,5 +78,31 @@ export const getApprovalInstance = (id: string) =>
|
|||||||
export const decideApproval = (
|
export const decideApproval = (
|
||||||
id: string,
|
id: string,
|
||||||
step_index: number,
|
step_index: number,
|
||||||
payload: { approver_id: string; approved: boolean; comment?: string },
|
payload: { approved: boolean; comment?: string },
|
||||||
) => post<ApprovalInstance>(`/approvals/${id}/steps/${step_index}/decision`, payload)
|
) => post<ApprovalInstance>(`/approvals/${id}/steps/${step_index}/decision`, payload)
|
||||||
|
|
||||||
|
export const createResourceAccessRequest = (payload: {
|
||||||
|
resource_type: string
|
||||||
|
resource_id: string
|
||||||
|
principal_type?: 'user' | 'role'
|
||||||
|
principal_id?: string
|
||||||
|
requested_permissions: string[]
|
||||||
|
reason?: string
|
||||||
|
expires_at?: string
|
||||||
|
}) => post<ResourceAccessRequest>('/approvals/resource-access/requests', payload)
|
||||||
|
|
||||||
|
export const getResourceAccessRequests = (status?: string) =>
|
||||||
|
get<ResourceAccessRequest[]>('/approvals/resource-access/requests', { status })
|
||||||
|
|
||||||
|
export const cancelResourceAccessRequest = (id: string) =>
|
||||||
|
post<ResourceAccessRequest>(`/approvals/resource-access/requests/${id}/cancel`, {})
|
||||||
|
|
||||||
|
export const getGpuRequestOptions = () =>
|
||||||
|
get<{ nodes: GpuRequestOption[] }>('/approvals/gpu-options')
|
||||||
|
|
||||||
|
export const requestGpuAccess = (payload: {
|
||||||
|
assignments: Array<{ node_id: string; gpu_index: number }>
|
||||||
|
reason?: string
|
||||||
|
}) => post<{ approval_required?: boolean; approval_id?: string; approval?: ApprovalInstance }>(
|
||||||
|
'/approvals/gpu-requests', payload,
|
||||||
|
)
|
||||||
|
|||||||
@@ -11,6 +11,11 @@ export interface AuditLog {
|
|||||||
target_id?: string
|
target_id?: string
|
||||||
detail?: string
|
detail?: string
|
||||||
client_ip?: string
|
client_ip?: string
|
||||||
|
result?: string
|
||||||
|
reason?: string
|
||||||
|
request_id?: string
|
||||||
|
session_id?: string
|
||||||
|
metadata?: string | Record<string, unknown>
|
||||||
time?: string
|
time?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { get, post, del } from '../request'
|
import { get, post, del, getAuthHeaders } from '../request'
|
||||||
import type { CompareTask, CompareModelRef } from '@/types'
|
import type { CompareTask, CompareModelRef } from '@/types'
|
||||||
|
|
||||||
const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000
|
const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000
|
||||||
@@ -78,7 +78,10 @@ export const streamChatReal = (data: any): Promise<Response> => {
|
|||||||
}
|
}
|
||||||
return fetch('/modelTF/model-compare/stream-chat', {
|
return fetch('/modelTF/model-compare/stream-chat', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
...getAuthHeaders(),
|
||||||
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
messages,
|
messages,
|
||||||
temperature: data.temperature ?? 0.7,
|
temperature: data.temperature ?? 0.7,
|
||||||
|
|||||||
@@ -11,6 +11,11 @@ export interface DataConvertTask {
|
|||||||
error_message: string
|
error_message: string
|
||||||
create_time: string
|
create_time: string
|
||||||
update_time?: string
|
update_time?: string
|
||||||
|
created_by?: string | null
|
||||||
|
creator_name?: string | null
|
||||||
|
processed_by?: string | null
|
||||||
|
processor_name?: string | null
|
||||||
|
processed_at?: string | null
|
||||||
input_files?: Array<{ name: string; size: number }>
|
input_files?: Array<{ name: string; size: number }>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -49,6 +49,15 @@ export interface FineTuneGpuStatus {
|
|||||||
error?: string
|
error?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FineTuneCheckpoint {
|
||||||
|
id: string
|
||||||
|
step: number
|
||||||
|
name: string
|
||||||
|
path: string
|
||||||
|
size_bytes?: number
|
||||||
|
create_time?: string
|
||||||
|
}
|
||||||
|
|
||||||
/** 训练任务列表 */
|
/** 训练任务列表 */
|
||||||
export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
|
export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
|
||||||
|
|
||||||
@@ -89,6 +98,13 @@ export const updateFineTune = (id: string | number, data: Partial<FineTuneTask>)
|
|||||||
/** 停止训练任务 */
|
/** 停止训练任务 */
|
||||||
export const stopFineTune = (id: string | number) => post(`/fine-tune/stop/${id}`)
|
export const stopFineTune = (id: string | number) => post(`/fine-tune/stop/${id}`)
|
||||||
|
|
||||||
|
/** 从最新 checkpoint 继续训练 */
|
||||||
|
export const resumeFineTune = (id: string | number) => post(`/fine-tune/${id}/resume`)
|
||||||
|
|
||||||
|
/** 获取训练断点 */
|
||||||
|
export const getFineTuneCheckpoints = (id: string | number) =>
|
||||||
|
get<FineTuneCheckpoint[]>(`/fine-tune/${id}/checkpoints`)
|
||||||
|
|
||||||
/** 删除训练任务 */
|
/** 删除训练任务 */
|
||||||
export const deleteFineTune = (id: string | number) => del(`/fine-tune/${id}`)
|
export const deleteFineTune = (id: string | number) => del(`/fine-tune/${id}`)
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,10 @@ export interface ModelExportJob {
|
|||||||
payload?: Record<string, unknown>
|
payload?: Record<string, unknown>
|
||||||
create_time?: string
|
create_time?: string
|
||||||
completed_at?: string
|
completed_at?: string
|
||||||
|
created_by?: string
|
||||||
|
tenant_id?: string
|
||||||
|
archive_status?: string
|
||||||
|
archive_error?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 模型列表 */
|
/** 模型列表 */
|
||||||
@@ -51,7 +55,7 @@ export const getModelByName = (name: string) => get<ModelItem>(`/model-manage/na
|
|||||||
|
|
||||||
/** 本地模型路径列表 */
|
/** 本地模型路径列表 */
|
||||||
export const getLocalModels = () =>
|
export const getLocalModels = () =>
|
||||||
get<{ models: { path: string; name: string; source?: string }[] }>('/model-manage/local-models')
|
get<{ models: { path: string; name: string; storage_status?: string }[] }>('/model-manage/local-models')
|
||||||
|
|
||||||
/** 已训练模型列表 */
|
/** 已训练模型列表 */
|
||||||
export const getTrainedModels = () =>
|
export const getTrainedModels = () =>
|
||||||
@@ -97,9 +101,16 @@ export const mergeModel = (data: {
|
|||||||
output_model_name?: string
|
output_model_name?: string
|
||||||
}) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 })
|
}) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 })
|
||||||
|
|
||||||
/** 导出已训练模型权重 */
|
/** 导出已训练模型权重,沿用节点缓存和 MinIO 归档流程 */
|
||||||
export const exportModelUrl = (modelName: string) =>
|
export const exportModel = (data: {
|
||||||
`/modelTF/model-manage/trained-models/${encodeURIComponent(modelName)}/export`
|
trained_model_id?: string | number
|
||||||
|
model_name?: string
|
||||||
|
base_model_path?: string
|
||||||
|
adapter_path?: string
|
||||||
|
compute_node_id?: string
|
||||||
|
output_model_name?: string
|
||||||
|
export_quantization_bit?: 0 | 4 | 8
|
||||||
|
}) => post('/model-manage/export', data, { timeout: 15 * 60 * 1000 })
|
||||||
|
|
||||||
/** 测试在线模型连通性 */
|
/** 测试在线模型连通性 */
|
||||||
export const testOnlineModel = (data: {
|
export const testOnlineModel = (data: {
|
||||||
|
|||||||
@@ -9,11 +9,37 @@ export interface Tenant {
|
|||||||
quota: Record<string, unknown>
|
quota: Record<string, unknown>
|
||||||
retention_policy_id?: string | null
|
retention_policy_id?: string | null
|
||||||
create_time?: string
|
create_time?: string
|
||||||
|
deleted_at?: string | null
|
||||||
|
deleted_by?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TenantMember {
|
||||||
|
tenant_id: string
|
||||||
|
user_id: string
|
||||||
|
username?: string
|
||||||
|
display_name?: string
|
||||||
|
role: 'owner' | 'admin' | 'member' | 'viewer'
|
||||||
|
status: 'active' | 'pending' | 'disabled' | 'expired'
|
||||||
|
invited_by?: string | null
|
||||||
|
joined_at?: string | null
|
||||||
|
expires_at?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TenantQuotaUsage {
|
||||||
|
tenant_id: string
|
||||||
|
quota: Record<string, unknown>
|
||||||
|
gpu_limit: number
|
||||||
|
gpu_reserved: number
|
||||||
|
storage_reserved: number
|
||||||
|
reservations: number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 租户列表 */
|
/** 租户列表 */
|
||||||
export const getTenants = () => get<Tenant[]>('/tenants')
|
export const getTenants = () => get<Tenant[]>('/tenants')
|
||||||
|
|
||||||
|
/** 当前用户可切换的 active 租户及成员角色 */
|
||||||
|
export const getMyTenants = () => get<Tenant[]>('/tenants/mine')
|
||||||
|
|
||||||
/** 租户详情 */
|
/** 租户详情 */
|
||||||
export const getTenant = (id: string) => get<Tenant>(`/tenants/${id}`)
|
export const getTenant = (id: string) => get<Tenant>(`/tenants/${id}`)
|
||||||
|
|
||||||
@@ -28,6 +54,10 @@ export const updateTenant = (id: string, payload: Partial<Tenant>) =>
|
|||||||
/** 删除租户 */
|
/** 删除租户 */
|
||||||
export const deleteTenant = (id: string) => del(`/tenants/${id}`)
|
export const deleteTenant = (id: string) => del(`/tenants/${id}`)
|
||||||
|
|
||||||
|
/** 恢复软删除租户 */
|
||||||
|
export const restoreTenant = (id: string) =>
|
||||||
|
post<Tenant>(`/tenants/${id}/restore`, {})
|
||||||
|
|
||||||
/** 设置租户配额 */
|
/** 设置租户配额 */
|
||||||
export const setTenantQuota = (id: string, quota: Record<string, unknown>) =>
|
export const setTenantQuota = (id: string, quota: Record<string, unknown>) =>
|
||||||
put<Tenant>(`/tenants/${id}/quota`, quota)
|
put<Tenant>(`/tenants/${id}/quota`, quota)
|
||||||
@@ -35,3 +65,26 @@ export const setTenantQuota = (id: string, quota: Record<string, unknown>) =>
|
|||||||
/** 设置租户留存策略 */
|
/** 设置租户留存策略 */
|
||||||
export const setTenantRetention = (id: string, retention_policy_id: string) =>
|
export const setTenantRetention = (id: string, retention_policy_id: string) =>
|
||||||
put<Tenant>(`/tenants/${id}/retention-policy`, { retention_policy_id })
|
put<Tenant>(`/tenants/${id}/retention-policy`, { retention_policy_id })
|
||||||
|
|
||||||
|
export const getTenantMembers = (id: string) =>
|
||||||
|
get<TenantMember[]>(`/tenants/${id}/members`)
|
||||||
|
|
||||||
|
export const inviteTenantMember = (
|
||||||
|
id: string,
|
||||||
|
payload: { user_id: string; role?: TenantMember['role']; expires_at?: string },
|
||||||
|
) => post<TenantMember>(`/tenants/${id}/members/invite`, payload)
|
||||||
|
|
||||||
|
export const updateTenantMember = (id: string, userId: string, payload: Partial<TenantMember>) =>
|
||||||
|
put<TenantMember>(`/tenants/${id}/members/${userId}`, payload)
|
||||||
|
|
||||||
|
export const removeTenantMember = (id: string, userId: string) =>
|
||||||
|
del(`/tenants/${id}/members/${userId}`)
|
||||||
|
|
||||||
|
export const acceptTenantInvitation = (id: string, userId: string) =>
|
||||||
|
post<TenantMember>(`/tenants/${id}/members/${userId}/accept`, {})
|
||||||
|
|
||||||
|
export const getTenantInvitations = () =>
|
||||||
|
get<TenantMember[]>('/tenants/invitations')
|
||||||
|
|
||||||
|
export const getTenantQuotaUsage = (id: string) =>
|
||||||
|
get<TenantQuotaUsage>(`/tenants/${id}/quota/usage`)
|
||||||
|
|||||||
@@ -70,13 +70,27 @@ function getAuthToken(): string | null {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getActiveTenantId(): string | null {
|
||||||
|
return localStorage.getItem('activeTenantId')
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Headers shared by Axios and raw fetch requests such as SSE streaming. */
|
||||||
|
export function getAuthHeaders(): Record<string, string> {
|
||||||
|
const headers: Record<string, string> = {}
|
||||||
|
const token = getAuthToken()
|
||||||
|
if (token) headers.Authorization = `Bearer ${token}`
|
||||||
|
const tenantId = getActiveTenantId()
|
||||||
|
if (tenantId) headers['X-Tenant-ID'] = tenantId
|
||||||
|
return headers
|
||||||
|
}
|
||||||
|
|
||||||
// 请求拦截器:注入 Authorization header
|
// 请求拦截器:注入 Authorization header
|
||||||
service.interceptors.request.use(
|
service.interceptors.request.use(
|
||||||
(config) => {
|
(config) => {
|
||||||
const token = getAuthToken()
|
const authHeaders = getAuthHeaders()
|
||||||
if (token) {
|
if (Object.keys(authHeaders).length) {
|
||||||
config.headers = config.headers || {}
|
config.headers = config.headers || {}
|
||||||
config.headers['Authorization'] = `Bearer ${token}`
|
Object.assign(config.headers, authHeaders)
|
||||||
}
|
}
|
||||||
return config
|
return config
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -3,11 +3,17 @@ import { computed, ref, onMounted, onUnmounted } from 'vue'
|
|||||||
import { storeToRefs } from 'pinia'
|
import { storeToRefs } from 'pinia'
|
||||||
import { useSystemStore } from '@/stores/system'
|
import { useSystemStore } from '@/stores/system'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { getMyTenants, type Tenant } from '@/api/modules/tenant'
|
||||||
|
import { useAuthStore } from '@/stores/auth'
|
||||||
|
|
||||||
const systemStore = useSystemStore()
|
const systemStore = useSystemStore()
|
||||||
const { metrics } = storeToRefs(systemStore)
|
const { metrics } = storeToRefs(systemStore)
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
const auth = useAuthStore()
|
||||||
|
const tenants = ref<Tenant[]>([])
|
||||||
|
const activeTenantId = ref(localStorage.getItem('activeTenantId') || auth.currentUser?.tenant_id || (auth.isAdmin ? 'admin' : 'default'))
|
||||||
|
|
||||||
const showBackButton = computed(() => {
|
const showBackButton = computed(() => {
|
||||||
return route.path.split('/').filter(Boolean).length > 1
|
return route.path.split('/').filter(Boolean).length > 1
|
||||||
@@ -41,11 +47,27 @@ function openGuide() {
|
|||||||
window.open(guideUrl, '_blank', 'noopener,noreferrer')
|
window.open(guideUrl, '_blank', 'noopener,noreferrer')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadTenants() {
|
||||||
|
tenants.value = await getMyTenants().catch(() => [])
|
||||||
|
if (!tenants.value.some((tenant) => tenant.id === activeTenantId.value)) {
|
||||||
|
activeTenantId.value = tenants.value[0]?.id || (auth.isAdmin ? 'admin' : 'default')
|
||||||
|
localStorage.setItem('activeTenantId', activeTenantId.value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function switchTenant(value: string) {
|
||||||
|
activeTenantId.value = value
|
||||||
|
localStorage.setItem('activeTenantId', value)
|
||||||
|
ElMessage.success('当前租户已切换,正在刷新页面数据')
|
||||||
|
window.location.reload()
|
||||||
|
}
|
||||||
|
|
||||||
const serverIp = ref(window.location.hostname)
|
const serverIp = ref(window.location.hostname)
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
updateTime()
|
updateTime()
|
||||||
timer = setInterval(updateTime, 1000)
|
timer = setInterval(updateTime, 1000)
|
||||||
|
void loadTenants()
|
||||||
})
|
})
|
||||||
|
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
@@ -102,6 +124,13 @@ onUnmounted(() => {
|
|||||||
|
|
||||||
<div class="divider"></div>
|
<div class="divider"></div>
|
||||||
|
|
||||||
|
<div class="tenant-switcher">
|
||||||
|
<i class="fa fa-building-o" />
|
||||||
|
<el-select v-model="activeTenantId" size="small" @change="switchTenant">
|
||||||
|
<el-option v-for="tenant in tenants" :key="tenant.id" :label="tenant.name || tenant.id" :value="tenant.id" />
|
||||||
|
</el-select>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- 时间日期与服务器IP -->
|
<!-- 时间日期与服务器IP -->
|
||||||
<div class="system-info">
|
<div class="system-info">
|
||||||
<div class="info-item">
|
<div class="info-item">
|
||||||
@@ -256,6 +285,22 @@ onUnmounted(() => {
|
|||||||
background-color: #e2e8f0;
|
background-color: #e2e8f0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.tenant-switcher {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
min-width: 150px;
|
||||||
|
color: #64748b;
|
||||||
|
|
||||||
|
.fa {
|
||||||
|
color: var(--primary-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.el-select) {
|
||||||
|
width: 132px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
.system-info {
|
.system-info {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ interface MenuItem {
|
|||||||
icon: string
|
icon: string
|
||||||
to: string
|
to: string
|
||||||
permission: PermissionCode
|
permission: PermissionCode
|
||||||
|
allowNonAdmin?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
interface MenuGroup {
|
interface MenuGroup {
|
||||||
@@ -83,7 +84,7 @@ const menuGroups: MenuGroup[] = [
|
|||||||
items: [
|
items: [
|
||||||
{ key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' },
|
{ key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' },
|
||||||
{ key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' },
|
{ key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' },
|
||||||
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' },
|
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances?tab=mine', permission: 'user-settings', allowNonAdmin: true },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -117,8 +118,10 @@ const visibleMenuGroups = computed(() =>
|
|||||||
items: group.items.filter((item) => {
|
items: group.items.filter((item) => {
|
||||||
// admin 可以看到所有菜单
|
// admin 可以看到所有菜单
|
||||||
if (auth.isAdmin) return true
|
if (auth.isAdmin) return true
|
||||||
|
// 运行日志是普通用户可选的自助权限;没有权限时不显示入口。
|
||||||
|
if (item.permission === 'logs') return auth.hasPermission('logs')
|
||||||
// 非 admin 用户:仅隐藏管理员专属菜单
|
// 非 admin 用户:仅隐藏管理员专属菜单
|
||||||
return !ADMIN_ONLY_PERMISSIONS.includes(item.permission)
|
return item.allowNonAdmin || !ADMIN_ONLY_PERMISSIONS.includes(item.permission)
|
||||||
}),
|
}),
|
||||||
}))
|
}))
|
||||||
.filter((group) => group.items.length > 0),
|
.filter((group) => group.items.length > 0),
|
||||||
|
|||||||
@@ -43,11 +43,12 @@ function defaultUsers(): SystemUser[] {
|
|||||||
{
|
{
|
||||||
id: 'u_admin',
|
id: 'u_admin',
|
||||||
username: 'admin',
|
username: 'admin',
|
||||||
display_name: 'Platform Admin',
|
display_name: 'Admin',
|
||||||
role: 'admin',
|
role: 'admin',
|
||||||
status: 'active',
|
status: 'active',
|
||||||
permissions: allPermissions,
|
permissions: allPermissions,
|
||||||
create_time: '2026-01-01T00:00:00Z',
|
create_time: '2026-01-01T00:00:00Z',
|
||||||
|
tenant_id: 'admin',
|
||||||
protected: true,
|
protected: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -58,6 +59,7 @@ function defaultUsers(): SystemUser[] {
|
|||||||
status: 'active',
|
status: 'active',
|
||||||
permissions: allPermissions.filter((item) => item !== 'user-settings'),
|
permissions: allPermissions.filter((item) => item !== 'user-settings'),
|
||||||
create_time: '2026-01-01T00:00:00Z',
|
create_time: '2026-01-01T00:00:00Z',
|
||||||
|
tenant_id: 'default',
|
||||||
protected: false,
|
protected: false,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
@@ -88,7 +90,7 @@ export function authenticateMockUser(username: string, password: string): LoginR
|
|||||||
if (!user || user.status !== 'active') {
|
if (!user || user.status !== 'active') {
|
||||||
throw new UserMutationError('Invalid username or disabled account', 401)
|
throw new UserMutationError('Invalid username or disabled account', 401)
|
||||||
}
|
}
|
||||||
const expected = defaultPasswords[username] || 'platform123'
|
const expected = defaultPasswords[username] || '123456'
|
||||||
if (password !== expected) {
|
if (password !== expected) {
|
||||||
throw new UserMutationError('Invalid username or password', 401)
|
throw new UserMutationError('Invalid username or password', 401)
|
||||||
}
|
}
|
||||||
@@ -115,7 +117,7 @@ export function createMockUser(payload: CreateUserPayload): SystemUser {
|
|||||||
create_time: new Date().toISOString(),
|
create_time: new Date().toISOString(),
|
||||||
protected: false,
|
protected: false,
|
||||||
}
|
}
|
||||||
defaultPasswords[user.username] = payload.password || 'platform123'
|
defaultPasswords[user.username] = payload.password || '123456'
|
||||||
users.push(user)
|
users.push(user)
|
||||||
writeUsers(users)
|
writeUsers(users)
|
||||||
return user
|
return user
|
||||||
|
|||||||
@@ -50,18 +50,6 @@ const routes: RouteRecordRaw[] = [
|
|||||||
component: () => import('@/views/tenants/TenantDetailView.vue'),
|
component: () => import('@/views/tenants/TenantDetailView.vue'),
|
||||||
meta: { title: '租户详情', permission: 'user-settings' },
|
meta: { title: '租户详情', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
|
||||||
path: 'projects',
|
|
||||||
name: 'projects',
|
|
||||||
redirect: '/organization?tab=users',
|
|
||||||
meta: { title: '组织与权限', permission: 'user-settings' },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: 'projects/:id',
|
|
||||||
name: 'project-detail',
|
|
||||||
redirect: '/organization?tab=users',
|
|
||||||
meta: { title: '组织与权限', permission: 'user-settings' },
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
path: 'audit-logs',
|
path: 'audit-logs',
|
||||||
name: 'audit-logs',
|
name: 'audit-logs',
|
||||||
@@ -72,7 +60,7 @@ const routes: RouteRecordRaw[] = [
|
|||||||
path: 'operation-logs',
|
path: 'operation-logs',
|
||||||
name: 'operation-logs',
|
name: 'operation-logs',
|
||||||
redirect: '/logs?tab=operations',
|
redirect: '/logs?tab=operations',
|
||||||
meta: { title: '运行日志', permission: 'user-settings' },
|
meta: { title: '运行日志', permission: 'logs' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'approval-templates',
|
path: 'approval-templates',
|
||||||
@@ -84,7 +72,17 @@ const routes: RouteRecordRaw[] = [
|
|||||||
path: 'approval-instances',
|
path: 'approval-instances',
|
||||||
name: 'approval-instances',
|
name: 'approval-instances',
|
||||||
component: () => import('@/views/approvals/ApprovalCenterView.vue'),
|
component: () => import('@/views/approvals/ApprovalCenterView.vue'),
|
||||||
meta: { title: '审批中心', permission: 'user-settings' },
|
meta: { title: '审批中心', permission: 'user-settings', selfService: true },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: 'tenant-invitations',
|
||||||
|
redirect: '/approval-instances?tab=invitations',
|
||||||
|
meta: { title: '审批中心', selfService: true },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: 'resource-access-requests',
|
||||||
|
redirect: '/approval-instances?tab=access',
|
||||||
|
meta: { title: '审批中心', selfService: true },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'resource-acl',
|
path: 'resource-acl',
|
||||||
@@ -366,7 +364,6 @@ const permissionBySegment: Record<string, PermissionCode> = {
|
|||||||
organization: 'user-settings',
|
organization: 'user-settings',
|
||||||
'user-settings': 'user-settings',
|
'user-settings': 'user-settings',
|
||||||
tenants: 'user-settings',
|
tenants: 'user-settings',
|
||||||
projects: 'user-settings',
|
|
||||||
'audit-logs': 'user-settings',
|
'audit-logs': 'user-settings',
|
||||||
'operation-logs': 'user-settings',
|
'operation-logs': 'user-settings',
|
||||||
'approval-templates': 'user-settings',
|
'approval-templates': 'user-settings',
|
||||||
@@ -411,8 +408,9 @@ router.beforeEach((to, _from, next) => {
|
|||||||
if (!to.meta.skipPermission) {
|
if (!to.meta.skipPermission) {
|
||||||
const permission = requiredPermission(to.path, to.meta.permission)
|
const permission = requiredPermission(to.path, to.meta.permission)
|
||||||
// 仅限制管理员专属页面的访问权限
|
// 仅限制管理员专属页面的访问权限
|
||||||
// user-settings(组织与权限、资源授权、审批中心、运行日志)仅 admin 可访问
|
// user-settings(组织与权限、资源授权、审批中心)仅 admin 可访问
|
||||||
if (permission === 'user-settings' && !auth.isAdmin) {
|
const selfService = to.meta.selfService === true && (!to.query.tab || ['mine', 'access', 'invitations', 'compute'].includes(String(to.query.tab)))
|
||||||
|
if (permission === 'user-settings' && !auth.isAdmin && !selfService) {
|
||||||
next({ name: 'permission-denied', replace: true })
|
next({ name: 'permission-denied', replace: true })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -421,6 +419,10 @@ router.beforeEach((to, _from, next) => {
|
|||||||
next({ name: 'permission-denied', replace: true })
|
next({ name: 'permission-denied', replace: true })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (permission === 'logs' && !auth.isAdmin && !auth.hasPermission('logs')) {
|
||||||
|
next({ name: 'permission-denied', replace: true })
|
||||||
|
return
|
||||||
|
}
|
||||||
// 其他所有业务页面对已登录用户开放,不再检查权限码
|
// 其他所有业务页面对已登录用户开放,不再检查权限码
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,13 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
return currentUser.value?.permissions.includes(permission) ?? false
|
return currentUser.value?.permissions.includes(permission) ?? false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Button-level decision for a module action. The API remains the final authority. */
|
||||||
|
function can(permission?: PermissionCode, action?: 'read' | 'write' | 'execute' | 'download' | 'delete' | 'admin') {
|
||||||
|
if (!hasPermission(permission)) return false
|
||||||
|
if (action === 'admin') return isAdmin.value
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
/** 退出 */
|
/** 退出 */
|
||||||
async function logout() {
|
async function logout() {
|
||||||
const sessionId = sessionStorage.getItem(SESSION_STORAGE_KEY)
|
const sessionId = sessionStorage.getItem(SESSION_STORAGE_KEY)
|
||||||
@@ -88,6 +95,7 @@ export const useAuthStore = defineStore('auth', () => {
|
|||||||
isLoggedIn,
|
isLoggedIn,
|
||||||
isAdmin,
|
isAdmin,
|
||||||
hasPermission,
|
hasPermission,
|
||||||
|
can,
|
||||||
login,
|
login,
|
||||||
logout,
|
logout,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,6 +52,9 @@ export interface DataProcessTask {
|
|||||||
error_count?: number
|
error_count?: number
|
||||||
creator_name?: string | null
|
creator_name?: string | null
|
||||||
creator?: string | null
|
creator?: string | null
|
||||||
|
processor_name?: string | null
|
||||||
|
processor?: string | null
|
||||||
|
updated_by?: string | number | null
|
||||||
created_by?: string | number | null
|
created_by?: string | number | null
|
||||||
create_time?: string
|
create_time?: string
|
||||||
created_at?: string
|
created_at?: string
|
||||||
|
|||||||
@@ -24,7 +24,11 @@ export interface ModelItem {
|
|||||||
path?: string
|
path?: string
|
||||||
api_url?: string
|
api_url?: string
|
||||||
api_key?: string
|
api_key?: string
|
||||||
|
/** Returned by the server without exposing the actual credential. */
|
||||||
|
api_key_configured?: boolean
|
||||||
online_model_name?: string
|
online_model_name?: string
|
||||||
|
storage_status?: 'pending' | 'archiving' | 'available' | 'failed' | 'not_applicable' | string
|
||||||
|
storage_error?: string
|
||||||
create_time?: string
|
create_time?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,6 +46,8 @@ export interface TrainedModel {
|
|||||||
merged?: boolean
|
merged?: boolean
|
||||||
merging?: boolean
|
merging?: boolean
|
||||||
merged_path?: string
|
merged_path?: string
|
||||||
|
created_by?: string
|
||||||
|
tenant_id?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 创建/编辑模型请求体 */
|
/** 创建/编辑模型请求体 */
|
||||||
@@ -89,6 +95,8 @@ export interface DatasetItem {
|
|||||||
size_bytes?: number
|
size_bytes?: number
|
||||||
count?: number
|
count?: number
|
||||||
description?: string
|
description?: string
|
||||||
|
created_by?: string | number | null
|
||||||
|
creator_name?: string | null
|
||||||
create_time?: string
|
create_time?: string
|
||||||
files?: DatasetFile[]
|
files?: DatasetFile[]
|
||||||
current_version_no?: number | null
|
current_version_no?: number | null
|
||||||
@@ -168,6 +176,8 @@ export interface FineTuneTask {
|
|||||||
log_file?: string
|
log_file?: string
|
||||||
train_duration?: string
|
train_duration?: string
|
||||||
create_time?: string
|
create_time?: string
|
||||||
|
created_by?: string
|
||||||
|
tenant_id?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export type FineTuneStartPayload = Omit<
|
export type FineTuneStartPayload = Omit<
|
||||||
@@ -264,6 +274,19 @@ export interface EvalTask {
|
|||||||
score?: number
|
score?: number
|
||||||
status?: string
|
status?: string
|
||||||
create_time?: string
|
create_time?: string
|
||||||
|
progress?: number
|
||||||
|
progress_detail?: EvalProgress
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvalProgress {
|
||||||
|
status?: string
|
||||||
|
stage?: string
|
||||||
|
total?: number
|
||||||
|
completed?: number
|
||||||
|
percentage?: number
|
||||||
|
current_index?: number
|
||||||
|
message?: string
|
||||||
|
updated_at?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 启动评测时提交的可选基础指标配置。 */
|
/** 启动评测时提交的可选基础指标配置。 */
|
||||||
@@ -291,7 +314,8 @@ export interface StartEvalPayload {
|
|||||||
gpus?: number[]
|
gpus?: number[]
|
||||||
compute_node_id?: string
|
compute_node_id?: string
|
||||||
dataset_id: string | number
|
dataset_id: string | number
|
||||||
dimension_id: string | number
|
dimension_id?: string | number
|
||||||
|
dimension?: Partial<Dimension>
|
||||||
data_source: 'dataset' | 'inference'
|
data_source: 'dataset' | 'inference'
|
||||||
leaderboard: boolean
|
leaderboard: boolean
|
||||||
basic_metrics: BasicEvalMetricsConfig
|
basic_metrics: BasicEvalMetricsConfig
|
||||||
@@ -316,6 +340,8 @@ export interface EvalSampleResult {
|
|||||||
score: number
|
score: number
|
||||||
max_score: number
|
max_score: number
|
||||||
}>
|
}>
|
||||||
|
raw_score?: number | null
|
||||||
|
raw_max_score?: number | null
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 评测任务详情,包含逐样本结果和综合评价 */
|
/** 评测任务详情,包含逐样本结果和综合评价 */
|
||||||
@@ -334,8 +360,14 @@ export interface EvalTaskDetail extends EvalTask {
|
|||||||
score: number
|
score: number
|
||||||
max_score: number
|
max_score: number
|
||||||
pass_rate: number
|
pass_rate: number
|
||||||
|
sample_count?: number
|
||||||
|
available?: boolean
|
||||||
|
error?: string
|
||||||
}>
|
}>
|
||||||
samples: EvalSampleResult[]
|
samples: EvalSampleResult[]
|
||||||
|
progress_detail?: EvalProgress
|
||||||
|
basic_metrics?: Record<string, Record<string, unknown>>
|
||||||
|
metric_summary_version?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Dimension {
|
export interface Dimension {
|
||||||
@@ -446,20 +478,27 @@ export type PermissionCode =
|
|||||||
| 'logs'
|
| 'logs'
|
||||||
| 'user-settings'
|
| 'user-settings'
|
||||||
|
|
||||||
export type UserRole = 'admin' | 'operator' | 'viewer'
|
export type UserRole = 'admin' | 'operator' | 'viewer' | 'user'
|
||||||
|
|
||||||
export type UserStatus = 'active' | 'disabled'
|
export type UserStatus = 'active' | 'disabled' | 'pending' | 'deleted'
|
||||||
|
|
||||||
export interface SystemUser {
|
export interface SystemUser {
|
||||||
id: string
|
id: string
|
||||||
username: string
|
username: string
|
||||||
display_name: string
|
display_name: string
|
||||||
role: UserRole
|
role: UserRole
|
||||||
|
/** Canonical platform scope; role is retained for legacy API clients. */
|
||||||
|
platform_role?: 'platform_admin' | 'platform_user'
|
||||||
status: UserStatus
|
status: UserStatus
|
||||||
permissions: PermissionCode[]
|
permissions: PermissionCode[]
|
||||||
create_time: string
|
create_time: string
|
||||||
last_login?: string
|
last_login?: string
|
||||||
protected?: boolean
|
protected?: boolean
|
||||||
|
tenant_id?: string
|
||||||
|
deleted_at?: string | null
|
||||||
|
deleted_by?: string | null
|
||||||
|
tenant_memberships?: Array<{ tenant_id: string; role: string; status: string; expires_at?: string | null }>
|
||||||
|
tenant_count?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LoginResponse {
|
export interface LoginResponse {
|
||||||
@@ -475,6 +514,8 @@ export interface CreateUserPayload {
|
|||||||
role: UserRole
|
role: UserRole
|
||||||
status?: UserStatus
|
status?: UserStatus
|
||||||
permissions?: PermissionCode[]
|
permissions?: PermissionCode[]
|
||||||
|
tenant_id?: string
|
||||||
|
tenant_role?: 'owner' | 'admin' | 'member' | 'viewer'
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UpdateUserAccessPayload {
|
export interface UpdateUserAccessPayload {
|
||||||
|
|||||||
@@ -1,14 +1,24 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed } from 'vue'
|
import { computed } from 'vue'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import { useAuthStore } from '@/stores/auth'
|
||||||
import ApprovalInstanceView from './ApprovalInstanceView.vue'
|
import ApprovalInstanceView from './ApprovalInstanceView.vue'
|
||||||
import ApprovalTemplateView from './ApprovalTemplateView.vue'
|
import ApprovalTemplateView from './ApprovalTemplateView.vue'
|
||||||
|
import ResourceAccessRequestView from './ResourceAccessRequestView.vue'
|
||||||
|
import TenantInvitationsView from './TenantInvitationsView.vue'
|
||||||
|
import ComputeAccessRequestView from './ComputeAccessRequestView.vue'
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
const auth = useAuthStore()
|
||||||
|
|
||||||
const activeTab = computed<'instances' | 'mine' | 'strategies'>({
|
const activeTab = computed<'instances' | 'mine' | 'strategies' | 'access' | 'invitations' | 'compute'>({
|
||||||
get: () => route.query.tab === 'strategies' ? 'strategies' : route.query.tab === 'mine' ? 'mine' : 'instances',
|
get: () => {
|
||||||
|
const tab = String(route.query.tab || '')
|
||||||
|
if (tab === 'access' || tab === 'invitations' || tab === 'mine' || tab === 'compute') return tab
|
||||||
|
if (auth.isAdmin && tab === 'strategies') return 'strategies'
|
||||||
|
return auth.isAdmin ? 'instances' : 'mine'
|
||||||
|
},
|
||||||
set: (value: string) => {
|
set: (value: string) => {
|
||||||
void router.replace({ query: value === 'instances' ? {} : { tab: value } })
|
void router.replace({ query: value === 'instances' ? {} : { tab: value } })
|
||||||
},
|
},
|
||||||
@@ -25,15 +35,24 @@ const activeTab = computed<'instances' | 'mine' | 'strategies'>({
|
|||||||
</header>
|
</header>
|
||||||
|
|
||||||
<el-tabs v-model="activeTab">
|
<el-tabs v-model="activeTab">
|
||||||
<el-tab-pane label="审批申请" name="instances">
|
<el-tab-pane v-if="auth.isAdmin" label="审批申请" name="instances">
|
||||||
<ApprovalInstanceView v-if="activeTab === 'instances'" />
|
<ApprovalInstanceView v-if="activeTab === 'instances'" />
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
<el-tab-pane label="我的申请" name="mine">
|
<el-tab-pane label="我的申请" name="mine">
|
||||||
<ApprovalInstanceView v-if="activeTab === 'mine'" mine />
|
<ApprovalInstanceView v-if="activeTab === 'mine'" mine />
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
<el-tab-pane label="审批策略" name="strategies">
|
<el-tab-pane v-if="auth.isAdmin" label="审批策略" name="strategies">
|
||||||
<ApprovalTemplateView v-if="activeTab === 'strategies'" />
|
<ApprovalTemplateView v-if="activeTab === 'strategies'" />
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="访问申请" name="access">
|
||||||
|
<ResourceAccessRequestView v-if="activeTab === 'access'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="租户邀请" name="invitations">
|
||||||
|
<TenantInvitationsView v-if="activeTab === 'invitations'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="算力申请" name="compute">
|
||||||
|
<ComputeAccessRequestView v-if="activeTab === 'compute'" />
|
||||||
|
</el-tab-pane>
|
||||||
</el-tabs>
|
</el-tabs>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -11,12 +11,13 @@ const props = defineProps<{ mine?: boolean }>()
|
|||||||
const auth = useAuthStore()
|
const auth = useAuthStore()
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
const loadError = ref('')
|
||||||
const instances = ref<ApprovalInstance[]>([])
|
const instances = ref<ApprovalInstance[]>([])
|
||||||
const users = ref<SystemUser[]>([])
|
const users = ref<SystemUser[]>([])
|
||||||
const statusFilter = ref<string | undefined>(undefined)
|
const statusFilter = ref<string | undefined>(undefined)
|
||||||
const showDecide = ref(false)
|
const showDecide = ref(false)
|
||||||
const current = ref<ApprovalInstance | null>(null)
|
const current = ref<ApprovalInstance | null>(null)
|
||||||
const decision = ref({ step_index: 0, approver_id: '', approved: true, comment: '' })
|
const decision = ref({ step_index: 0, approved: true, comment: '' })
|
||||||
|
|
||||||
const visibleInstances = computed(() => {
|
const visibleInstances = computed(() => {
|
||||||
if (!props.mine) return instances.value
|
if (!props.mine) return instances.value
|
||||||
@@ -31,17 +32,26 @@ const statusOptions = [
|
|||||||
{ label: '已通过', value: 'approved' },
|
{ label: '已通过', value: 'approved' },
|
||||||
{ label: '已拒绝', value: 'rejected' },
|
{ label: '已拒绝', value: 'rejected' },
|
||||||
]
|
]
|
||||||
|
const statusLabel = (status?: string) => ({ pending: '待审批', approved: '已通过', rejected: '已拒绝', cancelled: '已撤回', expired: '已过期' }[status || ''] || status || '—')
|
||||||
|
const statusType = (status?: string): 'success' | 'warning' | 'danger' | 'info' => status === 'approved' ? 'success' : status === 'rejected' || status === 'expired' ? 'danger' : status === 'pending' ? 'warning' : 'info'
|
||||||
|
const actionLabel = (action?: string | null) => ({ 'resource.access': '资源访问', 'gpu.assign': '算力卡分配', 'tenant.member.add': '添加租户成员', 'tenant.member.remove': '移除租户成员', 'tenant.quota.update': '修改租户配额', 'model.use': '使用模型', 'dataset.use': '使用数据集' }[action || ''] || action || '权限申请')
|
||||||
|
const resourceLabel = (type?: string) => ({ dataset: '数据集', trained_model: '训练模型', model: '基座模型', compare: '推理任务', eval: '评测任务', compute: '算力资源' }[type || ''] || type || '资源')
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
instances.value = await getApprovalInstances(statusFilter.value)
|
loadError.value = ''
|
||||||
|
instances.value = await getApprovalInstances(statusFilter.value, props.mine)
|
||||||
|
} catch {
|
||||||
|
loadError.value = '审批记录加载失败,请刷新后重试。'
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadUsers() {
|
async function loadUsers() {
|
||||||
|
// 普通用户只查看自己的申请,不需要请求管理员用户列表。
|
||||||
|
if (!auth.isAdmin) return
|
||||||
try {
|
try {
|
||||||
users.value = await getUsers()
|
users.value = await getUsers()
|
||||||
} catch {
|
} catch {
|
||||||
@@ -49,7 +59,7 @@ async function loadUsers() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function userName(id?: string) {
|
function userName(id?: string | null) {
|
||||||
if (!id) return '—'
|
if (!id) return '—'
|
||||||
return users.value.find((u) => u.id === id)?.username || id
|
return users.value.find((u) => u.id === id)?.username || id
|
||||||
}
|
}
|
||||||
@@ -57,22 +67,23 @@ function userName(id?: string) {
|
|||||||
function openDecide(inst: ApprovalInstance) {
|
function openDecide(inst: ApprovalInstance) {
|
||||||
current.value = inst
|
current.value = inst
|
||||||
const step = inst.steps.find((s) => s.status === 'pending')
|
const step = inst.steps.find((s) => s.status === 'pending')
|
||||||
decision.value = { step_index: step ? step.step_index : 0, approver_id: '', approved: true, comment: '' }
|
decision.value = { step_index: step ? step.step_index : 0, approved: true, comment: '' }
|
||||||
showDecide.value = true
|
showDecide.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function canDecide(inst: ApprovalInstance) {
|
||||||
|
if (props.mine || inst.status !== 'pending' || !auth.can('user-settings', 'write')) return false
|
||||||
|
const step = inst.steps.find((item) => item.status === 'pending')
|
||||||
|
return !!step && (!step.approver_id || step.approver_id === auth.currentUser?.id || auth.isAdmin)
|
||||||
|
}
|
||||||
|
|
||||||
function asApprovalInstance(row: unknown): ApprovalInstance {
|
function asApprovalInstance(row: unknown): ApprovalInstance {
|
||||||
return row as ApprovalInstance
|
return row as ApprovalInstance
|
||||||
}
|
}
|
||||||
|
|
||||||
async function submitDecision() {
|
async function submitDecision() {
|
||||||
if (!current.value) return
|
if (!current.value) return
|
||||||
if (!decision.value.approver_id) {
|
|
||||||
ElMessage.warning('请选择审批人')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
await decideApproval(current.value.id, decision.value.step_index, {
|
await decideApproval(current.value.id, decision.value.step_index, {
|
||||||
approver_id: decision.value.approver_id,
|
|
||||||
approved: decision.value.approved,
|
approved: decision.value.approved,
|
||||||
comment: decision.value.comment,
|
comment: decision.value.comment,
|
||||||
})
|
})
|
||||||
@@ -82,13 +93,14 @@ async function submitDecision() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
loadUsers()
|
void loadUsers()
|
||||||
load()
|
void load()
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
<div class="page">
|
<div class="page">
|
||||||
|
<el-alert v-if="loadError" type="error" show-icon :closable="false" :title="loadError" class="state-alert" />
|
||||||
<DataTablePage :title="props.mine ? '我的申请' : '审批申请'" :data="visibleInstances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
|
<DataTablePage :title="props.mine ? '我的申请' : '审批申请'" :data="visibleInstances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
|
||||||
<template #toolbar-extra>
|
<template #toolbar-extra>
|
||||||
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
|
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
|
||||||
@@ -96,17 +108,18 @@ onMounted(() => {
|
|||||||
</el-select>
|
</el-select>
|
||||||
</template>
|
</template>
|
||||||
<template #columns>
|
<template #columns>
|
||||||
<el-table-column prop="resource_type" label="资源类型" min-width="120" />
|
<el-table-column label="申请事项" min-width="160"><template #default="{ row }">{{ actionLabel(asApprovalInstance(row).action) }}</template></el-table-column>
|
||||||
|
<el-table-column label="资源类型" min-width="120"><template #default="{ row }">{{ resourceLabel(asApprovalInstance(row).resource_type) }}</template></el-table-column>
|
||||||
<el-table-column prop="resource_id" label="资源 ID" min-width="160" show-overflow-tooltip />
|
<el-table-column prop="resource_id" label="资源 ID" min-width="160" show-overflow-tooltip />
|
||||||
<el-table-column prop="applicant_id" label="申请人" min-width="120">
|
<el-table-column prop="applicant_id" label="申请人" min-width="120">
|
||||||
<template #default="{ row }">{{ userName(asApprovalInstance(row).applicant_id) }}</template>
|
<template #default="{ row }">{{ userName(asApprovalInstance(row).applicant_id) }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="status" label="状态" min-width="100" />
|
<el-table-column label="状态" min-width="100"><template #default="{ row }"><el-tag size="small" :type="statusType(asApprovalInstance(row).status)">{{ statusLabel(asApprovalInstance(row).status) }}</el-tag></template></el-table-column>
|
||||||
<el-table-column prop="current_step" label="当前步骤" min-width="100" />
|
<el-table-column prop="current_step" label="当前步骤" min-width="100" />
|
||||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||||
</template>
|
</template>
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
<el-button v-if="asApprovalInstance(row).status === 'pending'" link type="primary" @click="openDecide(asApprovalInstance(row))">审批</el-button>
|
<el-button v-if="canDecide(asApprovalInstance(row))" link type="primary" @click="openDecide(asApprovalInstance(row))">审批</el-button>
|
||||||
</template>
|
</template>
|
||||||
</DataTablePage>
|
</DataTablePage>
|
||||||
<el-dialog v-model="showDecide" title="审批决策" width="480px">
|
<el-dialog v-model="showDecide" title="审批决策" width="480px">
|
||||||
@@ -117,10 +130,10 @@ onMounted(() => {
|
|||||||
<el-form-item label="步骤">
|
<el-form-item label="步骤">
|
||||||
第 {{ decision.step_index + 1 }} 步
|
第 {{ decision.step_index + 1 }} 步
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="审批人" required>
|
<el-form-item label="审批人">
|
||||||
<el-select v-model="decision.approver_id" filterable style="width: 100%">
|
{{ current.steps.find((s) => s.step_index === decision.step_index)?.approver_id
|
||||||
<el-option v-for="u in users" :key="u.id" :label="u.username" :value="u.id" />
|
? userName(current.steps.find((s) => s.step_index === decision.step_index)?.approver_id)
|
||||||
</el-select>
|
: '平台管理员' }}
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="结果">
|
<el-form-item label="结果">
|
||||||
<el-radio-group v-model="decision.approved">
|
<el-radio-group v-model="decision.approved">
|
||||||
@@ -142,4 +155,5 @@ onMounted(() => {
|
|||||||
|
|
||||||
<style scoped lang="scss">
|
<style scoped lang="scss">
|
||||||
.page { padding: 16px; }
|
.page { padding: 16px; }
|
||||||
|
.state-alert { margin-bottom: 16px; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,77 +1,68 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
import { Plus } from '@element-plus/icons-vue'
|
import { Plus } from '@element-plus/icons-vue'
|
||||||
import DataTablePage from '@/components/DataTablePage.vue'
|
import DataTablePage from '@/components/DataTablePage.vue'
|
||||||
import { createApprovalTemplate, getApprovalTemplates, type ApprovalTemplate } from '@/api/modules/approval'
|
import { createApprovalTemplate, getApprovalTemplates, type ApprovalTemplate } from '@/api/modules/approval'
|
||||||
|
import { getUsers } from '@/api/modules/system'
|
||||||
|
import type { SystemUser } from '@/types'
|
||||||
|
|
||||||
|
interface TemplateStep { approver_id?: string; approver_type?: 'user' | 'admin' | 'tenant_admin' }
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const templates = ref<ApprovalTemplate[]>([])
|
const templates = ref<ApprovalTemplate[]>([])
|
||||||
|
const users = ref<SystemUser[]>([])
|
||||||
const showCreate = ref(false)
|
const showCreate = ref(false)
|
||||||
const form = ref({ name: '', stepsText: '[]' })
|
const form = reactive<{ name: string; steps: TemplateStep[] }>({ name: '', steps: [{ approver_type: 'user', approver_id: '' }] })
|
||||||
|
|
||||||
|
function userName(id?: string) { return users.value.find((user) => user.id === id)?.display_name || users.value.find((user) => user.id === id)?.username || id || '未指定' }
|
||||||
|
function stepLabel(step: TemplateStep) { return step.approver_type === 'admin' ? '平台管理员' : step.approver_type === 'tenant_admin' ? '租户管理员' : userName(step.approver_id) }
|
||||||
|
function addStep() { form.steps.push({ approver_type: 'user', approver_id: '' }) }
|
||||||
|
function removeStep(index: number) { if (form.steps.length > 1) form.steps.splice(index, 1) }
|
||||||
|
function resetForm() { form.name = ''; form.steps = [{ approver_type: 'user', approver_id: '' }] }
|
||||||
async function load() {
|
async function load() {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
templates.value = await getApprovalTemplates()
|
templates.value = await getApprovalTemplates()
|
||||||
} finally {
|
users.value = await getUsers().catch(() => [])
|
||||||
loading.value = false
|
} finally { loading.value = false }
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function submitCreate() {
|
async function submitCreate() {
|
||||||
if (!form.value.name) {
|
if (!form.name.trim()) return ElMessage.warning('请填写模板名称')
|
||||||
ElMessage.warning('请填写模板名称')
|
const steps = form.steps.filter((step) => step.approver_type !== 'user' || step.approver_id)
|
||||||
return
|
if (!steps.length) return ElMessage.warning('请至少配置一个有效审批人')
|
||||||
}
|
await createApprovalTemplate({ name: form.name.trim(), steps: steps as any })
|
||||||
let steps: unknown[] = []
|
ElMessage.success('审批策略创建成功')
|
||||||
try {
|
|
||||||
steps = JSON.parse(form.value.stepsText || '[]')
|
|
||||||
} catch {
|
|
||||||
ElMessage.error('步骤需为合法 JSON 数组')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
await createApprovalTemplate({ name: form.value.name, steps: steps as any })
|
|
||||||
ElMessage.success('模板创建成功')
|
|
||||||
showCreate.value = false
|
showCreate.value = false
|
||||||
form.value = { name: '', stepsText: '[]' }
|
resetForm()
|
||||||
load()
|
await load()
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(load)
|
onMounted(load)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
<div class="page">
|
<div class="page">
|
||||||
<DataTablePage title="审批模板" :data="templates" :loading="loading">
|
<DataTablePage title="审批策略" :data="templates" :loading="loading">
|
||||||
<template #toolbar-extra>
|
<template #toolbar-extra><el-button type="primary" :icon="Plus" @click="showCreate = true">新建策略</el-button></template>
|
||||||
<el-button type="primary" :icon="Plus" @click="showCreate = true">新建模板</el-button>
|
|
||||||
</template>
|
|
||||||
<template #columns>
|
<template #columns>
|
||||||
<el-table-column prop="name" label="模板名" min-width="160" />
|
<el-table-column prop="name" label="策略名称" min-width="180" />
|
||||||
<el-table-column label="步骤数" min-width="100">
|
<el-table-column label="审批链路" min-width="280"><template #default="{ row }"><el-space wrap><el-tag v-for="(step, index) in row.steps || []" :key="index" size="small">{{ index + 1 }}. {{ stepLabel(step) }}</el-tag></el-space></template></el-table-column>
|
||||||
<template #default="{ row }">{{ (row.steps || []).length }}</template>
|
|
||||||
</el-table-column>
|
|
||||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||||
</template>
|
</template>
|
||||||
</DataTablePage>
|
</DataTablePage>
|
||||||
<el-dialog v-model="showCreate" title="新建审批模板" width="560px">
|
<el-dialog v-model="showCreate" title="新建审批策略" width="620px">
|
||||||
<el-form label-width="90px">
|
<el-form label-width="90px">
|
||||||
<el-form-item label="名称" required>
|
<el-form-item label="策略名称" required><el-input v-model="form.name" placeholder="例如:数据集访问审批" /></el-form-item>
|
||||||
<el-input v-model="form.name" placeholder="模板名" />
|
<el-form-item label="审批链路"><div class="steps-form"><div v-for="(step, index) in form.steps" :key="index" class="step-row"><span class="step-number">第 {{ index + 1 }} 步</span><el-select v-model="step.approver_type" style="width: 135px"><el-option label="指定用户" value="user" /><el-option label="租户管理员" value="tenant_admin" /><el-option label="平台管理员" value="admin" /></el-select><el-select v-if="step.approver_type === 'user'" v-model="step.approver_id" filterable placeholder="选择审批人" style="width: 190px"><el-option v-for="user in users" :key="user.id" :label="`${user.display_name || user.username}(${user.username})`" :value="user.id" /></el-select><span v-else class="role-hint">{{ stepLabel(step) }}</span><el-button text type="danger" :disabled="form.steps.length === 1" @click="removeStep(index)">移除</el-button></div><el-button text type="primary" @click="addStep">+ 添加审批步骤</el-button></div></el-form-item>
|
||||||
</el-form-item>
|
|
||||||
<el-form-item label="步骤 JSON">
|
|
||||||
<el-input v-model="form.stepsText" type="textarea" :rows="5" placeholder='[{"approver_id":"u1"},{"approver_id":"u2"}]' />
|
|
||||||
</el-form-item>
|
|
||||||
</el-form>
|
</el-form>
|
||||||
<template #footer>
|
<template #footer><el-button @click="showCreate = false">取消</el-button><el-button type="primary" @click="submitCreate">创建策略</el-button></template>
|
||||||
<el-button @click="showCreate = false">取消</el-button>
|
|
||||||
<el-button type="primary" @click="submitCreate">创建</el-button>
|
|
||||||
</template>
|
|
||||||
</el-dialog>
|
</el-dialog>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped lang="scss">
|
<style scoped>
|
||||||
.page { padding: 16px; }
|
.page { padding: 16px; }
|
||||||
|
.steps-form { width: 100%; }
|
||||||
|
.step-row { display: flex; align-items: center; gap: 8px; margin-bottom: 10px; }
|
||||||
|
.step-number { width: 52px; color: #64748b; font-size: 13px; }
|
||||||
|
.role-hint { min-width: 190px; color: #64748b; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
92
frontend/src/views/approvals/ComputeAccessRequestView.vue
Normal file
92
frontend/src/views/approvals/ComputeAccessRequestView.vue
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { getGpuRequestOptions, requestGpuAccess, type GpuRequestOption } from '@/api/modules/approval'
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const submitting = ref(false)
|
||||||
|
const loadError = ref('')
|
||||||
|
const nodes = ref<GpuRequestOption[]>([])
|
||||||
|
const form = reactive({ node_id: '', gpu_indices: [] as number[], reason: '' })
|
||||||
|
|
||||||
|
const selectedNode = computed(() => nodes.value.find((node) => node.id === form.node_id) || null)
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
loadError.value = ''
|
||||||
|
const result = await getGpuRequestOptions()
|
||||||
|
nodes.value = result.nodes || []
|
||||||
|
if (!nodes.value.some((node) => node.id === form.node_id)) {
|
||||||
|
form.node_id = nodes.value[0]?.id || ''
|
||||||
|
form.gpu_indices = []
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
loadError.value = '可申请算力加载失败,请刷新后重试。'
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function changeNode() {
|
||||||
|
form.gpu_indices = []
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submit() {
|
||||||
|
if (!form.node_id) return ElMessage.warning('请选择算力节点')
|
||||||
|
if (!form.gpu_indices.length) return ElMessage.warning('请选择至少一张算力卡')
|
||||||
|
submitting.value = true
|
||||||
|
try {
|
||||||
|
const result = await requestGpuAccess({
|
||||||
|
assignments: form.gpu_indices.map((gpu_index) => ({ node_id: form.node_id, gpu_index })),
|
||||||
|
reason: form.reason.trim() || undefined,
|
||||||
|
})
|
||||||
|
ElMessage.success(result.approval_required === false ? '算力分配成功' : '算力申请已提交,等待审批')
|
||||||
|
form.gpu_indices = []
|
||||||
|
form.reason = ''
|
||||||
|
} finally {
|
||||||
|
submitting.value = false
|
||||||
|
await load()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(load)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="compute-request" v-loading="loading">
|
||||||
|
<el-alert v-if="loadError" type="error" show-icon :closable="false" :title="loadError" class="state-alert" />
|
||||||
|
<el-alert v-else-if="!nodes.length && !loading" type="info" show-icon :closable="false" title="当前没有可申请的空闲算力卡,请稍后重试或联系管理员。" class="state-alert" />
|
||||||
|
<el-card shadow="never">
|
||||||
|
<template #header><span>申请算力卡</span></template>
|
||||||
|
<el-form label-width="100px" class="request-form">
|
||||||
|
<el-form-item label="算力节点">
|
||||||
|
<el-select v-model="form.node_id" filterable placeholder="选择在线算力节点" style="width: min(520px, 100%)" @change="changeNode">
|
||||||
|
<el-option v-for="node in nodes" :key="node.id" :label="`${node.name}(${node.code})`" :value="node.id" />
|
||||||
|
</el-select>
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="算力卡">
|
||||||
|
<el-checkbox-group v-if="selectedNode" v-model="form.gpu_indices">
|
||||||
|
<el-checkbox v-for="gpu in selectedNode.gpus" :key="gpu.index" :value="gpu.index">
|
||||||
|
GPU {{ gpu.index }} · {{ gpu.name }} · {{ gpu.memory_total_gb }} GB
|
||||||
|
</el-checkbox>
|
||||||
|
</el-checkbox-group>
|
||||||
|
<span v-else class="muted">请先选择算力节点</span>
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="申请理由">
|
||||||
|
<el-input v-model="form.reason" type="textarea" :rows="3" maxlength="500" show-word-limit placeholder="说明训练、推理或评测用途" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item>
|
||||||
|
<el-button type="primary" :loading="submitting" :disabled="!nodes.length" @click="submit">提交算力申请</el-button>
|
||||||
|
</el-form-item>
|
||||||
|
</el-form>
|
||||||
|
</el-card>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.compute-request { padding: 16px; }
|
||||||
|
.state-alert { margin-bottom: 16px; }
|
||||||
|
.request-form { max-width: 760px; }
|
||||||
|
.muted { color: #94a3b8; }
|
||||||
|
</style>
|
||||||
123
frontend/src/views/approvals/ResourceAccessRequestView.vue
Normal file
123
frontend/src/views/approvals/ResourceAccessRequestView.vue
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { onMounted, reactive, ref } from 'vue'
|
||||||
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
|
import { getDatasetList, type DatasetItem } from '@/api/modules/dataset'
|
||||||
|
import { getTrainedModels } from '@/api/modules/model'
|
||||||
|
import {
|
||||||
|
cancelResourceAccessRequest,
|
||||||
|
createResourceAccessRequest,
|
||||||
|
getResourceAccessRequests,
|
||||||
|
type ResourceAccessRequest,
|
||||||
|
} from '@/api/modules/approval'
|
||||||
|
import type { TrainedModel } from '@/types'
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const resourcesLoading = ref(false)
|
||||||
|
const submitting = ref(false)
|
||||||
|
const datasets = ref<DatasetItem[]>([])
|
||||||
|
const models = ref<TrainedModel[]>([])
|
||||||
|
const requests = ref<ResourceAccessRequest[]>([])
|
||||||
|
const loadError = ref('')
|
||||||
|
const statusFilter = ref('')
|
||||||
|
const form = reactive({ resource_type: 'dataset', resource_id: '', requested_permissions: ['read', 'execute'], reason: '', expires_at: '' })
|
||||||
|
const permissionOptions = [
|
||||||
|
{ label: '查看', value: 'read' },
|
||||||
|
{ label: '使用', value: 'execute' },
|
||||||
|
{ label: '下载', value: 'download' },
|
||||||
|
]
|
||||||
|
|
||||||
|
function resourceOptions() {
|
||||||
|
if (form.resource_type === 'dataset') return datasets.value.map((item) => ({ id: String(item.id), name: item.name || String(item.id) }))
|
||||||
|
return models.value.map((item) => ({ id: String(item.id || item.name), name: item.name || String(item.id) }))
|
||||||
|
}
|
||||||
|
function resetResource() { form.resource_id = '' }
|
||||||
|
function resourceTypeLabel(type: string) { return type === 'trained_model' ? '训练模型' : type === 'dataset' ? '数据集' : type }
|
||||||
|
function permissionLabel(permission: string) { return ({ read: '查看', write: '编辑', execute: '使用', download: '下载' } as Record<string, string>)[permission] || permission }
|
||||||
|
function statusLabel(status: string) { return ({ pending: '审批中', approved: '已通过', rejected: '已拒绝', cancelled: '已撤回', expired: '已过期' } as Record<string, string>)[status] || status }
|
||||||
|
function statusType(status: string): 'success' | 'warning' | 'danger' | 'info' { return status === 'approved' ? 'success' : status === 'rejected' || status === 'expired' ? 'danger' : status === 'pending' ? 'warning' : 'info' }
|
||||||
|
function resourceName(row: ResourceAccessRequest) { return resourceOptions().find((item) => item.id === row.resource_id)?.name || row.resource_id }
|
||||||
|
function asRequest(row: unknown) { return row as ResourceAccessRequest }
|
||||||
|
async function loadResources() {
|
||||||
|
if (resourcesLoading.value) return
|
||||||
|
resourcesLoading.value = true
|
||||||
|
try {
|
||||||
|
const [datasetItems, trained] = await Promise.all([getDatasetList().catch(() => []), getTrainedModels().catch(() => ({ models: [] }))])
|
||||||
|
datasets.value = datasetItems || []
|
||||||
|
models.value = trained?.models || []
|
||||||
|
} catch {
|
||||||
|
// 资源下拉列表加载失败不影响申请历史展示。
|
||||||
|
} finally { resourcesLoading.value = false }
|
||||||
|
}
|
||||||
|
async function load() {
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
loadError.value = ''
|
||||||
|
requests.value = await getResourceAccessRequests(statusFilter.value || undefined)
|
||||||
|
} catch {
|
||||||
|
loadError.value = '访问申请加载失败,请刷新后重试。'
|
||||||
|
} finally { loading.value = false }
|
||||||
|
}
|
||||||
|
async function submit() {
|
||||||
|
if (!form.resource_id) return ElMessage.warning('请选择资源')
|
||||||
|
if (!form.requested_permissions.length) return ElMessage.warning('请选择申请权限')
|
||||||
|
submitting.value = true
|
||||||
|
try {
|
||||||
|
await createResourceAccessRequest({ resource_type: form.resource_type, resource_id: form.resource_id, requested_permissions: form.requested_permissions, reason: form.reason || undefined, expires_at: form.expires_at || undefined })
|
||||||
|
ElMessage.success('访问申请已提交')
|
||||||
|
form.reason = ''
|
||||||
|
form.expires_at = ''
|
||||||
|
await load()
|
||||||
|
} finally { submitting.value = false }
|
||||||
|
}
|
||||||
|
async function cancel(id: string) {
|
||||||
|
await ElMessageBox.confirm('撤回后需要重新提交申请,是否继续?', '撤回访问申请', { type: 'warning' })
|
||||||
|
await cancelResourceAccessRequest(id)
|
||||||
|
ElMessage.success('申请已撤销')
|
||||||
|
await load()
|
||||||
|
}
|
||||||
|
onMounted(() => {
|
||||||
|
// 先显示申请记录;资源选项在后台加载,避免慢资源查询阻塞页面。
|
||||||
|
void load()
|
||||||
|
void loadResources()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="access-request" v-loading="loading">
|
||||||
|
<el-alert v-if="loadError" type="error" show-icon :closable="false" :title="loadError" class="state-alert" />
|
||||||
|
<el-card shadow="never">
|
||||||
|
<template #header><span>申请资源访问</span></template>
|
||||||
|
<el-form label-width="100px" class="request-form">
|
||||||
|
<el-form-item label="资源类型">
|
||||||
|
<el-select v-model="form.resource_type" style="width: 220px" @change="resetResource"><el-option label="数据集" value="dataset" /><el-option label="训练模型" value="trained_model" /></el-select>
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="资源">
|
||||||
|
<el-select v-model="form.resource_id" filterable placeholder="选择资源" :loading="resourcesLoading" style="width: min(520px, 100%)" @focus="loadResources"><el-option v-for="item in resourceOptions()" :key="item.id" :label="item.name" :value="item.id" /></el-select>
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="申请权限"><el-checkbox-group v-model="form.requested_permissions"><el-checkbox v-for="item in permissionOptions" :key="item.value" :value="item.value">{{ item.label }}</el-checkbox></el-checkbox-group></el-form-item>
|
||||||
|
<el-form-item label="有效期至"><el-date-picker v-model="form.expires_at" type="datetime" value-format="YYYY-MM-DDTHH:mm:ssZ" placeholder="可选" /></el-form-item>
|
||||||
|
<el-form-item label="申请理由"><el-input v-model="form.reason" type="textarea" :rows="3" maxlength="500" show-word-limit /></el-form-item>
|
||||||
|
<el-form-item><el-button type="primary" :loading="submitting" @click="submit">提交申请</el-button></el-form-item>
|
||||||
|
</el-form>
|
||||||
|
</el-card>
|
||||||
|
<el-card shadow="never" class="history-card">
|
||||||
|
<template #header><div class="history-header"><span>我的访问申请</span><el-select v-model="statusFilter" clearable placeholder="筛选状态" style="width: 130px" @change="load"><el-option label="审批中" value="pending" /><el-option label="已通过" value="approved" /><el-option label="已拒绝" value="rejected" /><el-option label="已撤回" value="cancelled" /><el-option label="已过期" value="expired" /></el-select></div></template>
|
||||||
|
<el-table :data="requests" empty-text="暂无访问申请">
|
||||||
|
<el-table-column label="资源" min-width="220"><template #default="{ row }"><div>{{ resourceName(asRequest(row)) }}</div><small>{{ resourceTypeLabel(asRequest(row).resource_type) }} · {{ asRequest(row).resource_id }}</small></template></el-table-column>
|
||||||
|
<el-table-column label="申请权限" min-width="150"><template #default="{ row }">{{ row.requested_permissions.map(permissionLabel).join('、') }}</template></el-table-column>
|
||||||
|
<el-table-column label="状态" width="100"><template #default="{ row }"><el-tag size="small" :type="statusType(row.status)">{{ statusLabel(row.status) }}</el-tag></template></el-table-column><el-table-column prop="created_at" label="申请时间" min-width="180" />
|
||||||
|
<el-table-column label="操作" width="90"><template #default="{ row }"><el-button v-if="row.status === 'pending'" link type="danger" @click="cancel(row.id)">撤销</el-button></template></el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.access-request { padding: 16px; }
|
||||||
|
.state-alert { margin-bottom: 16px; }
|
||||||
|
.history-header { display: flex; align-items: center; justify-content: space-between; gap: 12px; }
|
||||||
|
.history-header span { font-weight: 600; }
|
||||||
|
.access-request small { color: #94a3b8; }
|
||||||
|
.request-form { max-width: 760px; }
|
||||||
|
.history-card { margin-top: 16px; }
|
||||||
|
</style>
|
||||||
46
frontend/src/views/approvals/TenantInvitationsView.vue
Normal file
46
frontend/src/views/approvals/TenantInvitationsView.vue
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { onMounted, ref } from 'vue'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { acceptTenantInvitation, getTenantInvitations, type TenantMember } from '@/api/modules/tenant'
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const accepting = ref('')
|
||||||
|
const invitations = ref<TenantMember[]>([])
|
||||||
|
const error = ref('')
|
||||||
|
const statusLabel = (status: string) => ({ pending: '待接受', expired: '已过期', disabled: '已失效', active: '已加入' }[status] || status)
|
||||||
|
const statusType = (status: string): 'success' | 'warning' | 'danger' | 'info' => status === 'active' ? 'success' : status === 'expired' || status === 'disabled' ? 'danger' : 'warning'
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
loading.value = true
|
||||||
|
try { error.value = ''; invitations.value = await getTenantInvitations() } catch { error.value = '租户邀请加载失败,请刷新后重试。' } finally { loading.value = false }
|
||||||
|
}
|
||||||
|
async function accept(item: TenantMember) {
|
||||||
|
accepting.value = item.tenant_id
|
||||||
|
try {
|
||||||
|
await acceptTenantInvitation(item.tenant_id, item.user_id)
|
||||||
|
ElMessage.success('已加入租户')
|
||||||
|
await load()
|
||||||
|
} finally { accepting.value = '' }
|
||||||
|
}
|
||||||
|
function asMember(row: unknown) { return row as TenantMember }
|
||||||
|
onMounted(load)
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="page" v-loading="loading">
|
||||||
|
<el-alert v-if="error" type="error" show-icon :closable="false" :title="error" class="alert" />
|
||||||
|
<el-table :data="invitations" empty-text="暂无租户邀请">
|
||||||
|
<el-table-column prop="tenant_id" label="租户 ID" min-width="180" />
|
||||||
|
<el-table-column prop="role" label="加入角色" width="120"><template #default="{ row }">{{ row.role === 'admin' ? '租户管理员' : row.role === 'viewer' ? '只读成员' : '租户成员' }}</template></el-table-column>
|
||||||
|
<el-table-column prop="invited_by" label="邀请人" min-width="150" />
|
||||||
|
<el-table-column prop="expires_at" label="有效期至" min-width="180" />
|
||||||
|
<el-table-column label="状态" width="100"><template #default="{ row }"><el-tag size="small" :type="statusType(row.status)">{{ statusLabel(row.status) }}</el-tag></template></el-table-column>
|
||||||
|
<el-table-column label="操作" width="100"><template #default="{ row }"><el-button v-if="asMember(row).status === 'pending'" link type="primary" :loading="accepting === asMember(row).tenant_id" @click="accept(asMember(row))">接受邀请</el-button></template></el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.page { padding: 16px; }
|
||||||
|
.alert { margin-bottom: 16px; }
|
||||||
|
</style>
|
||||||
@@ -10,6 +10,8 @@ const logs = ref<AuditLog[]>([])
|
|||||||
const total = ref(0)
|
const total = ref(0)
|
||||||
const users = ref<SystemUser[]>([])
|
const users = ref<SystemUser[]>([])
|
||||||
const tenants = ref<Tenant[]>([])
|
const tenants = ref<Tenant[]>([])
|
||||||
|
const detailVisible = ref(false)
|
||||||
|
const detailLog = ref<AuditLog | null>(null)
|
||||||
const query = reactive<AuditQuery>({
|
const query = reactive<AuditQuery>({
|
||||||
tenant_id: '',
|
tenant_id: '',
|
||||||
actor_id: '',
|
actor_id: '',
|
||||||
@@ -50,7 +52,17 @@ const actionOptions = [
|
|||||||
{ value: 'grant_acl', label: '授予资源权限' },
|
{ value: 'grant_acl', label: '授予资源权限' },
|
||||||
{ value: 'revoke_acl', label: '撤销资源权限' },
|
{ value: 'revoke_acl', label: '撤销资源权限' },
|
||||||
{ value: 'gpu.assign', label: '分配算力卡' },
|
{ value: 'gpu.assign', label: '分配算力卡' },
|
||||||
|
{ value: 'gpu.assign.request', label: '申请算力卡' },
|
||||||
{ value: 'gpu.release', label: '释放算力卡' },
|
{ value: 'gpu.release', label: '释放算力卡' },
|
||||||
|
{ value: 'approval.decision', label: '处理审批' },
|
||||||
|
{ value: 'dataset.download', label: '下载数据集' },
|
||||||
|
{ value: 'data-process', label: '访问数据处理' },
|
||||||
|
{ value: 'data-convert', label: '访问数据类型转换' },
|
||||||
|
{ value: 'fine-tune', label: '访问模型训练' },
|
||||||
|
{ value: 'model-eval', label: '访问模型评测' },
|
||||||
|
{ value: 'model-inference', label: '访问模型推理' },
|
||||||
|
{ value: 'model-manage', label: '访问模型管理' },
|
||||||
|
{ value: 'dashboard', label: '访问服务看板' },
|
||||||
{ value: 'create', label: '创建' },
|
{ value: 'create', label: '创建' },
|
||||||
{ value: 'update', label: '修改' },
|
{ value: 'update', label: '修改' },
|
||||||
{ value: 'delete', label: '删除' },
|
{ value: 'delete', label: '删除' },
|
||||||
@@ -83,10 +95,42 @@ const targetTypeOptions = [
|
|||||||
{ value: 'retention_policy', label: '留存策略' },
|
{ value: 'retention_policy', label: '留存策略' },
|
||||||
{ value: 'module', label: '业务模块' },
|
{ value: 'module', label: '业务模块' },
|
||||||
{ value: 'api', label: '接口' },
|
{ value: 'api', label: '接口' },
|
||||||
|
{ value: 'approval_instance', label: '审批实例' },
|
||||||
]
|
]
|
||||||
|
|
||||||
const actionLabels = Object.fromEntries(actionOptions.map((item) => [item.value, item.label]))
|
const actionLabels = Object.fromEntries(actionOptions.map((item) => [item.value, item.label]))
|
||||||
const targetTypeLabels = Object.fromEntries(targetTypeOptions.map((item) => [item.value, item.label]))
|
const targetTypeLabels = Object.fromEntries(targetTypeOptions.map((item) => [item.value, item.label]))
|
||||||
|
const actionVerbLabels: Record<string, string> = {
|
||||||
|
create: '创建',
|
||||||
|
update: '修改',
|
||||||
|
delete: '删除',
|
||||||
|
start: '启动',
|
||||||
|
stop: '停止',
|
||||||
|
upload: '上传',
|
||||||
|
download: '下载',
|
||||||
|
export: '导出',
|
||||||
|
import: '导入',
|
||||||
|
merge: '合并',
|
||||||
|
assign: '分配',
|
||||||
|
release: '释放',
|
||||||
|
request: '申请',
|
||||||
|
approve: '审批通过',
|
||||||
|
reject: '审批拒绝',
|
||||||
|
login: '登录',
|
||||||
|
logout: '退出登录',
|
||||||
|
}
|
||||||
|
const actionObjectLabels: Record<string, string> = {
|
||||||
|
'approval.decision': '处理审批',
|
||||||
|
'dataset.download': '下载数据集',
|
||||||
|
'gpu.assign.request': '申请算力卡',
|
||||||
|
'data-process': '访问数据处理',
|
||||||
|
'data-convert': '访问数据类型转换',
|
||||||
|
'fine-tune': '访问模型训练',
|
||||||
|
'model-eval': '访问模型评测',
|
||||||
|
'model-inference': '访问模型推理',
|
||||||
|
'model-manage': '访问模型管理',
|
||||||
|
dashboard: '访问服务看板',
|
||||||
|
}
|
||||||
|
|
||||||
function userName(id?: string) {
|
function userName(id?: string) {
|
||||||
if (!id) return '系统'
|
if (!id) return '系统'
|
||||||
@@ -100,13 +144,68 @@ function tenantName(id?: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function actionName(action?: string) {
|
function actionName(action?: string) {
|
||||||
return action ? actionLabels[action] || action : '未记录'
|
if (!action) return '未记录'
|
||||||
|
if (actionLabels[action]) return actionLabels[action]
|
||||||
|
if (actionObjectLabels[action]) return actionObjectLabels[action]
|
||||||
|
const parts = action.split('.')
|
||||||
|
const verb = actionVerbLabels[parts[parts.length - 1]]
|
||||||
|
const object = targetTypeName(parts.slice(0, -1).join('_'))
|
||||||
|
return verb ? verb + object : '其他系统操作'
|
||||||
}
|
}
|
||||||
|
|
||||||
function targetTypeName(type?: string) {
|
function targetTypeName(type?: string) {
|
||||||
return type ? targetTypeLabels[type] || type : '未指定'
|
return type ? targetTypeLabels[type] || type : '未指定'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resultName(result?: string) {
|
||||||
|
if (result === 'success') return '成功'
|
||||||
|
if (result === 'denied') return '已拒绝'
|
||||||
|
if (result === 'failure') return '失败'
|
||||||
|
return result || '已记录'
|
||||||
|
}
|
||||||
|
|
||||||
|
function resultTagType(result?: string) {
|
||||||
|
if (result === 'success') return 'success'
|
||||||
|
if (result === 'denied' || result === 'failure') return 'danger'
|
||||||
|
return 'info'
|
||||||
|
}
|
||||||
|
|
||||||
|
function detailDescription(row: AuditLog) {
|
||||||
|
const target = row.target_id
|
||||||
|
? '目标为“' + targetTypeName(row.target_type) + ' ' + row.target_id + '”'
|
||||||
|
: '对象为“' + targetTypeName(row.target_type) + '”'
|
||||||
|
const raw = String(row.detail || '').trim()
|
||||||
|
const translated = raw
|
||||||
|
.replace(/user tombstoned; sessions, memberships and grants revoked/gi, '用户已停用,同时撤销会话、租户成员关系和权限授权')
|
||||||
|
.replace(/module visit/gi, '访问平台业务模块')
|
||||||
|
.replace(/dataset bundle download/gi, '下载数据集文件包')
|
||||||
|
.replace(/evaluation report download/gi, '下载模型评测报告')
|
||||||
|
.replace(/approval decision/gi, '处理审批结果')
|
||||||
|
.replace(/resource request/gi, '提交资源申请')
|
||||||
|
.replace(/engine=merge/gi, '执行权重合并')
|
||||||
|
.replace(/engine=export/gi, '执行模型导出')
|
||||||
|
return actionName(row.action) + ',' + target + ',结果:' + resultName(row.result) + (translated ? ';' + translated : '')
|
||||||
|
}
|
||||||
|
|
||||||
|
function metadataText(value?: string | Record<string, unknown>) {
|
||||||
|
if (!value) return '无'
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
try { return JSON.stringify(JSON.parse(value), null, 2) } catch { return value }
|
||||||
|
}
|
||||||
|
return JSON.stringify(value, null, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
function showDetail(row: AuditLog) {
|
||||||
|
detailLog.value = row
|
||||||
|
detailVisible.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(value?: string) {
|
||||||
|
if (!value) return '-'
|
||||||
|
const date = new Date(value)
|
||||||
|
return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false })
|
||||||
|
}
|
||||||
|
|
||||||
function applyTimeRange() {
|
function applyTimeRange() {
|
||||||
if (timeRange.value && timeRange.value.length === 2) {
|
if (timeRange.value && timeRange.value.length === 2) {
|
||||||
query.start_time = timeRange.value[0]
|
query.start_time = timeRange.value[0]
|
||||||
@@ -169,8 +268,11 @@ onMounted(() => {
|
|||||||
<template>
|
<template>
|
||||||
<div class="page">
|
<div class="page">
|
||||||
<div class="page-header">
|
<div class="page-header">
|
||||||
<h2 class="page-title">审计日志</h2>
|
<div>
|
||||||
<el-button @click="handleExport">导出 CSV</el-button>
|
<h2 class="page-title">审计记录</h2>
|
||||||
|
<p class="page-subtitle">记录管理员和系统关键操作,支持按用户、资源、结果和来源地址追溯。</p>
|
||||||
|
</div>
|
||||||
|
<el-button type="primary" @click="handleExport">导出 CSV</el-button>
|
||||||
</div>
|
</div>
|
||||||
<el-card class="filter-card">
|
<el-card class="filter-card">
|
||||||
<el-form :inline="true" class="filter-form">
|
<el-form :inline="true" class="filter-form">
|
||||||
@@ -195,7 +297,7 @@ onMounted(() => {
|
|||||||
</el-select>
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="关键词">
|
<el-form-item label="关键词">
|
||||||
<el-input v-model="query.keyword" placeholder="资源 ID 或详情" clearable style="width: 220px" />
|
<el-input v-model="query.keyword" placeholder="资源名称、操作说明或 ID" clearable style="width: 240px" @keyup.enter="load" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="目标 ID">
|
<el-form-item label="目标 ID">
|
||||||
<el-input v-model="query.target_id" placeholder="精确查询,可选" clearable style="width: 180px" />
|
<el-input v-model="query.target_id" placeholder="精确查询,可选" clearable style="width: 180px" />
|
||||||
@@ -220,7 +322,9 @@ onMounted(() => {
|
|||||||
</el-form>
|
</el-form>
|
||||||
</el-card>
|
</el-card>
|
||||||
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
|
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
|
||||||
<el-table-column prop="time" label="时间" min-width="180" />
|
<el-table-column label="时间" min-width="170">
|
||||||
|
<template #default="{ row }">{{ formatTime(row.time) }}</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="租户" min-width="140">
|
<el-table-column label="租户" min-width="140">
|
||||||
<template #default="{ row }">{{ tenantName(row.tenant_id) }}</template>
|
<template #default="{ row }">{{ tenantName(row.tenant_id) }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
@@ -233,11 +337,45 @@ onMounted(() => {
|
|||||||
<el-table-column label="目标类型" min-width="120">
|
<el-table-column label="目标类型" min-width="120">
|
||||||
<template #default="{ row }">{{ targetTypeName(row.target_type) }}</template>
|
<template #default="{ row }">{{ targetTypeName(row.target_type) }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="target_id" label="目标 ID" min-width="140" show-overflow-tooltip />
|
<el-table-column label="操作说明" min-width="300" show-overflow-tooltip>
|
||||||
<el-table-column prop="detail" label="详情" min-width="200" show-overflow-tooltip />
|
<template #default="{ row }">{{ detailDescription(row as AuditLog) }}</template>
|
||||||
<el-table-column prop="client_ip" label="IP" min-width="120" />
|
</el-table-column>
|
||||||
|
<el-table-column label="结果" width="90" align="center">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-tag :type="resultTagType(row.result)" size="small">{{ resultName(row.result) }}</el-tag>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="来源 IP" width="130" prop="client_ip" />
|
||||||
|
<el-table-column label="操作" width="80" fixed="right">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-button link type="primary" @click="showDetail(row as AuditLog)">详情</el-button>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
<div class="pager">共 {{ total }} 条</div>
|
<div class="pager">
|
||||||
|
<span>共 {{ total }} 条</span>
|
||||||
|
<el-pagination background layout="total, prev, pager, next" :total="total" :page-size="query.limit" :current-page="Math.floor((query.offset || 0) / (query.limit || 50)) + 1" @current-change="(page: number) => { query.offset = (page - 1) * (query.limit || 50); void load() }" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<el-dialog v-model="detailVisible" title="审计记录详情" width="820px">
|
||||||
|
<el-descriptions v-if="detailLog" :column="2" border>
|
||||||
|
<el-descriptions-item label="发生时间">{{ formatTime(detailLog.time) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="结果">
|
||||||
|
<el-tag :type="resultTagType(detailLog.result)" size="small">{{ resultName(detailLog.result) }}</el-tag>
|
||||||
|
</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="操作人">{{ userName(detailLog.actor_id) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="所属租户">{{ tenantName(detailLog.tenant_id) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="来源 IP">{{ detailLog.client_ip || '未记录' }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="请求 ID">{{ detailLog.request_id || '未记录' }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="操作">{{ actionName(detailLog.action) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="资源类型">{{ targetTypeName(detailLog.target_type) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="目标 ID">{{ detailLog.target_id || '未指定' }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="中文说明" :span="2">{{ detailDescription(detailLog) }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="失败原因" :span="2">{{ detailLog.reason || '无' }}</el-descriptions-item>
|
||||||
|
<el-descriptions-item label="原始详情" :span="2"><pre class="detail-box">{{ detailLog.detail || '无' }}</pre></el-descriptions-item>
|
||||||
|
<el-descriptions-item label="扩展信息" :span="2"><pre class="detail-box">{{ metadataText(detailLog.metadata) }}</pre></el-descriptions-item>
|
||||||
|
</el-descriptions>
|
||||||
|
</el-dialog>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -245,8 +383,10 @@ onMounted(() => {
|
|||||||
.page { padding: 16px; }
|
.page { padding: 16px; }
|
||||||
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
|
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
|
||||||
.page-title { margin: 0; font-size: 18px; }
|
.page-title { margin: 0; font-size: 18px; }
|
||||||
|
.page-subtitle { margin: 6px 0 0; color: #909399; font-size: 13px; }
|
||||||
.filter-card { margin-bottom: 16px; }
|
.filter-card { margin-bottom: 16px; }
|
||||||
.filter-form { display: flex; flex-wrap: wrap; }
|
.filter-form { display: flex; flex-wrap: wrap; }
|
||||||
.log-table { margin-top: 8px; }
|
.log-table { margin-top: 8px; }
|
||||||
.pager { margin-top: 12px; text-align: right; color: #909399; }
|
.pager { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 12px; color: #909399; }
|
||||||
|
.detail-box { max-height: 180px; margin: 0; padding: 8px 10px; overflow: auto; white-space: pre-wrap; word-break: break-all; color: #606266; background: #f5f7fa; border-radius: 4px; font-size: 12px; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, reactive, ref, computed } from 'vue'
|
import { onMounted, reactive, ref, computed } from 'vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { useAuthStore } from '@/stores/auth'
|
||||||
import {
|
import {
|
||||||
getOperationLogs,
|
getOperationLogs,
|
||||||
getOperationLogStats,
|
getOperationLogStats,
|
||||||
@@ -10,17 +11,19 @@ import {
|
|||||||
} from '@/api/modules/operation-log'
|
} from '@/api/modules/operation-log'
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
const auth = useAuthStore()
|
||||||
|
const isAdmin = computed(() => auth.isAdmin)
|
||||||
const statsLoading = ref(false)
|
const statsLoading = ref(false)
|
||||||
const logs = ref<OperationLog[]>([])
|
const logs = ref<OperationLog[]>([])
|
||||||
const total = ref(0)
|
const total = ref(0)
|
||||||
const stats = ref<OperationLogStats | null>(null)
|
const stats = ref<OperationLogStats | null>(null)
|
||||||
|
|
||||||
// 默认筛选:只看失败
|
// 管理员默认查看完整操作流,失败记录通过状态筛选快速定位。
|
||||||
const query = reactive<OperationLogQuery>({
|
const query = reactive<OperationLogQuery>({
|
||||||
user_id: '',
|
user_id: '',
|
||||||
module: '',
|
module: '',
|
||||||
action: '',
|
action: '',
|
||||||
status: 'failure',
|
status: '',
|
||||||
keyword: '',
|
keyword: '',
|
||||||
start_time: '',
|
start_time: '',
|
||||||
end_time: '',
|
end_time: '',
|
||||||
@@ -143,6 +146,12 @@ function actionLabel(action?: string) {
|
|||||||
return actionOptions.find((a) => a.value === action)?.label || action || '-'
|
return actionOptions.find((a) => a.value === action)?.label || action || '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function operationDescription(row: OperationLog) {
|
||||||
|
const target = row.target_name || row.target_id
|
||||||
|
const suffix = target ? ',对象为“' + target + '”' : ''
|
||||||
|
return moduleLabel(row.module) + ':' + actionLabel(row.action) + suffix + ',结果:' + (row.status === 'success' ? '成功' : '失败')
|
||||||
|
}
|
||||||
|
|
||||||
function toggleOnlyFailures() {
|
function toggleOnlyFailures() {
|
||||||
query.status = onlyFailures.value ? '' : 'failure'
|
query.status = onlyFailures.value ? '' : 'failure'
|
||||||
handleSearch()
|
handleSearch()
|
||||||
@@ -157,7 +166,7 @@ onMounted(() => {
|
|||||||
<template>
|
<template>
|
||||||
<div class="page">
|
<div class="page">
|
||||||
<div class="page-header">
|
<div class="page-header">
|
||||||
<h2 class="page-title">系统操作日志</h2>
|
<h2 class="page-title">{{ isAdmin ? '系统操作日志' : '我的操作日志' }}</h2>
|
||||||
<el-button :type="onlyFailures ? 'danger' : 'default'" @click="toggleOnlyFailures">
|
<el-button :type="onlyFailures ? 'danger' : 'default'" @click="toggleOnlyFailures">
|
||||||
{{ onlyFailures ? '只看失败 ✅' : '显示全部' }}
|
{{ onlyFailures ? '只看失败 ✅' : '显示全部' }}
|
||||||
</el-button>
|
</el-button>
|
||||||
@@ -237,7 +246,7 @@ onMounted(() => {
|
|||||||
@keyup.enter="handleSearch"
|
@keyup.enter="handleSearch"
|
||||||
/>
|
/>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="用户">
|
<el-form-item v-if="isAdmin" label="用户">
|
||||||
<el-input v-model="query.user_id" placeholder="用户ID/用户名" clearable style="width: 140px" @keyup.enter="handleSearch" />
|
<el-input v-model="query.user_id" placeholder="用户ID/用户名" clearable style="width: 140px" @keyup.enter="handleSearch" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="时间范围">
|
<el-form-item label="时间范围">
|
||||||
@@ -273,7 +282,7 @@ onMounted(() => {
|
|||||||
{{ row.create_time ? new Date(row.create_time).toLocaleString('zh-CN') : '-' }}
|
{{ row.create_time ? new Date(row.create_time).toLocaleString('zh-CN') : '-' }}
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="用户" width="110" align="center" show-overflow-tooltip>
|
<el-table-column v-if="isAdmin" label="用户" width="110" align="center" show-overflow-tooltip>
|
||||||
<template #default="{ row }">{{ row.username || row.user_id || '-' }}</template>
|
<template #default="{ row }">{{ row.username || row.user_id || '-' }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="模块" width="110" align="center">
|
<el-table-column label="模块" width="110" align="center">
|
||||||
@@ -285,6 +294,9 @@ onMounted(() => {
|
|||||||
<el-table-column label="目标" min-width="140" align="center" show-overflow-tooltip>
|
<el-table-column label="目标" min-width="140" align="center" show-overflow-tooltip>
|
||||||
<template #default="{ row }">{{ row.target_name || row.target_id || '-' }}</template>
|
<template #default="{ row }">{{ row.target_name || row.target_id || '-' }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
|
<el-table-column label="来源 IP" width="125" align="center">
|
||||||
|
<template #default="{ row }">{{ row.client_ip || '未记录' }}</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="状态" width="80" align="center">
|
<el-table-column label="状态" width="80" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-tag :type="statusTagType(row.status)" size="small" effect="dark">
|
<el-tag :type="statusTagType(row.status)" size="small" effect="dark">
|
||||||
@@ -369,6 +381,7 @@ onMounted(() => {
|
|||||||
<span v-else style="color: #c0c4cc">无堆栈信息</span>
|
<span v-else style="color: #c0c4cc">无堆栈信息</span>
|
||||||
</el-descriptions-item>
|
</el-descriptions-item>
|
||||||
<el-descriptions-item label="操作详情" :span="2">
|
<el-descriptions-item label="操作详情" :span="2">
|
||||||
|
<div class="detail-summary">{{ operationDescription(detailLog) }}</div>
|
||||||
<pre v-if="detailLog.detail" class="detail-box">{{ detailLog.detail }}</pre>
|
<pre v-if="detailLog.detail" class="detail-box">{{ detailLog.detail }}</pre>
|
||||||
<span v-else style="color: #c0c4cc">-</span>
|
<span v-else style="color: #c0c4cc">-</span>
|
||||||
</el-descriptions-item>
|
</el-descriptions-item>
|
||||||
@@ -428,6 +441,15 @@ onMounted(() => {
|
|||||||
white-space: pre-wrap;
|
white-space: pre-wrap;
|
||||||
word-break: break-all;
|
word-break: break-all;
|
||||||
}
|
}
|
||||||
|
.detail-summary {
|
||||||
|
margin-bottom: 8px;
|
||||||
|
padding: 8px 10px;
|
||||||
|
color: #303133;
|
||||||
|
background: #f0f9ff;
|
||||||
|
border-left: 3px solid #409eff;
|
||||||
|
border-radius: 3px;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
:deep(.failure-row) {
|
:deep(.failure-row) {
|
||||||
background-color: #fef0f0 !important;
|
background-color: #fef0f0 !important;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -201,6 +201,12 @@ onMounted(load)
|
|||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column prop="output_filename" label="输出文件名" min-width="160" />
|
<el-table-column prop="output_filename" label="输出文件名" min-width="160" />
|
||||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||||
|
<el-table-column label="上传人" min-width="130" show-overflow-tooltip>
|
||||||
|
<template #default="{ row }">{{ row.creator_name || row.created_by || '-' }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="处理人" min-width="130" show-overflow-tooltip>
|
||||||
|
<template #default="{ row }">{{ row.processor_name || row.processed_by || '-' }}</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="操作" width="240" fixed="right">
|
<el-table-column label="操作" width="240" fixed="right">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-upload
|
<el-upload
|
||||||
|
|||||||
@@ -148,6 +148,12 @@ onMounted(() => void loadData())
|
|||||||
<el-table-column label="创建时间" align="center" width="190">
|
<el-table-column label="创建时间" align="center" width="190">
|
||||||
<template #default="{ row }">{{ formatDateTime(row.create_time || row.created_at) }}</template>
|
<template #default="{ row }">{{ formatDateTime(row.create_time || row.created_at) }}</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
|
<el-table-column label="上传/创建人" align="center" width="150" show-overflow-tooltip>
|
||||||
|
<template #default="{ row }">{{ row.creator_name || row.created_by || '-' }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="最后处理人" align="center" width="150" show-overflow-tooltip>
|
||||||
|
<template #default="{ row }">{{ row.processor_name || row.updated_by || '-' }}</template>
|
||||||
|
</el-table-column>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
|
|||||||
@@ -267,6 +267,11 @@ onMounted(loadData)
|
|||||||
{{ row.create_time ? new Date(row.create_time).toLocaleString('zh-CN') : '-' }}
|
{{ row.create_time ? new Date(row.create_time).toLocaleString('zh-CN') : '-' }}
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
|
<el-table-column label="上传人" align="center" width="140" show-overflow-tooltip>
|
||||||
|
<template #default="{ row }">
|
||||||
|
{{ row.creator_name || row.created_by || '-' }}
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template #actions="{ row }">
|
<template #actions="{ row }">
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import EvalTaskSetupStep, { type EvalTaskSetupDraft } from './create/EvalTaskSet
|
|||||||
import EvalRuleSetupStep, { type EvalRuleSetupDraft } from './create/EvalRuleSetupStep.vue'
|
import EvalRuleSetupStep, { type EvalRuleSetupDraft } from './create/EvalRuleSetupStep.vue'
|
||||||
import BasicMetricSetupStep, { type BasicMetricSetupDraft } from './create/BasicMetricSetupStep.vue'
|
import BasicMetricSetupStep, { type BasicMetricSetupDraft } from './create/BasicMetricSetupStep.vue'
|
||||||
import StartEvalStep from './create/StartEvalStep.vue'
|
import StartEvalStep from './create/StartEvalStep.vue'
|
||||||
import { createDimension, startEval } from '@/api/modules/eval'
|
import { startEval } from '@/api/modules/eval'
|
||||||
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
||||||
import { getDatasetList } from '@/api/modules/dataset'
|
import { getDatasetList } from '@/api/modules/dataset'
|
||||||
import { getComputeGpus } from '@/api/modules/compute'
|
import { getComputeGpus } from '@/api/modules/compute'
|
||||||
@@ -34,7 +34,6 @@ const trainedModels = ref<TrainedModel[]>([])
|
|||||||
const evalDatasets = ref<DatasetItem[]>([])
|
const evalDatasets = ref<DatasetItem[]>([])
|
||||||
const evalModels = ref<ModelItem[]>([])
|
const evalModels = ref<ModelItem[]>([])
|
||||||
const gpus = ref<GpuInfo[]>([])
|
const gpus = ref<GpuInfo[]>([])
|
||||||
const createdDimensionId = ref<string | number>('')
|
|
||||||
|
|
||||||
const taskForm = ref<EvalTaskSetupDraft>({
|
const taskForm = ref<EvalTaskSetupDraft>({
|
||||||
eval_task_name: '',
|
eval_task_name: '',
|
||||||
@@ -69,14 +68,6 @@ const basicMetricForm = ref<BasicMetricSetupDraft>({
|
|||||||
output_precision: 3,
|
output_precision: 3,
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(
|
|
||||||
ruleForm,
|
|
||||||
() => {
|
|
||||||
createdDimensionId.value = ''
|
|
||||||
},
|
|
||||||
{ deep: true },
|
|
||||||
)
|
|
||||||
|
|
||||||
async function loadData() {
|
async function loadData() {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
const results = await Promise.allSettled([
|
const results = await Promise.allSettled([
|
||||||
@@ -127,22 +118,11 @@ function buildDimensionPayload() {
|
|||||||
return payload
|
return payload
|
||||||
}
|
}
|
||||||
|
|
||||||
async function resolveDimensionId() {
|
|
||||||
if (createdDimensionId.value !== '') return createdDimensionId.value
|
|
||||||
|
|
||||||
const created = await createDimension(buildDimensionPayload())
|
|
||||||
if (created?.id === undefined || created.id === null || created.id === '') {
|
|
||||||
throw new Error('评测维度已提交,但未返回维度 ID,无法启动评测任务')
|
|
||||||
}
|
|
||||||
createdDimensionId.value = created.id
|
|
||||||
return created.id
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleSubmit() {
|
async function handleSubmit() {
|
||||||
if (loading.value || submitting.value) return
|
if (loading.value || submitting.value) return
|
||||||
submitting.value = true
|
submitting.value = true
|
||||||
try {
|
try {
|
||||||
const dimensionId = await resolveDimensionId()
|
const dimension = buildDimensionPayload()
|
||||||
// GPU 选择为「节点:GPU序号」复合值,解析出节点与 GPU 序号,
|
// GPU 选择为「节点:GPU序号」复合值,解析出节点与 GPU 序号,
|
||||||
// 多算力节点时必须把节点信息传给后端,否则会派发到错误的算力节点
|
// 多算力节点时必须把节点信息传给后端,否则会派发到错误的算力节点
|
||||||
const selectedGpuKeys = Array.isArray(taskForm.value.gpu_id)
|
const selectedGpuKeys = Array.isArray(taskForm.value.gpu_id)
|
||||||
@@ -165,7 +145,7 @@ async function handleSubmit() {
|
|||||||
gpus: gpuIndices,
|
gpus: gpuIndices,
|
||||||
compute_node_id: gpuNodeId || '',
|
compute_node_id: gpuNodeId || '',
|
||||||
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
||||||
dimension_id: dimensionId,
|
dimension,
|
||||||
data_source: taskForm.value.data_source,
|
data_source: taskForm.value.data_source,
|
||||||
leaderboard: taskForm.value.leaderboard,
|
leaderboard: taskForm.value.leaderboard,
|
||||||
basic_metrics: {
|
basic_metrics: {
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
import { useRoute } from 'vue-router'
|
import { useRoute } from 'vue-router'
|
||||||
|
import VChart from 'vue-echarts'
|
||||||
|
import '@/plugins/echarts'
|
||||||
|
import type { EChartsOption } from 'echarts'
|
||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||||
import { getEvalDetail } from '@/api/modules/eval'
|
import { getEvalDetail } from '@/api/modules/eval'
|
||||||
@@ -52,6 +55,78 @@ const passRate = computed(() => {
|
|||||||
const overallScore = computed(() => formatScore(detail.value?.overall_score, detail.value?.overall_score_max))
|
const overallScore = computed(() => formatScore(detail.value?.overall_score, detail.value?.overall_score_max))
|
||||||
const displayModelName = computed(() => detail.value?.model_name || String(detail.value?.model_id || '-'))
|
const displayModelName = computed(() => detail.value?.model_name || String(detail.value?.model_id || '-'))
|
||||||
const displayMetric = computed(() => detail.value?.metric_label || detail.value?.metric || '-')
|
const displayMetric = computed(() => detail.value?.metric_label || detail.value?.metric || '-')
|
||||||
|
const progressDetail = computed(() => detail.value?.progress_detail)
|
||||||
|
const progressTotal = computed(() => Math.max(
|
||||||
|
Number(detail.value?.sample_count || 0),
|
||||||
|
Number(progressDetail.value?.total || 0),
|
||||||
|
Number(detail.value?.samples?.length || 0),
|
||||||
|
))
|
||||||
|
const progressCompleted = computed(() => detail.value?.status === 'completed'
|
||||||
|
? progressTotal.value
|
||||||
|
: Math.min(progressTotal.value || Number(detail.value?.completed_count || 0), Number(progressDetail.value?.completed ?? detail.value?.completed_count ?? 0)))
|
||||||
|
const progressPercentage = computed(() => detail.value?.status === 'completed'
|
||||||
|
? 100
|
||||||
|
: detail.value?.status === 'failed' || detail.value?.status === 'stopped'
|
||||||
|
? Math.max(0, Math.min(100, Math.round(Number(progressDetail.value?.percentage ?? detail.value?.progress ?? completionRate.value))))
|
||||||
|
: Math.max(0, Math.min(100, Math.round(Number(progressDetail.value?.percentage ?? completionRate.value)))))
|
||||||
|
const progressStage = computed(() => ({
|
||||||
|
dataset: '准备数据集',
|
||||||
|
model_loading: '加载模型',
|
||||||
|
inference: '生成回答',
|
||||||
|
metrics: '计算指标',
|
||||||
|
completed: '评测完成',
|
||||||
|
failed: '评测失败',
|
||||||
|
}[detail.value?.status === 'completed' ? 'completed' : String(progressDetail.value?.stage || '')] || (detail.value?.status === 'running' ? '任务运行中' : '等待开始')))
|
||||||
|
|
||||||
|
const radarDimensions = computed(() => {
|
||||||
|
const dimensions = new Map<string, { name: string; value: number }>()
|
||||||
|
for (const item of detail.value?.dimension_summary || []) {
|
||||||
|
if (item.available === false || !Number.isFinite(Number(item.score))) continue
|
||||||
|
dimensions.set(item.name, {
|
||||||
|
name: item.name,
|
||||||
|
value: Math.max(0, Math.min(100, Number(item.score) / Math.max(Number(item.max_score) || 100, 1) * 100)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const metricLabels: Record<string, string> = {
|
||||||
|
bleu: 'BLEU',
|
||||||
|
rouge: 'ROUGE-L',
|
||||||
|
cosine: 'Cosine 相似度',
|
||||||
|
exact_match: '精确匹配',
|
||||||
|
text_similarity: '文本相似度',
|
||||||
|
}
|
||||||
|
for (const [key, metric] of Object.entries(detail.value?.basic_metrics || {})) {
|
||||||
|
const score = Number(metric?.score)
|
||||||
|
if (!Number.isFinite(score) || metric?.available === false) continue
|
||||||
|
const name = metricLabels[key] || key
|
||||||
|
if (!dimensions.has(name)) dimensions.set(name, { name, value: Math.max(0, Math.min(100, score)) })
|
||||||
|
}
|
||||||
|
return [...dimensions.values()]
|
||||||
|
})
|
||||||
|
|
||||||
|
const radarOption = computed<EChartsOption>(() => ({
|
||||||
|
tooltip: { trigger: 'item' },
|
||||||
|
radar: {
|
||||||
|
indicator: radarDimensions.value.map((item) => ({ name: item.name, max: 100 })),
|
||||||
|
radius: '62%',
|
||||||
|
splitNumber: 4,
|
||||||
|
axisName: { color: '#667085', fontSize: 11 },
|
||||||
|
splitArea: { areaStyle: { color: ['#fbfbfd', '#f2f4f8'] } },
|
||||||
|
splitLine: { lineStyle: { color: '#e4e7ec' } },
|
||||||
|
axisLine: { lineStyle: { color: '#e4e7ec' } },
|
||||||
|
},
|
||||||
|
series: [{
|
||||||
|
type: 'radar',
|
||||||
|
symbol: 'circle',
|
||||||
|
symbolSize: 4,
|
||||||
|
data: [{
|
||||||
|
value: radarDimensions.value.map((item) => item.value),
|
||||||
|
name: '评测指标',
|
||||||
|
areaStyle: { color: 'rgba(79, 70, 229, 0.18)' },
|
||||||
|
lineStyle: { color: '#4f46e5', width: 2 },
|
||||||
|
itemStyle: { color: '#4f46e5' },
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
}))
|
||||||
|
|
||||||
function formatDateTime(value?: string) {
|
function formatDateTime(value?: string) {
|
||||||
if (!value) return '-'
|
if (!value) return '-'
|
||||||
@@ -152,8 +227,9 @@ onUnmounted(stopPolling)
|
|||||||
</div>
|
</div>
|
||||||
<div class="overview-item">
|
<div class="overview-item">
|
||||||
<span>评测进度</span>
|
<span>评测进度</span>
|
||||||
<strong>{{ detail.completed_count }} / {{ detail.sample_count }}</strong>
|
<strong>{{ progressCompleted }} / {{ progressTotal }}</strong>
|
||||||
<el-progress :percentage="completionRate" :show-text="false" :stroke-width="5" />
|
<el-progress :percentage="progressPercentage" :show-text="false" :stroke-width="5" />
|
||||||
|
<small>{{ progressStage }}{{ progressDetail?.message ? ' · ' + progressDetail.message : '' }}</small>
|
||||||
</div>
|
</div>
|
||||||
<div class="overview-item overview-time">
|
<div class="overview-item overview-time">
|
||||||
<span>{{ detail.completed_time ? '完成时间' : '创建时间' }}</span>
|
<span>{{ detail.completed_time ? '完成时间' : '创建时间' }}</span>
|
||||||
@@ -196,6 +272,9 @@ onUnmounted(stopPolling)
|
|||||||
<p>查看各评测指标的得分与通过率</p>
|
<p>查看各评测指标的得分与通过率</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="dimension-layout">
|
||||||
|
<VChart v-if="radarDimensions.length >= 3" class="evaluation-radar" :option="radarOption" autoresize />
|
||||||
|
<div v-else class="radar-fallback">可用指标少于 3 个,暂以指标明细展示。</div>
|
||||||
<div class="dimension-grid">
|
<div class="dimension-grid">
|
||||||
<div v-for="dimension in detail.dimension_summary" :key="dimension.name" class="dimension-item">
|
<div v-for="dimension in detail.dimension_summary" :key="dimension.name" class="dimension-item">
|
||||||
<div class="dimension-label">
|
<div class="dimension-label">
|
||||||
@@ -203,6 +282,8 @@ onUnmounted(stopPolling)
|
|||||||
<span>{{ formatScore(dimension.score, dimension.max_score) }}</span>
|
<span>{{ formatScore(dimension.score, dimension.max_score) }}</span>
|
||||||
</div>
|
</div>
|
||||||
<el-progress :percentage="dimension.pass_rate" :stroke-width="7" />
|
<el-progress :percentage="dimension.pass_rate" :stroke-width="7" />
|
||||||
|
<small v-if="dimension.available === false" class="metric-error">{{ dimension.error || '指标依赖不可用' }}</small>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
@@ -491,6 +572,41 @@ onUnmounted(stopPolling)
|
|||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dimension-layout {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(260px, 0.85fr) minmax(0, 1.6fr);
|
||||||
|
gap: 16px;
|
||||||
|
align-items: stretch;
|
||||||
|
margin-top: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.evaluation-radar,
|
||||||
|
.radar-fallback {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 280px;
|
||||||
|
border: 1px solid #ebeef5;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: #fafafa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.radar-fallback {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 20px;
|
||||||
|
color: #909399;
|
||||||
|
font-size: 13px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.metric-error {
|
||||||
|
display: block;
|
||||||
|
margin-top: 6px;
|
||||||
|
color: #e6a23c;
|
||||||
|
font-size: 11px;
|
||||||
|
line-height: 1.4;
|
||||||
|
}
|
||||||
|
|
||||||
.dimension-item {
|
.dimension-item {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
padding: 14px 16px;
|
padding: 14px 16px;
|
||||||
@@ -665,6 +781,10 @@ onUnmounted(stopPolling)
|
|||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dimension-layout {
|
||||||
|
grid-template-columns: minmax(0, 1fr);
|
||||||
|
}
|
||||||
|
|
||||||
.dimension-item:nth-child(2) {
|
.dimension-item:nth-child(2) {
|
||||||
border-right: 0;
|
border-right: 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,6 +65,20 @@ function displayMetric(row: Partial<EvalTask>) {
|
|||||||
return row.metric_label || row.metric || '-'
|
return row.metric_label || row.metric || '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function progressPercentage(row: Partial<EvalTask>) {
|
||||||
|
if (row.status === 'completed') return 100
|
||||||
|
return Math.max(0, Math.min(100, Math.round(Number(row.progress_detail?.percentage ?? row.progress ?? 0))))
|
||||||
|
}
|
||||||
|
|
||||||
|
function progressCompleted(row: Partial<EvalTask>) {
|
||||||
|
if (row.status === 'completed') return row.progress_detail?.total ?? '-'
|
||||||
|
return row.progress_detail?.completed ?? 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function progressTotal(row: Partial<EvalTask>) {
|
||||||
|
return row.progress_detail?.total ?? '-'
|
||||||
|
}
|
||||||
|
|
||||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||||
async () => {
|
async () => {
|
||||||
await loadEvalList({ silent: true })
|
await loadEvalList({ silent: true })
|
||||||
@@ -125,7 +139,17 @@ onUnmounted(() => {
|
|||||||
</el-tooltip>
|
</el-tooltip>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="评分" prop="score" width="100" align="center" />
|
<el-table-column label="评分" prop="score" width="100" align="center">
|
||||||
|
<template #default="{ row }">
|
||||||
|
{{ row.score == null ? '-' : `${Number(row.score).toFixed(2)} / 100` }}
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="评测进度" width="130" align="center">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-progress :percentage="progressPercentage(row)" :stroke-width="6" :show-text="false" />
|
||||||
|
<small>{{ progressCompleted(row) }} / {{ progressTotal(row) }}</small>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column label="状态" width="100" align="center">
|
<el-table-column label="状态" width="100" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<ModelStatusTag :status="row.status" />
|
<ModelStatusTag :status="row.status" />
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ const availableGpus = computed(() => {
|
|||||||
)
|
)
|
||||||
result = result.filter((gpu) => {
|
result = result.filter((gpu) => {
|
||||||
const key = `${gpu.node_id}:${gpu.id ?? gpu.uuid ?? gpu.name}`
|
const key = `${gpu.node_id}:${gpu.id ?? gpu.uuid ?? gpu.name}`
|
||||||
return assignedKeys.has(key) || myAssignedGpus.value.length === 0
|
// 未完成算力审批时不展示未授权 GPU,避免接口失败时回退为全量资源。
|
||||||
|
return assignedKeys.has(key)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user