Compare commits
26 Commits
f97245b814
...
baseline/f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec7d8c0a3d | ||
|
|
0292bf5138 | ||
|
|
0271942ba5 | ||
|
|
250e060271 | ||
|
|
7b36bc774e | ||
|
|
4e5c43fad5 | ||
|
|
62a1d03eac | ||
|
|
94230cad16 | ||
|
|
0c601934a0 | ||
|
|
5cc306eb0a | ||
|
|
cc08b164d0 | ||
|
|
24c77a990a | ||
|
|
15c4223f2c | ||
|
|
b975de02da | ||
|
|
46d343fb63 | ||
|
|
0c39f2f5b9 | ||
|
|
c7c9ed925b | ||
|
|
f917a025e1 | ||
|
|
a9ab130d43 | ||
|
|
525fc55cef | ||
|
|
3fd9cf9100 | ||
|
|
a72e2a2520 | ||
|
|
01d2e6c76a | ||
|
|
8a6a6574bb | ||
|
|
9025437a37 | ||
|
|
4623e3fa1c |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -12,8 +12,6 @@ __pycache__/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
!frontend/dist/
|
||||
!frontend/dist/**
|
||||
node_modules/
|
||||
*.tsbuildinfo
|
||||
downloads/
|
||||
@@ -159,6 +157,8 @@ backend/config.yaml
|
||||
.codex-backups/
|
||||
.pnpm-store/
|
||||
.zcode/
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
||||
39
README.md
39
README.md
@@ -134,12 +134,45 @@ npm run dev
|
||||
|
||||
## 算力服务启动
|
||||
|
||||
算力服务是一个 FastAPI 应用,同时承载 Compute API(模型训练/推理/GPU 管理)和 File Gateway(文件上传下载)路由。Docker 部署时对外暴露两个端口(19100 和 19101)均指向同一服务,方便应用平台分别配置 `api_base_url` 和 `file_gateway_url`。本地开发只需启动一个进程。
|
||||
|
||||
### 方式一:Docker 启动(推荐)
|
||||
|
||||
```bash
|
||||
cd compute
|
||||
uvicorn api.main:app --reload --port 19100
|
||||
cd docker/compute
|
||||
cp .env.example .env
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
默认 `COMPUTE_MODE=real`。真实 GPU 接入时,在每台算力服务器上部署 Compute API、Agent、File Gateway 和 LLaMA-Factory,应用平台通过 `compute_nodes.api_base_url` 和 `compute_nodes.file_gateway_url` 主动轮询。仅在隔离联调环境可显式设置 `COMPUTE_MODE=simulator` 或 `COMPUTE_EXECUTION_MODE=simulator`。
|
||||
### 方式二:本地开发启动
|
||||
|
||||
**Windows (cmd):**
|
||||
|
||||
```cmd
|
||||
cd /d E:\yg_ft\compute
|
||||
set PYTHONPATH=E:\yg_ft
|
||||
.\.venv\Scripts\python.exe -m uvicorn api.main:app --reload --port 19100
|
||||
```
|
||||
|
||||
> `PYTHONPATH=E:\yg_ft` 是必需的,因为代码使用 `from compute.agent...` 绝对导入。
|
||||
|
||||
**Linux / macOS:**
|
||||
|
||||
```bash
|
||||
cd compute
|
||||
PYTHONPATH=.. uvicorn api.main:app --reload --port 19100
|
||||
```
|
||||
|
||||
### 环境变量说明
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `COMPUTE_MODE` | `real` | `real` / `simulator`,仅隔离联调用 simulator |
|
||||
| `COMPUTE_EXECUTION_MODE` | `real` | 训练执行模式 |
|
||||
| `COMPUTE_SERVICE_TOKEN` | `change_me` | 服务间认证 token |
|
||||
| `MODELTF_ROUTE_PREFIX` | `/modelTF` | API 路由前缀 |
|
||||
|
||||
应用平台通过数据库 `compute_nodes` 表中的 `api_base_url` 和 `file_gateway_url` 主动轮询算力节点状态。
|
||||
|
||||
## 日志
|
||||
|
||||
|
||||
10
backend/_check_sessions.py
Normal file
10
backend/_check_sessions.py
Normal file
@@ -0,0 +1,10 @@
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, user_id, login_at, logout_at, duration_seconds FROM sessions ORDER BY login_at DESC LIMIT 10"
|
||||
).fetchall()
|
||||
print(f"sessions count: {len(rows)}")
|
||||
for r in rows:
|
||||
print(f" user={r['user_id'][:25]}... login={r['login_at']} logout={r['logout_at']} dur={r['duration_seconds']}")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,5 +10,9 @@ logger = get_logger(__name__)
|
||||
@router.get("/health")
|
||||
async def health_check() -> dict[str, object]:
|
||||
logger.info("health check requested")
|
||||
return {"code": 0, "message": "ok", "data": get_platform_store().health_metrics()}
|
||||
return {
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": get_platform_store().health_metrics(),
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,8 +3,20 @@
|
||||
from app.api.v1.endpoints.data_process import router as data_process_router
|
||||
from app.api.v1.endpoints.platform import router as platform_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
from app.modules.tenant.router import router as tenant_router
|
||||
from app.modules.project.router import router as project_router
|
||||
from app.modules.approval.router import router as approval_router
|
||||
from app.modules.system.router import router as system_router
|
||||
from app.modules.retention.router import router as retention_router
|
||||
from app.modules.resource.router import router as resource_router
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health_router, tags=["health"])
|
||||
api_router.include_router(data_process_router, tags=["data-process"])
|
||||
api_router.include_router(platform_router, tags=["platform"])
|
||||
api_router.include_router(system_router, tags=["system"])
|
||||
api_router.include_router(tenant_router, tags=["tenant"])
|
||||
api_router.include_router(project_router, tags=["project"])
|
||||
api_router.include_router(approval_router, tags=["approval"])
|
||||
api_router.include_router(retention_router, tags=["retention"])
|
||||
api_router.include_router(resource_router, tags=["resource"])
|
||||
|
||||
138
backend/app/core/auth.py
Normal file
138
backend/app/core/auth.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""鉴权依赖:从 Authorization header 解析当前用户,提供权限校验。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Depends, HTTPException, Query, Request, status
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
# 无需鉴权的路径前缀(健康检查、登录等)
|
||||
PUBLIC_PATHS = ("/health", "/login", "/system-info")
|
||||
|
||||
|
||||
def _extract_token(request: Request) -> str | None:
|
||||
"""从 Authorization header 提取 token(格式: Bearer platform-token-{user_id})。"""
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
if token.startswith("platform-token-"):
|
||||
return token[len("platform-token-"):]
|
||||
return None
|
||||
|
||||
|
||||
def get_current_user(request: Request) -> dict[str, Any]:
|
||||
"""
|
||||
FastAPI 依赖:解析当前登录用户。
|
||||
- 公开路径(/health, /login 等)直接放行,返回匿名用户。
|
||||
- 无 token 或 token 无效时抛 401。
|
||||
- admin 用户标记为超级管理员,拥有全部权限。
|
||||
"""
|
||||
path = request.url.path
|
||||
# 去掉路由前缀后判断
|
||||
for prefix in PUBLIC_PATHS:
|
||||
if path.endswith(prefix):
|
||||
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
|
||||
|
||||
user_id = _extract_token(request)
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
|
||||
|
||||
store = get_platform_store()
|
||||
for u in store.users():
|
||||
if u.get("id") == user_id:
|
||||
return u
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
|
||||
|
||||
|
||||
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""FastAPI 依赖:要求当前用户是管理员(role=admin 或 protected)。"""
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return current_user
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
|
||||
|
||||
|
||||
def is_admin(user: dict[str, Any]) -> bool:
|
||||
"""判断用户是否为管理员(admin 角色或 protected 标记)。"""
|
||||
return user.get("role") == "admin" or user.get("protected", False)
|
||||
|
||||
|
||||
def has_resource_access(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
user: dict[str, Any],
|
||||
permission: str = "read",
|
||||
) -> bool:
|
||||
"""
|
||||
检查用户对某资源是否有指定权限。
|
||||
- admin/protected 用户直接放行(旁路)。
|
||||
- 其他用户检查 acls 表中是否有对应授权。
|
||||
"""
|
||||
if user.get("role") == "admin" or user.get("protected"):
|
||||
return True
|
||||
|
||||
store = get_platform_store()
|
||||
acls = store.get_acl(resource_type, resource_id)
|
||||
user_id = user.get("id")
|
||||
user_role = user.get("role")
|
||||
|
||||
for entry in acls:
|
||||
# 按 user 授权
|
||||
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
|
||||
if _permission_covers(entry.get("permission"), permission):
|
||||
return True
|
||||
# 按 role 授权
|
||||
if entry.get("principal_type") == "role" and entry.get("principal_id") == user_role:
|
||||
if _permission_covers(entry.get("permission"), permission):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _permission_covers(granted: str | None, required: str) -> bool:
|
||||
"""权限覆盖判断:write/execute 覆盖 read;admin 覆盖一切。"""
|
||||
if not granted:
|
||||
return False
|
||||
if granted == "admin":
|
||||
return True
|
||||
if granted == required:
|
||||
return True
|
||||
# write 覆盖 read
|
||||
if required == "read" and granted in ("write", "execute"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filter_accessible_resource_ids(
|
||||
resource_type: str,
|
||||
all_ids: list[str],
|
||||
user: dict[str, Any],
|
||||
) -> list[str]:
|
||||
"""
|
||||
从全部资源 ID 中过滤出当前用户可访问的 ID 列表。
|
||||
- admin 直接返回全部。
|
||||
- 普通用户查 acls 表取交集。
|
||||
"""
|
||||
if user.get("role") == "admin" or user.get("protected"):
|
||||
return all_ids
|
||||
|
||||
if not all_ids:
|
||||
return []
|
||||
|
||||
store = get_platform_store()
|
||||
user_id = user.get("id")
|
||||
user_role = user.get("role")
|
||||
|
||||
# 查询该用户在该资源类型下有 read 权限的所有 resource_id
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT resource_id FROM acls
|
||||
WHERE resource_type=? AND (
|
||||
(principal_type='user' AND principal_id=?)
|
||||
OR (principal_type='role' AND principal_id=?)
|
||||
)
|
||||
""",
|
||||
(resource_type, user_id, user_role),
|
||||
).fetchall()
|
||||
|
||||
accessible = {r["resource_id"] for r in rows}
|
||||
return [rid for rid in all_ids if rid in accessible]
|
||||
@@ -2,6 +2,18 @@
|
||||
from functools import lru_cache
|
||||
import os
|
||||
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 显式指定 backend 目录下的 .env,并强制覆盖已有环境变量,
|
||||
# 确保远程数据库配置生效,不被本地默认值或残留环境变量影响。
|
||||
_env_path = _Path(__file__).resolve().parent.parent.parent / ".env"
|
||||
load_dotenv(dotenv_path=_env_path, override=True)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = os.getenv(name)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,10 @@ CREATE TABLE IF NOT EXISTS trained_models (
|
||||
create_time TEXT NOT NULL,
|
||||
merged INTEGER NOT NULL DEFAULT 0,
|
||||
merging INTEGER NOT NULL DEFAULT 0,
|
||||
merged_path TEXT
|
||||
merged_path TEXT,
|
||||
artifact_dir TEXT,
|
||||
compute_node_id TEXT,
|
||||
compute_node_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||
@@ -282,3 +285,51 @@ CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(targe
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||
|
||||
-- ===================== Project / Tenant =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
name TEXT NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
description TEXT,
|
||||
quota TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
create_time TEXT NOT NULL,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS project_members (
|
||||
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL DEFAULT 'member',
|
||||
create_time TEXT NOT NULL,
|
||||
PRIMARY KEY (project_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL DEFAULT '[]',
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
issued_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
ip TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acls (
|
||||
id TEXT PRIMARY KEY,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
principal_type TEXT NOT NULL,
|
||||
principal_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
@@ -80,6 +80,17 @@ CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
||||
failure_reason TEXT,
|
||||
generation_run_id TEXT,
|
||||
results_confirmed BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
workflow_step VARCHAR(20) NOT NULL DEFAULT 'create'
|
||||
CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results')),
|
||||
preview_status VARCHAR(20) NOT NULL DEFAULT 'idle'
|
||||
CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled')),
|
||||
preview_progress NUMERIC(5,2) NOT NULL DEFAULT 0
|
||||
CHECK (preview_progress >= 0 AND preview_progress <= 100),
|
||||
preview_run_id TEXT,
|
||||
preview_failure_reason TEXT,
|
||||
preview_total_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_total_files >= 0),
|
||||
preview_completed_files INTEGER NOT NULL DEFAULT 0 CHECK (preview_completed_files >= 0),
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
owner_id TEXT,
|
||||
@@ -95,6 +106,87 @@ CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
);
|
||||
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
||||
-- 历史任务在引入六步确认流程前已经完成审核,默认保留为已确认;
|
||||
-- 新任务由创建接口显式写入 FALSE,并在第六步确认后转为 TRUE。
|
||||
ALTER TABLE data_process_tasks
|
||||
ADD COLUMN IF NOT EXISTS results_confirmed BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
UPDATE data_process_tasks
|
||||
SET results_confirmed=FALSE
|
||||
WHERE status <> 'completed' AND results_confirmed=TRUE;
|
||||
|
||||
-- 先以可空列接入旧库,才能只回填历史行;随后再收紧默认值与约束。
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2);
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_run_id TEXT;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_total_files INTEGER;
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER;
|
||||
|
||||
CREATE TEMP TABLE data_process_workflow_backfill_ids ON COMMIT DROP AS
|
||||
SELECT id FROM data_process_tasks WHERE workflow_step IS NULL;
|
||||
|
||||
UPDATE data_process_tasks task
|
||||
SET workflow_step = CASE
|
||||
WHEN task.status IN ('running', 'failed', 'stopped') THEN 'generate'
|
||||
WHEN task.status = 'completed' AND task.results_confirmed=FALSE THEN 'generate'
|
||||
WHEN task.status = 'completed' THEN 'results'
|
||||
ELSE 'create'
|
||||
END
|
||||
WHERE task.workflow_step IS NULL;
|
||||
UPDATE data_process_tasks
|
||||
SET preview_status='idle', preview_progress=0,
|
||||
preview_total_files=0, preview_completed_files=0
|
||||
WHERE preview_status IS NULL OR preview_progress IS NULL
|
||||
OR preview_total_files IS NULL OR preview_completed_files IS NULL;
|
||||
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET DEFAULT 'create';
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN workflow_step SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET DEFAULT 'idle';
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_status SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_progress SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_total_files SET NOT NULL;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET DEFAULT 0;
|
||||
ALTER TABLE data_process_tasks ALTER COLUMN preview_completed_files SET NOT NULL;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid='data_process_tasks'::regclass
|
||||
AND conname='ck_data_process_tasks_workflow_step'
|
||||
) THEN
|
||||
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_workflow_step
|
||||
CHECK (workflow_step IN ('create', 'model', 'upload', 'preview', 'generate', 'results'));
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid='data_process_tasks'::regclass
|
||||
AND conname='ck_data_process_tasks_preview_status'
|
||||
) THEN
|
||||
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_status
|
||||
CHECK (preview_status IN ('idle', 'queued', 'running', 'completed', 'failed', 'cancelled'));
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid='data_process_tasks'::regclass
|
||||
AND conname='ck_data_process_tasks_preview_progress'
|
||||
) THEN
|
||||
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_progress
|
||||
CHECK (preview_progress >= 0 AND preview_progress <= 100);
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid='data_process_tasks'::regclass
|
||||
AND conname='ck_data_process_tasks_preview_file_counts'
|
||||
) THEN
|
||||
ALTER TABLE data_process_tasks ADD CONSTRAINT ck_data_process_tasks_preview_file_counts
|
||||
CHECK (preview_total_files >= 0 AND preview_completed_files >= 0
|
||||
AND preview_completed_files <= preview_total_files);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
||||
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
||||
@@ -153,6 +245,22 @@ CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
||||
ON data_process_preview_items(task_id, source_file_id, created_at);
|
||||
|
||||
-- 子表在新库中到这里才存在;只修复本次新增 workflow_step 前的历史任务。
|
||||
UPDATE data_process_tasks task
|
||||
SET workflow_step = CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM data_process_preview_items preview
|
||||
WHERE preview.task_id=task.id
|
||||
) THEN 'preview'
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM data_process_source_files source_file
|
||||
WHERE source_file.task_id=task.id AND source_file.deleted_at IS NULL
|
||||
) THEN 'upload'
|
||||
ELSE task.workflow_step
|
||||
END
|
||||
WHERE task.id IN (SELECT id FROM data_process_workflow_backfill_ids)
|
||||
AND task.status='pending';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
|
||||
68
backend/app/db/sql/002_governance.sql
Normal file
68
backend/app/db/sql/002_governance.sql
Normal file
@@ -0,0 +1,68 @@
|
||||
-- 平台治理:租户 / 审批 / 审计(字段以 platform_store 实际写入为准)
|
||||
CREATE TABLE IF NOT EXISTS tenants (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
code TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
owner_user_id TEXT,
|
||||
quota TEXT,
|
||||
retention_policy_id TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_templates (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
steps TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
template_id TEXT,
|
||||
resource_type TEXT,
|
||||
resource_id TEXT,
|
||||
applicant_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
current_step INTEGER DEFAULT 0,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_steps (
|
||||
id TEXT PRIMARY KEY,
|
||||
instance_id TEXT,
|
||||
step_index INTEGER,
|
||||
approver_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
comment TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
actor_id TEXT,
|
||||
action TEXT,
|
||||
target_type TEXT,
|
||||
target_id TEXT,
|
||||
detail TEXT,
|
||||
client_ip TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
scope TEXT,
|
||||
rule TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
create_time TEXT,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
18
backend/app/db/sql/003_model_path_governance.sql
Normal file
18
backend/app/db/sql/003_model_path_governance.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- 003_model_path_governance
|
||||
-- 模型路径治理:增加 can_train 标识,区分本地可训练模型与 API / 远程模型。
|
||||
-- 训练预检阶段依赖该字段拦截不适合 LLaMA-Factory 本地训练的基座模型。
|
||||
|
||||
-- 1. models 表增加 can_train(默认 0,后设搬迁为 1 的规则如下)
|
||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
-- 2. 将已有模型按规则推定 can_train:
|
||||
-- - path 非空 且 model_source != 'api' → 可训练 (1)
|
||||
-- - 其余 → 不可训练 (0)
|
||||
UPDATE models
|
||||
SET can_train = CASE
|
||||
WHEN path IS NOT NULL AND path != '' AND model_source IS NOT NULL AND model_source != 'api' THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
|
||||
-- 3. 给 trained_models 增加 artifact_dir(训练产物目录扫描结果目录)
|
||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
|
||||
3
backend/app/db/sql/003_tenant_quota.sql
Normal file
3
backend/app/db/sql/003_tenant_quota.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- 租户配额与保留策略扩展(如后续治理表需补列,可在此追加)
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS gpu_quota TEXT;
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;
|
||||
91
backend/app/modules/approval/router.py
Normal file
91
backend/app/modules/approval/router.py
Normal file
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/approvals", tags=["approval"])
|
||||
|
||||
|
||||
@router.get("/templates")
|
||||
def list_templates() -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_templates())
|
||||
|
||||
|
||||
@router.post("/templates")
|
||||
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
if not payload.get("name"):
|
||||
raise fail(400, "name 必填")
|
||||
return ok(get_platform_store().create_approval_template(payload))
|
||||
|
||||
|
||||
@router.get("/templates/{template_id}")
|
||||
def get_template(template_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().approval_template(template_id))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.put("/templates/{template_id}")
|
||||
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_approval_template(template_id, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.delete("/templates/{template_id}")
|
||||
def delete_template(template_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().delete_approval_template(template_id))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_instances(status: str | None = None) -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_instances(status=status))
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
for field in ("resource_type", "resource_id", "applicant_id"):
|
||||
if not payload.get(field):
|
||||
raise fail(400, f"{field} 必填")
|
||||
try:
|
||||
return ok(get_platform_store().create_approval_instance(payload))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.get("/{instance_id}")
|
||||
def get_instance(instance_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().approval_instance(instance_id))
|
||||
except KeyError:
|
||||
raise fail(404, "instance not found")
|
||||
|
||||
|
||||
@router.post("/{instance_id}/steps/{step_index}/decision")
|
||||
def decide(
|
||||
instance_id: str,
|
||||
step_index: int,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
) -> dict[str, Any]:
|
||||
if not payload.get("approver_id"):
|
||||
raise fail(400, "approver_id 必填")
|
||||
try:
|
||||
return ok(
|
||||
get_platform_store().decide_approval_step(
|
||||
instance_id,
|
||||
step_index,
|
||||
approver_id=payload["approver_id"],
|
||||
approved=bool(payload.get("approved", False)),
|
||||
comment=payload.get("comment"),
|
||||
)
|
||||
)
|
||||
except (KeyError, ValueError) as e:
|
||||
raise fail(400, str(e))
|
||||
@@ -33,6 +33,16 @@ def _unwrap_dict(payload: Any) -> dict[str, Any]:
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
# Inference calls are intentionally short-timeout:
|
||||
# - load dispatch only confirms the compute node accepted the request
|
||||
# (the actual model load now runs asynchronously on the node).
|
||||
# - status/unload must never block the platform for long when a node is
|
||||
# unreachable but still marked online.
|
||||
INFERENCE_LOAD_TIMEOUT = httpx.Timeout(30, connect=10)
|
||||
INFERENCE_STATUS_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||
INFERENCE_UNLOAD_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||
|
||||
|
||||
class ComputeNodeClient:
|
||||
"""Application-side client for one compute node.
|
||||
|
||||
@@ -182,6 +192,36 @@ class ComputeNodeClient:
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
json_data: dict[str, Any] | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generic request method for compute API endpoints."""
|
||||
url = _join_url(self.api_base_url, f"{self.route_prefix}{path}")
|
||||
async with httpx.AsyncClient(timeout=timeout or 300, headers=self.headers()) as client:
|
||||
if method.upper() == "GET":
|
||||
response = await client.get(url)
|
||||
else:
|
||||
response = await client.post(url, json=json_data)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
# ── Inference helpers (short timeouts — see module constants) ──────────
|
||||
|
||||
async def inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Dispatch a model load. Returns as soon as the node accepts the
|
||||
request; the node now loads asynchronously (status goes 'loading')."""
|
||||
return await self._request("POST", "/inference/load", json_data=payload, timeout=INFERENCE_LOAD_TIMEOUT)
|
||||
|
||||
async def inference_status(self) -> dict[str, Any]:
|
||||
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
|
||||
|
||||
async def inference_unload(self) -> dict[str, Any]:
|
||||
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
filename: str,
|
||||
@@ -196,7 +236,8 @@ class ComputeNodeClient:
|
||||
"resource_id": resource_id or "",
|
||||
}
|
||||
files = {"file": (filename, content)}
|
||||
async with httpx.AsyncClient(timeout=max(self.timeout, 60), headers=self.headers()) as client:
|
||||
timeout = httpx.Timeout(max(self.timeout, 60), connect=self.timeout)
|
||||
async with httpx.AsyncClient(timeout=timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/upload"),
|
||||
data=data,
|
||||
|
||||
@@ -1,15 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
||||
MAX_STARTING_ATTEMPTS = 40
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
def _parse_inference_load_status(task: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
load_status = task.get("load_status") or {}
|
||||
if isinstance(load_status, str):
|
||||
try:
|
||||
load_status = json.loads(load_status)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
load_status = {}
|
||||
return load_status.get("loaded_models") or [], load_status
|
||||
|
||||
|
||||
async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
||||
"""推进处于 starting 状态的推理加载。
|
||||
|
||||
模型加载已改为异步派发:/model-compare/{id}/load 立即返回,这里在每次
|
||||
轮询时查询对应计算节点的 /inference/status,把任务从 starting 推进到
|
||||
ready/error。使用短超时,单节点不可达不会阻塞整轮轮询。
|
||||
"""
|
||||
reconciled: list[dict[str, Any]] = []
|
||||
now = time.time()
|
||||
for task in store.compare_tasks():
|
||||
items, _ = _parse_inference_load_status(task)
|
||||
if not any(item.get("status") == "starting" for item in items):
|
||||
continue
|
||||
# dirty 只要处理过任一 starting 项就置位:load_attempts / last_polled_at
|
||||
# 必须落库,否则节点不可达时计数不会累积,封顶逻辑永远触发不了
|
||||
dirty = False
|
||||
for item in items:
|
||||
if item.get("status") != "starting":
|
||||
continue
|
||||
# 节流:同一 item 每 3s 只查询一次
|
||||
if now - float(item.get("last_polled_at") or 0) < 3:
|
||||
continue
|
||||
item["last_polled_at"] = now
|
||||
item["load_attempts"] = int(item.get("load_attempts") or 0) + 1
|
||||
dirty = True
|
||||
node = next((n for n in store.compute_nodes() if n["id"] == item.get("node_id")), None)
|
||||
if not node:
|
||||
item["status"] = "error"
|
||||
item["error"] = "compute node deleted"
|
||||
store.mark_inference_unloaded(item.get("node_id") or "")
|
||||
continue
|
||||
if not node.get("enabled") or node.get("scheduler_status") != "online":
|
||||
item["status"] = "error"
|
||||
item["error"] = "compute node offline"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
continue
|
||||
try:
|
||||
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
|
||||
except Exception as exc: # noqa: BLE001 - node unreachable; keep retrying until cap
|
||||
if int(item.get("load_attempts") or 0) >= MAX_STARTING_ATTEMPTS:
|
||||
item["status"] = "error"
|
||||
item["error"] = f"compute node unreachable: {exc}"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
continue
|
||||
node_status = status.get("status")
|
||||
if node_status == "ready":
|
||||
item["status"] = "ready"
|
||||
item.pop("error", None)
|
||||
store.mark_inference_loaded(node["id"])
|
||||
elif node_status == "error":
|
||||
item["status"] = "error"
|
||||
item["error"] = status.get("error") or "model load failed on compute node"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
elif node_status == "idle":
|
||||
# 节点重启导致已加载模型丢失
|
||||
item["status"] = "error"
|
||||
item["error"] = "model disappeared from compute node (node may have restarted)"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
# node_status == "loading" -> 保持 starting,下轮再查
|
||||
if dirty:
|
||||
if any(i.get("status") in {"ready", "running"} for i in items):
|
||||
new_status = "loaded"
|
||||
elif any(i.get("status") == "starting" for i in items):
|
||||
new_status = "starting" # 仍在加载中,保持 starting
|
||||
else:
|
||||
new_status = "failed"
|
||||
store.update_compare_task(task["id"], {"status": new_status, "load_status": {"loaded_models": items}})
|
||||
reconciled.append({"task_id": task["id"], "status": new_status})
|
||||
return reconciled
|
||||
|
||||
|
||||
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
||||
output_dir = job.get("output_dir")
|
||||
if not output_dir:
|
||||
return None
|
||||
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
|
||||
data_root = "/data/yg-ft/"
|
||||
if full_path.startswith(data_root):
|
||||
full_path = full_path[len(data_root):]
|
||||
rel_path = full_path.lstrip("/")
|
||||
import httpx
|
||||
url = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
|
||||
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
|
||||
response = await http.get(url, params={"path": rel_path})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
@@ -27,6 +131,13 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
||||
except Exception:
|
||||
pass
|
||||
# P0-4: Force-fetch last log snippet when job reaches terminal state
|
||||
if job.get("status") in {"failed", "stopped"}:
|
||||
try:
|
||||
last_logs = await client.job_logs(task["compute_job_id"], tail_lines=200)
|
||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||
except Exception:
|
||||
pass
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
@@ -41,4 +152,40 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|
||||
|
||||
# ── Eval job sync ────────────────────────────────────────────────
|
||||
eval_synced = 0
|
||||
for eval_task in store.running_eval_tasks():
|
||||
node = next(
|
||||
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
|
||||
None,
|
||||
)
|
||||
if not node:
|
||||
failed.append({"eval_task_id": eval_task["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
job = await client.get_job(eval_task["compute_job_id"])
|
||||
result_content = None
|
||||
# Try to read eval_results.json from the job output directory
|
||||
if job.get("status") == "completed" and job.get("output_dir"):
|
||||
try:
|
||||
result_content = await fetch_eval_result_content(client, node, job)
|
||||
except Exception:
|
||||
pass
|
||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
||||
eval_synced += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
||||
|
||||
# ── Inference load reconciliation ─────────────────────────────────────
|
||||
try:
|
||||
inference_reconciled = await reconcile_inference_loads(store)
|
||||
except Exception as exc: # noqa: BLE001 - keep polling alive
|
||||
failed.append({"inference_reconcile": str(exc)})
|
||||
inference_reconciled = []
|
||||
|
||||
return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed,
|
||||
"items": synced, "standalone": standalone_synced, "eval_synced": eval_synced,
|
||||
"inference_reconciled": inference_reconciled}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
148
backend/app/modules/data_process/dataset_format.py
Normal file
148
backend/app/modules/data_process/dataset_format.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""Dataset format validation for Alpaca, ShareGPT, DPO, CPT formats.
|
||||
|
||||
Used by the training preflight flow to validate that uploaded dataset files
|
||||
conform to the declared format before submitting to the compute node.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_sample(path: str | None, content: str | None = None, max_samples: int = 20) -> list[dict[str, Any]]:
|
||||
"""Load up to max_samples records from JSONL file path or raw content string."""
|
||||
try:
|
||||
if content is not None:
|
||||
text = content.strip()
|
||||
elif path:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
text = fh.read().strip()
|
||||
else:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if not text:
|
||||
return []
|
||||
|
||||
lines = text.splitlines()[:max_samples]
|
||||
records: list[dict[str, Any]] = []
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(record, dict):
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
def _check_alpaca(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate Alpaca format: requires 'instruction' field."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("Alpaca 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_instruction = sum(1 for r in records if not r.get("instruction"))
|
||||
if missing_instruction:
|
||||
errors.append(
|
||||
f"Alpaca 格式要求每条记录包含 instruction 字段,"
|
||||
f"前{len(records)}条中有{missing_instruction}条缺失"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _check_sharegpt(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate ShareGPT format: requires 'messages' (list of dicts with role/content)."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("ShareGPT 格式数据集无有效记录")
|
||||
return errors
|
||||
bad = 0
|
||||
for r in records:
|
||||
messages = r.get("messages")
|
||||
if not isinstance(messages, list) or not messages:
|
||||
bad += 1
|
||||
continue
|
||||
for msg in messages:
|
||||
if not isinstance(msg, dict) or "role" not in msg or "content" not in msg:
|
||||
bad += 1
|
||||
break
|
||||
if bad:
|
||||
errors.append(
|
||||
f"ShareGPT 格式要求每条记录包含 messages 列表,"
|
||||
f"每条消息需有 role 和 content 字段,前{len(records)}条中有{bad}条不符合"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _check_dpo(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate DPO format: requires 'chosen' and 'rejected' fields."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("DPO 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_chosen = sum(1 for r in records if not r.get("chosen"))
|
||||
missing_rejected = sum(1 for r in records if not r.get("rejected"))
|
||||
if missing_chosen:
|
||||
errors.append(f"DPO 格式要求 chosen 字段,前{len(records)}条中有{missing_chosen}条缺失")
|
||||
if missing_rejected:
|
||||
errors.append(f"DPO 格式要求 rejected 字段,前{len(records)}条中有{missing_rejected}条缺失")
|
||||
return errors
|
||||
|
||||
|
||||
def _check_cpt(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate CPT format: requires 'text' field, should NOT have instruction/output."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("CPT 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_text = sum(1 for r in records if not r.get("text"))
|
||||
has_instruction = sum(1 for r in records if r.get("instruction") or r.get("output"))
|
||||
if missing_text:
|
||||
errors.append(f"CPT 格式要求 text 字段,前{len(records)}条中有{missing_text}条缺失")
|
||||
if has_instruction:
|
||||
errors.append(
|
||||
f"CPT 格式不应包含 instruction/output 字段(疑似 Alpaca 格式),"
|
||||
f"前{len(records)}条中有{has_instruction}条包含此类字段"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
FORMAT_VALIDATORS = {
|
||||
"alpaca": _check_alpaca,
|
||||
"alpaca_jsonl": _check_alpaca,
|
||||
"sharegpt": _check_sharegpt,
|
||||
"dpo": _check_dpo,
|
||||
"cpt": _check_cpt,
|
||||
"pt": _check_cpt,
|
||||
}
|
||||
|
||||
|
||||
def validate_dataset_format(
|
||||
dataset_format: str,
|
||||
content: str | None = None,
|
||||
path: str | None = None,
|
||||
max_samples: int = 20,
|
||||
) -> list[str]:
|
||||
"""Validate dataset content against expected format.
|
||||
|
||||
Args:
|
||||
dataset_format: One of 'alpaca', 'sharegpt', 'dpo', 'cpt'.
|
||||
content: Raw file content (JSONL text). Mutually exclusive with path.
|
||||
path: File path to read content from.
|
||||
max_samples: Maximum records to sample for validation.
|
||||
|
||||
Returns:
|
||||
List of error messages (empty if valid).
|
||||
"""
|
||||
fmt = str(dataset_format).lower().strip()
|
||||
validator = FORMAT_VALIDATORS.get(fmt)
|
||||
if not validator:
|
||||
return [f"不支持的数据集格式: {dataset_format},支持的格式: {', '.join(sorted(FORMAT_VALIDATORS))}"]
|
||||
records = _load_sample(path=path, content=content, max_samples=max_samples)
|
||||
return validator(records)
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
@@ -22,6 +23,10 @@ class ModelGenerationError(ValueError):
|
||||
"""模型配置、响应或调用失败。"""
|
||||
|
||||
|
||||
class _TerminalModelGenerationError(ModelGenerationError):
|
||||
"""使用相同参数重试也无法恢复的模型响应错误。"""
|
||||
|
||||
|
||||
OUTPUT_TYPE_STANDARD = "standard"
|
||||
OUTPUT_TYPE_REASONING = "reasoning"
|
||||
SUPPORTED_OUTPUT_TYPES = {OUTPUT_TYPE_STANDARD, OUTPUT_TYPE_REASONING}
|
||||
@@ -31,6 +36,25 @@ SUPPORTED_REASONING_DETAILS = {
|
||||
REASONING_DETAIL_NORMAL,
|
||||
REASONING_DETAIL_DETAILED,
|
||||
}
|
||||
MINIMAX_M3_API_HOSTS = {"api.minimax.io", "api.minimaxi.com"}
|
||||
MINIMAX_M3_MIN_COMPLETION_TOKENS = 4096
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_retryable_generation_error(exc: Exception) -> bool:
|
||||
if isinstance(exc, _TerminalModelGenerationError):
|
||||
return False
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
status_code = exc.response.status_code
|
||||
return status_code in {408, 425, 429} or status_code >= 500
|
||||
if isinstance(exc, httpx.RequestError):
|
||||
return True
|
||||
return isinstance(exc, (json.JSONDecodeError, ModelGenerationError))
|
||||
|
||||
|
||||
def _is_official_minimax_m3(endpoint: str, model_name: str) -> bool:
|
||||
host = (urlsplit(endpoint).hostname or "").casefold()
|
||||
return host in MINIMAX_M3_API_HOSTS and model_name.casefold() == "minimax-m3"
|
||||
|
||||
|
||||
def chat_completions_url(value: str) -> str:
|
||||
@@ -59,32 +83,139 @@ def chat_completions_url(value: str) -> str:
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
|
||||
|
||||
|
||||
def _message_content(payload: Mapping[str, Any]) -> str:
|
||||
def _response_choice(payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
try:
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
choice = payload["choices"][0]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ModelGenerationError(
|
||||
"model response does not contain choices[0].message.content"
|
||||
) from exc
|
||||
raise ModelGenerationError("模型响应缺少 choices[0]") from exc
|
||||
if not isinstance(choice, Mapping):
|
||||
raise ModelGenerationError("模型响应 choices[0] 不是对象")
|
||||
return choice
|
||||
|
||||
|
||||
def _response_finish_reason(payload: Mapping[str, Any]) -> str:
|
||||
try:
|
||||
return str(_response_choice(payload).get("finish_reason") or "").strip().lower()
|
||||
except ModelGenerationError:
|
||||
return ""
|
||||
|
||||
|
||||
def _response_content_length(payload: Mapping[str, Any]) -> int:
|
||||
try:
|
||||
message = _response_choice(payload).get("message")
|
||||
if not isinstance(message, Mapping):
|
||||
return 0
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return len(content)
|
||||
if isinstance(content, list):
|
||||
return sum(
|
||||
len(str(item.get("text") or ""))
|
||||
for item in content
|
||||
if isinstance(item, Mapping)
|
||||
)
|
||||
except ModelGenerationError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _raise_for_terminal_response(payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
choice = _response_choice(payload)
|
||||
base_response = payload.get("base_resp")
|
||||
status_code: Any = None
|
||||
status_message = ""
|
||||
if isinstance(base_response, Mapping):
|
||||
status_code = base_response.get("status_code")
|
||||
status_message = re.sub(
|
||||
r"\s+", " ", str(base_response.get("status_msg") or "")
|
||||
).strip()[:200]
|
||||
|
||||
if bool(payload.get("input_sensitive")) or status_code in {1026, "1026"}:
|
||||
raise _TerminalModelGenerationError(
|
||||
f"模型输入触发内容安全拦截(code={status_code or 1026})"
|
||||
)
|
||||
if bool(payload.get("output_sensitive")) or status_code in {1027, "1027"}:
|
||||
raise _TerminalModelGenerationError(
|
||||
f"模型输出触发内容安全拦截(code={status_code or 1027})"
|
||||
)
|
||||
|
||||
finish_reason = str(choice.get("finish_reason") or "").strip().lower()
|
||||
if finish_reason == "length":
|
||||
raise _TerminalModelGenerationError(
|
||||
"模型输出因达到 Token 上限被截断(finish_reason=length),"
|
||||
"请提高最大输出长度后重试"
|
||||
)
|
||||
if finish_reason == "content_filter":
|
||||
raise _TerminalModelGenerationError(
|
||||
"模型输出被内容安全策略拦截(finish_reason=content_filter)"
|
||||
)
|
||||
if finish_reason in {"tool_calls", "function_call"}:
|
||||
raise _TerminalModelGenerationError(
|
||||
f"模型返回了当前生成任务不支持的工具调用(finish_reason={finish_reason})"
|
||||
)
|
||||
if status_code not in {None, "", 0, "0"}:
|
||||
detail = f":{status_message}" if status_message else ""
|
||||
raise _TerminalModelGenerationError(
|
||||
f"模型服务返回业务错误(code={status_code}){detail}"
|
||||
)
|
||||
return choice
|
||||
|
||||
|
||||
def _message_content(payload: Mapping[str, Any]) -> str:
|
||||
choice = _raise_for_terminal_response(payload)
|
||||
message = choice.get("message")
|
||||
if not isinstance(message, Mapping):
|
||||
raise ModelGenerationError("模型响应缺少 choices[0].message")
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
result = content
|
||||
elif isinstance(content, list):
|
||||
parts = [
|
||||
str(item.get("text") or "")
|
||||
for item in content
|
||||
if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
|
||||
]
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
raise ModelGenerationError("model response content must be text")
|
||||
result = "".join(parts)
|
||||
elif content is None:
|
||||
result = ""
|
||||
else:
|
||||
raise ModelGenerationError("模型响应 content 必须是文本")
|
||||
if not result.strip():
|
||||
raise ModelGenerationError("模型返回的最终内容为空,未生成可解析的 JSON")
|
||||
return result
|
||||
|
||||
|
||||
def _json_documents(content: str) -> list[Any]:
|
||||
decoder = json.JSONDecoder()
|
||||
documents: list[Any] = []
|
||||
cursor = 0
|
||||
while cursor < len(content):
|
||||
match = re.search(r"[\[{]", content[cursor:])
|
||||
if not match:
|
||||
break
|
||||
start = cursor + match.start()
|
||||
try:
|
||||
value, end = decoder.raw_decode(content[start:])
|
||||
except json.JSONDecodeError:
|
||||
cursor = start + 1
|
||||
continue
|
||||
if isinstance(value, (Mapping, list)):
|
||||
documents.append(value)
|
||||
cursor = start + max(end, 1)
|
||||
return documents
|
||||
|
||||
|
||||
def _json_payload(content: str) -> Any:
|
||||
# 只移除模型在 JSON 之前自行输出的思考过程,不能破坏 JSON 字段中的训练内容。
|
||||
cleaned = content.strip()
|
||||
if re.match(r"^\s*<think>", cleaned, flags=re.IGNORECASE) and not re.match(
|
||||
r"^\s*<think>[\s\S]*?</think>", cleaned, flags=re.IGNORECASE
|
||||
):
|
||||
raise ModelGenerationError("模型思考内容未闭合,响应可能已被截断")
|
||||
cleaned = re.sub(
|
||||
r"^\s*<think>[\s\S]*?</think>\s*",
|
||||
r"^\s*(?:<think>[\s\S]*?</think>\s*)+",
|
||||
"",
|
||||
content,
|
||||
cleaned,
|
||||
count=1,
|
||||
flags=re.IGNORECASE,
|
||||
).strip()
|
||||
@@ -93,10 +224,16 @@ def _json_payload(content: str) -> Any:
|
||||
cleaned = fenced.group(1).strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError as exc:
|
||||
except json.JSONDecodeError as direct_error:
|
||||
documents = _json_documents(cleaned)
|
||||
if len(documents) == 1:
|
||||
return documents[0]
|
||||
if len(documents) > 1:
|
||||
raise ModelGenerationError("模型响应包含多个 JSON 对象,无法确定应使用哪一个")
|
||||
raise ModelGenerationError(
|
||||
f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
|
||||
) from exc
|
||||
"模型响应中没有找到唯一且完整的 JSON 对象"
|
||||
f"(第 {direct_error.lineno} 行,第 {direct_error.colno} 列)"
|
||||
) from direct_error
|
||||
|
||||
|
||||
def _result_items(payload: Any) -> list[Mapping[str, Any]]:
|
||||
@@ -206,6 +343,7 @@ def generate_model_records(
|
||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||
if not model_name:
|
||||
raise ModelGenerationError("generation model name is required")
|
||||
is_minimax_m3 = _is_official_minimax_m3(endpoint, model_name)
|
||||
|
||||
temperature = float(config.get("temperature", 0.7))
|
||||
max_tokens = int(config.get("max_tokens", 1024))
|
||||
@@ -246,9 +384,18 @@ def generate_model_records(
|
||||
reasoning_detail=reasoning_detail,
|
||||
),
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if bool(config.get("json_mode", False)):
|
||||
if is_minimax_m3:
|
||||
request_payload.update(
|
||||
reasoning_split=True,
|
||||
max_completion_tokens=max(
|
||||
max_tokens,
|
||||
MINIMAX_M3_MIN_COMPLETION_TOKENS,
|
||||
),
|
||||
)
|
||||
else:
|
||||
request_payload["max_tokens"] = max_tokens
|
||||
if bool(config.get("json_mode", False)) and not is_minimax_m3:
|
||||
request_payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
last_error: Exception | None = None
|
||||
@@ -264,7 +411,24 @@ def generate_model_records(
|
||||
body = response.json()
|
||||
if not isinstance(body, Mapping):
|
||||
raise ModelGenerationError("model response body must be a JSON object")
|
||||
candidate_items = _result_items(_json_payload(_message_content(body)))
|
||||
try:
|
||||
candidate_items = _result_items(
|
||||
_json_payload(_message_content(body))
|
||||
)
|
||||
except ModelGenerationError as exc:
|
||||
logger.warning(
|
||||
"data process model response rejected task_id=%s model=%s "
|
||||
"finish_reason=%s response_chars=%s input_sensitive=%s "
|
||||
"output_sensitive=%s reason=%s",
|
||||
task_id,
|
||||
model_name,
|
||||
_response_finish_reason(body) or "missing",
|
||||
_response_content_length(body),
|
||||
bool(body.get("input_sensitive")),
|
||||
bool(body.get("output_sensitive")),
|
||||
str(exc),
|
||||
)
|
||||
raise
|
||||
if len(candidate_items) < batch_count:
|
||||
raise ModelGenerationError(
|
||||
"model response contains fewer result objects than requested: "
|
||||
@@ -278,6 +442,8 @@ def generate_model_records(
|
||||
ModelGenerationError,
|
||||
) as exc:
|
||||
last_error = exc
|
||||
if not _is_retryable_generation_error(exc):
|
||||
break
|
||||
|
||||
if generated_items is None:
|
||||
error_message = str(last_error or "model generation failed")[:2000]
|
||||
|
||||
308
backend/app/modules/data_process/office_preview.py
Normal file
308
backend/app/modules/data_process/office_preview.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""Word 与 Excel 原文件的安全、受限预览模型。
|
||||
|
||||
预览只返回浏览器绘制所需的结构化数据,不返回或执行 Office 包中的活动内容。
|
||||
DOCX 的字符偏移与上传时的正文抽取规则保持一致,供前端定位当前切片。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from docx import Document
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
_MAX_WORKBOOK_COLUMNS,
|
||||
_MAX_WORKBOOK_HEADER_SCAN_ROWS,
|
||||
_infer_xlsx_header_region,
|
||||
_normalize_spreadsheet_value,
|
||||
_rewrite_xlsx_workbook_relationships,
|
||||
_validate_office_archive,
|
||||
_xlsx_sheet_merge_ranges,
|
||||
normalize_text,
|
||||
)
|
||||
|
||||
MAX_DOCX_PREVIEW_BLOCKS = 2_000
|
||||
MAX_XLSX_PREVIEW_ROWS = 200
|
||||
|
||||
|
||||
def _docx_alignment(paragraph: Paragraph) -> str:
|
||||
value = paragraph.alignment
|
||||
return {
|
||||
0: "left",
|
||||
1: "center",
|
||||
2: "right",
|
||||
3: "justify",
|
||||
4: "distribute",
|
||||
5: "justify",
|
||||
7: "justify",
|
||||
8: "distribute",
|
||||
9: "distribute",
|
||||
}.get(int(value) if value is not None else -1, "left")
|
||||
|
||||
|
||||
def _docx_heading_level(paragraph: Paragraph) -> int | None:
|
||||
style = paragraph.style
|
||||
if style is None:
|
||||
return None
|
||||
style_name = str(style.name or "")
|
||||
style_id = str(style.style_id or "")
|
||||
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def build_docx_preview(raw: bytes) -> dict[str, Any]:
|
||||
"""把 DOCX 转为保留标题、段落和表格顺序的浏览器预览模型。"""
|
||||
|
||||
_validate_office_archive(raw, "docx")
|
||||
try:
|
||||
document = Document(io.BytesIO(raw))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid DOCX file: {exc}") from exc
|
||||
|
||||
blocks: list[dict[str, Any]] = []
|
||||
source_cursor = 0
|
||||
has_source_content = False
|
||||
rendered_blocks = 0
|
||||
truncated = False
|
||||
|
||||
def source_range(value: str) -> tuple[str, int, int] | None:
|
||||
nonlocal source_cursor, has_source_content
|
||||
text = normalize_text(value)
|
||||
if not text:
|
||||
return None
|
||||
if has_source_content:
|
||||
source_cursor += 2
|
||||
start = source_cursor
|
||||
source_cursor += len(text)
|
||||
has_source_content = True
|
||||
return text, start, source_cursor
|
||||
|
||||
for child in document.element.body.iterchildren():
|
||||
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
||||
truncated = True
|
||||
break
|
||||
|
||||
if isinstance(child, CT_P):
|
||||
paragraph = Paragraph(child, document)
|
||||
located = source_range(paragraph.text)
|
||||
if located is None:
|
||||
continue
|
||||
text, start, end = located
|
||||
style_name = str(paragraph.style.name or "") if paragraph.style else ""
|
||||
blocks.append(
|
||||
{
|
||||
"type": "paragraph",
|
||||
"text": text,
|
||||
"style": style_name,
|
||||
"heading_level": _docx_heading_level(paragraph),
|
||||
"alignment": _docx_alignment(paragraph),
|
||||
"is_list": "list" in style_name.casefold() or "列表" in style_name,
|
||||
"source_start": start,
|
||||
"source_end": end,
|
||||
}
|
||||
)
|
||||
rendered_blocks += 1
|
||||
continue
|
||||
|
||||
if not isinstance(child, CT_Tbl):
|
||||
continue
|
||||
table = Table(child, document)
|
||||
preview_rows: list[dict[str, Any]] = []
|
||||
for row in table.rows:
|
||||
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
||||
truncated = True
|
||||
break
|
||||
cell_values = [normalize_text(cell.text) for cell in row.cells]
|
||||
located = source_range("\t".join(cell_values))
|
||||
if located is None:
|
||||
continue
|
||||
_, start, end = located
|
||||
preview_rows.append(
|
||||
{
|
||||
"cells": cell_values,
|
||||
"source_start": start,
|
||||
"source_end": end,
|
||||
}
|
||||
)
|
||||
rendered_blocks += 1
|
||||
if preview_rows:
|
||||
blocks.append({"type": "table", "rows": preview_rows})
|
||||
if truncated:
|
||||
break
|
||||
|
||||
return {
|
||||
"format": "docx",
|
||||
"blocks": blocks,
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
|
||||
def build_xlsx_preview(
|
||||
raw: bytes,
|
||||
*,
|
||||
sheet_index: int = 0,
|
||||
offset: int = 0,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
"""按工作表分页返回 XLSX 的表头和记录网格。"""
|
||||
|
||||
if sheet_index < 0 or offset < 0:
|
||||
raise ValueError("sheet_index and offset must be non-negative")
|
||||
if limit < 1 or limit > MAX_XLSX_PREVIEW_ROWS:
|
||||
raise ValueError(
|
||||
f"XLSX preview limit must be in [1, {MAX_XLSX_PREVIEW_ROWS}]"
|
||||
)
|
||||
|
||||
_validate_office_archive(raw, "xlsx")
|
||||
merged_by_sheet, normalized_targets = _xlsx_sheet_merge_ranges(raw)
|
||||
workbook_raw = (
|
||||
_rewrite_xlsx_workbook_relationships(raw, normalized_targets)
|
||||
if normalized_targets
|
||||
else raw
|
||||
)
|
||||
try:
|
||||
workbook = load_workbook(
|
||||
io.BytesIO(workbook_raw),
|
||||
read_only=True,
|
||||
data_only=True,
|
||||
keep_links=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid XLSX file: {exc}") from exc
|
||||
|
||||
try:
|
||||
sheets = [
|
||||
{
|
||||
"index": index,
|
||||
"name": worksheet.title,
|
||||
"state": worksheet.sheet_state,
|
||||
}
|
||||
for index, worksheet in enumerate(workbook.worksheets)
|
||||
]
|
||||
if not sheets:
|
||||
raise ValueError("XLSX workbook contains no worksheets")
|
||||
if sheet_index >= len(sheets):
|
||||
raise ValueError("XLSX worksheet index is out of range")
|
||||
|
||||
worksheet = workbook.worksheets[sheet_index]
|
||||
reset_dimensions = getattr(worksheet, "reset_dimensions", None)
|
||||
if callable(reset_dimensions):
|
||||
reset_dimensions()
|
||||
row_iterator = enumerate(worksheet.iter_rows(values_only=True), start=1)
|
||||
buffered_rows: dict[int, tuple[Any, ...]] = {}
|
||||
|
||||
def normalized_values(row: tuple[Any, ...]) -> list[Any]:
|
||||
values = list(row)
|
||||
while values and values[-1] in {None, ""}:
|
||||
values.pop()
|
||||
if len(values) > _MAX_WORKBOOK_COLUMNS:
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} exceeds "
|
||||
f"{_MAX_WORKBOOK_COLUMNS} columns"
|
||||
)
|
||||
return values
|
||||
|
||||
for row_number, row in row_iterator:
|
||||
values = normalized_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
buffered_rows[row_number] = tuple(values)
|
||||
if len(buffered_rows) >= _MAX_WORKBOOK_HEADER_SCAN_ROWS:
|
||||
break
|
||||
|
||||
if not buffered_rows:
|
||||
return {
|
||||
"format": "xlsx",
|
||||
"sheets": sheets,
|
||||
"active_sheet": {
|
||||
"index": sheet_index,
|
||||
"name": worksheet.title,
|
||||
"columns": [],
|
||||
"rows": [],
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"has_more": False,
|
||||
},
|
||||
}
|
||||
|
||||
_, header_end_row, headers = _infer_xlsx_header_region(
|
||||
worksheet.title,
|
||||
buffered_rows,
|
||||
merged_by_sheet.get(worksheet.title, ()),
|
||||
)
|
||||
preview_rows: list[dict[str, Any]] = []
|
||||
record_index = 0
|
||||
has_more = False
|
||||
|
||||
def append_row(row_number: int, values: tuple[Any, ...] | list[Any]) -> bool:
|
||||
nonlocal record_index, has_more
|
||||
row_values = list(values)
|
||||
if len(row_values) > len(headers):
|
||||
raise ValueError(
|
||||
f"XLSX worksheet {worksheet.title!r} has a row wider than its header"
|
||||
)
|
||||
row_values.extend([None] * (len(headers) - len(row_values)))
|
||||
record = {
|
||||
header: _normalize_spreadsheet_value(value)
|
||||
for header, value in zip(headers, row_values, strict=True)
|
||||
}
|
||||
if not any(value not in {"", None} for value in record.values()):
|
||||
return False
|
||||
current_index = record_index
|
||||
record_index += 1
|
||||
if current_index < offset:
|
||||
return False
|
||||
if len(preview_rows) >= limit:
|
||||
has_more = True
|
||||
return True
|
||||
preview_rows.append(
|
||||
{
|
||||
"row_number": row_number,
|
||||
"record_index": current_index,
|
||||
"values": [record[header] for header in headers],
|
||||
"record": record,
|
||||
}
|
||||
)
|
||||
return False
|
||||
|
||||
for row_number, values in buffered_rows.items():
|
||||
if row_number > header_end_row and append_row(row_number, values):
|
||||
break
|
||||
else:
|
||||
for row_number, row in row_iterator:
|
||||
values = normalized_values(row)
|
||||
if not values or all(value in {None, ""} for value in values):
|
||||
continue
|
||||
if append_row(row_number, values):
|
||||
break
|
||||
|
||||
return {
|
||||
"format": "xlsx",
|
||||
"sheets": sheets,
|
||||
"active_sheet": {
|
||||
"index": sheet_index,
|
||||
"name": worksheet.title,
|
||||
"columns": headers,
|
||||
"rows": preview_rows,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"has_more": has_more,
|
||||
},
|
||||
}
|
||||
finally:
|
||||
workbook.close()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_DOCX_PREVIEW_BLOCKS",
|
||||
"MAX_XLSX_PREVIEW_ROWS",
|
||||
"build_docx_preview",
|
||||
"build_xlsx_preview",
|
||||
]
|
||||
@@ -7,6 +7,18 @@ from urllib.parse import urlsplit
|
||||
|
||||
from app.modules.data_process.store import DataProcessStore
|
||||
|
||||
REQUIRED_TASK_COLUMNS = (
|
||||
"generation_run_id",
|
||||
"results_confirmed",
|
||||
"workflow_step",
|
||||
"preview_status",
|
||||
"preview_progress",
|
||||
"preview_run_id",
|
||||
"preview_failure_reason",
|
||||
"preview_total_files",
|
||||
"preview_completed_files",
|
||||
)
|
||||
|
||||
|
||||
def _target_label(database_url: str) -> str:
|
||||
parsed = urlsplit(database_url)
|
||||
@@ -18,14 +30,13 @@ def _schema_ready(store: DataProcessStore) -> bool:
|
||||
with store.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema=current_schema()
|
||||
AND table_name='data_process_tasks'
|
||||
AND column_name='generation_run_id'
|
||||
) AS ready
|
||||
"""
|
||||
SELECT COUNT(*) = %s AS ready
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema=current_schema()
|
||||
AND table_name='data_process_tasks'
|
||||
AND column_name = ANY(%s)
|
||||
""",
|
||||
(len(REQUIRED_TASK_COLUMNS), list(REQUIRED_TASK_COLUMNS)),
|
||||
).fetchone()
|
||||
return bool(row and row["ready"])
|
||||
|
||||
|
||||
@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def stage_copy(
|
||||
self,
|
||||
*,
|
||||
batch_id: str,
|
||||
source_reference: str,
|
||||
expected_source_task_id: str,
|
||||
expected_source_file_id: str,
|
||||
task_id: str,
|
||||
source_file_id: str,
|
||||
version: int,
|
||||
name: str,
|
||||
) -> StagedSourceObject:
|
||||
"""为不可变源对象创建独立目录项,不把大文件重新读入内存。"""
|
||||
|
||||
batch_id = _safe_component(batch_id, "batch id")
|
||||
task_id = _safe_component(task_id, "task id")
|
||||
source_file_id = _safe_component(source_file_id, "source file id")
|
||||
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
|
||||
raise DataProcessStorageError("invalid source file version")
|
||||
basename = _safe_basename(name)
|
||||
source_relative = self._relative_from_reference(source_reference)
|
||||
if source_relative is None:
|
||||
raise DataProcessStorageError("original source object is not available")
|
||||
self._assert_expected_owner(
|
||||
source_relative,
|
||||
expected_task_id=expected_source_task_id,
|
||||
expected_source_file_id=expected_source_file_id,
|
||||
)
|
||||
descriptor, source_info = self._open_read_descriptor(source_relative)
|
||||
os.close(descriptor)
|
||||
|
||||
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
|
||||
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
|
||||
source_path = self._path_for_relative(source_relative)
|
||||
try:
|
||||
os.link(source_path, temporary_path, follow_symlinks=False)
|
||||
copy_info = temporary_path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(copy_info.st_mode)
|
||||
or source_info.st_dev != copy_info.st_dev
|
||||
or source_info.st_ino != copy_info.st_ino
|
||||
):
|
||||
raise DataProcessStorageError("source storage object changed while copying")
|
||||
except Exception:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
relative_path = PurePosixPath(
|
||||
task_id,
|
||||
source_file_id,
|
||||
f"v{version}",
|
||||
basename,
|
||||
)
|
||||
reference = (
|
||||
"local://data-process/"
|
||||
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
||||
)
|
||||
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
|
||||
staged = list(objects)
|
||||
published: list[StagedSourceObject] = []
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
244
backend/app/modules/project/router.py
Normal file
244
backend/app/modules/project/router.py
Normal file
@@ -0,0 +1,244 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["project"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
def _require_no_pending_approval(resource_type: str, resource_id: str) -> None:
|
||||
"""第 4 周:写操作审批拦截——存在待审批实例时拒绝执行。"""
|
||||
store = get_platform_store()
|
||||
pending = [
|
||||
i for i in store.approval_instances(status="pending")
|
||||
if i["resource_type"] == resource_type and i["resource_id"] == resource_id
|
||||
]
|
||||
if pending:
|
||||
raise fail(409, "存在待审批的变更,请先完成审批")
|
||||
|
||||
|
||||
def _require_approval_or_admin(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
current_user: dict[str, Any],
|
||||
action_desc: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""高风险操作审批旁路:admin 直接放行,普通用户创建审批实例(code=202)。"""
|
||||
if is_admin(current_user):
|
||||
return None
|
||||
store = get_platform_store()
|
||||
instance = store.create_approval_instance({
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"applicant_id": current_user.get("id"),
|
||||
"template_id": None,
|
||||
})
|
||||
return {
|
||||
"code": 202,
|
||||
"message": f"操作已提交审批,等待管理员批准:{action_desc}",
|
||||
"data": {"approval_required": True, "approval_id": instance["id"]},
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_projects(
|
||||
tenant_id: str = "default",
|
||||
status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
projects = store.projects(tenant_id=tenant_id, status=status, keyword=keyword)
|
||||
# #1 ACL 过滤:admin 直接放行,普通用户只能看到自己被授权的项目
|
||||
accessible_ids = set(
|
||||
filter_accessible_resource_ids("project", [p["id"] for p in projects], current_user)
|
||||
)
|
||||
filtered = [p for p in projects if p["id"] in accessible_ids]
|
||||
return ok(filtered)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
proj = store.create_project(payload)
|
||||
store.record_audit(
|
||||
action="project.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=proj["id"],
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
detail=f"name={proj.get('name')}",
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.get("/{project_id}")
|
||||
def get_project(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
# #2 访问控制:普通用户无 read 权限则拒绝
|
||||
if not has_resource_access("project", project_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this project")
|
||||
try:
|
||||
return ok(get_platform_store().project(project_id))
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
|
||||
|
||||
@router.put("/{project_id}")
|
||||
def update_project(
|
||||
project_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to update this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
proj = store.update_project(project_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.post("/{project_id}/archive")
|
||||
def archive_project(
|
||||
project_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
_require_no_pending_approval("project", project_id)
|
||||
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
|
||||
if pending:
|
||||
return pending
|
||||
store = get_platform_store()
|
||||
try:
|
||||
proj = store.archive_project(project_id)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.archive",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.delete("/{project_id}")
|
||||
def delete_project(
|
||||
project_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
_require_no_pending_approval("project", project_id)
|
||||
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
|
||||
if pending:
|
||||
return pending
|
||||
store = get_platform_store()
|
||||
store.delete_project(project_id)
|
||||
store.record_audit(
|
||||
action="project.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
)
|
||||
return ok(None)
|
||||
|
||||
|
||||
@router.get("/{project_id}/members")
|
||||
def list_members(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this project")
|
||||
try:
|
||||
return ok(get_platform_store().project_members(project_id))
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
|
||||
|
||||
@router.post("/{project_id}/members")
|
||||
def add_member(
|
||||
project_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
member = store.add_project_member(project_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.member.add",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
|
||||
)
|
||||
return ok(member)
|
||||
|
||||
|
||||
@router.put("/{project_id}/members/{user_id}")
|
||||
def update_member(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
member = store.update_project_member_role(project_id, user_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project or member not found")
|
||||
store.record_audit(
|
||||
action="project.member.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={user_id},role={payload.get('role')}",
|
||||
)
|
||||
return ok(member)
|
||||
|
||||
|
||||
@router.delete("/{project_id}/members/{user_id}")
|
||||
def remove_member(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
store.remove_project_member(project_id, user_id)
|
||||
store.record_audit(
|
||||
action="project.member.remove",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={user_id}",
|
||||
)
|
||||
return ok(None)
|
||||
1
backend/app/modules/resource/__init__.py
Normal file
1
backend/app/modules/resource/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Resource access control list (ACL) module."""
|
||||
41
backend/app/modules/resource/router.py
Normal file
41
backend/app/modules/resource/router.py
Normal file
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/resources", tags=["resource"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("/{resource_type}/{resource_id}/acl")
|
||||
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]:
|
||||
"""查询资源 ACL,返回按主体分组的权限列表。"""
|
||||
return ok(get_platform_store().resource_acl(resource_type, resource_id))
|
||||
|
||||
|
||||
@router.put("/{resource_type}/{resource_id}/acl")
|
||||
def set_acl(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
"""设置资源 ACL,body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
|
||||
entries = payload.get("entries") or []
|
||||
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
|
||||
get_platform_store().record_audit(
|
||||
action="resource.acl.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type=resource_type,
|
||||
target_id=resource_id,
|
||||
detail=f"entries={len(entries)}",
|
||||
)
|
||||
return ok(result)
|
||||
75
backend/app/modules/retention/router.py
Normal file
75
backend/app/modules/retention/router.py
Normal file
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/retention-policies", tags=["retention"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_policies() -> dict[str, Any]:
|
||||
return ok(get_platform_store().retention_policies())
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
if not payload.get("name"):
|
||||
raise fail(400, "name 必填")
|
||||
policy = get_platform_store().create_retention_policy(payload)
|
||||
get_platform_store().record_audit(
|
||||
action="retention.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy["id"],
|
||||
detail=f"name={policy.get('name')}",
|
||||
)
|
||||
return ok(policy)
|
||||
|
||||
|
||||
@router.get("/{policy_id}")
|
||||
def get_policy(policy_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().retention_policy(policy_id))
|
||||
except KeyError:
|
||||
raise fail(404, "retention policy not found")
|
||||
|
||||
|
||||
@router.put("/{policy_id}")
|
||||
def update_policy(
|
||||
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
policy = store.update_retention_policy(policy_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "retention policy not found")
|
||||
store.record_audit(
|
||||
action="retention.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy_id,
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(policy)
|
||||
|
||||
|
||||
@router.delete("/{policy_id}")
|
||||
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
store.delete_retention_policy(policy_id)
|
||||
store.record_audit(
|
||||
action="retention.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy_id,
|
||||
)
|
||||
return ok({"deleted": policy_id})
|
||||
115
backend/app/modules/system/router.py
Normal file
115
backend/app/modules/system/router.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
|
||||
|
||||
|
||||
router = APIRouter(prefix="/system", tags=["system"])
|
||||
|
||||
|
||||
@router.post("/audit/visit")
|
||||
def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
|
||||
"""记录用户访问业务模块的行为,用于看板用户操作分布统计。"""
|
||||
action = str(payload.get("action") or payload.get("module") or "").strip()
|
||||
if not action:
|
||||
return {"code": 0, "message": "ok", "data": {"recorded": False}}
|
||||
actor_id = ""
|
||||
if request is not None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
if token.startswith("platform-token-"):
|
||||
actor_id = token[len("platform-token-"):]
|
||||
get_platform_store().record_audit(
|
||||
action=action,
|
||||
actor_id=actor_id or None,
|
||||
target_type="module",
|
||||
target_id=action,
|
||||
detail=str(payload.get("detail") or ""),
|
||||
)
|
||||
return {"code": 0, "message": "ok", "data": {"recorded": True}}
|
||||
|
||||
|
||||
@router.get("/permissions/codes")
|
||||
def permission_codes() -> dict:
|
||||
"""返回平台权限码清单(权限码接口)。"""
|
||||
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
|
||||
|
||||
|
||||
@router.get("/permissions")
|
||||
def permissions_overview() -> dict:
|
||||
"""返回权限码清单与角色定义。"""
|
||||
store = get_platform_store()
|
||||
return {
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": {"codes": ALL_PERMISSIONS, "roles": store.roles()},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/audit-logs")
|
||||
def audit_logs(
|
||||
tenant_id: str | None = Query(default=None, description="租户 ID"),
|
||||
project_id: str | None = Query(default=None, description="项目 ID"),
|
||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||
action: str | None = Query(default=None, description="动作类型"),
|
||||
target_type: str | None = Query(default=None, description="目标类型"),
|
||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict:
|
||||
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
actor_id=actor_id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return {"code": 0, "message": "ok", "data": result}
|
||||
|
||||
|
||||
@router.get("/audit-logs/export")
|
||||
def audit_logs_export(
|
||||
tenant_id: str | None = Query(default=None, description="租户 ID"),
|
||||
project_id: str | None = Query(default=None, description="项目 ID"),
|
||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||
action: str | None = Query(default=None, description="动作类型"),
|
||||
target_type: str | None = Query(default=None, description="目标类型"),
|
||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
) -> StreamingResponse:
|
||||
"""审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。"""
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
actor_id=actor_id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=10000,
|
||||
offset=0,
|
||||
)
|
||||
items = result["items"]
|
||||
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
|
||||
header = ",".join(columns) + "\n"
|
||||
|
||||
def iter_rows():
|
||||
yield header
|
||||
for row in items:
|
||||
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
|
||||
|
||||
return StreamingResponse(
|
||||
iter_rows(),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=audit_logs.csv"},
|
||||
)
|
||||
116
backend/app/modules/tenant/router.py
Normal file
116
backend/app/modules/tenant/router.py
Normal file
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/tenants", tags=["tenant"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_tenants() -> dict[str, Any]:
|
||||
return ok(get_platform_store().tenants())
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.create_tenant(payload)
|
||||
except KeyError as e:
|
||||
raise fail(400, f"missing field: {e}")
|
||||
store.record_audit(
|
||||
action="tenant.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant["id"],
|
||||
tenant_id=tenant["id"],
|
||||
detail=f"name={tenant.get('name')}",
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.get("/{tenant_id}")
|
||||
def get_tenant(tenant_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().tenant(tenant_id))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
|
||||
|
||||
@router.put("/{tenant_id}")
|
||||
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.update_tenant(tenant_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.put("/{tenant_id}/quota")
|
||||
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.set_tenant_quota(tenant_id, payload.get("quota", {}))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.quota.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.put("/{tenant_id}/retention-policy")
|
||||
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.retention.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.delete("/{tenant_id}")
|
||||
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.delete_tenant(tenant_id)
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
detail=f"name={tenant.get('name')}",
|
||||
)
|
||||
return ok(tenant)
|
||||
@@ -109,6 +109,24 @@ class DataProcessStatus(StrEnum):
|
||||
stopped = "stopped"
|
||||
|
||||
|
||||
class DataProcessWorkflowStep(StrEnum):
|
||||
create = "create"
|
||||
model = "model"
|
||||
upload = "upload"
|
||||
preview = "preview"
|
||||
generate = "generate"
|
||||
results = "results"
|
||||
|
||||
|
||||
class DataProcessPreviewStatus(StrEnum):
|
||||
idle = "idle"
|
||||
queued = "queued"
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
cancelled = "cancelled"
|
||||
|
||||
|
||||
class ProcessType(StrEnum):
|
||||
structured = "structured"
|
||||
unstructured = "unstructured"
|
||||
@@ -164,6 +182,14 @@ class DataProcessTaskUpdate(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessWorkflowStepUpdate(BaseModel):
|
||||
"""仅保存创建向导位置,不修改配置或使下游产物失效。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
workflow_step: DataProcessWorkflowStep
|
||||
|
||||
|
||||
class DataProcessRegenerateRequest(BaseModel):
|
||||
"""以一份完整配置准备任务重新生成。
|
||||
|
||||
@@ -194,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessRepeatRequest(BaseModel):
|
||||
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_updated_at: str = Field(min_length=1)
|
||||
request_id: str = Field(
|
||||
min_length=8,
|
||||
max_length=80,
|
||||
pattern=r"^[A-Za-z0-9_-]+$",
|
||||
)
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -288,6 +327,32 @@ class ResultUpdate(BaseModel):
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class ResultRegenerateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_updated_at: str = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class ResultBatchRegenerateItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
result_id: str = Field(min_length=1, max_length=100)
|
||||
expected_updated_at: str = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class ResultBatchRegenerateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: list[ResultBatchRegenerateItem] = Field(min_length=1, max_length=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_unique_results(self) -> "ResultBatchRegenerateRequest":
|
||||
result_ids = [item.result_id for item in self.items]
|
||||
if len(result_ids) != len(set(result_ids)):
|
||||
raise ValueError("result_id values must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class DatasetSplit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ dependencies = [
|
||||
"pydantic>=2.7.0",
|
||||
"sqlalchemy>=2.0.30",
|
||||
"psycopg[binary]>=3.2.1",
|
||||
"psycopg-pool>=3.2.1",
|
||||
"alembic>=1.13.1",
|
||||
"redis>=5.0.4",
|
||||
"httpx>=0.27.0",
|
||||
|
||||
@@ -4,6 +4,7 @@ python-multipart>=0.0.9
|
||||
pydantic>=2.7.0
|
||||
sqlalchemy>=2.0.30
|
||||
psycopg[binary]>=3.2.1
|
||||
psycopg-pool>=3.2.1
|
||||
alembic>=1.13.1
|
||||
redis>=5.0.4
|
||||
httpx>=0.27.0
|
||||
@@ -18,3 +19,7 @@ llama-index-core==0.14.23
|
||||
llama-index-embeddings-huggingface==0.6.1
|
||||
docling==2.115.0
|
||||
tiktoken>=0.7.0
|
||||
|
||||
# 测试与代码检查
|
||||
pytest>=8.2.0
|
||||
ruff>=0.5.0
|
||||
|
||||
276
backend/tests/test_compare_inference_async.py
Normal file
276
backend/tests/test_compare_inference_async.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""
|
||||
模型推理异步加载改造的单元测试。
|
||||
|
||||
覆盖:
|
||||
- model_compare_load:异步派发,立即返回 starting + 节点信息(不等待加载完成)
|
||||
- model_compare_delete:先删记录,卸载失败也不阻塞删除
|
||||
- reconcile_inference_loads:starting -> ready/error/idle/不可达的状态迁移与封顶
|
||||
- _unload_from_compute_node:任务感知,只命中记录中的节点
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import model_compare_delete, model_compare_load
|
||||
import app.api.v1.endpoints.platform as platform
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import MAX_STARTING_ATTEMPTS, reconcile_inference_loads
|
||||
|
||||
|
||||
class FakeInferenceStore:
|
||||
"""内存 store,仅实现推理加载/对账用到的接口。"""
|
||||
|
||||
def __init__(self, tasks: list[dict[str, Any]] | None = None, nodes: list[dict[str, Any]] | None = None) -> None:
|
||||
self._tasks: dict[str, dict[str, Any]] = {t["id"]: dict(t) for t in (tasks or [])}
|
||||
self._nodes = nodes or []
|
||||
self._inference_nodes: set[str] = set()
|
||||
|
||||
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||
if task_id not in self._tasks:
|
||||
raise KeyError(task_id)
|
||||
return dict(self._tasks[task_id])
|
||||
|
||||
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||
return [dict(t) for t in self._tasks.values()]
|
||||
|
||||
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self._tasks[task_id]
|
||||
merged = {**current, **payload, "id": task_id}
|
||||
self._tasks[task_id] = merged
|
||||
return dict(merged)
|
||||
|
||||
def delete_compare_task(self, task_id: str) -> None:
|
||||
self._tasks.pop(task_id, None)
|
||||
|
||||
def compute_nodes(self) -> list[dict[str, Any]]:
|
||||
return [dict(n) for n in self._nodes]
|
||||
|
||||
def model(self, model_id: str) -> dict[str, Any]:
|
||||
raise KeyError(model_id)
|
||||
|
||||
def trained_models(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
def mark_inference_loaded(self, node_id: str) -> None:
|
||||
self._inference_nodes.add(node_id)
|
||||
|
||||
def mark_inference_unloaded(self, node_id: str) -> None:
|
||||
self._inference_nodes.discard(node_id)
|
||||
|
||||
def is_inference_loaded(self, node_id: str) -> bool:
|
||||
return node_id in self._inference_nodes
|
||||
|
||||
|
||||
def _node(node_id: str, code: str = "") -> dict[str, Any]:
|
||||
return {
|
||||
"id": node_id,
|
||||
"code": code or node_id,
|
||||
"name": code or node_id,
|
||||
"api_base_url": f"http://{code or node_id}:19100",
|
||||
"enabled": True,
|
||||
"scheduler_status": "online",
|
||||
}
|
||||
|
||||
|
||||
def _task(task_id: str, *, node_id: str | None = None, load_status: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": f"task-{task_id}",
|
||||
"status": "pending",
|
||||
"models": [
|
||||
{"model_id": "m_1", "model_name": "qwen", "model_path": "/models/qwen", "node_id": node_id}
|
||||
],
|
||||
"load_status": load_status or {"loaded_models": []},
|
||||
}
|
||||
|
||||
|
||||
async def _fake_inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "loading", "request_id": "req-1"}
|
||||
|
||||
|
||||
async def _fake_inference_unload(self) -> dict[str, Any]:
|
||||
return {"unloaded": True, "status": "idle"}
|
||||
|
||||
|
||||
def _patch_store(monkeypatch, store: FakeInferenceStore) -> None:
|
||||
monkeypatch.setattr(platform, "get_platform_store", lambda: store)
|
||||
monkeypatch.setattr(platform, "get_settings", lambda: SimpleNamespace(compute_mode="real"))
|
||||
|
||||
|
||||
def test_select_eval_node_prefers_model_node(monkeypatch) -> None:
|
||||
from app.api.v1.endpoints.platform import _select_eval_node
|
||||
|
||||
store = FakeInferenceStore(nodes=[_node("n1"), _node("n2")])
|
||||
# 指定模型所在节点时优先返回该节点
|
||||
assert _select_eval_node(store, "n2")["id"] == "n2"
|
||||
# 无指定节点时回退到第一个在线节点
|
||||
assert _select_eval_node(store, None)["id"] == "n1"
|
||||
|
||||
|
||||
def test_select_eval_node_returns_none_when_model_node_offline(monkeypatch) -> None:
|
||||
from app.api.v1.endpoints.platform import _select_eval_node
|
||||
|
||||
nodes = [_node("n1"), _node("n2")]
|
||||
nodes[1]["enabled"] = False
|
||||
store = FakeInferenceStore(nodes=nodes)
|
||||
# 模型所在节点不可用 → 明确失败,不派发到其它节点
|
||||
assert _select_eval_node(store, "n2") is None
|
||||
# 无指定节点时仍回退第一个在线节点
|
||||
assert _select_eval_node(store, None)["id"] == "n1"
|
||||
|
||||
|
||||
def test_model_compare_load_dispatches_and_returns_starting(monkeypatch) -> None:
|
||||
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_load", _fake_inference_load)
|
||||
|
||||
result = asyncio.run(model_compare_load("t1"))
|
||||
assert result["code"] == 0
|
||||
updated = result["data"]
|
||||
assert updated["status"] == "starting"
|
||||
items = updated["load_status"]["loaded_models"]
|
||||
assert items[0]["status"] == "starting"
|
||||
assert items[0]["node_id"] == "n1"
|
||||
assert "n1" in store._inference_nodes
|
||||
|
||||
|
||||
def test_model_compare_load_marks_error_when_all_nodes_fail(monkeypatch) -> None:
|
||||
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
|
||||
async def _raise(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raise RuntimeError("conn refused")
|
||||
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_load", _raise)
|
||||
|
||||
result = asyncio.run(model_compare_load("t1"))
|
||||
updated = result["data"]
|
||||
assert updated["status"] == "failed"
|
||||
assert updated["load_status"]["loaded_models"][0]["status"] == "error"
|
||||
assert "conn refused" in updated["load_status"]["loaded_models"][0]["error"]
|
||||
|
||||
|
||||
def test_model_compare_delete_removes_record_even_if_unload_raises(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
|
||||
async def _raise(self) -> dict[str, Any]:
|
||||
raise RuntimeError("unload boom")
|
||||
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _raise)
|
||||
|
||||
result = asyncio.run(model_compare_delete("t1"))
|
||||
assert result["data"] == {"deleted": "t1"}
|
||||
assert "t1" not in store._tasks
|
||||
# finally 中仍清掉了节点标记
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
def test_unload_from_compute_node_only_hits_recorded_node(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1"), _node("n2")])
|
||||
_patch_store(monkeypatch, store)
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _fake_inference_unload)
|
||||
|
||||
from app.api.v1.endpoints.platform import _unload_from_compute_node
|
||||
|
||||
result = asyncio.run(_unload_from_compute_node(store, task=task))
|
||||
assert result["unloaded"] is True
|
||||
# 只命中任务记录中的节点 n1,n2 未被卸载
|
||||
assert [r["node_id"] for r in result["nodes"]] == ["n1"]
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
async def _status_ready(self) -> dict[str, Any]:
|
||||
return {"loaded": True, "status": "ready", "model_name": "qwen"}
|
||||
|
||||
|
||||
def test_reconcile_transitions_starting_to_ready(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_ready)
|
||||
|
||||
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||
assert reconciled == [{"task_id": "t1", "status": "loaded"}]
|
||||
updated = store._tasks["t1"]
|
||||
assert updated["status"] == "loaded"
|
||||
assert updated["load_status"]["loaded_models"][0]["status"] == "ready"
|
||||
assert "n1" in store._inference_nodes
|
||||
|
||||
|
||||
def test_reconcile_transitions_to_error_and_failed(monkeypatch) -> None:
|
||||
async def _status_error(self) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "error", "error": "CUDA out of memory"}
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_error)
|
||||
|
||||
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||
assert reconciled == [{"task_id": "t1", "status": "failed"}]
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "CUDA out of memory" in item["error"]
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
def test_reconcile_idle_marks_model_disappeared(monkeypatch) -> None:
|
||||
async def _status_idle(self) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "idle"}
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_idle)
|
||||
|
||||
asyncio.run(reconcile_inference_loads(store))
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "disappeared" in item["error"]
|
||||
assert store._tasks["t1"]["status"] == "failed"
|
||||
|
||||
|
||||
def test_reconcile_unreachable_node_flips_to_error_after_cap(monkeypatch) -> None:
|
||||
async def _raise(self) -> dict[str, Any]:
|
||||
raise RuntimeError("conn refused")
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _raise)
|
||||
|
||||
# 每次轮询前重置节流时间戳,逐次推进 load_attempts 到封顶
|
||||
for _ in range(MAX_STARTING_ATTEMPTS):
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
item["last_polled_at"] = 0
|
||||
asyncio.run(reconcile_inference_loads(store))
|
||||
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "unreachable" in item["error"]
|
||||
assert store._tasks["t1"]["status"] == "failed"
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
record_fingerprint,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
structured_json_dumps,
|
||||
)
|
||||
|
||||
|
||||
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
|
||||
root_json = parse_text_content('{"id":1}', filename="root.json")
|
||||
assert root_json.record_locators == (
|
||||
{
|
||||
"kind": "json",
|
||||
"record_index": 1,
|
||||
"json_pointer": "",
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
},
|
||||
)
|
||||
|
||||
wrapped_json = parse_text_content(
|
||||
'{"records":[{"id":1},{"id":1}]}',
|
||||
filename="wrapped.json",
|
||||
)
|
||||
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
|
||||
"/records/0",
|
||||
"/records/1",
|
||||
]
|
||||
|
||||
parsed_jsonl = parse_text_content(
|
||||
'{"id":1}\r\n\r\n{"id":1}',
|
||||
filename="records.jsonl",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == [(1, 1, 1), (2, 3, 3)]
|
||||
assert [
|
||||
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == ['{"id":1}', '{"id":1}']
|
||||
|
||||
parsed_csv = parse_text_content(
|
||||
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
|
||||
filename="records.csv",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_csv.record_locators
|
||||
] == [(1, 2, 3), (2, 5, 5)]
|
||||
assert [
|
||||
parsed_csv.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_csv.record_locators
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
|
||||
processed = preprocess_structured_records_with_lineage(
|
||||
[
|
||||
{"id": "A", "value": "first", "empty": ""},
|
||||
{"id": "", "value": "invalid", "empty": ""},
|
||||
{"id": "A", "value": "duplicate identity", "empty": ""},
|
||||
{"id": "B", "value": "second", "empty": ""},
|
||||
],
|
||||
["clean_invalid", "deduplicate"],
|
||||
)
|
||||
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
|
||||
assert [entry.record for entry in processed] == [
|
||||
{"id": "A", "value": "first"},
|
||||
{"id": "", "value": "invalid"},
|
||||
{"id": "A", "value": "duplicate identity"},
|
||||
{"id": "B", "value": "second"},
|
||||
]
|
||||
|
||||
|
||||
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
|
||||
assert parsed_pdf.format == "pdf"
|
||||
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
|
||||
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
|
||||
)
|
||||
assert parsed_xlsx.record_locators == (
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
)
|
||||
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
|
||||
|
||||
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
|
||||
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
assert parsed_pptx.records == ()
|
||||
|
||||
|
||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||
workbook = Workbook()
|
||||
first = workbook.active
|
||||
first.title = "甲表"
|
||||
first.append(["说明"])
|
||||
first.append([])
|
||||
first.append(["id", "value"])
|
||||
first.append([1, "same"])
|
||||
first.append([1, "same"])
|
||||
second = workbook.create_sheet("乙表")
|
||||
second.append(["id", "value"])
|
||||
second.append([1, "same"])
|
||||
output = io.BytesIO()
|
||||
workbook.save(output)
|
||||
workbook.close()
|
||||
|
||||
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
|
||||
assert parsed.records == (
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
)
|
||||
assert [
|
||||
(
|
||||
locator["record_index"],
|
||||
locator["sheet_index"],
|
||||
locator["sheet_name"],
|
||||
locator["row_number"],
|
||||
locator["sheet_record_index"],
|
||||
)
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(1, 0, "甲表", 4, 0),
|
||||
(2, 0, "甲表", 5, 1),
|
||||
(3, 1, "乙表", 2, 0),
|
||||
]
|
||||
|
||||
|
||||
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
|
||||
pages = _pdf_page_texts(
|
||||
"""
|
||||
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
|
||||
|
||||
assert extract_structured_records(
|
||||
'{"items":[{"text":" 内容 "}],"total":1}',
|
||||
"json",
|
||||
) == [{"text": " 内容 "}]
|
||||
|
||||
|
||||
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
|
||||
source = '{"code":"001","text":" 内容 ","quote":"""}'
|
||||
parsed = parse_text_content(source, filename="records.json")
|
||||
assert parsed.text == source
|
||||
assert parsed.records == (
|
||||
{"code": "001", "text": " 内容 ", "quote": """},
|
||||
)
|
||||
|
||||
invalid_values = (
|
||||
'{"id":1,"id":2}',
|
||||
'{"nested":{"id":1,"id":2}}',
|
||||
'{"value":NaN}',
|
||||
'{"value":Infinity}',
|
||||
'{"value":-Infinity}',
|
||||
'{"value":"bad\x00control"}',
|
||||
)
|
||||
for invalid in invalid_values:
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content(invalid, filename="invalid.json")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content("{\"id\":1}", filename="invalid.json")
|
||||
with pytest.raises(ValueError, match="nesting exceeds"):
|
||||
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
|
||||
|
||||
|
||||
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
|
||||
source = (
|
||||
' {"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}\r\n\r\n'
|
||||
'{"id":2}\r\n'
|
||||
)
|
||||
parsed = parse_text_content(source, filename="records.jsonl")
|
||||
assert parsed.text == source
|
||||
assert parsed.records[0] == {
|
||||
"code": "001",
|
||||
"text": " 内容 ",
|
||||
"value": Decimal("0.123456789012345678901234567890"),
|
||||
}
|
||||
assert [
|
||||
source[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(
|
||||
'{"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}'
|
||||
),
|
||||
'{"id":2}',
|
||||
]
|
||||
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
|
||||
|
||||
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
|
||||
parse_text_content(invalid, filename="invalid.jsonl")
|
||||
|
||||
|
||||
def test_json_record_contract_avoids_business_field_collisions() -> None:
|
||||
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
|
||||
{"id": 1},
|
||||
{"id": 2},
|
||||
]
|
||||
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
|
||||
{"id": 1, "data": [{"id": 2}]}
|
||||
]
|
||||
assert extract_structured_records(
|
||||
'{"records":[{"id":1}],"data":[{"id":2}]}',
|
||||
"json",
|
||||
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
|
||||
assert extract_structured_records(
|
||||
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
|
||||
"json",
|
||||
) == [{"id": 1}]
|
||||
assert extract_structured_records(
|
||||
'{"payload":{"data":[{"id":2}],"total":1}}',
|
||||
"json",
|
||||
) == [{"id": 2}]
|
||||
assert extract_structured_records('{"records":[],"total":0}', "json") == []
|
||||
# 包装数组中的非对象不是记录集合,整体按一条业务对象保留。
|
||||
assert extract_structured_records('{"data":[1,2]}', "json") == [
|
||||
{"data": [1, 2]}
|
||||
]
|
||||
|
||||
|
||||
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
|
||||
pretty = (
|
||||
'{\n "records": [\n {"id": 1},\n'
|
||||
' {\n "id": 2\n }\n ],\n "total": 2\n}'
|
||||
)
|
||||
parsed = parse_text_content(pretty, filename="pretty.json")
|
||||
assert [
|
||||
pretty[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id": 1}', '{\n "id": 2\n }']
|
||||
assert [
|
||||
(locator["start_line"], locator["end_line"])
|
||||
for locator in parsed.record_locators
|
||||
] == [(3, 3), (4, 6)]
|
||||
|
||||
minified = '[{"id":1},{"id":2}]'
|
||||
parsed = parse_text_content(minified, filename="minified.json")
|
||||
assert [
|
||||
minified[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
|
||||
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
|
||||
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
|
||||
parsed = parse_text_content(source, filename="precise.json")
|
||||
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
|
||||
assert parsed.records[1]["value"] == Decimal("1e400")
|
||||
assert structured_json_dumps(parsed.records[0]) == (
|
||||
'{"value":0.123456789012345678901234567890}'
|
||||
)
|
||||
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
|
||||
assert isinstance(parsed.records[0]["value"], Decimal)
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
assert preprocess_structured_records(clean_source, []) == clean_source
|
||||
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
|
||||
{"id": "1", "name": "有效"},
|
||||
{"id": "", "name": "缺少关键字段"},
|
||||
{"id": "2", "name": "有效"},
|
||||
]
|
||||
|
||||
hierarchy = [
|
||||
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
|
||||
{"id": "", "parent_id": "", "name": "", "empty": ""},
|
||||
]
|
||||
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
|
||||
{"id": "1", "parent_id": None, "name": "根节点"},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点"},
|
||||
]
|
||||
|
||||
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
|
||||
assert "profile" in preprocess_structured_records(nested, [])[0]
|
||||
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
|
||||
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
duplicates = [
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "C-1", "value": "updated"},
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "", "value": "blank-one"},
|
||||
{"customer_id": "", "value": "blank-two"},
|
||||
]
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 4
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 5
|
||||
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
|
||||
assert [record["value"] for record in deduplicated] == [
|
||||
"first",
|
||||
"updated",
|
||||
"blank-one",
|
||||
"blank-two",
|
||||
]
|
||||
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
|
||||
masked, counts = desensitize_structured_record(
|
||||
{
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "张三",
|
||||
"contact_name": "李四",
|
||||
"姓名": "王五",
|
||||
"profile.name": "赵六",
|
||||
}
|
||||
)
|
||||
|
||||
assert masked == {
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "[NAME]",
|
||||
"contact_name": "[NAME]",
|
||||
"姓名": "[NAME]",
|
||||
"profile.name": "[NAME]",
|
||||
}
|
||||
assert counts == {
|
||||
"email": 0,
|
||||
"phone": 0,
|
||||
"id_card": 0,
|
||||
"name": 4,
|
||||
"total": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址?",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -88,6 +88,344 @@ def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
|
||||
assert progress_updates == [(1, 1)]
|
||||
|
||||
|
||||
def test_minimax_m3_uses_split_reasoning_and_completion_token_budget() -> None:
|
||||
requests: list[dict[str, object]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content)
|
||||
requests.append(payload)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"finish_reason": "stop",
|
||||
"message": {
|
||||
"reasoning_content": "模型内部思考不应混入业务 JSON",
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "申请编号有什么作用?",
|
||||
"reasoning": "来源说明它用于标识报销申请。",
|
||||
"answer": "它用于唯一标识一笔报销申请。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
"output_sensitive": False,
|
||||
"base_resp": {"status_code": 0, "status_msg": ""},
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-minimax", "edited_content": "申请编号用于标识报销申请。"}],
|
||||
model={
|
||||
"name": "MiniMax",
|
||||
"online_model_name": "MiniMax-M3",
|
||||
"api_url": "https://api.minimaxi.com/v1",
|
||||
},
|
||||
config={
|
||||
"output_type": "reasoning",
|
||||
"json_mode": True,
|
||||
"max_tokens": 1024,
|
||||
"generation_retries": 0,
|
||||
},
|
||||
task_id="task-minimax",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert records[0]["status"] == "valid"
|
||||
assert len(requests) == 1
|
||||
assert requests[0]["reasoning_split"] is True
|
||||
assert requests[0]["max_completion_tokens"] >= 4096
|
||||
assert "max_tokens" not in requests[0]
|
||||
assert "response_format" not in requests[0]
|
||||
|
||||
|
||||
def test_minimax_m3_keeps_larger_configured_completion_budget() -> None:
|
||||
requests: list[dict[str, object]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "问题",
|
||||
"output": "这是满足测试要求的完整答案。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
generate_model_records(
|
||||
[{"id": "preview-minimax-budget", "edited_content": "来源正文"}],
|
||||
model={
|
||||
"online_model_name": "MiniMax-M3",
|
||||
"api_url": "https://api.minimax.io/v1",
|
||||
},
|
||||
config={"max_tokens": 8192, "generation_retries": 0},
|
||||
task_id="task-minimax-budget",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert requests[0]["max_completion_tokens"] == 8192
|
||||
|
||||
|
||||
def test_minimax_m3_name_on_custom_proxy_keeps_generic_openai_parameters() -> None:
|
||||
requests: list[dict[str, object]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "问题",
|
||||
"output": "这是代理服务返回的完整答案。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
generate_model_records(
|
||||
[{"id": "preview-minimax-proxy", "edited_content": "来源正文"}],
|
||||
model={
|
||||
"online_model_name": "MiniMax-M3",
|
||||
"api_url": "https://model-proxy.example/v1",
|
||||
},
|
||||
config={"max_tokens": 1024, "json_mode": True, "generation_retries": 0},
|
||||
task_id="task-minimax-proxy",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert requests[0]["max_tokens"] == 1024
|
||||
assert requests[0]["response_format"] == {"type": "json_object"}
|
||||
assert "reasoning_split" not in requests[0]
|
||||
assert "max_completion_tokens" not in requests[0]
|
||||
|
||||
|
||||
def test_generate_model_records_extracts_json_surrounded_by_model_explanation() -> None:
|
||||
content = "模型结果如下:\n```json\n" + json.dumps(
|
||||
{
|
||||
"items": [{
|
||||
"instruction": "字段有什么作用?",
|
||||
"output": "该字段用于唯一标识记录。",
|
||||
}],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
) + "\n```\n生成完毕。"
|
||||
client = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda _: httpx.Response(
|
||||
200,
|
||||
json={"choices": [{"message": {"content": content}}]},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-explanation", "edited_content": "字段用于唯一标识记录。"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 0},
|
||||
task_id="task-explanation",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert records[0]["status"] == "valid"
|
||||
assert records[0]["output"] == "该字段用于唯一标识记录。"
|
||||
|
||||
|
||||
def test_generate_model_records_reports_token_truncation_instead_of_json_error() -> None:
|
||||
client = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda _: httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"finish_reason": "length",
|
||||
"message": {"content": ""},
|
||||
}],
|
||||
"output_sensitive": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-truncated", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 0},
|
||||
task_id="task-truncated",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert "Token" in records[0]["error"]
|
||||
assert "截断" in records[0]["error"]
|
||||
|
||||
|
||||
def test_token_truncation_is_not_retried_even_when_json_looks_complete() -> None:
|
||||
request_count = 0
|
||||
content = json.dumps({
|
||||
"items": [{
|
||||
"instruction": "问题",
|
||||
"output": "表面完整但服务端已声明截断。",
|
||||
}],
|
||||
}, ensure_ascii=False)
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"finish_reason": "length",
|
||||
"message": {"content": content},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-length", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 5},
|
||||
task_id="task-length",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 1
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert "finish_reason=length" in records[0]["error"]
|
||||
|
||||
|
||||
def test_sensitive_model_response_is_not_retried_or_saved() -> None:
|
||||
request_count = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"finish_reason": "stop",
|
||||
"message": {"content": "{}"},
|
||||
}],
|
||||
"output_sensitive": True,
|
||||
"base_resp": {"status_code": 1027, "status_msg": "output sensitive"},
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-sensitive", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 5},
|
||||
task_id="task-sensitive",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 1
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert "安全拦截" in records[0]["error"]
|
||||
assert "1027" in records[0]["error"]
|
||||
|
||||
|
||||
def test_empty_model_content_can_retry_then_succeed() -> None:
|
||||
request_count = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
if request_count == 1:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"choices": [{"finish_reason": "stop", "message": {"content": ""}}]},
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"finish_reason": "stop",
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "问题",
|
||||
"output": "第二次请求返回了完整答案。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-empty-retry", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 1},
|
||||
task_id="task-empty-retry",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 2
|
||||
assert records[0]["status"] == "valid"
|
||||
|
||||
|
||||
def test_multiple_top_level_json_documents_are_rejected_as_ambiguous() -> None:
|
||||
first = json.dumps({
|
||||
"items": [{"instruction": "问题一", "output": "答案一"}],
|
||||
}, ensure_ascii=False)
|
||||
second = json.dumps({
|
||||
"items": [{"instruction": "问题二", "output": "答案二"}],
|
||||
}, ensure_ascii=False)
|
||||
client = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda _: httpx.Response(
|
||||
200,
|
||||
json={"choices": [{"message": {"content": f"{first}\n{second}"}}]},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-ambiguous", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 0},
|
||||
task_id="task-ambiguous",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert "多个 JSON" in records[0]["error"]
|
||||
|
||||
|
||||
def test_generate_model_records_builds_reasoning_output_with_think_tags() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content)
|
||||
@@ -414,6 +752,112 @@ def test_generate_model_records_retries_short_batch_then_marks_it_invalid() -> N
|
||||
assert "expected 10, got 1" in records[0]["error"]
|
||||
|
||||
|
||||
def test_generate_model_records_does_not_retry_non_retryable_http_errors() -> None:
|
||||
request_count = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
return httpx.Response(401, json={"error": {"message": "unauthorized"}})
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-auth", "edited_content": "来源内容"}],
|
||||
model={
|
||||
"api_url": "https://model.example/v1",
|
||||
"online_model_name": "test-model",
|
||||
"api_key": "invalid",
|
||||
},
|
||||
config={"generation_retries": 5},
|
||||
task_id="task-auth",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 1
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert "401" in records[0]["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [408, 425, 429, 500])
|
||||
def test_generate_model_records_retries_retryable_http_statuses(
|
||||
status_code: int,
|
||||
) -> None:
|
||||
request_count = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
if request_count == 1:
|
||||
return httpx.Response(status_code)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "来源内容是什么?",
|
||||
"output": "这是用于验证可重试错误的来源内容。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-retryable", "edited_content": "来源内容"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 1},
|
||||
task_id="task-retryable",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 2
|
||||
assert records[0]["status"] == "valid"
|
||||
|
||||
|
||||
def test_generate_model_records_retries_transient_network_errors() -> None:
|
||||
request_count = 0
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal request_count
|
||||
request_count += 1
|
||||
if request_count == 1:
|
||||
raise httpx.ConnectError("temporary connection failure", request=request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": json.dumps({
|
||||
"items": [{
|
||||
"instruction": "网络恢复了吗?",
|
||||
"output": "临时连接错误后,第二次模型请求已经成功。",
|
||||
}],
|
||||
}, ensure_ascii=False),
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-network", "edited_content": "网络重试来源"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 1},
|
||||
task_id="task-network",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
|
||||
assert request_count == 2
|
||||
assert records[0]["status"] == "valid"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qa_pairs_per_item", [0, 51])
|
||||
def test_generate_model_records_rejects_out_of_range_count(
|
||||
qa_pairs_per_item: int,
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.modules.data_process.schema_cli import _target_label
|
||||
from app.modules.data_process.schema_cli import REQUIRED_TASK_COLUMNS, _target_label
|
||||
|
||||
|
||||
def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
|
||||
@@ -18,6 +18,24 @@ def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
|
||||
assert "requires 001_platform_runtime.sql first" in sql
|
||||
assert "supports only the current TEXT runtime schema" in sql
|
||||
assert "generation_run_id" in sql
|
||||
assert "results_confirmed BOOLEAN NOT NULL DEFAULT TRUE" in sql
|
||||
assert "WHERE status <> 'completed' AND results_confirmed=TRUE" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS workflow_step VARCHAR(20)" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_status VARCHAR(20)" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_progress NUMERIC(5,2)" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_run_id TEXT" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_failure_reason TEXT" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_total_files INTEGER" in sql
|
||||
assert "ADD COLUMN IF NOT EXISTS preview_completed_files INTEGER" in sql
|
||||
assert "data_process_workflow_backfill_ids" in sql
|
||||
assert "ck_data_process_tasks_workflow_step" in sql
|
||||
assert "ck_data_process_tasks_preview_status" in sql
|
||||
assert "ck_data_process_tasks_preview_progress" in sql
|
||||
assert "ck_data_process_tasks_preview_file_counts" in sql
|
||||
for value in ("create", "model", "upload", "preview", "generate", "results"):
|
||||
assert f"'{value}'" in sql
|
||||
for value in ("idle", "queued", "running", "completed", "failed", "cancelled"):
|
||||
assert f"'{value}'" in sql
|
||||
assert "CREATE TABLE IF NOT EXISTS data_process_results" in sql
|
||||
assert sql.count("BEGIN;") == 1
|
||||
assert sql.rstrip().endswith("COMMIT;")
|
||||
@@ -27,3 +45,17 @@ def test_schema_cli_target_label_never_contains_credentials() -> None:
|
||||
label = _target_label("postgresql://secret-user:secret-password@db.example:5433/yg_ft")
|
||||
assert label == "db.example:5433/yg_ft"
|
||||
assert "secret" not in label
|
||||
|
||||
|
||||
def test_schema_check_requires_current_runtime_columns() -> None:
|
||||
assert REQUIRED_TASK_COLUMNS == (
|
||||
"generation_run_id",
|
||||
"results_confirmed",
|
||||
"workflow_step",
|
||||
"preview_status",
|
||||
"preview_progress",
|
||||
"preview_run_id",
|
||||
"preview_failure_reason",
|
||||
"preview_total_files",
|
||||
"preview_completed_files",
|
||||
)
|
||||
|
||||
@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_stage_copy_creates_an_independently_deletable_source_object(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
original = _stage(storage, content=b"immutable source")
|
||||
storage.publish([original])
|
||||
|
||||
copied = storage.stage_copy(
|
||||
batch_id="batch-copy",
|
||||
source_reference=original.reference,
|
||||
expected_source_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
task_id="task-2",
|
||||
source_file_id="source-2",
|
||||
version=1,
|
||||
name="source.txt",
|
||||
)
|
||||
storage.publish([copied])
|
||||
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
original.reference,
|
||||
expected_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
) is True
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
copied.reference,
|
||||
expected_task_id="task-2",
|
||||
expected_source_file_id="source-2",
|
||||
) is True
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
@@ -17,9 +18,18 @@ from app.modules.data_process.store import (
|
||||
_preview_config_changed,
|
||||
_reasoning_output_is_valid,
|
||||
_source_storage_descriptor,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
def test_preview_replace_sql_never_uses_untyped_null_placeholders() -> None:
|
||||
source = inspect.getsource(DataProcessStore.replace_preview_items)
|
||||
|
||||
assert "%s IS NULL" not in source
|
||||
assert "is_direct_build = preview_run_id is None" in source
|
||||
assert "workflow_step=CASE WHEN %s THEN 'preview'" in source
|
||||
|
||||
|
||||
class _Result:
|
||||
def __init__(self, *, row: dict[str, Any] | None = None, rows: list[dict[str, Any]] | None = None):
|
||||
self.row = row
|
||||
@@ -146,6 +156,7 @@ class _PublishStore(DataProcessStore):
|
||||
return {
|
||||
"id": task_id,
|
||||
"status": "completed",
|
||||
"results_confirmed": True,
|
||||
"description": "",
|
||||
"config": self._task_config,
|
||||
"output_dataset_id": train_dataset and train_dataset["id"],
|
||||
@@ -321,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
|
||||
yield self._conn
|
||||
|
||||
|
||||
class _RepeatConnection:
|
||||
def __init__(self) -> None:
|
||||
self.source_files = [
|
||||
{
|
||||
"id": "source-old",
|
||||
"name": "source.jsonl",
|
||||
"size_bytes": 12,
|
||||
"record_count": 1,
|
||||
"file_format": "jsonl",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"content": '{"id":1}\n',
|
||||
"content_preview": '{"id":1}',
|
||||
"metadata": {"storage_backend": "local"},
|
||||
"created_by": "user-1",
|
||||
}
|
||||
]
|
||||
self.source_previews = [
|
||||
{
|
||||
"id": "preview-old",
|
||||
"source_file_id": "source-old",
|
||||
"original_content": '{"id":1}',
|
||||
"edited_content": '{"id":1,"checked":true}',
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"source_start_line": 1,
|
||||
"source_end_line": 1,
|
||||
"token_count": 5,
|
||||
"status": "modified",
|
||||
"quality_score": {"overall": 90},
|
||||
}
|
||||
]
|
||||
self.created_task: dict[str, Any] | None = None
|
||||
self.created_files: list[dict[str, Any]] = []
|
||||
self.created_previews: list[dict[str, Any]] = []
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if params is not None:
|
||||
assert normalized.count("%s") == len(params)
|
||||
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
|
||||
return _Result(row=None)
|
||||
if normalized.startswith("SELECT * FROM data_process_source_files"):
|
||||
return _Result(rows=[dict(item) for item in self.source_files])
|
||||
if normalized.startswith("SELECT * FROM data_process_preview_items"):
|
||||
return _Result(rows=[dict(item) for item in self.source_previews])
|
||||
if normalized.startswith("INSERT INTO data_process_tasks"):
|
||||
self.created_task = {
|
||||
"id": params[0],
|
||||
"name": params[1],
|
||||
"description": params[2],
|
||||
"status": "pending",
|
||||
"process_type": params[3],
|
||||
"source_dataset_id": params[4],
|
||||
"config": params[5],
|
||||
"progress": 20,
|
||||
"input_count": params[6],
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_total_files": params[7],
|
||||
"preview_completed_files": params[8],
|
||||
"created_at": params[15],
|
||||
"updated_at": params[16],
|
||||
}
|
||||
return _Result(row=dict(self.created_task))
|
||||
if normalized.startswith("INSERT INTO data_process_source_files"):
|
||||
self.created_files.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"storage_object_id": params[2],
|
||||
"content": params[8],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("INSERT INTO data_process_preview_items"):
|
||||
self.created_previews.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"source_file_id": params[2],
|
||||
"edited_content": params[4],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
|
||||
return _Result(
|
||||
row={
|
||||
"source_file_count": len(self.created_files),
|
||||
"preview_count": len(self.created_previews),
|
||||
}
|
||||
)
|
||||
raise AssertionError(f"unexpected SQL: {normalized}")
|
||||
|
||||
|
||||
class _RepeatStore(DataProcessStore):
|
||||
def __init__(self, conn: _RepeatConnection) -> None:
|
||||
self._conn = conn
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[_RepeatConnection]:
|
||||
yield self._conn
|
||||
|
||||
def _task_in_connection(
|
||||
self,
|
||||
conn: Any,
|
||||
task_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
assert task_id == "task-source"
|
||||
assert for_update is True
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": "原任务",
|
||||
"description": "原描述",
|
||||
"status": "completed",
|
||||
"process_type": "structured",
|
||||
"source_dataset_id": None,
|
||||
"config": {
|
||||
"temperature": 0.3,
|
||||
"_regeneration_prepared": {"prepared": True},
|
||||
},
|
||||
"results_confirmed": True,
|
||||
"preview_status": "completed",
|
||||
"tenant_id": "tenant-1",
|
||||
"project_id": "project-1",
|
||||
"owner_id": "owner-1",
|
||||
"created_by": "user-1",
|
||||
"updated_at": "2026-07-28T12:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
class _TaskListConnection:
|
||||
def __init__(self) -> None:
|
||||
self.task = {
|
||||
@@ -454,12 +599,14 @@ class _LegacyRecoveryConnection:
|
||||
record["preview_item_id"] = params[1]
|
||||
return _Result()
|
||||
if normalized.startswith("UPDATE data_process_tasks SET status='completed'"):
|
||||
assert "workflow_step='results'" in normalized
|
||||
self.task.update(
|
||||
{
|
||||
"status": "completed",
|
||||
"progress": 100,
|
||||
"output_dataset_id": params[0],
|
||||
"output_count": params[1],
|
||||
"workflow_step": "results",
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
@@ -496,6 +643,7 @@ class _StartGenerationConnection:
|
||||
config=config,
|
||||
output_dataset_id="dataset_train" if published_prepared else None,
|
||||
output_count=28,
|
||||
results_confirmed=published_prepared,
|
||||
),
|
||||
"generation_run_id": None,
|
||||
}
|
||||
@@ -512,6 +660,7 @@ class _StartGenerationConnection:
|
||||
assert normalized.startswith("UPDATE data_process_tasks SET config=%s, status='running'")
|
||||
assert "output_dataset_id=NULL" in normalized
|
||||
assert "output_count=0" in normalized
|
||||
assert "results_confirmed=FALSE" in normalized
|
||||
self.task.update(
|
||||
{
|
||||
"config": params[0],
|
||||
@@ -526,6 +675,7 @@ class _StartGenerationConnection:
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"generation_run_id": params[2],
|
||||
"results_confirmed": False,
|
||||
"updated_at": params[3],
|
||||
}
|
||||
)
|
||||
@@ -559,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
|
||||
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
|
||||
|
||||
|
||||
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
|
||||
conn = _RepeatConnection()
|
||||
store = _RepeatStore(conn)
|
||||
request_id = "repeat-request-0001"
|
||||
target_task_id = repeat_task_id("task-source", request_id)
|
||||
|
||||
repeated = store.repeat_task(
|
||||
"task-source",
|
||||
expected_updated_at="2026-07-28T12:00:00Z",
|
||||
request_id=request_id,
|
||||
file_copies={
|
||||
"source-old": {
|
||||
"id": "source-new",
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert repeated["created"] is True
|
||||
assert repeated["task"]["id"] == target_task_id
|
||||
assert repeated["task"]["config"] == {"temperature": 0.3}
|
||||
assert repeated["task"]["results_confirmed"] is False
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == 1
|
||||
assert conn.created_files == [
|
||||
{
|
||||
"id": "source-new",
|
||||
"task_id": target_task_id,
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
"content": '{"id":1}\n',
|
||||
}
|
||||
]
|
||||
assert conn.created_previews[0]["task_id"] == target_task_id
|
||||
assert conn.created_previews[0]["source_file_id"] == "source-new"
|
||||
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
|
||||
|
||||
|
||||
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
|
||||
decoded = _decode_row(
|
||||
{
|
||||
@@ -713,6 +904,7 @@ def test_start_generation_clears_previous_output_count() -> None:
|
||||
|
||||
assert task["status"] == "running"
|
||||
assert task["output_count"] == 0
|
||||
assert task["results_confirmed"] is False
|
||||
assert conn.results == []
|
||||
|
||||
|
||||
@@ -737,6 +929,7 @@ def test_prepared_published_task_survives_generation_preflight_failure() -> None
|
||||
assert conn.task["status"] == "completed"
|
||||
assert conn.task["output_dataset_id"] == "dataset_train"
|
||||
assert conn.task["output_count"] == 28
|
||||
assert conn.task["results_confirmed"] is True
|
||||
assert "_regeneration_prepared" in conn.task["config"]
|
||||
assert conn.results == [{"id": "old-result"}]
|
||||
|
||||
@@ -752,6 +945,7 @@ def test_legacy_aborted_regeneration_recovers_results_and_published_state() -> N
|
||||
assert conn.task["progress"] == 100
|
||||
assert conn.task["output_dataset_id"] == "dataset_train"
|
||||
assert conn.task["output_count"] == 2
|
||||
assert conn.task["workflow_step"] == "results"
|
||||
assert conn.task["started_at"] is None
|
||||
assert conn.task["completed_at"] is None
|
||||
assert [item["id"] for item in conn.results] == ["result_train", "result_test"]
|
||||
@@ -1185,3 +1379,96 @@ def test_source_storage_descriptor_rejects_unowned_or_unsupported_references(
|
||||
"dpt_task",
|
||||
"dpsf_source",
|
||||
)
|
||||
|
||||
|
||||
class _LifecycleConnection:
|
||||
def __init__(self) -> None:
|
||||
self.task: dict[str, Any] = {
|
||||
"id": "task-lifecycle",
|
||||
"status": "running",
|
||||
"generation_run_id": "generation-active",
|
||||
"workflow_step": "generate",
|
||||
"preview_status": "running",
|
||||
"preview_progress": Decimal("40.00"),
|
||||
"preview_run_id": "preview-active",
|
||||
"preview_failure_reason": None,
|
||||
"preview_total_files": 5,
|
||||
"preview_completed_files": 2,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
self.last_update_sql = ""
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if params is not None:
|
||||
assert normalized.count("%s") == len(params)
|
||||
if normalized.startswith("SELECT * FROM data_process_tasks"):
|
||||
row = None if self.task["deleted_at"] is not None else dict(self.task)
|
||||
return _Result(row=row)
|
||||
if normalized.startswith("UPDATE data_process_tasks SET workflow_step="):
|
||||
self.last_update_sql = normalized
|
||||
workflow_step, updated_at, task_id = params
|
||||
assert task_id == self.task["id"]
|
||||
self.task.update(workflow_step=workflow_step, updated_at=updated_at)
|
||||
return _Result(row=dict(self.task))
|
||||
if normalized.startswith("UPDATE data_process_tasks SET status=CASE"):
|
||||
self.last_update_sql = normalized
|
||||
deleted_at, deleted_by, updated_at, task_id = params
|
||||
assert task_id == self.task["id"]
|
||||
self.task.update(
|
||||
status="stopped",
|
||||
generation_run_id=None,
|
||||
preview_status="cancelled",
|
||||
preview_run_id=None,
|
||||
deleted_at=deleted_at,
|
||||
deleted_by=deleted_by,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("SELECT status, generation_run_id"):
|
||||
row = None if self.task["deleted_at"] is not None else dict(self.task)
|
||||
return _Result(row=row)
|
||||
if normalized.startswith("SELECT preview_status, preview_run_id"):
|
||||
row = None if self.task["deleted_at"] is not None else dict(self.task)
|
||||
return _Result(row=row)
|
||||
raise AssertionError(f"unexpected SQL: {normalized}")
|
||||
|
||||
|
||||
class _LifecycleStore(DataProcessStore):
|
||||
def __init__(self, conn: _LifecycleConnection) -> None:
|
||||
self._conn = conn
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[_LifecycleConnection]:
|
||||
yield self._conn
|
||||
|
||||
|
||||
def test_workflow_step_update_does_not_invalidate_active_runs() -> None:
|
||||
conn = _LifecycleConnection()
|
||||
|
||||
task = _LifecycleStore(conn).update_workflow_step("task-lifecycle", "results")
|
||||
|
||||
assert task["workflow_step"] == "results"
|
||||
assert task["status"] == "running"
|
||||
assert task["generation_run_id"] == "generation-active"
|
||||
assert task["preview_status"] == "running"
|
||||
assert task["preview_run_id"] == "preview-active"
|
||||
assert "generation_run_id" not in conn.last_update_sql
|
||||
assert "preview_run_id" not in conn.last_update_sql
|
||||
|
||||
|
||||
def test_delete_atomically_invalidates_generation_and_preview_runs() -> None:
|
||||
conn = _LifecycleConnection()
|
||||
store = _LifecycleStore(conn)
|
||||
|
||||
store.delete_task("task-lifecycle", deleted_by="user-1")
|
||||
|
||||
assert conn.task["status"] == "stopped"
|
||||
assert conn.task["generation_run_id"] is None
|
||||
assert conn.task["preview_status"] == "cancelled"
|
||||
assert conn.task["preview_run_id"] is None
|
||||
assert conn.task["deleted_by"] == "user-1"
|
||||
assert conn.task["deleted_at"] is not None
|
||||
assert store.generation_is_running("task-lifecycle", "generation-active") is False
|
||||
assert store.preview_is_running("task-lifecycle", "preview-active") is False
|
||||
|
||||
774
backend/tests/test_governance.py
Normal file
774
backend/tests/test_governance.py
Normal file
@@ -0,0 +1,774 @@
|
||||
"""
|
||||
平台治理功能集成测试 —— 覆盖第 1-4 周交付内容。
|
||||
|
||||
测试策略:
|
||||
- 在导入 app 模块前 mock psycopg / psycopg_pool,避免依赖真实数据库驱动
|
||||
- 使用 FastAPI TestClient 对真实路由栈发起请求
|
||||
- 通过 mock.get_platform_store 替换为内存 FakeStore
|
||||
- 每周交付内容对应一组 test class,方便分阶段验收
|
||||
|
||||
覆盖范围:
|
||||
第 1 周 — 登录、当前用户、用户列表、权限码、日志查询
|
||||
第 2 周 — 租户、项目、项目成员、资源 ACL
|
||||
第 3 周 — 审批实例、审批模板、审计日志查询和导出
|
||||
第 4 周 — 写操作审计、审批拦截、权限校验
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# ============================================================
|
||||
# 在导入 app 之前 mock psycopg / psycopg_pool
|
||||
# ============================================================
|
||||
|
||||
_psycopg_mock = types.ModuleType("psycopg")
|
||||
_psycopg_mock.PgConn = type("PgConn", (), {})
|
||||
_psycopg_mock.PostgresConnectionPool = MagicMock()
|
||||
_psycopg_mock.connection = MagicMock()
|
||||
sys.modules.setdefault("psycopg", _psycopg_mock)
|
||||
|
||||
_psycopg_pool_mock = types.ModuleType("psycopg_pool")
|
||||
_psycopg_pool_mock.ConnectionPool = MagicMock()
|
||||
sys.modules.setdefault("psycopg_pool", _psycopg_pool_mock)
|
||||
|
||||
# 现在安全导入 app 模块
|
||||
from app.api.v1.endpoints.platform import ok, fail # noqa: E402
|
||||
from app.modules.tenant.router import router as tenant_router # noqa: E402
|
||||
from app.modules.project.router import router as project_router # noqa: E402
|
||||
from app.modules.approval.router import router as approval_router # noqa: E402
|
||||
from app.modules.system.router import router as system_router # noqa: E402
|
||||
from app.modules.retention.router import router as retention_router # noqa: E402
|
||||
from app.modules.resource.router import router as resource_router # noqa: E402
|
||||
from app.api.v1.endpoints.platform import router as platform_router # noqa: E402
|
||||
|
||||
PREFIX = "/modelTF"
|
||||
ADMIN_TOKEN = "platform-token-u_admin"
|
||||
OP_TOKEN = "platform-token-u_op"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FakePlatformStore —— 内存实现,模拟 PlatformStore 全部治理接口
|
||||
# ============================================================
|
||||
|
||||
class FakePlatformStore:
|
||||
"""平台治理测试专用内存 store,确保测试不连接真实数据库。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._users: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "u_admin",
|
||||
"username": "admin",
|
||||
"display_name": "Admin",
|
||||
"role": "admin",
|
||||
"status": "active",
|
||||
"permissions": [
|
||||
"dashboard", "fine-tune", "model-eval", "model-inference",
|
||||
"model-manage", "dataset", "data-process", "data-convert",
|
||||
"compute", "hardware", "logs", "user-settings",
|
||||
],
|
||||
"last_login": "2026-08-01T10:00:00Z",
|
||||
"protected": True,
|
||||
},
|
||||
{
|
||||
"id": "u_op",
|
||||
"username": "operator",
|
||||
"display_name": "Operator",
|
||||
"role": "operator",
|
||||
"status": "active",
|
||||
"permissions": ["dashboard", "fine-tune"],
|
||||
"last_login": "2026-08-01T11:00:00Z",
|
||||
"protected": False,
|
||||
},
|
||||
]
|
||||
self._tenants: dict[str, dict[str, Any]] = {}
|
||||
self._projects: dict[str, dict[str, Any]] = {}
|
||||
self._members: dict[str, list[dict[str, Any]]] = {}
|
||||
self._acl: dict[str, list[dict[str, Any]]] = {}
|
||||
self._audit_logs: list[dict[str, Any]] = []
|
||||
self._approval_templates: dict[str, dict[str, Any]] = {}
|
||||
self._approval_instances: dict[str, dict[str, Any]] = {}
|
||||
self._retention_policies: dict[str, dict[str, Any]] = {}
|
||||
self._models: list[dict[str, Any]] = []
|
||||
self._datasets: list[dict[str, Any]] = []
|
||||
self._tasks: list[dict[str, Any]] = []
|
||||
self._compute_nodes: list[dict[str, Any]] = []
|
||||
self._gpus: list[dict[str, Any]] = []
|
||||
self._sessions: list[dict[str, Any]] = []
|
||||
self._seq = 0
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[Any]:
|
||||
class FakeConn:
|
||||
def execute(self, *a, **kw):
|
||||
return []
|
||||
|
||||
def commit(self):
|
||||
pass
|
||||
|
||||
def rollback(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
yield FakeConn()
|
||||
|
||||
# ---- helpers ----
|
||||
|
||||
def _next_id(self, prefix: str) -> str:
|
||||
self._seq += 1
|
||||
return f"{prefix}_{self._seq}"
|
||||
|
||||
# ==================== 第1周:登录 / 用户 / 权限码 / 日志 ====================
|
||||
|
||||
def login(self, username: str, password: str) -> dict[str, Any] | None:
|
||||
for u in self._users:
|
||||
if u["username"] == username and u["status"] == "active":
|
||||
if password in ("admin123", "operator123", "test123"):
|
||||
return dict(u)
|
||||
return None
|
||||
|
||||
def users(self) -> list[dict[str, Any]]:
|
||||
return [dict(u) for u in self._users]
|
||||
|
||||
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
u = {"id": self._next_id("u"), "protected": False, **payload}
|
||||
self._users.append(u)
|
||||
return u
|
||||
|
||||
def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
for u in self._users:
|
||||
if u["id"] == user_id:
|
||||
u.update(payload)
|
||||
return u
|
||||
raise KeyError(user_id)
|
||||
|
||||
def delete_user(self, user_id: str) -> None:
|
||||
self._users = [u for u in self._users if u["id"] != user_id]
|
||||
|
||||
def roles(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": "admin", "display_name": "管理员"},
|
||||
{"name": "operator", "display_name": "操作员"},
|
||||
{"name": "viewer", "display_name": "访客"},
|
||||
]
|
||||
|
||||
def log_files(self, date: str | None = None) -> list[dict[str, Any]]:
|
||||
return [{"name": "backend-2026-08-01.log", "size": "1 KB", "date": "2026-08-01"}]
|
||||
|
||||
def log_content(self, file: str) -> dict[str, Any]:
|
||||
return {"file": file, "content": "[INFO] test line", "size": "1 KB"}
|
||||
|
||||
def training_log_files(self) -> list[dict[str, Any]]:
|
||||
return [{"task_id": "ft_001", "name": "ft_001.log", "size": "2 KB"}]
|
||||
|
||||
def training_log_content(self, file: str) -> dict[str, Any]:
|
||||
return {"file": file, "content": "epoch 0 loss 1.0", "size": "2 KB"}
|
||||
|
||||
# ==================== 第2周:租户 / 项目 / 成员 / ACL ====================
|
||||
|
||||
def tenants(self) -> list[dict[str, Any]]:
|
||||
return list(self._tenants.values())
|
||||
|
||||
def tenant(self, tenant_id: str) -> dict[str, Any]:
|
||||
if tenant_id not in self._tenants:
|
||||
raise KeyError(tenant_id)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
tid = self._next_id("tnt")
|
||||
t = {"id": tid, "status": "active", "quota": "{}", "retention_policy_id": None,
|
||||
"create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._tenants[tid] = t
|
||||
return dict(t)
|
||||
|
||||
def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._tenants[tenant_id].update(payload)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def set_tenant_quota(self, tenant_id: str, quota: dict[str, Any]) -> dict[str, Any]:
|
||||
self._tenants[tenant_id]["quota"] = json.dumps(quota)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def set_tenant_retention(self, tenant_id: str, retention_policy_id: str | None) -> dict[str, Any]:
|
||||
self._tenants[tenant_id]["retention_policy_id"] = retention_policy_id
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def projects(self, *, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for p in self._projects.values():
|
||||
if p.get("tenant_id") != tenant_id:
|
||||
continue
|
||||
if status and p.get("status") != status:
|
||||
continue
|
||||
if keyword and keyword.lower() not in p.get("name", "").lower():
|
||||
continue
|
||||
result.append(dict(p))
|
||||
return result
|
||||
|
||||
def project(self, project_id: str) -> dict[str, Any]:
|
||||
if project_id not in self._projects:
|
||||
raise KeyError(project_id)
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def create_project(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
pid = self._next_id("prj")
|
||||
p = {"id": pid, "status": "active", "quota": "{}", "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._projects[pid] = p
|
||||
self._members[pid] = []
|
||||
return dict(p)
|
||||
|
||||
def update_project(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._projects[project_id].update(payload)
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def archive_project(self, project_id: str) -> dict[str, Any]:
|
||||
self._projects[project_id]["status"] = "archived"
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def delete_project(self, project_id: str) -> None:
|
||||
self._projects.pop(project_id, None)
|
||||
self._members.pop(project_id, None)
|
||||
|
||||
def project_members(self, project_id: str) -> list[dict[str, Any]]:
|
||||
return [dict(m) for m in self._members.get(project_id, [])]
|
||||
|
||||
def add_project_member(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
m = {"joined_at": "2026-08-01T00:00:00Z", **payload}
|
||||
self._members.setdefault(project_id, []).append(m)
|
||||
return m
|
||||
|
||||
def update_project_member_role(self, project_id: str, user_id: str, role: str) -> dict[str, Any]:
|
||||
for m in self._members.get(project_id, []):
|
||||
if m["user_id"] == user_id:
|
||||
m["role"] = role
|
||||
return m
|
||||
raise KeyError(user_id)
|
||||
|
||||
def remove_project_member(self, project_id: str, user_id: str) -> None:
|
||||
self._members[project_id] = [m for m in self._members.get(project_id, []) if m["user_id"] != user_id]
|
||||
|
||||
# ---- ACL ----
|
||||
|
||||
def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
key = f"{resource_type}:{resource_id}"
|
||||
return [dict(a) for a in self._acl.get(key, [])]
|
||||
|
||||
def set_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
key = f"{resource_type}:{resource_id}"
|
||||
self._acl[key] = [dict(e) for e in entries]
|
||||
return self.get_acl(resource_type, resource_id)
|
||||
|
||||
def resource_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
rows = self.get_acl(resource_type, resource_id)
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
k = f"{r.get('principal_type')}:{r.get('principal_id')}"
|
||||
bucket = grouped.setdefault(k, {
|
||||
"subject_type": r.get("principal_type"),
|
||||
"subject_id": r.get("principal_id"),
|
||||
"permissions": [],
|
||||
})
|
||||
perm = r.get("permission")
|
||||
if perm and perm not in bucket["permissions"]:
|
||||
bucket["permissions"].append(perm)
|
||||
return list(grouped.values())
|
||||
|
||||
def set_resource_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
flat: list[dict[str, Any]] = []
|
||||
for e in entries:
|
||||
for perm in e.get("permissions") or []:
|
||||
flat.append({
|
||||
"principal_type": e.get("subject_type"),
|
||||
"principal_id": e.get("subject_id"),
|
||||
"permission": perm,
|
||||
})
|
||||
self.set_acl(resource_type, resource_id, flat)
|
||||
return self.resource_acl(resource_type, resource_id)
|
||||
|
||||
# ==================== 第3周:审批 / 审计 / 留存 ====================
|
||||
|
||||
def approval_templates(self) -> list[dict[str, Any]]:
|
||||
return list(self._approval_templates.values())
|
||||
|
||||
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
tid = payload.get("id") or self._next_id("tpl")
|
||||
t = {"id": tid, "steps": [], "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._approval_templates[tid] = t
|
||||
return dict(t)
|
||||
|
||||
def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for i in self._approval_instances.values():
|
||||
if status and i.get("status") != status:
|
||||
continue
|
||||
result.append(dict(i))
|
||||
return result
|
||||
|
||||
def approval_instance(self, instance_id: str) -> dict[str, Any]:
|
||||
if instance_id not in self._approval_instances:
|
||||
raise KeyError(instance_id)
|
||||
return dict(self._approval_instances[instance_id])
|
||||
|
||||
def create_approval_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
iid = self._next_id("appr")
|
||||
inst = {
|
||||
"id": iid,
|
||||
"status": "pending",
|
||||
"current_step": 0,
|
||||
"steps": [],
|
||||
"create_time": "2026-08-01T00:00:00Z",
|
||||
**payload,
|
||||
}
|
||||
self._approval_instances[iid] = inst
|
||||
return dict(inst)
|
||||
|
||||
def decide_approval_step(self, instance_id: str, step_index: int, *, approver_id: str, approved: bool, comment: str | None = None) -> dict[str, Any]:
|
||||
inst = self._approval_instances[instance_id]
|
||||
inst["status"] = "approved" if approved else "rejected"
|
||||
inst["current_step"] = step_index + 1
|
||||
return dict(inst)
|
||||
|
||||
def audit_logs(self, **kw) -> dict[str, Any]:
|
||||
items = [dict(l) for l in self._audit_logs]
|
||||
for filter_key in ("tenant_id", "project_id", "actor_id", "action", "target_type"):
|
||||
val = kw.get(filter_key)
|
||||
if val:
|
||||
items = [l for l in items if l.get(filter_key) == val]
|
||||
limit = kw.get("limit", 50)
|
||||
offset = kw.get("offset", 0)
|
||||
total = len(items)
|
||||
items = items[offset:offset + limit]
|
||||
return {"items": items, "total": total}
|
||||
|
||||
def record_audit(self, **kw) -> None:
|
||||
log = {"id": self._next_id("log"), "time": "2026-08-01T12:00:00Z", **kw}
|
||||
self._audit_logs.append(log)
|
||||
|
||||
# ---- 留存策略 ----
|
||||
|
||||
def retention_policies(self) -> list[dict[str, Any]]:
|
||||
return list(self._retention_policies.values())
|
||||
|
||||
def retention_policy(self, policy_id: str) -> dict[str, Any]:
|
||||
if policy_id not in self._retention_policies:
|
||||
raise KeyError(policy_id)
|
||||
return dict(self._retention_policies[policy_id])
|
||||
|
||||
def create_retention_policy(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
pid = payload.get("id") or self._next_id("rpol")
|
||||
p = {"id": pid, "status": "active", "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._retention_policies[pid] = p
|
||||
return dict(p)
|
||||
|
||||
def update_retention_policy(self, policy_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._retention_policies[policy_id].update(payload)
|
||||
return dict(self._retention_policies[policy_id])
|
||||
|
||||
def delete_retention_policy(self, policy_id: str) -> None:
|
||||
self._retention_policies.pop(policy_id, None)
|
||||
|
||||
# ---- dashboard & other stubs ----
|
||||
|
||||
def login_duration_rank(self, limit: int = 8, days: int = 30) -> list[dict[str, Any]]:
|
||||
return [{"user": "admin", "role": "admin", "duration": 10.0}]
|
||||
|
||||
def models(self) -> list[dict[str, Any]]:
|
||||
return self._models
|
||||
|
||||
def datasets(self) -> list[dict[str, Any]]:
|
||||
return self._datasets
|
||||
|
||||
def tasks(self) -> list[dict[str, Any]]:
|
||||
return self._tasks
|
||||
|
||||
def compute_nodes(self) -> list[dict[str, Any]]:
|
||||
return self._compute_nodes
|
||||
|
||||
def gpus(self) -> list[dict[str, Any]]:
|
||||
return self._gpus
|
||||
|
||||
def system_info(self) -> dict[str, Any]:
|
||||
return {"cpu": {}, "memory": {}}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 测试 fixtures
|
||||
# ============================================================
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def fake_store() -> FakePlatformStore:
|
||||
return FakePlatformStore()
|
||||
|
||||
|
||||
def _build_client(store: FakePlatformStore) -> TestClient:
|
||||
"""构建 TestClient,patch 所有治理模块的 get_platform_store。"""
|
||||
app = FastAPI()
|
||||
app.include_router(platform_router, prefix=PREFIX)
|
||||
app.include_router(system_router, prefix=PREFIX)
|
||||
app.include_router(tenant_router, prefix=PREFIX)
|
||||
app.include_router(project_router, prefix=PREFIX)
|
||||
app.include_router(approval_router, prefix=PREFIX)
|
||||
app.include_router(retention_router, prefix=PREFIX)
|
||||
app.include_router(resource_router, prefix=PREFIX)
|
||||
|
||||
patches = [
|
||||
patch("app.db.platform_store.get_platform_store", return_value=store),
|
||||
patch("app.core.auth.get_platform_store", return_value=store),
|
||||
patch("app.api.v1.endpoints.platform.get_platform_store", return_value=store),
|
||||
patch("app.modules.system.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.tenant.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.project.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.approval.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.retention.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.resource.router.get_platform_store", return_value=store),
|
||||
]
|
||||
for p in patches:
|
||||
p.start()
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
client._fake_store = store # type: ignore[attr-defined]
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client(fake_store: FakePlatformStore) -> TestClient:
|
||||
c = _build_client(fake_store)
|
||||
yield c
|
||||
|
||||
|
||||
def _admin_headers() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {ADMIN_TOKEN}"}
|
||||
|
||||
|
||||
def _op_headers() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {OP_TOKEN}"}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 1 周测试:登录、当前用户、用户列表、权限码、日志查询
|
||||
# ============================================================
|
||||
|
||||
class TestWeek1AuthUserPermissionsLogs:
|
||||
"""第 1 周:登录、当前用户、用户列表、权限码、日志查询接口。"""
|
||||
|
||||
def test_login_success(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "admin123"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert data["token"] == ADMIN_TOKEN
|
||||
assert data["user"]["username"] == "admin"
|
||||
|
||||
def test_login_invalid(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "wrong"})
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_me_with_valid_token(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/me", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["username"] == "admin"
|
||||
|
||||
def test_me_without_token(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/me")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_users_list(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/users", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
users = resp.json()["data"]
|
||||
assert len(users) >= 2
|
||||
assert any(u["username"] == "admin" for u in users)
|
||||
|
||||
def test_create_user(self, client: TestClient):
|
||||
resp = client.post(
|
||||
f"{PREFIX}/users",
|
||||
json={"username": "tester", "display_name": "Tester", "role": "viewer", "password": "test123"},
|
||||
headers=_admin_headers(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["username"] == "tester"
|
||||
|
||||
def test_permission_codes(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/permissions/codes")
|
||||
assert resp.status_code == 200
|
||||
codes = resp.json()["data"]["codes"]
|
||||
assert "dashboard" in codes
|
||||
assert "user-settings" in codes
|
||||
|
||||
def test_permissions_overview(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/permissions")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "codes" in data
|
||||
assert "roles" in data
|
||||
|
||||
def test_log_files(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/log-files", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
files = resp.json()["data"]
|
||||
assert len(files) >= 1
|
||||
|
||||
def test_log_content(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/log-content", params={"file": "backend.log"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "content" in resp.json()["data"]
|
||||
|
||||
def test_training_log_files(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/training-log-files", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) >= 1
|
||||
|
||||
def test_training_log_content(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/training-log-content", params={"file": "ft_001.log"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "content" in resp.json()["data"]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 2 周测试:租户、项目、项目成员、资源 ACL
|
||||
# ============================================================
|
||||
|
||||
class TestWeek2TenantProjectACL:
|
||||
"""第 2 周:租户、项目、项目成员、资源 ACL。"""
|
||||
|
||||
def test_tenant_crud(self, client: TestClient):
|
||||
# 创建
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "Tenant-A", "code": "ta"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
tid = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/tenants", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(t["id"] == tid for t in resp.json()["data"])
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/tenants/{tid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Tenant-A"
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}", json={"name": "Tenant-A2"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Tenant-A2"
|
||||
|
||||
def test_tenant_quota(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "Q-Tenant", "code": "qt"}, headers=_admin_headers())
|
||||
tid = resp.json()["data"]["id"]
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}/quota", json={"quota": {"gpu": 4}}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_tenant_retention(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "R-Tenant", "code": "rt"}, headers=_admin_headers())
|
||||
tid = resp.json()["data"]["id"]
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}/retention-policy", json={"retention_policy_id": "rpol_1"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_project_crud(self, client: TestClient):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-1", "code": "p1", "tenant_id": "default"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/projects", params={"tenant_id": "default"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(p["id"] == pid for p in resp.json()["data"])
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/projects/{pid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Proj-1"
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/projects/{pid}", json={"description": "updated"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 归档
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "archived"
|
||||
|
||||
def test_project_members(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-M", "code": "pm", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 加成员
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/members", json={"user_id": "u_op", "role": "developer"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 列成员
|
||||
resp = client.get(f"{PREFIX}/projects/{pid}/members", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) >= 1
|
||||
# 改角色
|
||||
resp = client.put(f"{PREFIX}/projects/{pid}/members/u_op", json={"role": "maintainer"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 删成员
|
||||
resp = client.delete(f"{PREFIX}/projects/{pid}/members/u_op", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_resource_acl(self, client: TestClient):
|
||||
# 设置 ACL
|
||||
resp = client.put(
|
||||
f"{PREFIX}/resources/model/m001/acl",
|
||||
json={"entries": [{"subject_type": "user", "subject_id": "u_op", "permissions": ["read", "write"]}]},
|
||||
headers=_admin_headers(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
result = resp.json()["data"]
|
||||
assert len(result) == 1
|
||||
assert set(result[0]["permissions"]) == {"read", "write"}
|
||||
# 查询 ACL
|
||||
resp = client.get(f"{PREFIX}/resources/model/m001/acl", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 3 周测试:审批实例、审批模板、审计日志查询和导出
|
||||
# ============================================================
|
||||
|
||||
class TestWeek3ApprovalAudit:
|
||||
"""第 3 周:审批实例、审批模板、审计日志查询和导出。"""
|
||||
|
||||
def test_approval_template_crud(self, client: TestClient):
|
||||
# 创建模板
|
||||
resp = client.post(f"{PREFIX}/approvals/templates", json={"name": "delete-approval", "steps": [{"approver_id": "u_admin", "status": "pending"}]}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
tpl_id = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/approvals/templates", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(t["id"] == tpl_id for t in resp.json()["data"])
|
||||
|
||||
def test_approval_instance_flow(self, client: TestClient):
|
||||
# 创建审批实例
|
||||
resp = client.post(f"{PREFIX}/approvals", json={
|
||||
"resource_type": "dataset", "resource_id": "ds_001",
|
||||
"applicant_id": "u_op",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
iid = resp.json()["data"]["id"]
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/approvals/{iid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "pending"
|
||||
# 审批决策
|
||||
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
|
||||
"approver_id": "u_admin", "approved": True, "comment": "ok",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "approved"
|
||||
|
||||
def test_approval_instance_reject(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/approvals", json={
|
||||
"resource_type": "model", "resource_id": "m_002",
|
||||
"applicant_id": "u_op",
|
||||
}, headers=_admin_headers())
|
||||
iid = resp.json()["data"]["id"]
|
||||
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
|
||||
"approver_id": "u_admin", "approved": False, "comment": "no",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "rejected"
|
||||
|
||||
def test_approval_missing_field(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/approvals", json={"resource_type": "dataset"}, headers=_admin_headers())
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_audit_logs_query(self, client: TestClient):
|
||||
# 通过 API 写操作触发审计
|
||||
client.post(f"{PREFIX}/tenants", json={"name": "Audit-Tenant", "code": "at"}, headers=_admin_headers())
|
||||
# 查询
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs", params={"limit": 50}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "items" in data
|
||||
assert "total" in data
|
||||
assert data["total"] >= 1
|
||||
|
||||
def test_audit_logs_filter_by_action(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs", params={"action": "tenant.create"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
items = resp.json()["data"]["items"]
|
||||
assert all(i.get("action") == "tenant.create" for i in items)
|
||||
|
||||
def test_audit_logs_export_csv(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs/export", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "text/csv" in resp.headers.get("content-type", "")
|
||||
# CSV 首行是表头
|
||||
lines = resp.text.strip().split("\n")
|
||||
assert "time" in lines[0]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 4 周测试:写操作审计、审批拦截、权限校验
|
||||
# ============================================================
|
||||
|
||||
class TestWeek4AuditInterceptPermission:
|
||||
"""第 4 周:写操作审计、审批拦截、权限校验。"""
|
||||
|
||||
def test_write_operation_produces_audit(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 清空审计日志便于断言
|
||||
fake_store._audit_logs.clear()
|
||||
# 创建租户 → 应产生 tenant.create 审计
|
||||
client.post(f"{PREFIX}/tenants", json={"name": "W-Tenant", "code": "wt"}, headers=_admin_headers())
|
||||
assert any(l["action"] == "tenant.create" for l in fake_store._audit_logs)
|
||||
# 创建项目 → 应产生 project.create 审计
|
||||
client.post(f"{PREFIX}/projects", json={"name": "W-Proj", "code": "wp", "tenant_id": "default"}, headers=_admin_headers())
|
||||
assert any(l["action"] == "project.create" for l in fake_store._audit_logs)
|
||||
# 设置 ACL → 应产生 resource.acl.set 审计
|
||||
client.put(f"{PREFIX}/resources/model/w001/acl", json={"entries": []}, headers=_admin_headers())
|
||||
assert any(l["action"] == "resource.acl.set" for l in fake_store._audit_logs)
|
||||
|
||||
def test_approval_intercept_on_project_archive(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "I-Proj", "code": "ip", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 无待审批 → 可归档
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_approval_intercept_blocks_when_pending(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "B-Proj", "code": "bp", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 注入一条待审批实例
|
||||
fake_store.create_approval_instance({
|
||||
"resource_type": "project",
|
||||
"resource_id": pid,
|
||||
"applicant_id": "u_op",
|
||||
})
|
||||
# 有待审批 → 归档应被拒绝
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 409
|
||||
|
||||
def test_retention_policy_crud_with_audit(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
fake_store._audit_logs.clear()
|
||||
# 创建
|
||||
resp = client.post(f"{PREFIX}/retention-policies", json={"name": "30d-keep", "scope": "tenant"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
rpid = resp.json()["data"]["id"]
|
||||
assert any(l["action"] == "retention.create" for l in fake_store._audit_logs)
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/retention-policies", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(p["id"] == rpid for p in resp.json()["data"])
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/retention-policies/{rpid}", json={"status": "inactive"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "inactive"
|
||||
# 删除
|
||||
resp = client.delete(f"{PREFIX}/retention-policies/{rpid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_login_duration_rank_in_dashboard(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/dashboard/stats", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "login_duration_rank" in data
|
||||
assert "recent_login_users" in data
|
||||
assert "service_status" in data
|
||||
assert "training_7d" in data
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
import hashlib
|
||||
@@ -10,10 +12,11 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
|
||||
from compute.agent.process_manager import ProcessManager
|
||||
from compute.engines.llama_factory.adapter import build_command, parse_log_line, prepare_runtime_files
|
||||
from compute.engines.llama_factory.inference import get_inference_session
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@@ -448,6 +451,29 @@ def create_app() -> FastAPI:
|
||||
accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload)
|
||||
errors.extend(accelerator_errors)
|
||||
warnings.extend(accelerator_warnings)
|
||||
elif engine == "eval":
|
||||
# Eval engine: validate model path and dataset path
|
||||
if not payload.get("model_name_or_path"):
|
||||
errors.append("model_name_or_path is required for eval")
|
||||
else:
|
||||
path_checks.append(_check_path_item({
|
||||
"name": "model_name_or_path",
|
||||
"path": payload.get("model_name_or_path", ""),
|
||||
"type": "any",
|
||||
"required": True,
|
||||
}))
|
||||
if payload.get("dataset_path"):
|
||||
path_checks.append(_check_path_item({
|
||||
"name": "dataset_path",
|
||||
"path": payload.get("dataset_path", ""),
|
||||
"type": "file",
|
||||
"required": True,
|
||||
}))
|
||||
else:
|
||||
errors.append("dataset_path is required for eval")
|
||||
if shutil.which("python") is None:
|
||||
errors.append("python runtime not found")
|
||||
|
||||
elif engine == "smoke":
|
||||
warnings.append("smoke engine skips model and dataset path checks")
|
||||
|
||||
@@ -666,6 +692,88 @@ def create_app() -> FastAPI:
|
||||
metrics = [parse_log_line(line) for line in window["content"].splitlines()]
|
||||
return {"job_id": job_id, **window, "metrics": [m for m in metrics if m]}
|
||||
|
||||
# ── Inference Endpoints ───────────────────────────────────────────
|
||||
|
||||
@app.post(f"{route_prefix}/inference/load")
|
||||
async def inference_load(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Load a model for inference using LLaMA-Factory ChatModel.
|
||||
|
||||
Expected payload:
|
||||
model_name_or_path: str (required)
|
||||
adapter_name_or_path: str (optional, for LoRA adapters)
|
||||
template: str (default: "qwen")
|
||||
infer_backend: str (default: "huggingface")
|
||||
infer_dtype: str (default: "auto")
|
||||
"""
|
||||
session = get_inference_session()
|
||||
result = session.load(
|
||||
model_name_or_path=payload.get("model_name_or_path", ""),
|
||||
adapter_name_or_path=payload.get("adapter_name_or_path", ""),
|
||||
template=payload.get("template", "qwen"),
|
||||
infer_backend=payload.get("infer_backend", "huggingface"),
|
||||
infer_dtype=payload.get("infer_dtype", "auto"),
|
||||
)
|
||||
return result
|
||||
|
||||
@app.post(f"{route_prefix}/inference/unload")
|
||||
async def inference_unload() -> dict[str, Any]:
|
||||
"""Unload the currently loaded model and free GPU memory."""
|
||||
# Teardown (gc.collect + cuda.empty_cache) can take a while; run it off
|
||||
# the event loop so /health and /inference/status stay responsive.
|
||||
return await asyncio.to_thread(get_inference_session().unload)
|
||||
|
||||
@app.get(f"{route_prefix}/inference/status")
|
||||
async def inference_status() -> dict[str, Any]:
|
||||
"""Get the current inference session status."""
|
||||
return get_inference_session().info()
|
||||
|
||||
@app.post(f"{route_prefix}/inference/chat")
|
||||
async def inference_chat(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Chat with the loaded model (non-streaming).
|
||||
|
||||
Expected payload:
|
||||
messages: list[dict] (OpenAI format)
|
||||
temperature: float (default 0.95)
|
||||
top_p: float (default 0.7)
|
||||
max_new_tokens: int (default 1024)
|
||||
"""
|
||||
messages = payload.get("messages") or []
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages is required")
|
||||
# Generation is long-running; run it in a thread so the event loop keeps
|
||||
# serving /inference/status and /health during inference.
|
||||
result = await asyncio.to_thread(
|
||||
get_inference_session().chat,
|
||||
messages=messages,
|
||||
temperature=float(payload.get("temperature", 0.95)),
|
||||
top_p=float(payload.get("top_p", 0.7)),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
)
|
||||
if result.get("error"):
|
||||
raise HTTPException(status_code=500, detail=result["error"])
|
||||
return {"response": result["response"]}
|
||||
|
||||
@app.post(f"{route_prefix}/inference/chat/stream")
|
||||
async def inference_chat_stream(payload: dict[str, Any]) -> StreamingResponse:
|
||||
"""Chat with streaming response (Server-Sent Events)."""
|
||||
messages = payload.get("messages") or []
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages is required")
|
||||
|
||||
def generate():
|
||||
session = get_inference_session()
|
||||
for chunk in session.chat_stream(
|
||||
messages=messages,
|
||||
temperature=float(payload.get("temperature", 0.95)),
|
||||
top_p=float(payload.get("top_p", 0.7)),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/upload")
|
||||
async def upload_file(
|
||||
file: UploadFile | None = File(default=None),
|
||||
@@ -733,6 +841,22 @@ def create_app() -> FastAPI:
|
||||
"checksum_sha256": checksum,
|
||||
}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/read")
|
||||
async def read_file(path: str = Query(...)) -> JSONResponse:
|
||||
"""Read a text file from within YG_FT_DATA_ROOT. Used by the backend
|
||||
to fetch eval results and other job outputs."""
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
target = (data_root / path.lstrip("/\\")).resolve()
|
||||
if not _path_inside(data_root, target):
|
||||
raise HTTPException(status_code=400, detail="path must stay inside YG_FT_DATA_ROOT")
|
||||
if not target.is_file():
|
||||
raise HTTPException(status_code=404, detail="file not found")
|
||||
try:
|
||||
content = target.read_text(encoding="utf-8")
|
||||
return JSONResponse(json.loads(content) if content.strip().startswith("{") else {"content": content})
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
|
||||
async def download_file(file_id: str) -> FileResponse:
|
||||
upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads"
|
||||
|
||||
@@ -204,6 +204,31 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||
return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
|
||||
|
||||
if engine == "eval":
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'eval-job')}"
|
||||
eval_config_path = str(Path(output_dir) / "eval_config.json")
|
||||
eval_config = {
|
||||
"model_name_or_path": config.get("model_name_or_path", ""),
|
||||
"adapter_name_or_path": config.get("adapter_name_or_path", ""),
|
||||
"template": config.get("template", "qwen"),
|
||||
"dataset_path": config.get("dataset_path", ""),
|
||||
"output_dir": output_dir,
|
||||
"basic_metrics": config.get("basic_metrics", {}),
|
||||
"dimension": config.get("dimension", {}),
|
||||
"temperature": config.get("temperature", 0.1),
|
||||
"top_p": config.get("top_p", 0.95),
|
||||
"max_new_tokens": config.get("max_new_tokens", 512),
|
||||
"infer_backend": config.get("infer_backend", "huggingface"),
|
||||
"infer_dtype": config.get("infer_dtype", "auto"),
|
||||
}
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(eval_config_path).write_text(json.dumps(eval_config, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return LlamaFactoryCommand(
|
||||
command=["python", "-u", "-m", "compute.engines.llama_factory.eval_runner", "--config", eval_config_path],
|
||||
work_dir="/app",
|
||||
env={},
|
||||
)
|
||||
|
||||
errors = validate_config(config)
|
||||
if errors:
|
||||
raise ValueError("; ".join(errors))
|
||||
|
||||
485
compute/engines/llama_factory/eval_runner.py
Normal file
485
compute/engines/llama_factory/eval_runner.py
Normal file
@@ -0,0 +1,485 @@
|
||||
"""
|
||||
Evaluation runner — executes model evaluation as a subprocess job.
|
||||
|
||||
Usage:
|
||||
python -m compute.engines.llama_factory.eval_runner --config <config_json_path>
|
||||
|
||||
The config JSON is written by the compute API before spawning this subprocess.
|
||||
Results are written to ``output_dir/eval_results.json`` and progress is printed
|
||||
to stdout (captured as job logs).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_dataset(path: str) -> list[dict[str, Any]]:
|
||||
"""Load a JSON or JSONL dataset file.
|
||||
|
||||
Supports common field names used across the platform:
|
||||
* ``instruction`` + ``input`` + ``output`` (Alpaca-style)
|
||||
* ``question`` + ``answer``
|
||||
* ``messages`` (ShareGPT-style – the last assistant message is treated as reference)
|
||||
"""
|
||||
file_path = Path(path)
|
||||
text = file_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return []
|
||||
if file_path.suffix.lower() == ".json":
|
||||
value = json.loads(text)
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return [value] if isinstance(value, dict) else []
|
||||
|
||||
samples: list[dict[str, Any]] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
samples.append(obj)
|
||||
return samples
|
||||
|
||||
|
||||
def _sample_question(sample: dict[str, Any]) -> str:
|
||||
"""Extract the user-facing question / instruction from a sample."""
|
||||
if sample.get("instruction"):
|
||||
text = sample["instruction"]
|
||||
if sample.get("input"):
|
||||
text += "\n" + sample["input"]
|
||||
return text
|
||||
if sample.get("question"):
|
||||
return sample["question"]
|
||||
# ShareGPT-style: use the last user message as question
|
||||
messages = sample.get("messages") or []
|
||||
user_msgs = [m["content"] for m in messages if m.get("role") == "user"]
|
||||
return user_msgs[-1] if user_msgs else ""
|
||||
|
||||
|
||||
def _sample_reference(sample: dict[str, Any]) -> str:
|
||||
"""Extract the reference answer from a sample."""
|
||||
if sample.get("output"):
|
||||
return sample["output"]
|
||||
if sample.get("answer"):
|
||||
return sample["answer"]
|
||||
messages = sample.get("messages") or []
|
||||
assistant_msgs = [m["content"] for m in messages if m.get("role") == "assistant"]
|
||||
return assistant_msgs[-1] if assistant_msgs else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Basic metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4) -> dict[str, Any]:
|
||||
"""Compute BLEU score via sacrebleu (corpus-level)."""
|
||||
try:
|
||||
from sacrebleu.metrics import BLEU
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
|
||||
bleu = BLEU(max_ngram_order=ngram)
|
||||
# sacrebleu expects list-of-strings; we have one reference per prediction
|
||||
score = bleu.corpus_score(predictions, [references])
|
||||
return {
|
||||
"enabled": True,
|
||||
"score": round(score.score, 2),
|
||||
"bleu": round(score.score, 2),
|
||||
}
|
||||
|
||||
|
||||
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
|
||||
"""Compute ROUGE scores via rouge-score."""
|
||||
try:
|
||||
from rouge_score import rouge_scorer
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "rouge-score not installed", "score": 0}
|
||||
methods = methods or ["rouge1", "rouge2", "rougeL"]
|
||||
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
|
||||
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
|
||||
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
|
||||
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
|
||||
totals: dict[str, float] = {}
|
||||
n = max(len(predictions), 1)
|
||||
for ref, pred in zip(references, predictions):
|
||||
result = scorer.score(ref, pred)
|
||||
for key in methods:
|
||||
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
||||
avg = {k: round(v / n, 4) for k, v in totals.items()}
|
||||
return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
|
||||
|
||||
|
||||
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
"""Compute average cosine similarity via sklearn."""
|
||||
try:
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
|
||||
try:
|
||||
vectorizer = TfidfVectorizer()
|
||||
tfidf = vectorizer.fit_transform(references + predictions)
|
||||
n = len(references)
|
||||
ref_vec = tfidf[:n]
|
||||
pred_vec = tfidf[n:]
|
||||
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
|
||||
return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
|
||||
except ValueError:
|
||||
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
||||
|
||||
|
||||
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
total = len(predictions)
|
||||
if not total:
|
||||
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
|
||||
matched = sum(
|
||||
1
|
||||
for ref, pred in zip(references, predictions)
|
||||
if _normalize_text(ref) == _normalize_text(pred)
|
||||
)
|
||||
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
|
||||
|
||||
|
||||
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
if not predictions:
|
||||
return {"enabled": True, "score": 0}
|
||||
scores = [
|
||||
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
||||
for ref, pred in zip(references, predictions)
|
||||
]
|
||||
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM Judge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _judge_sample(
|
||||
question: str,
|
||||
reference: str,
|
||||
prediction: str,
|
||||
config: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Call an OpenAI-compatible LLM to judge a single sample.
|
||||
|
||||
Returns a dict with keys:
|
||||
score, max_score, passed, judgement, evaluation_reason, error_type
|
||||
"""
|
||||
api_url = (config.get("api_url") or "").strip().rstrip("/")
|
||||
api_key = (config.get("api_key") or "").strip()
|
||||
eval_model = (config.get("eval_model") or "").strip()
|
||||
# 优先使用模型记录里配置的真实 API 模型名(如 deepseek-chat),
|
||||
# 否则回退到平台内部模型名
|
||||
api_model = (config.get("api_model") or "").strip() or eval_model
|
||||
eval_prompt = (config.get("eval_prompt") or "").strip()
|
||||
score_min = float(config.get("score_min", 0))
|
||||
score_max = float(config.get("score_max", 5))
|
||||
pass_threshold = float(config.get("pass_threshold", 3))
|
||||
|
||||
if not api_url or not eval_model:
|
||||
return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
|
||||
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
|
||||
|
||||
system_msg = (
|
||||
eval_prompt
|
||||
or "你是一个专业的评测专家。请根据参考答-案对被测模型的输出进行评分。"
|
||||
)
|
||||
user_msg = (
|
||||
f"## 问题\n{question}\n\n"
|
||||
f"## 参考答案\n{reference}\n\n"
|
||||
f"## 模型输出\n{prediction}\n\n"
|
||||
f"请给出 {score_min}-{score_max} 分的评分,并说明理由。"
|
||||
)
|
||||
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
body = json.dumps({
|
||||
"model": api_model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_msg},
|
||||
{"role": "user", "content": user_msg},
|
||||
],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 512,
|
||||
}).encode("utf-8")
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{api_url}/v1/chat/completions",
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
},
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=120)
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
reply = data["choices"][0]["message"]["content"]
|
||||
except Exception as exc:
|
||||
return {"score": 0, "max_score": score_max, "passed": False,
|
||||
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
|
||||
"error_type": "其他"}
|
||||
|
||||
# Parse score from reply — look for patterns like "4分" or "Score: 4"
|
||||
score = 0
|
||||
import re
|
||||
score_patterns = [
|
||||
r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
|
||||
r'(\d+(?:\.\d+)?)\s*分',
|
||||
r'(\d+(?:\.\d+)?)\s*/\s*\d+',
|
||||
]
|
||||
for pat in score_patterns:
|
||||
m = re.search(pat, reply, re.IGNORECASE)
|
||||
if m:
|
||||
try:
|
||||
score = float(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
break
|
||||
score = max(score_min, min(score_max, score))
|
||||
passed = score >= pass_threshold
|
||||
|
||||
# Determine judgement label
|
||||
if score >= pass_threshold + 1:
|
||||
judgement = "正确"
|
||||
elif score >= pass_threshold:
|
||||
judgement = "部分正确"
|
||||
else:
|
||||
judgement = "错误"
|
||||
|
||||
# Guess error type from reply
|
||||
reply_lower = reply.lower()
|
||||
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
|
||||
error_type = "幻觉"
|
||||
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
|
||||
error_type = "不完整"
|
||||
elif any(w in reply_lower for w in ["格式", "format"]):
|
||||
error_type = "格式偏差"
|
||||
elif any(w in reply_lower for w in ["混淆", "confusion", "错误"]):
|
||||
error_type = "混淆"
|
||||
else:
|
||||
error_type = "其他"
|
||||
|
||||
return {
|
||||
"score": score,
|
||||
"max_score": score_max,
|
||||
"passed": passed,
|
||||
"judgement": judgement,
|
||||
"evaluation_reason": reply[:2000],
|
||||
"error_type": error_type,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Execute a full evaluation run. Returns the result dict (also written to file)."""
|
||||
model_path = config["model_name_or_path"]
|
||||
adapter_path = config.get("adapter_name_or_path", "")
|
||||
template = config.get("template", "qwen")
|
||||
dataset_path = config["dataset_path"]
|
||||
output_dir = Path(config["output_dir"])
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
basic_cfg = config.get("basic_metrics", {})
|
||||
dimension_cfg = config.get("dimension", {}) or {}
|
||||
output_precision = int(basic_cfg.get("output_precision", 2))
|
||||
|
||||
# ---- 1. Load dataset ----
|
||||
print(f"[eval] loading dataset: {dataset_path}")
|
||||
raw_samples = _load_dataset(dataset_path)
|
||||
print(f"[eval] loaded {len(raw_samples)} samples")
|
||||
|
||||
# ---- 2. Load model ----
|
||||
print(f"[eval] loading model: {model_path}")
|
||||
from compute.engines.llama_factory.inference import InferenceSession
|
||||
session = InferenceSession()
|
||||
session.load(
|
||||
model_name_or_path=model_path,
|
||||
adapter_name_or_path=adapter_path,
|
||||
template=template,
|
||||
infer_backend=config.get("infer_backend", "huggingface"),
|
||||
infer_dtype=config.get("infer_dtype", "auto"),
|
||||
)
|
||||
# load() 为异步加载(立即返回 loading),必须等待后台线程完成后再进行推理
|
||||
load_result = session.wait_until_loaded(timeout=float(config.get("load_timeout", 1800)))
|
||||
if not load_result.get("loaded"):
|
||||
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
|
||||
print(f"[eval] model loaded OK")
|
||||
|
||||
# ---- 3. Run inference on each sample ----
|
||||
samples: list[dict[str, Any]] = []
|
||||
predictions: list[str] = []
|
||||
references: list[str] = []
|
||||
questions: list[str] = []
|
||||
|
||||
total = len(raw_samples)
|
||||
judge_enabled = bool(dimension_cfg.get("eval_model") and dimension_cfg.get("api_url"))
|
||||
print(f"[eval] starting inference on {total} samples, judge={'enabled' if judge_enabled else 'disabled'}")
|
||||
|
||||
for idx, raw in enumerate(raw_samples, start=1):
|
||||
question = _sample_question(raw)
|
||||
reference = _sample_reference(raw)
|
||||
if not question:
|
||||
print(f"[eval] sample {idx}/{total}: skipped (no question)")
|
||||
continue
|
||||
|
||||
# Inference
|
||||
chat_msgs = [{"role": "user", "content": question}]
|
||||
result = session.chat(
|
||||
chat_msgs,
|
||||
temperature=float(config.get("temperature", 0.1)),
|
||||
top_p=float(config.get("top_p", 0.95)),
|
||||
max_new_tokens=int(config.get("max_new_tokens", 512)),
|
||||
do_sample=False,
|
||||
)
|
||||
prediction = result.get("response", "") if not result.get("error") else f"[ERROR] {result['error']}"
|
||||
|
||||
predictions.append(prediction)
|
||||
references.append(reference)
|
||||
questions.append(question)
|
||||
|
||||
# LLM Judge
|
||||
judge_result: dict[str, Any] = {}
|
||||
if judge_enabled:
|
||||
judge_result = _judge_sample(question, reference, prediction, dimension_cfg)
|
||||
|
||||
samples.append({
|
||||
"index": idx,
|
||||
"input": question,
|
||||
"reference_answer": reference,
|
||||
"model_output": prediction,
|
||||
"score": judge_result.get("score"),
|
||||
"max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5)),
|
||||
"passed": judge_result.get("passed"),
|
||||
"judgement": judge_result.get("judgement"),
|
||||
"evaluation_reason": judge_result.get("evaluation_reason", ""),
|
||||
"error_type": judge_result.get("error_type"),
|
||||
"dimension_scores": [
|
||||
{"name": "judge_score", "score": judge_result.get("score", 0),
|
||||
"max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5))},
|
||||
] if judge_result else [],
|
||||
"status": "completed",
|
||||
})
|
||||
|
||||
progress_pct = int(idx / max(total, 1) * 100)
|
||||
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
|
||||
|
||||
# ---- 4. Compute basic metrics ----
|
||||
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
|
||||
metrics_result: dict[str, Any] = {}
|
||||
|
||||
bleu_cfg = basic_cfg.get("bleu", {})
|
||||
if bleu_cfg.get("enabled"):
|
||||
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
|
||||
|
||||
rouge_cfg = basic_cfg.get("rouge", {})
|
||||
if rouge_cfg.get("enabled"):
|
||||
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
|
||||
|
||||
cosine_cfg = basic_cfg.get("cosine", {})
|
||||
if cosine_cfg.get("enabled"):
|
||||
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
||||
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
||||
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
||||
|
||||
# ---- 5. Summarise ----
|
||||
completed = len(samples)
|
||||
if judge_enabled:
|
||||
scored = [s for s in samples if s.get("score") is not None]
|
||||
passed_count = len([s for s in scored if s.get("passed")])
|
||||
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
|
||||
max_score = dimension_cfg.get("score_max", 5)
|
||||
overall_score = round(avg_score / max_score * 100, output_precision)
|
||||
overall_score_max = 100
|
||||
dimension_summary = [{
|
||||
"name": "综合评分",
|
||||
"score": overall_score,
|
||||
"max_score": 100,
|
||||
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
|
||||
}]
|
||||
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/{max_score} 分"
|
||||
else:
|
||||
passed_count = 0
|
||||
enabled_scores = [
|
||||
float(item.get("score") or 0)
|
||||
for item in metrics_result.values()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
|
||||
overall_score_max = 100
|
||||
dimension_summary = [
|
||||
{
|
||||
"name": name,
|
||||
"score": float(item.get("score") or 0),
|
||||
"max_score": 100,
|
||||
"pass_rate": float(item.get("score") or 0),
|
||||
}
|
||||
for name, item in metrics_result.items()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
|
||||
|
||||
result = {
|
||||
"overall_score": overall_score,
|
||||
"overall_score_max": overall_score_max,
|
||||
"overall_evaluation": overall_evaluation,
|
||||
"improvement_suggestions": [],
|
||||
"dimension_summary": dimension_summary,
|
||||
"samples": samples,
|
||||
"sample_count": total,
|
||||
"completed_count": completed,
|
||||
"passed_count": passed_count,
|
||||
"basic_metrics": metrics_result,
|
||||
}
|
||||
|
||||
# ---- 6. Write results ----
|
||||
result_path = output_dir / "eval_results.json"
|
||||
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"[eval] results written to {result_path}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="YG-FT Evaluation Runner")
|
||||
parser.add_argument("--config", required=True, help="Path to eval config JSON file")
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config)
|
||||
if not config_path.exists():
|
||||
print(f"FATAL: config file not found: {args.config}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
start = time.time()
|
||||
try:
|
||||
run_eval(config)
|
||||
elapsed = time.time() - start
|
||||
print(f"[eval] DONE in {elapsed:.1f}s")
|
||||
except Exception as exc:
|
||||
print(f"[eval] FAILED: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
272
compute/engines/llama_factory/inference.py
Normal file
272
compute/engines/llama_factory/inference.py
Normal file
@@ -0,0 +1,272 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
class InferenceSession:
|
||||
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
||||
|
||||
Model loading is asynchronous: ``load()`` spawns a background daemon thread
|
||||
and returns immediately with ``status == "loading"``. ``info()`` (served by
|
||||
``/inference/status``) is always responsive, so the platform backend can
|
||||
poll loading progress without being blocked by a minutes-long model load —
|
||||
which previously froze the whole compute node event loop.
|
||||
|
||||
State machine: idle -> loading -> ready | error, ready -> idle (unload),
|
||||
loading -> idle (cancelled). Long operations (ChatModel build, teardown,
|
||||
generation) never run while holding ``_state_lock``; they either run in the
|
||||
worker thread or under ``_chat_lock`` only.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._state_lock = threading.Lock() # brief state transitions only
|
||||
self._chat_lock = threading.Lock() # serialize chat/teardown
|
||||
self._status: str = "idle"
|
||||
self._error: str = ""
|
||||
self._request_id: str = ""
|
||||
self._load_args: dict[str, Any] = {}
|
||||
self._teardown_old = False # load-while-ready: unload old before loading new
|
||||
self._cancel_requested = False # unload-while-loading: tear down after load finishes
|
||||
self._load_thread: threading.Thread | None = None
|
||||
self._model: Any = None
|
||||
self._tokenizer: Any = None
|
||||
self._generating_args: dict[str, Any] = {}
|
||||
self._model_name: str = ""
|
||||
self._adapter_path: str = ""
|
||||
self._loaded_at: float = 0.0
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
with self._state_lock:
|
||||
return self._status
|
||||
|
||||
def info(self) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
return {
|
||||
"loaded": self._status == "ready",
|
||||
"status": self._status,
|
||||
"model_name": self._model_name,
|
||||
"adapter_path": self._adapter_path,
|
||||
"loaded_at": self._loaded_at,
|
||||
"request_id": self._request_id,
|
||||
"error": self._error,
|
||||
}
|
||||
|
||||
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
|
||||
"""Wait for an in-flight async load to finish and return its outcome.
|
||||
|
||||
供同步消费方(如 eval_runner 子进程)使用:``load()`` 立即返回 loading 后,
|
||||
调用本方法等待后台加载线程完成,拿到最终的 loaded/error 结果。
|
||||
若在 timeout 秒内仍未加载完成,返回 ``status == "loading"`` 并附上超时提示。
|
||||
"""
|
||||
with self._state_lock:
|
||||
thread = self._load_thread
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=timeout)
|
||||
with self._state_lock:
|
||||
loaded = self._status == "ready"
|
||||
status = self._status
|
||||
error = self._error
|
||||
if not loaded and status == "loading":
|
||||
error = error or f"model load timed out after {timeout or 'N/A'}s"
|
||||
return {
|
||||
"loaded": loaded,
|
||||
"status": status,
|
||||
"model_name": self._model_name,
|
||||
"adapter_path": self._adapter_path,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
def load(
|
||||
self,
|
||||
model_name_or_path,
|
||||
adapter_name_or_path="",
|
||||
template="qwen",
|
||||
infer_backend="huggingface",
|
||||
infer_dtype="auto",
|
||||
**kwargs,
|
||||
) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
# A model is already loading — dedupe, reuse the same request id.
|
||||
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||
self._teardown_old = self._status == "ready"
|
||||
self._status = "loading"
|
||||
self._error = ""
|
||||
self._request_id = uuid.uuid4().hex[:12]
|
||||
self._cancel_requested = False
|
||||
self._load_args = {
|
||||
"model_name_or_path": model_name_or_path,
|
||||
"template": template,
|
||||
"infer_backend": infer_backend,
|
||||
"infer_dtype": infer_dtype,
|
||||
}
|
||||
if adapter_name_or_path:
|
||||
self._load_args["adapter_name_or_path"] = adapter_name_or_path
|
||||
self._load_args.update(kwargs)
|
||||
self._model_name = model_name_or_path
|
||||
self._adapter_path = adapter_name_or_path
|
||||
self._load_thread = threading.Thread(target=self._load_worker, daemon=True)
|
||||
self._load_thread.start()
|
||||
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||
|
||||
def _load_worker(self) -> None:
|
||||
"""Build the ChatModel off the state lock so info() never blocks."""
|
||||
model = None
|
||||
tokenizer = None
|
||||
generating_args: dict[str, Any] = {}
|
||||
error = ""
|
||||
try:
|
||||
if self._teardown_old:
|
||||
self._release_model()
|
||||
from llamafactory.chat import ChatModel
|
||||
from llamafactory.hparams import get_infer_args
|
||||
|
||||
args = dict(self._load_args)
|
||||
infer_result = get_infer_args(args)
|
||||
model = ChatModel(args)
|
||||
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
||||
generating_args = infer_result[-1]
|
||||
if hasattr(generating_args, "__dataclass_fields__"):
|
||||
generating_args = {
|
||||
k: v for k, v in vars(generating_args).items() if not k.startswith("_")
|
||||
}
|
||||
else:
|
||||
generating_args = dict(generating_args)
|
||||
except Exception as exc: # noqa: BLE001 - surface load failure via status
|
||||
error = str(exc)
|
||||
with self._state_lock:
|
||||
if error:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "error"
|
||||
self._error = error
|
||||
return
|
||||
if self._cancel_requested:
|
||||
# Unload was requested while loading — drop the fresh model.
|
||||
model = None
|
||||
tokenizer = None
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "idle"
|
||||
return
|
||||
self._model = model
|
||||
self._tokenizer = tokenizer
|
||||
self._generating_args = generating_args
|
||||
self._loaded_at = time.time()
|
||||
self._status = "ready"
|
||||
|
||||
def _release_model(self) -> None:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
self._status = "unloading"
|
||||
model = self._model
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
if model is not None:
|
||||
try:
|
||||
del model
|
||||
except Exception: # noqa: BLE001 - best-effort teardown
|
||||
pass
|
||||
# 强制释放 PyTorch CUDA 缓存,真正归还 GPU 显存
|
||||
try:
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.synchronize()
|
||||
except Exception: # noqa: BLE001 - teardown must not raise
|
||||
pass
|
||||
with self._state_lock:
|
||||
self._status = "idle"
|
||||
self._model_name = ""
|
||||
self._adapter_path = ""
|
||||
self._loaded_at = 0.0
|
||||
self._error = ""
|
||||
|
||||
def unload(self) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
# Ask the worker to tear down right after the load finishes.
|
||||
self._cancel_requested = True
|
||||
return {"unloaded": False, "status": "cancelling", "request_id": self._request_id}
|
||||
was_ready = self._status == "ready"
|
||||
if was_ready:
|
||||
self._release_model()
|
||||
else:
|
||||
with self._state_lock:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "idle"
|
||||
self._model_name = ""
|
||||
self._adapter_path = ""
|
||||
self._loaded_at = 0.0
|
||||
self._error = ""
|
||||
return {"unloaded": True, "status": "idle"}
|
||||
|
||||
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
return {
|
||||
"error": f"model is still loading (request_id={self._request_id}); please retry",
|
||||
"response": "",
|
||||
}
|
||||
if self._status == "error":
|
||||
return {"error": f"model load failed: {self._error}", "response": ""}
|
||||
if self._status != "ready" or self._model is None:
|
||||
return {"error": "model not loaded", "response": ""}
|
||||
try:
|
||||
generate_kwargs = {
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"max_new_tokens": max_new_tokens,
|
||||
"do_sample": do_sample,
|
||||
}
|
||||
generate_kwargs.update(kwargs)
|
||||
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
||||
user_messages = [m for m in messages if m["role"] != "system"]
|
||||
responses = []
|
||||
for response in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
||||
responses.append(response)
|
||||
full_response = "".join(str(r) for r in responses)
|
||||
return {"response": full_response}
|
||||
except Exception as exc: # noqa: BLE001 - return generation error to caller
|
||||
return {"error": str(exc), "response": ""}
|
||||
|
||||
def chat_stream(self, messages, **kwargs) -> Iterator[str]:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
yield 'data: {"error": "model is still loading; please retry"}\n\n'
|
||||
return
|
||||
if self._status == "error":
|
||||
yield 'data: {"error": "model load failed: ' + str(self._error) + '"}\n\n'
|
||||
return
|
||||
if self._status != "ready" or self._model is None:
|
||||
yield 'data: {"error": "model not loaded"}\n\n'
|
||||
return
|
||||
try:
|
||||
generate_kwargs = {**kwargs}
|
||||
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
||||
user_messages = [m for m in messages if m["role"] != "system"]
|
||||
for new_text in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
||||
yield new_text
|
||||
except Exception as exc: # noqa: BLE001 - stream error as SSE event
|
||||
yield 'data: {"error": "' + str(exc) + '"}\n\n'
|
||||
|
||||
|
||||
_inference_session = None
|
||||
|
||||
|
||||
def get_inference_session() -> InferenceSession:
|
||||
global _inference_session
|
||||
if _inference_session is None:
|
||||
_inference_session = InferenceSession()
|
||||
return _inference_session
|
||||
@@ -4,3 +4,9 @@ python-multipart>=0.0.9
|
||||
pydantic>=2.7.0
|
||||
python-dotenv>=1.0.1
|
||||
httpx>=0.27.0
|
||||
# 模型评测指标
|
||||
sacrebleu>=2.4.0
|
||||
rouge-score>=0.1.2
|
||||
scikit-learn>=1.3.0
|
||||
# LLaMA-Factory 训练引擎
|
||||
llamafactory
|
||||
|
||||
146
compute/tests/test_inference_session.py
Normal file
146
compute/tests/test_inference_session.py
Normal file
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from compute.engines.llama_factory.inference import InferenceSession
|
||||
|
||||
# 模拟模型加载耗时,用于验证 load() 立即返回、info() 不阻塞
|
||||
LOAD_DELAY = 0.2
|
||||
|
||||
|
||||
class FakeChatModel:
|
||||
def __init__(self, args: dict[str, Any]) -> None:
|
||||
time.sleep(LOAD_DELAY)
|
||||
self.tokenizer = object()
|
||||
self.engine = types.SimpleNamespace(tokenizer=object())
|
||||
self._output = "hello from model"
|
||||
|
||||
def stream_chat(self, *args, **kwargs):
|
||||
for _ in range(1):
|
||||
yield self._output
|
||||
|
||||
|
||||
class FailingChatModel:
|
||||
def __init__(self, args: dict[str, Any]) -> None:
|
||||
time.sleep(LOAD_DELAY)
|
||||
raise RuntimeError("boom: fake load failure")
|
||||
|
||||
|
||||
def _get_infer_args(args: dict[str, Any]) -> list[Any]:
|
||||
# 最后一个元素为 generating_args,worker 会转成 dict
|
||||
return [None, None, {"temperature": 0.7}]
|
||||
|
||||
|
||||
def _install_llamafactory(monkeypatch, chat_model: type) -> None:
|
||||
llmf = types.ModuleType("llamafactory")
|
||||
chat_mod = types.ModuleType("llamafactory.chat")
|
||||
hparams_mod = types.ModuleType("llamafactory.hparams")
|
||||
chat_mod.ChatModel = chat_model
|
||||
hparams_mod.get_infer_args = _get_infer_args
|
||||
llmf.chat = chat_mod
|
||||
llmf.hparams = hparams_mod
|
||||
monkeypatch.setitem(sys.modules, "llamafactory", llmf)
|
||||
monkeypatch.setitem(sys.modules, "llamafactory.chat", chat_mod)
|
||||
monkeypatch.setitem(sys.modules, "llamafactory.hparams", hparams_mod)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_llamafactory(monkeypatch) -> None:
|
||||
_install_llamafactory(monkeypatch, FakeChatModel)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_failing_llamafactory(monkeypatch) -> None:
|
||||
_install_llamafactory(monkeypatch, FailingChatModel)
|
||||
|
||||
|
||||
def _wait_for_status(session: InferenceSession, status: str, timeout: float = 3.0) -> bool:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if session.info()["status"] == status:
|
||||
return True
|
||||
time.sleep(0.02)
|
||||
return False
|
||||
|
||||
|
||||
def test_load_returns_immediately_then_ready(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
started = time.time()
|
||||
result = session.load("/models/qwen")
|
||||
assert result["status"] == "loading"
|
||||
assert result["loaded"] is False
|
||||
assert result["request_id"]
|
||||
# 在慢加载完成前就返回,且 info() 加载期间可响应
|
||||
assert time.time() - started < LOAD_DELAY
|
||||
assert session.info()["status"] == "loading"
|
||||
assert _wait_for_status(session, "ready")
|
||||
info = session.info()
|
||||
assert info["loaded"] is True
|
||||
assert info["status"] == "ready"
|
||||
assert info["model_name"] == "/models/qwen"
|
||||
|
||||
|
||||
def test_second_load_while_loading_deduped(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
r1 = session.load("/models/a")
|
||||
r2 = session.load("/models/b")
|
||||
assert r2["status"] == "loading"
|
||||
assert r2["request_id"] == r1["request_id"]
|
||||
assert _wait_for_status(session, "ready")
|
||||
assert session.info()["status"] == "ready"
|
||||
|
||||
|
||||
def test_load_error_surfaces_in_status(stub_failing_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/bad")
|
||||
assert _wait_for_status(session, "error")
|
||||
assert "boom" in session.info()["error"]
|
||||
|
||||
|
||||
def test_unload_while_loading_cancels(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
result = session.unload()
|
||||
assert result["status"] == "cancelling"
|
||||
assert _wait_for_status(session, "idle")
|
||||
|
||||
|
||||
def test_chat_while_loading_returns_loading_error(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
out = session.chat([{"role": "user", "content": "hi"}])
|
||||
assert "still loading" in (out.get("error") or "")
|
||||
assert _wait_for_status(session, "ready")
|
||||
out = session.chat([{"role": "user", "content": "hi"}])
|
||||
assert out.get("response") == "hello from model"
|
||||
|
||||
|
||||
def test_chat_stream_while_loading_yields_error(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
chunks = list(session.chat_stream([{"role": "user", "content": "hi"}]))
|
||||
assert any("still loading" in c for c in chunks)
|
||||
|
||||
|
||||
def test_wait_until_loaded_blocks_until_ready(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
result = session.load("/models/qwen")
|
||||
assert result["status"] == "loading"
|
||||
# 同步等待后台加载线程完成
|
||||
outcome = session.wait_until_loaded(timeout=3.0)
|
||||
assert outcome["loaded"] is True
|
||||
assert outcome["status"] == "ready"
|
||||
|
||||
|
||||
def test_wait_until_loaded_reports_load_error(stub_failing_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/bad")
|
||||
outcome = session.wait_until_loaded(timeout=3.0)
|
||||
assert outcome["loaded"] is False
|
||||
assert outcome["status"] == "error"
|
||||
assert "boom" in outcome["error"]
|
||||
51
design-qa.md
51
design-qa.md
@@ -328,6 +328,57 @@ final result: blocked
|
||||
|
||||
---
|
||||
|
||||
# Failed Result Regeneration Design QA
|
||||
|
||||
## Evidence
|
||||
|
||||
- Source visual truth: `/var/folders/nk/yks07zp14wb4rv3jqq0pt_4h0000gn/T/codex-clipboard-0c2e6f90-aaac-4817-9084-bc31496aed0a.png`.
|
||||
- Implementation route: `http://localhost:16801/data-process/:id/workflow`, step 6.
|
||||
- Browser evidence: `/Users/caoxiaozhu/.codex/visualizations/2026/07/27/019fa277-626b-76a1-af02-48948d66c7ec/data-process-empty-list.png`.
|
||||
- Viewport: 1280 × 720.
|
||||
- Target state: a completed task on step 6 with a selected invalid result.
|
||||
- Available browser state: authenticated data-process list with zero tasks; no representative invalid result could be opened without creating or mutating local business data.
|
||||
|
||||
## Static and automated evidence
|
||||
|
||||
- Invalid results render a primary plain `重新生成` button in the right side of the result header.
|
||||
- Valid results keep the existing `恢复生成结果` action.
|
||||
- While regeneration is running, the selected row shows a spinner, the button shows loading, editors are disabled, and the footer confirmation action is disabled.
|
||||
- A successful response replaces only the selected row in place; a failed second generation leaves the original invalid row untouched.
|
||||
- `regression-data-process-wizard.mjs`: passed.
|
||||
- `regression-data-process-detail.mjs`: passed.
|
||||
- `regression-data-process-list.mjs`: passed.
|
||||
- `vue-tsc -b --noEmit`: passed.
|
||||
- Relevant backend tests: 97 passed, with one third-party deprecation warning.
|
||||
|
||||
## Required fidelity surfaces
|
||||
|
||||
- Typography and hierarchy: the action uses the existing Element Plus small-button hierarchy and remains secondary to result content.
|
||||
- Spacing and layout rhythm: the action is placed in the existing flex header's right action slot, matching the source location.
|
||||
- Colors and tokens: the action uses the existing product primary color and existing Font Awesome refresh icon.
|
||||
- Copy and content: the visible label is exactly `重新生成`; only server-saved invalid results expose it.
|
||||
- Interaction: optimistic concurrency protects against overwriting a newer edit, and persistence happens only after the replacement passes generation and quality validation.
|
||||
|
||||
## Findings
|
||||
|
||||
- [P2] Representative rendered comparison unavailable
|
||||
Location: step 6 result header with an invalid result selected.
|
||||
Evidence: the authenticated local database currently contains zero data-processing tasks, so the target state cannot be reached without creating test business data or invoking the configured model.
|
||||
Impact: automated and static checks prove the contract and placement, but cannot prove pixel-level spacing against the supplied screenshot.
|
||||
Fix: open any existing failed result after one is available, capture the same 1280 × 720 state, and compare it side by side with the source screenshot.
|
||||
|
||||
## Comparison history
|
||||
|
||||
### Iteration 1 — blocked
|
||||
|
||||
- Source screenshot was inspected and the implementation was aligned to its existing right-header action slot.
|
||||
- Local login and list navigation succeeded.
|
||||
- The target result state was unavailable because the local task list was empty.
|
||||
|
||||
final result: blocked
|
||||
|
||||
---
|
||||
|
||||
# Dataset Version Actions Design QA
|
||||
|
||||
## Evidence
|
||||
|
||||
@@ -11,7 +11,7 @@ RUN pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
&& pip install -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
&& rm -f /tmp/requirements.txt
|
||||
|
||||
RUN python -c "import fastapi, uvicorn, psycopg, sqlalchemy, redis, jwt, passlib, httpx, alembic; print('backend dependency check ok')"
|
||||
RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, alembic; print('backend dependency check ok')"
|
||||
|
||||
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \
|
||||
&& chmod -R 0775 /opt/yg-ft /data/yg-ft
|
||||
|
||||
@@ -35,6 +35,6 @@ COMPUTE_GPU_MEMORY_GB=80
|
||||
COMPUTE_GPU_POWER_LIMIT_W=300
|
||||
|
||||
LOG_DIR=/opt/yg-ft/logs/compute
|
||||
CUDA_VISIBLE_DEVICES=all
|
||||
CUDA_VISIBLE_DEVICES=0
|
||||
NVIDIA_VISIBLE_DEVICES=all
|
||||
NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
|
||||
@@ -14,8 +14,8 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
}
|
||||
|
||||
location = /modelTF {
|
||||
@@ -25,8 +25,8 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
}
|
||||
|
||||
location ~* \.(?:js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf)$ {
|
||||
|
||||
121
docs/模型评测功能总结.md
Normal file
121
docs/模型评测功能总结.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# 模型评测功能总结
|
||||
|
||||
本项目(基于 LLaMA-Factory 的微调训练平台)包含 **4 套相对独立** 的模型评测能力,分别面向不同的使用场景:
|
||||
|
||||
| 能力 | 入口/目录 | 评测类型 | 打分方式 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1. 学术 Benchmark 评测 | `llamafactory/eval/` | 选择题式基准(类 MMLU/C-Eval) | 选项匹配 + few-shot |
|
||||
| 2. 评估工作台 | `backend/app/api/v1/eval/` | 生成式问答(指令跟随) | BLEU / ROUGE / ExactMatch + 可选 LLM 评审 |
|
||||
| 3. 平台评估系统 | `backend/app/api/v1/evaluation/` | 基于评估数据集的问答 | 判卷模型(judge model)打分(0–5 分) |
|
||||
| 4. 训练时验证评估 | `backend/app/services/task_runner.py` | 训练验证集 | loss 指标 |
|
||||
|
||||
下面分别说明。
|
||||
|
||||
---
|
||||
|
||||
## 1. 学术 Benchmark 评测(LLaMA-Factory 原生)
|
||||
|
||||
面向标准学术选择题基准(如 MMLU、C-Eval 等),复用 LLaMA-Factory 原生的评测框架。
|
||||
|
||||
**核心文件**
|
||||
- `llamafactory/eval/evaluator.py`:`Evaluator` 类 + `run_eval()` 入口
|
||||
- `llamafactory/eval/template.py`:评测 prompt 模板(中/英,含 few-shot 示例构建)
|
||||
- `llamafactory/hparams/evaluation_args.py`:`EvaluationArguments` 配置类
|
||||
|
||||
**工作流程**
|
||||
1. 按 `task`(benchmark 名称)加载数据集,按科目(subject)拆分。
|
||||
2. 每个样本构造 few-shot 提示词(`n_shot` 控制示例数,由 `lang` 决定中/英模板),将题干与候选选项拼入 prompt。
|
||||
3. 调用模型推理得到预测,与标准答案比对,统计每个科目及整体的 `accuracy`。
|
||||
4. 结果写入 `save_dir`,打印各科目与平均准确率。
|
||||
|
||||
**关键参数(`EvaluationArguments`)**
|
||||
- `task`:基准数据集名
|
||||
- `batch_size` / `n_shot` / `lang` / `save_dir` / `seed`
|
||||
- `model_name_or_path`、`template`、`trust_remote_code` 等模型相关参数
|
||||
|
||||
> 该能力属于框架底层,本平台前端未直接提供操作入口,主要通过配置文件/脚本调用。
|
||||
|
||||
---
|
||||
|
||||
## 2. 评估工作台(生成式评测 + 指标计算)
|
||||
|
||||
后端路由位于 `backend/app/api/v1/eval/__init__.py`,前端称为「评估工作台」。**适用于评测模型的指令跟随与生成质量**,并支持 LLM 作为裁判(LLM-as-a-Judge)。
|
||||
|
||||
**API 端点**
|
||||
- `GET /evaluation/tasks`:列出评测任务(`frontend/src/api/evaluation.ts:listTasks`)
|
||||
- `POST /evaluation/run`:提交一次评测(`runEval`)
|
||||
- `GET /evaluation/report/{task_id}`:拉取评测报告(`getReport`)
|
||||
- `DELETE /evaluation/tasks/{task_id}`:删除任务(`deleteTask`)
|
||||
|
||||
**评测流程(`run_eval`)**
|
||||
1. 通过 **LLaMA-Factory 数据管道**(`get_dataset`) 加载数据集,支持 `subset` 与抽样(`eval_sample`)。
|
||||
2. 用 **原生 transformers** 加载模型在本地做生成推理(单进程顺序生成,便于展示样本)。
|
||||
3. 计算客观指标(`compute_score`):
|
||||
- `BLEU`(sacrebleu)
|
||||
- `ROUGE-1 / ROUGE-2 / ROUGE-L`(rouge-score)
|
||||
- `Exact Match`
|
||||
4. **可选 LLM 评审**(judge):当配置了 `judge_model` / `judge_api_base` / `judge_api_key` 时,调用 OpenAI 兼容接口对每条样本打分(10 分制),并输出 4 个维度与理由:
|
||||
- 核心事实正确性 `factual`
|
||||
- 信息完整性 `completeness`
|
||||
- 无幻觉 `no_hallucination`
|
||||
- 格式合规性 `format`
|
||||
- 综合分 `score` + `reason`
|
||||
5. 任务状态持久化在后端 `eval_tasks.json`(支持 running/completed/failed/stopped),前端轮询进度。
|
||||
|
||||
**前端页面**
|
||||
- `frontend/src/views/evaluation/EvaluateTask.vue`:任务列表、创建评测对话框(选模型、数据集、指标、可选 judge 配置)
|
||||
- `frontend/src/views/evaluation/EvaluateReport.vue`:报告页,展示综合得分、BLEU、ROUGE-L、各维度指标及「参考答案 vs 模型预测 vs LLM 评审」对比样例
|
||||
|
||||
---
|
||||
|
||||
## 3. 平台评估系统(基于评估数据集 + 判卷模型)
|
||||
|
||||
后端路由位于 `backend/app/api/v1/evaluation/__init__.py`,是平台业务层自研的评测体系。通过「评估数据集」组织题目,可一次性对 **多个被测模型 + 指定判卷模型** 进行批量评分。
|
||||
|
||||
**核心概念(数据模型 `backend/app/models/models.py`)**
|
||||
- `EvalDataset`(`models.py:131`):评估数据集,从项目问答对(`Question`/`Chunk`)中按 `question_type`(mixed/fact/reasoning)选题构建,状态 `pending/running/completed/failed`。
|
||||
- `EvalResult`(`models.py:147`):单条评测结果,含 `judge_score`(0–5 分)、`is_correct`(true/false/partial)、`feedback`、`expected_answer` 等。
|
||||
- `Task`(`models.py:184`):后台任务,`task_type="model-evaluation"`,记录进度与 `model_info`(存放平均分等汇总)。
|
||||
|
||||
**评测流程(`process_evaluation_task`,`backend/app/services/task_processor.py:336` 起)**
|
||||
1. 加载评估数据集关联的题目,可选带入 `chunk` 上下文(RAG 场景)。
|
||||
2. 对每道题,先用 `build_eval_prompt` 组合「上下文 + 题目 + 参考答案」,调用 **判卷模型**(`call_model`,temperature=0.3)生成评分。
|
||||
3. `parse_eval_result` 解析出 `score`(0–5)、`is_correct`、`feedback`,写入 `EvalResult`。
|
||||
4. 逐题提交进度(`completed_count` / `progress`),支持中途 `stopped`。
|
||||
5. 汇总:`avg_score = 总分/有效数 × 20`(换算百分制),`avg_score_5 = 总分/有效数`(5 分制),存入 `task.model_info`。判定规则:得分 **≥3 视为正确**。
|
||||
|
||||
**特点**
|
||||
- 判卷与被测模型解耦:被测模型给出答案,判卷模型(judge)独立评分,降低自评偏差。
|
||||
- 支持失败隔离:单题异常写入 `evaluation_status: failed` 记录而不中断整体任务。
|
||||
|
||||
---
|
||||
|
||||
## 4. 训练时验证评估
|
||||
|
||||
在微调训练任务执行期间,由 `backend/app/services/task_runner.py` 的 `do_eval` 触发:
|
||||
|
||||
- 在训练过程中对验证集(validation set)计算 `eval_loss`,用于监控过拟合。
|
||||
- 结果回填到 `Task` 的 `loss_info` / `detail`,前端绘制 loss 曲线。
|
||||
- 属于训练配套的轻量评估,不参与上述 1–3 的业务评测。
|
||||
|
||||
---
|
||||
|
||||
## 附属:前端评测相关页面
|
||||
|
||||
| 文件 | 作用 |
|
||||
| --- | --- |
|
||||
| `frontend/src/views/evaluation/EvaluateTask.vue` | 评估工作台:任务列表 + 创建评测 |
|
||||
| `frontend/src/views/evaluation/EvaluateReport.vue` | 评估报告:指标卡 + 维度标签 + 对比样例 |
|
||||
| `frontend/src/api/evaluation.ts` | 评估工作台接口封装 |
|
||||
| 平台评估系统入口 | 评估数据集管理 + 评估任务(model-evaluation)创建与结果查看 |
|
||||
|
||||
---
|
||||
|
||||
## 小结
|
||||
|
||||
- **想要学术榜单式准确率** → 用能力 1(LLaMA-Factory `eval/`)。
|
||||
- **想要开放式生成质量(BLEU/ROUGE + LLM 评审)** → 用能力 2(评估工作台 `/evaluation/run`)。
|
||||
- **想要基于自有问答数据、用判卷模型批量打分** → 用能力 3(平台评估系统 `model-evaluation` 任务)。
|
||||
- **训练过程监控** → 能力 4(`do_eval` 验证集 loss)。
|
||||
|
||||
三种业务评测(1/2/3)相互独立,可并存于同一平台;数据模型(`EvalDataset`/`EvalResult`/`Task`)主要服务于能力 3,而能力 2 使用独立的 `eval_tasks.json` 文件持久化。
|
||||
2
frontend/.gitignore
vendored
2
frontend/.gitignore
vendored
@@ -1,7 +1,5 @@
|
||||
node_modules
|
||||
dist
|
||||
!dist/
|
||||
!dist/**
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
import{d as C,bf as g,D as E,H as O,o as _,e as B,s as D,Z as $,w as A,c as I,aL as K,q as t,n as v,aa as r,g as q,bg as M,y as u,z as N,P as k}from"./index-CGB0A5x_.js";import{_ as R}from"./_plugin-vue_export-helper-DlAUqK2U.js";const V={class:"app-confirm-header"},z={class:"app-confirm-heading"},H={class:"app-confirm-icon","aria-hidden":"true"},L={class:"app-confirm-body"},P={class:"app-confirm-actions"},S=C({__name:"AppConfirmDialog",setup(j,{expose:x}){const c=u(!1),m=u(),p=u(),d=`app-confirm-title-${g()}`,y=`app-confirm-message-${g()}`,n=N({title:"请确认操作",message:"",confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1});let o=null,i=null;function s(a){c.value=!1;const e=o;o=null,e==null||e(a)}function h(a){return o&&s(!1),Object.assign(n,{confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1,...a}),c.value=!0,new Promise(e=>{o=e})}function w(){n.closeOnOverlay&&s(!1)}function T(a){var b;if(a.key==="Escape"){a.preventDefault(),s(!1);return}if(a.key!=="Tab")return;const e=Array.from(((b=m.value)==null?void 0:b.querySelectorAll("button:not([disabled])"))??[]),l=e[0],f=e[e.length-1];!l||!f||(a.shiftKey&&document.activeElement===l?(a.preventDefault(),f.focus()):!a.shiftKey&&document.activeElement===f&&(a.preventDefault(),l.focus()))}return E(c,async a=>{var e;if(a){i=document.activeElement instanceof HTMLElement?document.activeElement:null,await k(),(e=p.value)==null||e.focus();return}await k(),i==null||i.focus(),i=null}),O(()=>{o==null||o(!1),o=null}),x({open:h}),(a,e)=>(_(),B(M,{to:"body"},[D($,{name:"app-confirm"},{default:A(()=>[c.value?(_(),I("div",{key:0,class:"app-confirm-overlay",onMousedown:K(w,["self"])},[t("section",{ref_key:"dialogRef",ref:m,class:v(["app-confirm-dialog",`is-${n.tone}`]),role:"alertdialog","aria-modal":!0,"aria-labelledby":d,"aria-describedby":y,onKeydown:T},[t("header",V,[t("div",z,[t("span",H,[t("i",{class:v(n.tone==="primary"?"fa fa-question-circle":"fa fa-exclamation-triangle")},null,2)]),t("h2",{id:d},r(n.title),1)]),t("button",{class:"app-confirm-close",type:"button","aria-label":"关闭确认弹窗",onClick:e[0]||(e[0]=l=>s(!1))},[...e[3]||(e[3]=[t("i",{class:"fa fa-times","aria-hidden":"true"},null,-1)])])]),t("div",L,[t("p",{id:y},r(n.message),1)]),t("footer",P,[t("button",{ref_key:"cancelButtonRef",ref:p,class:"app-confirm-button is-cancel",type:"button",onClick:e[1]||(e[1]=l=>s(!1))},r(n.cancelText),513),t("button",{class:"app-confirm-button is-confirm",type:"button",onClick:e[2]||(e[2]=l=>s(!0))},r(n.confirmText),1)])],34)],32)):q("",!0)]),_:1})]))}}),G=R(S,[["__scopeId","data-v-8d3bf6ad"]]);export{G as A};
|
||||
@@ -1 +0,0 @@
|
||||
.app-confirm-overlay[data-v-8d3bf6ad]{position:fixed;z-index:2000;top:0;right:0;bottom:0;left:0;display:grid;place-items:center;padding:20px;box-sizing:border-box;background:#0f172a70}.app-confirm-dialog[data-v-8d3bf6ad]{position:relative;width:min(480px,100%);overflow:hidden;background:#fff;border:1px solid #dfe3ea;border-radius:8px;box-shadow:0 12px 28px #0f172a29}.app-confirm-header[data-v-8d3bf6ad]{display:flex;min-height:52px;align-items:center;justify-content:space-between;gap:16px;padding:0 10px 0 20px;border-bottom:1px solid #e7eaf0}.app-confirm-heading[data-v-8d3bf6ad]{display:flex;min-width:0;align-items:center;gap:10px}.app-confirm-heading h2[data-v-8d3bf6ad]{margin:0;overflow:hidden;color:#273142;font-size:15px;font-weight:650;line-height:1.4;text-overflow:ellipsis;white-space:nowrap}.app-confirm-close[data-v-8d3bf6ad]{display:inline-grid;width:32px;height:32px;flex:0 0 32px;place-items:center;padding:0;color:#7b8495;background:transparent;border:0;border-radius:4px;cursor:pointer;transition:color .18s ease,background-color .18s ease}.app-confirm-close[data-v-8d3bf6ad]:hover{color:#273142;background:#f2f4f7}.app-confirm-close[data-v-8d3bf6ad]:focus-visible{outline:2px solid rgba(91,80,242,.45);outline-offset:1px}.app-confirm-icon[data-v-8d3bf6ad]{display:inline-grid;width:28px;height:28px;flex:0 0 28px;place-items:center;color:#a15c07;background:#fff8e6;border:1px solid #f3dfad;border-radius:6px;font-size:13px}.app-confirm-dialog.is-danger .app-confirm-icon[data-v-8d3bf6ad]{color:#c43232;background:#fff1f1;border-color:#f2c7c7}.app-confirm-dialog.is-primary .app-confirm-icon[data-v-8d3bf6ad]{color:#4f46e5;background:#f3f2ff;border-color:#d9d6ff}.app-confirm-body[data-v-8d3bf6ad]{padding:16px 20px 18px}.app-confirm-body p[data-v-8d3bf6ad]{margin:0;color:#5f6878;font-size:13px;line-height:1.7}.app-confirm-actions[data-v-8d3bf6ad]{display:flex;justify-content:flex-end;gap:8px;padding:10px 14px;background:#f8f9fb;border-top:1px solid #e7eaf0}.app-confirm-button[data-v-8d3bf6ad]{height:34px;min-width:72px;padding:0 13px;color:#344054;font-size:13px;font-weight:500;background:#fff;border:1px solid #cfd5df;border-radius:4px;cursor:pointer;transition:border-color .18s ease,background-color .18s ease,color .18s ease}.app-confirm-button[data-v-8d3bf6ad]:hover{color:#273142;background:#f2f4f7;border-color:#b9c1cd}.app-confirm-button[data-v-8d3bf6ad]:focus-visible{outline:2px solid rgba(91,80,242,.45);outline-offset:1px}.app-confirm-button.is-confirm[data-v-8d3bf6ad]{color:#fff;background:#a15c07;border-color:#a15c07}.app-confirm-button.is-confirm[data-v-8d3bf6ad]:hover{background:#844b06;border-color:#844b06}.app-confirm-dialog.is-danger .app-confirm-button.is-confirm[data-v-8d3bf6ad]{background:#c43232;border-color:#c43232}.app-confirm-dialog.is-danger .app-confirm-button.is-confirm[data-v-8d3bf6ad]:hover{background:#a92828;border-color:#a92828}.app-confirm-dialog.is-primary .app-confirm-button.is-confirm[data-v-8d3bf6ad]{background:#4f46e5;border-color:#4f46e5}.app-confirm-dialog.is-primary .app-confirm-button.is-confirm[data-v-8d3bf6ad]:hover{background:#4338ca;border-color:#4338ca}.app-confirm-enter-active[data-v-8d3bf6ad],.app-confirm-leave-active[data-v-8d3bf6ad]{transition:opacity .18s ease}.app-confirm-enter-active .app-confirm-dialog[data-v-8d3bf6ad],.app-confirm-leave-active .app-confirm-dialog[data-v-8d3bf6ad]{transition:opacity .18s ease,transform .18s ease}.app-confirm-enter-from[data-v-8d3bf6ad],.app-confirm-leave-to[data-v-8d3bf6ad]{opacity:0}.app-confirm-enter-from .app-confirm-dialog[data-v-8d3bf6ad],.app-confirm-leave-to .app-confirm-dialog[data-v-8d3bf6ad]{opacity:0;transform:translateY(4px)}@media(max-width:520px){.app-confirm-overlay[data-v-8d3bf6ad]{padding:12px}.app-confirm-actions[data-v-8d3bf6ad]{display:grid;grid-template-columns:repeat(2,minmax(0,1fr))}.app-confirm-button[data-v-8d3bf6ad]{height:auto;min-width:0;min-height:44px}}@media(prefers-reduced-motion:reduce){.app-confirm-enter-active[data-v-8d3bf6ad],.app-confirm-leave-active[data-v-8d3bf6ad],.app-confirm-enter-active .app-confirm-dialog[data-v-8d3bf6ad],.app-confirm-leave-active .app-confirm-dialog[data-v-8d3bf6ad]{transition:none}}
|
||||
@@ -1 +0,0 @@
|
||||
import{a as B,E as K}from"./el-form-item-Ct_zBuY0.js";import{E as M}from"./index-CbmPSBt0.js";import{E as h}from"./index-CCavIuTY.js";import{E as I}from"./el-tag-C_D80BPW.js";import{E as R}from"./el-divider-DF2VMDw-.js";import{E as F}from"./el-slider-CM3VEvCM.js";import{d as L,G as z,e as V,w as s,ac as D,y as $,o as f,s as a,x as p,q as j,c as k,ad as A,aa as b,f as E,M as G,g as J,v as O,z as H,j as _,A as P}from"./index-CGB0A5x_.js";import"./el-popper-DvBtRO32.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-input-number-VAjRO6-I.js";import{P as Q}from"./PageCard-D1lOaeEG.js";import{u as W}from"./usePolling-B4Zfd6BY.js";import{a as X}from"./compare-ClXsFp7Q.js";import{a as Y}from"./status-Dl1fykxa.js";import{_ as Z}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-Cgg396Sr.js";import"./_baseClone-CX1E8HTZ.js";import"./raf-CCivjDro.js";import"./index-D2-3cnGo.js";import"./index-DMVRI44M.js";import"./debounce-Bqww2DS8.js";import"./toNumber-IRhFg6gN.js";import"./clamp-BqqU4KQ-.js";import"./index-BPwPlcbL.js";import"./index-CL2S8yHq.js";import"./el-card-BWcR1iD2.js";const tt={class:"model-list"},et={key:0,class:"empty-hint"},ot=L({__name:"CompareChatView",setup(at){const T=D(),y=O(),x=T.params.id,n=$(null),e=H({systemPrompt:"",question:"",temperature:.7,topP:.9,topK:40,maxTokens:2048}),m=_(()=>{var l;if(!((l=n.value)!=null&&l.load_status))return[];try{return(typeof n.value.load_status=="string"?JSON.parse(n.value.load_status):n.value.load_status).loaded_models||[]}catch{return[]}}),C=_(()=>m.value.length>0&&m.value.every(l=>l.status==="ready"||l.status==="running")),g=_(()=>m.value.some(l=>l.status==="starting"));async function c(){try{n.value=await X(x)}catch{}}function S(){var u,i;if(!e.question.trim()){P.warning("请输入问题");return}if(g.value){P.warning("模型仍在启动中,请稍候");return}const l=new URLSearchParams({taskId:x,taskName:((u=n.value)==null?void 0:u.model_name)||((i=n.value)==null?void 0:i.name)||"",question:e.question,systemPrompt:e.systemPrompt,temperature:String(e.temperature),topP:String(e.topP),topK:String(e.topK),maxTokens:String(e.maxTokens)}),t=y.resolve(`/model-compare/result?${l.toString()}`).href;window.open(t,"_blank")}const{start:q}=W(c,5e3,{immediate:!1});return z(async()=>{await c(),q()}),(l,t)=>{const u=R,i=I,v=M,r=B,d=F,w=h,U=K;return f(),V(Q,{title:"模型对比配置"},{default:s(()=>[a(u,{"content-position":"left"},{default:s(()=>[...t[7]||(t[7]=[p("已启动模型",-1)])]),_:1}),j("div",tt,[(f(!0),k(G,null,A(m.value,(o,N)=>(f(),V(i,{key:N,type:o.status==="ready"||o.status==="running"?"success":o.status==="starting"?"warning":"danger",size:"large"},{default:s(()=>[p(b(o.model_name)+" ("+b(E(Y)(o.status))+") ",1)]),_:2},1032,["type"]))),128)),m.value.length?J("",!0):(f(),k("span",et,"暂无已启动模型"))]),a(u,{"content-position":"left"},{default:s(()=>[...t[8]||(t[8]=[p("对话配置",-1)])]),_:1}),a(U,{"label-width":"120px",style:{"max-width":"700px"}},{default:s(()=>[a(r,{label:"系统提示词"},{default:s(()=>[a(v,{modelValue:e.systemPrompt,"onUpdate:modelValue":t[0]||(t[0]=o=>e.systemPrompt=o),type:"textarea",rows:3,placeholder:"可选"},null,8,["modelValue"])]),_:1}),a(r,{label:"问题"},{default:s(()=>[a(v,{modelValue:e.question,"onUpdate:modelValue":t[1]||(t[1]=o=>e.question=o),type:"textarea",rows:4,placeholder:"请输入要对比的问题"},null,8,["modelValue"])]),_:1}),a(r,{label:"Temperature"},{default:s(()=>[a(d,{modelValue:e.temperature,"onUpdate:modelValue":t[2]||(t[2]=o=>e.temperature=o),min:0,max:2,step:.1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-p"},{default:s(()=>[a(d,{modelValue:e.topP,"onUpdate:modelValue":t[3]||(t[3]=o=>e.topP=o),min:0,max:1,step:.05,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-k"},{default:s(()=>[a(d,{modelValue:e.topK,"onUpdate:modelValue":t[4]||(t[4]=o=>e.topK=o),min:1,max:100,step:1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Max Tokens"},{default:s(()=>[a(d,{modelValue:e.maxTokens,"onUpdate:modelValue":t[5]||(t[5]=o=>e.maxTokens=o),min:256,max:4096,step:128,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,null,{default:s(()=>[a(w,{type:"primary",disabled:!C.value||g.value,onClick:S},{default:s(()=>[...t[9]||(t[9]=[p(" 开始对比 ",-1)])]),_:1},8,["disabled"]),a(w,{onClick:t[6]||(t[6]=o=>E(y).back())},{default:s(()=>[...t[10]||(t[10]=[p("返回",-1)])]),_:1})]),_:1})]),_:1})]),_:1})}}}),Ut=Z(ot,[["__scopeId","data-v-fbe74cb0"]]);export{Ut as default};
|
||||
@@ -1 +0,0 @@
|
||||
.model-list[data-v-fbe74cb0]{display:flex;flex-wrap:wrap;gap:12px;margin-bottom:12px}.empty-hint[data-v-fbe74cb0]{color:#909399}
|
||||
@@ -1 +0,0 @@
|
||||
import{E as D}from"./el-alert-DBfM1mZE.js";import{E as $}from"./index-CCavIuTY.js";import{E as z}from"./el-tag-C_D80BPW.js";import{E as F}from"./el-card-BWcR1iD2.js";import{d as O,G as U,H as J,c as d,q as i,aa as c,f as v,s as N,w as m,M as K,ad as A,ac as G,y as g,o as r,x as p,e as y,g as H}from"./index-CGB0A5x_.js";import{_ as L}from"./MarkdownView.vue_vue_type_style_index_0_lang-DqO2Wm8-.js";import{a as W,c as j}from"./compare-ClXsFp7Q.js";import{_ as Q}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./vnode-DFpZw5lG.js";const X={class:"compare-result"},Y={class:"result-header"},Z={class:"header-actions"},tt={class:"result-grid"},et={class:"card-header"},st={class:"model-name"},ot={key:0,class:"error-text"},at={key:1,class:"streaming-text"},nt={key:3,class:"loading-text"},lt={key:4,class:"result-stats"},rt=O({__name:"CompareResultView",setup(it){const u=G(),q=u.query.taskId,k=decodeURIComponent(u.query.question||""),h=decodeURIComponent(u.query.systemPrompt||""),E=Number(u.query.temperature||.7),P=Number(u.query.topP||.9),I=Number(u.query.topK||40),b=Number(u.query.maxTokens||2048),w=u.query.taskName,l=g([]),C=g(!1),x=g([]),f=new Set;async function B(){if(!C.value){C.value=!0;try{const s=await W(q);let t=[];s.load_status&&(t=(typeof s.load_status=="string"?JSON.parse(s.load_status):s.load_status).loaded_models||[]),x.value=t,l.value=t.map(o=>({name:o.model_name||"模型",content:"",displayContent:"",isTyping:!1,status:"loading"})),await Promise.all(t.map((o,e)=>S(o,e)))}catch{}}}async function S(s,t){const o=Date.now();try{const e=await V(j({port:s.port,model_name:s.model_name,messages:[...h?[{role:"system",content:h}]:[],{role:"user",content:k}],temperature:E,top_p:P,top_k:I,max_tokens:b}),3e5),a=(e==null?void 0:e.response)||(e==null?void 0:e.content)||(e==null?void 0:e.data)||JSON.stringify(e),_=(Date.now()-o)/1e3;l.value[t].content=a,l.value[t].status="done",l.value[t].stats={totalTime:_,charsPerSec:_>0?Number((a.length/_).toFixed(1)):0},M(t,a)}catch(e){l.value[t].content="推理失败: "+(e.message||""),l.value[t].status="error"}}async function V(s,t){let o=null;try{return await Promise.race([s,new Promise((e,a)=>{o=setTimeout(()=>a(new Error("推理超时")),t)})])}finally{o&&clearTimeout(o)}}function M(s,t){let o=0;l.value[s].isTyping=!0;const e=Math.max(2,Math.ceil(t.length/30)),a=setInterval(()=>{o+=e,l.value[s].displayContent=t.slice(0,o),o>=t.length&&(clearInterval(a),f.delete(a),l.value[s].displayContent=t,l.value[s].isTyping=!1)},50);f.add(a)}return U(B),J(()=>{f.forEach(clearInterval),f.clear()}),(s,t)=>{const o=$,e=D,a=z,_=F;return r(),d("div",X,[i("div",Y,[i("h2",null,"对比结果"+c(v(w)?` - ${v(w)}`:""),1),i("div",Z,[N(o,{onClick:t[0]||(t[0]=n=>s.$router.push("/model-inference"))},{default:m(()=>[...t[1]||(t[1]=[p("返回列表",-1)])]),_:1})])]),N(e,{type:"info",closable:!1,"show-icon":"",class:"question-box"},{title:m(()=>[t[2]||(t[2]=i("strong",null,"问题:",-1)),p(c(v(k)),1)]),_:1}),i("div",tt,[(r(!0),d(K,null,A(l.value,(n,R)=>(r(),y(_,{key:R,shadow:"hover",class:"result-card"},{header:m(()=>[i("div",et,[i("span",st,c(n.name),1),n.status==="loading"?(r(),y(a,{key:0,type:"warning",size:"small"},{default:m(()=>[...t[3]||(t[3]=[p("生成中...",-1)])]),_:1})):n.status==="done"?(r(),y(a,{key:1,type:"success",size:"small"},{default:m(()=>[...t[4]||(t[4]=[p("完成",-1)])]),_:1})):(r(),y(a,{key:2,type:"danger",size:"small"},{default:m(()=>[...t[5]||(t[5]=[p("失败",-1)])]),_:1}))])]),default:m(()=>{var T;return[n.status==="error"?(r(),d("div",ot,c(n.content),1)):n.isTyping?(r(),d("div",at,c(n.displayContent),1)):n.displayContent?(r(),y(L,{key:2,content:n.displayContent},null,8,["content"])):(r(),d("div",nt,[...t[6]||(t[6]=[i("i",{class:"fa fa-spinner fa-spin"},null,-1),p(" 正在生成回答... ",-1)])])),n.stats?(r(),d("div",lt,[i("span",null,"耗时 "+c((T=n.stats.totalTime)==null?void 0:T.toFixed(1))+"s",1),i("span",null,"速度 "+c(n.stats.charsPerSec)+" 字/秒",1)])):H("",!0)]}),_:2},1024))),128))])])}}}),gt=Q(rt,[["__scopeId","data-v-8391f389"]]);export{gt as default};
|
||||
@@ -1 +0,0 @@
|
||||
.compare-result[data-v-8391f389]{max-width:1200px;margin:0 auto}.result-header[data-v-8391f389]{display:flex;align-items:center;justify-content:space-between;margin-bottom:16px}.result-header h2[data-v-8391f389]{font-size:18px;font-weight:500;margin:0}.question-box[data-v-8391f389]{margin-bottom:20px}.result-grid[data-v-8391f389]{display:grid;grid-template-columns:repeat(auto-fit,minmax(420px,1fr));gap:16px}.result-card .card-header[data-v-8391f389]{display:flex;align-items:center;justify-content:space-between}.result-card .card-header .model-name[data-v-8391f389]{font-weight:500;color:#303133}.result-card .loading-text[data-v-8391f389],.result-card .error-text[data-v-8391f389]{color:#909399;min-height:80px;display:flex;align-items:center;justify-content:center}.result-card .error-text[data-v-8391f389]{color:#f56c6c}.result-card .streaming-text[data-v-8391f389]{min-height:80px;line-height:1.7;white-space:pre-wrap;word-break:break-word}.result-card .result-stats[data-v-8391f389]{display:flex;gap:16px;margin-top:12px;padding-top:12px;border-top:1px solid #ebeef5;font-size:12px;color:#909399}
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.compute-page[data-v-569994b3]{display:flex;flex-direction:column;gap:16px;min-height:0;height:100%;padding:24px;background:#fff}.compute-header[data-v-569994b3]{display:flex;justify-content:space-between;gap:16px;align-items:flex-start}.compute-header h1[data-v-569994b3]{margin:0;font-size:24px;font-weight:650;color:#111827}.compute-header p[data-v-569994b3]{margin:8px 0 0;color:#64748b}.header-actions[data-v-569994b3],.replica-toolbar[data-v-569994b3]{display:flex;align-items:center;gap:12px}.last-updated[data-v-569994b3],.muted[data-v-569994b3]{color:#64748b;font-size:12px}.last-updated[data-v-569994b3]{min-width:92px;text-align:right}.summary-grid[data-v-569994b3]{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.summary-tile[data-v-569994b3]{border:1px solid #e5e7eb;border-radius:8px;padding:14px 16px;background:#f8fafc}.summary-tile span[data-v-569994b3]{display:block;color:#64748b;font-size:12px}.summary-tile strong[data-v-569994b3]{display:block;margin-top:8px;color:#111827;font-size:24px}.compute-tabs[data-v-569994b3]{flex:1;min-height:0}.compute-tabs[data-v-569994b3] .el-tabs__content{height:calc(100% - 56px)}.compute-tabs[data-v-569994b3] .el-tab-pane{height:100%}.mono[data-v-569994b3]{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px}.replica-toolbar[data-v-569994b3]{margin-bottom:12px}.replica-toolbar .el-select[data-v-569994b3]{width:280px}.sync-progress[data-v-569994b3]{display:grid;gap:8px;margin-bottom:12px;border:1px solid #e5e7eb;border-radius:8px;padding:12px 14px;background:#f8fafc}.sync-progress>div[data-v-569994b3]{display:flex;align-items:center;gap:10px}.node-form-grid[data-v-569994b3]{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));column-gap:12px}.node-form-grid[data-v-569994b3] .el-input-number,.node-form-grid[data-v-569994b3] .el-select{width:100%}@media(max-width:960px){.compute-header[data-v-569994b3],.header-actions[data-v-569994b3],.replica-toolbar[data-v-569994b3]{flex-direction:column;align-items:stretch}.summary-grid[data-v-569994b3]{grid-template-columns:repeat(2,minmax(0,1fr))}.node-form-grid[data-v-569994b3]{grid-template-columns:1fr}}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
import{a as N,E as v}from"./el-form-item-Ct_zBuY0.js";import{E as b}from"./el-popper-DvBtRO32.js";import{E as O}from"./index-CbmPSBt0.js";import{E as S}from"./index-CCavIuTY.js";import{E as g,a as E}from"./el-select-C59URVI0.js";import{d as J,e as V,w as e,o as w,q as t,s as l,x as i,z as x,A as C}from"./index-CGB0A5x_.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-scrollbar-p6_f_LH2.js";import"./el-tag-C_D80BPW.js";import{P as U}from"./PageCard-D1lOaeEG.js";import{_ as y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-Cgg396Sr.js";import"./_baseClone-CX1E8HTZ.js";import"./index-DMVRI44M.js";import"./index-BPwPlcbL.js";import"./raf-CCivjDro.js";import"./index-D2-3cnGo.js";import"./vnode-DFpZw5lG.js";import"./index-khuXz7sd.js";import"./scroll-C0BSy_Rk.js";import"./clamp-BqqU4KQ-.js";import"./toNumber-IRhFg6gN.js";import"./_baseIteratee-uVh0OyD5.js";import"./el-card-BWcR1iD2.js";const F={class:"converter-panel"},L={class:"form-row"},T={class:"panel-footer"},B={class:"actions"},k=J({__name:"DataConvertView",setup(I){const a=x({outputName:"converted-data",encoding:"UTF-8"});function p(){C.info("当前仅完成界面设计,转换功能将在后续接入")}function d(){Object.assign(a,{outputName:"converted-data",encoding:"UTF-8"})}return(j,o)=>{const n=N,m=O,u=E,f=g,c=v,r=S,_=b;return w(),V(U,{class:"data-convert-page",title:"数据类型转换",subtitle:"将 JSON 文件转换为便于训练和评测使用的 JSONL 格式"},{default:e(()=>[t("div",F,[o[9]||(o[9]=t("div",{class:"panel-header"},[t("div",{class:"tool-icon","aria-hidden":"true"},[t("i",{class:"fa fa-exchange"})]),t("div",null,[t("h3",null,"JSON 转 JSONL"),t("p",null,"每条 JSON 数据将输出为 JSONL 文件中的一行记录")])],-1)),l(c,{class:"converter-form","label-position":"top"},{default:e(()=>[l(n,{label:"转换类型"},{default:e(()=>[...o[2]||(o[2]=[t("div",{class:"format-field","aria-label":"JSON 转 JSONL"},[t("span",null,"JSON"),t("i",{class:"fa fa-long-arrow-right","aria-hidden":"true"}),t("span",null,"JSONL")],-1)])]),_:1}),l(n,{label:"源文件",required:""},{default:e(()=>[t("button",{class:"upload-zone",type:"button",onClick:p},[...o[3]||(o[3]=[t("i",{class:"fa fa-cloud-upload","aria-hidden":"true"},null,-1),t("span",{class:"upload-content"},[t("strong",null,"点击选择或拖拽 JSON 文件到此处"),t("small",null,"仅支持 .json 格式,单文件不超过 200 MB")],-1),t("span",{class:"select-button"},"选择文件",-1)])])]),_:1}),t("div",L,[l(n,{label:"输出文件名"},{default:e(()=>[l(m,{modelValue:a.outputName,"onUpdate:modelValue":o[0]||(o[0]=s=>a.outputName=s)},{append:e(()=>[...o[4]||(o[4]=[i(".jsonl",-1)])]),_:1},8,["modelValue"])]),_:1}),l(n,{label:"字符编码"},{default:e(()=>[l(f,{modelValue:a.encoding,"onUpdate:modelValue":o[1]||(o[1]=s=>a.encoding=s),style:{width:"100%"}},{default:e(()=>[l(u,{label:"UTF-8",value:"UTF-8"})]),_:1},8,["modelValue"])]),_:1})]),o[5]||(o[5]=t("div",{class:"format-tip"},[t("i",{class:"fa fa-info-circle","aria-hidden":"true"}),t("span",null,"支持由 JSON 数组转换为 JSONL,每个数组元素输出为一行。")],-1))]),_:1}),t("div",T,[o[8]||(o[8]=t("span",{class:"prototype-label"},"当前为 UI 原型,暂不执行实际转换",-1)),t("div",B,[l(r,{onClick:d},{default:e(()=>[...o[6]||(o[6]=[i("重置",-1)])]),_:1}),l(_,{content:"转换功能将在后续开发中接入",placement:"top"},{default:e(()=>[t("span",null,[l(r,{type:"primary",disabled:""},{default:e(()=>[...o[7]||(o[7]=[i("开始转换",-1)])]),_:1})])]),_:1})])])])]),_:1})}}}),it=y(k,[["__scopeId","data-v-89a8493f"]]);export{it as default};
|
||||
@@ -1 +0,0 @@
|
||||
.converter-panel[data-v-89a8493f]{width:100%;min-height:calc(100vh - 220px);border:1px solid #e4e7ed;border-radius:8px;background:#fff;display:flex;flex-direction:column}.panel-header[data-v-89a8493f]{min-height:72px;padding:16px 20px;border-bottom:1px solid #ebeef5;background:#fafafa;display:flex;align-items:center;gap:12px;box-sizing:border-box}.panel-header .tool-icon[data-v-89a8493f]{width:38px;height:38px;flex:0 0 auto;border-radius:6px;background:var(--el-color-primary-light-9);color:var(--primary-color);display:flex;align-items:center;justify-content:center}.panel-header h3[data-v-89a8493f]{margin:0;color:#303133;font-size:15px;font-weight:600}.panel-header p[data-v-89a8493f]{margin:4px 0 0;color:#909399;font-size:12px}.converter-form[data-v-89a8493f]{flex:1;padding:22px 24px 6px}.converter-form[data-v-89a8493f] .el-form-item{margin-bottom:20px}.converter-form[data-v-89a8493f] .el-form-item__label{padding-bottom:8px;color:#606266;font-size:13px}.format-field[data-v-89a8493f]{width:100%;min-height:40px;padding:0 14px;border:1px solid #dcdfe6;border-radius:4px;background:#f5f7fa;color:#303133;display:flex;align-items:center;gap:14px;box-sizing:border-box;font-size:13px;font-weight:500}.format-field i[data-v-89a8493f]{color:#909399}.upload-zone[data-v-89a8493f]{width:100%;min-height:112px;padding:20px;border:1px dashed #b8c4d1;border-radius:6px;background:#fafcff;color:#606266;cursor:pointer;font:inherit;display:flex;align-items:center;gap:14px;text-align:left;transition:border-color .2s ease,background .2s ease}.upload-zone[data-v-89a8493f]:hover,.upload-zone[data-v-89a8493f]:focus-visible{border-color:var(--primary-color);background:var(--el-color-primary-light-9);outline:none}.upload-zone>i[data-v-89a8493f]{color:var(--primary-color);font-size:24px}.upload-zone .upload-content[data-v-89a8493f]{min-width:0;display:flex;flex:1;flex-direction:column;gap:5px}.upload-zone strong[data-v-89a8493f]{color:#303133;font-size:13px;font-weight:500}.upload-zone small[data-v-89a8493f]{color:#909399;font-size:12px}.upload-zone .select-button[data-v-89a8493f]{min-height:32px;padding:0 14px;border:1px solid #dcdfe6;border-radius:4px;background:#fff;color:#606266;display:inline-flex;align-items:center;white-space:nowrap}.form-row[data-v-89a8493f]{display:grid;grid-template-columns:minmax(0,2fr) minmax(180px,1fr);gap:16px}.format-tip[data-v-89a8493f]{min-height:38px;padding:9px 12px;border-radius:4px;background:var(--el-color-primary-light-9);color:#606266;display:flex;align-items:center;gap:8px;box-sizing:border-box;font-size:12px}.format-tip i[data-v-89a8493f]{color:var(--primary-color)}.panel-footer[data-v-89a8493f]{min-height:64px;padding:12px 24px;border-top:1px solid #ebeef5;background:#fafafa;display:flex;align-items:center;justify-content:space-between;gap:20px;box-sizing:border-box}.panel-footer .prototype-label[data-v-89a8493f]{color:#909399;font-size:12px}.panel-footer .actions[data-v-89a8493f]{display:flex;gap:10px}@media(max-width:640px){.converter-form[data-v-89a8493f]{padding:18px 16px 4px}.form-row[data-v-89a8493f]{grid-template-columns:1fr;gap:0}.upload-zone[data-v-89a8493f]{align-items:flex-start;flex-wrap:wrap}.upload-zone .select-button[data-v-89a8493f]{margin-left:38px}.panel-footer[data-v-89a8493f]{padding:12px 16px;align-items:flex-end;flex-direction:column}}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
import{E as z}from"./index-CCavIuTY.js";import{E as L}from"./el-tag-C_D80BPW.js";import{E as M}from"./el-table-DkdQ6gPU.js";import{d as P,G as V,c as h,s as o,w as s,y as c,o as d,q as f,x as n,f as b,aa as r,e as $,A as I,v as S}from"./index-CGB0A5x_.js";import{E as q}from"./index-BMrs0kLP.js";import"./el-checkbox-Ci1DH0Aj.js";import{D as A}from"./DataTablePage-CURtmoAI.js";import{g as F,d as G}from"./dataProcess-D7vRt2n_.js";import{d as v}from"./dataProcessStatus-BNY61_45.js";import{_ as R}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-p6_f_LH2.js";import"./index-DMVRI44M.js";import"./el-popper-DvBtRO32.js";import"./index-BPwPlcbL.js";import"./_baseClone-CX1E8HTZ.js";import"./_baseIteratee-uVh0OyD5.js";import"./castArray-Cgg396Sr.js";import"./debounce-Bqww2DS8.js";import"./toNumber-IRhFg6gN.js";import"./raf-CCivjDro.js";import"./validator-D7tSoy2w.js";import"./index-Coa9QRND.js";import"./clamp-BqqU4KQ-.js";import"./index-C5EKbiob.js";import"./scroll-C0BSy_Rk.js";import"./index-CbmPSBt0.js";import"./index-D2-3cnGo.js";import"./omit-DaFv1hTo.js";import"./el-card-BWcR1iD2.js";import"./el-pagination-DGQ0vO_O.js";import"./el-select-C59URVI0.js";import"./vnode-DFpZw5lG.js";import"./index-khuXz7sd.js";import"./directive-vzjADwzS.js";import"./el-tooltip-l0sNRNKZ.js";const j={class:"data-process-page",style:{height:"100%"}},H={key:1},J={class:"action-buttons"},K=P({__name:"DataProcessListView",setup(O){const x={structured:"结构化数据",unstructured:"非结构化数据",external:"外来数据源拉取"},k=S(),l=c([]),p=c(!1),m=c(null),u=c("");async function D(t=!1){t||(p.value=!0),u.value="";try{const a=await F({page:1,page_size:200});l.value=a.items}catch{u.value="数据处理任务加载失败,请稍后重试。"}finally{t||(p.value=!1)}}function C(t){const a=t.id;k.push({name:"data-process-detail",params:{id:a}})}async function E(t){try{await q.confirm(`确定删除数据处理任务“${t.name}”吗?删除后无法恢复。`,"确认删除",{type:"warning",confirmButtonText:"删除",cancelButtonText:"取消",confirmButtonClass:"el-button--danger"})}catch{return}m.value=t.id;try{await G(t.id),l.value=l.value.filter(a=>a.id!==t.id),I.success("数据处理任务已删除")}catch{}finally{m.value=null}}function T(t){if(!t)return"-";const a=new Date(t);return Number.isNaN(a.getTime())?t:a.toLocaleString("zh-CN",{hour12:!1})}function _(t){const a=Number(t);return Number.isFinite(a)?Math.max(0,Math.trunc(a)):0}function w(t){return`${_(t.source_file_count)} 个`}function B(t){return`${_(t.output_count)} 条`}return V(D),(t,a)=>{const i=M,g=L,y=z;return d(),h("div",j,[o(A,{title:"",data:l.value,loading:p.value,searchable:"","search-fields":["name"],"create-text":"新建数据处理","create-to":"/data-process/create","row-key":"id","page-size":10,"empty-text":u.value||"暂无数据处理任务"},{columns:s(()=>[o(i,{label:"任务ID",prop:"id",align:"center",width:"100"}),o(i,{label:"任务名称",prop:"name",align:"center","show-overflow-tooltip":""}),o(i,{label:"任务状态",align:"center",width:"110"},{default:s(({row:e})=>[o(g,{type:b(v)(e).type,size:"small",effect:"light"},{default:s(()=>[n(r(b(v)(e).label),1)]),_:2},1032,["type"])]),_:1}),o(i,{label:"处理类型",align:"center",width:"140"},{default:s(({row:e})=>[e.process_type?(d(),$(g,{key:0,size:"small",type:"info",effect:"plain"},{default:s(()=>[n(r(x[e.process_type]||e.process_type),1)]),_:2},1024)):(d(),h("span",H,"-"))]),_:1}),o(i,{label:"文档数量",align:"center",width:"120"},{default:s(({row:e})=>[n(r(w(e)),1)]),_:1}),o(i,{label:"生成个数",align:"center",width:"120"},{default:s(({row:e})=>[n(r(B(e)),1)]),_:1}),o(i,{label:"创建时间",align:"center",width:"190"},{default:s(({row:e})=>[n(r(T(e.create_time||e.created_at)),1)]),_:1})]),actions:s(({row:e})=>[f("div",J,[o(y,{type:"primary",link:"",size:"small",onClick:N=>C(e)},{default:s(()=>[...a[0]||(a[0]=[f("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),n("详情 ",-1)])]),_:1},8,["onClick"]),o(y,{type:"danger",link:"",size:"small",loading:m.value===e.id,onClick:N=>E(e)},{default:s(()=>[...a[1]||(a[1]=[f("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),n("删除 ",-1)])]),_:1},8,["loading","onClick"])])]),_:1},8,["data","loading","empty-text"])])}}}),Nt=R(K,[["__scopeId","data-v-c2486738"]]);export{Nt as default};
|
||||
@@ -1 +0,0 @@
|
||||
.action-buttons[data-v-c2486738]{display:flex;justify-content:center;gap:8px}
|
||||
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";.data-table-page[data-v-19a2982f]{width:100%;height:100%;box-sizing:border-box;display:flex;flex-direction:column;min-height:0}.data-table-page .table-card[data-v-19a2982f]{margin-bottom:0;width:100%;flex:1;display:flex;flex-direction:column;min-height:0;overflow:hidden}.data-table-page .table-toolbar[data-v-19a2982f]{display:flex;align-items:center;justify-content:space-between;margin-bottom:8px;padding:0 12px;flex-wrap:wrap;gap:12px}.data-table-page .table-title[data-v-19a2982f]{font-size:18px;font-weight:600;color:#1e293b;margin:0;letter-spacing:-.3px}.data-table-page .toolbar-actions[data-v-19a2982f]{display:flex;align-items:center;gap:12px}.data-table-page .search-input[data-v-19a2982f]{width:280px}.data-table-page .search-input[data-v-19a2982f] .el-input__wrapper{border-radius:8px;box-shadow:0 0 0 1px #e2e8f0 inset;transition:all .2s}.data-table-page .search-input[data-v-19a2982f] .el-input__wrapper.is-focus{box-shadow:0 0 0 1px var(--primary-color) inset}.data-table-page .batch-bar[data-v-19a2982f]{display:flex;align-items:center;justify-content:space-between;background:#4f46e50d;border:1px solid rgba(79,70,229,.1);border-radius:8px;padding:10px 16px;margin:0 24px 16px}.data-table-page .batch-bar .batch-info[data-v-19a2982f]{font-size:13px;color:var(--primary-color)}.data-table-page .batch-bar .batch-info strong[data-v-19a2982f]{font-weight:600;margin:0 4px}.data-table-page .table-body[data-v-19a2982f]{flex:1;min-height:0}.data-table-page .table-pagination[data-v-19a2982f]{display:flex;justify-content:flex-end;align-items:center;padding:16px 24px;border-top:1px solid #f1f5f9}.data-table-page .table-pagination[data-v-19a2982f] .el-pagination{margin-top:0}.data-table-page[data-v-19a2982f] .el-card__body{display:flex;flex-direction:column;padding:12px 0 0;flex:1;min-height:0}.data-table-page[data-v-19a2982f] .el-table__inner-wrapper:before{display:none}.data-table-page[data-v-19a2982f] .el-table td.el-table__cell{padding:18px 0!important}.data-table-page[data-v-19a2982f] .el-table__cell .cell{line-height:22px;white-space:nowrap}.data-table-page[data-v-19a2982f] .el-table .action-buttons{display:flex;align-items:center;justify-content:center;flex-wrap:nowrap;gap:8px}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.file-item[data-v-9270ca02]{display:flex;align-items:center;justify-content:space-between;padding:8px 12px;border:1px solid #ebeef5;border-radius:4px;margin-top:8px}.file-info[data-v-9270ca02]{display:flex;align-items:center;gap:8px}.file-name[data-v-9270ca02]{color:#303133}.file-size[data-v-9270ca02]{color:#909399;font-size:12px}.record-count[data-v-9270ca02]{margin-top:8px;font-size:13px;color:#909399}
|
||||
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";.capsule-tabs[data-v-75b95d11]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-75b95d11]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-75b95d11]:hover{color:#1e293b}.capsule-tab-item.active[data-v-75b95d11]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
@@ -1 +0,0 @@
|
||||
import{E}from"./index-CCavIuTY.js";import{E as T}from"./el-tag-C_D80BPW.js";import{E as L}from"./el-table-DkdQ6gPU.js";import{d as A,y as f,G as V,e as k,w as e,ac as $,j as B,o as y,q as r,s as l,x as i,g as N,aa as p,f as b,aY as P,aZ as S,n as w,v as M,A as R}from"./index-CGB0A5x_.js";import"./el-checkbox-Ci1DH0Aj.js";import{D as q}from"./DataTablePage-CURtmoAI.js";import{g as G,d as I,a as Y}from"./dataset-C_hkNkI5.js";import{_ as j}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-p6_f_LH2.js";import"./index-DMVRI44M.js";import"./el-popper-DvBtRO32.js";import"./index-BPwPlcbL.js";import"./_baseClone-CX1E8HTZ.js";import"./_baseIteratee-uVh0OyD5.js";import"./castArray-Cgg396Sr.js";import"./debounce-Bqww2DS8.js";import"./toNumber-IRhFg6gN.js";import"./raf-CCivjDro.js";import"./omit-DaFv1hTo.js";import"./index-CbmPSBt0.js";import"./index-D2-3cnGo.js";import"./el-card-BWcR1iD2.js";import"./el-pagination-DGQ0vO_O.js";import"./el-select-C59URVI0.js";import"./vnode-DFpZw5lG.js";import"./index-khuXz7sd.js";import"./scroll-C0BSy_Rk.js";import"./clamp-BqqU4KQ-.js";import"./directive-vzjADwzS.js";import"./index-BMrs0kLP.js";import"./validator-D7tSoy2w.js";import"./index-Coa9QRND.js";import"./index-C5EKbiob.js";import"./el-tooltip-l0sNRNKZ.js";const O={class:"capsule-tabs"},U={class:"action-buttons"},Z=A({__name:"DatasetListView",setup(F){const C=M(),D=$(),c=f(!1),u=f([]),n=f(D.query.tab==="task"?"task":"upload"),h=B(()=>n.value==="task"?u.value.filter(s=>s.source==="task"):u.value.filter(s=>s.source!=="task"));async function d(){c.value=!0;try{u.value=await G()||[]}catch{}finally{c.value=!1}}async function _(s){await I(s.id),u.value=u.value.filter(a=>a.id!==s.id),await d(),R.success("删除成功")}function x(s){C.push(`/dataset/${s.id}/preview`)}function z(s){window.open(Y(s.id),"_blank")}return V(d),(s,a)=>{const o=L,v=T,m=E;return y(),k(q,{title:"数据集管理",data:h.value,loading:c.value,searchable:"","search-fields":["name","description"],"create-text":n.value==="upload"?"上传数据集":"","create-to":"/dataset/create","delete-fn":_,"row-key":"id",onRefresh:d},{title:e(()=>[r("div",O,[r("button",{class:w(["capsule-tab-item",{active:n.value==="upload"}]),onClick:a[0]||(a[0]=t=>n.value="upload")}," 上传任务 ",2),r("button",{class:w(["capsule-tab-item",{active:n.value==="task"}]),onClick:a[1]||(a[1]=t=>n.value="task")}," 数据任务 ",2)])]),columns:e(()=>[n.value==="task"?(y(),k(o,{key:0,label:"任务ID",prop:"task_id",align:"center",width:"100"})):N("",!0),l(o,{label:"数据集名称",prop:"name",align:"center"}),l(o,{label:"数据类型",align:"center",width:"110"},{default:e(({row:t})=>[l(v,{type:"primary",size:"small"},{default:e(()=>[i(p(b(P)[String(t.type).toLowerCase()]||t.type||"-"),1)]),_:2},1024)]),_:1}),l(o,{label:"存储位置",align:"center",width:"110"},{default:e(({row:t})=>[l(v,{type:"success",size:"small"},{default:e(()=>[i(p(b(S)[t.storage_type]||t.storage_type||"-"),1)]),_:2},1024)]),_:1}),l(o,{label:"大小",align:"center",width:"100"},{default:e(({row:t})=>[i(p(t.size&&t.size!=="0 B"&&t.size!=="0"?t.size:"-"),1)]),_:1}),l(o,{label:"数据条数",align:"center",width:"100"},{default:e(({row:t})=>[i(p(t.count||0),1)]),_:1}),l(o,{label:"描述",align:"center","show-overflow-tooltip":""},{default:e(({row:t})=>[i(p(t.description||"-"),1)]),_:1}),l(o,{label:"创建时间",align:"center",width:"180"},{default:e(({row:t})=>[i(p(t.create_time?new Date(t.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:e(({row:t})=>[r("div",U,[l(m,{type:"primary",link:"",size:"small",onClick:g=>x(t)},{default:e(()=>[...a[2]||(a[2]=[r("i",{class:"fa fa-eye",style:{"margin-right":"4px"}},null,-1),i("详情 ",-1)])]),_:1},8,["onClick"]),l(m,{type:"success",link:"",size:"small",onClick:g=>z(t)},{default:e(()=>[...a[3]||(a[3]=[r("i",{class:"fa fa-download",style:{"margin-right":"4px"}},null,-1),i("下载 ",-1)])]),_:1},8,["onClick"]),l(m,{type:"danger",link:"",size:"small",onClick:g=>_(t)},{default:e(()=>[...a[4]||(a[4]=[r("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),i("删除 ",-1)])]),_:1},8,["onClick"])])]),_:1},8,["data","loading","create-text"])}}}),Tt=j(Z,[["__scopeId","data-v-75b95d11"]]);export{Tt as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
import{a as k,E as M}from"./el-form-item-Ct_zBuY0.js";import{E as q}from"./index-CbmPSBt0.js";import{E as R}from"./index-CCavIuTY.js";import{d as S,G as B,e as F,w as i,j as y,y as _,z as I,o as O,s as r,f as m,aB as P,x as h,aa as j,A as x,v as A,ac as L}from"./index-CGB0A5x_.js";import{P as N}from"./PageCard-D1lOaeEG.js";import{E as T,D as U}from"./DimensionFormFields-Dz4OI87j.js";import{b as z,u as G,c as H}from"./eval-CNlv4GHr.js";import{g as J}from"./model-xlHsHa68.js";import"./castArray-Cgg396Sr.js";import"./_baseClone-CX1E8HTZ.js";import"./raf-CCivjDro.js";import"./index-D2-3cnGo.js";import"./index-DMVRI44M.js";import"./el-card-BWcR1iD2.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-select-C59URVI0.js";import"./index-BPwPlcbL.js";import"./vnode-DFpZw5lG.js";import"./el-popper-DvBtRO32.js";import"./el-scrollbar-p6_f_LH2.js";import"./index-khuXz7sd.js";import"./el-tag-C_D80BPW.js";import"./scroll-C0BSy_Rk.js";import"./clamp-BqqU4KQ-.js";import"./toNumber-IRhFg6gN.js";import"./_baseIteratee-uVh0OyD5.js";import"./el-checkbox-Ci1DH0Aj.js";import"./omit-DaFv1hTo.js";import"./el-radio-group-B7nPUe8-.js";import"./el-input-number-VAjRO6-I.js";import"./index-CL2S8yHq.js";import"./el-slider-CM3VEvCM.js";import"./debounce-Bqww2DS8.js";import"./el-switch-ZjdGiO6T.js";import"./validator-D7tSoy2w.js";import"./el-tooltip-l0sNRNKZ.js";/* empty css */const Se=S({__name:"DimensionCreateView",setup(K){const c=L(),f=A(),n=_(),p=_(!1),u=y(()=>!!c.params.id),s=y(()=>c.params.id),d=_([]);let e=I({name:"",type:"",description:"",eval_model:"",eval_method:"",eval_prompt:"",is_active:!0,is_default:!1,bleu_n:1,output_precision:3,score_min:0,score_max:5,pass_threshold:3});const b={name:[{required:!0,message:"请输入维度名称",trigger:"blur"},{max:50,message:"不超过 50 字符",trigger:"blur"}],type:[{required:!0,message:"请选择指标类型",trigger:"change"}],eval_model:[{required:!0,message:"请选择大模型",trigger:"change"}],eval_method:[{required:!0,message:"请选择评估方式",trigger:"change"}],eval_prompt:[{required:!0,message:"请填写评估 Prompt",trigger:"blur"}]};async function w(){var o,t;if(s.value)try{const a=await z(s.value);Object.assign(e,{name:a.name||"",type:a.type||"",description:a.description||"",eval_model:a.eval_model||"",eval_method:a.eval_method||(a.type?(t=(o=T[a.type])==null?void 0:o[0])==null?void 0:t.value:""),eval_prompt:a.eval_prompt||"",is_active:a.is_active!==!1,is_default:!!a.is_default,bleu_n:a.bleu_n??1,output_precision:a.output_precision??3,score_min:a.score_min??0,score_max:a.score_max??5,pass_threshold:a.pass_threshold??3})}catch{}}async function E(){try{const o=await J()||[];d.value=o.filter(t=>t.purpose==="evaluation")}catch{d.value=[]}}async function D(){n.value&&await n.value.validate(async o=>{if(o){p.value=!0;try{const t={name:e.name,type:e.type,description:e.description,eval_model:e.type==="text_similarity"?null:e.eval_model,eval_method:e.eval_method,eval_prompt:e.type==="text_similarity"?null:e.eval_prompt,is_active:e.is_active,is_default:e.is_default,create_time:new Date().toISOString()};e.type==="text_similarity"&&(t.bleu_n=e.bleu_n,t.output_precision=e.output_precision),e.type==="metric"&&(t.score_min=e.score_min,t.score_max=e.score_max,t.pass_threshold=e.pass_threshold),u.value&&s.value?(await G(s.value,t),x.success("更新成功")):(await H(t),x.success("创建成功")),f.push("/model-eval")}catch{}finally{p.value=!1}}})}function V(){f.back()}return B(()=>{E(),w()}),(o,t)=>{const a=q,v=k,g=R,C=M;return O(),F(N,{title:u.value?"编辑评测维度":"添加评测维度"},{default:i(()=>[r(C,{ref_key:"formRef",ref:n,model:m(e),rules:b,"label-width":"130px",style:{"max-width":"760px"}},{default:i(()=>[r(v,{label:"维度名称",prop:"name"},{default:i(()=>[r(a,{modelValue:m(e).name,"onUpdate:modelValue":t[0]||(t[0]=l=>m(e).name=l),placeholder:"请输入维度名称",maxlength:"50","show-word-limit":""},null,8,["modelValue"])]),_:1}),r(U,{modelValue:m(e),"onUpdate:modelValue":t[1]||(t[1]=l=>P(e)?e.value=l:e=l),"eval-models":d.value},null,8,["modelValue","eval-models"]),r(v,null,{default:i(()=>[r(g,{type:"primary",loading:p.value,onClick:D},{default:i(()=>[h(j(u.value?"保存":"创建"),1)]),_:1},8,["loading"]),r(g,{onClick:V},{default:i(()=>[...t[2]||(t[2]=[h("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1},8,["title"])}}});export{Se as default};
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.method-desc[data-v-c194cd75]{color:#909399;font-size:12px;margin-left:6px}
|
||||
@@ -1 +0,0 @@
|
||||
.step-form[data-v-684556c4]{max-width:760px}.field-tip[data-v-684556c4]{color:#909399;font-size:12px;line-height:20px}.source-alert[data-v-684556c4]{margin:-4px 0 20px 120px;max-width:640px}.step-form[data-v-91940aa0]{max-width:760px}.step-form-wide[data-v-91940aa0]{max-width:920px}.basic-metric-form[data-v-dac20296]{max-width:760px}.start-eval-step[data-v-7800c08e]{max-width:920px}.create-wizard-layout[data-v-95b7e7f1]{display:flex;flex-direction:column;min-height:620px;margin:-20px;background:#fff}.wizard-main[data-v-95b7e7f1]{flex:1;padding:32px}.wizard-steps-container[data-v-95b7e7f1]{margin-bottom:32px;padding-bottom:24px;border-bottom:1px dashed #e2e8f0}.custom-wizard-steps[data-v-95b7e7f1]{display:flex;align-items:center;justify-content:space-between;max-width:1120px;margin:0 auto;padding:0 20px}.step-item[data-v-95b7e7f1]{display:flex;flex:none;align-items:center}.step-connector[data-v-95b7e7f1]{flex:1;height:2px;margin:0 12px;background:#e2e8f0;transition:background-color .2s ease}.step-connector.is-active[data-v-95b7e7f1]{background:#5146e5}.step-node[data-v-95b7e7f1]{display:flex;align-items:center;gap:12px}.step-icon[data-v-95b7e7f1]{display:flex;width:30px;height:30px;flex:0 0 30px;align-items:center;justify-content:center;box-sizing:border-box;border:2px solid #cbd5e1;border-radius:50%;background:#fff;color:#64748b;font-size:13px;font-weight:650;transition:all .2s ease}.step-title[data-v-95b7e7f1]{color:#64748b;font-size:15px;font-weight:600;white-space:nowrap}.step-description[data-v-95b7e7f1]{margin-top:3px;color:#94a3b8;font-size:12px;white-space:nowrap}.step-item.is-active .step-icon[data-v-95b7e7f1]{border-color:#5146e5;background:#eef2ff;color:#5146e5}.step-item.is-active .step-title[data-v-95b7e7f1],.step-item.is-completed .step-title[data-v-95b7e7f1]{color:#1e293b}.step-item.is-completed .step-icon[data-v-95b7e7f1]{border-color:#5146e5;background:#5146e5;color:#fff}.wizard-content[data-v-95b7e7f1]{max-width:920px;min-height:400px;margin:0 auto}.wizard-footer[data-v-95b7e7f1]{display:flex;min-height:64px;flex-shrink:0;align-items:center;justify-content:flex-end;gap:12px;padding:0 32px;border-top:1px solid #e2e8f0;background:#fff;box-shadow:0 -4px 6px -1px #0f172a05}.footer-back[data-v-95b7e7f1]{margin-right:auto}.footer-button-icon[data-v-95b7e7f1]{margin-right:6px}.footer-button-icon.is-right[data-v-95b7e7f1]{margin-right:0;margin-left:6px}@media(max-width:1100px){.step-description[data-v-95b7e7f1]{display:none}}@media(max-width:760px){.wizard-main[data-v-95b7e7f1]{padding:24px 20px}.custom-wizard-steps[data-v-95b7e7f1]{padding:0}.step-connector[data-v-95b7e7f1]{margin:0 8px}.step-text[data-v-95b7e7f1]{display:none}.wizard-footer[data-v-95b7e7f1]{padding:0 20px}}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/EvalView-CH6pm7i4.css
vendored
1
frontend/dist/assets/EvalView-CH6pm7i4.css
vendored
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";.eval-page[data-v-894b0bb7]{width:100%;height:100%;display:flex;flex-direction:column;min-height:0}.capsule-tabs[data-v-894b0bb7]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-894b0bb7]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-894b0bb7]:hover{color:#1e293b}.capsule-tab-item.active[data-v-894b0bb7]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
1
frontend/dist/assets/EvalView-CnH5RcpQ.js
vendored
1
frontend/dist/assets/EvalView-CnH5RcpQ.js
vendored
@@ -1 +0,0 @@
|
||||
import{E as V}from"./index-CCavIuTY.js";import{E as $}from"./el-table-DkdQ6gPU.js";import{d as B,G as D,c as T,e as _,w as o,g as C,y as p,o as v,s as a,q as i,x as b,aa as L,n as c,v as N,A as z}from"./index-CGB0A5x_.js";import{E as M}from"./index-BMrs0kLP.js";import"./el-tag-C_D80BPW.js";import"./el-checkbox-Ci1DH0Aj.js";import{D as y}from"./DataTablePage-CURtmoAI.js";import{_ as A}from"./ModelStatusTag.vue_vue_type_script_setup_true_lang-NhrqSCT2.js";import{g as G,d as P}from"./eval-CNlv4GHr.js";import{_ as R}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-p6_f_LH2.js";import"./index-DMVRI44M.js";import"./el-popper-DvBtRO32.js";import"./index-BPwPlcbL.js";import"./_baseClone-CX1E8HTZ.js";import"./_baseIteratee-uVh0OyD5.js";import"./castArray-Cgg396Sr.js";import"./debounce-Bqww2DS8.js";import"./toNumber-IRhFg6gN.js";import"./raf-CCivjDro.js";import"./validator-D7tSoy2w.js";import"./index-Coa9QRND.js";import"./clamp-BqqU4KQ-.js";import"./index-C5EKbiob.js";import"./scroll-C0BSy_Rk.js";import"./index-CbmPSBt0.js";import"./index-D2-3cnGo.js";import"./omit-DaFv1hTo.js";import"./el-card-BWcR1iD2.js";import"./el-pagination-DGQ0vO_O.js";import"./el-select-C59URVI0.js";import"./vnode-DFpZw5lG.js";import"./index-khuXz7sd.js";import"./directive-vzjADwzS.js";import"./el-tooltip-l0sNRNKZ.js";import"./status-Dl1fykxa.js";const S={class:"eval-page"},q={class:"capsule-tabs"},I={class:"capsule-tabs"},Q=B({__name:"EvalView",setup(j){const f=N(),s=p("tasks"),d=p(!1),n=p([]),w=p([{rank:1,name:"GPT-4",score:92.5},{rank:2,name:"Claude-3",score:90.1},{rank:3,name:"Qwen-Max",score:85.3}]);async function u(){d.value=!0;try{n.value=await G()||[]}catch{n.value=[]}finally{d.value=!1}}async function k(r){await M.confirm("确定要删除这个评测任务吗?","确认删除",{type:"warning"}),await P(r.id),n.value=n.value.filter(e=>e.id!==r.id),z.success("删除成功"),await u()}function E(){f.push("/model-eval/create")}function m(r){return s.value===r}function x(r){f.push({name:"model-eval-detail",params:{id:r.id}})}return D(()=>{u()}),(r,e)=>{const l=$,g=V;return v(),T("div",S,[s.value==="tasks"?(v(),_(y,{key:0,title:"",data:n.value,loading:d.value,"delete-fn":k,"create-text":"创建评测任务",onCreate:E,"row-key":"id",onRefresh:u},{title:o(()=>[i("div",q,[i("button",{class:c(["capsule-tab-item",{active:m("tasks")}]),onClick:e[0]||(e[0]=t=>s.value="tasks")},"评测任务",2),i("button",{class:c(["capsule-tab-item",{active:m("leaderboard")}]),onClick:e[1]||(e[1]=t=>s.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(l,{label:"任务名称",prop:"eval_task_name",align:"center"}),a(l,{label:"评测模型",prop:"model_name",align:"center"}),a(l,{label:"数据集",prop:"dataset",align:"center"}),a(l,{label:"指标",prop:"metric",align:"center"}),a(l,{label:"评分",prop:"score",width:"100",align:"center"}),a(l,{label:"状态",width:"100",align:"center"},{default:o(({row:t})=>[a(A,{status:t.status},null,8,["status"])]),_:1}),a(l,{label:"创建时间",align:"center",width:"180"},{default:o(({row:t})=>[b(L(t.create_time?new Date(t.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:o(({row:t})=>[a(g,{link:"",type:"primary",size:"small",onClick:h=>x(t)},{default:o(()=>[...e[4]||(e[4]=[i("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),b("详情 ",-1)])]),_:1},8,["onClick"]),a(g,{link:"",type:"danger",size:"small",onClick:h=>k(t)},{default:o(()=>[...e[5]||(e[5]=[i("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),b("删除 ",-1)])]),_:1},8,["onClick"])]),_:1},8,["data","loading"])):C("",!0),s.value==="leaderboard"?(v(),_(y,{key:1,title:"",data:w.value,"row-key":"rank"},{title:o(()=>[i("div",I,[i("button",{class:c(["capsule-tab-item",{active:m("tasks")}]),onClick:e[2]||(e[2]=t=>s.value="tasks")},"评测任务",2),i("button",{class:c(["capsule-tab-item",{active:m("leaderboard")}]),onClick:e[3]||(e[3]=t=>s.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(l,{label:"排名",prop:"rank",width:"80",align:"center"}),a(l,{label:"模型名称",prop:"name",align:"center"}),a(l,{label:"综合评分",prop:"score",align:"center"})]),_:1},8,["data"])):C("",!0)])}}}),Ve=R(Q,[["__scopeId","data-v-894b0bb7"]]);export{Ve as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.el-popover{--el-popover-bg-color:var(--el-bg-color-overlay);--el-popover-font-size:var(--el-font-size-base);--el-popover-border-color:var(--el-border-color-lighter);--el-popover-padding:12px;--el-popover-padding-large:18px 20px;--el-popover-title-font-size:16px;--el-popover-title-text-color:var(--el-text-color-primary);--el-popover-border-radius:4px}.el-popover.el-popper{background:var(--el-popover-bg-color);border-radius:var(--el-popover-border-radius);border:1px solid var(--el-popover-border-color);min-width:150px;padding:var(--el-popover-padding);z-index:var(--el-index-popper);color:var(--el-text-color-regular);line-height:1.4;font-size:var(--el-popover-font-size);box-shadow:var(--el-box-shadow-light);overflow-wrap:break-word;box-sizing:border-box}.el-popover.el-popper--plain{padding:var(--el-popover-padding-large)}.el-popover__title{color:var(--el-popover-title-text-color);font-size:var(--el-popover-title-font-size);margin-bottom:12px;line-height:1}.el-popover__reference:focus:not(.focusing),.el-popover__reference:focus:hover{outline-width:0}.el-popover.el-popper.is-dark{--el-popover-bg-color:var(--el-text-color-primary);--el-popover-border-color:var(--el-text-color-primary);--el-popover-title-text-color:var(--el-bg-color);color:var(--el-bg-color)}.el-popover.el-popper:focus:active,.el-popover.el-popper:focus{outline-width:0}.progress-value[data-v-4ab0bbc0]{color:var(--primary-color);font-weight:600}.filter-header[data-v-4ab0bbc0]{display:inline-flex;align-items:center;gap:6px}.filter-badge[data-v-4ab0bbc0]{line-height:1}.filter-icon[data-v-4ab0bbc0]{cursor:pointer;font-size:12px;color:#c0c4cc;transition:color .2s}.filter-icon[data-v-4ab0bbc0]:hover,.filter-icon.active[data-v-4ab0bbc0]{color:#1890ff}.filter-options[data-v-4ab0bbc0]{display:flex;flex-direction:column;gap:8px;max-height:240px;overflow-y:auto}.filter-actions[data-v-4ab0bbc0]{text-align:right;margin-top:8px;border-top:1px solid #ebeef5;padding-top:8px}
|
||||
1
frontend/dist/assets/GuideView-0IY716sm.js
vendored
1
frontend/dist/assets/GuideView-0IY716sm.js
vendored
@@ -1 +0,0 @@
|
||||
import{_ as s}from"./_plugin-vue_export-helper-DlAUqK2U.js";import{o as t,c,q as o}from"./index-CGB0A5x_.js";const n={},r={class:"simple-page"};function a(_,e){return t(),c("section",r,[...e[0]||(e[0]=[o("h1",null,"使用文档",-1),o("p",null,"第一版系统已接入后端、数据集、模型、微调任务、算力节点和训练日志主链路。",-1)])])}const p=s(n,[["render",a],["__scopeId","data-v-962c8803"]]);export{p as default};
|
||||
1
frontend/dist/assets/GuideView-BuVTsF9M.css
vendored
1
frontend/dist/assets/GuideView-BuVTsF9M.css
vendored
@@ -1 +0,0 @@
|
||||
.simple-page[data-v-962c8803]{padding:24px}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
import{E as T}from"./el-alert-DBfM1mZE.js";import{a as A,E as D}from"./el-form-item-Ct_zBuY0.js";import{E as L}from"./index-CbmPSBt0.js";import{E as j}from"./index-CCavIuTY.js";import{E as z,b as H,a as J}from"./el-select-C59URVI0.js";import{E as K}from"./el-divider-DF2VMDw-.js";import{d as Q,G as W,e as i,w as o,at as X,y as u,z as Y,o as s,s as a,x as b,c as v,ad as k,M as w,g as Z,j as p,A as B,v as ee}from"./index-CGB0A5x_.js";import"./el-scrollbar-p6_f_LH2.js";import"./el-popper-DvBtRO32.js";import"./el-tag-C_D80BPW.js";import{P as te}from"./PageCard-D1lOaeEG.js";import{g as le,a as ae}from"./model-xlHsHa68.js";import"./vnode-DFpZw5lG.js";import"./castArray-Cgg396Sr.js";import"./_baseClone-CX1E8HTZ.js";import"./raf-CCivjDro.js";import"./index-D2-3cnGo.js";import"./index-DMVRI44M.js";import"./index-BPwPlcbL.js";import"./index-khuXz7sd.js";import"./scroll-C0BSy_Rk.js";import"./clamp-BqqU4KQ-.js";import"./toNumber-IRhFg6gN.js";import"./_baseIteratee-uVh0OyD5.js";import"./el-card-BWcR1iD2.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";const Ge=Q({__name:"InferenceCreateView",setup(oe){const E=ee(),f=u(),c=u(!1),m=u(""),V=u([]),h=u([]),_=u([]),x=p(()=>V.value.map(t=>({key:`db-${t.id}`,id:t.id,name:t.name,source:"database",model_path:t.path||""}))),M=p(()=>h.value.map(t=>({key:`trained-${t.id}`,id:t.id,name:t.name,source:"trained",model_path:t.merged_path||t.base_model_path||"",merged:t.merged,merging:t.merging,disabled:t.merged===!1}))),G=p(()=>{const t={};for(const e of[...x.value,...M.value])t[e.key]=e;return t}),n=Y({name:"",description:"",model_key:"",gpu_id:0}),I={name:[{required:!0,message:"请输入推理名称",trigger:"blur"}],model_key:[{required:!0,message:"请选择模型",trigger:"change"}]},O=p(()=>G.value[n.model_key]);async function S(){f.value&&await f.value.validate(async t=>{if(!t)return;const e=O.value;if(!e){B.warning("请选择模型");return}c.value=!0,m.value="正在启动模型服务...";try{await new Promise(r=>setTimeout(r,1200)),B.success("模型已启动"),E.push({path:"/model-inference/chat/mock",query:{model:e.name}})}finally{c.value=!1,m.value=""}})}function $(){E.back()}async function q(){try{const[t,e,r]=await Promise.all([le(),ae(),X()]);V.value=t||[],h.value=(e==null?void 0:e.models)||[],_.value=(r==null?void 0:r.gpu)||[],_.value.length>0&&(n.gpu_id=0)}catch{}}return W(q),(t,e)=>{const r=L,d=A,F=K,g=J,C=H,P=z,N=T,U=j,R=D;return s(),i(te,{title:"新建推理"},{default:o(()=>[a(R,{ref_key:"formRef",ref:f,model:n,rules:I,"label-width":"100px"},{default:o(()=>[a(d,{label:"推理名称",prop:"name"},{default:o(()=>[a(r,{modelValue:n.name,"onUpdate:modelValue":e[0]||(e[0]=l=>n.name=l),placeholder:"请输入推理名称",maxlength:"50","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(d,{label:"描述"},{default:o(()=>[a(r,{modelValue:n.description,"onUpdate:modelValue":e[1]||(e[1]=l=>n.description=l),type:"textarea",rows:2,maxlength:"200","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(F,{"content-position":"left"},{default:o(()=>[...e[4]||(e[4]=[b("选择模型",-1)])]),_:1}),a(d,{label:"选择模型",prop:"model_key"},{default:o(()=>[a(P,{modelValue:n.model_key,"onUpdate:modelValue":e[2]||(e[2]=l=>n.model_key=l),placeholder:"请选择模型",filterable:"",style:{width:"400px"}},{default:o(()=>[a(C,{label:"本地模型"},{default:o(()=>[(s(!0),v(w,null,k(x.value,l=>(s(),i(g,{key:l.key,label:l.name,value:l.key},null,8,["label","value"]))),128))]),_:1}),a(C,{label:"已训练模型"},{default:o(()=>[(s(!0),v(w,null,k(M.value,l=>(s(),i(g,{key:l.key,label:l.name+(l.disabled?"(未合并)":""),value:l.key,disabled:l.disabled},null,8,["label","value","disabled"]))),128))]),_:1})]),_:1},8,["modelValue"])]),_:1}),a(d,{label:"GPU"},{default:o(()=>[a(P,{modelValue:n.gpu_id,"onUpdate:modelValue":e[3]||(e[3]=l=>n.gpu_id=l),style:{width:"400px"}},{default:o(()=>[(s(!0),v(w,null,k(_.value,(l,y)=>(s(),i(g,{key:y,label:`${l.name} (GPU${y})`,value:y},null,8,["label","value"]))),128))]),_:1},8,["modelValue"])]),_:1}),m.value?(s(),i(d,{key:0,label:"启动状态"},{default:o(()=>[a(N,{title:m.value,type:"info",closable:!1,"show-icon":""},null,8,["title"])]),_:1})):Z("",!0),a(d,null,{default:o(()=>[a(U,{type:"primary",loading:c.value,onClick:S},{default:o(()=>[...e[5]||(e[5]=[b("开始推理",-1)])]),_:1},8,["loading"]),a(U,{onClick:$},{default:o(()=>[...e[6]||(e[6]=[b("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1})}}});export{Ge as default};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user