Compare commits
20 Commits
3fd9cf9100
...
baseline/f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec7d8c0a3d | ||
|
|
0292bf5138 | ||
|
|
0271942ba5 | ||
|
|
250e060271 | ||
|
|
7b36bc774e | ||
|
|
4e5c43fad5 | ||
|
|
62a1d03eac | ||
|
|
94230cad16 | ||
|
|
0c601934a0 | ||
|
|
5cc306eb0a | ||
|
|
cc08b164d0 | ||
|
|
24c77a990a | ||
|
|
15c4223f2c | ||
|
|
b975de02da | ||
|
|
46d343fb63 | ||
|
|
0c39f2f5b9 | ||
|
|
c7c9ed925b | ||
|
|
f917a025e1 | ||
|
|
a9ab130d43 | ||
|
|
525fc55cef |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -157,6 +157,8 @@ backend/config.yaml
|
||||
.codex-backups/
|
||||
.pnpm-store/
|
||||
.zcode/
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
||||
39
README.md
39
README.md
@@ -134,12 +134,45 @@ npm run dev
|
||||
|
||||
## 算力服务启动
|
||||
|
||||
算力服务是一个 FastAPI 应用,同时承载 Compute API(模型训练/推理/GPU 管理)和 File Gateway(文件上传下载)路由。Docker 部署时对外暴露两个端口(19100 和 19101)均指向同一服务,方便应用平台分别配置 `api_base_url` 和 `file_gateway_url`。本地开发只需启动一个进程。
|
||||
|
||||
### 方式一:Docker 启动(推荐)
|
||||
|
||||
```bash
|
||||
cd compute
|
||||
uvicorn api.main:app --reload --port 19100
|
||||
cd docker/compute
|
||||
cp .env.example .env
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
默认 `COMPUTE_MODE=real`。真实 GPU 接入时,在每台算力服务器上部署 Compute API、Agent、File Gateway 和 LLaMA-Factory,应用平台通过 `compute_nodes.api_base_url` 和 `compute_nodes.file_gateway_url` 主动轮询。仅在隔离联调环境可显式设置 `COMPUTE_MODE=simulator` 或 `COMPUTE_EXECUTION_MODE=simulator`。
|
||||
### 方式二:本地开发启动
|
||||
|
||||
**Windows (cmd):**
|
||||
|
||||
```cmd
|
||||
cd /d E:\yg_ft\compute
|
||||
set PYTHONPATH=E:\yg_ft
|
||||
.\.venv\Scripts\python.exe -m uvicorn api.main:app --reload --port 19100
|
||||
```
|
||||
|
||||
> `PYTHONPATH=E:\yg_ft` 是必需的,因为代码使用 `from compute.agent...` 绝对导入。
|
||||
|
||||
**Linux / macOS:**
|
||||
|
||||
```bash
|
||||
cd compute
|
||||
PYTHONPATH=.. uvicorn api.main:app --reload --port 19100
|
||||
```
|
||||
|
||||
### 环境变量说明
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|---|---|---|
|
||||
| `COMPUTE_MODE` | `real` | `real` / `simulator`,仅隔离联调用 simulator |
|
||||
| `COMPUTE_EXECUTION_MODE` | `real` | 训练执行模式 |
|
||||
| `COMPUTE_SERVICE_TOKEN` | `change_me` | 服务间认证 token |
|
||||
| `MODELTF_ROUTE_PREFIX` | `/modelTF` | API 路由前缀 |
|
||||
|
||||
应用平台通过数据库 `compute_nodes` 表中的 `api_base_url` 和 `file_gateway_url` 主动轮询算力节点状态。
|
||||
|
||||
## 日志
|
||||
|
||||
|
||||
10
backend/_check_sessions.py
Normal file
10
backend/_check_sessions.py
Normal file
@@ -0,0 +1,10 @@
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
store = get_platform_store()
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, user_id, login_at, logout_at, duration_seconds FROM sessions ORDER BY login_at DESC LIMIT 10"
|
||||
).fetchall()
|
||||
print(f"sessions count: {len(rows)}")
|
||||
for r in rows:
|
||||
print(f" user={r['user_id'][:25]}... login={r['login_at']} logout={r['logout_at']} dur={r['duration_seconds']}")
|
||||
@@ -8,12 +8,14 @@ import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from threading import BoundedSemaphore, Lock
|
||||
from typing import Any, Iterator, Literal
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
import httpx
|
||||
@@ -32,6 +34,7 @@ from fastapi import (
|
||||
from fastapi.responses import StreamingResponse
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
||||
from app.modules.data_process.algorithms import (
|
||||
ParsedText,
|
||||
canonical_record_json,
|
||||
@@ -45,9 +48,10 @@ from app.modules.data_process.algorithms import (
|
||||
is_near_duplicate,
|
||||
near_duplicate_fingerprint,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
structured_json_dumps,
|
||||
)
|
||||
from app.modules.data_process.document_chunking import (
|
||||
DocumentChunk,
|
||||
@@ -75,9 +79,11 @@ from app.modules.data_process.store import (
|
||||
NotFoundError,
|
||||
get_data_process_store,
|
||||
new_id,
|
||||
repeat_task_id,
|
||||
)
|
||||
from app.schemas.data_process import (
|
||||
DataProcessRegenerateRequest,
|
||||
DataProcessRepeatRequest,
|
||||
DataProcessStatus,
|
||||
DataProcessTaskCreate,
|
||||
DataProcessTaskUpdate,
|
||||
@@ -261,7 +267,14 @@ def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
|
||||
content = str(source.get("content") or "")
|
||||
file_format = str(source.get("file_format") or "").lower()
|
||||
if file_format == "xlsx":
|
||||
# XLSX 上传阶段已安全解析为 JSONL 后入库。
|
||||
raw_content = source.get("raw_content")
|
||||
if isinstance(raw_content, bytes):
|
||||
return parse_text_content(
|
||||
raw_content,
|
||||
filename=str(source.get("name") or "source.xlsx"),
|
||||
file_format="xlsx",
|
||||
)
|
||||
# 兼容原始对象已缺失的历史文件:退化为上传阶段生成的 JSONL。
|
||||
return parse_text_content(content, file_format="jsonl")
|
||||
if file_format in {"pdf", "docx", "pptx"}:
|
||||
# 文档上传阶段已抽取文本,预览阶段只需要对正文切片。
|
||||
@@ -381,11 +394,12 @@ def _build_preview_items(
|
||||
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
def append_item(item: dict[str, Any]) -> None:
|
||||
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
|
||||
content = str(item.get("edited_content") or "").strip()
|
||||
if should_clean_invalid and not content:
|
||||
return
|
||||
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
# 去重必须基于脱敏前内容,否则不同原文可能在替换 PII 后被错误合并。
|
||||
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
|
||||
if should_deduplicate and content_hash in seen_content_hashes:
|
||||
return
|
||||
seen_content_hashes.add(content_hash)
|
||||
@@ -447,6 +461,7 @@ def _build_preview_items(
|
||||
continue
|
||||
for key in band_keys:
|
||||
seen_near_duplicate_bands.setdefault(key, []).append(content)
|
||||
dedup_content = content
|
||||
pii_counts: dict[str, int] = {}
|
||||
if should_desensitize:
|
||||
content, pii_counts = desensitize_pii(content)
|
||||
@@ -476,7 +491,8 @@ def _build_preview_items(
|
||||
else "original"
|
||||
),
|
||||
"quality_score": quality,
|
||||
}
|
||||
},
|
||||
dedup_content=dedup_content,
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -488,48 +504,76 @@ def _build_preview_items(
|
||||
"filter_anomaly",
|
||||
}
|
||||
source_records = list(parsed.records)
|
||||
processed_records = preprocess_structured_records(
|
||||
processed_records = preprocess_structured_records_with_lineage(
|
||||
source_records,
|
||||
structured_options,
|
||||
)
|
||||
if not processed_records and parsed.text and not source_records:
|
||||
processed_records = [{"value": parsed.text}]
|
||||
same_cardinality = len(processed_records) == len(source_records)
|
||||
for index, record in enumerate(processed_records):
|
||||
original_record = source_records[index] if same_cardinality else record
|
||||
original_content = json.dumps(
|
||||
original_record,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
for processed in processed_records:
|
||||
source_index = processed.source_index
|
||||
record = processed.record
|
||||
original_record = (
|
||||
source_records[source_index]
|
||||
if source_index < len(source_records)
|
||||
else record
|
||||
)
|
||||
source_locator = (
|
||||
deepcopy(parsed.record_locators[source_index])
|
||||
if source_index < len(parsed.record_locators)
|
||||
else None
|
||||
)
|
||||
original_content = structured_json_dumps(original_record)
|
||||
pii_counts: dict[str, int] = {}
|
||||
edited_record = record
|
||||
dedup_content = (
|
||||
canonical_record_json(record)
|
||||
if "normalize_format" in preprocess_options
|
||||
else structured_json_dumps(record)
|
||||
)
|
||||
if should_desensitize:
|
||||
edited_record, pii_counts = desensitize_structured_record(record)
|
||||
content = (
|
||||
canonical_record_json(edited_record)
|
||||
if "normalize_format" in preprocess_options
|
||||
else json.dumps(
|
||||
edited_record,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
else structured_json_dumps(edited_record)
|
||||
)
|
||||
quality = _preview_quality(content, config)
|
||||
quality["pii_replacements"] = pii_counts
|
||||
if source_locator is not None:
|
||||
quality["source_locator"] = source_locator
|
||||
source_start = (
|
||||
source_locator.get("source_start")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_end = (
|
||||
source_locator.get("source_end")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_start_line = (
|
||||
source_locator.get("start_line")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
source_end_line = (
|
||||
source_locator.get("end_line")
|
||||
if source_locator is not None
|
||||
else None
|
||||
)
|
||||
append_item(
|
||||
{
|
||||
"source_file_id": source["id"],
|
||||
"original_content": original_content,
|
||||
"edited_content": content,
|
||||
"source_start": None,
|
||||
"source_end": None,
|
||||
"source_start_line": None,
|
||||
"source_end_line": None,
|
||||
"source_start": source_start,
|
||||
"source_end": source_end,
|
||||
"source_start_line": source_start_line,
|
||||
"source_end_line": source_end_line,
|
||||
"token_count": estimate_token_count(content),
|
||||
"status": "modified" if content != original_content else "original",
|
||||
"quality_score": quality,
|
||||
}
|
||||
},
|
||||
dedup_content=dedup_content,
|
||||
)
|
||||
return items
|
||||
|
||||
@@ -772,18 +816,33 @@ def list_tasks(
|
||||
keyword: str | None = Query(default=None),
|
||||
status: DataProcessStatus | None = Query(default=None),
|
||||
process_type: ProcessType | None = Query(default=None),
|
||||
tenant_id: str | None = Query(default=None),
|
||||
project_id: str | None = Query(default=None),
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
return ok(
|
||||
store.list_tasks(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
process_type=process_type,
|
||||
)
|
||||
tasks = store.list_tasks(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
keyword=keyword,
|
||||
status=status,
|
||||
process_type=process_type,
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
)
|
||||
# #4 资源 ACL 过滤:admin 放行,普通用户只看到自己被授权的数据处理任务
|
||||
items = tasks.get("items", [])
|
||||
if not is_admin(current_user) and items:
|
||||
accessible_ids = set(
|
||||
filter_accessible_resource_ids(
|
||||
"data-process", [t["id"] for t in items], current_user
|
||||
)
|
||||
)
|
||||
items = [t for t in items if t["id"] in accessible_ids]
|
||||
tasks["items"] = items
|
||||
tasks["total"] = len(items)
|
||||
return ok(tasks)
|
||||
|
||||
|
||||
@router.post("")
|
||||
@@ -850,6 +909,135 @@ def prepare_regeneration(
|
||||
)
|
||||
|
||||
|
||||
def _repeat_file_copies(
|
||||
store: DataProcessStore,
|
||||
storage: LocalDataProcessStorage,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
|
||||
"""为新任务创建独立的源文件引用,避免删除任一任务时互相影响。"""
|
||||
|
||||
target_task_id = repeat_task_id(source_task_id, request_id)
|
||||
copies: dict[str, dict[str, str]] = {}
|
||||
staged: list[StagedSourceObject] = []
|
||||
batch_id = storage.new_batch_id()
|
||||
for summary in store.list_source_files(source_task_id):
|
||||
old_file_id = str(summary["id"])
|
||||
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
||||
new_file_id = new_id("dpsf")
|
||||
old_reference = str(source.get("storage_object_id") or "")
|
||||
if old_reference.startswith("local://data-process/"):
|
||||
staged_object = storage.stage_copy(
|
||||
batch_id=batch_id,
|
||||
source_reference=old_reference,
|
||||
expected_source_task_id=source_task_id,
|
||||
expected_source_file_id=old_file_id,
|
||||
task_id=target_task_id,
|
||||
source_file_id=new_file_id,
|
||||
version=1,
|
||||
name=str(source["name"]),
|
||||
)
|
||||
staged.append(staged_object)
|
||||
new_reference = staged_object.reference
|
||||
elif old_reference.startswith("db://data-process/") or not old_reference:
|
||||
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
|
||||
else:
|
||||
raise ValueError("源任务包含不受支持的文件存储引用")
|
||||
copies[old_file_id] = {
|
||||
"id": new_file_id,
|
||||
"storage_object_id": new_reference,
|
||||
}
|
||||
return copies, staged
|
||||
|
||||
|
||||
def _remove_repeated_storage_objects(
|
||||
storage: LocalDataProcessStorage,
|
||||
task_id: str,
|
||||
staged: list[StagedSourceObject],
|
||||
copies: dict[str, dict[str, str]],
|
||||
) -> None:
|
||||
source_file_ids = {
|
||||
str(copy["storage_object_id"]): str(copy["id"])
|
||||
for copy in copies.values()
|
||||
}
|
||||
for item in staged:
|
||||
try:
|
||||
storage.delete(
|
||||
item.reference,
|
||||
expected_task_id=task_id,
|
||||
expected_source_file_id=source_file_ids[item.reference],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"failed to roll back repeated data process source object task_id=%s",
|
||||
task_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/repeat", status_code=202)
|
||||
def repeat_generation(
|
||||
task_id: str,
|
||||
payload: DataProcessRepeatRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
store: DataProcessStore = Depends(get_data_process_store),
|
||||
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
||||
) -> dict[str, Any]:
|
||||
"""按原任务快照创建独立任务,并立即在后台开始新一批生成。"""
|
||||
|
||||
with api_errors():
|
||||
repeated = store.find_repeated_task(task_id, payload.request_id)
|
||||
staged: list[StagedSourceObject] = []
|
||||
target_task_id = repeat_task_id(task_id, payload.request_id)
|
||||
if repeated is None:
|
||||
copies, staged = _repeat_file_copies(
|
||||
store,
|
||||
storage,
|
||||
task_id,
|
||||
payload.request_id,
|
||||
)
|
||||
storage.publish(staged)
|
||||
try:
|
||||
repeated = store.repeat_task(
|
||||
task_id,
|
||||
expected_updated_at=payload.expected_updated_at,
|
||||
request_id=payload.request_id,
|
||||
file_copies=copies,
|
||||
)
|
||||
except Exception:
|
||||
_remove_repeated_storage_objects(
|
||||
storage,
|
||||
target_task_id,
|
||||
staged,
|
||||
copies,
|
||||
)
|
||||
raise
|
||||
if not repeated["created"]:
|
||||
_remove_repeated_storage_objects(
|
||||
storage,
|
||||
target_task_id,
|
||||
staged,
|
||||
copies,
|
||||
)
|
||||
|
||||
repeated_task = repeated["task"]
|
||||
if repeated_task.get("status") == "pending":
|
||||
try:
|
||||
started = store.start_generation(target_task_id, replace_existing=True)
|
||||
background_tasks.add_task(
|
||||
_run_generation,
|
||||
store,
|
||||
target_task_id,
|
||||
str(started["generation_run_id"]),
|
||||
)
|
||||
except ConflictError:
|
||||
latest = store.get_task(target_task_id)
|
||||
if latest.get("status") != "running":
|
||||
raise
|
||||
repeated["task"] = store.get_task(target_task_id)
|
||||
repeated["progress"] = store.progress(target_task_id)
|
||||
return ok(repeated, "已按原配置创建新任务并开始后台生成")
|
||||
|
||||
|
||||
@router.delete("/{task_id}")
|
||||
def delete_task(
|
||||
task_id: str,
|
||||
@@ -919,7 +1107,7 @@ async def upload_source_files(
|
||||
f"{suffix} is not supported for {process_type} data processing",
|
||||
)
|
||||
parsed = parse_text_content(raw, filename=name)
|
||||
if not parsed.text:
|
||||
if not parsed.text.strip():
|
||||
raise fail(400, f"source file is empty: {name}")
|
||||
batch_size += len(raw)
|
||||
if batch_size > MAX_SOURCE_BATCH_BYTES:
|
||||
@@ -934,7 +1122,11 @@ async def upload_source_files(
|
||||
content=raw,
|
||||
)
|
||||
staged.append(staged_object)
|
||||
record_count = len(parsed.records) or (1 if parsed.text else 0)
|
||||
record_count = (
|
||||
len(parsed.records)
|
||||
if process_type == "structured"
|
||||
else (1 if parsed.text else 0)
|
||||
)
|
||||
prepared.append(
|
||||
{
|
||||
"id": source_file_id,
|
||||
@@ -1362,15 +1554,28 @@ def _prepare_preview_items(
|
||||
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
|
||||
)
|
||||
is_unstructured = task.get("process_type") == "unstructured"
|
||||
if is_unstructured and (
|
||||
needs_unstructured_raw = is_unstructured and (
|
||||
chunk_method == "layout_hybrid"
|
||||
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
):
|
||||
)
|
||||
has_structured_xlsx = not is_unstructured and any(
|
||||
str(source.get("file_format") or "").lower() == "xlsx"
|
||||
for source in sources
|
||||
)
|
||||
if needs_unstructured_raw or has_structured_xlsx:
|
||||
for index, source in enumerate(sources):
|
||||
if (
|
||||
chunk_method != "layout_hybrid"
|
||||
and str(source.get("file_format") or "").lower() != "pdf"
|
||||
):
|
||||
source_format = str(source.get("file_format") or "").lower()
|
||||
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
|
||||
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
|
||||
needs_pdf_noise = (
|
||||
is_unstructured
|
||||
and not needs_layout_raw
|
||||
and source_format == "pdf"
|
||||
and bool(
|
||||
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
||||
)
|
||||
)
|
||||
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
|
||||
continue
|
||||
storage_object_id = str(source.get("storage_object_id") or "")
|
||||
actual_size = storage.file_size(
|
||||
@@ -1379,7 +1584,7 @@ def _prepare_preview_items(
|
||||
expected_source_file_id=str(source["id"]),
|
||||
)
|
||||
if actual_size is None:
|
||||
if chunk_method == "layout_hybrid":
|
||||
if needs_layout_raw:
|
||||
raise InvalidStateError(
|
||||
"版面结构混合切分无法读取原始文件,请重新上传后再处理"
|
||||
)
|
||||
@@ -1396,12 +1601,10 @@ def _prepare_preview_items(
|
||||
)
|
||||
)
|
||||
enriched = dict(source)
|
||||
if chunk_method == "layout_hybrid":
|
||||
if needs_structured_xlsx or needs_layout_raw:
|
||||
enriched["raw_content"] = raw
|
||||
sources[index] = enriched
|
||||
continue
|
||||
if str(source.get("file_format") or "").lower() != "pdf":
|
||||
continue
|
||||
pages = extract_pdf_page_texts(raw)
|
||||
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
||||
if extracted_text != str(source.get("content") or ""):
|
||||
@@ -1413,7 +1616,7 @@ def _prepare_preview_items(
|
||||
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
||||
sources[index] = enriched
|
||||
items = _build_preview_items(task, sources)
|
||||
if not items and source_file_ids is None:
|
||||
if not items and source_file_ids is None and is_unstructured:
|
||||
raise InvalidStateError("source files did not produce preview items")
|
||||
return items
|
||||
|
||||
@@ -1435,6 +1638,7 @@ def _run_preview(
|
||||
len(source_file_ids),
|
||||
)
|
||||
try:
|
||||
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
|
||||
if not store.mark_preview_running(task_id, preview_run_id):
|
||||
logger.info(
|
||||
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
|
||||
@@ -1461,7 +1665,7 @@ def _run_preview(
|
||||
storage,
|
||||
[source_file_id],
|
||||
)
|
||||
if not items:
|
||||
if not items and is_unstructured:
|
||||
raise InvalidStateError(
|
||||
f"source file did not produce preview items: {source_file_id}"
|
||||
)
|
||||
@@ -1657,8 +1861,13 @@ def update_preview_item(
|
||||
) -> dict[str, Any]:
|
||||
with api_errors():
|
||||
task = store.get_task(task_id)
|
||||
existing = store.get_preview_item(task_id, preview_id)
|
||||
update = payload.model_dump(exclude_unset=True, mode="json")
|
||||
update["quality_score"] = _preview_quality(payload.edited_content, task.get("config") or {})
|
||||
quality = _preview_quality(payload.edited_content, task.get("config") or {})
|
||||
source_locator = (existing.get("quality_score") or {}).get("source_locator")
|
||||
if isinstance(source_locator, Mapping):
|
||||
quality["source_locator"] = deepcopy(dict(source_locator))
|
||||
update["quality_score"] = quality
|
||||
item = store.update_preview_item(
|
||||
task_id,
|
||||
preview_id,
|
||||
@@ -2089,13 +2298,11 @@ def regenerate_results_batch(
|
||||
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||
)
|
||||
# httpx.Client 支持跨线程复用,批次内共享连接池可减少重复建连开销。
|
||||
with (
|
||||
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
|
||||
ThreadPoolExecutor(
|
||||
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||
thread_name_prefix="data-result-regeneration",
|
||||
) as executor,
|
||||
):
|
||||
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
|
||||
ThreadPoolExecutor(
|
||||
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||
thread_name_prefix="data-result-regeneration",
|
||||
) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
_regenerate_result_in_place,
|
||||
|
||||
@@ -10,5 +10,9 @@ logger = get_logger(__name__)
|
||||
@router.get("/health")
|
||||
async def health_check() -> dict[str, object]:
|
||||
logger.info("health check requested")
|
||||
return {"code": 0, "message": "ok", "data": get_platform_store().health_metrics()}
|
||||
return {
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": get_platform_store().health_metrics(),
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,8 +3,20 @@
|
||||
from app.api.v1.endpoints.data_process import router as data_process_router
|
||||
from app.api.v1.endpoints.platform import router as platform_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
from app.modules.tenant.router import router as tenant_router
|
||||
from app.modules.project.router import router as project_router
|
||||
from app.modules.approval.router import router as approval_router
|
||||
from app.modules.system.router import router as system_router
|
||||
from app.modules.retention.router import router as retention_router
|
||||
from app.modules.resource.router import router as resource_router
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health_router, tags=["health"])
|
||||
api_router.include_router(data_process_router, tags=["data-process"])
|
||||
api_router.include_router(platform_router, tags=["platform"])
|
||||
api_router.include_router(system_router, tags=["system"])
|
||||
api_router.include_router(tenant_router, tags=["tenant"])
|
||||
api_router.include_router(project_router, tags=["project"])
|
||||
api_router.include_router(approval_router, tags=["approval"])
|
||||
api_router.include_router(retention_router, tags=["retention"])
|
||||
api_router.include_router(resource_router, tags=["resource"])
|
||||
|
||||
138
backend/app/core/auth.py
Normal file
138
backend/app/core/auth.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""鉴权依赖:从 Authorization header 解析当前用户,提供权限校验。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Depends, HTTPException, Query, Request, status
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
# 无需鉴权的路径前缀(健康检查、登录等)
|
||||
PUBLIC_PATHS = ("/health", "/login", "/system-info")
|
||||
|
||||
|
||||
def _extract_token(request: Request) -> str | None:
|
||||
"""从 Authorization header 提取 token(格式: Bearer platform-token-{user_id})。"""
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
if token.startswith("platform-token-"):
|
||||
return token[len("platform-token-"):]
|
||||
return None
|
||||
|
||||
|
||||
def get_current_user(request: Request) -> dict[str, Any]:
|
||||
"""
|
||||
FastAPI 依赖:解析当前登录用户。
|
||||
- 公开路径(/health, /login 等)直接放行,返回匿名用户。
|
||||
- 无 token 或 token 无效时抛 401。
|
||||
- admin 用户标记为超级管理员,拥有全部权限。
|
||||
"""
|
||||
path = request.url.path
|
||||
# 去掉路由前缀后判断
|
||||
for prefix in PUBLIC_PATHS:
|
||||
if path.endswith(prefix):
|
||||
return {"id": None, "username": "anonymous", "role": "viewer", "permissions": [], "protected": False}
|
||||
|
||||
user_id = _extract_token(request)
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing or invalid token")
|
||||
|
||||
store = get_platform_store()
|
||||
for u in store.users():
|
||||
if u.get("id") == user_id:
|
||||
return u
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="user not found")
|
||||
|
||||
|
||||
def require_admin(current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
|
||||
"""FastAPI 依赖:要求当前用户是管理员(role=admin 或 protected)。"""
|
||||
if current_user.get("role") == "admin" or current_user.get("protected"):
|
||||
return current_user
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin permission required")
|
||||
|
||||
|
||||
def is_admin(user: dict[str, Any]) -> bool:
|
||||
"""判断用户是否为管理员(admin 角色或 protected 标记)。"""
|
||||
return user.get("role") == "admin" or user.get("protected", False)
|
||||
|
||||
|
||||
def has_resource_access(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
user: dict[str, Any],
|
||||
permission: str = "read",
|
||||
) -> bool:
|
||||
"""
|
||||
检查用户对某资源是否有指定权限。
|
||||
- admin/protected 用户直接放行(旁路)。
|
||||
- 其他用户检查 acls 表中是否有对应授权。
|
||||
"""
|
||||
if user.get("role") == "admin" or user.get("protected"):
|
||||
return True
|
||||
|
||||
store = get_platform_store()
|
||||
acls = store.get_acl(resource_type, resource_id)
|
||||
user_id = user.get("id")
|
||||
user_role = user.get("role")
|
||||
|
||||
for entry in acls:
|
||||
# 按 user 授权
|
||||
if entry.get("principal_type") == "user" and entry.get("principal_id") == user_id:
|
||||
if _permission_covers(entry.get("permission"), permission):
|
||||
return True
|
||||
# 按 role 授权
|
||||
if entry.get("principal_type") == "role" and entry.get("principal_id") == user_role:
|
||||
if _permission_covers(entry.get("permission"), permission):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _permission_covers(granted: str | None, required: str) -> bool:
|
||||
"""权限覆盖判断:write/execute 覆盖 read;admin 覆盖一切。"""
|
||||
if not granted:
|
||||
return False
|
||||
if granted == "admin":
|
||||
return True
|
||||
if granted == required:
|
||||
return True
|
||||
# write 覆盖 read
|
||||
if required == "read" and granted in ("write", "execute"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def filter_accessible_resource_ids(
|
||||
resource_type: str,
|
||||
all_ids: list[str],
|
||||
user: dict[str, Any],
|
||||
) -> list[str]:
|
||||
"""
|
||||
从全部资源 ID 中过滤出当前用户可访问的 ID 列表。
|
||||
- admin 直接返回全部。
|
||||
- 普通用户查 acls 表取交集。
|
||||
"""
|
||||
if user.get("role") == "admin" or user.get("protected"):
|
||||
return all_ids
|
||||
|
||||
if not all_ids:
|
||||
return []
|
||||
|
||||
store = get_platform_store()
|
||||
user_id = user.get("id")
|
||||
user_role = user.get("role")
|
||||
|
||||
# 查询该用户在该资源类型下有 read 权限的所有 resource_id
|
||||
with store.connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT resource_id FROM acls
|
||||
WHERE resource_type=? AND (
|
||||
(principal_type='user' AND principal_id=?)
|
||||
OR (principal_type='role' AND principal_id=?)
|
||||
)
|
||||
""",
|
||||
(resource_type, user_id, user_role),
|
||||
).fetchall()
|
||||
|
||||
accessible = {r["resource_id"] for r in rows}
|
||||
return [rid for rid in all_ids if rid in accessible]
|
||||
@@ -2,6 +2,18 @@
|
||||
from functools import lru_cache
|
||||
import os
|
||||
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 显式指定 backend 目录下的 .env,并强制覆盖已有环境变量,
|
||||
# 确保远程数据库配置生效,不被本地默认值或残留环境变量影响。
|
||||
_env_path = _Path(__file__).resolve().parent.parent.parent / ".env"
|
||||
load_dotenv(dotenv_path=_env_path, override=True)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = os.getenv(name)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,10 @@ CREATE TABLE IF NOT EXISTS trained_models (
|
||||
create_time TEXT NOT NULL,
|
||||
merged INTEGER NOT NULL DEFAULT 0,
|
||||
merging INTEGER NOT NULL DEFAULT 0,
|
||||
merged_path TEXT
|
||||
merged_path TEXT,
|
||||
artifact_dir TEXT,
|
||||
compute_node_id TEXT,
|
||||
compute_node_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_lineage (
|
||||
@@ -282,3 +285,51 @@ CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(targe
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||
|
||||
-- ===================== Project / Tenant =====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
name TEXT NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
description TEXT,
|
||||
quota TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
create_time TEXT NOT NULL,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS project_members (
|
||||
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL DEFAULT 'member',
|
||||
create_time TEXT NOT NULL,
|
||||
PRIMARY KEY (project_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
permissions TEXT NOT NULL DEFAULT '[]',
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
issued_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
ip TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS acls (
|
||||
id TEXT PRIMARY KEY,
|
||||
resource_type TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
principal_type TEXT NOT NULL,
|
||||
principal_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
68
backend/app/db/sql/002_governance.sql
Normal file
68
backend/app/db/sql/002_governance.sql
Normal file
@@ -0,0 +1,68 @@
|
||||
-- 平台治理:租户 / 审批 / 审计(字段以 platform_store 实际写入为准)
|
||||
CREATE TABLE IF NOT EXISTS tenants (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
code TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
owner_user_id TEXT,
|
||||
quota TEXT,
|
||||
retention_policy_id TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_templates (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
steps TEXT,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
template_id TEXT,
|
||||
resource_type TEXT,
|
||||
resource_id TEXT,
|
||||
applicant_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
current_step INTEGER DEFAULT 0,
|
||||
create_time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS approval_steps (
|
||||
id TEXT PRIMARY KEY,
|
||||
instance_id TEXT,
|
||||
step_index INTEGER,
|
||||
approver_id TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
comment TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
actor_id TEXT,
|
||||
action TEXT,
|
||||
target_type TEXT,
|
||||
target_id TEXT,
|
||||
detail TEXT,
|
||||
client_ip TEXT,
|
||||
time TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_logs(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_project ON audit_logs(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_time ON audit_logs(time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_policies (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
scope TEXT,
|
||||
rule TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
create_time TEXT,
|
||||
create_by TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
18
backend/app/db/sql/003_model_path_governance.sql
Normal file
18
backend/app/db/sql/003_model_path_governance.sql
Normal file
@@ -0,0 +1,18 @@
|
||||
-- 003_model_path_governance
|
||||
-- 模型路径治理:增加 can_train 标识,区分本地可训练模型与 API / 远程模型。
|
||||
-- 训练预检阶段依赖该字段拦截不适合 LLaMA-Factory 本地训练的基座模型。
|
||||
|
||||
-- 1. models 表增加 can_train(默认 0,后设搬迁为 1 的规则如下)
|
||||
ALTER TABLE models ADD COLUMN IF NOT EXISTS can_train INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
-- 2. 将已有模型按规则推定 can_train:
|
||||
-- - path 非空 且 model_source != 'api' → 可训练 (1)
|
||||
-- - 其余 → 不可训练 (0)
|
||||
UPDATE models
|
||||
SET can_train = CASE
|
||||
WHEN path IS NOT NULL AND path != '' AND model_source IS NOT NULL AND model_source != 'api' THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
|
||||
-- 3. 给 trained_models 增加 artifact_dir(训练产物目录扫描结果目录)
|
||||
ALTER TABLE trained_models ADD COLUMN IF NOT EXISTS artifact_dir TEXT;
|
||||
3
backend/app/db/sql/003_tenant_quota.sql
Normal file
3
backend/app/db/sql/003_tenant_quota.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
-- 租户配额与保留策略扩展(如后续治理表需补列,可在此追加)
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS gpu_quota TEXT;
|
||||
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS storage_quota TEXT;
|
||||
91
backend/app/modules/approval/router.py
Normal file
91
backend/app/modules/approval/router.py
Normal file
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/approvals", tags=["approval"])
|
||||
|
||||
|
||||
@router.get("/templates")
|
||||
def list_templates() -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_templates())
|
||||
|
||||
|
||||
@router.post("/templates")
|
||||
def create_template(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
if not payload.get("name"):
|
||||
raise fail(400, "name 必填")
|
||||
return ok(get_platform_store().create_approval_template(payload))
|
||||
|
||||
|
||||
@router.get("/templates/{template_id}")
|
||||
def get_template(template_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().approval_template(template_id))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.put("/templates/{template_id}")
|
||||
def update_template(template_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_approval_template(template_id, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.delete("/templates/{template_id}")
|
||||
def delete_template(template_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().delete_approval_template(template_id))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_instances(status: str | None = None) -> dict[str, Any]:
|
||||
return ok(get_platform_store().approval_instances(status=status))
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_instance(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
for field in ("resource_type", "resource_id", "applicant_id"):
|
||||
if not payload.get(field):
|
||||
raise fail(400, f"{field} 必填")
|
||||
try:
|
||||
return ok(get_platform_store().create_approval_instance(payload))
|
||||
except KeyError:
|
||||
raise fail(404, "template not found")
|
||||
|
||||
|
||||
@router.get("/{instance_id}")
|
||||
def get_instance(instance_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().approval_instance(instance_id))
|
||||
except KeyError:
|
||||
raise fail(404, "instance not found")
|
||||
|
||||
|
||||
@router.post("/{instance_id}/steps/{step_index}/decision")
|
||||
def decide(
|
||||
instance_id: str,
|
||||
step_index: int,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
) -> dict[str, Any]:
|
||||
if not payload.get("approver_id"):
|
||||
raise fail(400, "approver_id 必填")
|
||||
try:
|
||||
return ok(
|
||||
get_platform_store().decide_approval_step(
|
||||
instance_id,
|
||||
step_index,
|
||||
approver_id=payload["approver_id"],
|
||||
approved=bool(payload.get("approved", False)),
|
||||
comment=payload.get("comment"),
|
||||
)
|
||||
)
|
||||
except (KeyError, ValueError) as e:
|
||||
raise fail(400, str(e))
|
||||
@@ -33,6 +33,16 @@ def _unwrap_dict(payload: Any) -> dict[str, Any]:
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
# Inference calls are intentionally short-timeout:
|
||||
# - load dispatch only confirms the compute node accepted the request
|
||||
# (the actual model load now runs asynchronously on the node).
|
||||
# - status/unload must never block the platform for long when a node is
|
||||
# unreachable but still marked online.
|
||||
INFERENCE_LOAD_TIMEOUT = httpx.Timeout(30, connect=10)
|
||||
INFERENCE_STATUS_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||
INFERENCE_UNLOAD_TIMEOUT = httpx.Timeout(30, connect=5)
|
||||
|
||||
|
||||
class ComputeNodeClient:
|
||||
"""Application-side client for one compute node.
|
||||
|
||||
@@ -182,6 +192,36 @@ class ComputeNodeClient:
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
json_data: dict[str, Any] | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generic request method for compute API endpoints."""
|
||||
url = _join_url(self.api_base_url, f"{self.route_prefix}{path}")
|
||||
async with httpx.AsyncClient(timeout=timeout or 300, headers=self.headers()) as client:
|
||||
if method.upper() == "GET":
|
||||
response = await client.get(url)
|
||||
else:
|
||||
response = await client.post(url, json=json_data)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
# ── Inference helpers (short timeouts — see module constants) ──────────
|
||||
|
||||
async def inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Dispatch a model load. Returns as soon as the node accepts the
|
||||
request; the node now loads asynchronously (status goes 'loading')."""
|
||||
return await self._request("POST", "/inference/load", json_data=payload, timeout=INFERENCE_LOAD_TIMEOUT)
|
||||
|
||||
async def inference_status(self) -> dict[str, Any]:
|
||||
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
|
||||
|
||||
async def inference_unload(self) -> dict[str, Any]:
|
||||
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
filename: str,
|
||||
@@ -196,7 +236,8 @@ class ComputeNodeClient:
|
||||
"resource_id": resource_id or "",
|
||||
}
|
||||
files = {"file": (filename, content)}
|
||||
async with httpx.AsyncClient(timeout=max(self.timeout, 60), headers=self.headers()) as client:
|
||||
timeout = httpx.Timeout(max(self.timeout, 60), connect=self.timeout)
|
||||
async with httpx.AsyncClient(timeout=timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/upload"),
|
||||
data=data,
|
||||
|
||||
@@ -1,15 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
||||
MAX_STARTING_ATTEMPTS = 40
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
def _parse_inference_load_status(task: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
load_status = task.get("load_status") or {}
|
||||
if isinstance(load_status, str):
|
||||
try:
|
||||
load_status = json.loads(load_status)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
load_status = {}
|
||||
return load_status.get("loaded_models") or [], load_status
|
||||
|
||||
|
||||
async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
||||
"""推进处于 starting 状态的推理加载。
|
||||
|
||||
模型加载已改为异步派发:/model-compare/{id}/load 立即返回,这里在每次
|
||||
轮询时查询对应计算节点的 /inference/status,把任务从 starting 推进到
|
||||
ready/error。使用短超时,单节点不可达不会阻塞整轮轮询。
|
||||
"""
|
||||
reconciled: list[dict[str, Any]] = []
|
||||
now = time.time()
|
||||
for task in store.compare_tasks():
|
||||
items, _ = _parse_inference_load_status(task)
|
||||
if not any(item.get("status") == "starting" for item in items):
|
||||
continue
|
||||
# dirty 只要处理过任一 starting 项就置位:load_attempts / last_polled_at
|
||||
# 必须落库,否则节点不可达时计数不会累积,封顶逻辑永远触发不了
|
||||
dirty = False
|
||||
for item in items:
|
||||
if item.get("status") != "starting":
|
||||
continue
|
||||
# 节流:同一 item 每 3s 只查询一次
|
||||
if now - float(item.get("last_polled_at") or 0) < 3:
|
||||
continue
|
||||
item["last_polled_at"] = now
|
||||
item["load_attempts"] = int(item.get("load_attempts") or 0) + 1
|
||||
dirty = True
|
||||
node = next((n for n in store.compute_nodes() if n["id"] == item.get("node_id")), None)
|
||||
if not node:
|
||||
item["status"] = "error"
|
||||
item["error"] = "compute node deleted"
|
||||
store.mark_inference_unloaded(item.get("node_id") or "")
|
||||
continue
|
||||
if not node.get("enabled") or node.get("scheduler_status") != "online":
|
||||
item["status"] = "error"
|
||||
item["error"] = "compute node offline"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
continue
|
||||
try:
|
||||
status = await ComputeNodeClient(node["api_base_url"]).inference_status()
|
||||
except Exception as exc: # noqa: BLE001 - node unreachable; keep retrying until cap
|
||||
if int(item.get("load_attempts") or 0) >= MAX_STARTING_ATTEMPTS:
|
||||
item["status"] = "error"
|
||||
item["error"] = f"compute node unreachable: {exc}"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
continue
|
||||
node_status = status.get("status")
|
||||
if node_status == "ready":
|
||||
item["status"] = "ready"
|
||||
item.pop("error", None)
|
||||
store.mark_inference_loaded(node["id"])
|
||||
elif node_status == "error":
|
||||
item["status"] = "error"
|
||||
item["error"] = status.get("error") or "model load failed on compute node"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
elif node_status == "idle":
|
||||
# 节点重启导致已加载模型丢失
|
||||
item["status"] = "error"
|
||||
item["error"] = "model disappeared from compute node (node may have restarted)"
|
||||
store.mark_inference_unloaded(node["id"])
|
||||
# node_status == "loading" -> 保持 starting,下轮再查
|
||||
if dirty:
|
||||
if any(i.get("status") in {"ready", "running"} for i in items):
|
||||
new_status = "loaded"
|
||||
elif any(i.get("status") == "starting" for i in items):
|
||||
new_status = "starting" # 仍在加载中,保持 starting
|
||||
else:
|
||||
new_status = "failed"
|
||||
store.update_compare_task(task["id"], {"status": new_status, "load_status": {"loaded_models": items}})
|
||||
reconciled.append({"task_id": task["id"], "status": new_status})
|
||||
return reconciled
|
||||
|
||||
|
||||
async def fetch_eval_result_content(client: ComputeNodeClient, node: dict[str, Any], job: dict[str, Any]) -> dict[str, Any] | None:
|
||||
output_dir = job.get("output_dir")
|
||||
if not output_dir:
|
||||
return None
|
||||
full_path = f"{str(output_dir).rstrip('/')}/eval_results.json"
|
||||
data_root = "/data/yg-ft/"
|
||||
if full_path.startswith(data_root):
|
||||
full_path = full_path[len(data_root):]
|
||||
rel_path = full_path.lstrip("/")
|
||||
import httpx
|
||||
url = f"{node['api_base_url'].rstrip('/')}/modelTF/compute/files/read"
|
||||
async with httpx.AsyncClient(timeout=30, headers=client.headers()) as http:
|
||||
response = await http.get(url, params={"path": rel_path})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
@@ -27,6 +131,13 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store.record_training_log_metrics(task["id"], str(logs.get("content") or ""))
|
||||
except Exception:
|
||||
pass
|
||||
# P0-4: Force-fetch last log snippet when job reaches terminal state
|
||||
if job.get("status") in {"failed", "stopped"}:
|
||||
try:
|
||||
last_logs = await client.job_logs(task["compute_job_id"], tail_lines=200)
|
||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||
except Exception:
|
||||
pass
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
@@ -41,4 +152,40 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||
return {"synced": len(synced) + len(standalone_synced), "failed": failed, "items": synced, "standalone": standalone_synced}
|
||||
|
||||
# ── Eval job sync ────────────────────────────────────────────────
|
||||
eval_synced = 0
|
||||
for eval_task in store.running_eval_tasks():
|
||||
node = next(
|
||||
(item for item in store.compute_nodes() if item["id"] == eval_task.get("compute_node_id")),
|
||||
None,
|
||||
)
|
||||
if not node:
|
||||
failed.append({"eval_task_id": eval_task["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
job = await client.get_job(eval_task["compute_job_id"])
|
||||
result_content = None
|
||||
# Try to read eval_results.json from the job output directory
|
||||
if job.get("status") == "completed" and job.get("output_dir"):
|
||||
try:
|
||||
result_content = await fetch_eval_result_content(client, node, job)
|
||||
except Exception:
|
||||
pass
|
||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
||||
eval_synced += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
failed.append({"eval_task_id": eval_task["id"], "error": str(exc)})
|
||||
|
||||
# ── Inference load reconciliation ─────────────────────────────────────
|
||||
try:
|
||||
inference_reconciled = await reconcile_inference_loads(store)
|
||||
except Exception as exc: # noqa: BLE001 - keep polling alive
|
||||
failed.append({"inference_reconcile": str(exc)})
|
||||
inference_reconciled = []
|
||||
|
||||
return {"synced": len(synced) + len(standalone_synced) + eval_synced, "failed": failed,
|
||||
"items": synced, "standalone": standalone_synced, "eval_synced": eval_synced,
|
||||
"inference_reconciled": inference_reconciled}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
148
backend/app/modules/data_process/dataset_format.py
Normal file
148
backend/app/modules/data_process/dataset_format.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""Dataset format validation for Alpaca, ShareGPT, DPO, CPT formats.
|
||||
|
||||
Used by the training preflight flow to validate that uploaded dataset files
|
||||
conform to the declared format before submitting to the compute node.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_sample(path: str | None, content: str | None = None, max_samples: int = 20) -> list[dict[str, Any]]:
|
||||
"""Load up to max_samples records from JSONL file path or raw content string."""
|
||||
try:
|
||||
if content is not None:
|
||||
text = content.strip()
|
||||
elif path:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
text = fh.read().strip()
|
||||
else:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if not text:
|
||||
return []
|
||||
|
||||
lines = text.splitlines()[:max_samples]
|
||||
records: list[dict[str, Any]] = []
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(record, dict):
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
def _check_alpaca(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate Alpaca format: requires 'instruction' field."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("Alpaca 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_instruction = sum(1 for r in records if not r.get("instruction"))
|
||||
if missing_instruction:
|
||||
errors.append(
|
||||
f"Alpaca 格式要求每条记录包含 instruction 字段,"
|
||||
f"前{len(records)}条中有{missing_instruction}条缺失"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _check_sharegpt(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate ShareGPT format: requires 'messages' (list of dicts with role/content)."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("ShareGPT 格式数据集无有效记录")
|
||||
return errors
|
||||
bad = 0
|
||||
for r in records:
|
||||
messages = r.get("messages")
|
||||
if not isinstance(messages, list) or not messages:
|
||||
bad += 1
|
||||
continue
|
||||
for msg in messages:
|
||||
if not isinstance(msg, dict) or "role" not in msg or "content" not in msg:
|
||||
bad += 1
|
||||
break
|
||||
if bad:
|
||||
errors.append(
|
||||
f"ShareGPT 格式要求每条记录包含 messages 列表,"
|
||||
f"每条消息需有 role 和 content 字段,前{len(records)}条中有{bad}条不符合"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def _check_dpo(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate DPO format: requires 'chosen' and 'rejected' fields."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("DPO 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_chosen = sum(1 for r in records if not r.get("chosen"))
|
||||
missing_rejected = sum(1 for r in records if not r.get("rejected"))
|
||||
if missing_chosen:
|
||||
errors.append(f"DPO 格式要求 chosen 字段,前{len(records)}条中有{missing_chosen}条缺失")
|
||||
if missing_rejected:
|
||||
errors.append(f"DPO 格式要求 rejected 字段,前{len(records)}条中有{missing_rejected}条缺失")
|
||||
return errors
|
||||
|
||||
|
||||
def _check_cpt(records: list[dict[str, Any]]) -> list[str]:
|
||||
"""Validate CPT format: requires 'text' field, should NOT have instruction/output."""
|
||||
errors: list[str] = []
|
||||
if not records:
|
||||
errors.append("CPT 格式数据集无有效记录")
|
||||
return errors
|
||||
missing_text = sum(1 for r in records if not r.get("text"))
|
||||
has_instruction = sum(1 for r in records if r.get("instruction") or r.get("output"))
|
||||
if missing_text:
|
||||
errors.append(f"CPT 格式要求 text 字段,前{len(records)}条中有{missing_text}条缺失")
|
||||
if has_instruction:
|
||||
errors.append(
|
||||
f"CPT 格式不应包含 instruction/output 字段(疑似 Alpaca 格式),"
|
||||
f"前{len(records)}条中有{has_instruction}条包含此类字段"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
FORMAT_VALIDATORS = {
|
||||
"alpaca": _check_alpaca,
|
||||
"alpaca_jsonl": _check_alpaca,
|
||||
"sharegpt": _check_sharegpt,
|
||||
"dpo": _check_dpo,
|
||||
"cpt": _check_cpt,
|
||||
"pt": _check_cpt,
|
||||
}
|
||||
|
||||
|
||||
def validate_dataset_format(
|
||||
dataset_format: str,
|
||||
content: str | None = None,
|
||||
path: str | None = None,
|
||||
max_samples: int = 20,
|
||||
) -> list[str]:
|
||||
"""Validate dataset content against expected format.
|
||||
|
||||
Args:
|
||||
dataset_format: One of 'alpaca', 'sharegpt', 'dpo', 'cpt'.
|
||||
content: Raw file content (JSONL text). Mutually exclusive with path.
|
||||
path: File path to read content from.
|
||||
max_samples: Maximum records to sample for validation.
|
||||
|
||||
Returns:
|
||||
List of error messages (empty if valid).
|
||||
"""
|
||||
fmt = str(dataset_format).lower().strip()
|
||||
validator = FORMAT_VALIDATORS.get(fmt)
|
||||
if not validator:
|
||||
return [f"不支持的数据集格式: {dataset_format},支持的格式: {', '.join(sorted(FORMAT_VALIDATORS))}"]
|
||||
records = _load_sample(path=path, content=content, max_samples=max_samples)
|
||||
return validator(records)
|
||||
@@ -137,6 +137,67 @@ class LocalDataProcessStorage:
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def stage_copy(
|
||||
self,
|
||||
*,
|
||||
batch_id: str,
|
||||
source_reference: str,
|
||||
expected_source_task_id: str,
|
||||
expected_source_file_id: str,
|
||||
task_id: str,
|
||||
source_file_id: str,
|
||||
version: int,
|
||||
name: str,
|
||||
) -> StagedSourceObject:
|
||||
"""为不可变源对象创建独立目录项,不把大文件重新读入内存。"""
|
||||
|
||||
batch_id = _safe_component(batch_id, "batch id")
|
||||
task_id = _safe_component(task_id, "task id")
|
||||
source_file_id = _safe_component(source_file_id, "source file id")
|
||||
if isinstance(version, bool) or not isinstance(version, int) or version < 1:
|
||||
raise DataProcessStorageError("invalid source file version")
|
||||
basename = _safe_basename(name)
|
||||
source_relative = self._relative_from_reference(source_reference)
|
||||
if source_relative is None:
|
||||
raise DataProcessStorageError("original source object is not available")
|
||||
self._assert_expected_owner(
|
||||
source_relative,
|
||||
expected_task_id=expected_source_task_id,
|
||||
expected_source_file_id=expected_source_file_id,
|
||||
)
|
||||
descriptor, source_info = self._open_read_descriptor(source_relative)
|
||||
os.close(descriptor)
|
||||
|
||||
batch_directory = self._ensure_directory(self._root / ".staging" / batch_id)
|
||||
temporary_path = batch_directory / f"{source_file_id}-{uuid.uuid4().hex}.tmp"
|
||||
source_path = self._path_for_relative(source_relative)
|
||||
try:
|
||||
os.link(source_path, temporary_path, follow_symlinks=False)
|
||||
copy_info = temporary_path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(copy_info.st_mode)
|
||||
or source_info.st_dev != copy_info.st_dev
|
||||
or source_info.st_ino != copy_info.st_ino
|
||||
):
|
||||
raise DataProcessStorageError("source storage object changed while copying")
|
||||
except Exception:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
relative_path = PurePosixPath(
|
||||
task_id,
|
||||
source_file_id,
|
||||
f"v{version}",
|
||||
basename,
|
||||
)
|
||||
reference = (
|
||||
"local://data-process/"
|
||||
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
||||
)
|
||||
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
||||
self._issued_staged_objects[temporary_path] = staged
|
||||
return staged
|
||||
|
||||
def publish(self, objects: Iterable[StagedSourceObject]) -> None:
|
||||
staged = list(objects)
|
||||
published: list[StagedSourceObject] = []
|
||||
|
||||
@@ -45,6 +45,13 @@ _UNSTRUCTURED_PREVIEW_DEFAULTS: dict[str, Any] = {
|
||||
"preserve_lists": True,
|
||||
}
|
||||
_REGENERATION_MARKER_KEY = "_regeneration_prepared"
|
||||
_REPEAT_SOURCE_TASK_KEY = "_repeat_source_task_id"
|
||||
_REPEAT_REQUEST_KEY = "_repeat_request_id"
|
||||
_INTERNAL_CONFIG_KEYS = {
|
||||
_REGENERATION_MARKER_KEY,
|
||||
_REPEAT_SOURCE_TASK_KEY,
|
||||
_REPEAT_REQUEST_KEY,
|
||||
}
|
||||
|
||||
|
||||
class DataProcessStoreError(RuntimeError):
|
||||
@@ -71,6 +78,13 @@ def new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:20]}"
|
||||
|
||||
|
||||
def repeat_task_id(source_task_id: str, request_id: str) -> str:
|
||||
"""按源任务和请求幂等键生成稳定的新任务 ID。"""
|
||||
|
||||
digest = hashlib.sha256(f"{source_task_id}:{request_id}".encode()).hexdigest()
|
||||
return f"dpt_{digest[:20]}"
|
||||
|
||||
|
||||
def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
@@ -183,17 +197,25 @@ def _is_regeneration_prepared(task: dict[str, Any]) -> bool:
|
||||
return _regeneration_marker(task) is not None
|
||||
|
||||
|
||||
def _business_config(config: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""过滤只供服务端维护的工作流标记。"""
|
||||
|
||||
return {
|
||||
key: value
|
||||
for key, value in (config or {}).items()
|
||||
if key not in _INTERNAL_CONFIG_KEYS
|
||||
}
|
||||
|
||||
|
||||
def _public_task(item: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
"""从 API 任务快照中移除服务端内部重新生成标记。"""
|
||||
"""从 API 任务快照中移除服务端内部工作流标记。"""
|
||||
|
||||
if item is None:
|
||||
return None
|
||||
public = dict(item)
|
||||
config = public.get("config")
|
||||
if isinstance(config, dict) and _REGENERATION_MARKER_KEY in config:
|
||||
public["config"] = {
|
||||
key: value for key, value in config.items() if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
if isinstance(config, dict):
|
||||
public["config"] = _business_config(config)
|
||||
return public
|
||||
|
||||
|
||||
@@ -353,13 +375,7 @@ class DataProcessStore:
|
||||
payload.get("description") or "",
|
||||
payload["process_type"],
|
||||
payload.get("source_dataset_id"),
|
||||
json_dumps(
|
||||
{
|
||||
key: value
|
||||
for key, value in (payload.get("config") or {}).items()
|
||||
if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
),
|
||||
json_dumps(_business_config(payload.get("config"))),
|
||||
payload.get("tenant_id"),
|
||||
payload.get("project_id"),
|
||||
payload.get("owner_id"),
|
||||
@@ -373,6 +389,268 @@ class DataProcessStore:
|
||||
raise ConflictError("data process task name already exists") from exc
|
||||
return _public_task(_decode_row(row)) or {}
|
||||
|
||||
@staticmethod
|
||||
def _repeat_response(
|
||||
conn: psycopg.Connection[dict[str, Any]],
|
||||
row: dict[str, Any],
|
||||
*,
|
||||
source_task_id: str,
|
||||
created: bool,
|
||||
) -> dict[str, Any]:
|
||||
task_id = str(row["id"])
|
||||
counts = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL) AS source_file_count,
|
||||
(SELECT COUNT(*) FROM data_process_preview_items
|
||||
WHERE task_id=%s) AS preview_count
|
||||
""",
|
||||
(task_id, task_id),
|
||||
).fetchone() or {}
|
||||
task = _public_task(_decode_row(row)) or {}
|
||||
task["source_file_count"] = int(counts.get("source_file_count") or 0)
|
||||
task["preview_count"] = int(counts.get("preview_count") or 0)
|
||||
return {
|
||||
"task": task,
|
||||
"source_task_id": source_task_id,
|
||||
"created": created,
|
||||
"copied_source_file_count": task["source_file_count"],
|
||||
"copied_preview_count": task["preview_count"],
|
||||
}
|
||||
|
||||
def find_repeated_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""查找同一幂等请求已创建的新任务。"""
|
||||
|
||||
task_id = repeat_task_id(source_task_id, request_id)
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM data_process_tasks WHERE id=%s",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
decoded = _decode_row(row) or {}
|
||||
config = decoded.get("config") or {}
|
||||
if (
|
||||
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
|
||||
or config.get(_REPEAT_REQUEST_KEY) != request_id
|
||||
):
|
||||
raise ConflictError("再次生成请求与现有任务冲突")
|
||||
if decoded.get("deleted_at"):
|
||||
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
row,
|
||||
source_task_id=source_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
def repeat_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
request_id: str,
|
||||
file_copies: dict[str, dict[str, str]],
|
||||
) -> dict[str, Any]:
|
||||
"""复制已确认任务的配置、源文件和预览,结果与发布数据保持独立。"""
|
||||
|
||||
task_id = repeat_task_id(source_task_id, request_id)
|
||||
now = utcnow()
|
||||
try:
|
||||
with self.connect() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM data_process_tasks WHERE id=%s FOR UPDATE",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
decoded = _decode_row(existing) or {}
|
||||
config = decoded.get("config") or {}
|
||||
if (
|
||||
config.get(_REPEAT_SOURCE_TASK_KEY) != source_task_id
|
||||
or config.get(_REPEAT_REQUEST_KEY) != request_id
|
||||
):
|
||||
raise ConflictError("再次生成请求与现有任务冲突")
|
||||
if decoded.get("deleted_at"):
|
||||
raise ConflictError("此次再次生成创建的任务已被删除,请重新发起")
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
existing,
|
||||
source_task_id=source_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
source_task = self._task_in_connection(
|
||||
conn,
|
||||
source_task_id,
|
||||
for_update=True,
|
||||
)
|
||||
if (
|
||||
source_task.get("status") != "completed"
|
||||
or source_task.get("results_confirmed") is False
|
||||
):
|
||||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||||
if source_task.get("preview_status") in ACTIVE_PREVIEW_STATUSES:
|
||||
raise ConflictError("源任务仍在处理切分,暂时不能再次生成")
|
||||
if expected_updated_at != _serialize_value(source_task.get("updated_at")):
|
||||
raise ConflictError("源任务已被其他操作修改,请刷新后重试")
|
||||
|
||||
source_files = conn.execute(
|
||||
"""
|
||||
SELECT * FROM data_process_source_files
|
||||
WHERE task_id=%s AND deleted_at IS NULL
|
||||
ORDER BY created_at, id
|
||||
""",
|
||||
(source_task_id,),
|
||||
).fetchall()
|
||||
source_file_ids = {str(row["id"]) for row in source_files}
|
||||
if source_file_ids != set(file_copies):
|
||||
raise ConflictError("源文件快照已变化,请刷新后重试")
|
||||
previews = conn.execute(
|
||||
"""
|
||||
SELECT * FROM data_process_preview_items
|
||||
WHERE task_id=%s
|
||||
ORDER BY source_file_id NULLS LAST, source_start NULLS LAST,
|
||||
created_at, id
|
||||
""",
|
||||
(source_task_id,),
|
||||
).fetchall()
|
||||
if not previews:
|
||||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||||
|
||||
suffix = f"(再次生成-{task_id[-6:]})"
|
||||
base_name = str(source_task.get("name") or "数据处理任务")
|
||||
repeated_name = f"{base_name[: max(1, 150 - len(suffix))]}{suffix}"
|
||||
repeated_config = _business_config(source_task.get("config") or {})
|
||||
repeated_config[_REPEAT_SOURCE_TASK_KEY] = source_task_id
|
||||
repeated_config[_REPEAT_REQUEST_KEY] = request_id
|
||||
input_count = sum(int(row.get("record_count") or 0) for row in source_files)
|
||||
task_row = conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_tasks
|
||||
(id, name, description, status, process_type, source_dataset_id,
|
||||
output_dataset_id, config, progress, input_count, output_count,
|
||||
filtered_count, duplicate_count, error_count, failure_reason,
|
||||
generation_run_id, results_confirmed, workflow_step,
|
||||
preview_status, preview_progress, preview_run_id,
|
||||
preview_failure_reason, preview_total_files,
|
||||
preview_completed_files, tenant_id, project_id, owner_id,
|
||||
approval_status, created_by, updated_by, created_at, updated_at)
|
||||
VALUES
|
||||
(%s, %s, %s, 'pending', %s, %s, NULL, %s, 20, %s, 0,
|
||||
0, 0, 0, NULL, NULL, FALSE, 'preview', 'completed', 100,
|
||||
NULL, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
task_id,
|
||||
repeated_name,
|
||||
source_task.get("description") or "",
|
||||
source_task["process_type"],
|
||||
source_task.get("source_dataset_id"),
|
||||
json_dumps(repeated_config),
|
||||
input_count,
|
||||
len(source_files),
|
||||
len(source_files),
|
||||
source_task.get("tenant_id"),
|
||||
source_task.get("project_id"),
|
||||
source_task.get("owner_id"),
|
||||
source_task.get("approval_status") or "not_required",
|
||||
source_task.get("created_by"),
|
||||
source_task.get("created_by"),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
file_id_map: dict[str, str] = {}
|
||||
for source in source_files:
|
||||
old_file_id = str(source["id"])
|
||||
copy = file_copies[old_file_id]
|
||||
new_file_id = str(copy["id"])
|
||||
storage_object_id, metadata = _source_storage_descriptor(
|
||||
{
|
||||
"storage_object_id": copy["storage_object_id"],
|
||||
"metadata": _json_value(source.get("metadata"), {}),
|
||||
},
|
||||
task_id,
|
||||
new_file_id,
|
||||
)
|
||||
file_id_map[old_file_id] = new_file_id
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_source_files
|
||||
(id, task_id, storage_object_id, name, size_bytes, record_count,
|
||||
file_format, checksum_sha256, version_no, content,
|
||||
content_preview, metadata, tenant_id, project_id, created_by,
|
||||
created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 1, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
new_file_id,
|
||||
task_id,
|
||||
storage_object_id,
|
||||
source["name"],
|
||||
source.get("size_bytes") or 0,
|
||||
source.get("record_count") or 0,
|
||||
source.get("file_format"),
|
||||
source["checksum_sha256"],
|
||||
source.get("content") or "",
|
||||
source.get("content_preview"),
|
||||
json_dumps(metadata),
|
||||
source_task.get("tenant_id"),
|
||||
source_task.get("project_id"),
|
||||
source.get("created_by") or source_task.get("created_by"),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
|
||||
for preview in previews:
|
||||
old_source_file_id = preview.get("source_file_id")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_process_preview_items
|
||||
(id, task_id, source_file_id, original_content, edited_content,
|
||||
source_start, source_end, source_start_line, source_end_line,
|
||||
token_count, status, quality_score, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s)
|
||||
""",
|
||||
(
|
||||
new_id("dpp"),
|
||||
task_id,
|
||||
file_id_map.get(str(old_source_file_id))
|
||||
if old_source_file_id
|
||||
else None,
|
||||
preview.get("original_content") or "",
|
||||
preview.get("edited_content") or "",
|
||||
preview.get("source_start"),
|
||||
preview.get("source_end"),
|
||||
preview.get("source_start_line"),
|
||||
preview.get("source_end_line"),
|
||||
max(0, int(preview.get("token_count") or 0)),
|
||||
preview.get("status") or "original",
|
||||
json_dumps(_json_value(preview.get("quality_score"), {})),
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
return self._repeat_response(
|
||||
conn,
|
||||
task_row or {},
|
||||
source_task_id=source_task_id,
|
||||
created=True,
|
||||
)
|
||||
except psycopg.errors.UniqueViolation as exc:
|
||||
raise ConflictError("再次生成任务名称或请求发生冲突,请重试") from exc
|
||||
|
||||
def get_task(self, task_id: str, *, for_update: bool = False) -> dict[str, Any]:
|
||||
lock = " FOR UPDATE" if for_update else ""
|
||||
with self.connect() as conn:
|
||||
@@ -654,14 +932,11 @@ class DataProcessStore:
|
||||
"process type and source dataset cannot change during regeneration"
|
||||
)
|
||||
if payload.get("config") is not None:
|
||||
next_config = {
|
||||
key: value
|
||||
for key, value in payload["config"].items()
|
||||
if key != _REGENERATION_MARKER_KEY
|
||||
}
|
||||
current_marker = _regeneration_marker(task)
|
||||
if current_marker:
|
||||
next_config[_REGENERATION_MARKER_KEY] = current_marker
|
||||
next_config = _business_config(payload["config"])
|
||||
current_config = dict(task.get("config") or {})
|
||||
for key in _INTERNAL_CONFIG_KEYS:
|
||||
if key in current_config:
|
||||
next_config[key] = current_config[key]
|
||||
values["config"] = json_dumps(next_config)
|
||||
invalidates_results = (
|
||||
("config" in payload and payload.get("config") != task.get("config"))
|
||||
@@ -753,8 +1028,10 @@ class DataProcessStore:
|
||||
raise InvalidStateError("process_type cannot be changed during regeneration")
|
||||
|
||||
current_config = dict(task.get("config") or {})
|
||||
next_config = dict(payload.get("config") or {})
|
||||
next_config.pop(_REGENERATION_MARKER_KEY, None)
|
||||
next_config = _business_config(payload.get("config"))
|
||||
for key in (_REPEAT_SOURCE_TASK_KEY, _REPEAT_REQUEST_KEY):
|
||||
if key in current_config:
|
||||
next_config[key] = current_config[key]
|
||||
preview_invalidated = _preview_config_changed(
|
||||
process_type,
|
||||
current_config,
|
||||
|
||||
244
backend/app/modules/project/router.py
Normal file
244
backend/app/modules/project/router.py
Normal file
@@ -0,0 +1,244 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, has_resource_access, is_admin
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["project"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
def _require_no_pending_approval(resource_type: str, resource_id: str) -> None:
|
||||
"""第 4 周:写操作审批拦截——存在待审批实例时拒绝执行。"""
|
||||
store = get_platform_store()
|
||||
pending = [
|
||||
i for i in store.approval_instances(status="pending")
|
||||
if i["resource_type"] == resource_type and i["resource_id"] == resource_id
|
||||
]
|
||||
if pending:
|
||||
raise fail(409, "存在待审批的变更,请先完成审批")
|
||||
|
||||
|
||||
def _require_approval_or_admin(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
current_user: dict[str, Any],
|
||||
action_desc: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""高风险操作审批旁路:admin 直接放行,普通用户创建审批实例(code=202)。"""
|
||||
if is_admin(current_user):
|
||||
return None
|
||||
store = get_platform_store()
|
||||
instance = store.create_approval_instance({
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"applicant_id": current_user.get("id"),
|
||||
"template_id": None,
|
||||
})
|
||||
return {
|
||||
"code": 202,
|
||||
"message": f"操作已提交审批,等待管理员批准:{action_desc}",
|
||||
"data": {"approval_required": True, "approval_id": instance["id"]},
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_projects(
|
||||
tenant_id: str = "default",
|
||||
status: str | None = None,
|
||||
keyword: str | None = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
projects = store.projects(tenant_id=tenant_id, status=status, keyword=keyword)
|
||||
# #1 ACL 过滤:admin 直接放行,普通用户只能看到自己被授权的项目
|
||||
accessible_ids = set(
|
||||
filter_accessible_resource_ids("project", [p["id"] for p in projects], current_user)
|
||||
)
|
||||
filtered = [p for p in projects if p["id"] in accessible_ids]
|
||||
return ok(filtered)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_project(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
proj = store.create_project(payload)
|
||||
store.record_audit(
|
||||
action="project.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=proj["id"],
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
detail=f"name={proj.get('name')}",
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.get("/{project_id}")
|
||||
def get_project(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
# #2 访问控制:普通用户无 read 权限则拒绝
|
||||
if not has_resource_access("project", project_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this project")
|
||||
try:
|
||||
return ok(get_platform_store().project(project_id))
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
|
||||
|
||||
@router.put("/{project_id}")
|
||||
def update_project(
|
||||
project_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to update this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
proj = store.update_project(project_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.post("/{project_id}/archive")
|
||||
def archive_project(
|
||||
project_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
_require_no_pending_approval("project", project_id)
|
||||
pending = _require_approval_or_admin("project", project_id, current_user, f"归档项目 {project_id}")
|
||||
if pending:
|
||||
return pending
|
||||
store = get_platform_store()
|
||||
try:
|
||||
proj = store.archive_project(project_id)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.archive",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
tenant_id=proj.get("tenant_id"),
|
||||
)
|
||||
return ok(proj)
|
||||
|
||||
|
||||
@router.delete("/{project_id}")
|
||||
def delete_project(
|
||||
project_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
_require_no_pending_approval("project", project_id)
|
||||
pending = _require_approval_or_admin("project", project_id, current_user, f"删除项目 {project_id}")
|
||||
if pending:
|
||||
return pending
|
||||
store = get_platform_store()
|
||||
store.delete_project(project_id)
|
||||
store.record_audit(
|
||||
action="project.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project",
|
||||
target_id=project_id,
|
||||
)
|
||||
return ok(None)
|
||||
|
||||
|
||||
@router.get("/{project_id}/members")
|
||||
def list_members(project_id: str, current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "read"):
|
||||
raise fail(403, "no permission to access this project")
|
||||
try:
|
||||
return ok(get_platform_store().project_members(project_id))
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
|
||||
|
||||
@router.post("/{project_id}/members")
|
||||
def add_member(
|
||||
project_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
member = store.add_project_member(project_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project not found")
|
||||
store.record_audit(
|
||||
action="project.member.add",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={payload.get('user_id')},role={payload.get('role')}",
|
||||
)
|
||||
return ok(member)
|
||||
|
||||
|
||||
@router.put("/{project_id}/members/{user_id}")
|
||||
def update_member(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
try:
|
||||
member = store.update_project_member_role(project_id, user_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "project or member not found")
|
||||
store.record_audit(
|
||||
action="project.member.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={user_id},role={payload.get('role')}",
|
||||
)
|
||||
return ok(member)
|
||||
|
||||
|
||||
@router.delete("/{project_id}/members/{user_id}")
|
||||
def remove_member(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
request: Request = None,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
if not has_resource_access("project", project_id, current_user, "write"):
|
||||
raise fail(403, "no permission to manage members of this project")
|
||||
store = get_platform_store()
|
||||
store.remove_project_member(project_id, user_id)
|
||||
store.record_audit(
|
||||
action="project.member.remove",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="project.member",
|
||||
target_id=project_id,
|
||||
detail=f"user_id={user_id}",
|
||||
)
|
||||
return ok(None)
|
||||
1
backend/app/modules/resource/__init__.py
Normal file
1
backend/app/modules/resource/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Resource access control list (ACL) module."""
|
||||
41
backend/app/modules/resource/router.py
Normal file
41
backend/app/modules/resource/router.py
Normal file
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/resources", tags=["resource"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("/{resource_type}/{resource_id}/acl")
|
||||
def get_acl(resource_type: str, resource_id: str) -> dict[str, Any]:
|
||||
"""查询资源 ACL,返回按主体分组的权限列表。"""
|
||||
return ok(get_platform_store().resource_acl(resource_type, resource_id))
|
||||
|
||||
|
||||
@router.put("/{resource_type}/{resource_id}/acl")
|
||||
def set_acl(
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
payload: dict[str, Any] = Body(...),
|
||||
request: Request = None,
|
||||
) -> dict[str, Any]:
|
||||
"""设置资源 ACL,body: { entries: [{ subject_type, subject_id, permissions: [] }] }"""
|
||||
entries = payload.get("entries") or []
|
||||
result = get_platform_store().set_resource_acl(resource_type, resource_id, entries)
|
||||
get_platform_store().record_audit(
|
||||
action="resource.acl.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type=resource_type,
|
||||
target_id=resource_id,
|
||||
detail=f"entries={len(entries)}",
|
||||
)
|
||||
return ok(result)
|
||||
75
backend/app/modules/retention/router.py
Normal file
75
backend/app/modules/retention/router.py
Normal file
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/retention-policies", tags=["retention"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_policies() -> dict[str, Any]:
|
||||
return ok(get_platform_store().retention_policies())
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_policy(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
if not payload.get("name"):
|
||||
raise fail(400, "name 必填")
|
||||
policy = get_platform_store().create_retention_policy(payload)
|
||||
get_platform_store().record_audit(
|
||||
action="retention.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy["id"],
|
||||
detail=f"name={policy.get('name')}",
|
||||
)
|
||||
return ok(policy)
|
||||
|
||||
|
||||
@router.get("/{policy_id}")
|
||||
def get_policy(policy_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().retention_policy(policy_id))
|
||||
except KeyError:
|
||||
raise fail(404, "retention policy not found")
|
||||
|
||||
|
||||
@router.put("/{policy_id}")
|
||||
def update_policy(
|
||||
policy_id: str, payload: dict[str, Any] = Body(...), request: Request = None
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
policy = store.update_retention_policy(policy_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "retention policy not found")
|
||||
store.record_audit(
|
||||
action="retention.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy_id,
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(policy)
|
||||
|
||||
|
||||
@router.delete("/{policy_id}")
|
||||
def delete_policy(policy_id: str, request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
store.delete_retention_policy(policy_id)
|
||||
store.record_audit(
|
||||
action="retention.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="retention_policy",
|
||||
target_id=policy_id,
|
||||
)
|
||||
return ok({"deleted": policy_id})
|
||||
115
backend/app/modules/system/router.py
Normal file
115
backend/app/modules/system/router.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.db.platform_store import ALL_PERMISSIONS, get_platform_store
|
||||
|
||||
|
||||
router = APIRouter(prefix="/system", tags=["system"])
|
||||
|
||||
|
||||
@router.post("/audit/visit")
|
||||
def record_visit(payload: dict = Body(...), request: Request = None) -> dict:
|
||||
"""记录用户访问业务模块的行为,用于看板用户操作分布统计。"""
|
||||
action = str(payload.get("action") or payload.get("module") or "").strip()
|
||||
if not action:
|
||||
return {"code": 0, "message": "ok", "data": {"recorded": False}}
|
||||
actor_id = ""
|
||||
if request is not None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
if token.startswith("platform-token-"):
|
||||
actor_id = token[len("platform-token-"):]
|
||||
get_platform_store().record_audit(
|
||||
action=action,
|
||||
actor_id=actor_id or None,
|
||||
target_type="module",
|
||||
target_id=action,
|
||||
detail=str(payload.get("detail") or ""),
|
||||
)
|
||||
return {"code": 0, "message": "ok", "data": {"recorded": True}}
|
||||
|
||||
|
||||
@router.get("/permissions/codes")
|
||||
def permission_codes() -> dict:
|
||||
"""返回平台权限码清单(权限码接口)。"""
|
||||
return {"code": 0, "message": "ok", "data": {"codes": ALL_PERMISSIONS}}
|
||||
|
||||
|
||||
@router.get("/permissions")
|
||||
def permissions_overview() -> dict:
|
||||
"""返回权限码清单与角色定义。"""
|
||||
store = get_platform_store()
|
||||
return {
|
||||
"code": 0,
|
||||
"message": "ok",
|
||||
"data": {"codes": ALL_PERMISSIONS, "roles": store.roles()},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/audit-logs")
|
||||
def audit_logs(
|
||||
tenant_id: str | None = Query(default=None, description="租户 ID"),
|
||||
project_id: str | None = Query(default=None, description="项目 ID"),
|
||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||
action: str | None = Query(default=None, description="动作类型"),
|
||||
target_type: str | None = Query(default=None, description="目标类型"),
|
||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict:
|
||||
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
actor_id=actor_id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return {"code": 0, "message": "ok", "data": result}
|
||||
|
||||
|
||||
@router.get("/audit-logs/export")
|
||||
def audit_logs_export(
|
||||
tenant_id: str | None = Query(default=None, description="租户 ID"),
|
||||
project_id: str | None = Query(default=None, description="项目 ID"),
|
||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||
action: str | None = Query(default=None, description="动作类型"),
|
||||
target_type: str | None = Query(default=None, description="目标类型"),
|
||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||
) -> StreamingResponse:
|
||||
"""审计日志导出:返回 CSV 流,与应用查询相同的过滤条件。"""
|
||||
store = get_platform_store()
|
||||
result = store.audit_logs(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
actor_id=actor_id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=10000,
|
||||
offset=0,
|
||||
)
|
||||
items = result["items"]
|
||||
columns = ["time", "tenant_id", "project_id", "actor_id", "action", "target_type", "target_id", "detail", "client_ip"]
|
||||
header = ",".join(columns) + "\n"
|
||||
|
||||
def iter_rows():
|
||||
yield header
|
||||
for row in items:
|
||||
yield ",".join(f'"{str(row.get(c, "") or "")}"' for c in columns) + "\n"
|
||||
|
||||
return StreamingResponse(
|
||||
iter_rows(),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=audit_logs.csv"},
|
||||
)
|
||||
116
backend/app/modules/tenant/router.py
Normal file
116
backend/app/modules/tenant/router.py
Normal file
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Body, Request
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import ok, fail
|
||||
from app.db.platform_store import get_platform_store
|
||||
|
||||
router = APIRouter(prefix="/tenants", tags=["tenant"])
|
||||
|
||||
|
||||
def _actor(request: Request) -> str | None:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth.replace("Bearer ", "").strip()
|
||||
return token or None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_tenants() -> dict[str, Any]:
|
||||
return ok(get_platform_store().tenants())
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_tenant(payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.create_tenant(payload)
|
||||
except KeyError as e:
|
||||
raise fail(400, f"missing field: {e}")
|
||||
store.record_audit(
|
||||
action="tenant.create",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant["id"],
|
||||
tenant_id=tenant["id"],
|
||||
detail=f"name={tenant.get('name')}",
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.get("/{tenant_id}")
|
||||
def get_tenant(tenant_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().tenant(tenant_id))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
|
||||
|
||||
@router.put("/{tenant_id}")
|
||||
def update_tenant(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.update_tenant(tenant_id, payload)
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.update",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
detail=f"fields={','.join(payload.keys())}",
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.put("/{tenant_id}/quota")
|
||||
def set_quota(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.set_tenant_quota(tenant_id, payload.get("quota", {}))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.quota.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.put("/{tenant_id}/retention-policy")
|
||||
def set_retention(tenant_id: str, payload: dict[str, Any] = Body(...), request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.set_tenant_retention(tenant_id, payload.get("retention_policy_id"))
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.retention.set",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
return ok(tenant)
|
||||
|
||||
|
||||
@router.delete("/{tenant_id}")
|
||||
def delete_tenant(tenant_id: str, request: Request = None) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
tenant = store.delete_tenant(tenant_id)
|
||||
except KeyError:
|
||||
raise fail(404, "tenant not found")
|
||||
store.record_audit(
|
||||
action="tenant.delete",
|
||||
actor_id=_actor(request) if request else None,
|
||||
target_type="tenant",
|
||||
target_id=tenant_id,
|
||||
tenant_id=tenant_id,
|
||||
detail=f"name={tenant.get('name')}",
|
||||
)
|
||||
return ok(tenant)
|
||||
@@ -220,6 +220,19 @@ class DataProcessRegenerateRequest(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessRepeatRequest(BaseModel):
|
||||
"""按已确认任务的完整快照创建一批独立的新生成结果。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
expected_updated_at: str = Field(min_length=1)
|
||||
request_id: str = Field(
|
||||
min_length=8,
|
||||
max_length=80,
|
||||
pattern=r"^[A-Za-z0-9_-]+$",
|
||||
)
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ dependencies = [
|
||||
"pydantic>=2.7.0",
|
||||
"sqlalchemy>=2.0.30",
|
||||
"psycopg[binary]>=3.2.1",
|
||||
"psycopg-pool>=3.2.1",
|
||||
"alembic>=1.13.1",
|
||||
"redis>=5.0.4",
|
||||
"httpx>=0.27.0",
|
||||
|
||||
@@ -4,6 +4,7 @@ python-multipart>=0.0.9
|
||||
pydantic>=2.7.0
|
||||
sqlalchemy>=2.0.30
|
||||
psycopg[binary]>=3.2.1
|
||||
psycopg-pool>=3.2.1
|
||||
alembic>=1.13.1
|
||||
redis>=5.0.4
|
||||
httpx>=0.27.0
|
||||
@@ -18,3 +19,7 @@ llama-index-core==0.14.23
|
||||
llama-index-embeddings-huggingface==0.6.1
|
||||
docling==2.115.0
|
||||
tiktoken>=0.7.0
|
||||
|
||||
# 测试与代码检查
|
||||
pytest>=8.2.0
|
||||
ruff>=0.5.0
|
||||
|
||||
276
backend/tests/test_compare_inference_async.py
Normal file
276
backend/tests/test_compare_inference_async.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""
|
||||
模型推理异步加载改造的单元测试。
|
||||
|
||||
覆盖:
|
||||
- model_compare_load:异步派发,立即返回 starting + 节点信息(不等待加载完成)
|
||||
- model_compare_delete:先删记录,卸载失败也不阻塞删除
|
||||
- reconcile_inference_loads:starting -> ready/error/idle/不可达的状态迁移与封顶
|
||||
- _unload_from_compute_node:任务感知,只命中记录中的节点
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from app.api.v1.endpoints.platform import model_compare_delete, model_compare_load
|
||||
import app.api.v1.endpoints.platform as platform
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import MAX_STARTING_ATTEMPTS, reconcile_inference_loads
|
||||
|
||||
|
||||
class FakeInferenceStore:
|
||||
"""内存 store,仅实现推理加载/对账用到的接口。"""
|
||||
|
||||
def __init__(self, tasks: list[dict[str, Any]] | None = None, nodes: list[dict[str, Any]] | None = None) -> None:
|
||||
self._tasks: dict[str, dict[str, Any]] = {t["id"]: dict(t) for t in (tasks or [])}
|
||||
self._nodes = nodes or []
|
||||
self._inference_nodes: set[str] = set()
|
||||
|
||||
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||
if task_id not in self._tasks:
|
||||
raise KeyError(task_id)
|
||||
return dict(self._tasks[task_id])
|
||||
|
||||
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||
return [dict(t) for t in self._tasks.values()]
|
||||
|
||||
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self._tasks[task_id]
|
||||
merged = {**current, **payload, "id": task_id}
|
||||
self._tasks[task_id] = merged
|
||||
return dict(merged)
|
||||
|
||||
def delete_compare_task(self, task_id: str) -> None:
|
||||
self._tasks.pop(task_id, None)
|
||||
|
||||
def compute_nodes(self) -> list[dict[str, Any]]:
|
||||
return [dict(n) for n in self._nodes]
|
||||
|
||||
def model(self, model_id: str) -> dict[str, Any]:
|
||||
raise KeyError(model_id)
|
||||
|
||||
def trained_models(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
def mark_inference_loaded(self, node_id: str) -> None:
|
||||
self._inference_nodes.add(node_id)
|
||||
|
||||
def mark_inference_unloaded(self, node_id: str) -> None:
|
||||
self._inference_nodes.discard(node_id)
|
||||
|
||||
def is_inference_loaded(self, node_id: str) -> bool:
|
||||
return node_id in self._inference_nodes
|
||||
|
||||
|
||||
def _node(node_id: str, code: str = "") -> dict[str, Any]:
|
||||
return {
|
||||
"id": node_id,
|
||||
"code": code or node_id,
|
||||
"name": code or node_id,
|
||||
"api_base_url": f"http://{code or node_id}:19100",
|
||||
"enabled": True,
|
||||
"scheduler_status": "online",
|
||||
}
|
||||
|
||||
|
||||
def _task(task_id: str, *, node_id: str | None = None, load_status: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": f"task-{task_id}",
|
||||
"status": "pending",
|
||||
"models": [
|
||||
{"model_id": "m_1", "model_name": "qwen", "model_path": "/models/qwen", "node_id": node_id}
|
||||
],
|
||||
"load_status": load_status or {"loaded_models": []},
|
||||
}
|
||||
|
||||
|
||||
async def _fake_inference_load(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "loading", "request_id": "req-1"}
|
||||
|
||||
|
||||
async def _fake_inference_unload(self) -> dict[str, Any]:
|
||||
return {"unloaded": True, "status": "idle"}
|
||||
|
||||
|
||||
def _patch_store(monkeypatch, store: FakeInferenceStore) -> None:
|
||||
monkeypatch.setattr(platform, "get_platform_store", lambda: store)
|
||||
monkeypatch.setattr(platform, "get_settings", lambda: SimpleNamespace(compute_mode="real"))
|
||||
|
||||
|
||||
def test_select_eval_node_prefers_model_node(monkeypatch) -> None:
|
||||
from app.api.v1.endpoints.platform import _select_eval_node
|
||||
|
||||
store = FakeInferenceStore(nodes=[_node("n1"), _node("n2")])
|
||||
# 指定模型所在节点时优先返回该节点
|
||||
assert _select_eval_node(store, "n2")["id"] == "n2"
|
||||
# 无指定节点时回退到第一个在线节点
|
||||
assert _select_eval_node(store, None)["id"] == "n1"
|
||||
|
||||
|
||||
def test_select_eval_node_returns_none_when_model_node_offline(monkeypatch) -> None:
|
||||
from app.api.v1.endpoints.platform import _select_eval_node
|
||||
|
||||
nodes = [_node("n1"), _node("n2")]
|
||||
nodes[1]["enabled"] = False
|
||||
store = FakeInferenceStore(nodes=nodes)
|
||||
# 模型所在节点不可用 → 明确失败,不派发到其它节点
|
||||
assert _select_eval_node(store, "n2") is None
|
||||
# 无指定节点时仍回退第一个在线节点
|
||||
assert _select_eval_node(store, None)["id"] == "n1"
|
||||
|
||||
|
||||
def test_model_compare_load_dispatches_and_returns_starting(monkeypatch) -> None:
|
||||
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_load", _fake_inference_load)
|
||||
|
||||
result = asyncio.run(model_compare_load("t1"))
|
||||
assert result["code"] == 0
|
||||
updated = result["data"]
|
||||
assert updated["status"] == "starting"
|
||||
items = updated["load_status"]["loaded_models"]
|
||||
assert items[0]["status"] == "starting"
|
||||
assert items[0]["node_id"] == "n1"
|
||||
assert "n1" in store._inference_nodes
|
||||
|
||||
|
||||
def test_model_compare_load_marks_error_when_all_nodes_fail(monkeypatch) -> None:
|
||||
store = FakeInferenceStore(tasks=[_task("t1", node_id="n1")], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
|
||||
async def _raise(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raise RuntimeError("conn refused")
|
||||
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_load", _raise)
|
||||
|
||||
result = asyncio.run(model_compare_load("t1"))
|
||||
updated = result["data"]
|
||||
assert updated["status"] == "failed"
|
||||
assert updated["load_status"]["loaded_models"][0]["status"] == "error"
|
||||
assert "conn refused" in updated["load_status"]["loaded_models"][0]["error"]
|
||||
|
||||
|
||||
def test_model_compare_delete_removes_record_even_if_unload_raises(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
_patch_store(monkeypatch, store)
|
||||
|
||||
async def _raise(self) -> dict[str, Any]:
|
||||
raise RuntimeError("unload boom")
|
||||
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _raise)
|
||||
|
||||
result = asyncio.run(model_compare_delete("t1"))
|
||||
assert result["data"] == {"deleted": "t1"}
|
||||
assert "t1" not in store._tasks
|
||||
# finally 中仍清掉了节点标记
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
def test_unload_from_compute_node_only_hits_recorded_node(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "ready", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1"), _node("n2")])
|
||||
_patch_store(monkeypatch, store)
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_unload", _fake_inference_unload)
|
||||
|
||||
from app.api.v1.endpoints.platform import _unload_from_compute_node
|
||||
|
||||
result = asyncio.run(_unload_from_compute_node(store, task=task))
|
||||
assert result["unloaded"] is True
|
||||
# 只命中任务记录中的节点 n1,n2 未被卸载
|
||||
assert [r["node_id"] for r in result["nodes"]] == ["n1"]
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
async def _status_ready(self) -> dict[str, Any]:
|
||||
return {"loaded": True, "status": "ready", "model_name": "qwen"}
|
||||
|
||||
|
||||
def test_reconcile_transitions_starting_to_ready(monkeypatch) -> None:
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_ready)
|
||||
|
||||
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||
assert reconciled == [{"task_id": "t1", "status": "loaded"}]
|
||||
updated = store._tasks["t1"]
|
||||
assert updated["status"] == "loaded"
|
||||
assert updated["load_status"]["loaded_models"][0]["status"] == "ready"
|
||||
assert "n1" in store._inference_nodes
|
||||
|
||||
|
||||
def test_reconcile_transitions_to_error_and_failed(monkeypatch) -> None:
|
||||
async def _status_error(self) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "error", "error": "CUDA out of memory"}
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_error)
|
||||
|
||||
reconciled = asyncio.run(reconcile_inference_loads(store))
|
||||
assert reconciled == [{"task_id": "t1", "status": "failed"}]
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "CUDA out of memory" in item["error"]
|
||||
assert "n1" not in store._inference_nodes
|
||||
|
||||
|
||||
def test_reconcile_idle_marks_model_disappeared(monkeypatch) -> None:
|
||||
async def _status_idle(self) -> dict[str, Any]:
|
||||
return {"loaded": False, "status": "idle"}
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _status_idle)
|
||||
|
||||
asyncio.run(reconcile_inference_loads(store))
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "disappeared" in item["error"]
|
||||
assert store._tasks["t1"]["status"] == "failed"
|
||||
|
||||
|
||||
def test_reconcile_unreachable_node_flips_to_error_after_cap(monkeypatch) -> None:
|
||||
async def _raise(self) -> dict[str, Any]:
|
||||
raise RuntimeError("conn refused")
|
||||
|
||||
task = _task(
|
||||
"t1",
|
||||
node_id="n1",
|
||||
load_status={"loaded_models": [{"model_id": "m_1", "status": "starting", "node_id": "n1"}]},
|
||||
)
|
||||
store = FakeInferenceStore(tasks=[task], nodes=[_node("n1")])
|
||||
monkeypatch.setattr(ComputeNodeClient, "inference_status", _raise)
|
||||
|
||||
# 每次轮询前重置节流时间戳,逐次推进 load_attempts 到封顶
|
||||
for _ in range(MAX_STARTING_ATTEMPTS):
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
item["last_polled_at"] = 0
|
||||
asyncio.run(reconcile_inference_loads(store))
|
||||
|
||||
item = store._tasks["t1"]["load_status"]["loaded_models"][0]
|
||||
assert item["status"] == "error"
|
||||
assert "unreachable" in item["error"]
|
||||
assert store._tasks["t1"]["status"] == "failed"
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
@@ -29,11 +30,13 @@ from app.modules.data_process.algorithms import (
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
preprocess_structured_records,
|
||||
preprocess_structured_records_with_lineage,
|
||||
record_fingerprint,
|
||||
remove_document_noise,
|
||||
score_quality,
|
||||
stable_split,
|
||||
stable_split_assignments,
|
||||
structured_json_dumps,
|
||||
)
|
||||
|
||||
|
||||
@@ -194,6 +197,75 @@ def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_structured_text_record_locators_preserve_logical_source_positions() -> None:
|
||||
root_json = parse_text_content('{"id":1}', filename="root.json")
|
||||
assert root_json.record_locators == (
|
||||
{
|
||||
"kind": "json",
|
||||
"record_index": 1,
|
||||
"json_pointer": "",
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
},
|
||||
)
|
||||
|
||||
wrapped_json = parse_text_content(
|
||||
'{"records":[{"id":1},{"id":1}]}',
|
||||
filename="wrapped.json",
|
||||
)
|
||||
assert [locator["json_pointer"] for locator in wrapped_json.record_locators] == [
|
||||
"/records/0",
|
||||
"/records/1",
|
||||
]
|
||||
|
||||
parsed_jsonl = parse_text_content(
|
||||
'{"id":1}\r\n\r\n{"id":1}',
|
||||
filename="records.jsonl",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == [(1, 1, 1), (2, 3, 3)]
|
||||
assert [
|
||||
parsed_jsonl.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_jsonl.record_locators
|
||||
] == ['{"id":1}', '{"id":1}']
|
||||
|
||||
parsed_csv = parse_text_content(
|
||||
'id,note\r\n1,"hello\r\nworld"\r\n\r\n2,plain',
|
||||
filename="records.csv",
|
||||
)
|
||||
assert [
|
||||
(locator["record_index"], locator["start_line"], locator["end_line"])
|
||||
for locator in parsed_csv.record_locators
|
||||
] == [(1, 2, 3), (2, 5, 5)]
|
||||
assert [
|
||||
parsed_csv.text[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed_csv.record_locators
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_preprocess_lineage_survives_column_cleanup_and_row_removal() -> None:
|
||||
processed = preprocess_structured_records_with_lineage(
|
||||
[
|
||||
{"id": "A", "value": "first", "empty": ""},
|
||||
{"id": "", "value": "invalid", "empty": ""},
|
||||
{"id": "A", "value": "duplicate identity", "empty": ""},
|
||||
{"id": "B", "value": "second", "empty": ""},
|
||||
],
|
||||
["clean_invalid", "deduplicate"],
|
||||
)
|
||||
assert [entry.source_index for entry in processed] == [0, 1, 2, 3]
|
||||
assert [entry.record for entry in processed] == [
|
||||
{"id": "A", "value": "first"},
|
||||
{"id": "", "value": "invalid"},
|
||||
{"id": "A", "value": "duplicate identity"},
|
||||
{"id": "B", "value": "second"},
|
||||
]
|
||||
|
||||
|
||||
def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
parsed_pdf = parse_text_content(_minimal_pdf(), filename="manual.pdf")
|
||||
assert parsed_pdf.format == "pdf"
|
||||
@@ -220,6 +292,24 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
{"name": "Alice", "score": 95, "created_at": "2026-07-23T10:30:00"},
|
||||
{"name": "Bob", "score": 88, "created_at": "2026-07-24T09:00:00"},
|
||||
)
|
||||
assert parsed_xlsx.record_locators == (
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "数据",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
)
|
||||
assert json.loads(parsed_xlsx.text.splitlines()[0]) == parsed_xlsx.records[0]
|
||||
|
||||
parsed_pptx = parse_text_content(_pptx_bytes(), filename="slides.pptx")
|
||||
@@ -228,6 +318,44 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
||||
assert parsed_pptx.records == ()
|
||||
|
||||
|
||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||
workbook = Workbook()
|
||||
first = workbook.active
|
||||
first.title = "甲表"
|
||||
first.append(["说明"])
|
||||
first.append([])
|
||||
first.append(["id", "value"])
|
||||
first.append([1, "same"])
|
||||
first.append([1, "same"])
|
||||
second = workbook.create_sheet("乙表")
|
||||
second.append(["id", "value"])
|
||||
second.append([1, "same"])
|
||||
output = io.BytesIO()
|
||||
workbook.save(output)
|
||||
workbook.close()
|
||||
|
||||
parsed = parse_text_content(output.getvalue(), filename="duplicate.xlsx")
|
||||
assert parsed.records == (
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
{"id": 1, "value": "same"},
|
||||
)
|
||||
assert [
|
||||
(
|
||||
locator["record_index"],
|
||||
locator["sheet_index"],
|
||||
locator["sheet_name"],
|
||||
locator["row_number"],
|
||||
locator["sheet_record_index"],
|
||||
)
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(1, 0, "甲表", 4, 0),
|
||||
(2, 0, "甲表", 5, 1),
|
||||
(3, 1, "乙表", 2, 0),
|
||||
]
|
||||
|
||||
|
||||
def test_pdf_document_noise_removes_headers_page_numbers_and_toc_safely() -> None:
|
||||
pages = _pdf_page_texts(
|
||||
"""
|
||||
@@ -568,7 +696,130 @@ def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
assert result == [{"items": [{"text": " 内容 "}], "ignored": 1}]
|
||||
|
||||
assert extract_structured_records(
|
||||
'{"items":[{"text":" 内容 "}],"total":1}',
|
||||
"json",
|
||||
) == [{"text": " 内容 "}]
|
||||
|
||||
|
||||
def test_json_parsing_is_strict_and_preserves_field_values() -> None:
|
||||
source = '{"code":"001","text":" 内容 ","quote":"""}'
|
||||
parsed = parse_text_content(source, filename="records.json")
|
||||
assert parsed.text == source
|
||||
assert parsed.records == (
|
||||
{"code": "001", "text": " 内容 ", "quote": """},
|
||||
)
|
||||
|
||||
invalid_values = (
|
||||
'{"id":1,"id":2}',
|
||||
'{"nested":{"id":1,"id":2}}',
|
||||
'{"value":NaN}',
|
||||
'{"value":Infinity}',
|
||||
'{"value":-Infinity}',
|
||||
'{"value":"bad\x00control"}',
|
||||
)
|
||||
for invalid in invalid_values:
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content(invalid, filename="invalid.json")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_text_content("{\"id\":1}", filename="invalid.json")
|
||||
with pytest.raises(ValueError, match="nesting exceeds"):
|
||||
parse_text_content("[" * 65 + "0" + "]" * 65, filename="deep.json")
|
||||
|
||||
|
||||
def test_jsonl_uses_the_same_strict_lossless_number_and_text_contract() -> None:
|
||||
source = (
|
||||
' {"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}\r\n\r\n'
|
||||
'{"id":2}\r\n'
|
||||
)
|
||||
parsed = parse_text_content(source, filename="records.jsonl")
|
||||
assert parsed.text == source
|
||||
assert parsed.records[0] == {
|
||||
"code": "001",
|
||||
"text": " 内容 ",
|
||||
"value": Decimal("0.123456789012345678901234567890"),
|
||||
}
|
||||
assert [
|
||||
source[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == [
|
||||
(
|
||||
'{"code":"001","text":" 内容 ",'
|
||||
'"value":0.123456789012345678901234567890}'
|
||||
),
|
||||
'{"id":2}',
|
||||
]
|
||||
assert [locator["start_line"] for locator in parsed.record_locators] == [1, 3]
|
||||
|
||||
for invalid in ('{"id":1,"id":2}', '{"value":NaN}'):
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 1"):
|
||||
parse_text_content(invalid, filename="invalid.jsonl")
|
||||
|
||||
|
||||
def test_json_record_contract_avoids_business_field_collisions() -> None:
|
||||
assert extract_structured_records('[{"id":1},{"id":2}]', "json") == [
|
||||
{"id": 1},
|
||||
{"id": 2},
|
||||
]
|
||||
assert extract_structured_records('{"id":1,"data":[{"id":2}]}', "json") == [
|
||||
{"id": 1, "data": [{"id": 2}]}
|
||||
]
|
||||
assert extract_structured_records(
|
||||
'{"records":[{"id":1}],"data":[{"id":2}]}',
|
||||
"json",
|
||||
) == [{"records": [{"id": 1}], "data": [{"id": 2}]}]
|
||||
assert extract_structured_records(
|
||||
'{"response":{"data":[{"id":1}],"status":"ok"},"success":true,"code":0}',
|
||||
"json",
|
||||
) == [{"id": 1}]
|
||||
assert extract_structured_records(
|
||||
'{"payload":{"data":[{"id":2}],"total":1}}',
|
||||
"json",
|
||||
) == [{"id": 2}]
|
||||
assert extract_structured_records('{"records":[],"total":0}', "json") == []
|
||||
# 包装数组中的非对象不是记录集合,整体按一条业务对象保留。
|
||||
assert extract_structured_records('{"data":[1,2]}', "json") == [
|
||||
{"data": [1, 2]}
|
||||
]
|
||||
|
||||
|
||||
def test_json_record_locators_cover_pretty_and_minified_sources() -> None:
|
||||
pretty = (
|
||||
'{\n "records": [\n {"id": 1},\n'
|
||||
' {\n "id": 2\n }\n ],\n "total": 2\n}'
|
||||
)
|
||||
parsed = parse_text_content(pretty, filename="pretty.json")
|
||||
assert [
|
||||
pretty[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id": 1}', '{\n "id": 2\n }']
|
||||
assert [
|
||||
(locator["start_line"], locator["end_line"])
|
||||
for locator in parsed.record_locators
|
||||
] == [(3, 3), (4, 6)]
|
||||
|
||||
minified = '[{"id":1},{"id":2}]'
|
||||
parsed = parse_text_content(minified, filename="minified.json")
|
||||
assert [
|
||||
minified[locator["source_start"] : locator["source_end"]]
|
||||
for locator in parsed.record_locators
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
|
||||
def test_high_precision_json_numbers_serialize_without_type_or_value_loss() -> None:
|
||||
source = '[{"value":0.123456789012345678901234567890},{"value":1e400}]'
|
||||
parsed = parse_text_content(source, filename="precise.json")
|
||||
assert parsed.records[0]["value"] == Decimal("0.123456789012345678901234567890")
|
||||
assert parsed.records[1]["value"] == Decimal("1e400")
|
||||
assert structured_json_dumps(parsed.records[0]) == (
|
||||
'{"value":0.123456789012345678901234567890}'
|
||||
)
|
||||
assert structured_json_dumps(parsed.records[1]) == '{"value":1E+400}'
|
||||
assert isinstance(parsed.records[0]["value"], Decimal)
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
@@ -587,9 +838,20 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
assert preprocess_structured_records(clean_source, []) == clean_source
|
||||
assert preprocess_structured_records(clean_source, ["clean_invalid"]) == [
|
||||
{"id": "1", "name": "有效"},
|
||||
{"id": "", "name": "缺少关键字段"},
|
||||
{"id": "2", "name": "有效"},
|
||||
]
|
||||
|
||||
hierarchy = [
|
||||
{"id": "1", "parent_id": None, "name": "根节点", "empty": ""},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点", "empty": ""},
|
||||
{"id": "", "parent_id": "", "name": "", "empty": ""},
|
||||
]
|
||||
assert preprocess_structured_records(hierarchy, ["clean_invalid"]) == [
|
||||
{"id": "1", "parent_id": None, "name": "根节点"},
|
||||
{"id": "2", "parent_id": "1", "name": "子节点"},
|
||||
]
|
||||
|
||||
nested = [{"id": 1, "profile": {"name": "张三", "level": 2}}]
|
||||
assert "profile" in preprocess_structured_records(nested, [])[0]
|
||||
assert preprocess_structured_records(nested, ["detect_structure"])[0] == {
|
||||
@@ -601,13 +863,15 @@ def test_every_structured_preprocess_option_has_independent_behavior() -> None:
|
||||
duplicates = [
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "C-1", "value": "updated"},
|
||||
{"customer_id": "C-1", "value": "first"},
|
||||
{"customer_id": "", "value": "blank-one"},
|
||||
{"customer_id": "", "value": "blank-two"},
|
||||
]
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 4
|
||||
assert len(preprocess_structured_records(duplicates, [])) == 5
|
||||
deduplicated = preprocess_structured_records(duplicates, ["deduplicate"])
|
||||
assert [record["value"] for record in deduplicated] == [
|
||||
"first",
|
||||
"updated",
|
||||
"blank-one",
|
||||
"blank-two",
|
||||
]
|
||||
@@ -660,6 +924,41 @@ def test_structured_desensitization_counts_and_document_helpers() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_structured_desensitization_only_masks_explicit_person_name_fields() -> None:
|
||||
masked, counts = desensitize_structured_record(
|
||||
{
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "张三",
|
||||
"contact_name": "李四",
|
||||
"姓名": "王五",
|
||||
"profile.name": "赵六",
|
||||
}
|
||||
)
|
||||
|
||||
assert masked == {
|
||||
"table_name": "customer_profile",
|
||||
"chinese_name": "zh_CN",
|
||||
"english_name": "en_US",
|
||||
"product_name": "智能助手",
|
||||
"metadata.table_name": "customer_archive",
|
||||
"name": "[NAME]",
|
||||
"contact_name": "[NAME]",
|
||||
"姓名": "[NAME]",
|
||||
"profile.name": "[NAME]",
|
||||
}
|
||||
assert counts == {
|
||||
"email": 0,
|
||||
"phone": 0,
|
||||
"id_card": 0,
|
||||
"name": 4,
|
||||
"total": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址?",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -21,7 +22,12 @@ from app.modules.data_process.storage import (
|
||||
LocalDataProcessStorage,
|
||||
get_data_process_storage,
|
||||
)
|
||||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||||
from app.modules.data_process.store import (
|
||||
InvalidStateError,
|
||||
NotFoundError,
|
||||
get_data_process_store,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
class FakeDataProcessStore:
|
||||
@@ -35,6 +41,7 @@ class FakeDataProcessStore:
|
||||
self.datasets: dict[str, dict[str, Any]] = {}
|
||||
self.models: dict[str, dict[str, Any]] = {}
|
||||
self.regeneration_prepared: set[str] = set()
|
||||
self.repeat_requests: dict[tuple[str, str], str] = {}
|
||||
self.sequence = 0
|
||||
|
||||
def _id(self, prefix: str) -> str:
|
||||
@@ -150,6 +157,121 @@ class FakeDataProcessStore:
|
||||
"published_outputs_preserved": published_outputs_preserved,
|
||||
}
|
||||
|
||||
def _repeat_response(
|
||||
self,
|
||||
source_task_id: str,
|
||||
repeated_task_id: str,
|
||||
*,
|
||||
created: bool,
|
||||
) -> dict[str, Any]:
|
||||
task = self.get_task(repeated_task_id)
|
||||
task["source_file_count"] = len(self.sources[repeated_task_id])
|
||||
task["preview_count"] = len(self.previews[repeated_task_id])
|
||||
return {
|
||||
"task": task,
|
||||
"source_task_id": source_task_id,
|
||||
"created": created,
|
||||
"copied_source_file_count": len(self.sources[repeated_task_id]),
|
||||
"copied_preview_count": len(self.previews[repeated_task_id]),
|
||||
}
|
||||
|
||||
def find_repeated_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
request_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
repeated_task_id = self.repeat_requests.get((source_task_id, request_id))
|
||||
if repeated_task_id is None:
|
||||
return None
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=False,
|
||||
)
|
||||
|
||||
def repeat_task(
|
||||
self,
|
||||
source_task_id: str,
|
||||
*,
|
||||
expected_updated_at: str,
|
||||
request_id: str,
|
||||
file_copies: dict[str, dict[str, str]],
|
||||
) -> dict[str, Any]:
|
||||
existing = self.find_repeated_task(source_task_id, request_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
source_task = self.get_task(source_task_id)
|
||||
if source_task["status"] != "completed" or source_task.get("results_confirmed") is False:
|
||||
raise InvalidStateError("只有已完成并确认结果的任务可以再次生成")
|
||||
if source_task.get("updated_at") != expected_updated_at:
|
||||
raise InvalidStateError("源任务已被其他操作修改,请刷新后重试")
|
||||
source_files = self.sources[source_task_id]
|
||||
if set(file_copies) != {str(item["id"]) for item in source_files}:
|
||||
raise InvalidStateError("源文件快照已变化,请刷新后重试")
|
||||
if not self.previews[source_task_id]:
|
||||
raise InvalidStateError("源任务没有可用于再次生成的切分结果")
|
||||
|
||||
repeated_task_id = repeat_task_id(source_task_id, request_id)
|
||||
suffix = f"(再次生成-{repeated_task_id[-6:]})"
|
||||
task = {
|
||||
**deepcopy(source_task),
|
||||
"id": repeated_task_id,
|
||||
"name": f"{source_task['name'][: max(1, 150 - len(suffix))]}{suffix}",
|
||||
"status": "pending",
|
||||
"progress": 20,
|
||||
"output_dataset_id": None,
|
||||
"output_datasets": [],
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"generation_run_id": None,
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_run_id": None,
|
||||
"preview_failure_reason": None,
|
||||
"preview_total_files": len(source_files),
|
||||
"preview_completed_files": len(source_files),
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
}
|
||||
self.tasks[repeated_task_id] = task
|
||||
self.sources[repeated_task_id] = []
|
||||
file_id_map: dict[str, str] = {}
|
||||
for source in source_files:
|
||||
old_file_id = str(source["id"])
|
||||
copy = file_copies[old_file_id]
|
||||
file_id_map[old_file_id] = copy["id"]
|
||||
self.sources[repeated_task_id].append(
|
||||
{
|
||||
**deepcopy(source),
|
||||
"id": copy["id"],
|
||||
"task_id": repeated_task_id,
|
||||
"storage_object_id": copy["storage_object_id"],
|
||||
}
|
||||
)
|
||||
self.previews[repeated_task_id] = [
|
||||
{
|
||||
**deepcopy(item),
|
||||
"id": self._id("dpp"),
|
||||
"task_id": repeated_task_id,
|
||||
"source_file_id": file_id_map.get(str(item.get("source_file_id")))
|
||||
if item.get("source_file_id")
|
||||
else None,
|
||||
}
|
||||
for item in self.previews[source_task_id]
|
||||
]
|
||||
self.results[repeated_task_id] = []
|
||||
self.repeat_requests[(source_task_id, request_id)] = repeated_task_id
|
||||
return self._repeat_response(
|
||||
source_task_id,
|
||||
repeated_task_id,
|
||||
created=True,
|
||||
)
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
del self.tasks[task_id]
|
||||
@@ -899,6 +1021,15 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||||
assert listed_preview.json()["data"]["total"] == 2
|
||||
preview_item = listed_preview.json()["data"]["items"][0]
|
||||
source_locator = preview_item["quality_score"]["source_locator"]
|
||||
assert source_locator == {
|
||||
"kind": "jsonl",
|
||||
"record_index": 1,
|
||||
"start_line": 1,
|
||||
"end_line": 1,
|
||||
"source_start": 0,
|
||||
"source_end": len(preview_item["original_content"]),
|
||||
}
|
||||
updated_preview = client.put(
|
||||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||||
json={
|
||||
@@ -907,6 +1038,7 @@ def test_data_process_full_contract_without_database(tmp_path: Path) -> None:
|
||||
},
|
||||
)
|
||||
assert "quality_score" in updated_preview.json()["data"]
|
||||
assert updated_preview.json()["data"]["quality_score"]["source_locator"] == source_locator
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
@@ -1760,6 +1892,118 @@ def test_regenerate_endpoint_prepares_an_existing_published_task(tmp_path: Path)
|
||||
assert [item["id"] for item in detail["output_datasets"]] == ["dataset_train"]
|
||||
|
||||
|
||||
def test_completed_task_can_repeat_into_an_independent_background_task(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
client, store, storage = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "原始生成任务",
|
||||
"process_type": "structured",
|
||||
"config": {"qa_pairs_per_row": 1, "temperature": 0.3},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("source.jsonl", b'{"name":"alpha"}\n', "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
source_id = uploaded.json()["data"]["files"][0]["id"]
|
||||
built = client.post(
|
||||
f"/modelTF/data-process/{task_id}/preview/build",
|
||||
json={"replace_existing": True},
|
||||
)
|
||||
assert built.status_code == 200
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
progress=100,
|
||||
results_confirmed=True,
|
||||
workflow_step="results",
|
||||
output_count=1,
|
||||
output_dataset_id="dataset-original",
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
store.results[task_id] = [{"id": "result-original", "output": "原结果"}]
|
||||
store.datasets["dataset-original"] = {
|
||||
"id": "dataset-original",
|
||||
"name": "原数据集",
|
||||
"type": "train",
|
||||
"source_task_id": task_id,
|
||||
"deleted_at": None,
|
||||
}
|
||||
original_task = deepcopy(store.tasks[task_id])
|
||||
original_sources = deepcopy(store.sources[task_id])
|
||||
original_previews = deepcopy(store.previews[task_id])
|
||||
original_results = deepcopy(store.results[task_id])
|
||||
original_datasets = deepcopy(store.datasets)
|
||||
monkeypatch.setattr(data_process_endpoint, "_run_generation", lambda *_: None)
|
||||
|
||||
payload = {
|
||||
"expected_updated_at": "2026-07-28T12:00:00Z",
|
||||
"request_id": "repeat-request-0001",
|
||||
}
|
||||
response = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
|
||||
assert response.status_code == 202
|
||||
repeated = response.json()["data"]
|
||||
repeated_task_id = repeated["task"]["id"]
|
||||
assert repeated["created"] is True
|
||||
assert repeated_task_id != task_id
|
||||
assert repeated["task"]["status"] == "running"
|
||||
assert repeated["task"]["workflow_step"] == "generate"
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == len(original_previews)
|
||||
assert store.tasks[task_id] == original_task
|
||||
assert store.sources[task_id] == original_sources
|
||||
assert store.previews[task_id] == original_previews
|
||||
assert store.results[task_id] == original_results
|
||||
assert store.datasets == original_datasets
|
||||
|
||||
repeated_source = store.sources[repeated_task_id][0]
|
||||
repeated_preview = store.previews[repeated_task_id][0]
|
||||
assert repeated_source["id"] != source_id
|
||||
assert repeated_source["storage_object_id"] != original_sources[0]["storage_object_id"]
|
||||
assert repeated_preview["id"] != original_previews[0]["id"]
|
||||
assert repeated_preview["source_file_id"] == repeated_source["id"]
|
||||
assert storage.read(repeated_source["storage_object_id"]) == b'{"name":"alpha"}\n'
|
||||
|
||||
replay = client.post(f"/modelTF/data-process/{task_id}/repeat", json=payload)
|
||||
assert replay.status_code == 202
|
||||
assert replay.json()["data"]["created"] is False
|
||||
assert replay.json()["data"]["task"]["id"] == repeated_task_id
|
||||
assert len(store.tasks) == 2
|
||||
assert len(store.sources[repeated_task_id]) == 1
|
||||
|
||||
|
||||
def test_repeat_rejects_a_stale_source_snapshot_without_creating_a_task(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, store, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "源任务", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id].update(
|
||||
status="completed",
|
||||
results_confirmed=True,
|
||||
updated_at="2026-07-28T12:00:00Z",
|
||||
)
|
||||
before = deepcopy(store.tasks)
|
||||
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/repeat",
|
||||
json={
|
||||
"expected_updated_at": "2026-07-28T11:59:59Z",
|
||||
"request_id": "repeat-request-stale",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 409
|
||||
assert store.tasks == before
|
||||
|
||||
|
||||
def test_published_split_datasets_remain_in_detail_after_regeneration(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -2112,6 +2356,91 @@ def test_preprocess_deduplicates_and_quality_filter_removes_short_results(
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||||
|
||||
|
||||
def test_structured_deduplication_preserves_distinct_rows_after_desensitization(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "先去重再脱敏",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"names.jsonl",
|
||||
(
|
||||
'{"name":"张三","role":"开发"}\n'
|
||||
'{"name":"李四","role":"开发"}\n'
|
||||
),
|
||||
"application/jsonl",
|
||||
)
|
||||
},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert len(items) == 2
|
||||
assert len({item["original_content"] for item in items}) == 2
|
||||
assert {item["edited_content"] for item in items} == {
|
||||
'{"name":"[NAME]","role":"开发"}'
|
||||
}
|
||||
|
||||
|
||||
def test_structured_deduplication_removes_identical_rows_across_sources(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "跨源原文去重",
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": ["deduplicate", "desensitize"]},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"first.jsonl",
|
||||
'{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
(
|
||||
"files",
|
||||
(
|
||||
"second.jsonl",
|
||||
'\n{"name":"张三","role":"开发"}\n',
|
||||
"application/jsonl",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
first_source, second_source = uploaded.json()["data"]["files"]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
|
||||
assert preview.status_code == 200
|
||||
data = preview.json()["data"]
|
||||
assert data["total"] == 1
|
||||
assert data["file_counts"] == {
|
||||
first_source["id"]: 1,
|
||||
second_source["id"]: 0,
|
||||
}
|
||||
|
||||
|
||||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||||
store = FakeDataProcessStore()
|
||||
task = store.create_task(
|
||||
@@ -2364,7 +2693,26 @@ def test_xlsx_upload_is_accepted_as_structured_records(tmp_path: Path) -> None:
|
||||
)
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
preview_items = preview.json()["data"]["items"]
|
||||
assert len(preview_items) == 2
|
||||
assert [item["quality_score"]["source_locator"] for item in preview_items] == [
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 1,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 2,
|
||||
"sheet_record_index": 0,
|
||||
},
|
||||
{
|
||||
"kind": "xlsx",
|
||||
"record_index": 2,
|
||||
"sheet_index": 0,
|
||||
"sheet_name": "Sheet",
|
||||
"row_number": 3,
|
||||
"sheet_record_index": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_docx_preview_preserves_document_block_order_and_source_offsets(
|
||||
@@ -2757,6 +3105,218 @@ def _preview_task(
|
||||
)
|
||||
|
||||
|
||||
def _structured_preview_task(
|
||||
content: str,
|
||||
*,
|
||||
file_format: str,
|
||||
options: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return data_process_endpoint._build_preview_items(
|
||||
{
|
||||
"process_type": "structured",
|
||||
"config": {"preprocess_options": options or []},
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "structured-source",
|
||||
"name": f"records.{file_format}",
|
||||
"file_format": file_format,
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_structured_preview_exposes_json_jsonl_and_csv_source_locators() -> None:
|
||||
json_source = '{"records":[{"id":1},{"id":2}]}'
|
||||
json_items = _structured_preview_task(
|
||||
json_source,
|
||||
file_format="json",
|
||||
)
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"]
|
||||
for item in json_items
|
||||
] == ["/records/0", "/records/1"]
|
||||
assert [
|
||||
json_source[item["source_start"] : item["source_end"]]
|
||||
for item in json_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
assert [item["source_start_line"] for item in json_items] == [1, 1]
|
||||
|
||||
jsonl_source = '{"id":1}\n\n{"id":2}'
|
||||
jsonl_items = _structured_preview_task(jsonl_source, file_format="jsonl")
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in jsonl_items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in jsonl_items] == [1, 3]
|
||||
assert [
|
||||
jsonl_source[item["source_start"] : item["source_end"]]
|
||||
for item in jsonl_items
|
||||
] == ['{"id":1}', '{"id":2}']
|
||||
|
||||
csv_source = 'id,note\n1,"hello\nworld"\n\n2,plain'
|
||||
csv_items = _structured_preview_task(csv_source, file_format="csv")
|
||||
assert [
|
||||
(item["source_start_line"], item["source_end_line"])
|
||||
for item in csv_items
|
||||
] == [(2, 3), (5, 5)]
|
||||
assert [
|
||||
csv_source[item["source_start"] : item["source_end"]]
|
||||
for item in csv_items
|
||||
] == ['1,"hello\nworld"', "2,plain"]
|
||||
|
||||
|
||||
def test_structured_empty_json_upload_and_preview_remain_empty(tmp_path: Path) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "空 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("empty-array.json", "[]", "application/json")),
|
||||
(
|
||||
"files",
|
||||
("empty-wrapper.json", '{"records":[],"total":0}', "application/json"),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [0, 0]
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["items"] == []
|
||||
assert preview.json()["data"]["total"] == 0
|
||||
assert set(preview.json()["data"]["file_counts"].values()) == {0}
|
||||
|
||||
|
||||
def test_structured_json_upload_rejects_ambiguous_or_invalid_numbers(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "严格 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
invalid_sources = (
|
||||
("duplicate.json", '{"id":1,"id":2}'),
|
||||
("duplicate.jsonl", '{"id":1,"id":2}\n'),
|
||||
("nan.json", '{"value":NaN}'),
|
||||
("infinity.json", '{"value":Infinity}'),
|
||||
("control.json", '{"value":"bad\x00control"}'),
|
||||
("deep.json", "[" * 10_000 + "0" + "]" * 10_000),
|
||||
)
|
||||
|
||||
for filename, content in invalid_sources:
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": (filename, content, "application/json")},
|
||||
)
|
||||
assert response.status_code == 400, (filename, response.text)
|
||||
|
||||
|
||||
def test_structured_json_preview_preserves_precision_and_business_data_field(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
client, _, _ = make_client(tmp_path)
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "无损 JSON", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
precise = '{"value":0.123456789012345678901234567890}'
|
||||
business = '{"id":7,"data":[{"id":8}]}'
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("precise.json", precise, "application/json")),
|
||||
("files", ("business.json", business, "application/json")),
|
||||
],
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
assert [item["record_count"] for item in uploaded.json()["data"]["files"]] == [1, 1]
|
||||
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.status_code == 200
|
||||
items = preview.json()["data"]["items"]
|
||||
assert [item["original_content"] for item in items] == [precise, business]
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["json_pointer"] for item in items
|
||||
] == ["", ""]
|
||||
assert [item["source_start"] for item in items] == [0, 0]
|
||||
|
||||
|
||||
def test_structured_preview_lineage_survives_clean_deduplicate_and_filter() -> None:
|
||||
source_records = [
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "A", "amount": 10, "empty": ""},
|
||||
{"id": "", "amount": 11, "empty": ""},
|
||||
{"id": "B", "amount": 11, "empty": ""},
|
||||
{"id": "C", "amount": 12, "empty": ""},
|
||||
{"id": "D", "amount": 12, "empty": ""},
|
||||
{"id": "E", "amount": 13, "empty": ""},
|
||||
{"id": "F", "amount": 13, "empty": ""},
|
||||
{"id": "G", "amount": 14, "empty": ""},
|
||||
{"id": "H", "amount": 1000, "empty": ""},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate", "filter_anomaly"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [item["source_start_line"] for item in items] == [1, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert [json.loads(item["original_content"])["id"] for item in items] == [
|
||||
"A",
|
||||
"",
|
||||
"B",
|
||||
"C",
|
||||
"D",
|
||||
"E",
|
||||
"F",
|
||||
"G",
|
||||
]
|
||||
|
||||
|
||||
def test_structured_preview_deduplicates_exact_rows_not_matching_identifiers() -> None:
|
||||
source_records = [
|
||||
{"customer_id": "C-1", "status": "old"},
|
||||
{"customer_id": "C-1", "status": "new"},
|
||||
{"status": "old", "customer_id": "C-1"},
|
||||
]
|
||||
source = "\n".join(
|
||||
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
for record in source_records
|
||||
)
|
||||
|
||||
items = _structured_preview_task(
|
||||
source,
|
||||
file_format="jsonl",
|
||||
options=["clean_invalid", "deduplicate"],
|
||||
)
|
||||
|
||||
assert [
|
||||
item["quality_score"]["source_locator"]["record_index"]
|
||||
for item in items
|
||||
] == [1, 2]
|
||||
assert [item["source_start_line"] for item in items] == [1, 2]
|
||||
assert [json.loads(item["original_content"])["status"] for item in items] == [
|
||||
"old",
|
||||
"new",
|
||||
]
|
||||
|
||||
|
||||
def test_fixed_preview_preserves_source_offsets() -> None:
|
||||
content = (
|
||||
"# 第一章\n"
|
||||
|
||||
@@ -63,6 +63,40 @@ def test_stage_publish_read_delete_roundtrip_with_unicode_filename(tmp_path: Pat
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_stage_copy_creates_an_independently_deletable_source_object(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
original = _stage(storage, content=b"immutable source")
|
||||
storage.publish([original])
|
||||
|
||||
copied = storage.stage_copy(
|
||||
batch_id="batch-copy",
|
||||
source_reference=original.reference,
|
||||
expected_source_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
task_id="task-2",
|
||||
source_file_id="source-2",
|
||||
version=1,
|
||||
name="source.txt",
|
||||
)
|
||||
storage.publish([copied])
|
||||
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
original.reference,
|
||||
expected_task_id="task-1",
|
||||
expected_source_file_id="source-1",
|
||||
) is True
|
||||
assert storage.read(copied.reference) == b"immutable source"
|
||||
assert storage.delete(
|
||||
copied.reference,
|
||||
expected_task_id="task-2",
|
||||
expected_source_file_id="source-2",
|
||||
) is True
|
||||
_assert_staging_empty(storage)
|
||||
|
||||
|
||||
def test_db_reference_is_left_to_database_storage(tmp_path: Path) -> None:
|
||||
storage = LocalDataProcessStorage(tmp_path / "storage")
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.modules.data_process.store import (
|
||||
_preview_config_changed,
|
||||
_reasoning_output_is_valid,
|
||||
_source_storage_descriptor,
|
||||
repeat_task_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -331,6 +332,140 @@ class _TaskDetailStore(DataProcessStore):
|
||||
yield self._conn
|
||||
|
||||
|
||||
class _RepeatConnection:
|
||||
def __init__(self) -> None:
|
||||
self.source_files = [
|
||||
{
|
||||
"id": "source-old",
|
||||
"name": "source.jsonl",
|
||||
"size_bytes": 12,
|
||||
"record_count": 1,
|
||||
"file_format": "jsonl",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"content": '{"id":1}\n',
|
||||
"content_preview": '{"id":1}',
|
||||
"metadata": {"storage_backend": "local"},
|
||||
"created_by": "user-1",
|
||||
}
|
||||
]
|
||||
self.source_previews = [
|
||||
{
|
||||
"id": "preview-old",
|
||||
"source_file_id": "source-old",
|
||||
"original_content": '{"id":1}',
|
||||
"edited_content": '{"id":1,"checked":true}',
|
||||
"source_start": 0,
|
||||
"source_end": 8,
|
||||
"source_start_line": 1,
|
||||
"source_end_line": 1,
|
||||
"token_count": 5,
|
||||
"status": "modified",
|
||||
"quality_score": {"overall": 90},
|
||||
}
|
||||
]
|
||||
self.created_task: dict[str, Any] | None = None
|
||||
self.created_files: list[dict[str, Any]] = []
|
||||
self.created_previews: list[dict[str, Any]] = []
|
||||
|
||||
def execute(self, sql: str, params: Any = None) -> _Result:
|
||||
normalized = " ".join(sql.split())
|
||||
if params is not None:
|
||||
assert normalized.count("%s") == len(params)
|
||||
if normalized.startswith("SELECT * FROM data_process_tasks WHERE id="):
|
||||
return _Result(row=None)
|
||||
if normalized.startswith("SELECT * FROM data_process_source_files"):
|
||||
return _Result(rows=[dict(item) for item in self.source_files])
|
||||
if normalized.startswith("SELECT * FROM data_process_preview_items"):
|
||||
return _Result(rows=[dict(item) for item in self.source_previews])
|
||||
if normalized.startswith("INSERT INTO data_process_tasks"):
|
||||
self.created_task = {
|
||||
"id": params[0],
|
||||
"name": params[1],
|
||||
"description": params[2],
|
||||
"status": "pending",
|
||||
"process_type": params[3],
|
||||
"source_dataset_id": params[4],
|
||||
"config": params[5],
|
||||
"progress": 20,
|
||||
"input_count": params[6],
|
||||
"results_confirmed": False,
|
||||
"workflow_step": "preview",
|
||||
"preview_status": "completed",
|
||||
"preview_progress": 100,
|
||||
"preview_total_files": params[7],
|
||||
"preview_completed_files": params[8],
|
||||
"created_at": params[15],
|
||||
"updated_at": params[16],
|
||||
}
|
||||
return _Result(row=dict(self.created_task))
|
||||
if normalized.startswith("INSERT INTO data_process_source_files"):
|
||||
self.created_files.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"storage_object_id": params[2],
|
||||
"content": params[8],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("INSERT INTO data_process_preview_items"):
|
||||
self.created_previews.append(
|
||||
{
|
||||
"id": params[0],
|
||||
"task_id": params[1],
|
||||
"source_file_id": params[2],
|
||||
"edited_content": params[4],
|
||||
}
|
||||
)
|
||||
return _Result()
|
||||
if normalized.startswith("SELECT (SELECT COUNT(*) FROM data_process_source_files"):
|
||||
return _Result(
|
||||
row={
|
||||
"source_file_count": len(self.created_files),
|
||||
"preview_count": len(self.created_previews),
|
||||
}
|
||||
)
|
||||
raise AssertionError(f"unexpected SQL: {normalized}")
|
||||
|
||||
|
||||
class _RepeatStore(DataProcessStore):
|
||||
def __init__(self, conn: _RepeatConnection) -> None:
|
||||
self._conn = conn
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[_RepeatConnection]:
|
||||
yield self._conn
|
||||
|
||||
def _task_in_connection(
|
||||
self,
|
||||
conn: Any,
|
||||
task_id: str,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
assert task_id == "task-source"
|
||||
assert for_update is True
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": "原任务",
|
||||
"description": "原描述",
|
||||
"status": "completed",
|
||||
"process_type": "structured",
|
||||
"source_dataset_id": None,
|
||||
"config": {
|
||||
"temperature": 0.3,
|
||||
"_regeneration_prepared": {"prepared": True},
|
||||
},
|
||||
"results_confirmed": True,
|
||||
"preview_status": "completed",
|
||||
"tenant_id": "tenant-1",
|
||||
"project_id": "project-1",
|
||||
"owner_id": "owner-1",
|
||||
"created_by": "user-1",
|
||||
"updated_at": "2026-07-28T12:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
class _TaskListConnection:
|
||||
def __init__(self) -> None:
|
||||
self.task = {
|
||||
@@ -574,6 +709,47 @@ def test_decode_row_serializes_postgres_numeric_values_as_json_numbers() -> None
|
||||
assert decoded == {"progress": 100.0, "duration_seconds": 389.0}
|
||||
|
||||
|
||||
def test_repeat_task_copies_business_snapshot_with_new_resource_ids() -> None:
|
||||
conn = _RepeatConnection()
|
||||
store = _RepeatStore(conn)
|
||||
request_id = "repeat-request-0001"
|
||||
target_task_id = repeat_task_id("task-source", request_id)
|
||||
|
||||
repeated = store.repeat_task(
|
||||
"task-source",
|
||||
expected_updated_at="2026-07-28T12:00:00Z",
|
||||
request_id=request_id,
|
||||
file_copies={
|
||||
"source-old": {
|
||||
"id": "source-new",
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert repeated["created"] is True
|
||||
assert repeated["task"]["id"] == target_task_id
|
||||
assert repeated["task"]["config"] == {"temperature": 0.3}
|
||||
assert repeated["task"]["results_confirmed"] is False
|
||||
assert repeated["copied_source_file_count"] == 1
|
||||
assert repeated["copied_preview_count"] == 1
|
||||
assert conn.created_files == [
|
||||
{
|
||||
"id": "source-new",
|
||||
"task_id": target_task_id,
|
||||
"storage_object_id": (
|
||||
f"local://data-process/{target_task_id}/source-new/v1/source.jsonl"
|
||||
),
|
||||
"content": '{"id":1}\n',
|
||||
}
|
||||
]
|
||||
assert conn.created_previews[0]["task_id"] == target_task_id
|
||||
assert conn.created_previews[0]["source_file_id"] == "source-new"
|
||||
assert conn.created_previews[0]["edited_content"] == '{"id":1,"checked":true}'
|
||||
|
||||
|
||||
def test_decode_row_decodes_aggregated_output_datasets_json() -> None:
|
||||
decoded = _decode_row(
|
||||
{
|
||||
|
||||
774
backend/tests/test_governance.py
Normal file
774
backend/tests/test_governance.py
Normal file
@@ -0,0 +1,774 @@
|
||||
"""
|
||||
平台治理功能集成测试 —— 覆盖第 1-4 周交付内容。
|
||||
|
||||
测试策略:
|
||||
- 在导入 app 模块前 mock psycopg / psycopg_pool,避免依赖真实数据库驱动
|
||||
- 使用 FastAPI TestClient 对真实路由栈发起请求
|
||||
- 通过 mock.get_platform_store 替换为内存 FakeStore
|
||||
- 每周交付内容对应一组 test class,方便分阶段验收
|
||||
|
||||
覆盖范围:
|
||||
第 1 周 — 登录、当前用户、用户列表、权限码、日志查询
|
||||
第 2 周 — 租户、项目、项目成员、资源 ACL
|
||||
第 3 周 — 审批实例、审批模板、审计日志查询和导出
|
||||
第 4 周 — 写操作审计、审批拦截、权限校验
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Iterator
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# ============================================================
|
||||
# 在导入 app 之前 mock psycopg / psycopg_pool
|
||||
# ============================================================
|
||||
|
||||
_psycopg_mock = types.ModuleType("psycopg")
|
||||
_psycopg_mock.PgConn = type("PgConn", (), {})
|
||||
_psycopg_mock.PostgresConnectionPool = MagicMock()
|
||||
_psycopg_mock.connection = MagicMock()
|
||||
sys.modules.setdefault("psycopg", _psycopg_mock)
|
||||
|
||||
_psycopg_pool_mock = types.ModuleType("psycopg_pool")
|
||||
_psycopg_pool_mock.ConnectionPool = MagicMock()
|
||||
sys.modules.setdefault("psycopg_pool", _psycopg_pool_mock)
|
||||
|
||||
# 现在安全导入 app 模块
|
||||
from app.api.v1.endpoints.platform import ok, fail # noqa: E402
|
||||
from app.modules.tenant.router import router as tenant_router # noqa: E402
|
||||
from app.modules.project.router import router as project_router # noqa: E402
|
||||
from app.modules.approval.router import router as approval_router # noqa: E402
|
||||
from app.modules.system.router import router as system_router # noqa: E402
|
||||
from app.modules.retention.router import router as retention_router # noqa: E402
|
||||
from app.modules.resource.router import router as resource_router # noqa: E402
|
||||
from app.api.v1.endpoints.platform import router as platform_router # noqa: E402
|
||||
|
||||
PREFIX = "/modelTF"
|
||||
ADMIN_TOKEN = "platform-token-u_admin"
|
||||
OP_TOKEN = "platform-token-u_op"
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FakePlatformStore —— 内存实现,模拟 PlatformStore 全部治理接口
|
||||
# ============================================================
|
||||
|
||||
class FakePlatformStore:
|
||||
"""平台治理测试专用内存 store,确保测试不连接真实数据库。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._users: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "u_admin",
|
||||
"username": "admin",
|
||||
"display_name": "Admin",
|
||||
"role": "admin",
|
||||
"status": "active",
|
||||
"permissions": [
|
||||
"dashboard", "fine-tune", "model-eval", "model-inference",
|
||||
"model-manage", "dataset", "data-process", "data-convert",
|
||||
"compute", "hardware", "logs", "user-settings",
|
||||
],
|
||||
"last_login": "2026-08-01T10:00:00Z",
|
||||
"protected": True,
|
||||
},
|
||||
{
|
||||
"id": "u_op",
|
||||
"username": "operator",
|
||||
"display_name": "Operator",
|
||||
"role": "operator",
|
||||
"status": "active",
|
||||
"permissions": ["dashboard", "fine-tune"],
|
||||
"last_login": "2026-08-01T11:00:00Z",
|
||||
"protected": False,
|
||||
},
|
||||
]
|
||||
self._tenants: dict[str, dict[str, Any]] = {}
|
||||
self._projects: dict[str, dict[str, Any]] = {}
|
||||
self._members: dict[str, list[dict[str, Any]]] = {}
|
||||
self._acl: dict[str, list[dict[str, Any]]] = {}
|
||||
self._audit_logs: list[dict[str, Any]] = []
|
||||
self._approval_templates: dict[str, dict[str, Any]] = {}
|
||||
self._approval_instances: dict[str, dict[str, Any]] = {}
|
||||
self._retention_policies: dict[str, dict[str, Any]] = {}
|
||||
self._models: list[dict[str, Any]] = []
|
||||
self._datasets: list[dict[str, Any]] = []
|
||||
self._tasks: list[dict[str, Any]] = []
|
||||
self._compute_nodes: list[dict[str, Any]] = []
|
||||
self._gpus: list[dict[str, Any]] = []
|
||||
self._sessions: list[dict[str, Any]] = []
|
||||
self._seq = 0
|
||||
|
||||
@contextmanager
|
||||
def connect(self) -> Iterator[Any]:
|
||||
class FakeConn:
|
||||
def execute(self, *a, **kw):
|
||||
return []
|
||||
|
||||
def commit(self):
|
||||
pass
|
||||
|
||||
def rollback(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
yield FakeConn()
|
||||
|
||||
# ---- helpers ----
|
||||
|
||||
def _next_id(self, prefix: str) -> str:
|
||||
self._seq += 1
|
||||
return f"{prefix}_{self._seq}"
|
||||
|
||||
# ==================== 第1周:登录 / 用户 / 权限码 / 日志 ====================
|
||||
|
||||
def login(self, username: str, password: str) -> dict[str, Any] | None:
|
||||
for u in self._users:
|
||||
if u["username"] == username and u["status"] == "active":
|
||||
if password in ("admin123", "operator123", "test123"):
|
||||
return dict(u)
|
||||
return None
|
||||
|
||||
def users(self) -> list[dict[str, Any]]:
|
||||
return [dict(u) for u in self._users]
|
||||
|
||||
def create_user(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
u = {"id": self._next_id("u"), "protected": False, **payload}
|
||||
self._users.append(u)
|
||||
return u
|
||||
|
||||
def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
for u in self._users:
|
||||
if u["id"] == user_id:
|
||||
u.update(payload)
|
||||
return u
|
||||
raise KeyError(user_id)
|
||||
|
||||
def delete_user(self, user_id: str) -> None:
|
||||
self._users = [u for u in self._users if u["id"] != user_id]
|
||||
|
||||
def roles(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": "admin", "display_name": "管理员"},
|
||||
{"name": "operator", "display_name": "操作员"},
|
||||
{"name": "viewer", "display_name": "访客"},
|
||||
]
|
||||
|
||||
def log_files(self, date: str | None = None) -> list[dict[str, Any]]:
|
||||
return [{"name": "backend-2026-08-01.log", "size": "1 KB", "date": "2026-08-01"}]
|
||||
|
||||
def log_content(self, file: str) -> dict[str, Any]:
|
||||
return {"file": file, "content": "[INFO] test line", "size": "1 KB"}
|
||||
|
||||
def training_log_files(self) -> list[dict[str, Any]]:
|
||||
return [{"task_id": "ft_001", "name": "ft_001.log", "size": "2 KB"}]
|
||||
|
||||
def training_log_content(self, file: str) -> dict[str, Any]:
|
||||
return {"file": file, "content": "epoch 0 loss 1.0", "size": "2 KB"}
|
||||
|
||||
# ==================== 第2周:租户 / 项目 / 成员 / ACL ====================
|
||||
|
||||
def tenants(self) -> list[dict[str, Any]]:
|
||||
return list(self._tenants.values())
|
||||
|
||||
def tenant(self, tenant_id: str) -> dict[str, Any]:
|
||||
if tenant_id not in self._tenants:
|
||||
raise KeyError(tenant_id)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def create_tenant(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
tid = self._next_id("tnt")
|
||||
t = {"id": tid, "status": "active", "quota": "{}", "retention_policy_id": None,
|
||||
"create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._tenants[tid] = t
|
||||
return dict(t)
|
||||
|
||||
def update_tenant(self, tenant_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._tenants[tenant_id].update(payload)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def set_tenant_quota(self, tenant_id: str, quota: dict[str, Any]) -> dict[str, Any]:
|
||||
self._tenants[tenant_id]["quota"] = json.dumps(quota)
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def set_tenant_retention(self, tenant_id: str, retention_policy_id: str | None) -> dict[str, Any]:
|
||||
self._tenants[tenant_id]["retention_policy_id"] = retention_policy_id
|
||||
return dict(self._tenants[tenant_id])
|
||||
|
||||
def projects(self, *, tenant_id: str = "default", status: str | None = None, keyword: str | None = None) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for p in self._projects.values():
|
||||
if p.get("tenant_id") != tenant_id:
|
||||
continue
|
||||
if status and p.get("status") != status:
|
||||
continue
|
||||
if keyword and keyword.lower() not in p.get("name", "").lower():
|
||||
continue
|
||||
result.append(dict(p))
|
||||
return result
|
||||
|
||||
def project(self, project_id: str) -> dict[str, Any]:
|
||||
if project_id not in self._projects:
|
||||
raise KeyError(project_id)
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def create_project(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
pid = self._next_id("prj")
|
||||
p = {"id": pid, "status": "active", "quota": "{}", "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._projects[pid] = p
|
||||
self._members[pid] = []
|
||||
return dict(p)
|
||||
|
||||
def update_project(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._projects[project_id].update(payload)
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def archive_project(self, project_id: str) -> dict[str, Any]:
|
||||
self._projects[project_id]["status"] = "archived"
|
||||
return dict(self._projects[project_id])
|
||||
|
||||
def delete_project(self, project_id: str) -> None:
|
||||
self._projects.pop(project_id, None)
|
||||
self._members.pop(project_id, None)
|
||||
|
||||
def project_members(self, project_id: str) -> list[dict[str, Any]]:
|
||||
return [dict(m) for m in self._members.get(project_id, [])]
|
||||
|
||||
def add_project_member(self, project_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
m = {"joined_at": "2026-08-01T00:00:00Z", **payload}
|
||||
self._members.setdefault(project_id, []).append(m)
|
||||
return m
|
||||
|
||||
def update_project_member_role(self, project_id: str, user_id: str, role: str) -> dict[str, Any]:
|
||||
for m in self._members.get(project_id, []):
|
||||
if m["user_id"] == user_id:
|
||||
m["role"] = role
|
||||
return m
|
||||
raise KeyError(user_id)
|
||||
|
||||
def remove_project_member(self, project_id: str, user_id: str) -> None:
|
||||
self._members[project_id] = [m for m in self._members.get(project_id, []) if m["user_id"] != user_id]
|
||||
|
||||
# ---- ACL ----
|
||||
|
||||
def get_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
key = f"{resource_type}:{resource_id}"
|
||||
return [dict(a) for a in self._acl.get(key, [])]
|
||||
|
||||
def set_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
key = f"{resource_type}:{resource_id}"
|
||||
self._acl[key] = [dict(e) for e in entries]
|
||||
return self.get_acl(resource_type, resource_id)
|
||||
|
||||
def resource_acl(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||
rows = self.get_acl(resource_type, resource_id)
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
k = f"{r.get('principal_type')}:{r.get('principal_id')}"
|
||||
bucket = grouped.setdefault(k, {
|
||||
"subject_type": r.get("principal_type"),
|
||||
"subject_id": r.get("principal_id"),
|
||||
"permissions": [],
|
||||
})
|
||||
perm = r.get("permission")
|
||||
if perm and perm not in bucket["permissions"]:
|
||||
bucket["permissions"].append(perm)
|
||||
return list(grouped.values())
|
||||
|
||||
def set_resource_acl(self, resource_type: str, resource_id: str, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
flat: list[dict[str, Any]] = []
|
||||
for e in entries:
|
||||
for perm in e.get("permissions") or []:
|
||||
flat.append({
|
||||
"principal_type": e.get("subject_type"),
|
||||
"principal_id": e.get("subject_id"),
|
||||
"permission": perm,
|
||||
})
|
||||
self.set_acl(resource_type, resource_id, flat)
|
||||
return self.resource_acl(resource_type, resource_id)
|
||||
|
||||
# ==================== 第3周:审批 / 审计 / 留存 ====================
|
||||
|
||||
def approval_templates(self) -> list[dict[str, Any]]:
|
||||
return list(self._approval_templates.values())
|
||||
|
||||
def create_approval_template(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
tid = payload.get("id") or self._next_id("tpl")
|
||||
t = {"id": tid, "steps": [], "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._approval_templates[tid] = t
|
||||
return dict(t)
|
||||
|
||||
def approval_instances(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for i in self._approval_instances.values():
|
||||
if status and i.get("status") != status:
|
||||
continue
|
||||
result.append(dict(i))
|
||||
return result
|
||||
|
||||
def approval_instance(self, instance_id: str) -> dict[str, Any]:
|
||||
if instance_id not in self._approval_instances:
|
||||
raise KeyError(instance_id)
|
||||
return dict(self._approval_instances[instance_id])
|
||||
|
||||
def create_approval_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
iid = self._next_id("appr")
|
||||
inst = {
|
||||
"id": iid,
|
||||
"status": "pending",
|
||||
"current_step": 0,
|
||||
"steps": [],
|
||||
"create_time": "2026-08-01T00:00:00Z",
|
||||
**payload,
|
||||
}
|
||||
self._approval_instances[iid] = inst
|
||||
return dict(inst)
|
||||
|
||||
def decide_approval_step(self, instance_id: str, step_index: int, *, approver_id: str, approved: bool, comment: str | None = None) -> dict[str, Any]:
|
||||
inst = self._approval_instances[instance_id]
|
||||
inst["status"] = "approved" if approved else "rejected"
|
||||
inst["current_step"] = step_index + 1
|
||||
return dict(inst)
|
||||
|
||||
def audit_logs(self, **kw) -> dict[str, Any]:
|
||||
items = [dict(l) for l in self._audit_logs]
|
||||
for filter_key in ("tenant_id", "project_id", "actor_id", "action", "target_type"):
|
||||
val = kw.get(filter_key)
|
||||
if val:
|
||||
items = [l for l in items if l.get(filter_key) == val]
|
||||
limit = kw.get("limit", 50)
|
||||
offset = kw.get("offset", 0)
|
||||
total = len(items)
|
||||
items = items[offset:offset + limit]
|
||||
return {"items": items, "total": total}
|
||||
|
||||
def record_audit(self, **kw) -> None:
|
||||
log = {"id": self._next_id("log"), "time": "2026-08-01T12:00:00Z", **kw}
|
||||
self._audit_logs.append(log)
|
||||
|
||||
# ---- 留存策略 ----
|
||||
|
||||
def retention_policies(self) -> list[dict[str, Any]]:
|
||||
return list(self._retention_policies.values())
|
||||
|
||||
def retention_policy(self, policy_id: str) -> dict[str, Any]:
|
||||
if policy_id not in self._retention_policies:
|
||||
raise KeyError(policy_id)
|
||||
return dict(self._retention_policies[policy_id])
|
||||
|
||||
def create_retention_policy(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
pid = payload.get("id") or self._next_id("rpol")
|
||||
p = {"id": pid, "status": "active", "create_time": "2026-08-01T00:00:00Z", **payload}
|
||||
self._retention_policies[pid] = p
|
||||
return dict(p)
|
||||
|
||||
def update_retention_policy(self, policy_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self._retention_policies[policy_id].update(payload)
|
||||
return dict(self._retention_policies[policy_id])
|
||||
|
||||
def delete_retention_policy(self, policy_id: str) -> None:
|
||||
self._retention_policies.pop(policy_id, None)
|
||||
|
||||
# ---- dashboard & other stubs ----
|
||||
|
||||
def login_duration_rank(self, limit: int = 8, days: int = 30) -> list[dict[str, Any]]:
|
||||
return [{"user": "admin", "role": "admin", "duration": 10.0}]
|
||||
|
||||
def models(self) -> list[dict[str, Any]]:
|
||||
return self._models
|
||||
|
||||
def datasets(self) -> list[dict[str, Any]]:
|
||||
return self._datasets
|
||||
|
||||
def tasks(self) -> list[dict[str, Any]]:
|
||||
return self._tasks
|
||||
|
||||
def compute_nodes(self) -> list[dict[str, Any]]:
|
||||
return self._compute_nodes
|
||||
|
||||
def gpus(self) -> list[dict[str, Any]]:
|
||||
return self._gpus
|
||||
|
||||
def system_info(self) -> dict[str, Any]:
|
||||
return {"cpu": {}, "memory": {}}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 测试 fixtures
|
||||
# ============================================================
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def fake_store() -> FakePlatformStore:
|
||||
return FakePlatformStore()
|
||||
|
||||
|
||||
def _build_client(store: FakePlatformStore) -> TestClient:
|
||||
"""构建 TestClient,patch 所有治理模块的 get_platform_store。"""
|
||||
app = FastAPI()
|
||||
app.include_router(platform_router, prefix=PREFIX)
|
||||
app.include_router(system_router, prefix=PREFIX)
|
||||
app.include_router(tenant_router, prefix=PREFIX)
|
||||
app.include_router(project_router, prefix=PREFIX)
|
||||
app.include_router(approval_router, prefix=PREFIX)
|
||||
app.include_router(retention_router, prefix=PREFIX)
|
||||
app.include_router(resource_router, prefix=PREFIX)
|
||||
|
||||
patches = [
|
||||
patch("app.db.platform_store.get_platform_store", return_value=store),
|
||||
patch("app.core.auth.get_platform_store", return_value=store),
|
||||
patch("app.api.v1.endpoints.platform.get_platform_store", return_value=store),
|
||||
patch("app.modules.system.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.tenant.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.project.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.approval.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.retention.router.get_platform_store", return_value=store),
|
||||
patch("app.modules.resource.router.get_platform_store", return_value=store),
|
||||
]
|
||||
for p in patches:
|
||||
p.start()
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
client._fake_store = store # type: ignore[attr-defined]
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client(fake_store: FakePlatformStore) -> TestClient:
|
||||
c = _build_client(fake_store)
|
||||
yield c
|
||||
|
||||
|
||||
def _admin_headers() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {ADMIN_TOKEN}"}
|
||||
|
||||
|
||||
def _op_headers() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {OP_TOKEN}"}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 1 周测试:登录、当前用户、用户列表、权限码、日志查询
|
||||
# ============================================================
|
||||
|
||||
class TestWeek1AuthUserPermissionsLogs:
|
||||
"""第 1 周:登录、当前用户、用户列表、权限码、日志查询接口。"""
|
||||
|
||||
def test_login_success(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "admin123"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert data["token"] == ADMIN_TOKEN
|
||||
assert data["user"]["username"] == "admin"
|
||||
|
||||
def test_login_invalid(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/login", json={"username": "admin", "password": "wrong"})
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_me_with_valid_token(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/me", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["username"] == "admin"
|
||||
|
||||
def test_me_without_token(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/me")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_users_list(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/users", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
users = resp.json()["data"]
|
||||
assert len(users) >= 2
|
||||
assert any(u["username"] == "admin" for u in users)
|
||||
|
||||
def test_create_user(self, client: TestClient):
|
||||
resp = client.post(
|
||||
f"{PREFIX}/users",
|
||||
json={"username": "tester", "display_name": "Tester", "role": "viewer", "password": "test123"},
|
||||
headers=_admin_headers(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["username"] == "tester"
|
||||
|
||||
def test_permission_codes(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/permissions/codes")
|
||||
assert resp.status_code == 200
|
||||
codes = resp.json()["data"]["codes"]
|
||||
assert "dashboard" in codes
|
||||
assert "user-settings" in codes
|
||||
|
||||
def test_permissions_overview(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/permissions")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "codes" in data
|
||||
assert "roles" in data
|
||||
|
||||
def test_log_files(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/log-files", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
files = resp.json()["data"]
|
||||
assert len(files) >= 1
|
||||
|
||||
def test_log_content(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/log-content", params={"file": "backend.log"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "content" in resp.json()["data"]
|
||||
|
||||
def test_training_log_files(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/training-log-files", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) >= 1
|
||||
|
||||
def test_training_log_content(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/training-log-content", params={"file": "ft_001.log"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "content" in resp.json()["data"]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 2 周测试:租户、项目、项目成员、资源 ACL
|
||||
# ============================================================
|
||||
|
||||
class TestWeek2TenantProjectACL:
|
||||
"""第 2 周:租户、项目、项目成员、资源 ACL。"""
|
||||
|
||||
def test_tenant_crud(self, client: TestClient):
|
||||
# 创建
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "Tenant-A", "code": "ta"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
tid = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/tenants", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(t["id"] == tid for t in resp.json()["data"])
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/tenants/{tid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Tenant-A"
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}", json={"name": "Tenant-A2"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Tenant-A2"
|
||||
|
||||
def test_tenant_quota(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "Q-Tenant", "code": "qt"}, headers=_admin_headers())
|
||||
tid = resp.json()["data"]["id"]
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}/quota", json={"quota": {"gpu": 4}}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_tenant_retention(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/tenants", json={"name": "R-Tenant", "code": "rt"}, headers=_admin_headers())
|
||||
tid = resp.json()["data"]["id"]
|
||||
resp = client.put(f"{PREFIX}/tenants/{tid}/retention-policy", json={"retention_policy_id": "rpol_1"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_project_crud(self, client: TestClient):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-1", "code": "p1", "tenant_id": "default"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/projects", params={"tenant_id": "default"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(p["id"] == pid for p in resp.json()["data"])
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/projects/{pid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["name"] == "Proj-1"
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/projects/{pid}", json={"description": "updated"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 归档
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "archived"
|
||||
|
||||
def test_project_members(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "Proj-M", "code": "pm", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 加成员
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/members", json={"user_id": "u_op", "role": "developer"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 列成员
|
||||
resp = client.get(f"{PREFIX}/projects/{pid}/members", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) >= 1
|
||||
# 改角色
|
||||
resp = client.put(f"{PREFIX}/projects/{pid}/members/u_op", json={"role": "maintainer"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
# 删成员
|
||||
resp = client.delete(f"{PREFIX}/projects/{pid}/members/u_op", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_resource_acl(self, client: TestClient):
|
||||
# 设置 ACL
|
||||
resp = client.put(
|
||||
f"{PREFIX}/resources/model/m001/acl",
|
||||
json={"entries": [{"subject_type": "user", "subject_id": "u_op", "permissions": ["read", "write"]}]},
|
||||
headers=_admin_headers(),
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
result = resp.json()["data"]
|
||||
assert len(result) == 1
|
||||
assert set(result[0]["permissions"]) == {"read", "write"}
|
||||
# 查询 ACL
|
||||
resp = client.get(f"{PREFIX}/resources/model/m001/acl", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert len(resp.json()["data"]) == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 3 周测试:审批实例、审批模板、审计日志查询和导出
|
||||
# ============================================================
|
||||
|
||||
class TestWeek3ApprovalAudit:
|
||||
"""第 3 周:审批实例、审批模板、审计日志查询和导出。"""
|
||||
|
||||
def test_approval_template_crud(self, client: TestClient):
|
||||
# 创建模板
|
||||
resp = client.post(f"{PREFIX}/approvals/templates", json={"name": "delete-approval", "steps": [{"approver_id": "u_admin", "status": "pending"}]}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
tpl_id = resp.json()["data"]["id"]
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/approvals/templates", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(t["id"] == tpl_id for t in resp.json()["data"])
|
||||
|
||||
def test_approval_instance_flow(self, client: TestClient):
|
||||
# 创建审批实例
|
||||
resp = client.post(f"{PREFIX}/approvals", json={
|
||||
"resource_type": "dataset", "resource_id": "ds_001",
|
||||
"applicant_id": "u_op",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
iid = resp.json()["data"]["id"]
|
||||
# 查详情
|
||||
resp = client.get(f"{PREFIX}/approvals/{iid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "pending"
|
||||
# 审批决策
|
||||
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
|
||||
"approver_id": "u_admin", "approved": True, "comment": "ok",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "approved"
|
||||
|
||||
def test_approval_instance_reject(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/approvals", json={
|
||||
"resource_type": "model", "resource_id": "m_002",
|
||||
"applicant_id": "u_op",
|
||||
}, headers=_admin_headers())
|
||||
iid = resp.json()["data"]["id"]
|
||||
resp = client.post(f"{PREFIX}/approvals/{iid}/steps/0/decision", json={
|
||||
"approver_id": "u_admin", "approved": False, "comment": "no",
|
||||
}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "rejected"
|
||||
|
||||
def test_approval_missing_field(self, client: TestClient):
|
||||
resp = client.post(f"{PREFIX}/approvals", json={"resource_type": "dataset"}, headers=_admin_headers())
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_audit_logs_query(self, client: TestClient):
|
||||
# 通过 API 写操作触发审计
|
||||
client.post(f"{PREFIX}/tenants", json={"name": "Audit-Tenant", "code": "at"}, headers=_admin_headers())
|
||||
# 查询
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs", params={"limit": 50}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "items" in data
|
||||
assert "total" in data
|
||||
assert data["total"] >= 1
|
||||
|
||||
def test_audit_logs_filter_by_action(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs", params={"action": "tenant.create"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
items = resp.json()["data"]["items"]
|
||||
assert all(i.get("action") == "tenant.create" for i in items)
|
||||
|
||||
def test_audit_logs_export_csv(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/system/audit-logs/export", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert "text/csv" in resp.headers.get("content-type", "")
|
||||
# CSV 首行是表头
|
||||
lines = resp.text.strip().split("\n")
|
||||
assert "time" in lines[0]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 第 4 周测试:写操作审计、审批拦截、权限校验
|
||||
# ============================================================
|
||||
|
||||
class TestWeek4AuditInterceptPermission:
|
||||
"""第 4 周:写操作审计、审批拦截、权限校验。"""
|
||||
|
||||
def test_write_operation_produces_audit(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 清空审计日志便于断言
|
||||
fake_store._audit_logs.clear()
|
||||
# 创建租户 → 应产生 tenant.create 审计
|
||||
client.post(f"{PREFIX}/tenants", json={"name": "W-Tenant", "code": "wt"}, headers=_admin_headers())
|
||||
assert any(l["action"] == "tenant.create" for l in fake_store._audit_logs)
|
||||
# 创建项目 → 应产生 project.create 审计
|
||||
client.post(f"{PREFIX}/projects", json={"name": "W-Proj", "code": "wp", "tenant_id": "default"}, headers=_admin_headers())
|
||||
assert any(l["action"] == "project.create" for l in fake_store._audit_logs)
|
||||
# 设置 ACL → 应产生 resource.acl.set 审计
|
||||
client.put(f"{PREFIX}/resources/model/w001/acl", json={"entries": []}, headers=_admin_headers())
|
||||
assert any(l["action"] == "resource.acl.set" for l in fake_store._audit_logs)
|
||||
|
||||
def test_approval_intercept_on_project_archive(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "I-Proj", "code": "ip", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 无待审批 → 可归档
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_approval_intercept_blocks_when_pending(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
# 创建项目
|
||||
resp = client.post(f"{PREFIX}/projects", json={"name": "B-Proj", "code": "bp", "tenant_id": "default"}, headers=_admin_headers())
|
||||
pid = resp.json()["data"]["id"]
|
||||
# 注入一条待审批实例
|
||||
fake_store.create_approval_instance({
|
||||
"resource_type": "project",
|
||||
"resource_id": pid,
|
||||
"applicant_id": "u_op",
|
||||
})
|
||||
# 有待审批 → 归档应被拒绝
|
||||
resp = client.post(f"{PREFIX}/projects/{pid}/archive", headers=_admin_headers())
|
||||
assert resp.status_code == 409
|
||||
|
||||
def test_retention_policy_crud_with_audit(self, client: TestClient, fake_store: FakePlatformStore):
|
||||
fake_store._audit_logs.clear()
|
||||
# 创建
|
||||
resp = client.post(f"{PREFIX}/retention-policies", json={"name": "30d-keep", "scope": "tenant"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
rpid = resp.json()["data"]["id"]
|
||||
assert any(l["action"] == "retention.create" for l in fake_store._audit_logs)
|
||||
# 查列表
|
||||
resp = client.get(f"{PREFIX}/retention-policies", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert any(p["id"] == rpid for p in resp.json()["data"])
|
||||
# 更新
|
||||
resp = client.put(f"{PREFIX}/retention-policies/{rpid}", json={"status": "inactive"}, headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["status"] == "inactive"
|
||||
# 删除
|
||||
resp = client.delete(f"{PREFIX}/retention-policies/{rpid}", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_login_duration_rank_in_dashboard(self, client: TestClient):
|
||||
resp = client.get(f"{PREFIX}/dashboard/stats", headers=_admin_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert "login_duration_rank" in data
|
||||
assert "recent_login_users" in data
|
||||
assert "service_status" in data
|
||||
assert "training_7d" in data
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
import hashlib
|
||||
@@ -10,10 +12,11 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
|
||||
from compute.agent.process_manager import ProcessManager
|
||||
from compute.engines.llama_factory.adapter import build_command, parse_log_line, prepare_runtime_files
|
||||
from compute.engines.llama_factory.inference import get_inference_session
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@@ -448,6 +451,29 @@ def create_app() -> FastAPI:
|
||||
accelerator_errors, accelerator_warnings, accelerator = _validate_training_accelerator(payload)
|
||||
errors.extend(accelerator_errors)
|
||||
warnings.extend(accelerator_warnings)
|
||||
elif engine == "eval":
|
||||
# Eval engine: validate model path and dataset path
|
||||
if not payload.get("model_name_or_path"):
|
||||
errors.append("model_name_or_path is required for eval")
|
||||
else:
|
||||
path_checks.append(_check_path_item({
|
||||
"name": "model_name_or_path",
|
||||
"path": payload.get("model_name_or_path", ""),
|
||||
"type": "any",
|
||||
"required": True,
|
||||
}))
|
||||
if payload.get("dataset_path"):
|
||||
path_checks.append(_check_path_item({
|
||||
"name": "dataset_path",
|
||||
"path": payload.get("dataset_path", ""),
|
||||
"type": "file",
|
||||
"required": True,
|
||||
}))
|
||||
else:
|
||||
errors.append("dataset_path is required for eval")
|
||||
if shutil.which("python") is None:
|
||||
errors.append("python runtime not found")
|
||||
|
||||
elif engine == "smoke":
|
||||
warnings.append("smoke engine skips model and dataset path checks")
|
||||
|
||||
@@ -666,6 +692,88 @@ def create_app() -> FastAPI:
|
||||
metrics = [parse_log_line(line) for line in window["content"].splitlines()]
|
||||
return {"job_id": job_id, **window, "metrics": [m for m in metrics if m]}
|
||||
|
||||
# ── Inference Endpoints ───────────────────────────────────────────
|
||||
|
||||
@app.post(f"{route_prefix}/inference/load")
|
||||
async def inference_load(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Load a model for inference using LLaMA-Factory ChatModel.
|
||||
|
||||
Expected payload:
|
||||
model_name_or_path: str (required)
|
||||
adapter_name_or_path: str (optional, for LoRA adapters)
|
||||
template: str (default: "qwen")
|
||||
infer_backend: str (default: "huggingface")
|
||||
infer_dtype: str (default: "auto")
|
||||
"""
|
||||
session = get_inference_session()
|
||||
result = session.load(
|
||||
model_name_or_path=payload.get("model_name_or_path", ""),
|
||||
adapter_name_or_path=payload.get("adapter_name_or_path", ""),
|
||||
template=payload.get("template", "qwen"),
|
||||
infer_backend=payload.get("infer_backend", "huggingface"),
|
||||
infer_dtype=payload.get("infer_dtype", "auto"),
|
||||
)
|
||||
return result
|
||||
|
||||
@app.post(f"{route_prefix}/inference/unload")
|
||||
async def inference_unload() -> dict[str, Any]:
|
||||
"""Unload the currently loaded model and free GPU memory."""
|
||||
# Teardown (gc.collect + cuda.empty_cache) can take a while; run it off
|
||||
# the event loop so /health and /inference/status stay responsive.
|
||||
return await asyncio.to_thread(get_inference_session().unload)
|
||||
|
||||
@app.get(f"{route_prefix}/inference/status")
|
||||
async def inference_status() -> dict[str, Any]:
|
||||
"""Get the current inference session status."""
|
||||
return get_inference_session().info()
|
||||
|
||||
@app.post(f"{route_prefix}/inference/chat")
|
||||
async def inference_chat(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Chat with the loaded model (non-streaming).
|
||||
|
||||
Expected payload:
|
||||
messages: list[dict] (OpenAI format)
|
||||
temperature: float (default 0.95)
|
||||
top_p: float (default 0.7)
|
||||
max_new_tokens: int (default 1024)
|
||||
"""
|
||||
messages = payload.get("messages") or []
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages is required")
|
||||
# Generation is long-running; run it in a thread so the event loop keeps
|
||||
# serving /inference/status and /health during inference.
|
||||
result = await asyncio.to_thread(
|
||||
get_inference_session().chat,
|
||||
messages=messages,
|
||||
temperature=float(payload.get("temperature", 0.95)),
|
||||
top_p=float(payload.get("top_p", 0.7)),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
)
|
||||
if result.get("error"):
|
||||
raise HTTPException(status_code=500, detail=result["error"])
|
||||
return {"response": result["response"]}
|
||||
|
||||
@app.post(f"{route_prefix}/inference/chat/stream")
|
||||
async def inference_chat_stream(payload: dict[str, Any]) -> StreamingResponse:
|
||||
"""Chat with streaming response (Server-Sent Events)."""
|
||||
messages = payload.get("messages") or []
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages is required")
|
||||
|
||||
def generate():
|
||||
session = get_inference_session()
|
||||
for chunk in session.chat_stream(
|
||||
messages=messages,
|
||||
temperature=float(payload.get("temperature", 0.95)),
|
||||
top_p=float(payload.get("top_p", 0.7)),
|
||||
max_new_tokens=int(payload.get("max_new_tokens", 1024)),
|
||||
do_sample=bool(payload.get("do_sample", True)),
|
||||
):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/upload")
|
||||
async def upload_file(
|
||||
file: UploadFile | None = File(default=None),
|
||||
@@ -733,6 +841,22 @@ def create_app() -> FastAPI:
|
||||
"checksum_sha256": checksum,
|
||||
}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/read")
|
||||
async def read_file(path: str = Query(...)) -> JSONResponse:
|
||||
"""Read a text file from within YG_FT_DATA_ROOT. Used by the backend
|
||||
to fetch eval results and other job outputs."""
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
target = (data_root / path.lstrip("/\\")).resolve()
|
||||
if not _path_inside(data_root, target):
|
||||
raise HTTPException(status_code=400, detail="path must stay inside YG_FT_DATA_ROOT")
|
||||
if not target.is_file():
|
||||
raise HTTPException(status_code=404, detail="file not found")
|
||||
try:
|
||||
content = target.read_text(encoding="utf-8")
|
||||
return JSONResponse(json.loads(content) if content.strip().startswith("{") else {"content": content})
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
|
||||
async def download_file(file_id: str) -> FileResponse:
|
||||
upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads"
|
||||
|
||||
@@ -204,6 +204,31 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||
return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
|
||||
|
||||
if engine == "eval":
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'eval-job')}"
|
||||
eval_config_path = str(Path(output_dir) / "eval_config.json")
|
||||
eval_config = {
|
||||
"model_name_or_path": config.get("model_name_or_path", ""),
|
||||
"adapter_name_or_path": config.get("adapter_name_or_path", ""),
|
||||
"template": config.get("template", "qwen"),
|
||||
"dataset_path": config.get("dataset_path", ""),
|
||||
"output_dir": output_dir,
|
||||
"basic_metrics": config.get("basic_metrics", {}),
|
||||
"dimension": config.get("dimension", {}),
|
||||
"temperature": config.get("temperature", 0.1),
|
||||
"top_p": config.get("top_p", 0.95),
|
||||
"max_new_tokens": config.get("max_new_tokens", 512),
|
||||
"infer_backend": config.get("infer_backend", "huggingface"),
|
||||
"infer_dtype": config.get("infer_dtype", "auto"),
|
||||
}
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(eval_config_path).write_text(json.dumps(eval_config, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return LlamaFactoryCommand(
|
||||
command=["python", "-u", "-m", "compute.engines.llama_factory.eval_runner", "--config", eval_config_path],
|
||||
work_dir="/app",
|
||||
env={},
|
||||
)
|
||||
|
||||
errors = validate_config(config)
|
||||
if errors:
|
||||
raise ValueError("; ".join(errors))
|
||||
|
||||
485
compute/engines/llama_factory/eval_runner.py
Normal file
485
compute/engines/llama_factory/eval_runner.py
Normal file
@@ -0,0 +1,485 @@
|
||||
"""
|
||||
Evaluation runner — executes model evaluation as a subprocess job.
|
||||
|
||||
Usage:
|
||||
python -m compute.engines.llama_factory.eval_runner --config <config_json_path>
|
||||
|
||||
The config JSON is written by the compute API before spawning this subprocess.
|
||||
Results are written to ``output_dir/eval_results.json`` and progress is printed
|
||||
to stdout (captured as job logs).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_dataset(path: str) -> list[dict[str, Any]]:
|
||||
"""Load a JSON or JSONL dataset file.
|
||||
|
||||
Supports common field names used across the platform:
|
||||
* ``instruction`` + ``input`` + ``output`` (Alpaca-style)
|
||||
* ``question`` + ``answer``
|
||||
* ``messages`` (ShareGPT-style – the last assistant message is treated as reference)
|
||||
"""
|
||||
file_path = Path(path)
|
||||
text = file_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return []
|
||||
if file_path.suffix.lower() == ".json":
|
||||
value = json.loads(text)
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return [value] if isinstance(value, dict) else []
|
||||
|
||||
samples: list[dict[str, Any]] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
samples.append(obj)
|
||||
return samples
|
||||
|
||||
|
||||
def _sample_question(sample: dict[str, Any]) -> str:
|
||||
"""Extract the user-facing question / instruction from a sample."""
|
||||
if sample.get("instruction"):
|
||||
text = sample["instruction"]
|
||||
if sample.get("input"):
|
||||
text += "\n" + sample["input"]
|
||||
return text
|
||||
if sample.get("question"):
|
||||
return sample["question"]
|
||||
# ShareGPT-style: use the last user message as question
|
||||
messages = sample.get("messages") or []
|
||||
user_msgs = [m["content"] for m in messages if m.get("role") == "user"]
|
||||
return user_msgs[-1] if user_msgs else ""
|
||||
|
||||
|
||||
def _sample_reference(sample: dict[str, Any]) -> str:
|
||||
"""Extract the reference answer from a sample."""
|
||||
if sample.get("output"):
|
||||
return sample["output"]
|
||||
if sample.get("answer"):
|
||||
return sample["answer"]
|
||||
messages = sample.get("messages") or []
|
||||
assistant_msgs = [m["content"] for m in messages if m.get("role") == "assistant"]
|
||||
return assistant_msgs[-1] if assistant_msgs else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Basic metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4) -> dict[str, Any]:
|
||||
"""Compute BLEU score via sacrebleu (corpus-level)."""
|
||||
try:
|
||||
from sacrebleu.metrics import BLEU
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
|
||||
bleu = BLEU(max_ngram_order=ngram)
|
||||
# sacrebleu expects list-of-strings; we have one reference per prediction
|
||||
score = bleu.corpus_score(predictions, [references])
|
||||
return {
|
||||
"enabled": True,
|
||||
"score": round(score.score, 2),
|
||||
"bleu": round(score.score, 2),
|
||||
}
|
||||
|
||||
|
||||
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
|
||||
"""Compute ROUGE scores via rouge-score."""
|
||||
try:
|
||||
from rouge_score import rouge_scorer
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "rouge-score not installed", "score": 0}
|
||||
methods = methods or ["rouge1", "rouge2", "rougeL"]
|
||||
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
|
||||
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
|
||||
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
|
||||
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
|
||||
totals: dict[str, float] = {}
|
||||
n = max(len(predictions), 1)
|
||||
for ref, pred in zip(references, predictions):
|
||||
result = scorer.score(ref, pred)
|
||||
for key in methods:
|
||||
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
||||
avg = {k: round(v / n, 4) for k, v in totals.items()}
|
||||
return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
|
||||
|
||||
|
||||
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
"""Compute average cosine similarity via sklearn."""
|
||||
try:
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
|
||||
try:
|
||||
vectorizer = TfidfVectorizer()
|
||||
tfidf = vectorizer.fit_transform(references + predictions)
|
||||
n = len(references)
|
||||
ref_vec = tfidf[:n]
|
||||
pred_vec = tfidf[n:]
|
||||
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
|
||||
return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
|
||||
except ValueError:
|
||||
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
||||
|
||||
|
||||
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
total = len(predictions)
|
||||
if not total:
|
||||
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
|
||||
matched = sum(
|
||||
1
|
||||
for ref, pred in zip(references, predictions)
|
||||
if _normalize_text(ref) == _normalize_text(pred)
|
||||
)
|
||||
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
|
||||
|
||||
|
||||
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
if not predictions:
|
||||
return {"enabled": True, "score": 0}
|
||||
scores = [
|
||||
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
||||
for ref, pred in zip(references, predictions)
|
||||
]
|
||||
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM Judge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _judge_sample(
|
||||
question: str,
|
||||
reference: str,
|
||||
prediction: str,
|
||||
config: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Call an OpenAI-compatible LLM to judge a single sample.
|
||||
|
||||
Returns a dict with keys:
|
||||
score, max_score, passed, judgement, evaluation_reason, error_type
|
||||
"""
|
||||
api_url = (config.get("api_url") or "").strip().rstrip("/")
|
||||
api_key = (config.get("api_key") or "").strip()
|
||||
eval_model = (config.get("eval_model") or "").strip()
|
||||
# 优先使用模型记录里配置的真实 API 模型名(如 deepseek-chat),
|
||||
# 否则回退到平台内部模型名
|
||||
api_model = (config.get("api_model") or "").strip() or eval_model
|
||||
eval_prompt = (config.get("eval_prompt") or "").strip()
|
||||
score_min = float(config.get("score_min", 0))
|
||||
score_max = float(config.get("score_max", 5))
|
||||
pass_threshold = float(config.get("pass_threshold", 3))
|
||||
|
||||
if not api_url or not eval_model:
|
||||
return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
|
||||
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
|
||||
|
||||
system_msg = (
|
||||
eval_prompt
|
||||
or "你是一个专业的评测专家。请根据参考答-案对被测模型的输出进行评分。"
|
||||
)
|
||||
user_msg = (
|
||||
f"## 问题\n{question}\n\n"
|
||||
f"## 参考答案\n{reference}\n\n"
|
||||
f"## 模型输出\n{prediction}\n\n"
|
||||
f"请给出 {score_min}-{score_max} 分的评分,并说明理由。"
|
||||
)
|
||||
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
body = json.dumps({
|
||||
"model": api_model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_msg},
|
||||
{"role": "user", "content": user_msg},
|
||||
],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 512,
|
||||
}).encode("utf-8")
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{api_url}/v1/chat/completions",
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
},
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=120)
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
reply = data["choices"][0]["message"]["content"]
|
||||
except Exception as exc:
|
||||
return {"score": 0, "max_score": score_max, "passed": False,
|
||||
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
|
||||
"error_type": "其他"}
|
||||
|
||||
# Parse score from reply — look for patterns like "4分" or "Score: 4"
|
||||
score = 0
|
||||
import re
|
||||
score_patterns = [
|
||||
r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
|
||||
r'(\d+(?:\.\d+)?)\s*分',
|
||||
r'(\d+(?:\.\d+)?)\s*/\s*\d+',
|
||||
]
|
||||
for pat in score_patterns:
|
||||
m = re.search(pat, reply, re.IGNORECASE)
|
||||
if m:
|
||||
try:
|
||||
score = float(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
break
|
||||
score = max(score_min, min(score_max, score))
|
||||
passed = score >= pass_threshold
|
||||
|
||||
# Determine judgement label
|
||||
if score >= pass_threshold + 1:
|
||||
judgement = "正确"
|
||||
elif score >= pass_threshold:
|
||||
judgement = "部分正确"
|
||||
else:
|
||||
judgement = "错误"
|
||||
|
||||
# Guess error type from reply
|
||||
reply_lower = reply.lower()
|
||||
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
|
||||
error_type = "幻觉"
|
||||
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
|
||||
error_type = "不完整"
|
||||
elif any(w in reply_lower for w in ["格式", "format"]):
|
||||
error_type = "格式偏差"
|
||||
elif any(w in reply_lower for w in ["混淆", "confusion", "错误"]):
|
||||
error_type = "混淆"
|
||||
else:
|
||||
error_type = "其他"
|
||||
|
||||
return {
|
||||
"score": score,
|
||||
"max_score": score_max,
|
||||
"passed": passed,
|
||||
"judgement": judgement,
|
||||
"evaluation_reason": reply[:2000],
|
||||
"error_type": error_type,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Execute a full evaluation run. Returns the result dict (also written to file)."""
|
||||
model_path = config["model_name_or_path"]
|
||||
adapter_path = config.get("adapter_name_or_path", "")
|
||||
template = config.get("template", "qwen")
|
||||
dataset_path = config["dataset_path"]
|
||||
output_dir = Path(config["output_dir"])
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
basic_cfg = config.get("basic_metrics", {})
|
||||
dimension_cfg = config.get("dimension", {}) or {}
|
||||
output_precision = int(basic_cfg.get("output_precision", 2))
|
||||
|
||||
# ---- 1. Load dataset ----
|
||||
print(f"[eval] loading dataset: {dataset_path}")
|
||||
raw_samples = _load_dataset(dataset_path)
|
||||
print(f"[eval] loaded {len(raw_samples)} samples")
|
||||
|
||||
# ---- 2. Load model ----
|
||||
print(f"[eval] loading model: {model_path}")
|
||||
from compute.engines.llama_factory.inference import InferenceSession
|
||||
session = InferenceSession()
|
||||
session.load(
|
||||
model_name_or_path=model_path,
|
||||
adapter_name_or_path=adapter_path,
|
||||
template=template,
|
||||
infer_backend=config.get("infer_backend", "huggingface"),
|
||||
infer_dtype=config.get("infer_dtype", "auto"),
|
||||
)
|
||||
# load() 为异步加载(立即返回 loading),必须等待后台线程完成后再进行推理
|
||||
load_result = session.wait_until_loaded(timeout=float(config.get("load_timeout", 1800)))
|
||||
if not load_result.get("loaded"):
|
||||
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
|
||||
print(f"[eval] model loaded OK")
|
||||
|
||||
# ---- 3. Run inference on each sample ----
|
||||
samples: list[dict[str, Any]] = []
|
||||
predictions: list[str] = []
|
||||
references: list[str] = []
|
||||
questions: list[str] = []
|
||||
|
||||
total = len(raw_samples)
|
||||
judge_enabled = bool(dimension_cfg.get("eval_model") and dimension_cfg.get("api_url"))
|
||||
print(f"[eval] starting inference on {total} samples, judge={'enabled' if judge_enabled else 'disabled'}")
|
||||
|
||||
for idx, raw in enumerate(raw_samples, start=1):
|
||||
question = _sample_question(raw)
|
||||
reference = _sample_reference(raw)
|
||||
if not question:
|
||||
print(f"[eval] sample {idx}/{total}: skipped (no question)")
|
||||
continue
|
||||
|
||||
# Inference
|
||||
chat_msgs = [{"role": "user", "content": question}]
|
||||
result = session.chat(
|
||||
chat_msgs,
|
||||
temperature=float(config.get("temperature", 0.1)),
|
||||
top_p=float(config.get("top_p", 0.95)),
|
||||
max_new_tokens=int(config.get("max_new_tokens", 512)),
|
||||
do_sample=False,
|
||||
)
|
||||
prediction = result.get("response", "") if not result.get("error") else f"[ERROR] {result['error']}"
|
||||
|
||||
predictions.append(prediction)
|
||||
references.append(reference)
|
||||
questions.append(question)
|
||||
|
||||
# LLM Judge
|
||||
judge_result: dict[str, Any] = {}
|
||||
if judge_enabled:
|
||||
judge_result = _judge_sample(question, reference, prediction, dimension_cfg)
|
||||
|
||||
samples.append({
|
||||
"index": idx,
|
||||
"input": question,
|
||||
"reference_answer": reference,
|
||||
"model_output": prediction,
|
||||
"score": judge_result.get("score"),
|
||||
"max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5)),
|
||||
"passed": judge_result.get("passed"),
|
||||
"judgement": judge_result.get("judgement"),
|
||||
"evaluation_reason": judge_result.get("evaluation_reason", ""),
|
||||
"error_type": judge_result.get("error_type"),
|
||||
"dimension_scores": [
|
||||
{"name": "judge_score", "score": judge_result.get("score", 0),
|
||||
"max_score": judge_result.get("max_score", dimension_cfg.get("score_max", 5))},
|
||||
] if judge_result else [],
|
||||
"status": "completed",
|
||||
})
|
||||
|
||||
progress_pct = int(idx / max(total, 1) * 100)
|
||||
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
|
||||
|
||||
# ---- 4. Compute basic metrics ----
|
||||
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
|
||||
metrics_result: dict[str, Any] = {}
|
||||
|
||||
bleu_cfg = basic_cfg.get("bleu", {})
|
||||
if bleu_cfg.get("enabled"):
|
||||
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
|
||||
|
||||
rouge_cfg = basic_cfg.get("rouge", {})
|
||||
if rouge_cfg.get("enabled"):
|
||||
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
|
||||
|
||||
cosine_cfg = basic_cfg.get("cosine", {})
|
||||
if cosine_cfg.get("enabled"):
|
||||
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
||||
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
||||
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
||||
|
||||
# ---- 5. Summarise ----
|
||||
completed = len(samples)
|
||||
if judge_enabled:
|
||||
scored = [s for s in samples if s.get("score") is not None]
|
||||
passed_count = len([s for s in scored if s.get("passed")])
|
||||
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
|
||||
max_score = dimension_cfg.get("score_max", 5)
|
||||
overall_score = round(avg_score / max_score * 100, output_precision)
|
||||
overall_score_max = 100
|
||||
dimension_summary = [{
|
||||
"name": "综合评分",
|
||||
"score": overall_score,
|
||||
"max_score": 100,
|
||||
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
|
||||
}]
|
||||
overall_evaluation = f"评测完成:{completed} 样本,{passed_count} 通过,平均 {avg_score}/{max_score} 分"
|
||||
else:
|
||||
passed_count = 0
|
||||
enabled_scores = [
|
||||
float(item.get("score") or 0)
|
||||
for item in metrics_result.values()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_score = round(sum(enabled_scores) / len(enabled_scores), output_precision) if enabled_scores else 0
|
||||
overall_score_max = 100
|
||||
dimension_summary = [
|
||||
{
|
||||
"name": name,
|
||||
"score": float(item.get("score") or 0),
|
||||
"max_score": 100,
|
||||
"pass_rate": float(item.get("score") or 0),
|
||||
}
|
||||
for name, item in metrics_result.items()
|
||||
if isinstance(item, dict) and item.get("enabled", True) and item.get("score") is not None
|
||||
]
|
||||
overall_evaluation = f"评测完成:{completed} 样本(未配置 LLM 评委)"
|
||||
|
||||
result = {
|
||||
"overall_score": overall_score,
|
||||
"overall_score_max": overall_score_max,
|
||||
"overall_evaluation": overall_evaluation,
|
||||
"improvement_suggestions": [],
|
||||
"dimension_summary": dimension_summary,
|
||||
"samples": samples,
|
||||
"sample_count": total,
|
||||
"completed_count": completed,
|
||||
"passed_count": passed_count,
|
||||
"basic_metrics": metrics_result,
|
||||
}
|
||||
|
||||
# ---- 6. Write results ----
|
||||
result_path = output_dir / "eval_results.json"
|
||||
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"[eval] results written to {result_path}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="YG-FT Evaluation Runner")
|
||||
parser.add_argument("--config", required=True, help="Path to eval config JSON file")
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config)
|
||||
if not config_path.exists():
|
||||
print(f"FATAL: config file not found: {args.config}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
start = time.time()
|
||||
try:
|
||||
run_eval(config)
|
||||
elapsed = time.time() - start
|
||||
print(f"[eval] DONE in {elapsed:.1f}s")
|
||||
except Exception as exc:
|
||||
print(f"[eval] FAILED: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
272
compute/engines/llama_factory/inference.py
Normal file
272
compute/engines/llama_factory/inference.py
Normal file
@@ -0,0 +1,272 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
class InferenceSession:
|
||||
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
||||
|
||||
Model loading is asynchronous: ``load()`` spawns a background daemon thread
|
||||
and returns immediately with ``status == "loading"``. ``info()`` (served by
|
||||
``/inference/status``) is always responsive, so the platform backend can
|
||||
poll loading progress without being blocked by a minutes-long model load —
|
||||
which previously froze the whole compute node event loop.
|
||||
|
||||
State machine: idle -> loading -> ready | error, ready -> idle (unload),
|
||||
loading -> idle (cancelled). Long operations (ChatModel build, teardown,
|
||||
generation) never run while holding ``_state_lock``; they either run in the
|
||||
worker thread or under ``_chat_lock`` only.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._state_lock = threading.Lock() # brief state transitions only
|
||||
self._chat_lock = threading.Lock() # serialize chat/teardown
|
||||
self._status: str = "idle"
|
||||
self._error: str = ""
|
||||
self._request_id: str = ""
|
||||
self._load_args: dict[str, Any] = {}
|
||||
self._teardown_old = False # load-while-ready: unload old before loading new
|
||||
self._cancel_requested = False # unload-while-loading: tear down after load finishes
|
||||
self._load_thread: threading.Thread | None = None
|
||||
self._model: Any = None
|
||||
self._tokenizer: Any = None
|
||||
self._generating_args: dict[str, Any] = {}
|
||||
self._model_name: str = ""
|
||||
self._adapter_path: str = ""
|
||||
self._loaded_at: float = 0.0
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
with self._state_lock:
|
||||
return self._status
|
||||
|
||||
def info(self) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
return {
|
||||
"loaded": self._status == "ready",
|
||||
"status": self._status,
|
||||
"model_name": self._model_name,
|
||||
"adapter_path": self._adapter_path,
|
||||
"loaded_at": self._loaded_at,
|
||||
"request_id": self._request_id,
|
||||
"error": self._error,
|
||||
}
|
||||
|
||||
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
|
||||
"""Wait for an in-flight async load to finish and return its outcome.
|
||||
|
||||
供同步消费方(如 eval_runner 子进程)使用:``load()`` 立即返回 loading 后,
|
||||
调用本方法等待后台加载线程完成,拿到最终的 loaded/error 结果。
|
||||
若在 timeout 秒内仍未加载完成,返回 ``status == "loading"`` 并附上超时提示。
|
||||
"""
|
||||
with self._state_lock:
|
||||
thread = self._load_thread
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=timeout)
|
||||
with self._state_lock:
|
||||
loaded = self._status == "ready"
|
||||
status = self._status
|
||||
error = self._error
|
||||
if not loaded and status == "loading":
|
||||
error = error or f"model load timed out after {timeout or 'N/A'}s"
|
||||
return {
|
||||
"loaded": loaded,
|
||||
"status": status,
|
||||
"model_name": self._model_name,
|
||||
"adapter_path": self._adapter_path,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
def load(
|
||||
self,
|
||||
model_name_or_path,
|
||||
adapter_name_or_path="",
|
||||
template="qwen",
|
||||
infer_backend="huggingface",
|
||||
infer_dtype="auto",
|
||||
**kwargs,
|
||||
) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
# A model is already loading — dedupe, reuse the same request id.
|
||||
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||
self._teardown_old = self._status == "ready"
|
||||
self._status = "loading"
|
||||
self._error = ""
|
||||
self._request_id = uuid.uuid4().hex[:12]
|
||||
self._cancel_requested = False
|
||||
self._load_args = {
|
||||
"model_name_or_path": model_name_or_path,
|
||||
"template": template,
|
||||
"infer_backend": infer_backend,
|
||||
"infer_dtype": infer_dtype,
|
||||
}
|
||||
if adapter_name_or_path:
|
||||
self._load_args["adapter_name_or_path"] = adapter_name_or_path
|
||||
self._load_args.update(kwargs)
|
||||
self._model_name = model_name_or_path
|
||||
self._adapter_path = adapter_name_or_path
|
||||
self._load_thread = threading.Thread(target=self._load_worker, daemon=True)
|
||||
self._load_thread.start()
|
||||
return {"loaded": False, "status": "loading", "request_id": self._request_id}
|
||||
|
||||
def _load_worker(self) -> None:
|
||||
"""Build the ChatModel off the state lock so info() never blocks."""
|
||||
model = None
|
||||
tokenizer = None
|
||||
generating_args: dict[str, Any] = {}
|
||||
error = ""
|
||||
try:
|
||||
if self._teardown_old:
|
||||
self._release_model()
|
||||
from llamafactory.chat import ChatModel
|
||||
from llamafactory.hparams import get_infer_args
|
||||
|
||||
args = dict(self._load_args)
|
||||
infer_result = get_infer_args(args)
|
||||
model = ChatModel(args)
|
||||
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
||||
generating_args = infer_result[-1]
|
||||
if hasattr(generating_args, "__dataclass_fields__"):
|
||||
generating_args = {
|
||||
k: v for k, v in vars(generating_args).items() if not k.startswith("_")
|
||||
}
|
||||
else:
|
||||
generating_args = dict(generating_args)
|
||||
except Exception as exc: # noqa: BLE001 - surface load failure via status
|
||||
error = str(exc)
|
||||
with self._state_lock:
|
||||
if error:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "error"
|
||||
self._error = error
|
||||
return
|
||||
if self._cancel_requested:
|
||||
# Unload was requested while loading — drop the fresh model.
|
||||
model = None
|
||||
tokenizer = None
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "idle"
|
||||
return
|
||||
self._model = model
|
||||
self._tokenizer = tokenizer
|
||||
self._generating_args = generating_args
|
||||
self._loaded_at = time.time()
|
||||
self._status = "ready"
|
||||
|
||||
def _release_model(self) -> None:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
self._status = "unloading"
|
||||
model = self._model
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
if model is not None:
|
||||
try:
|
||||
del model
|
||||
except Exception: # noqa: BLE001 - best-effort teardown
|
||||
pass
|
||||
# 强制释放 PyTorch CUDA 缓存,真正归还 GPU 显存
|
||||
try:
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.synchronize()
|
||||
except Exception: # noqa: BLE001 - teardown must not raise
|
||||
pass
|
||||
with self._state_lock:
|
||||
self._status = "idle"
|
||||
self._model_name = ""
|
||||
self._adapter_path = ""
|
||||
self._loaded_at = 0.0
|
||||
self._error = ""
|
||||
|
||||
def unload(self) -> dict[str, Any]:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
# Ask the worker to tear down right after the load finishes.
|
||||
self._cancel_requested = True
|
||||
return {"unloaded": False, "status": "cancelling", "request_id": self._request_id}
|
||||
was_ready = self._status == "ready"
|
||||
if was_ready:
|
||||
self._release_model()
|
||||
else:
|
||||
with self._state_lock:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._status = "idle"
|
||||
self._model_name = ""
|
||||
self._adapter_path = ""
|
||||
self._loaded_at = 0.0
|
||||
self._error = ""
|
||||
return {"unloaded": True, "status": "idle"}
|
||||
|
||||
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
return {
|
||||
"error": f"model is still loading (request_id={self._request_id}); please retry",
|
||||
"response": "",
|
||||
}
|
||||
if self._status == "error":
|
||||
return {"error": f"model load failed: {self._error}", "response": ""}
|
||||
if self._status != "ready" or self._model is None:
|
||||
return {"error": "model not loaded", "response": ""}
|
||||
try:
|
||||
generate_kwargs = {
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"max_new_tokens": max_new_tokens,
|
||||
"do_sample": do_sample,
|
||||
}
|
||||
generate_kwargs.update(kwargs)
|
||||
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
||||
user_messages = [m for m in messages if m["role"] != "system"]
|
||||
responses = []
|
||||
for response in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
||||
responses.append(response)
|
||||
full_response = "".join(str(r) for r in responses)
|
||||
return {"response": full_response}
|
||||
except Exception as exc: # noqa: BLE001 - return generation error to caller
|
||||
return {"error": str(exc), "response": ""}
|
||||
|
||||
def chat_stream(self, messages, **kwargs) -> Iterator[str]:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
if self._status == "loading":
|
||||
yield 'data: {"error": "model is still loading; please retry"}\n\n'
|
||||
return
|
||||
if self._status == "error":
|
||||
yield 'data: {"error": "model load failed: ' + str(self._error) + '"}\n\n'
|
||||
return
|
||||
if self._status != "ready" or self._model is None:
|
||||
yield 'data: {"error": "model not loaded"}\n\n'
|
||||
return
|
||||
try:
|
||||
generate_kwargs = {**kwargs}
|
||||
system = next((m["content"] for m in messages if m["role"] == "system"), None)
|
||||
user_messages = [m for m in messages if m["role"] != "system"]
|
||||
for new_text in self._model.stream_chat(user_messages, system=system, **generate_kwargs):
|
||||
yield new_text
|
||||
except Exception as exc: # noqa: BLE001 - stream error as SSE event
|
||||
yield 'data: {"error": "' + str(exc) + '"}\n\n'
|
||||
|
||||
|
||||
_inference_session = None
|
||||
|
||||
|
||||
def get_inference_session() -> InferenceSession:
|
||||
global _inference_session
|
||||
if _inference_session is None:
|
||||
_inference_session = InferenceSession()
|
||||
return _inference_session
|
||||
@@ -4,3 +4,9 @@ python-multipart>=0.0.9
|
||||
pydantic>=2.7.0
|
||||
python-dotenv>=1.0.1
|
||||
httpx>=0.27.0
|
||||
# 模型评测指标
|
||||
sacrebleu>=2.4.0
|
||||
rouge-score>=0.1.2
|
||||
scikit-learn>=1.3.0
|
||||
# LLaMA-Factory 训练引擎
|
||||
llamafactory
|
||||
|
||||
146
compute/tests/test_inference_session.py
Normal file
146
compute/tests/test_inference_session.py
Normal file
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from compute.engines.llama_factory.inference import InferenceSession
|
||||
|
||||
# 模拟模型加载耗时,用于验证 load() 立即返回、info() 不阻塞
|
||||
LOAD_DELAY = 0.2
|
||||
|
||||
|
||||
class FakeChatModel:
|
||||
def __init__(self, args: dict[str, Any]) -> None:
|
||||
time.sleep(LOAD_DELAY)
|
||||
self.tokenizer = object()
|
||||
self.engine = types.SimpleNamespace(tokenizer=object())
|
||||
self._output = "hello from model"
|
||||
|
||||
def stream_chat(self, *args, **kwargs):
|
||||
for _ in range(1):
|
||||
yield self._output
|
||||
|
||||
|
||||
class FailingChatModel:
|
||||
def __init__(self, args: dict[str, Any]) -> None:
|
||||
time.sleep(LOAD_DELAY)
|
||||
raise RuntimeError("boom: fake load failure")
|
||||
|
||||
|
||||
def _get_infer_args(args: dict[str, Any]) -> list[Any]:
|
||||
# 最后一个元素为 generating_args,worker 会转成 dict
|
||||
return [None, None, {"temperature": 0.7}]
|
||||
|
||||
|
||||
def _install_llamafactory(monkeypatch, chat_model: type) -> None:
|
||||
llmf = types.ModuleType("llamafactory")
|
||||
chat_mod = types.ModuleType("llamafactory.chat")
|
||||
hparams_mod = types.ModuleType("llamafactory.hparams")
|
||||
chat_mod.ChatModel = chat_model
|
||||
hparams_mod.get_infer_args = _get_infer_args
|
||||
llmf.chat = chat_mod
|
||||
llmf.hparams = hparams_mod
|
||||
monkeypatch.setitem(sys.modules, "llamafactory", llmf)
|
||||
monkeypatch.setitem(sys.modules, "llamafactory.chat", chat_mod)
|
||||
monkeypatch.setitem(sys.modules, "llamafactory.hparams", hparams_mod)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_llamafactory(monkeypatch) -> None:
|
||||
_install_llamafactory(monkeypatch, FakeChatModel)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_failing_llamafactory(monkeypatch) -> None:
|
||||
_install_llamafactory(monkeypatch, FailingChatModel)
|
||||
|
||||
|
||||
def _wait_for_status(session: InferenceSession, status: str, timeout: float = 3.0) -> bool:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if session.info()["status"] == status:
|
||||
return True
|
||||
time.sleep(0.02)
|
||||
return False
|
||||
|
||||
|
||||
def test_load_returns_immediately_then_ready(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
started = time.time()
|
||||
result = session.load("/models/qwen")
|
||||
assert result["status"] == "loading"
|
||||
assert result["loaded"] is False
|
||||
assert result["request_id"]
|
||||
# 在慢加载完成前就返回,且 info() 加载期间可响应
|
||||
assert time.time() - started < LOAD_DELAY
|
||||
assert session.info()["status"] == "loading"
|
||||
assert _wait_for_status(session, "ready")
|
||||
info = session.info()
|
||||
assert info["loaded"] is True
|
||||
assert info["status"] == "ready"
|
||||
assert info["model_name"] == "/models/qwen"
|
||||
|
||||
|
||||
def test_second_load_while_loading_deduped(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
r1 = session.load("/models/a")
|
||||
r2 = session.load("/models/b")
|
||||
assert r2["status"] == "loading"
|
||||
assert r2["request_id"] == r1["request_id"]
|
||||
assert _wait_for_status(session, "ready")
|
||||
assert session.info()["status"] == "ready"
|
||||
|
||||
|
||||
def test_load_error_surfaces_in_status(stub_failing_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/bad")
|
||||
assert _wait_for_status(session, "error")
|
||||
assert "boom" in session.info()["error"]
|
||||
|
||||
|
||||
def test_unload_while_loading_cancels(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
result = session.unload()
|
||||
assert result["status"] == "cancelling"
|
||||
assert _wait_for_status(session, "idle")
|
||||
|
||||
|
||||
def test_chat_while_loading_returns_loading_error(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
out = session.chat([{"role": "user", "content": "hi"}])
|
||||
assert "still loading" in (out.get("error") or "")
|
||||
assert _wait_for_status(session, "ready")
|
||||
out = session.chat([{"role": "user", "content": "hi"}])
|
||||
assert out.get("response") == "hello from model"
|
||||
|
||||
|
||||
def test_chat_stream_while_loading_yields_error(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/qwen")
|
||||
chunks = list(session.chat_stream([{"role": "user", "content": "hi"}]))
|
||||
assert any("still loading" in c for c in chunks)
|
||||
|
||||
|
||||
def test_wait_until_loaded_blocks_until_ready(stub_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
result = session.load("/models/qwen")
|
||||
assert result["status"] == "loading"
|
||||
# 同步等待后台加载线程完成
|
||||
outcome = session.wait_until_loaded(timeout=3.0)
|
||||
assert outcome["loaded"] is True
|
||||
assert outcome["status"] == "ready"
|
||||
|
||||
|
||||
def test_wait_until_loaded_reports_load_error(stub_failing_llamafactory) -> None:
|
||||
session = InferenceSession()
|
||||
session.load("/models/bad")
|
||||
outcome = session.wait_until_loaded(timeout=3.0)
|
||||
assert outcome["loaded"] is False
|
||||
assert outcome["status"] == "error"
|
||||
assert "boom" in outcome["error"]
|
||||
@@ -11,7 +11,7 @@ RUN pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
&& pip install -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
&& rm -f /tmp/requirements.txt
|
||||
|
||||
RUN python -c "import fastapi, uvicorn, psycopg, sqlalchemy, redis, jwt, passlib, httpx, alembic; print('backend dependency check ok')"
|
||||
RUN python -c "import fastapi, uvicorn, psycopg, psycopg_pool, sqlalchemy, redis, jwt, passlib, httpx, alembic; print('backend dependency check ok')"
|
||||
|
||||
RUN mkdir -p /opt/yg-ft/logs/backend /data/yg-ft \
|
||||
&& chmod -R 0775 /opt/yg-ft /data/yg-ft
|
||||
|
||||
@@ -35,6 +35,6 @@ COMPUTE_GPU_MEMORY_GB=80
|
||||
COMPUTE_GPU_POWER_LIMIT_W=300
|
||||
|
||||
LOG_DIR=/opt/yg-ft/logs/compute
|
||||
CUDA_VISIBLE_DEVICES=all
|
||||
CUDA_VISIBLE_DEVICES=0
|
||||
NVIDIA_VISIBLE_DEVICES=all
|
||||
NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
|
||||
@@ -14,8 +14,8 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
}
|
||||
|
||||
location = /modelTF {
|
||||
@@ -25,8 +25,8 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 900s;
|
||||
proxy_send_timeout 900s;
|
||||
}
|
||||
|
||||
location ~* \.(?:js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf)$ {
|
||||
|
||||
121
docs/模型评测功能总结.md
Normal file
121
docs/模型评测功能总结.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# 模型评测功能总结
|
||||
|
||||
本项目(基于 LLaMA-Factory 的微调训练平台)包含 **4 套相对独立** 的模型评测能力,分别面向不同的使用场景:
|
||||
|
||||
| 能力 | 入口/目录 | 评测类型 | 打分方式 |
|
||||
| --- | --- | --- | --- |
|
||||
| 1. 学术 Benchmark 评测 | `llamafactory/eval/` | 选择题式基准(类 MMLU/C-Eval) | 选项匹配 + few-shot |
|
||||
| 2. 评估工作台 | `backend/app/api/v1/eval/` | 生成式问答(指令跟随) | BLEU / ROUGE / ExactMatch + 可选 LLM 评审 |
|
||||
| 3. 平台评估系统 | `backend/app/api/v1/evaluation/` | 基于评估数据集的问答 | 判卷模型(judge model)打分(0–5 分) |
|
||||
| 4. 训练时验证评估 | `backend/app/services/task_runner.py` | 训练验证集 | loss 指标 |
|
||||
|
||||
下面分别说明。
|
||||
|
||||
---
|
||||
|
||||
## 1. 学术 Benchmark 评测(LLaMA-Factory 原生)
|
||||
|
||||
面向标准学术选择题基准(如 MMLU、C-Eval 等),复用 LLaMA-Factory 原生的评测框架。
|
||||
|
||||
**核心文件**
|
||||
- `llamafactory/eval/evaluator.py`:`Evaluator` 类 + `run_eval()` 入口
|
||||
- `llamafactory/eval/template.py`:评测 prompt 模板(中/英,含 few-shot 示例构建)
|
||||
- `llamafactory/hparams/evaluation_args.py`:`EvaluationArguments` 配置类
|
||||
|
||||
**工作流程**
|
||||
1. 按 `task`(benchmark 名称)加载数据集,按科目(subject)拆分。
|
||||
2. 每个样本构造 few-shot 提示词(`n_shot` 控制示例数,由 `lang` 决定中/英模板),将题干与候选选项拼入 prompt。
|
||||
3. 调用模型推理得到预测,与标准答案比对,统计每个科目及整体的 `accuracy`。
|
||||
4. 结果写入 `save_dir`,打印各科目与平均准确率。
|
||||
|
||||
**关键参数(`EvaluationArguments`)**
|
||||
- `task`:基准数据集名
|
||||
- `batch_size` / `n_shot` / `lang` / `save_dir` / `seed`
|
||||
- `model_name_or_path`、`template`、`trust_remote_code` 等模型相关参数
|
||||
|
||||
> 该能力属于框架底层,本平台前端未直接提供操作入口,主要通过配置文件/脚本调用。
|
||||
|
||||
---
|
||||
|
||||
## 2. 评估工作台(生成式评测 + 指标计算)
|
||||
|
||||
后端路由位于 `backend/app/api/v1/eval/__init__.py`,前端称为「评估工作台」。**适用于评测模型的指令跟随与生成质量**,并支持 LLM 作为裁判(LLM-as-a-Judge)。
|
||||
|
||||
**API 端点**
|
||||
- `GET /evaluation/tasks`:列出评测任务(`frontend/src/api/evaluation.ts:listTasks`)
|
||||
- `POST /evaluation/run`:提交一次评测(`runEval`)
|
||||
- `GET /evaluation/report/{task_id}`:拉取评测报告(`getReport`)
|
||||
- `DELETE /evaluation/tasks/{task_id}`:删除任务(`deleteTask`)
|
||||
|
||||
**评测流程(`run_eval`)**
|
||||
1. 通过 **LLaMA-Factory 数据管道**(`get_dataset`) 加载数据集,支持 `subset` 与抽样(`eval_sample`)。
|
||||
2. 用 **原生 transformers** 加载模型在本地做生成推理(单进程顺序生成,便于展示样本)。
|
||||
3. 计算客观指标(`compute_score`):
|
||||
- `BLEU`(sacrebleu)
|
||||
- `ROUGE-1 / ROUGE-2 / ROUGE-L`(rouge-score)
|
||||
- `Exact Match`
|
||||
4. **可选 LLM 评审**(judge):当配置了 `judge_model` / `judge_api_base` / `judge_api_key` 时,调用 OpenAI 兼容接口对每条样本打分(10 分制),并输出 4 个维度与理由:
|
||||
- 核心事实正确性 `factual`
|
||||
- 信息完整性 `completeness`
|
||||
- 无幻觉 `no_hallucination`
|
||||
- 格式合规性 `format`
|
||||
- 综合分 `score` + `reason`
|
||||
5. 任务状态持久化在后端 `eval_tasks.json`(支持 running/completed/failed/stopped),前端轮询进度。
|
||||
|
||||
**前端页面**
|
||||
- `frontend/src/views/evaluation/EvaluateTask.vue`:任务列表、创建评测对话框(选模型、数据集、指标、可选 judge 配置)
|
||||
- `frontend/src/views/evaluation/EvaluateReport.vue`:报告页,展示综合得分、BLEU、ROUGE-L、各维度指标及「参考答案 vs 模型预测 vs LLM 评审」对比样例
|
||||
|
||||
---
|
||||
|
||||
## 3. 平台评估系统(基于评估数据集 + 判卷模型)
|
||||
|
||||
后端路由位于 `backend/app/api/v1/evaluation/__init__.py`,是平台业务层自研的评测体系。通过「评估数据集」组织题目,可一次性对 **多个被测模型 + 指定判卷模型** 进行批量评分。
|
||||
|
||||
**核心概念(数据模型 `backend/app/models/models.py`)**
|
||||
- `EvalDataset`(`models.py:131`):评估数据集,从项目问答对(`Question`/`Chunk`)中按 `question_type`(mixed/fact/reasoning)选题构建,状态 `pending/running/completed/failed`。
|
||||
- `EvalResult`(`models.py:147`):单条评测结果,含 `judge_score`(0–5 分)、`is_correct`(true/false/partial)、`feedback`、`expected_answer` 等。
|
||||
- `Task`(`models.py:184`):后台任务,`task_type="model-evaluation"`,记录进度与 `model_info`(存放平均分等汇总)。
|
||||
|
||||
**评测流程(`process_evaluation_task`,`backend/app/services/task_processor.py:336` 起)**
|
||||
1. 加载评估数据集关联的题目,可选带入 `chunk` 上下文(RAG 场景)。
|
||||
2. 对每道题,先用 `build_eval_prompt` 组合「上下文 + 题目 + 参考答案」,调用 **判卷模型**(`call_model`,temperature=0.3)生成评分。
|
||||
3. `parse_eval_result` 解析出 `score`(0–5)、`is_correct`、`feedback`,写入 `EvalResult`。
|
||||
4. 逐题提交进度(`completed_count` / `progress`),支持中途 `stopped`。
|
||||
5. 汇总:`avg_score = 总分/有效数 × 20`(换算百分制),`avg_score_5 = 总分/有效数`(5 分制),存入 `task.model_info`。判定规则:得分 **≥3 视为正确**。
|
||||
|
||||
**特点**
|
||||
- 判卷与被测模型解耦:被测模型给出答案,判卷模型(judge)独立评分,降低自评偏差。
|
||||
- 支持失败隔离:单题异常写入 `evaluation_status: failed` 记录而不中断整体任务。
|
||||
|
||||
---
|
||||
|
||||
## 4. 训练时验证评估
|
||||
|
||||
在微调训练任务执行期间,由 `backend/app/services/task_runner.py` 的 `do_eval` 触发:
|
||||
|
||||
- 在训练过程中对验证集(validation set)计算 `eval_loss`,用于监控过拟合。
|
||||
- 结果回填到 `Task` 的 `loss_info` / `detail`,前端绘制 loss 曲线。
|
||||
- 属于训练配套的轻量评估,不参与上述 1–3 的业务评测。
|
||||
|
||||
---
|
||||
|
||||
## 附属:前端评测相关页面
|
||||
|
||||
| 文件 | 作用 |
|
||||
| --- | --- |
|
||||
| `frontend/src/views/evaluation/EvaluateTask.vue` | 评估工作台:任务列表 + 创建评测 |
|
||||
| `frontend/src/views/evaluation/EvaluateReport.vue` | 评估报告:指标卡 + 维度标签 + 对比样例 |
|
||||
| `frontend/src/api/evaluation.ts` | 评估工作台接口封装 |
|
||||
| 平台评估系统入口 | 评估数据集管理 + 评估任务(model-evaluation)创建与结果查看 |
|
||||
|
||||
---
|
||||
|
||||
## 小结
|
||||
|
||||
- **想要学术榜单式准确率** → 用能力 1(LLaMA-Factory `eval/`)。
|
||||
- **想要开放式生成质量(BLEU/ROUGE + LLM 评审)** → 用能力 2(评估工作台 `/evaluation/run`)。
|
||||
- **想要基于自有问答数据、用判卷模型批量打分** → 用能力 3(平台评估系统 `model-evaluation` 任务)。
|
||||
- **训练过程监控** → 能力 4(`do_eval` 验证集 loss)。
|
||||
|
||||
三种业务评测(1/2/3)相互独立,可并存于同一平台;数据模型(`EvalDataset`/`EvalResult`/`Task`)主要服务于能力 3,而能力 2 使用独立的 `eval_tasks.json` 文件持久化。
|
||||
@@ -83,6 +83,11 @@ assert.match(detailSource, /\.el-button\s*>\s*span[\s\S]*?width:\s*100%[\s\S]*?d
|
||||
assert.match(detailSource, /\.el-button i[\s\S]*?margin-left:\s*auto/, '输出数据集跳转图标没有统一右对齐')
|
||||
assert.match(detailSource, /将发布三个独立数据集/, '发布说明仍未明确生成三个独立数据集')
|
||||
assert.match(detailSource, /function startRegeneration\(\)[\s\S]*?name: 'data-process-regenerate'[\s\S]*?params: \{ id: taskId\.value \}/, '重新生成按钮没有携带原任务 ID 进入命名路由')
|
||||
assert.match(detailSource, /const canRepeatGeneration = computed[\s\S]*?status === 'completed'[\s\S]*?results_confirmed !== false[\s\S]*?previewCount\.value > 0/, '已完成任务缺少再次生成资格判断')
|
||||
assert.match(detailSource, /repeatDataProcessTask\(taskId\.value,[\s\S]*?expected_updated_at: detail\.value\.updated_at[\s\S]*?request_id: repeatRequestId\.value/, '再次生成没有携带源任务版本和幂等请求 ID')
|
||||
assert.match(detailSource, /name: 'data-process-workflow'[\s\S]*?params: \{ id: repeated\.task\.id \}/, '再次生成成功后没有进入新任务工作流')
|
||||
assert.match(detailSource, /原任务和原结果不会被修改/, '再次生成确认提示没有说明原任务保持不变')
|
||||
assert.match(detailSource, /v-if="canRepeatGeneration"[\s\S]*?@click="repeatGeneration"[\s\S]*?按原配置再生成一批/, '已完成任务详情缺少再次生成新批次入口')
|
||||
assert.match(detailSource, /const canRegenerate = computed\(\(\) => \{[\s\S]*?status === 'pending'[\s\S]*?status === 'failed'[\s\S]*?status === 'stopped'[\s\S]*?status === 'completed'[\s\S]*?outputDatasetId\.value[\s\S]*?hasPublishedOutputs\.value/, '详情页没有覆盖指针已清空但旧发布数据集仍存在的重新生成任务')
|
||||
assert.match(detailSource, /v-if="canRegenerate"[\s\S]*?@click="startRegeneration"[\s\S]*?重新生成/, '可恢复任务没有收敛为单一重新生成入口')
|
||||
assert.match(detailSource, /v-if="detail\.status === 'completed' && !hasCurrentPublishedDataset"[\s\S]*?@click="openPublishDialog"[\s\S]*?发布为三个数据集/, '未发布或发布指针失效的完成任务没有保留发布入口')
|
||||
@@ -115,6 +120,11 @@ assert.match(detailSource, /inputMetricCount\.toLocaleString\(\) \}\} \{\{ input
|
||||
assert.match(detailSource, /sourceFileCount\.toLocaleString\(\) \}\} 个/, '源文件数量缺少个数单位')
|
||||
assert.match(detailSource, /<span>生成结果<\/span><strong>\{\{ numeric\(detail\.output_count\)\.toLocaleString\(\) \}\} 条<\/strong>/, '生成结果数量缺少条数单位或仍误称成功输出')
|
||||
assert.match(detailSource, /const configExpanded = ref\(false\)/, '处理配置没有默认收起')
|
||||
assert.match(detailSource, /appendGroup\(\['clean_invalid', 'deduplicate'\], '数据清洗'\)/, '详情页没有将完整清洗配置合并为数据清洗')
|
||||
assert.match(detailSource, /appendGroup\(\['detect_structure', 'normalize_format'\], '结构标准化'\)/, '详情页没有将完整结构配置合并为结构标准化')
|
||||
assert.match(detailSource, /历史部分配置/, '详情页没有标识旧任务的半组选项')
|
||||
assert.match(detailSource, /异常数据过滤(历史规则)/, '详情页没有标识已停用的历史异常过滤规则')
|
||||
assert.match(detailSource, /new Set\(value\.map/, '详情页没有去除历史预处理配置中的重复值')
|
||||
assert.match(detailSource, /:aria-expanded="configExpanded"/, '处理配置折叠按钮缺少无障碍状态')
|
||||
assert.match(detailSource, /<el-collapse-transition>[\s\S]*?v-show="configExpanded"/, '处理配置没有折叠过渡或内容状态')
|
||||
assert.doesNotMatch(detailSource, /const (?:detailMap|completedResults)\b|TODO: 接入真实接口/, '详情页仍包含本地 Mock 数据')
|
||||
@@ -126,6 +136,7 @@ for (const apiName of [
|
||||
'updateDataProcessResult',
|
||||
'restoreDataProcessResult',
|
||||
'publishDataProcess',
|
||||
'repeatDataProcessTask',
|
||||
]) {
|
||||
assert.match(
|
||||
apiSource,
|
||||
@@ -136,5 +147,8 @@ for (const apiName of [
|
||||
assert.match(apiSource, /keyword\?: string; status\?: string; split\?: string/, '结果列表 API 缺少服务端筛选参数')
|
||||
assert.match(apiSource, /\/results\/\$\{encodeURIComponent\(resultId\)\}/, '结果资源路径没有安全编码结果 ID')
|
||||
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/publish`/, '发布 API 路径不正确')
|
||||
assert.match(apiSource, /`\/data-process\/\$\{encodeURIComponent\(taskId\)\}\/repeat`/, '再次生成 API 路径不正确')
|
||||
assert.match(typesSource, /interface DataProcessRepeatPayload[\s\S]*?expected_updated_at: string[\s\S]*?request_id: string/, '再次生成请求契约不完整')
|
||||
assert.match(typesSource, /interface DataProcessRepeatResult[\s\S]*?task: DataProcessTask[\s\S]*?source_task_id: string[\s\S]*?created: boolean/, '再次生成响应契约不完整')
|
||||
|
||||
console.log('数据处理任务详情真实 API 回归检查通过')
|
||||
|
||||
@@ -183,8 +183,36 @@ for (const field of ['sourceStart', 'sourceEnd', 'originalContent', 'editedConte
|
||||
assert.ok(typesSource.includes(field), `PreviewItem 缺少字段:${field}`)
|
||||
}
|
||||
assert.match(typesSource, /sourceFileId/, 'PreviewItem 缺少来源文件标识')
|
||||
assert.match(typesSource, /sourceLocator\?: PreviewSourceLocator/, 'PreviewItem 缺少结构化来源定位契约')
|
||||
assert.match(typesSource, /headingPath\?: string\[\]/, 'PreviewItem 缺少非结构化标题路径')
|
||||
assert.match(typesSource, /PreviewSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, '前端来源定位 kind 未使用明确联合类型')
|
||||
assert.match(contractTypesSource, /DataProcessSourceLocatorKind = 'json' \| 'jsonl' \| 'csv' \| 'xlsx'/, 'API 来源定位 kind 未使用明确联合类型')
|
||||
for (const field of ['kind', 'record_index', 'start_line', 'end_line', 'source_start', 'source_end', 'json_pointer', 'sheet_index', 'sheet_name', 'row_number', 'sheet_record_index']) {
|
||||
assert.ok(typesSource.includes(field), `PreviewSourceLocator 缺少字段:${field}`)
|
||||
assert.ok(contractTypesSource.includes(field), `后端来源定位契约缺少字段:${field}`)
|
||||
}
|
||||
assert.match(contractTypesSource, /source_locator\?: DataProcessSourceLocator/, '质量信息缺少来源定位契约')
|
||||
assert.match(contractTypesSource, /heading_path\?: string\[\]/, '质量信息缺少标题路径契约')
|
||||
assert.match(viewSource, /const sourceLocator = item\.quality_score\?\.source_locator/, '预览映射丢失来源定位')
|
||||
assert.match(viewSource, /sourceStart:\s*item\.source_start\s*\?\?\s*sourceLocator\?\.source_start/, 'JSON locator 的字符起点没有映射到预览项')
|
||||
assert.match(viewSource, /sourceEnd:\s*item\.source_end\s*\?\?\s*sourceLocator\?\.source_end/, 'JSON locator 的字符终点没有映射到预览项')
|
||||
assert.match(viewSource, /sourceStartLine:\s*item\.source_start_line\s*\?\?\s*sourceLocator\?\.start_line/, 'JSON locator 的起始行没有映射到预览项')
|
||||
assert.match(viewSource, /sourceEndLine:\s*item\.source_end_line\s*\?\?\s*sourceLocator\?\.end_line/, 'JSON locator 的结束行没有映射到预览项')
|
||||
assert.match(viewSource, /headingPath:[\s\S]*?item\.quality_score\?\.heading_path/, '预览映射丢失标题路径')
|
||||
assert.match(typesSource, /export type StepId = 'create' \| 'model' \| 'upload' \| 'preview' \| 'generate' \| 'results'/, '步骤类型缺少独立大模型选择步骤')
|
||||
assert.match(modelSource, /export function sourceLines/, '缺少源文件行偏移生成函数')
|
||||
assert.match(modelSource, /export function sourceLineWindow/, '缺少有界源文件行窗口函数')
|
||||
assert.match(modelSource, /maxLines:\s*number/, '源文件行窗口缺少最大渲染行数参数')
|
||||
assert.doesNotMatch(modelSource, /\.split\(\s*['"]\\n['"]\s*\)/, '源文件行窗口仍会先对全文 split')
|
||||
assert.match(modelSource, /lines\.length < limit/, '源文件行扫描没有受最大行数约束')
|
||||
assert.match(modelSource, /unicodeCodePointLength/, '源文件字符偏移未与后端 Unicode code point 计数保持一致')
|
||||
assert.match(modelSource, /export function sourceLineNumberAtOffset/, '字符偏移缺少无数组的行号解析函数')
|
||||
const manualPreviewHelperStart = modelSource.indexOf('export function isManualPreviewItem(')
|
||||
const manualPreviewHelperEnd = modelSource.indexOf('\n}', manualPreviewHelperStart)
|
||||
assert.ok(manualPreviewHelperStart >= 0, '缺少统一的手动预览项判定函数')
|
||||
const manualPreviewHelperSource = modelSource.slice(manualPreviewHelperStart, manualPreviewHelperEnd + 2)
|
||||
for (const field of ['status', 'originalContent', 'sourceStart', 'sourceEnd', 'sourceStartLine', 'sourceEndLine', 'sourcePages', 'sourceLocator']) {
|
||||
assert.ok(manualPreviewHelperSource.includes(field), `手动预览项判定缺少来源字段:${field}`)
|
||||
}
|
||||
assert.doesNotMatch(modelSource, /buildPreviewItems/, '前端不应保留与后端重复的本地切片算法')
|
||||
assert.match(viewSource, /selectedPreviewFileId/, '父页面缺少当前预览文件状态')
|
||||
const previewBuildBindingStart = viewSource.indexOf('useDataProcessPreviewBuild()')
|
||||
@@ -210,6 +238,30 @@ for (const marker of [
|
||||
}
|
||||
assert.match(previewSource, /sourceStart/, '第四步未使用来源起始偏移')
|
||||
assert.match(previewSource, /sourceEnd/, '第四步未使用来源结束偏移')
|
||||
const lineRangeStart = previewSource.indexOf('function lineRange(item: PreviewItem)')
|
||||
const lineRangeEnd = previewSource.indexOf('\n}', lineRangeStart)
|
||||
const lineRangeSource = previewSource.slice(lineRangeStart, lineRangeEnd + 2)
|
||||
assert.match(lineRangeSource, /isManualPreviewItem\(item\)[\s\S]*?手动新增,无源文件定位/, '来源标签仍会把缺少行偏移的正常记录误判为手动新增')
|
||||
assert.match(lineRangeSource, /props\.processType === 'unstructured'[\s\S]*?来源:源文件记录/, '结构化来源记录缺少无行偏移时的准确标签')
|
||||
assert.doesNotMatch(lineRangeSource, /sourceStartLine == null[^\n]*手动新增/, '来源标签仍直接以缺少行号判定手动新增')
|
||||
assert.match(lineRangeSource, /sheet_name[\s\S]*?row_number[\s\S]*?来源:\$\{sheet\} · 第 \$\{locator\.row_number\} 行/, 'XLSX 来源标签没有展示工作表和物理行号')
|
||||
assert.match(lineRangeSource, /json_pointer[\s\S]*?JSON 路径/, 'JSON 来源标签没有展示 JSON 路径')
|
||||
assert.match(lineRangeSource, /locator\?\.kind === 'json'[\s\S]*?JSON 根对象/, 'JSON 根对象来源标签被空 JSON Pointer 错误降级')
|
||||
assert.match(lineRangeSource, /locatedLines[\s\S]*?第 \$\{locatedLines\.start\}[\s\S]*?locatedLines\.end/, 'JSONL/CSV 来源标签没有展示行范围')
|
||||
assert.match(lineRangeSource, /headingPath[\s\S]*?章节:/, '非结构化来源标签没有合并标题路径')
|
||||
assert.match(previewSource, /sourceLocator\?\.start_line[\s\S]*?sourceLocator\?\.end_line/, '文本预览没有优先使用后端行号定位')
|
||||
assert.match(previewSource, /sourceLocator\?\.source_start\s*\?\?\s*item\.sourceStart/, '文本预览没有优先使用 locator 字符起点')
|
||||
assert.match(previewSource, /sourceLocator\?\.source_end\s*\?\?\s*item\.sourceEnd/, '文本预览没有优先使用 locator 字符终点')
|
||||
assert.match(previewSource, /data-line-number="line\.number"/, '文本预览行缺少稳定行号定位标识')
|
||||
assert.match(previewSource, /isLineHighlighted\(line\.number, line\.start, line\.end\)/, '文本预览没有按物理行号高亮')
|
||||
assert.match(previewSource, /querySelector<HTMLElement>\(`\[data-line-number=/, '选中记录后没有按物理行号滚动定位')
|
||||
assert.match(previewSource, /const SOURCE_LINE_RENDER_LIMIT = 240/, '源文件查看器缺少安全渲染上限')
|
||||
assert.match(previewSource, /const SOURCE_LINE_CHARACTER_LIMIT = 4_000/, '源文件查看器缺少单行字符渲染上限')
|
||||
assert.match(previewSource, /sourceLineWindow\([\s\S]*?SOURCE_LINE_RENDER_LIMIT/, '源文件查看器没有使用有界行窗口')
|
||||
assert.match(previewSource, /SOURCE_LINE_RENDER_LIMIT,[\s\S]*?SOURCE_LINE_CHARACTER_LIMIT,[\s\S]*?selectedSourceLine\.value,[\s\S]*?selectedSourceOffset\.value/, '单行超大 JSON 没有围绕选中来源构建字符窗口')
|
||||
assert.match(previewSource, /sourceWindowStartLine/, '源文件查看器缺少窗口起始行状态')
|
||||
assert.match(previewSource, /showPreviousSourceWindow[\s\S]*?showNextSourceWindow/, '源文件查看器缺少前后窗口导航')
|
||||
assert.match(previewSource, /sourceLineNumberAtOffset\(props\.sourceText/, '仅有字符偏移时没有解析目标物理行')
|
||||
assert.match(previewSource, /filterable/, '文件选择器必须可搜索')
|
||||
assert.match(previewSource, /当前文件/, '预览缺少当前文件切换器')
|
||||
assert.doesNotMatch(previewSource, /located-badge|sync-label|已定位到/, '源文件栏不应显示冗余定位提示')
|
||||
@@ -269,6 +321,18 @@ for (const marker of [
|
||||
]) {
|
||||
assert.ok(officeViewerSource.includes(marker), `Word/XLSX 预览缺少结构或行为:${marker}`)
|
||||
}
|
||||
assert.match(officeViewerSource, /const selectedXlsxLocator = computed/, 'XLSX 查看器没有读取精确来源定位')
|
||||
assert.match(officeViewerSource, /row\.row_number === locator\.row_number/, 'XLSX 查看器没有按物理行号精确高亮')
|
||||
assert.match(officeViewerSource, /row\.record_index === locator\.sheet_record_index/, 'XLSX 查看器没有按工作表记录序号精确高亮')
|
||||
assert.match(officeViewerSource, /Math\.floor\(locator\.sheet_record_index \/ XLSX_PAGE_SIZE\) \* XLSX_PAGE_SIZE/, 'XLSX 查看器没有按记录序号自动计算分页')
|
||||
assert.match(officeViewerSource, /activeSheetIndex\.value = targetSheet[\s\S]*?pageOffset\.value = targetOffset[\s\S]*?loadPreview\(\)/, '切换记录时 XLSX 查看器没有自动切工作表和分页')
|
||||
const xlsxHighlightStart = officeViewerSource.indexOf('function xlsxRowHighlighted(')
|
||||
const xlsxHighlightEnd = officeViewerSource.indexOf('\n}', xlsxHighlightStart)
|
||||
const xlsxHighlightSource = officeViewerSource.slice(xlsxHighlightStart, xlsxHighlightEnd + 2)
|
||||
assert.ok(
|
||||
xlsxHighlightSource.indexOf('locator.row_number') < xlsxHighlightSource.indexOf('selectedRecordKey.value'),
|
||||
'XLSX 查看器没有把精确定位放在原内容比对 fallback 之前',
|
||||
)
|
||||
|
||||
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
|
||||
const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
|
||||
@@ -454,7 +518,7 @@ assert.match(
|
||||
)
|
||||
assert.match(
|
||||
workflowInitializationSource,
|
||||
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?goToStep\(resumeStep\)[\s\S]*?resumeGeneration/,
|
||||
/sourceTask\.status === 'running'[\s\S]*?resumeStep = 'generate'[\s\S]*?resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)/,
|
||||
'生成运行中时没有强制回到第五步并接管后台进度',
|
||||
)
|
||||
const startGenerationHandler = viewSource.slice(
|
||||
@@ -463,7 +527,35 @@ const startGenerationHandler = viewSource.slice(
|
||||
)
|
||||
assert.match(startGenerationHandler, /await persistWorkflowStep\('generate'\)[\s\S]*?await startGeneration\(\)[\s\S]*?dirty\.value = false/, '开始生成没有持久化第五步或启动真实后台任务')
|
||||
assert.doesNotMatch(startGenerationHandler, /router\.(?:push|replace)|allowLeave\s*=\s*true/, '开始生成后应停留在第五步,不得自动跳回列表')
|
||||
assert.match(viewSource, /:disabled="currentStepId === 'generate' \|\| previewBuilding \|\| sourceUploading"/, '第五步底部返回按钮没有固定禁用')
|
||||
assert.match(
|
||||
generationSource,
|
||||
/const canReturnFromGeneration = computed\(\(\) => \([\s\S]*?generation\.status === 'idle'[\s\S]*?!generationStarting\.value[\s\S]*?!generationRestoring\.value/,
|
||||
'第五步返回权限没有区分未启动、启动中和恢复中状态',
|
||||
)
|
||||
assert.match(
|
||||
viewSource,
|
||||
/:disabled="\(currentStepId === 'generate' && !canReturnFromGeneration\) \|\| previewBuilding \|\| sourceUploading"/,
|
||||
'第五步尚未启动生成时返回按钮仍被禁用',
|
||||
)
|
||||
const handleBackStart = viewSource.indexOf('async function handleBack()')
|
||||
const handleBackEnd = viewSource.indexOf('\n}', handleBackStart)
|
||||
const handleBackSource = viewSource.slice(handleBackStart, handleBackEnd + 2)
|
||||
assert.match(
|
||||
handleBackSource,
|
||||
/currentStepId\.value === 'generate' && !canReturnFromGeneration\.value/,
|
||||
'第五步处理函数仍无条件拦截返回',
|
||||
)
|
||||
assert.match(
|
||||
generationSource,
|
||||
/async function resumeGeneration\(\)[\s\S]*?generationRestoring\.value = true[\s\S]*?await getDataProcessProgress\(taskId\)[\s\S]*?generationRestoring\.value = false/,
|
||||
'恢复已启动任务时存在短暂可返回的 idle 窗口',
|
||||
)
|
||||
assert.match(viewSource, /const resume = resumeGeneration\(\)[\s\S]*?goToStep\(resumeStep\)[\s\S]*?await resume/, '第五步展示时未先启动恢复锁')
|
||||
assert.match(
|
||||
generationSource,
|
||||
/const generationStarting = ref\(false\)[\s\S]*?generationStarting\.value = true[\s\S]*?generationStarting\.value = false/,
|
||||
'点击开始生成后到请求启动前没有锁定返回状态',
|
||||
)
|
||||
assert.match(
|
||||
viewSource,
|
||||
/generation\.status === 'success'[\s\S]*?persistWorkflowStep\('results'\)/,
|
||||
@@ -506,32 +598,34 @@ assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTy
|
||||
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
|
||||
|
||||
assert.match(taskSetupSource, /v-if="processType === 'structured'"/, '结构化配置必须仅在结构化数据类型下显示')
|
||||
const expectedStructuredOptions = [
|
||||
['clean_invalid', '清理无效数据', '清理全空列,并剔除关键字段残缺的数据行'],
|
||||
const expectedStructuredGroups = [
|
||||
[
|
||||
'detect_structure',
|
||||
'嵌套结构展平',
|
||||
'展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||
"values: ['clean_invalid', 'deduplicate']",
|
||||
'数据清洗',
|
||||
'清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
|
||||
],
|
||||
[
|
||||
'deduplicate',
|
||||
'重复记录去重',
|
||||
'按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||
"values: ['detect_structure', 'normalize_format']",
|
||||
'结构标准化',
|
||||
'展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
|
||||
],
|
||||
['normalize_format', '数据格式标准化', '按所选规则统一编码、空白、字段名及 JSON 序列化格式'],
|
||||
['filter_anomaly', '异常数据过滤', '使用 IQR 识别数值离群值,并过滤乱码等异常记录'],
|
||||
['desensitize', '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||
["values: ['desensitize']", '敏感信息脱敏', '识别并脱敏姓名、手机号、邮箱和身份证号'],
|
||||
]
|
||||
for (const [value, label, description] of expectedStructuredOptions) {
|
||||
assert.ok(structuredOptionsSource.includes(`value: '${value}'`), `结构化预处理缺少值:${value}`)
|
||||
for (const [values, label, description] of expectedStructuredGroups) {
|
||||
assert.ok(structuredOptionsSource.includes(values), `结构化预处理组合值不准确:${label}`)
|
||||
assert.ok(structuredOptionsSource.includes(`label: '${label}'`), `结构化预处理缺少标签:${label}`)
|
||||
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${value}`)
|
||||
assert.ok(structuredOptionsSource.includes(`description: '${description}'`), `结构化预处理语义不准确:${label}`)
|
||||
}
|
||||
const structuredOptionValues = [...structuredOptionsSource.matchAll(/\{\s*value: '([^']+)',\s*label:/g)]
|
||||
.map((match) => match[1])
|
||||
assert.deepEqual(structuredOptionValues, expectedStructuredOptions.map(([value]) => value), '结构化预处理值集合不准确')
|
||||
assert.equal(new Set(structuredOptionValues).size, structuredOptionValues.length, '结构化预处理 value 必须唯一')
|
||||
assert.match(structuredOptionsSource, /Array\.from\(new Set\(value\.filter\(/, '结构化预处理选中值没有去重')
|
||||
assert.equal(expectedStructuredGroups.length, 3, '结构化预处理应收敛为 3 项')
|
||||
const preprocessGroupsSource = structuredOptionsSource.slice(
|
||||
structuredOptionsSource.indexOf('const PREPROCESS_GROUPS'),
|
||||
structuredOptionsSource.indexOf('const legacyAnomalyFilterEnabled'),
|
||||
)
|
||||
assert.doesNotMatch(preprocessGroupsSource, /异常数据过滤|filter_anomaly|IQR/, '结构化新任务仍暴露异常数据过滤')
|
||||
assert.match(structuredOptionsSource, /:indeterminate="groupIndeterminate\(group\.values\)"/, '历史部分选中的组合项没有半选回显')
|
||||
assert.match(structuredOptionsSource, /function updatePreprocessGroup\([\s\S]*?new Set\(props\.options\.preprocessOptions\)[\s\S]*?next\.add\(value\)[\s\S]*?next\.delete\(value\)[\s\S]*?\[\.\.\.next\]/, '结构化预处理组合开关没有原子化更新或去重内部选项')
|
||||
assert.match(typesSource, /仅用于恢复历史任务[\s\S]*?\| 'filter_anomaly'/, '异常数据过滤缺少历史兼容类型')
|
||||
assert.match(structuredOptionsSource, /legacyAnomalyFilterEnabled[\s\S]*?历史任务[\s\S]*?结果可复现/, '历史异常过滤配置没有透明提示')
|
||||
assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
|
||||
for (const splitName of ['训练集', '验证集', '测试集']) {
|
||||
assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分:${splitName}`)
|
||||
@@ -585,8 +679,20 @@ for (const extension of ['txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json',
|
||||
}
|
||||
assert.match(sourceUploadWorkerSource, /LEGACY_OFFICE_EXTENSIONS = new Set\(\['doc', 'xls', 'ppt'\]\)/, '缺少旧版 Office 格式识别')
|
||||
assert.ok(sourceUploadWorkerSource.includes('请分别转换为 DOCX、XLSX、PPTX 后上传'), '旧版 Office 文件缺少转换提示')
|
||||
assert.match(sourceUploadWorkerSource, /if \(!BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?TextDecoder/, '文本格式没有执行 UTF-8 客户端校验')
|
||||
assert.match(sourceUploadWorkerSource, /if \(BINARY_FILE_EXTENSIONS\.has\(job\.extension\)\) \{[\s\S]*?getDataProcessSourceContent\(currentTaskId, source\.id,[\s\S]*?start_line:\s*1,[\s\S]*?line_count:\s*10_000/, '二进制文档上传后没有读取后端解析文本')
|
||||
const sourceValidationStart = sourceUploadWorkerSource.indexOf('export function validateSourceFileSelection(')
|
||||
const sourceValidationEnd = sourceUploadWorkerSource.indexOf('\n}\n\nfunction unicodeCodePointLength', sourceValidationStart)
|
||||
assert.ok(sourceValidationStart >= 0 && sourceValidationEnd > sourceValidationStart, '无法定位源文件选择校验函数')
|
||||
const sourceValidationSource = sourceUploadWorkerSource.slice(sourceValidationStart, sourceValidationEnd + 2)
|
||||
assert.doesNotMatch(sourceValidationSource, /file\.name === raw\.name[\s\S]{0,160}file\.size === raw\.size|同名且同大小/, '不同内容但同名同大小的文件仍会被前端误拒绝')
|
||||
assert.match(sourceValidationSource, /selectedFiles\.length >= MAX_SOURCE_FILE_COUNT/, '移除伪重复校验时误删了文件数量限制')
|
||||
assert.match(sourceValidationSource, /selectedBytes \+ raw\.size > MAX_SOURCE_BATCH_BYTES/, '移除伪重复校验时误删了批次大小限制')
|
||||
assert.doesNotMatch(sourceUploadWorkerSource, /job\.file\.arrayBuffer\(|new TextDecoder/, '上传前仍把整个文本文件读入浏览器内存')
|
||||
assert.match(sourceUploadWorkerSource, /export async function loadCanonicalSourceContent[\s\S]*?offset,[\s\S]*?limit: SOURCE_CONTENT_PAGE_CHARS/, '服务端 canonical content 没有按有界字符窗口读取')
|
||||
assert.match(sourceUploadWorkerSource, /pending\.content = await loadCanonicalSourceContent\(currentTaskId, source\.id\)/, '上传成功后没有统一使用服务端 canonical content')
|
||||
assert.doesNotMatch(sourceUploadWorkerSource, /\brawFile:\s*job\.file\b/, '上传成功状态仍长期保留原始 File')
|
||||
assert.doesNotMatch(typesSource, /\brawFile\??:\s*File\b/, '上传状态类型仍长期持有原始 File')
|
||||
assert.doesNotMatch(viewSource, /\brawFile:\s*raw\b/, '待上传列表仍复制保存原始 File')
|
||||
assert.match(apiSource, /params:\s*\{[\s\S]*?offset\?: number[\s\S]*?limit\?: number[\s\S]*?\}/, '正文 API 前端契约缺少字符窗口参数')
|
||||
assert.match(apiSource, /formData\.append\('files', file\)/, '上传 API 没有使用 files 多文件表单字段')
|
||||
assert.match(apiSource, /onUploadProgress:[\s\S]*?event\.loaded \/ event\.total[\s\S]*?Math\.min\(99,/, '上传 API 没有接入真实字节进度或响应前未限制在 99%')
|
||||
assert.match(apiSource, /source-files`[\s\S]*?timeout: 5 \* 60 \* 1000/, '源文件上传缺少 5 分钟超时')
|
||||
@@ -609,7 +715,7 @@ assert.match(
|
||||
/export interface DataProcessPreviewProgress[\s\S]*?workflow_step: DataProcessWorkflowStep[\s\S]*?preview_status: DataProcessPreviewStatus[\s\S]*?preview_progress: number[\s\S]*?preview_run_id/,
|
||||
'后台切分进度契约缺少步骤、状态、进度或任务代次',
|
||||
)
|
||||
for (const field of ['rawFile', 'status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
|
||||
for (const field of ['status', 'uploadProgress', 'uploadError', 'previewStatus', 'previewProgress', 'previewError', 'previewConfigSignature']) {
|
||||
assert.ok(typesSource.includes(field), `上传文件缺少逐文件预览字段:${field}`)
|
||||
}
|
||||
assert.match(typesSource, /status: 'queued' \| 'uploading' \| 'ready' \| 'failed'/, '上传文件状态机不完整')
|
||||
@@ -853,13 +959,23 @@ const defaultStructuredPreprocess = defaultPreprocessValues(
|
||||
)
|
||||
assert.deepEqual(
|
||||
defaultStructuredPreprocess,
|
||||
['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
'结构化默认预处理配置不准确',
|
||||
[],
|
||||
'结构化新任务不应默认勾选预处理',
|
||||
)
|
||||
assert.equal(new Set(defaultStructuredPreprocess).size, defaultStructuredPreprocess.length, '结构化默认预处理值重复')
|
||||
const defaultUnstructuredPreprocess = defaultPreprocessValues('createDefaultUnstructuredOptions')
|
||||
assert.deepEqual(defaultUnstructuredPreprocess, expectedSmartPreprocessOptions, '智能预处理默认值不完整')
|
||||
assert.deepEqual(defaultUnstructuredPreprocess, [], '非结构化新任务不应默认勾选预处理')
|
||||
assert.equal(new Set(defaultUnstructuredPreprocess).size, defaultUnstructuredPreprocess.length, '非结构化默认预处理值重复')
|
||||
for (const field of ['preserveTables', 'preserveCodeBlocks', 'preserveLists']) {
|
||||
assert.match(
|
||||
stateSource,
|
||||
new RegExp(`${field}:\\s*false`),
|
||||
`非结构化预处理选项 ${field} 不应默认开启`,
|
||||
)
|
||||
}
|
||||
assert.match(structuredOptionsSource, /默认不执行预处理,请按数据情况自行选择/, '结构化预处理缺少默认不勾选说明')
|
||||
assert.match(unstructuredOptionsSource, /默认不执行预处理,请按文档情况自行选择/, '非结构化预处理缺少默认不勾选说明')
|
||||
assert.doesNotMatch(unstructuredOptionsSource, /默认启用结构感知/, '非结构化预处理仍保留默认启用的误导文案')
|
||||
|
||||
const backendConfigStart = viewSource.indexOf('function toBackendConfig()')
|
||||
const backendConfigEnd = viewSource.indexOf('function taskPayload()', backendConfigStart)
|
||||
@@ -934,7 +1050,7 @@ for (const [field, fallback] of [
|
||||
)
|
||||
}
|
||||
assert.match(regenerationSource, /getDataProcessTask\(sourceTaskId\.value\)/, '重新生成没有加载原任务')
|
||||
assert.match(regenerationSource, /while \(true\)[\s\S]*?getDataProcessSourceContent[\s\S]*?has_more/, '重新生成没有分页加载完整源正文')
|
||||
assert.match(regenerationSource, /loadCanonicalSourceContent\(taskId, file\.id\)/, '重新生成没有复用分页 canonical 正文加载器')
|
||||
assert.match(regenerationSource, /getDataProcessPreview\(taskId, \{ page: 1, page_size: 500 \}\)[\s\S]*?for \(let page = 2; page <= pages;/, '重新生成没有分页加载全部现有切片')
|
||||
assert.match(viewSource, /if \(hydrating\.value\) return/, '任务水合期间仍可能触发重置副作用')
|
||||
assert.match(regenerationSource, /currentSignature !== originalPreviewConfigSignature\.value[\s\S]*?currentSignature === confirmedPreviewConfigSignature\.value/, '切分变更确认没有按原签名和已确认签名去重')
|
||||
@@ -948,7 +1064,7 @@ assert.match(regenerationSource, /if \(regenerationPrepared\.value\) \{[\s\S]*?g
|
||||
assert.match(regenerationSource, /regenerationPrepared\.value = true/, '重新生成提交成功后没有记录服务端已变更状态')
|
||||
assert.match(regenerationSource, /hydrateWorkspace\(regeneratedTask, !regenerated\.preview_invalidated\)/, '重新生成没有按 preview_invalidated 决定保留或清空切片')
|
||||
assert.match(regenerationSource, /重新生成配置已保存,但工作区恢复失败/, '重新生成配置已保存但水合失败时缺少可恢复错误状态')
|
||||
assert.match(regenerationSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
|
||||
assert.match(sourceUploadWorkerSource, /return chunks\.join\(''\)/, '分页恢复源正文时不应额外插入换行')
|
||||
assert.doesNotMatch(regenerationSource, /binaryDocument[\s\S]*?mapDataProcessSourceFile\(file, ''\)/, '二进制源正文加载失败时不能静默降级为空内容')
|
||||
assert.match(nextFromModelSource, /if \(isRegeneration\.value\) \{[\s\S]*?prepareRegeneration\(taskPayload\(\)\)/, '重新生成每次从模型步骤继续时没有调用专用接口')
|
||||
assert.doesNotMatch(nextFromModelSource, /isRegeneration\.value && !taskId\.value/, '重新生成提交一次后可能错误转为普通任务更新')
|
||||
@@ -1017,6 +1133,17 @@ for (const mutationFunction of [
|
||||
const mutationSource = viewSource.slice(mutationStart, mutationEnd === -1 ? undefined : mutationEnd)
|
||||
assert.ok(mutationSource.includes('resetDownstream()'), `预览变更 ${mutationFunction} 后没有失效旧生成结果`)
|
||||
}
|
||||
const updatePreviewContentStart = viewSource.indexOf('function updatePreviewContent(')
|
||||
const updatePreviewContentEnd = viewSource.indexOf('\n}', updatePreviewContentStart)
|
||||
const updatePreviewContentSource = viewSource.slice(updatePreviewContentStart, updatePreviewContentEnd + 2)
|
||||
assert.match(updatePreviewContentSource, /isManualPreviewItem\(item\)/, '编辑预览内容仍未按稳定来源信息区分手动项')
|
||||
assert.doesNotMatch(updatePreviewContentSource, /sourceStart == null/, '结构化来源记录编辑后仍会被误标为手动项')
|
||||
const restorePreviewItemStart = viewSource.indexOf('function restorePreviewItem(')
|
||||
const restorePreviewItemEnd = viewSource.indexOf('\n}', restorePreviewItemStart)
|
||||
const restorePreviewItemSource = viewSource.slice(restorePreviewItemStart, restorePreviewItemEnd + 2)
|
||||
assert.match(restorePreviewItemSource, /isManualPreviewItem\(item\)/, '恢复预览内容没有使用统一的手动项判定')
|
||||
assert.doesNotMatch(restorePreviewItemSource, /sourceStart == null/, '结构化来源记录仍因缺少字符偏移而无法恢复')
|
||||
assert.match(previewSource, /v-if="!isManualPreviewItem\(editingItem\)"/, '结构化来源记录的恢复原文按钮仍被错误隐藏')
|
||||
assert.doesNotMatch(modelSource, /createResults\(/, '纯预览映射模块不应承担结果生成职责')
|
||||
|
||||
function findNextStyleBlockStart(source, startIndex) {
|
||||
|
||||
@@ -1,9 +1,62 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, onUnmounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import zhCn from 'element-plus/es/locale/lang/zh-cn'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { routeLoading } from '@/router'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import { SESSION_TIMEOUT } from '@/constants'
|
||||
|
||||
const router = useRouter()
|
||||
const auth = useAuthStore()
|
||||
|
||||
/**
|
||||
* 离开页面超时:
|
||||
* - 标签页切走/最小化(document.hidden)时记录时间
|
||||
* - 切回来时若超过 SESSION_TIMEOUT(5分钟),强制跳登录
|
||||
* - 不管是否在操作,只要离开页面超过 5 分钟就跳
|
||||
*/
|
||||
let hiddenAt = 0
|
||||
|
||||
async function handleVisibility() {
|
||||
if (document.hidden) {
|
||||
hiddenAt = Date.now()
|
||||
} else {
|
||||
if (hiddenAt > 0 && Date.now() - hiddenAt >= SESSION_TIMEOUT) {
|
||||
await auth.logout()
|
||||
ElMessage.warning('登录已过期,请重新登录')
|
||||
router.push('/login')
|
||||
}
|
||||
hiddenAt = 0
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
document.addEventListener('visibilitychange', handleVisibility)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
document.removeEventListener('visibilitychange', handleVisibility)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<el-config-provider :locale="zhCn">
|
||||
<router-view />
|
||||
<div v-loading="routeLoading" element-loading-text="加载中..." class="app-root">
|
||||
<router-view />
|
||||
</div>
|
||||
</el-config-provider>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
}
|
||||
.app-root {
|
||||
height: 100%;
|
||||
position: relative;
|
||||
}
|
||||
</style>
|
||||
|
||||
15
frontend/src/api/modules/acl.ts
Normal file
15
frontend/src/api/modules/acl.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { get, put } from '../request'
|
||||
|
||||
export interface AclEntry {
|
||||
subject_type: string
|
||||
subject_id: string
|
||||
permissions: string[]
|
||||
}
|
||||
|
||||
/** 资源 ACL 查询 */
|
||||
export const getAcl = (resourceType: string, resourceId: string) =>
|
||||
get<AclEntry[]>(`/resources/${resourceType}/${resourceId}/acl`)
|
||||
|
||||
/** 资源 ACL 设置 */
|
||||
export const setAcl = (resourceType: string, resourceId: string, entries: AclEntry[]) =>
|
||||
put<AclEntry[]>(`/resources/${resourceType}/${resourceId}/acl`, { entries })
|
||||
50
frontend/src/api/modules/approval.ts
Normal file
50
frontend/src/api/modules/approval.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import { get, post } from '../request'
|
||||
|
||||
export interface ApprovalStep {
|
||||
approver_id?: string | null
|
||||
status: string
|
||||
}
|
||||
|
||||
export interface ApprovalTemplate {
|
||||
id: string
|
||||
name: string
|
||||
steps: ApprovalStep[]
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
export interface ApprovalInstance {
|
||||
id: string
|
||||
template_id?: string | null
|
||||
resource_type: string
|
||||
resource_id: string
|
||||
applicant_id: string
|
||||
status: string
|
||||
current_step: number
|
||||
create_time?: string
|
||||
steps: Array<ApprovalStep & { step_index: number; comment?: string | null; time?: string | null }>
|
||||
}
|
||||
|
||||
export const getApprovalTemplates = () =>
|
||||
get<ApprovalTemplate[]>('/approvals/templates')
|
||||
|
||||
export const createApprovalTemplate = (payload: { name: string; steps: ApprovalStep[] }) =>
|
||||
post<ApprovalTemplate>('/approvals/templates', payload)
|
||||
|
||||
export const getApprovalInstances = (status?: string) =>
|
||||
get<ApprovalInstance[]>('/approvals', { status })
|
||||
|
||||
export const createApprovalInstance = (payload: {
|
||||
template_id?: string
|
||||
resource_type: string
|
||||
resource_id: string
|
||||
applicant_id: string
|
||||
}) => post<ApprovalInstance>('/approvals', payload)
|
||||
|
||||
export const getApprovalInstance = (id: string) =>
|
||||
get<ApprovalInstance>(`/approvals/${id}`)
|
||||
|
||||
export const decideApproval = (
|
||||
id: string,
|
||||
step_index: number,
|
||||
payload: { approver_id: string; approved: boolean; comment?: string },
|
||||
) => post<ApprovalInstance>(`/approvals/${id}/steps/${step_index}/decision`, payload)
|
||||
40
frontend/src/api/modules/audit.ts
Normal file
40
frontend/src/api/modules/audit.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { get } from '../request'
|
||||
import request from '../request'
|
||||
|
||||
export interface AuditLog {
|
||||
id: string
|
||||
tenant_id?: string
|
||||
project_id?: string
|
||||
actor_id?: string
|
||||
action?: string
|
||||
target_type?: string
|
||||
target_id?: string
|
||||
detail?: string
|
||||
client_ip?: string
|
||||
time?: string
|
||||
}
|
||||
|
||||
export interface AuditQuery {
|
||||
tenant_id?: string
|
||||
project_id?: string
|
||||
actor_id?: string
|
||||
action?: string
|
||||
target_type?: string
|
||||
start_time?: string
|
||||
end_time?: string
|
||||
limit?: number
|
||||
offset?: number
|
||||
}
|
||||
|
||||
/** 审计日志查询:使用 get 辅助函数,拦截器已解包,直接返回 { items, total } */
|
||||
export const getAuditLogs = (query: AuditQuery = {}) =>
|
||||
get<{ items: AuditLog[]; total: number }>('/system/audit-logs', query)
|
||||
|
||||
/** 审计日志导出 CSV:blob 响应走完整 axios response,需手动取 data */
|
||||
export const exportAuditLogs = (query: AuditQuery = {}) =>
|
||||
request<Blob>({
|
||||
url: '/system/audit-logs/export',
|
||||
method: 'get',
|
||||
params: query,
|
||||
responseType: 'blob',
|
||||
}).then((res) => res.data)
|
||||
@@ -1,6 +1,8 @@
|
||||
import { get, post, del } from '../request'
|
||||
import type { CompareTask, CompareModelRef } from '@/types'
|
||||
|
||||
const INFERENCE_START_TIMEOUT_MS = 15 * 60 * 1000
|
||||
|
||||
/** 推理/对比任务列表 */
|
||||
export const getCompareList = () => get<CompareTask[]>('/model-compare')
|
||||
|
||||
@@ -12,7 +14,7 @@ export const createCompare = (data: Partial<CompareTask>) =>
|
||||
post<{ id: string | number }>('/model-compare', data)
|
||||
|
||||
/** 删除任务 */
|
||||
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`)
|
||||
export const deleteCompare = (id: string | number) => del(`/model-compare/${id}`, undefined, { timeout: 60_000 })
|
||||
|
||||
/** 更新任务加载状态 */
|
||||
export const updateLoadStatus = (id: string | number, load_status: any) =>
|
||||
@@ -34,7 +36,8 @@ export const stopModelByPid = (pid: number) =>
|
||||
post('/model-compare/stop-by-pid', { pid })
|
||||
|
||||
/** 加载任务 */
|
||||
export const loadCompare = (id: string | number) => post(`/model-compare/${id}/load`)
|
||||
export const loadCompare = (id: string | number) =>
|
||||
post(`/model-compare/${id}/load`, undefined, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
/** 卸载任务 */
|
||||
export const unloadCompare = (id: string | number) => post(`/model-compare/${id}/unload`)
|
||||
@@ -64,6 +67,31 @@ export const streamChat = async (data: any): Promise<any> => {
|
||||
}
|
||||
}
|
||||
|
||||
/** 真实流式对话 — 使用 fetch 调用后端 SSE 端点,返回 Response 供 ReadableStream 消费 */
|
||||
export const streamChatReal = (data: any): Promise<Response> => {
|
||||
const messages = data.messages || []
|
||||
if (!messages.length && data.user_question) {
|
||||
if (data.system_prompt) {
|
||||
messages.push({ role: 'system', content: data.system_prompt })
|
||||
}
|
||||
messages.push({ role: 'user', content: data.user_question })
|
||||
}
|
||||
return fetch('/modelTF/model-compare/stream-chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
messages,
|
||||
temperature: data.temperature ?? 0.7,
|
||||
top_p: data.top_p ?? 0.95,
|
||||
max_tokens: data.max_tokens ?? 2048,
|
||||
// 透传 task_id/node_id,让后端按 load_status 路由到真正加载了模型的算力节点,
|
||||
// 避免在多节点时回退到“第一个在线节点”导致连接失败
|
||||
task_id: data.task_id,
|
||||
node_id: data.node_id,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/** 非流式对话(按端口代理) */
|
||||
export const chatWithPort = (data: any) => post('/model-compare/chat-with-port', data)
|
||||
|
||||
@@ -73,8 +101,8 @@ export const batchChat = (data: any) => post('/model-chat/batch', data)
|
||||
/** 本地 transformers 模型对话 */
|
||||
export const localChat = (data: any) => post('/model-chat/local/chat', data)
|
||||
|
||||
/** 预加载本地模型 */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data)
|
||||
/** 预加载本地模型(模型加载耗时长,超时 15 分钟) */
|
||||
export const preloadLocalModel = (data: any) => post('/model-chat/local/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
/** 预加载已训练模型 */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data)
|
||||
/** 预加载已训练模型(超时 15 分钟) */
|
||||
export const preloadTrainedModel = (data: any) => post('/model-chat/trained/preload', data, { timeout: INFERENCE_START_TIMEOUT_MS })
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { get, post, put } from '../request'
|
||||
import { del, get, post, put } from '../request'
|
||||
|
||||
export interface ComputeNode {
|
||||
id: string
|
||||
@@ -95,6 +95,9 @@ export const createComputeNode = (data: ComputeNodePayload) =>
|
||||
export const updateComputeNode = (id: string, data: Partial<ComputeNode>) =>
|
||||
put<ComputeNode>(`/compute/nodes/${id}`, data)
|
||||
|
||||
export const deleteComputeNode = (id: string) =>
|
||||
del<{ deleted: string }>(`/compute/nodes/${id}`)
|
||||
|
||||
export const testComputeNode = (id: string) =>
|
||||
post<{ node_id: string; success: boolean; latency_ms: number; gpu_count: number; error?: string }>(`/compute/nodes/${id}/test-connection`)
|
||||
|
||||
|
||||
35
frontend/src/api/modules/dashboard.ts
Normal file
35
frontend/src/api/modules/dashboard.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { get } from '../request'
|
||||
|
||||
export interface ServiceStatusStat {
|
||||
type: string
|
||||
status: 'normal' | 'busy' | 'error'
|
||||
count: number
|
||||
}
|
||||
|
||||
export interface TrainingTaskStat {
|
||||
id: string
|
||||
name: string
|
||||
status: string
|
||||
train_type: string
|
||||
train_method: string
|
||||
base_model: string
|
||||
progress: number
|
||||
accuracy: number | null
|
||||
started_at: string
|
||||
}
|
||||
|
||||
export interface DashboardStats {
|
||||
online_services: number
|
||||
running_tasks: number
|
||||
pending_alerts: number
|
||||
training_7d: { date: string; train: number; gpu: number; accuracy: number | null }[]
|
||||
service_status: ServiceStatusStat[]
|
||||
training_tasks: TrainingTaskStat[]
|
||||
operation_distribution: { name: string; value: number }[]
|
||||
login_duration_rank: { user: string; role: string; duration: number }[]
|
||||
recent_login_users: { user: string; role: string; last_login: string }[]
|
||||
}
|
||||
|
||||
export function getDashboardStats() {
|
||||
return get<DashboardStats>('/dashboard/stats')
|
||||
}
|
||||
@@ -14,6 +14,8 @@ import type {
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessRepeatPayload,
|
||||
DataProcessRepeatResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
@@ -56,6 +58,8 @@ export type {
|
||||
DataProcessProgress,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessRegenerateResult,
|
||||
DataProcessRepeatPayload,
|
||||
DataProcessRepeatResult,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessPublishResult,
|
||||
DataProcessQualityScore,
|
||||
@@ -117,6 +121,15 @@ export const regenerateDataProcessTask = (
|
||||
payload,
|
||||
)
|
||||
|
||||
export const repeatDataProcessTask = (
|
||||
taskId: string | number,
|
||||
payload: DataProcessRepeatPayload,
|
||||
) => post<DataProcessRepeatResult>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/repeat`,
|
||||
payload,
|
||||
{ timeout: 5 * 60 * 1000 },
|
||||
)
|
||||
|
||||
export const deleteDataProcessTask = (taskId: string | number) =>
|
||||
del<{ deleted: string | number }>(`/data-process/${encodeURIComponent(taskId)}`)
|
||||
|
||||
@@ -150,7 +163,12 @@ export const deleteDataProcessSourceFile = (taskId: string | number, fileId: str
|
||||
export const getDataProcessSourceContent = (
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
params: { start_line?: number; line_count?: number } = {},
|
||||
params: {
|
||||
start_line?: number
|
||||
line_count?: number
|
||||
offset?: number
|
||||
limit?: number
|
||||
} = {},
|
||||
) => get<DataProcessSourceContent>(
|
||||
`/data-process/${encodeURIComponent(taskId)}/source-files/${encodeURIComponent(fileId)}/content`,
|
||||
params,
|
||||
|
||||
@@ -29,6 +29,7 @@ export const uploadDatasetFiles = (datasetId: string | number, files: File[]) =>
|
||||
files.forEach((f) => formData.append('files', f))
|
||||
return post(`/dataset-manage/upload/${datasetId}`, formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' },
|
||||
timeout: 120000,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
import { get, post, put, del } from '../request'
|
||||
import type { FineTuneStartPayload, FineTuneTask, TrainingProgress, LogContent } from '@/types'
|
||||
|
||||
export interface FineTuneMetricPoint {
|
||||
step: number
|
||||
epoch?: number | null
|
||||
loss?: number | null
|
||||
grad_norm?: number | null
|
||||
learning_rate?: number | null
|
||||
raw?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
export interface TrainingDiagnostic {
|
||||
level: string
|
||||
title: string
|
||||
@@ -80,6 +90,10 @@ export const getFineTuneLogs = (
|
||||
params: { tail_lines?: number; offset?: number; limit?: number } = {},
|
||||
) => get<LogContent & { job_id?: string; source?: string }>(`/fine-tune/${id}/logs`, params)
|
||||
|
||||
/** 获取训练指标曲线数据 */
|
||||
export const getFineTuneMetrics = (id: string | number) =>
|
||||
get<FineTuneMetricPoint[]>(`/fine-tune/${id}/metrics`)
|
||||
|
||||
/** 启动 TensorBoard */
|
||||
export const startTensorboard = () => post('/fine-tune/tensorboard/start')
|
||||
|
||||
|
||||
@@ -88,10 +88,14 @@ export const updateModelPurpose = (id: string | number, purpose: string) =>
|
||||
|
||||
/** 合并 LoRA 权重 */
|
||||
export const mergeModel = (data: {
|
||||
trained_model_id?: string | number
|
||||
model_name: string
|
||||
train_method: string
|
||||
base_model_path: string
|
||||
}) => post('/model-manage/merge', data)
|
||||
adapter_path?: string
|
||||
compute_node_id?: string
|
||||
output_model_name?: string
|
||||
}) => post('/model-manage/merge', data, { timeout: 15 * 60 * 1000 })
|
||||
|
||||
/** 导出已训练模型权重 */
|
||||
export const exportModelUrl = (modelName: string) =>
|
||||
|
||||
58
frontend/src/api/modules/project.ts
Normal file
58
frontend/src/api/modules/project.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { del, get, post, put } from '../request'
|
||||
|
||||
export interface Project {
|
||||
id: string
|
||||
tenant_id: string
|
||||
name: string
|
||||
code: string
|
||||
description?: string
|
||||
status: string
|
||||
quota?: Record<string, unknown>
|
||||
member_count?: number
|
||||
task_count?: number
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
export interface ProjectMember {
|
||||
user_id: string
|
||||
role: string
|
||||
joined_at?: string
|
||||
}
|
||||
|
||||
/** 项目列表(按租户过滤,默认 default) */
|
||||
export const getProjects = (tenantId = 'default') =>
|
||||
get<Project[]>('/projects', { tenant_id: tenantId })
|
||||
|
||||
/** 项目详情 */
|
||||
export const getProject = (id: string) => get<Project>(`/projects/${id}`)
|
||||
|
||||
/** 创建项目 */
|
||||
export const createProject = (payload: Partial<Project>) =>
|
||||
post<Project>('/projects', payload)
|
||||
|
||||
/** 更新项目 */
|
||||
export const updateProject = (id: string, payload: Partial<Project>) =>
|
||||
put<Project>(`/projects/${id}`, payload)
|
||||
|
||||
/** 归档项目 */
|
||||
export const archiveProject = (id: string) =>
|
||||
post<Project>(`/projects/${id}/archive`)
|
||||
|
||||
/** 项目成员列表 */
|
||||
export const getProjectMembers = (id: string) =>
|
||||
get<ProjectMember[]>(`/projects/${id}/members`)
|
||||
|
||||
/** 添加成员 */
|
||||
export const addProjectMember = (id: string, payload: { user_id: string; role: string }) =>
|
||||
post<ProjectMember>(`/projects/${id}/members`, payload)
|
||||
|
||||
/** 更新成员角色 */
|
||||
export const updateProjectMember = (id: string, userId: string, role: string) =>
|
||||
put<ProjectMember>(`/projects/${id}/members/${userId}`, { role })
|
||||
|
||||
/** 移除成员 */
|
||||
export const removeProjectMember = (id: string, userId: string) =>
|
||||
del(`/projects/${id}/members/${userId}`)
|
||||
|
||||
/** 删除项目 */
|
||||
export const deleteProject = (id: string) => del(`/projects/${id}`)
|
||||
29
frontend/src/api/modules/retention.ts
Normal file
29
frontend/src/api/modules/retention.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { del, get, post, put } from '../request'
|
||||
|
||||
export interface RetentionPolicy {
|
||||
id: string
|
||||
name: string
|
||||
scope?: string | null
|
||||
rule?: string | null
|
||||
status: string
|
||||
create_time?: string
|
||||
create_by?: string | null
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
/** 留存策略列表 */
|
||||
export const getRetentionPolicies = () => get<RetentionPolicy[]>('/retention-policies')
|
||||
|
||||
/** 留存策略详情 */
|
||||
export const getRetentionPolicy = (id: string) => get<RetentionPolicy>(`/retention-policies/${id}`)
|
||||
|
||||
/** 创建留存策略 */
|
||||
export const createRetentionPolicy = (payload: Partial<RetentionPolicy>) =>
|
||||
post<RetentionPolicy>('/retention-policies', payload)
|
||||
|
||||
/** 更新留存策略 */
|
||||
export const updateRetentionPolicy = (id: string, payload: Partial<RetentionPolicy>) =>
|
||||
put<RetentionPolicy>(`/retention-policies/${id}`, payload)
|
||||
|
||||
/** 删除留存策略 */
|
||||
export const deleteRetentionPolicy = (id: string) => del(`/retention-policies/${id}`)
|
||||
@@ -8,6 +8,8 @@ import type {
|
||||
UpdateUserAccessPayload,
|
||||
} from '@/types'
|
||||
|
||||
export type { SystemUser } from '@/types'
|
||||
|
||||
/** 系统信息(CPU/内存/磁盘/GPU/网络/系统) */
|
||||
export const getSystemInfo = () => get<SystemInfo>('/system-info')
|
||||
|
||||
@@ -18,6 +20,10 @@ export const getHealth = () => get<HealthMetrics>('/health')
|
||||
export const login = (username: string, password: string) =>
|
||||
post<LoginResponse>('/login', { username, password })
|
||||
|
||||
/** 登出 */
|
||||
export const logout = (sessionId?: string) =>
|
||||
post('/logout', { session_id: sessionId || '' })
|
||||
|
||||
/** 用户列表 */
|
||||
export const getUsers = () => get<SystemUser[]>('/users')
|
||||
|
||||
@@ -25,12 +31,14 @@ export const getUsers = () => get<SystemUser[]>('/users')
|
||||
export const createUser = (payload: CreateUserPayload) =>
|
||||
post<SystemUser>('/users', payload)
|
||||
|
||||
/** 删除用户,currentUsername 用于防止删除当前登录账号 */
|
||||
export const deleteUser = (id: string, currentUsername: string) =>
|
||||
del<{ deleted: string }>(`/users/${encodeURIComponent(id)}`, {
|
||||
current_username: currentUsername,
|
||||
})
|
||||
|
||||
/** 更新用户角色、状态及页面权限 */
|
||||
export const updateUserAccess = (id: string, payload: UpdateUserAccessPayload) =>
|
||||
put<SystemUser>(`/users/${encodeURIComponent(id)}`, payload)
|
||||
|
||||
/** 重置用户密码 */
|
||||
export const resetUserPassword = (id: string, password?: string) =>
|
||||
post<{ reset: string }>(`/users/${encodeURIComponent(id)}/reset-password`, { password })
|
||||
|
||||
/** 删除用户(protected 管理员账号不允许删除) */
|
||||
export const deleteUser = (id: string) =>
|
||||
del<{ deleted: string }>(`/users/${encodeURIComponent(id)}`)
|
||||
|
||||
37
frontend/src/api/modules/tenant.ts
Normal file
37
frontend/src/api/modules/tenant.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { del, get, post, put } from '../request'
|
||||
|
||||
export interface Tenant {
|
||||
id: string
|
||||
name: string
|
||||
code: string
|
||||
status: string
|
||||
owner_user_id?: string | null
|
||||
quota: Record<string, unknown>
|
||||
retention_policy_id?: string | null
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
/** 租户列表 */
|
||||
export const getTenants = () => get<Tenant[]>('/tenants')
|
||||
|
||||
/** 租户详情 */
|
||||
export const getTenant = (id: string) => get<Tenant>(`/tenants/${id}`)
|
||||
|
||||
/** 创建租户 */
|
||||
export const createTenant = (payload: Partial<Tenant>) =>
|
||||
post<Tenant>('/tenants', payload)
|
||||
|
||||
/** 更新租户 */
|
||||
export const updateTenant = (id: string, payload: Partial<Tenant>) =>
|
||||
put<Tenant>(`/tenants/${id}`, payload)
|
||||
|
||||
/** 删除租户 */
|
||||
export const deleteTenant = (id: string) => del(`/tenants/${id}`)
|
||||
|
||||
/** 设置租户配额 */
|
||||
export const setTenantQuota = (id: string, quota: Record<string, unknown>) =>
|
||||
put<Tenant>(`/tenants/${id}/quota`, { quota })
|
||||
|
||||
/** 设置租户留存策略 */
|
||||
export const setTenantRetention = (id: string, retention_policy_id: string) =>
|
||||
put<Tenant>(`/tenants/${id}/retention-policy`, { retention_policy_id })
|
||||
@@ -1,6 +1,5 @@
|
||||
import axios, { type AxiosInstance, type AxiosRequestConfig } from 'axios'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { touchSessionActivity } from '@/utils/sessionActivity'
|
||||
|
||||
/**
|
||||
* 后端统一响应格式
|
||||
@@ -15,12 +14,40 @@ export interface ApiResult<T = any> {
|
||||
const service: AxiosInstance = axios.create({
|
||||
// Use a relative path; Vite proxies /modelTF to http://localhost:17861 in local development.
|
||||
baseURL: '/modelTF',
|
||||
timeout: 30000,
|
||||
timeout: 120000,
|
||||
})
|
||||
|
||||
// 请求拦截器
|
||||
/**
|
||||
* 从 localStorage 取当前用户 token(登录时后端返回 platform-token-{user_id})。
|
||||
* 后端鉴权中间件依赖此 header 解析当前用户身份。
|
||||
*/
|
||||
function getAuthToken(): string | null {
|
||||
const USER_STORAGE_KEY = 'currentUser'
|
||||
const raw = localStorage.getItem(USER_STORAGE_KEY)
|
||||
if (raw) {
|
||||
try {
|
||||
const user = JSON.parse(raw)
|
||||
// 后端 login 返回的 token 格式为 platform-token-{user.id}
|
||||
if (user?.id) return `platform-token-${user.id}`
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
// 兼容改造前 admin 会话
|
||||
if (localStorage.getItem('username') === 'admin') return 'platform-token-admin'
|
||||
return null
|
||||
}
|
||||
|
||||
// 请求拦截器:注入 Authorization header
|
||||
service.interceptors.request.use(
|
||||
(config) => config,
|
||||
(config) => {
|
||||
const token = getAuthToken()
|
||||
if (token) {
|
||||
config.headers = config.headers || {}
|
||||
config.headers['Authorization'] = `Bearer ${token}`
|
||||
}
|
||||
return config
|
||||
},
|
||||
(error) => Promise.reject(error),
|
||||
)
|
||||
|
||||
@@ -30,12 +57,9 @@ service.interceptors.response.use(
|
||||
const res = response.data as ApiResult
|
||||
// 二进制流等非 JSON 响应直接返回
|
||||
if (response.config.responseType === 'blob' || response.config.responseType === 'arraybuffer') {
|
||||
touchSessionActivity()
|
||||
return response
|
||||
}
|
||||
if (res.code === 0) {
|
||||
// 生成进度轮询也属于用户正在使用系统,避免长任务结束后被误判为会话过期。
|
||||
touchSessionActivity()
|
||||
return res.data
|
||||
}
|
||||
// 业务错误
|
||||
|
||||
92
frontend/src/components/AclDialog.vue
Normal file
92
frontend/src/components/AclDialog.vue
Normal file
@@ -0,0 +1,92 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { getAcl, setAcl, type AclEntry } from '@/api/modules/acl'
|
||||
import { getUsers, type SystemUser } from '@/api/modules/system'
|
||||
|
||||
const props = defineProps<{
|
||||
modelValue: boolean
|
||||
resourceType: string
|
||||
resourceId: string
|
||||
}>()
|
||||
const emit = defineEmits<{ 'update:modelValue': [boolean] }>()
|
||||
|
||||
const visible = computed({
|
||||
get: () => props.modelValue,
|
||||
set: (v) => emit('update:modelValue', v),
|
||||
})
|
||||
const entries = ref<AclEntry[]>([])
|
||||
const users = ref<SystemUser[]>([])
|
||||
const loading = ref(false)
|
||||
const ALL_PERMS = ['read', 'write', 'execute', 'download', 'delete', 'share']
|
||||
const PROJECT_ROLES = ['member', 'admin', 'viewer']
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const [acl, us] = await Promise.all([
|
||||
getAcl(props.resourceType, props.resourceId),
|
||||
getUsers().catch(() => [] as SystemUser[]),
|
||||
])
|
||||
entries.value = acl
|
||||
users.value = us
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch(visible, (v) => { if (v) load() })
|
||||
|
||||
function addEntry() {
|
||||
entries.value.push({ subject_type: 'user', subject_id: '', permissions: [] })
|
||||
}
|
||||
|
||||
function removeEntry(idx: number) {
|
||||
entries.value.splice(idx, 1)
|
||||
}
|
||||
|
||||
async function save() {
|
||||
await setAcl(props.resourceType, props.resourceId, entries.value)
|
||||
ElMessage.success('ACL 已保存')
|
||||
visible.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<el-dialog v-model="visible" title="资源授权 (ACL)" width="640px">
|
||||
<div v-loading="loading">
|
||||
<el-button type="primary" size="small" @click="addEntry">添加授权项</el-button>
|
||||
<div v-for="(entry, idx) in entries" :key="idx" class="acl-row">
|
||||
<el-select v-model="entry.subject_type" style="width: 140px">
|
||||
<el-option label="用户" value="user" />
|
||||
<el-option label="项目角色" value="project_role" />
|
||||
</el-select>
|
||||
<el-select v-if="entry.subject_type === 'user'" v-model="entry.subject_id" placeholder="选择用户" style="width: 200px" filterable>
|
||||
<el-option v-for="u in users" :key="u.id" :label="`${u.username} (${u.id})`" :value="u.id" />
|
||||
</el-select>
|
||||
<el-select v-else v-model="entry.subject_id" placeholder="选择角色" style="width: 200px">
|
||||
<el-option v-for="r in PROJECT_ROLES" :key="r" :label="r" :value="r" />
|
||||
</el-select>
|
||||
<el-checkbox-group v-model="entry.permissions">
|
||||
<el-checkbox v-for="p in ALL_PERMS" :key="p" :value="p">{{ p }}</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
<el-button link type="danger" @click="removeEntry(idx)">删除</el-button>
|
||||
</div>
|
||||
<el-empty v-if="entries.length === 0" description="暂无授权" />
|
||||
</div>
|
||||
<template #footer>
|
||||
<el-button @click="visible = false">取消</el-button>
|
||||
<el-button type="primary" @click="save">保存</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.acl-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
</style>
|
||||
@@ -74,6 +74,16 @@ const menuGroups: MenuGroup[] = [
|
||||
{ key: 'compute', label: '算力节点', icon: 'fa-microchip', to: '/compute', permission: 'compute' },
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '平台治理',
|
||||
items: [
|
||||
{ key: 'tenants', label: '租户管理', icon: 'fa-building', to: '/tenants', permission: 'user-settings' },
|
||||
{ key: 'projects', label: '项目空间', icon: 'fa-folder', to: '/projects', permission: 'user-settings' },
|
||||
{ key: 'audit-logs', label: '审计日志', icon: 'fa-history', to: '/audit-logs', permission: 'user-settings' },
|
||||
{ key: 'approval-templates', label: '审批模板', icon: 'fa-list-alt', to: '/approval-templates', permission: 'user-settings' },
|
||||
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' },
|
||||
],
|
||||
},
|
||||
{
|
||||
title: '系统设置',
|
||||
items: [
|
||||
@@ -122,8 +132,8 @@ async function handleSelect(key: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function handleLogout() {
|
||||
auth.logout()
|
||||
async function handleLogout() {
|
||||
await auth.logout()
|
||||
router.push('/login')
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ref } from 'vue'
|
||||
import { streamChat } from '@/api/modules/compare'
|
||||
import { streamChat, streamChatReal } from '@/api/modules/compare'
|
||||
|
||||
export interface StreamMessage {
|
||||
/** 用户问题 */
|
||||
@@ -20,6 +20,11 @@ export interface StreamMessage {
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface SendOptions {
|
||||
/** 是否使用 mock 模式(默认 true,向后兼容) */
|
||||
useMock?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式对话 composable
|
||||
* 移植自原 model-chat.html:
|
||||
@@ -39,6 +44,24 @@ export function useStreamChat() {
|
||||
})
|
||||
const loading = ref(false)
|
||||
|
||||
/** 从 SSE 帧中提取错误信息(后端/计算节点错误以 data: {"error": "..."} 形式下发) */
|
||||
function extractSseError(buffer: string): string | null {
|
||||
const trimmed = buffer.trim()
|
||||
if (!trimmed.startsWith('data: ')) return null
|
||||
const lines = trimmed.split(/\r?\n/)
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const line = lines[i].trim()
|
||||
if (!line.startsWith('data: ')) continue
|
||||
try {
|
||||
const obj = JSON.parse(line.slice(6))
|
||||
if (obj && typeof obj.error === 'string' && obj.error) return obj.error
|
||||
} catch {
|
||||
/* 非 JSON 的 data 行忽略 */
|
||||
}
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
/** 从内容中解析 think 标签 */
|
||||
function parseContent(content: string) {
|
||||
const thinkRegex = /<think>([\s\S]*?)(<\/think>)?/g
|
||||
@@ -65,8 +88,10 @@ export function useStreamChat() {
|
||||
/**
|
||||
* 发起流式对话
|
||||
* @param payload 后端请求体 { port, model_name, model_path, system_prompt, user_question, ... }
|
||||
* @param options 可选配置 { useMock?: boolean }
|
||||
*/
|
||||
async function send(payload: any) {
|
||||
async function send(payload: any, options?: SendOptions) {
|
||||
const useMock = options?.useMock ?? true
|
||||
loading.value = true
|
||||
message.value = {
|
||||
question: payload.user_question || '',
|
||||
@@ -82,7 +107,10 @@ export function useStreamChat() {
|
||||
const UPDATE_INTERVAL = 50 // 50ms 节流
|
||||
|
||||
try {
|
||||
const response = await streamChat(payload)
|
||||
const response = useMock
|
||||
? await streamChat(payload)
|
||||
: await streamChatReal(payload)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`)
|
||||
}
|
||||
@@ -111,6 +139,16 @@ export function useStreamChat() {
|
||||
}
|
||||
|
||||
// 最终更新
|
||||
// 若整段响应是 SSE 错误帧,提取 error 字段以干净文案展示
|
||||
const sseError = extractSseError(buffer)
|
||||
if (sseError) {
|
||||
message.value.isThinking = false
|
||||
message.value.isStreaming = false
|
||||
message.value.done = true
|
||||
message.value.error = sseError
|
||||
message.value.displayContent = sseError
|
||||
return
|
||||
}
|
||||
const parsed = parseContent(buffer)
|
||||
message.value.thinkContent = parsed.think
|
||||
message.value.displayContent = parsed.display
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { createRouter, createWebHistory, type RouteRecordRaw } from 'vue-router'
|
||||
import { ref } from 'vue'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import type { PermissionCode } from '@/types'
|
||||
|
||||
/** 路由切换时的全局加载态,供 App.vue 显示全屏转圈遮罩,消除懒加载时的空白卡顿感 */
|
||||
export const routeLoading = ref(false)
|
||||
|
||||
const routes: RouteRecordRaw[] = [
|
||||
{
|
||||
path: '/login',
|
||||
@@ -27,6 +31,49 @@ const routes: RouteRecordRaw[] = [
|
||||
component: () => import('@/views/dashboard/DashboardView.vue'),
|
||||
meta: { title: '服务看板' },
|
||||
},
|
||||
// 平台治理
|
||||
{
|
||||
path: 'tenants',
|
||||
name: 'tenants',
|
||||
component: () => import('@/views/tenants/TenantListView.vue'),
|
||||
meta: { title: '租户管理', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'tenants/:id',
|
||||
name: 'tenant-detail',
|
||||
component: () => import('@/views/tenants/TenantDetailView.vue'),
|
||||
meta: { title: '租户详情', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'projects',
|
||||
name: 'projects',
|
||||
component: () => import('@/views/projects/ProjectListView.vue'),
|
||||
meta: { title: '项目空间', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'projects/:id',
|
||||
name: 'project-detail',
|
||||
component: () => import('@/views/projects/ProjectDetailView.vue'),
|
||||
meta: { title: '项目详情', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'audit-logs',
|
||||
name: 'audit-logs',
|
||||
component: () => import('@/views/audit/AuditLogView.vue'),
|
||||
meta: { title: '审计日志', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'approval-templates',
|
||||
name: 'approval-templates',
|
||||
component: () => import('@/views/approvals/ApprovalTemplateView.vue'),
|
||||
meta: { title: '审批模板', permission: 'user-settings' },
|
||||
},
|
||||
{
|
||||
path: 'approval-instances',
|
||||
name: 'approval-instances',
|
||||
component: () => import('@/views/approvals/ApprovalInstanceView.vue'),
|
||||
meta: { title: '审批中心', permission: 'user-settings' },
|
||||
},
|
||||
// 模型调优
|
||||
{
|
||||
path: 'fine-tune',
|
||||
@@ -299,6 +346,11 @@ const permissionBySegment: Record<string, PermissionCode> = {
|
||||
hardware: 'hardware',
|
||||
logs: 'logs',
|
||||
'user-settings': 'user-settings',
|
||||
tenants: 'user-settings',
|
||||
projects: 'user-settings',
|
||||
'audit-logs': 'user-settings',
|
||||
'approval-templates': 'user-settings',
|
||||
'approval-instances': 'user-settings',
|
||||
}
|
||||
|
||||
function requiredPermission(path: string, explicit?: unknown) {
|
||||
@@ -307,10 +359,11 @@ function requiredPermission(path: string, explicit?: unknown) {
|
||||
return permissionBySegment[segment]
|
||||
}
|
||||
|
||||
// 全局守卫:登录校验 + 会话超时
|
||||
// 全局守卫:登录校验
|
||||
// 离开页面超时由 App.vue 的 visibilitychange 监听接管
|
||||
router.beforeEach((to, _from, next) => {
|
||||
if (!to.meta.public) routeLoading.value = true
|
||||
const auth = useAuthStore()
|
||||
auth.syncSession()
|
||||
document.title = to.meta.title ? `${to.meta.title} - 远光软件微调平台` : '远光软件微调平台'
|
||||
|
||||
if (to.meta.public) {
|
||||
@@ -324,6 +377,7 @@ router.beforeEach((to, _from, next) => {
|
||||
}
|
||||
|
||||
if (!auth.isLoggedIn) {
|
||||
auth.logout() // fire-and-forget,无需阻塞跳转
|
||||
next({ name: 'login' })
|
||||
return
|
||||
}
|
||||
@@ -336,9 +390,11 @@ router.beforeEach((to, _from, next) => {
|
||||
}
|
||||
}
|
||||
|
||||
// 续期会话
|
||||
auth.refresh()
|
||||
next()
|
||||
})
|
||||
|
||||
router.afterEach(() => {
|
||||
routeLoading.value = false
|
||||
})
|
||||
|
||||
export default router
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
import { defineStore } from 'pinia'
|
||||
import { ref, computed } from 'vue'
|
||||
import { login as loginApi } from '@/api/modules/system'
|
||||
import { SESSION_TIMEOUT } from '@/constants'
|
||||
import { login as loginApi, logout as logoutApi } from '@/api/modules/system'
|
||||
import type { PermissionCode, SystemUser } from '@/types'
|
||||
import {
|
||||
clearSessionActivity,
|
||||
sessionActivityTime,
|
||||
startSessionActivity,
|
||||
syncSessionActivity,
|
||||
touchSessionActivity,
|
||||
} from '@/utils/sessionActivity'
|
||||
|
||||
const USER_STORAGE_KEY = 'currentUser'
|
||||
const SESSION_STORAGE_KEY = 'sessionId'
|
||||
|
||||
const allPermissions: PermissionCode[] = [
|
||||
'dashboard',
|
||||
@@ -37,26 +30,13 @@ function restoreUser(): SystemUser | null {
|
||||
localStorage.removeItem(USER_STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
|
||||
// 兼容改造前已经登录的 admin 会话。
|
||||
if (localStorage.getItem('username') === 'admin') {
|
||||
return {
|
||||
id: 'USR-0001',
|
||||
username: 'admin',
|
||||
display_name: '系统管理员',
|
||||
role: 'admin',
|
||||
status: 'active',
|
||||
permissions: allPermissions,
|
||||
create_time: '2026-01-01T08:00:00+08:00',
|
||||
protected: true,
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 认证 store
|
||||
* 沿用原项目 localStorage 的登录时间戳 + 5 分钟会话超时机制
|
||||
* 登录态管理:有 currentUser 即视为已登录。
|
||||
* 离开页面超时由 App.vue 的 visibilitychange 监听接管。
|
||||
*/
|
||||
export const useAuthStore = defineStore('auth', () => {
|
||||
const currentUser = ref<SystemUser | null>(restoreUser())
|
||||
@@ -67,20 +47,18 @@ export const useAuthStore = defineStore('auth', () => {
|
||||
if (currentUser.value?.role === 'operator') return '操作员'
|
||||
return '观察员'
|
||||
})
|
||||
const loginTime = sessionActivityTime
|
||||
|
||||
const isLoggedIn = computed(() => {
|
||||
if (!loginTime.value) return false
|
||||
return Date.now() - loginTime.value < SESSION_TIMEOUT
|
||||
})
|
||||
const isLoggedIn = computed(() => currentUser.value !== null)
|
||||
|
||||
/** 登录 */
|
||||
async function login(user: string, password: string) {
|
||||
const response = await loginApi(user, password)
|
||||
currentUser.value = response.user
|
||||
startSessionActivity()
|
||||
localStorage.setItem('username', response.user.username)
|
||||
localStorage.setItem(USER_STORAGE_KEY, JSON.stringify(response.user))
|
||||
if (response.session_id) {
|
||||
localStorage.setItem(SESSION_STORAGE_KEY, response.session_id)
|
||||
}
|
||||
}
|
||||
|
||||
/** 检查当前账号是否拥有指定模块权限。 */
|
||||
@@ -89,22 +67,16 @@ export const useAuthStore = defineStore('auth', () => {
|
||||
return currentUser.value?.permissions.includes(permission) ?? false
|
||||
}
|
||||
|
||||
/** 续期会话(活跃时刷新) */
|
||||
function refresh() {
|
||||
if (currentUser.value) touchSessionActivity()
|
||||
}
|
||||
|
||||
/** 在路由判断前吸收其他标签页写入的最后活跃时间。 */
|
||||
function syncSession() {
|
||||
syncSessionActivity()
|
||||
}
|
||||
|
||||
/** 退出 */
|
||||
function logout() {
|
||||
async function logout() {
|
||||
const sessionId = localStorage.getItem(SESSION_STORAGE_KEY)
|
||||
if (sessionId) {
|
||||
try { await logoutApi(sessionId) } catch { /* 静默 */ }
|
||||
}
|
||||
currentUser.value = null
|
||||
clearSessionActivity()
|
||||
localStorage.removeItem('username')
|
||||
localStorage.removeItem(USER_STORAGE_KEY)
|
||||
localStorage.removeItem(SESSION_STORAGE_KEY)
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -112,12 +84,9 @@ export const useAuthStore = defineStore('auth', () => {
|
||||
username,
|
||||
displayName,
|
||||
roleLabel,
|
||||
loginTime,
|
||||
isLoggedIn,
|
||||
hasPermission,
|
||||
login,
|
||||
refresh,
|
||||
syncSession,
|
||||
logout,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -99,6 +99,20 @@ export interface DataProcessRegenerateResult {
|
||||
published_outputs_preserved: boolean
|
||||
}
|
||||
|
||||
export interface DataProcessRepeatPayload {
|
||||
expected_updated_at: string
|
||||
request_id: string
|
||||
}
|
||||
|
||||
export interface DataProcessRepeatResult {
|
||||
task: DataProcessTask
|
||||
source_task_id: string
|
||||
created: boolean
|
||||
copied_source_file_count: number
|
||||
copied_preview_count: number
|
||||
progress: DataProcessProgress
|
||||
}
|
||||
|
||||
export type DataProcessTaskUpdatePayload = Partial<DataProcessTaskCreatePayload>
|
||||
|
||||
export interface DataProcessSourceFile {
|
||||
@@ -232,6 +246,22 @@ export interface DataProcessPreviewItem {
|
||||
updated_at?: string
|
||||
}
|
||||
|
||||
export type DataProcessSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
|
||||
|
||||
export interface DataProcessSourceLocator {
|
||||
kind: DataProcessSourceLocatorKind
|
||||
record_index?: number | null
|
||||
start_line?: number | null
|
||||
end_line?: number | null
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
json_pointer?: string | null
|
||||
sheet_index?: number | null
|
||||
sheet_name?: string | null
|
||||
row_number?: number | null
|
||||
sheet_record_index?: number | null
|
||||
}
|
||||
|
||||
export interface DataProcessPreviewBuildPayload {
|
||||
replace_existing?: true
|
||||
source_file_ids?: Array<string | number>
|
||||
@@ -369,6 +399,9 @@ export interface DataProcessQualityScore {
|
||||
is_valid?: boolean
|
||||
flags?: string[]
|
||||
fingerprint?: string
|
||||
source_pages?: number[]
|
||||
heading_path?: string[]
|
||||
source_locator?: DataProcessSourceLocator
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,10 @@ export interface TrainedModel {
|
||||
name: string
|
||||
train_methods?: TrainMethod[]
|
||||
base_model_path?: string
|
||||
artifact_dir?: string
|
||||
adapter_path?: string
|
||||
compute_node_id?: string
|
||||
compute_node_name?: string
|
||||
create_time?: string
|
||||
merged?: boolean
|
||||
merging?: boolean
|
||||
@@ -132,6 +136,7 @@ export interface FineTuneTask {
|
||||
train_dataset_id?: number | string
|
||||
auto_merge?: boolean
|
||||
output_model_name?: string
|
||||
compute_node_id?: string
|
||||
gpus?: number[]
|
||||
batch_size?: number
|
||||
learning_rate?: number
|
||||
@@ -211,6 +216,9 @@ export interface LoadedModel {
|
||||
status?: string
|
||||
pid?: number
|
||||
port?: number
|
||||
node_id?: string
|
||||
node_name?: string
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface CompareTask {
|
||||
@@ -229,6 +237,8 @@ export interface CompareModelRef {
|
||||
model_name: string
|
||||
model_path: string
|
||||
gpu_id: number
|
||||
node_id?: string
|
||||
node_name?: string
|
||||
source?: string
|
||||
port?: number
|
||||
}
|
||||
@@ -244,7 +254,9 @@ export interface EvalTask {
|
||||
model_name?: string
|
||||
model_id?: number | string
|
||||
dataset?: string
|
||||
dataset_id?: number | string
|
||||
metric?: string
|
||||
metric_label?: string
|
||||
score?: number
|
||||
status?: string
|
||||
create_time?: string
|
||||
@@ -271,6 +283,7 @@ export interface StartEvalPayload {
|
||||
eval_type: EvalType
|
||||
model_id: string | number
|
||||
gpu_id: string | number
|
||||
compute_node_id?: string
|
||||
dataset_id: string | number
|
||||
dimension_id: string | number
|
||||
data_source: 'dataset' | 'inference'
|
||||
@@ -355,12 +368,15 @@ export interface GpuInfo {
|
||||
power_w: number
|
||||
id?: number
|
||||
uuid?: string
|
||||
status?: 'idle' | 'busy' | 'warning' | 'offline'
|
||||
status?: 'idle' | 'busy' | 'reserved' | 'warning' | 'offline'
|
||||
memory_percent?: number
|
||||
power_limit_w?: number
|
||||
processes?: GpuProcess[]
|
||||
fan_speed?: number
|
||||
clock_mhz?: number
|
||||
node_id?: string
|
||||
node_code?: string
|
||||
node_name?: string
|
||||
driver_version?: string
|
||||
}
|
||||
|
||||
@@ -443,6 +459,7 @@ export interface SystemUser {
|
||||
export interface LoginResponse {
|
||||
token: string
|
||||
user: SystemUser
|
||||
session_id?: string
|
||||
}
|
||||
|
||||
export interface CreateUserPayload {
|
||||
|
||||
@@ -8,7 +8,7 @@ function storedActivityTime() {
|
||||
|
||||
/**
|
||||
* 会话按“最后活跃时间”计算,而不是从首次登录起固定倒计时。
|
||||
* 该 ref 被认证 store 与请求层共享,确保 API 活动可以立即影响路由守卫。
|
||||
* 该 ref 被认证 store 与路由守卫共享,确保真实用户活动可以立即影响超时判断。
|
||||
*/
|
||||
export const sessionActivityTime = ref(storedActivityTime())
|
||||
|
||||
@@ -30,3 +30,45 @@ export function clearSessionActivity() {
|
||||
sessionActivityTime.value = 0
|
||||
localStorage.removeItem(LOGIN_TIME_STORAGE_KEY)
|
||||
}
|
||||
|
||||
/**
|
||||
* 仅在用户真实活跃时续期会话:
|
||||
* - 鼠标移动 / 键盘 / 点击 / 触摸(说明用户正在操作)
|
||||
* - 标签页切回可见(说明用户回到界面)
|
||||
* 页面后台轮询接口、切走标签页不会续期,从而“无操作”或“不在当前界面”
|
||||
* 超过空闲时长才会被判定为会话过期并跳回登录。
|
||||
*/
|
||||
let userActivityBound = false
|
||||
let lastTouch = 0
|
||||
const ACTIVITY_THROTTLE = 5000 // 5s 内最多续期一次,避免 mousemove 过于频繁
|
||||
|
||||
const activityEvents = ['mousemove', 'mousedown', 'keydown', 'click', 'touchstart'] as const
|
||||
|
||||
function handleUserActivity() {
|
||||
const now = Date.now()
|
||||
if (now - lastTouch < ACTIVITY_THROTTLE) return
|
||||
lastTouch = now
|
||||
touchSessionActivity()
|
||||
}
|
||||
|
||||
function handleVisibility() {
|
||||
if (!document.hidden) {
|
||||
touchSessionActivity()
|
||||
}
|
||||
}
|
||||
|
||||
export function bindUserActivityListeners() {
|
||||
if (userActivityBound) return
|
||||
userActivityBound = true
|
||||
activityEvents.forEach((evt) =>
|
||||
window.addEventListener(evt, handleUserActivity, { passive: true })
|
||||
)
|
||||
document.addEventListener('visibilitychange', handleVisibility)
|
||||
}
|
||||
|
||||
export function unbindUserActivityListeners() {
|
||||
if (!userActivityBound) return
|
||||
userActivityBound = false
|
||||
activityEvents.forEach((evt) => window.removeEventListener(evt, handleUserActivity))
|
||||
document.removeEventListener('visibilitychange', handleVisibility)
|
||||
}
|
||||
|
||||
133
frontend/src/views/approvals/ApprovalInstanceView.vue
Normal file
133
frontend/src/views/approvals/ApprovalInstanceView.vue
Normal file
@@ -0,0 +1,133 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, reactive, ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import { getApprovalInstances, decideApproval, type ApprovalInstance } from '@/api/modules/approval'
|
||||
import { getUsers } from '@/api/modules/system'
|
||||
import type { SystemUser } from '@/types'
|
||||
|
||||
const loading = ref(false)
|
||||
const instances = ref<ApprovalInstance[]>([])
|
||||
const users = ref<SystemUser[]>([])
|
||||
const statusFilter = ref<string | undefined>(undefined)
|
||||
const showDecide = ref(false)
|
||||
const current = ref<ApprovalInstance | null>(null)
|
||||
const decision = ref({ step_index: 0, approver_id: '', approved: true, comment: '' })
|
||||
|
||||
const statusOptions = [
|
||||
{ label: '待审批', value: 'pending' },
|
||||
{ label: '已通过', value: 'approved' },
|
||||
{ label: '已拒绝', value: 'rejected' },
|
||||
]
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
instances.value = await getApprovalInstances(statusFilter.value)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadUsers() {
|
||||
try {
|
||||
users.value = await getUsers()
|
||||
} catch {
|
||||
users.value = []
|
||||
}
|
||||
}
|
||||
|
||||
function userName(id?: string) {
|
||||
if (!id) return '—'
|
||||
return users.value.find((u) => u.id === id)?.username || id
|
||||
}
|
||||
|
||||
function openDecide(inst: ApprovalInstance) {
|
||||
current.value = inst
|
||||
const step = inst.steps.find((s) => s.status === 'pending')
|
||||
decision.value = { step_index: step ? step.step_index : 0, approver_id: '', approved: true, comment: '' }
|
||||
showDecide.value = true
|
||||
}
|
||||
|
||||
function asApprovalInstance(row: unknown): ApprovalInstance {
|
||||
return row as ApprovalInstance
|
||||
}
|
||||
|
||||
async function submitDecision() {
|
||||
if (!current.value) return
|
||||
if (!decision.value.approver_id) {
|
||||
ElMessage.warning('请选择审批人')
|
||||
return
|
||||
}
|
||||
await decideApproval(current.value.id, decision.value.step_index, {
|
||||
approver_id: decision.value.approver_id,
|
||||
approved: decision.value.approved,
|
||||
comment: decision.value.comment,
|
||||
})
|
||||
ElMessage.success('审批已提交')
|
||||
showDecide.value = false
|
||||
load()
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadUsers()
|
||||
load()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="page">
|
||||
<DataTablePage title="审批实例" :data="instances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
|
||||
<template #toolbar-extra>
|
||||
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
|
||||
<el-option v-for="s in statusOptions" :key="s.value" :label="s.label" :value="s.value" />
|
||||
</el-select>
|
||||
</template>
|
||||
<template #columns>
|
||||
<el-table-column prop="resource_type" label="资源类型" min-width="120" />
|
||||
<el-table-column prop="resource_id" label="资源 ID" min-width="160" show-overflow-tooltip />
|
||||
<el-table-column prop="applicant_id" label="申请人" min-width="120">
|
||||
<template #default="{ row }">{{ userName(asApprovalInstance(row).applicant_id) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="status" label="状态" min-width="100" />
|
||||
<el-table-column prop="current_step" label="当前步骤" min-width="100" />
|
||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||
</template>
|
||||
<template #actions="{ row }">
|
||||
<el-button v-if="asApprovalInstance(row).status === 'pending'" link type="primary" @click="openDecide(asApprovalInstance(row))">审批</el-button>
|
||||
</template>
|
||||
</DataTablePage>
|
||||
<el-dialog v-model="showDecide" title="审批决策" width="480px">
|
||||
<el-form label-width="80px" v-if="current">
|
||||
<el-form-item label="实例">
|
||||
{{ current.resource_type }} / {{ current.resource_id }}
|
||||
</el-form-item>
|
||||
<el-form-item label="步骤">
|
||||
第 {{ decision.step_index + 1 }} 步
|
||||
</el-form-item>
|
||||
<el-form-item label="审批人" required>
|
||||
<el-select v-model="decision.approver_id" filterable style="width: 100%">
|
||||
<el-option v-for="u in users" :key="u.id" :label="u.username" :value="u.id" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="结果">
|
||||
<el-radio-group v-model="decision.approved">
|
||||
<el-radio :value="true">通过</el-radio>
|
||||
<el-radio :value="false">拒绝</el-radio>
|
||||
</el-radio-group>
|
||||
</el-form-item>
|
||||
<el-form-item label="意见">
|
||||
<el-input v-model="decision.comment" type="textarea" :rows="3" />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="showDecide = false">取消</el-button>
|
||||
<el-button type="primary" @click="submitDecision">提交</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.page { padding: 16px; }
|
||||
</style>
|
||||
77
frontend/src/views/approvals/ApprovalTemplateView.vue
Normal file
77
frontend/src/views/approvals/ApprovalTemplateView.vue
Normal file
@@ -0,0 +1,77 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { Plus } from '@element-plus/icons-vue'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import { createApprovalTemplate, getApprovalTemplates, type ApprovalTemplate } from '@/api/modules/approval'
|
||||
|
||||
const loading = ref(false)
|
||||
const templates = ref<ApprovalTemplate[]>([])
|
||||
const showCreate = ref(false)
|
||||
const form = ref({ name: '', stepsText: '[]' })
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
templates.value = await getApprovalTemplates()
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function submitCreate() {
|
||||
if (!form.value.name) {
|
||||
ElMessage.warning('请填写模板名称')
|
||||
return
|
||||
}
|
||||
let steps: unknown[] = []
|
||||
try {
|
||||
steps = JSON.parse(form.value.stepsText || '[]')
|
||||
} catch {
|
||||
ElMessage.error('步骤需为合法 JSON 数组')
|
||||
return
|
||||
}
|
||||
await createApprovalTemplate({ name: form.value.name, steps: steps as any })
|
||||
ElMessage.success('模板创建成功')
|
||||
showCreate.value = false
|
||||
form.value = { name: '', stepsText: '[]' }
|
||||
load()
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="page">
|
||||
<DataTablePage title="审批模板" :data="templates" :loading="loading">
|
||||
<template #toolbar-extra>
|
||||
<el-button type="primary" :icon="Plus" @click="showCreate = true">新建模板</el-button>
|
||||
</template>
|
||||
<template #columns>
|
||||
<el-table-column prop="name" label="模板名" min-width="160" />
|
||||
<el-table-column label="步骤数" min-width="100">
|
||||
<template #default="{ row }">{{ (row.steps || []).length }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="create_time" label="创建时间" min-width="180" />
|
||||
</template>
|
||||
</DataTablePage>
|
||||
<el-dialog v-model="showCreate" title="新建审批模板" width="560px">
|
||||
<el-form label-width="90px">
|
||||
<el-form-item label="名称" required>
|
||||
<el-input v-model="form.name" placeholder="模板名" />
|
||||
</el-form-item>
|
||||
<el-form-item label="步骤 JSON">
|
||||
<el-input v-model="form.stepsText" type="textarea" :rows="5" placeholder='[{"approver_id":"u1"},{"approver_id":"u2"}]' />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="showCreate = false">取消</el-button>
|
||||
<el-button type="primary" @click="submitCreate">创建</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.page { padding: 16px; }
|
||||
</style>
|
||||
125
frontend/src/views/audit/AuditLogView.vue
Normal file
125
frontend/src/views/audit/AuditLogView.vue
Normal file
@@ -0,0 +1,125 @@
|
||||
<script setup lang="ts">
|
||||
import { onMounted, reactive, ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { getAuditLogs, exportAuditLogs, type AuditLog, type AuditQuery } from '@/api/modules/audit'
|
||||
|
||||
const loading = ref(false)
|
||||
const logs = ref<AuditLog[]>([])
|
||||
const total = ref(0)
|
||||
const query = reactive<AuditQuery>({
|
||||
tenant_id: '',
|
||||
project_id: '',
|
||||
actor_id: '',
|
||||
action: '',
|
||||
target_type: '',
|
||||
start_time: '',
|
||||
end_time: '',
|
||||
limit: 50,
|
||||
offset: 0,
|
||||
})
|
||||
|
||||
// 时间范围(el-date-picker 双向绑定数组 [start, end])
|
||||
const timeRange = ref<[string, string] | null>(null)
|
||||
|
||||
function applyTimeRange() {
|
||||
if (timeRange.value && timeRange.value.length === 2) {
|
||||
query.start_time = timeRange.value[0]
|
||||
query.end_time = timeRange.value[1]
|
||||
} else {
|
||||
query.start_time = ''
|
||||
query.end_time = ''
|
||||
}
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const res = await getAuditLogs({ ...query })
|
||||
logs.value = res.items
|
||||
total.value = res.total
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function handleExport() {
|
||||
try {
|
||||
const blob = await exportAuditLogs({ ...query, limit: 10000, offset: 0 })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = url
|
||||
a.download = `audit_logs_${Date.now()}.csv`
|
||||
a.click()
|
||||
URL.revokeObjectURL(url)
|
||||
} catch {
|
||||
ElMessage.error('导出失败')
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="page">
|
||||
<div class="page-header">
|
||||
<h2 class="page-title">审计日志</h2>
|
||||
<el-button @click="handleExport">导出 CSV</el-button>
|
||||
</div>
|
||||
<el-card class="filter-card">
|
||||
<el-form :inline="true">
|
||||
<el-form-item label="租户">
|
||||
<el-input v-model="query.tenant_id" placeholder="tenant_id" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="项目">
|
||||
<el-input v-model="query.project_id" placeholder="project_id" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="操作人">
|
||||
<el-input v-model="query.actor_id" placeholder="actor_id" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="动作">
|
||||
<el-input v-model="query.action" placeholder="action" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="目标类型">
|
||||
<el-input v-model="query.target_type" placeholder="target_type" clearable />
|
||||
</el-form-item>
|
||||
<el-form-item label="时间范围">
|
||||
<el-date-picker
|
||||
v-model="timeRange"
|
||||
type="datetimerange"
|
||||
value-format="YYYY-MM-DDTHH:mm:ss"
|
||||
range-separator="至"
|
||||
start-placeholder="开始时间"
|
||||
end-placeholder="结束时间"
|
||||
clearable
|
||||
style="width: 360px"
|
||||
@change="applyTimeRange"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item>
|
||||
<el-button type="primary" @click="load">查询</el-button>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-card>
|
||||
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
|
||||
<el-table-column prop="time" label="时间" min-width="180" />
|
||||
<el-table-column prop="tenant_id" label="租户" min-width="120" />
|
||||
<el-table-column prop="project_id" label="项目" min-width="120" />
|
||||
<el-table-column prop="actor_id" label="操作人" min-width="120" />
|
||||
<el-table-column prop="action" label="动作" min-width="140" />
|
||||
<el-table-column prop="target_type" label="目标类型" min-width="120" />
|
||||
<el-table-column prop="target_id" label="目标 ID" min-width="140" show-overflow-tooltip />
|
||||
<el-table-column prop="detail" label="详情" min-width="200" show-overflow-tooltip />
|
||||
<el-table-column prop="client_ip" label="IP" min-width="120" />
|
||||
</el-table>
|
||||
<div class="pager">共 {{ total }} 条</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.page { padding: 16px; }
|
||||
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
|
||||
.page-title { margin: 0; font-size: 18px; }
|
||||
.filter-card { margin-bottom: 16px; }
|
||||
.log-table { margin-top: 8px; }
|
||||
.pager { margin-top: 12px; text-align: right; color: #909399; }
|
||||
</style>
|
||||
@@ -1,12 +1,12 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, onUnmounted, reactive, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import {
|
||||
checkNodeReplicaDrift,
|
||||
createComputeNode,
|
||||
deleteComputeNode,
|
||||
disableComputeNode,
|
||||
drainComputeNode,
|
||||
enableComputeNode,
|
||||
getComputeGpus,
|
||||
getComputeNodes,
|
||||
@@ -129,11 +129,10 @@ async function changeTab(name: string | number) {
|
||||
await router.replace({ path: '/compute', query: { tab: String(name) } })
|
||||
}
|
||||
|
||||
async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test', node: ComputeNode) {
|
||||
async function handleNodeAction(action: 'enable' | 'disable' | 'test', node: ComputeNode) {
|
||||
const nodeId = String(node.id)
|
||||
if (action === 'enable') await enableComputeNode(nodeId)
|
||||
if (action === 'disable') await disableComputeNode(nodeId)
|
||||
if (action === 'drain') await drainComputeNode(nodeId)
|
||||
if (action === 'test') {
|
||||
const result = await testComputeNode(nodeId)
|
||||
if (result.success) {
|
||||
@@ -145,6 +144,27 @@ async function handleNodeAction(action: 'enable' | 'disable' | 'drain' | 'test',
|
||||
await load()
|
||||
}
|
||||
|
||||
async function handleDeleteNode(node: ComputeNode) {
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
`确定删除算力节点「${node.name || node.code}」吗?节点删除后,其 GPU 设备和资源副本记录也会一并移除。`,
|
||||
'删除算力节点',
|
||||
{ type: 'warning', confirmButtonText: '删除', cancelButtonText: '取消' },
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
try {
|
||||
await deleteComputeNode(String(node.id))
|
||||
ElMessage.success('算力节点已删除')
|
||||
if (selectedNodeId.value === node.id) selectedNodeId.value = ''
|
||||
await load({ showButtonLoading: true })
|
||||
} catch (err: any) {
|
||||
const message = err?.response?.data?.detail?.message || err?.response?.data?.message || '删除算力节点失败'
|
||||
ElMessage.error(message)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReplicaDriftCheck() {
|
||||
if (!selectedNodeId.value) return
|
||||
checkingReplicas.value = true
|
||||
@@ -355,7 +375,7 @@ onUnmounted(() => {
|
||||
<el-button size="small" @click="handleNodeAction('test', asComputeNode(row))">测试</el-button>
|
||||
<el-button v-if="row.enabled" size="small" @click="handleNodeAction('disable', asComputeNode(row))">停用</el-button>
|
||||
<el-button v-else size="small" type="primary" @click="handleNodeAction('enable', asComputeNode(row))">启用</el-button>
|
||||
<el-button size="small" type="warning" plain @click="handleNodeAction('drain', asComputeNode(row))">维护</el-button>
|
||||
<el-button size="small" type="danger" plain @click="handleDeleteNode(asComputeNode(row))">删除</el-button>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref } from 'vue'
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import VChart from 'vue-echarts'
|
||||
import '@/plugins/echarts'
|
||||
import type { EChartsOption } from 'echarts'
|
||||
import { getDashboardStats } from '@/api/modules/dashboard'
|
||||
|
||||
type ServiceState = 'normal' | 'busy' | 'error'
|
||||
type TaskState = 'running' | 'pending' | 'completed' | 'failed'
|
||||
@@ -16,7 +17,7 @@ interface ServiceStatus {
|
||||
}
|
||||
|
||||
interface DashboardTask {
|
||||
id: number
|
||||
id: string
|
||||
name: string
|
||||
state: TaskState
|
||||
trainType: string
|
||||
@@ -43,59 +44,41 @@ interface RecentLoginUser {
|
||||
const router = useRouter()
|
||||
const period = ref('7d')
|
||||
|
||||
const serviceStatuses: ServiceStatus[] = [
|
||||
{ name: '模型推理', icon: 'fa-cube', state: 'normal', instances: '6 / 6' },
|
||||
{ name: '模型微调', icon: 'fa-sliders', state: 'busy', instances: '4 / 6' },
|
||||
{ name: '模型评测', icon: 'fa-bar-chart', state: 'normal', instances: '3 / 3' },
|
||||
{ name: '数据处理', icon: 'fa-filter', state: 'error', instances: '1 / 3' },
|
||||
]
|
||||
const onlineServices = ref(0)
|
||||
const runningTasks = ref(0)
|
||||
const pendingAlerts = ref(0)
|
||||
|
||||
const trainingTasks: DashboardTask[] = [
|
||||
{
|
||||
id: 103942,
|
||||
name: 'finance-sft-003',
|
||||
state: 'running',
|
||||
trainType: 'SFT',
|
||||
trainMethod: 'LoRA',
|
||||
baseModel: 'Qwen2.5-7B-Instruct',
|
||||
progress: 68,
|
||||
accuracy: 89.2,
|
||||
startedAt: '今天 09:18',
|
||||
},
|
||||
{
|
||||
id: 593021,
|
||||
name: 'legal-eval-008',
|
||||
state: 'pending',
|
||||
trainType: 'DPO',
|
||||
trainMethod: 'LoRA',
|
||||
baseModel: 'Qwen2.5-7B-Instruct',
|
||||
progress: 0,
|
||||
accuracy: null,
|
||||
startedAt: '今天 08:55',
|
||||
},
|
||||
{
|
||||
id: 849301,
|
||||
name: 'medical-cpt-002',
|
||||
state: 'completed',
|
||||
trainType: 'CPT',
|
||||
trainMethod: 'Full',
|
||||
baseModel: 'Qwen2.5-14B-Instruct',
|
||||
progress: 100,
|
||||
accuracy: 91.6,
|
||||
startedAt: '07/10 16:20',
|
||||
},
|
||||
{
|
||||
id: 201948,
|
||||
name: 'finance-sft-002',
|
||||
state: 'failed',
|
||||
trainType: 'SFT',
|
||||
trainMethod: 'LoRA',
|
||||
baseModel: 'Qwen2.5-7B-Instruct',
|
||||
progress: 42,
|
||||
accuracy: null,
|
||||
startedAt: '07/10 11:08',
|
||||
},
|
||||
]
|
||||
const serviceStatuses = ref<ServiceStatus[]>([])
|
||||
const trainingTasks = ref<DashboardTask[]>([])
|
||||
const loginDurationStats = ref<LoginDurationStat[]>([])
|
||||
const recentLoginUsers = ref<RecentLoginUser[]>([])
|
||||
const training7d = ref<{ date: string; train: number; gpu: number; accuracy: number | null }[]>([])
|
||||
|
||||
const onlineServicesHint = computed(() => {
|
||||
if (onlineServices.value === 0) return '暂无在线服务'
|
||||
const abnormal = serviceStatuses.value.filter(
|
||||
(s) => s.state === 'busy' || s.state === 'error'
|
||||
).length
|
||||
return abnormal > 0 ? `${abnormal} 个异常` : '全部在线'
|
||||
})
|
||||
const operationDistribution = ref<{ name: string; value: number }[]>([])
|
||||
|
||||
const serviceIcon: Record<string, string> = {
|
||||
'模型推理': 'fa-cube',
|
||||
'模型微调': 'fa-sliders',
|
||||
'模型训练': 'fa-sliders',
|
||||
'模型评测': 'fa-bar-chart',
|
||||
'模型管理': 'fa-cubes',
|
||||
'数据集管理': 'fa-file-text',
|
||||
'数据处理': 'fa-filter',
|
||||
'数据类型转换': 'fa-exchange',
|
||||
}
|
||||
const roleLabel: Record<string, string> = {
|
||||
admin: '超级管理员',
|
||||
operator: '操作员',
|
||||
observer: '观察员',
|
||||
guest: '访客',
|
||||
}
|
||||
|
||||
const serviceStateMeta: Record<ServiceState, { label: string; className: string }> = {
|
||||
normal: { label: '正常', className: 'is-normal' },
|
||||
@@ -136,11 +119,11 @@ const chartOption = computed<EChartsOption>(() => ({
|
||||
borderWidth: 0,
|
||||
padding: [10, 12],
|
||||
textStyle: { color: '#ffffff', fontSize: 12 },
|
||||
valueFormatter: (value) => `${value}`,
|
||||
valueFormatter: (value) => String(value ?? ''),
|
||||
},
|
||||
xAxis: {
|
||||
type: 'category',
|
||||
data: ['07/05', '07/06', '07/07', '07/08', '07/09', '07/10', '07/11\n今天'],
|
||||
data: training7d.value.map((d) => d.date),
|
||||
axisLine: { lineStyle: { color: '#e2e8f0' } },
|
||||
axisTick: { show: false },
|
||||
axisLabel: { color: '#64748b', fontSize: 11, lineHeight: 16, margin: 12 },
|
||||
@@ -175,7 +158,7 @@ const chartOption = computed<EChartsOption>(() => ({
|
||||
{
|
||||
name: '训练次数(次)',
|
||||
type: 'bar',
|
||||
data: [8, 12, 10, 15, 13, 18, 11],
|
||||
data: training7d.value.map((d) => d.train),
|
||||
barMaxWidth: 16,
|
||||
itemStyle: { borderRadius: [3, 3, 0, 0] },
|
||||
label: { show: true, position: 'top', color: '#64748b', fontSize: 10 },
|
||||
@@ -183,7 +166,7 @@ const chartOption = computed<EChartsOption>(() => ({
|
||||
{
|
||||
name: 'GPU 使用数(个)',
|
||||
type: 'bar',
|
||||
data: [3, 4, 4, 6, 5, 7, 5],
|
||||
data: training7d.value.map((d) => d.gpu),
|
||||
barMaxWidth: 16,
|
||||
itemStyle: { borderRadius: [3, 3, 0, 0] },
|
||||
label: { show: true, position: 'top', color: '#64748b', fontSize: 10 },
|
||||
@@ -192,7 +175,7 @@ const chartOption = computed<EChartsOption>(() => ({
|
||||
name: '平均准确率(%)',
|
||||
type: 'bar',
|
||||
yAxisIndex: 1,
|
||||
data: [82, 85, 84, 88, 87, 91, 89],
|
||||
data: training7d.value.map((d) => d.accuracy ?? null),
|
||||
barMaxWidth: 16,
|
||||
itemStyle: { borderRadius: [3, 3, 0, 0] },
|
||||
label: { show: true, position: 'top', color: '#d97706', fontSize: 10 },
|
||||
@@ -200,103 +183,115 @@ const chartOption = computed<EChartsOption>(() => ({
|
||||
],
|
||||
}))
|
||||
|
||||
const operationChartOption = computed<EChartsOption>(() => ({
|
||||
animationDuration: 500,
|
||||
tooltip: { trigger: 'item' },
|
||||
color: ['#4f46e5', '#10b981', '#f59e0b', '#3b82f6', '#ec4899'],
|
||||
series: [
|
||||
{
|
||||
name: '操作分类',
|
||||
type: 'pie',
|
||||
radius: ['40%', '64%'],
|
||||
center: ['50%', '50%'],
|
||||
avoidLabelOverlap: true,
|
||||
// 模块固定配色,按顺序循环分配颜色(与后端 OP_ORDER 一致:数据处理/模型训练/模型评测/模型推理)
|
||||
const OPERATION_COLORS = ['#4f46e5', '#10b981', '#f59e0b', '#3b82f6']
|
||||
const operationChartOption = computed<EChartsOption>(() => {
|
||||
const items = operationDistribution.value
|
||||
const total = items.reduce((s, d) => s + (d.value || 0), 0)
|
||||
// 按数据项顺序显式分配颜色,避免依赖 name 匹配或全局 color 数组;
|
||||
// value=0 的项给一个极小值(0.001)让扇区可见,从而显示各自颜色,
|
||||
// 但占比几乎为 0 不影响有数据项的百分比展示。
|
||||
const data = items.map((d, idx) => {
|
||||
const raw = d.value || 0
|
||||
return {
|
||||
value: total > 0 ? (raw > 0 ? raw : 0.001) : 1,
|
||||
name: d.name,
|
||||
itemStyle: {
|
||||
color: OPERATION_COLORS[idx % OPERATION_COLORS.length] || '#94a3b8',
|
||||
borderRadius: 6,
|
||||
borderColor: '#fff',
|
||||
borderWidth: 2
|
||||
borderWidth: 2,
|
||||
},
|
||||
label: {
|
||||
show: true,
|
||||
position: 'outside',
|
||||
formatter: '{b}',
|
||||
color: '#475569',
|
||||
fontSize: 11,
|
||||
lineHeight: 16,
|
||||
width: 70,
|
||||
overflow: 'truncate',
|
||||
},
|
||||
emphasis: {
|
||||
label: { show: true, fontSize: 12, fontWeight: 'bold', color: '#1e293b' }
|
||||
},
|
||||
labelLine: {
|
||||
show: true,
|
||||
length: 10,
|
||||
length2: 8,
|
||||
lineStyle: { color: '#94a3b8', width: 1 },
|
||||
},
|
||||
data: [
|
||||
{ value: 1048, name: '模型训练' },
|
||||
{ value: 735, name: '数据处理' },
|
||||
{ value: 580, name: '模型评测' },
|
||||
{ value: 484, name: '模型推理' },
|
||||
{ value: 300, name: '系统设置' }
|
||||
]
|
||||
}
|
||||
]
|
||||
}))
|
||||
|
||||
const loginDurationStats: LoginDurationStat[] = [
|
||||
{ id: 1, username: 'admin', duration: 124 },
|
||||
{ id: 2, username: 'zhangsan', duration: 86 },
|
||||
{ id: 3, username: 'lisi', duration: 42 },
|
||||
{ id: 4, username: 'wangwu', duration: 18 },
|
||||
]
|
||||
|
||||
const loginDurationChartOption = computed<EChartsOption>(() => ({
|
||||
animationDuration: 500,
|
||||
grid: { top: 8, right: 12, bottom: 6, left: 8, containLabel: true },
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
axisPointer: { type: 'shadow' },
|
||||
valueFormatter: (value) => `${value} 小时`,
|
||||
},
|
||||
xAxis: {
|
||||
type: 'value',
|
||||
max: Math.ceil(Math.max(...loginDurationStats.map((user) => user.duration)) * 1.15 / 10) * 10,
|
||||
splitNumber: 4,
|
||||
axisLabel: { color: '#94a3b8', fontSize: 11, formatter: '{value}h' },
|
||||
axisLine: { show: false },
|
||||
axisTick: { show: false },
|
||||
splitLine: { lineStyle: { color: '#eef2f7' } },
|
||||
},
|
||||
yAxis: {
|
||||
type: 'category',
|
||||
inverse: true,
|
||||
data: loginDurationStats.map((user) => user.username),
|
||||
axisLabel: { color: '#475569', fontSize: 12 },
|
||||
axisLine: { show: false },
|
||||
axisTick: { show: false },
|
||||
},
|
||||
series: [
|
||||
{
|
||||
name: '登录时长',
|
||||
type: 'bar',
|
||||
data: loginDurationStats.map((user) => user.duration),
|
||||
barMaxWidth: 18,
|
||||
barCategoryGap: '34%',
|
||||
itemStyle: { color: '#4f46e5', borderRadius: [0, 4, 4, 0] },
|
||||
label: { show: true, position: 'insideRight', distance: 6, color: '#ffffff', fontSize: 11, formatter: '{c} 小时' },
|
||||
})
|
||||
return {
|
||||
animationDuration: 500,
|
||||
tooltip: { trigger: 'item', formatter: '{b}: {c} ({d}%)' },
|
||||
legend: {
|
||||
type: 'scroll',
|
||||
bottom: 0,
|
||||
textStyle: { color: '#64748b', fontSize: 11 },
|
||||
itemWidth: 10,
|
||||
itemHeight: 10,
|
||||
},
|
||||
],
|
||||
}))
|
||||
series: [
|
||||
{
|
||||
name: '操作分类',
|
||||
type: 'pie',
|
||||
radius: ['38%', '60%'],
|
||||
center: ['50%', '42%'],
|
||||
avoidLabelOverlap: true,
|
||||
label: {
|
||||
show: true,
|
||||
position: 'outside',
|
||||
formatter: '{b}\n{d}%',
|
||||
color: '#475569',
|
||||
fontSize: 11,
|
||||
lineHeight: 15,
|
||||
},
|
||||
emphasis: {
|
||||
label: { show: true, fontSize: 12, fontWeight: 'bold', color: '#1e293b' },
|
||||
},
|
||||
labelLine: {
|
||||
show: true,
|
||||
length: 8,
|
||||
length2: 8,
|
||||
lineStyle: { color: '#94a3b8', width: 1 },
|
||||
},
|
||||
data,
|
||||
},
|
||||
],
|
||||
}
|
||||
})
|
||||
|
||||
const recentLoginUsers: RecentLoginUser[] = [
|
||||
{ id: 1, username: 'admin', role: '超级管理员', lastLogin: '10 分钟前' },
|
||||
{ id: 2, username: 'zhangsan', role: '操作员', lastLogin: '2 小时前' },
|
||||
{ id: 5, username: 'zhaoliu', role: '观察员', lastLogin: '5 小时前' },
|
||||
{ id: 3, username: 'lisi', role: '操作员', lastLogin: '昨天 15:30' },
|
||||
]
|
||||
const loginDurationChartOption = computed<EChartsOption>(() => {
|
||||
const stats = loginDurationStats.value
|
||||
const data = stats.map((u) => ({ name: u.username, value: u.duration }))
|
||||
const maxVal = data.length
|
||||
? Math.max(10, Math.ceil(Math.max(...data.map((d) => d.value), 0) * 1.15 / 10) * 10)
|
||||
: 10
|
||||
return {
|
||||
animationDuration: 500,
|
||||
grid: { top: 8, right: 12, bottom: 6, left: 8, containLabel: true },
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
axisPointer: { type: 'shadow' },
|
||||
valueFormatter: (value: unknown) => String(Number(Array.isArray(value) ? value[0] : value) || 0) + ' 小时',
|
||||
},
|
||||
xAxis: {
|
||||
type: 'value',
|
||||
max: maxVal,
|
||||
splitNumber: 4,
|
||||
axisLabel: { color: '#94a3b8', fontSize: 11, formatter: '{value}h' },
|
||||
axisLine: { show: false },
|
||||
axisTick: { show: false },
|
||||
splitLine: { lineStyle: { color: '#eef2f7' } },
|
||||
},
|
||||
yAxis: {
|
||||
type: 'category',
|
||||
inverse: true,
|
||||
data: data.map((d) => d.name),
|
||||
axisLabel: {
|
||||
color: '#1f2937',
|
||||
fontSize: 14,
|
||||
fontFamily: '"PingFang SC", "Microsoft YaHei", system-ui, -apple-system, sans-serif',
|
||||
margin: 12,
|
||||
},
|
||||
axisLine: { show: false },
|
||||
axisTick: { show: false },
|
||||
},
|
||||
series: [
|
||||
{
|
||||
name: '登录时长',
|
||||
type: 'bar',
|
||||
data: data.map((d) => d.value),
|
||||
barMaxWidth: 18,
|
||||
barCategoryGap: '34%',
|
||||
itemStyle: { color: '#4f46e5', borderRadius: [0, 4, 4, 0] },
|
||||
},
|
||||
],
|
||||
}
|
||||
})
|
||||
|
||||
const roleTagType: Record<string, 'danger' | 'primary' | 'info'> = {
|
||||
'超级管理员': 'danger',
|
||||
@@ -304,6 +299,45 @@ const roleTagType: Record<string, 'danger' | 'primary' | 'info'> = {
|
||||
'观察员': 'info',
|
||||
}
|
||||
|
||||
async function loadStats() {
|
||||
const stats = await getDashboardStats()
|
||||
onlineServices.value = stats.online_services
|
||||
runningTasks.value = stats.running_tasks
|
||||
pendingAlerts.value = stats.pending_alerts
|
||||
serviceStatuses.value = stats.service_status.map((s) => ({
|
||||
name: s.type,
|
||||
icon: serviceIcon[s.type] || 'fa-cube',
|
||||
state: s.status as ServiceState,
|
||||
instances: String(s.count),
|
||||
}))
|
||||
trainingTasks.value = stats.training_tasks.map((t) => ({
|
||||
id: String(t.id),
|
||||
name: t.name,
|
||||
state: t.status as TaskState,
|
||||
trainType: t.train_type,
|
||||
trainMethod: t.train_method,
|
||||
baseModel: t.base_model,
|
||||
progress: t.progress,
|
||||
accuracy: t.accuracy,
|
||||
startedAt: t.started_at,
|
||||
}))
|
||||
loginDurationStats.value = stats.login_duration_rank.map((u, i) => ({
|
||||
id: i + 1,
|
||||
username: u.user,
|
||||
duration: u.duration,
|
||||
}))
|
||||
recentLoginUsers.value = stats.recent_login_users.map((u, i) => ({
|
||||
id: i + 1,
|
||||
username: u.user,
|
||||
role: roleLabel[u.role] || u.role,
|
||||
lastLogin: u.last_login,
|
||||
}))
|
||||
training7d.value = stats.training_7d
|
||||
operationDistribution.value = stats.operation_distribution
|
||||
}
|
||||
|
||||
onMounted(loadStats)
|
||||
|
||||
function viewAllTasks() {
|
||||
router.push('/fine-tune')
|
||||
}
|
||||
@@ -329,17 +363,17 @@ function viewTask(task: DashboardTask) {
|
||||
<div class="overview-metrics">
|
||||
<div class="overview-metric">
|
||||
<span>在线服务</span>
|
||||
<strong>12</strong>
|
||||
<small>全部在线</small>
|
||||
<strong>{{ onlineServices }}</strong>
|
||||
<small>{{ onlineServicesHint }}</small>
|
||||
</div>
|
||||
<div class="overview-metric">
|
||||
<span>运行中任务</span>
|
||||
<strong>5</strong>
|
||||
<strong>{{ runningTasks }}</strong>
|
||||
<small>较昨日 +1</small>
|
||||
</div>
|
||||
<div class="overview-metric is-alert">
|
||||
<span>待处理告警</span>
|
||||
<strong>2</strong>
|
||||
<strong>{{ pendingAlerts }}</strong>
|
||||
<small>较昨日 -1</small>
|
||||
</div>
|
||||
</div>
|
||||
@@ -391,7 +425,9 @@ function viewTask(task: DashboardTask) {
|
||||
|
||||
<section class="stat-card" aria-labelledby="login-dur-title">
|
||||
<h2 id="login-dur-title" class="section-title">登录时长排行 (本月)</h2>
|
||||
<VChart class="duration-chart" :option="loginDurationChartOption" autoresize />
|
||||
<div class="chart-container">
|
||||
<VChart class="duration-chart" :option="loginDurationChartOption" autoresize />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="stat-card" aria-labelledby="recent-login-title">
|
||||
@@ -515,6 +551,15 @@ function viewTask(task: DashboardTask) {
|
||||
height: 224px;
|
||||
}
|
||||
|
||||
.empty-hint {
|
||||
flex: 1 1 auto;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
min-height: 224px;
|
||||
color: #94a3b8;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.duration-chart {
|
||||
width: 100%;
|
||||
height: 224px;
|
||||
@@ -697,10 +742,11 @@ function viewTask(task: DashboardTask) {
|
||||
|
||||
.service-table {
|
||||
display: grid;
|
||||
grid-template-rows: 36px repeat(4, minmax(48px, 1fr));
|
||||
grid-auto-rows: minmax(44px, auto);
|
||||
flex: 1 1 auto;
|
||||
margin-top: 12px;
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.service-row {
|
||||
@@ -894,7 +940,7 @@ function viewTask(task: DashboardTask) {
|
||||
}
|
||||
|
||||
.service-table {
|
||||
grid-template-rows: 32px repeat(4, minmax(40px, 1fr));
|
||||
grid-auto-rows: minmax(38px, auto);
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
useDataProcessSourceUpload,
|
||||
validateSourceFileSelection,
|
||||
@@ -33,7 +34,6 @@ import {
|
||||
deleteDataProcessPreview,
|
||||
deleteDataProcessSourceFile,
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
pullDataProcessExternalSource,
|
||||
testDataProcessExternalSource,
|
||||
updateDataProcessPreview,
|
||||
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
|
||||
let allowLeave = false
|
||||
const {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
|
||||
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
|
||||
return 'fa-arrow-right'
|
||||
})
|
||||
|
||||
const previousStepLabel = computed(() => currentStep.value > 0
|
||||
? WIZARD_STEPS[currentStep.value - 1].title
|
||||
: '')
|
||||
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
|
||||
}
|
||||
|
||||
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
const sourceLocator = item.quality_score?.source_locator
|
||||
return {
|
||||
id: String(item.id),
|
||||
sourceFileId: String(item.source_file_id),
|
||||
originalContent: item.original_content,
|
||||
editedContent: item.edited_content,
|
||||
savedEditedContent: item.edited_content,
|
||||
sourceStart: item.source_start,
|
||||
sourceEnd: item.source_end,
|
||||
sourceStartLine: item.source_start_line,
|
||||
sourceEndLine: item.source_end_line,
|
||||
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
|
||||
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
|
||||
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
|
||||
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
|
||||
tokenCount: item.token_count,
|
||||
status: item.status,
|
||||
sourcePages: Array.isArray(item.quality_score?.source_pages)
|
||||
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
|
||||
: [],
|
||||
sourceLocator,
|
||||
headingPath: Array.isArray(item.quality_score?.heading_path)
|
||||
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
|
||||
uploadedFiles.value.push({
|
||||
uid: localUid,
|
||||
rawFile: raw,
|
||||
name: raw.name,
|
||||
size: raw.size,
|
||||
count: 0,
|
||||
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
previewProgress: 0,
|
||||
})
|
||||
dirty.value = true
|
||||
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
|
||||
enqueueSourceUpload({ uid: localUid, file: raw })
|
||||
}
|
||||
|
||||
async function useSampleFile() {
|
||||
@@ -488,11 +493,8 @@ async function handlePullData() {
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
|
||||
const newFiles: UploadedDataFile[] = []
|
||||
for (const file of response.files) {
|
||||
const source = await getDataProcessSourceContent(taskId.value, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
newFiles.push(mapDataProcessSourceFile(file, source.content))
|
||||
const content = await loadCanonicalSourceContent(taskId.value, file.id)
|
||||
newFiles.push(mapDataProcessSourceFile(file, content))
|
||||
}
|
||||
uploadedFiles.value.push(...newFiles)
|
||||
externalConnected.value = true
|
||||
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
|
||||
function updatePreviewContent(id: string, value: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
const isManual = isManualPreviewItem(item)
|
||||
item.editedContent = value
|
||||
item.tokenCount = estimateTokenCount(value)
|
||||
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
||||
item.status = !value.trim()
|
||||
? 'invalid'
|
||||
: value === item.originalContent
|
||||
? 'original'
|
||||
: isManual ? 'manual' : 'modified'
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
|
||||
|
||||
function restorePreviewItem(id: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item || item.sourceStart == null) return
|
||||
if (!item || isManualPreviewItem(item)) return
|
||||
item.editedContent = item.originalContent
|
||||
item.tokenCount = estimateTokenCount(item.originalContent)
|
||||
item.status = 'original'
|
||||
@@ -897,7 +904,7 @@ async function handleBack() {
|
||||
ElMessage.warning('请等待当前文件切分完成')
|
||||
return
|
||||
}
|
||||
if (currentStepId.value === 'generate') return
|
||||
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
|
||||
if (currentStep.value > 0) {
|
||||
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
|
||||
if (!targetStep) return
|
||||
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
|
||||
if (sourceTask.status === 'running') resumeStep = 'generate'
|
||||
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
|
||||
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') {
|
||||
const resume = resumeGeneration()
|
||||
goToStep(resumeStep)
|
||||
await resume
|
||||
return
|
||||
}
|
||||
goToStep(resumeStep)
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
@@ -1154,7 +1166,7 @@ onMounted(() => {
|
||||
<div class="footer-left">
|
||||
<el-button
|
||||
v-if="currentStep > 0"
|
||||
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
|
||||
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
|
||||
@click="handleBack"
|
||||
>
|
||||
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回:{{ previousStepLabel }}
|
||||
@@ -1167,8 +1179,8 @@ onMounted(() => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
getDataProcessResults,
|
||||
getDataProcessTask,
|
||||
publishDataProcess,
|
||||
repeatDataProcessTask,
|
||||
restoreDataProcessResult,
|
||||
updateDataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -42,6 +43,8 @@ const savingResult = ref(false)
|
||||
const restoringResultId = ref<string | number | null>(null)
|
||||
const publishDialogVisible = ref(false)
|
||||
const publishing = ref(false)
|
||||
const repeatGenerating = ref(false)
|
||||
const repeatRequestId = ref('')
|
||||
const configExpanded = ref(false)
|
||||
const resultCellTooltipOptions = {
|
||||
popperClass: 'data-process-result-tooltip',
|
||||
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
|
||||
preserve_context: '保留上下文',
|
||||
}
|
||||
|
||||
const structuredPreprocessOptionKeys = new Set([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
|
||||
function formatStructuredPreprocessOptions(value: unknown[]) {
|
||||
const options = [...new Set(value.map((item) => String(item)))]
|
||||
const selected = new Set(options)
|
||||
const consumed = new Set<string>()
|
||||
const labels: string[] = []
|
||||
|
||||
function appendGroup(values: string[], groupLabel: string) {
|
||||
const selectedValues = values.filter((item) => selected.has(item))
|
||||
selectedValues.forEach((item) => consumed.add(item))
|
||||
if (selectedValues.length === values.length) {
|
||||
labels.push(groupLabel)
|
||||
return
|
||||
}
|
||||
selectedValues.forEach((item) => {
|
||||
labels.push(`${preprocessOptionLabelMap[item] || item}(历史部分配置)`)
|
||||
})
|
||||
}
|
||||
|
||||
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
|
||||
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
|
||||
|
||||
if (selected.has('desensitize')) {
|
||||
consumed.add('desensitize')
|
||||
labels.push('敏感信息脱敏')
|
||||
}
|
||||
if (selected.has('filter_anomaly')) {
|
||||
consumed.add('filter_anomaly')
|
||||
labels.push('异常数据过滤(历史规则)')
|
||||
}
|
||||
|
||||
options.forEach((item) => {
|
||||
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
|
||||
})
|
||||
return labels.length ? labels.join('、') : '-'
|
||||
}
|
||||
|
||||
function numeric(value: unknown) {
|
||||
const parsed = typeof value === 'number' ? value : Number(value)
|
||||
return Number.isFinite(parsed) ? parsed : 0
|
||||
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|
||||
|| status === 'stopped'
|
||||
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
|
||||
})
|
||||
const canRepeatGeneration = computed(() => (
|
||||
detail.value?.status === 'completed'
|
||||
&& detail.value.results_confirmed !== false
|
||||
&& previewCount.value > 0
|
||||
))
|
||||
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
|
||||
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
|
||||
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
|
||||
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (key === 'preprocess_options') {
|
||||
return value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
const containsStructuredOption = value.some((item) => (
|
||||
structuredPreprocessOptionKeys.has(String(item))
|
||||
))
|
||||
return containsStructuredOption
|
||||
? formatStructuredPreprocessOptions(value)
|
||||
: value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
}
|
||||
return value.length ? value.join('、') : '-'
|
||||
}
|
||||
@@ -503,6 +561,48 @@ function startRegeneration() {
|
||||
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
|
||||
}
|
||||
|
||||
function createRepeatRequestId() {
|
||||
if (typeof globalThis.crypto?.randomUUID === 'function') {
|
||||
return globalThis.crypto.randomUUID()
|
||||
}
|
||||
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
|
||||
}
|
||||
|
||||
async function repeatGeneration() {
|
||||
if (!detail.value?.updated_at || repeatGenerating.value) return
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
'系统会复制当前配置、源文件和切分结果,创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
|
||||
'按原配置再生成一批?',
|
||||
{
|
||||
confirmButtonText: '创建并开始生成',
|
||||
cancelButtonText: '取消',
|
||||
type: 'info',
|
||||
},
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
repeatGenerating.value = true
|
||||
repeatRequestId.value ||= createRepeatRequestId()
|
||||
try {
|
||||
const repeated = await repeatDataProcessTask(taskId.value, {
|
||||
expected_updated_at: detail.value.updated_at,
|
||||
request_id: repeatRequestId.value,
|
||||
})
|
||||
ElMessage.success(repeated.created ? '已创建新任务,正在后台生成' : '已恢复此前创建的新任务')
|
||||
await router.push({
|
||||
name: 'data-process-workflow',
|
||||
params: { id: repeated.task.id },
|
||||
})
|
||||
} catch {
|
||||
// 保留幂等请求 ID;网络超时后再次点击不会重复创建任务。
|
||||
} finally {
|
||||
repeatGenerating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch([currentPage, pageSize], () => void loadResults())
|
||||
|
||||
onMounted(loadPage)
|
||||
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
|
||||
<el-tag :type="displayStatus.type" size="small" effect="light">
|
||||
{{ displayStatus.label }}
|
||||
</el-tag>
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
|
||||
</el-button>
|
||||
<div class="heading-actions">
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRepeatGeneration"
|
||||
type="primary"
|
||||
:loading="repeatGenerating"
|
||||
:disabled="repeatGenerating"
|
||||
@click="repeatGeneration"
|
||||
>
|
||||
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
type="warning"
|
||||
plain
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<p>{{ detail.description || '暂无任务描述' }}</p>
|
||||
<dl class="heading-meta">
|
||||
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
|
||||
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
|
||||
}
|
||||
|
||||
.publish-button { margin-left: auto; }
|
||||
.heading-actions {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
|
||||
:deep(.el-button + .el-button) { margin-left: 0; }
|
||||
}
|
||||
.load-state-actions { display: flex; gap: 10px; }
|
||||
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
|
||||
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
|
||||
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
|
||||
@media (max-width: 720px) {
|
||||
.metric-grid, .config-grid { grid-template-columns: 1fr; }
|
||||
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
|
||||
.publish-button { width: 100%; margin-left: 0; }
|
||||
.heading-actions { width: 100%; margin-left: 0; }
|
||||
.heading-actions :deep(.el-button) { width: 100%; }
|
||||
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
|
||||
.result-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.result-filters { padding: 0 16px 16px; flex-direction: column; }
|
||||
|
||||
@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
const selectedXlsxLocator = computed(() => {
|
||||
const locator = props.selectedItem?.sourceLocator
|
||||
if (!locator) return null
|
||||
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
|
||||
const hasRow = locator.row_number != null || locator.sheet_record_index != null
|
||||
return hasSheet && hasRow ? locator : null
|
||||
})
|
||||
const visibleRowRange = computed(() => {
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
|
||||
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
|
||||
})
|
||||
|
||||
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
|
||||
const locator = selectedXlsxLocator.value
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (locator && sheet) {
|
||||
const sheetMatches = locator.sheet_index != null
|
||||
? sheet.index === locator.sheet_index
|
||||
: sheet.name === locator.sheet_name
|
||||
if (!sheetMatches) return false
|
||||
if (locator.row_number != null) return row.row_number === locator.row_number
|
||||
return row.record_index === locator.sheet_record_index
|
||||
}
|
||||
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
|
||||
}
|
||||
|
||||
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
|
||||
async function loadPreview(options: { reset?: boolean } = {}) {
|
||||
const sequence = ++loadSequence
|
||||
if (options.reset) {
|
||||
activeSheetIndex.value = 0
|
||||
pageOffset.value = 0
|
||||
const locator = selectedXlsxLocator.value
|
||||
activeSheetIndex.value = locator?.sheet_index ?? 0
|
||||
pageOffset.value = locator?.sheet_record_index == null
|
||||
? 0
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
preview.value = null
|
||||
}
|
||||
errorMessage.value = ''
|
||||
@@ -178,8 +198,34 @@ watch(
|
||||
)
|
||||
|
||||
watch(
|
||||
() => props.selectedItem?.id,
|
||||
() => void locateSelectedItem(),
|
||||
() => [
|
||||
props.selectedItem?.id,
|
||||
props.selectedItem?.sourceLocator?.sheet_index,
|
||||
props.selectedItem?.sourceLocator?.sheet_record_index,
|
||||
props.selectedItem?.sourceLocator?.row_number,
|
||||
],
|
||||
() => {
|
||||
const locator = selectedXlsxLocator.value
|
||||
if (!locator || isDocx.value) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
|
||||
const targetOffset = locator.sheet_record_index == null
|
||||
? pageOffset.value
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
const activeSheet = xlsxPreview.value?.active_sheet
|
||||
if (
|
||||
activeSheet?.index === targetSheet
|
||||
&& activeSheet.offset === targetOffset
|
||||
) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
activeSheetIndex.value = targetSheet
|
||||
pageOffset.value = targetOffset
|
||||
void loadPreview()
|
||||
},
|
||||
)
|
||||
</script>
|
||||
|
||||
@@ -301,6 +347,8 @@ watch(
|
||||
:key="row.row_number"
|
||||
class="xlsx-row"
|
||||
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
|
||||
:data-row-number="row.row_number"
|
||||
:data-record-index="row.record_index"
|
||||
>
|
||||
<th class="row-number-cell">{{ row.row_number }}</th>
|
||||
<td
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
import OfficeSourceViewer from './OfficeSourceViewer.vue'
|
||||
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||
import { sourceLines } from './previewModel'
|
||||
import {
|
||||
isManualPreviewItem,
|
||||
sourceLineNumberAtOffset,
|
||||
sourceLineWindow,
|
||||
} from './previewModel'
|
||||
import type { PreviewItem, ProcessType } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
|
||||
const search = ref('')
|
||||
const currentPage = ref(1)
|
||||
const PREVIEW_PAGE_SIZE = 10
|
||||
const SOURCE_LINE_RENDER_LIMIT = 240
|
||||
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
|
||||
const sourceWindowStartLine = ref(1)
|
||||
const editingItemId = ref<string | null>(null)
|
||||
const editorDraft = ref('')
|
||||
const lines = computed(() => sourceLines(props.sourceText))
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||
const normalizedFileFormat = computed(() => (
|
||||
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
|
||||
))
|
||||
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
|
||||
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
|
||||
const selectedSourceOffset = computed(() => {
|
||||
const item = selectedItem.value
|
||||
return item ? sourceOffsetRange(item)?.start ?? null : null
|
||||
})
|
||||
const selectedSourceLine = computed(() => {
|
||||
const item = selectedItem.value
|
||||
if (!item) return null
|
||||
return sourceLineRange(item)?.start
|
||||
?? (selectedSourceOffset.value == null
|
||||
? null
|
||||
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
|
||||
})
|
||||
const visibleSourceWindow = computed(() => sourceLineWindow(
|
||||
props.sourceText,
|
||||
sourceWindowStartLine.value,
|
||||
SOURCE_LINE_RENDER_LIMIT,
|
||||
SOURCE_LINE_CHARACTER_LIMIT,
|
||||
selectedSourceLine.value,
|
||||
selectedSourceOffset.value,
|
||||
))
|
||||
const lines = computed(() => visibleSourceWindow.value.lines)
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !search.value.trim()
|
||||
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
|
||||
|
||||
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
|
||||
|
||||
function isLineHighlighted(lineStart: number, lineEnd: number) {
|
||||
function sourceLineRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
|
||||
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
|
||||
return start == null ? null : { start, end: end ?? start }
|
||||
}
|
||||
|
||||
function sourceOffsetRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.source_start ?? item.sourceStart
|
||||
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
|
||||
return start == null ? null : { start, end: Math.max(start, end ?? start) }
|
||||
}
|
||||
|
||||
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
|
||||
const item = selectedItem.value
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
|
||||
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
|
||||
if (!item) return false
|
||||
const lineRange = sourceLineRange(item)
|
||||
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
|
||||
const offsetRange = sourceOffsetRange(item)
|
||||
if (!offsetRange) return false
|
||||
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
|
||||
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
|
||||
}
|
||||
|
||||
function selectItem(id: string) {
|
||||
@@ -107,34 +151,88 @@ watch(search, () => {
|
||||
|
||||
watch(() => props.selectedFileId, closeEditor)
|
||||
|
||||
watch(selectedItem, async (item) => {
|
||||
watch([selectedItem, () => props.sourceText], async ([item]) => {
|
||||
if (!item) return
|
||||
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
|
||||
if (visibleIndex >= 0) {
|
||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||
}
|
||||
|
||||
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
|
||||
if (isPdfSource.value || isOfficeSource.value) return
|
||||
const itemLineRange = sourceLineRange(item)
|
||||
const itemOffsetRange = sourceOffsetRange(item)
|
||||
if (!itemLineRange && !itemOffsetRange) {
|
||||
sourceWindowStartLine.value = 1
|
||||
return
|
||||
}
|
||||
const targetLine = selectedSourceLine.value
|
||||
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
|
||||
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
|
||||
await nextTick()
|
||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||
const exactTarget = sourceViewerRef.value
|
||||
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
|
||||
const target = exactTarget
|
||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
|
||||
}, { immediate: true })
|
||||
|
||||
async function showPreviousSourceWindow() {
|
||||
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
async function showNextSourceWindow() {
|
||||
if (!visibleSourceWindow.value.hasMore) return
|
||||
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
function itemNumber(item: PreviewItem) {
|
||||
return props.items.findIndex((entry) => entry.id === item.id) + 1
|
||||
}
|
||||
|
||||
function lineRange(item: PreviewItem) {
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
return first === last ? `来源:第 ${first} 页` : `来源:第 ${first}–${last} 页`
|
||||
if (isManualPreviewItem(item)) return '手动新增,无源文件定位'
|
||||
|
||||
const locator = item.sourceLocator
|
||||
const locatedLines = sourceLineRange(item)
|
||||
if (props.processType === 'unstructured') {
|
||||
const parts: string[] = []
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
|
||||
}
|
||||
if (locatedLines) {
|
||||
parts.push(
|
||||
locatedLines.start === locatedLines.end
|
||||
? `第 ${locatedLines.start} 行`
|
||||
: `第 ${locatedLines.start}–${locatedLines.end} 行`,
|
||||
)
|
||||
}
|
||||
if (item.headingPath?.length) parts.push(`章节:${item.headingPath.join(' / ')}`)
|
||||
return parts.length ? `来源:${parts.join(' · ')}` : '来源:源文件内容(无精确定位)'
|
||||
}
|
||||
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增,无源文件定位'
|
||||
return item.sourceStartLine === item.sourceEndLine
|
||||
? `来源:第 ${item.sourceStartLine} 行`
|
||||
: `来源:第 ${item.sourceStartLine}–${item.sourceEndLine} 行`
|
||||
|
||||
if (locator?.kind === 'xlsx') {
|
||||
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
|
||||
return locator.row_number != null
|
||||
? `来源:${sheet} · 第 ${locator.row_number} 行`
|
||||
: `来源:${sheet}`
|
||||
}
|
||||
if (locator?.kind === 'json') {
|
||||
return locator.json_pointer
|
||||
? `来源:JSON 路径 ${locator.json_pointer}`
|
||||
: '来源:JSON 根对象'
|
||||
}
|
||||
if (locatedLines) {
|
||||
return locatedLines.start === locatedLines.end
|
||||
? `来源:第 ${locatedLines.start} 行`
|
||||
: `来源:第 ${locatedLines.start}–${locatedLines.end} 行`
|
||||
}
|
||||
return '来源:源文件记录'
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
|
||||
<div>
|
||||
<strong>源文件 · {{ fileName }}</strong>
|
||||
</div>
|
||||
<div
|
||||
v-if="!isPdfSource && !isOfficeSource && lines.length"
|
||||
class="source-window-controls"
|
||||
aria-label="源文件行窗口"
|
||||
>
|
||||
<span>第 {{ visibleSourceWindow.startLine }}–{{ visibleSourceWindow.endLine }} 行</span>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看上一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasPrevious"
|
||||
@click="showPreviousSourceWindow"
|
||||
>
|
||||
上一段
|
||||
</el-button>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看下一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasMore"
|
||||
@click="showNextSourceWindow"
|
||||
>
|
||||
下一段
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<PdfSourceViewer
|
||||
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
|
||||
v-for="line in lines"
|
||||
:key="line.number"
|
||||
class="source-line"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
|
||||
:data-source-start="line.start"
|
||||
:data-line-number="line.number"
|
||||
>
|
||||
<span class="line-number">{{ line.number }}</span>
|
||||
<span class="line-content">{{ line.content || ' ' }}</span>
|
||||
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
|
||||
/>
|
||||
<div class="editor-actions">
|
||||
<el-button
|
||||
v-if="editingItem.sourceStart != null"
|
||||
v-if="!isManualPreviewItem(editingItem)"
|
||||
link
|
||||
@click="restoreItem"
|
||||
>
|
||||
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
|
||||
}
|
||||
}
|
||||
|
||||
.source-window-controls {
|
||||
flex: none;
|
||||
gap: 2px !important;
|
||||
|
||||
> span {
|
||||
margin-right: 4px;
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
:deep(.el-button) {
|
||||
margin-left: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.source-viewer {
|
||||
flex: 1;
|
||||
height: 538px;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
@@ -21,27 +22,32 @@ const emit = defineEmits<{
|
||||
'update:options': [value: StructuredProcessOptions]
|
||||
}>()
|
||||
|
||||
const PREPROCESS_OPTIONS: Array<{
|
||||
value: PreprocessOption
|
||||
const PREPROCESS_GROUPS: Array<{
|
||||
values: PreprocessOption[]
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
||||
{
|
||||
value: 'detect_structure',
|
||||
label: '嵌套结构展平',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||
values: ['clean_invalid', 'deduplicate'],
|
||||
label: '数据清洗',
|
||||
description: '清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
|
||||
},
|
||||
{
|
||||
value: 'deduplicate',
|
||||
label: '重复记录去重',
|
||||
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||
values: ['detect_structure', 'normalize_format'],
|
||||
label: '结构标准化',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
|
||||
},
|
||||
{
|
||||
values: ['desensitize'],
|
||||
label: '敏感信息脱敏',
|
||||
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
|
||||
},
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||
]
|
||||
|
||||
const legacyAnomalyFilterEnabled = computed(() => (
|
||||
props.options.preprocessOptions.includes('filter_anomaly')
|
||||
))
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
field: K,
|
||||
value: StructuredProcessOptions[K],
|
||||
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
|
||||
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
(option): option is PreprocessOption => (
|
||||
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||
),
|
||||
)))
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
function selectedCount(values: PreprocessOption[]) {
|
||||
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
|
||||
}
|
||||
|
||||
function groupSelected(values: PreprocessOption[]) {
|
||||
return selectedCount(values) === values.length
|
||||
}
|
||||
|
||||
function groupIndeterminate(values: PreprocessOption[]) {
|
||||
const count = selectedCount(values)
|
||||
return count > 0 && count < values.length
|
||||
}
|
||||
|
||||
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
|
||||
const next = new Set(props.options.preprocessOptions)
|
||||
values.forEach((value) => {
|
||||
if (Boolean(checked)) next.add(value)
|
||||
else next.delete(value)
|
||||
})
|
||||
updateField('preprocessOptions', [...next])
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>选择在生成问答对之前需要执行的数据处理方式</p>
|
||||
<p>默认不执行预处理,请按数据情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-checkbox-group
|
||||
:model-value="options.preprocessOptions"
|
||||
class="preprocess-option-grid"
|
||||
@update:model-value="updatePreprocessOptions"
|
||||
>
|
||||
<el-checkbox
|
||||
v-for="option in PREPROCESS_OPTIONS"
|
||||
:key="option.value"
|
||||
:value="option.value"
|
||||
<div class="preprocess-option-grid">
|
||||
<label
|
||||
v-for="group in PREPROCESS_GROUPS"
|
||||
:key="group.label"
|
||||
class="preprocess-option"
|
||||
:class="{ 'is-checked': groupSelected(group.values) }"
|
||||
>
|
||||
<el-checkbox
|
||||
:model-value="groupSelected(group.values)"
|
||||
:indeterminate="groupIndeterminate(group.values)"
|
||||
@update:model-value="updatePreprocessGroup(group.values, $event)"
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>{{ option.label }}</strong>
|
||||
<small>{{ option.description }}</small>
|
||||
<strong>{{ group.label }}</strong>
|
||||
<small>{{ group.description }}</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</label>
|
||||
</div>
|
||||
<el-alert
|
||||
v-if="legacyAnomalyFilterEnabled"
|
||||
class="legacy-preprocess-alert"
|
||||
type="warning"
|
||||
:closable="false"
|
||||
title="该历史任务仍启用了已停用的“异常数据过滤”;为保证结果可复现,本次继续保留"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="form-section generation-options-section">
|
||||
|
||||
@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认启用结构感知的推荐策略,只需决定是否需要脱敏</p>
|
||||
<p>默认不执行预处理,请按文档情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="preprocess-option-grid">
|
||||
|
||||
@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
preprocessOptions: [],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
|
||||
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
preprocessOptions: [],
|
||||
chunkMethod: 'layout_hybrid',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
semanticBreakpointPercentile: 95,
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
const supportedPreprocessOptions = new Set<PreprocessOption>([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
? Array.from(new Set(preprocessOptions.map(String).filter(
|
||||
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
|
||||
)))
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { SourceLine } from './types'
|
||||
import type { PreviewItem, SourceLine } from './types'
|
||||
|
||||
/** 仅用于“使用示例”上传;正式预览和切片全部由后端生成。 */
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
|
||||
'答:复利是将上一期利息加入本金,再计算下一期利息。',
|
||||
].join('\n')
|
||||
|
||||
/**
|
||||
* 把后端返回的字符偏移映射为源文件行,仅负责界面高亮,不参与切片。
|
||||
*/
|
||||
export function sourceLines(sourceText: string): SourceLine[] {
|
||||
const rawLines = sourceText.split('\n')
|
||||
let cursor = 0
|
||||
export interface SourceLineWindow {
|
||||
lines: SourceLine[]
|
||||
startLine: number
|
||||
endLine: number
|
||||
hasPrevious: boolean
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
return rawLines.map((content, index) => {
|
||||
const start = cursor
|
||||
const end = start + content.length
|
||||
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
|
||||
return { number: index + 1, content, start, end }
|
||||
})
|
||||
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
|
||||
let length = 0
|
||||
let index = start
|
||||
while (index < end) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
length += 1
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
function advanceCodePoints(value: string, start: number, end: number, count: number) {
|
||||
let index = start
|
||||
let remaining = Math.max(0, count)
|
||||
while (index < end && remaining > 0) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
remaining -= 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* 只扫描并返回当前可见行窗口,不对全文 split,避免大文件生成巨量字符串数组。
|
||||
* 字符定位场景可开启 code point 偏移,以与后端 Python 的字符计数保持一致。
|
||||
*/
|
||||
export function sourceLineWindow(
|
||||
sourceText: string,
|
||||
requestedStartLine: number,
|
||||
maxLines: number,
|
||||
maxCharactersPerLine: number,
|
||||
focusLine: number | null = null,
|
||||
focusOffset: number | null = null,
|
||||
): SourceLineWindow {
|
||||
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
|
||||
const limit = Math.max(1, Math.trunc(maxLines) || 1)
|
||||
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
|
||||
const trackUnicodeOffsets = focusOffset != null
|
||||
const lines: SourceLine[] = []
|
||||
let lineNumber = 1
|
||||
let jsCursor = 0
|
||||
let sourceCursor = 0
|
||||
|
||||
while (jsCursor <= sourceText.length && lineNumber < startLine) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
sourceCursor = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
|
||||
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
while (jsCursor <= sourceText.length && lines.length < limit) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
const fullSourceEnd = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
|
||||
: jsEnd
|
||||
const focusedStart = focusLine === lineNumber && focusOffset != null
|
||||
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
|
||||
: sourceCursor
|
||||
const segmentSourceStart = Math.min(
|
||||
focusedStart,
|
||||
Math.max(sourceCursor, fullSourceEnd - characterLimit),
|
||||
)
|
||||
const relativeSegmentStart = trackUnicodeOffsets
|
||||
? segmentSourceStart - sourceCursor
|
||||
: Math.max(0, segmentSourceStart - jsCursor)
|
||||
const segmentJsStart = advanceCodePoints(
|
||||
sourceText,
|
||||
jsCursor,
|
||||
jsEnd,
|
||||
relativeSegmentStart,
|
||||
)
|
||||
const segmentJsEnd = advanceCodePoints(
|
||||
sourceText,
|
||||
segmentJsStart,
|
||||
jsEnd,
|
||||
characterLimit,
|
||||
)
|
||||
const segmentLength = trackUnicodeOffsets
|
||||
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
|
||||
: segmentJsEnd - segmentJsStart
|
||||
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
|
||||
const end = start + segmentLength
|
||||
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
|
||||
lines.push({ number: lineNumber, content, start, end })
|
||||
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
return {
|
||||
lines,
|
||||
startLine: lines[0]?.number ?? startLine,
|
||||
endLine: lines[lines.length - 1]?.number ?? startLine,
|
||||
hasPrevious: startLine > 1,
|
||||
hasMore: jsCursor <= sourceText.length,
|
||||
}
|
||||
}
|
||||
|
||||
/** 根据后端 code point 偏移查找物理行号,不构建全文行数组。 */
|
||||
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
|
||||
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
|
||||
let offset = 0
|
||||
let lineNumber = 1
|
||||
for (const character of sourceText) {
|
||||
if (offset >= normalizedOffset) break
|
||||
if (character === '\n') lineNumber += 1
|
||||
offset += 1
|
||||
}
|
||||
return lineNumber
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动新增项可能先以空内容保存为 invalid,编辑后又由后端标记为 modified,
|
||||
* 因此不能只依赖可变的 status;空原文且完全没有来源定位才是稳定兜底。
|
||||
*/
|
||||
export function isManualPreviewItem(item: PreviewItem): boolean {
|
||||
const hasSourceLocation = item.sourceStart != null
|
||||
|| item.sourceEnd != null
|
||||
|| item.sourceStartLine != null
|
||||
|| item.sourceEndLine != null
|
||||
|| Boolean(item.sourcePages?.length)
|
||||
|| Boolean(item.sourceLocator)
|
||||
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
|
||||
}
|
||||
|
||||
/** 与后端预览 token 估算规则一致,仅用于编辑中的即时计数。 */
|
||||
|
||||
@@ -24,6 +24,7 @@ export type PreprocessOption =
|
||||
| 'detect_structure'
|
||||
| 'deduplicate'
|
||||
| 'normalize_format'
|
||||
/** 仅用于恢复历史任务,新任务界面不再提供。 */
|
||||
| 'filter_anomaly'
|
||||
| 'desensitize'
|
||||
|
||||
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
rawFile?: File
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
@@ -118,6 +118,22 @@ export interface SourceLine {
|
||||
end: number
|
||||
}
|
||||
|
||||
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
|
||||
|
||||
export interface PreviewSourceLocator {
|
||||
kind: PreviewSourceLocatorKind
|
||||
record_index?: number | null
|
||||
start_line?: number | null
|
||||
end_line?: number | null
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
json_pointer?: string | null
|
||||
sheet_index?: number | null
|
||||
sheet_name?: string | null
|
||||
row_number?: number | null
|
||||
sheet_record_index?: number | null
|
||||
}
|
||||
|
||||
export interface PreviewItem {
|
||||
id: string
|
||||
sourceFileId: string
|
||||
@@ -129,6 +145,8 @@ export interface PreviewItem {
|
||||
sourceStartLine: number | null
|
||||
sourceEndLine: number | null
|
||||
sourcePages?: number[]
|
||||
sourceLocator?: PreviewSourceLocator
|
||||
headingPath?: string[]
|
||||
tokenCount: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
qualityScore?: number
|
||||
|
||||
@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
let generationTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let generationRun = 0
|
||||
let pollFailureCount = 0
|
||||
let generationStarting = false
|
||||
const generationStarting = ref(false)
|
||||
const generationRestoring = ref(false)
|
||||
const canReturnFromGeneration = computed(() => (
|
||||
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
|
||||
))
|
||||
|
||||
function stopGenerationTimer() {
|
||||
generationRun += 1
|
||||
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
}
|
||||
|
||||
async function startGeneration() {
|
||||
if (generationStarting || generation.status === 'running') return false
|
||||
if (generationStarting.value || generation.status === 'running') return false
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) {
|
||||
ElMessage.error('任务尚未创建,请返回上一步重试')
|
||||
return false
|
||||
}
|
||||
|
||||
generationStarting = true
|
||||
generationStarting.value = true
|
||||
let runId: number | null = null
|
||||
try {
|
||||
const canStart = await bindings.beforeGenerate?.()
|
||||
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
generation.message = error instanceof Error ? error.message : '启动数据处理失败,请重试。'
|
||||
return false
|
||||
} finally {
|
||||
generationStarting = false
|
||||
generationStarting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
generationRestoring.value = true
|
||||
try {
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (activeRunId !== generationRun) return
|
||||
if (progress.status === 'running') {
|
||||
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
} catch (error) {
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败,请重试。'
|
||||
} finally {
|
||||
generationRestoring.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
|
||||
return {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
|
||||
@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import {
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
regenerateDataProcessTask,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -15,7 +14,10 @@ import {
|
||||
createStructuredOptionsFromConfig,
|
||||
createUnstructuredOptionsFromConfig,
|
||||
} from './dataProcessCreateState'
|
||||
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
} from './useDataProcessSourceUpload'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
@@ -50,24 +52,6 @@ interface RegenerationBindings {
|
||||
resetDownstream: () => void
|
||||
}
|
||||
|
||||
async function loadSourceContent(taskId: string, fileId: string | number) {
|
||||
const chunks: string[] = []
|
||||
let startLine = 1
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
start_line: startLine,
|
||||
line_count: 10_000,
|
||||
})
|
||||
chunks.push(source.content || '')
|
||||
if (!source.has_more) break
|
||||
const nextLine = Number(source.end_line || startLine) + 1
|
||||
if (nextLine <= startLine) break
|
||||
startLine = nextLine
|
||||
}
|
||||
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
|
||||
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
|
||||
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
|
||||
const taskId = String(task.id)
|
||||
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
|
||||
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
|
||||
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
|
||||
)))
|
||||
bindings.previewItems.value = preservePreviews
|
||||
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
function unicodeCodePointLength(value: string) {
|
||||
let length = 0
|
||||
for (const _character of value) length += 1
|
||||
return length
|
||||
}
|
||||
|
||||
/** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */
|
||||
export async function loadCanonicalSourceContent(
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) {
|
||||
const chunks: string[] = []
|
||||
let offset = 0
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
offset,
|
||||
limit: SOURCE_CONTENT_PAGE_CHARS,
|
||||
})
|
||||
const content = source.content || ''
|
||||
chunks.push(content)
|
||||
if (!source.has_more) break
|
||||
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
|
||||
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试')
|
||||
offset = nextOffset
|
||||
}
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
|
||||
// 先登记后端 ID,确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
try {
|
||||
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
|
||||
} catch {
|
||||
throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试')
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
|
||||
@@ -50,6 +50,20 @@ const rules: FormRules = {
|
||||
}
|
||||
|
||||
/** 处理文件选择(替换模式:新文件覆盖旧文件) */
|
||||
function parseDatasetRecordValues(text: string, fileName: string): unknown[] {
|
||||
const content = text.trim()
|
||||
if (!content) return []
|
||||
if (fileName.toLowerCase().endsWith('.json')) {
|
||||
const parsed = JSON.parse(content)
|
||||
return Array.isArray(parsed) ? parsed : [parsed]
|
||||
}
|
||||
return content
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
}
|
||||
|
||||
async function handleFileChange(uploadFile: UploadFile) {
|
||||
const raw = uploadFile.raw
|
||||
if (!raw) return
|
||||
@@ -70,25 +84,19 @@ async function handleFileChange(uploadFile: UploadFile) {
|
||||
async function analyzeFile(file: File) {
|
||||
try {
|
||||
const text = await file.text()
|
||||
const lines = text.trim().split('\n').filter(Boolean)
|
||||
fileCount.value = lines.length
|
||||
const records = parseDatasetRecordValues(text, file.name)
|
||||
fileCount.value = records.length
|
||||
|
||||
// Alpaca 格式校验:每行 JSON 须含 instruction 字段
|
||||
let validCount = 0
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const obj = JSON.parse(line)
|
||||
if (obj.instruction !== undefined) validCount++
|
||||
} catch {
|
||||
// 非 JSON 行(如纯 JSONL 多行结构)
|
||||
}
|
||||
}
|
||||
if (validCount > 0 && validCount === lines.length) {
|
||||
const validCount = records.filter(
|
||||
(obj) => obj && typeof obj === 'object' && 'instruction' in obj,
|
||||
).length
|
||||
if (validCount > 0 && validCount === records.length) {
|
||||
formatValid.value = true
|
||||
formatMessage.value = `符合 Alpaca 格式(含 instruction 字段)`
|
||||
} else if (validCount > 0) {
|
||||
formatValid.value = true
|
||||
formatMessage.value = `部分符合 Alpaca 格式(${validCount}/${lines.length})`
|
||||
formatMessage.value = `部分符合 Alpaca 格式(${validCount}/${records.length})`
|
||||
} else {
|
||||
formatValid.value = false
|
||||
formatMessage.value = '未检测到标准 Alpaca 格式(缺少 instruction 字段),仍可上传'
|
||||
|
||||
@@ -79,7 +79,9 @@ async function loadEditData() {
|
||||
async function loadModels() {
|
||||
try {
|
||||
const all = (await getModelList()) || []
|
||||
evalModels.value = all.filter((m) => m.purpose === 'evaluation')
|
||||
evalModels.value = all.filter(
|
||||
(m) => m.purpose === 'evaluation' || (m.model_source === 'api' && !!m.api_url),
|
||||
)
|
||||
} catch {
|
||||
evalModels.value = []
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<script setup lang="ts">
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref, watch } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
@@ -10,7 +10,7 @@ import StartEvalStep from './create/StartEvalStep.vue'
|
||||
import { createDimension, startEval } from '@/api/modules/eval'
|
||||
import { getTrainedModels, getModelList } from '@/api/modules/model'
|
||||
import { getDatasetList } from '@/api/modules/dataset'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { getComputeGpus } from '@/api/modules/compute'
|
||||
import type { DatasetItem, Dimension, GpuInfo, ModelItem, TrainedModel } from '@/types'
|
||||
|
||||
type StepExposed = { validate: () => Promise<boolean> }
|
||||
@@ -82,17 +82,21 @@ async function loadData() {
|
||||
const results = await Promise.allSettled([
|
||||
getTrainedModels(),
|
||||
getDatasetList(),
|
||||
getSystemInfo(),
|
||||
getModelList(),
|
||||
getComputeGpus(),
|
||||
])
|
||||
|
||||
if (results[0].status === 'fulfilled') trainedModels.value = results[0].value?.models || []
|
||||
if (results[1].status === 'fulfilled') {
|
||||
evalDatasets.value = (results[1].value || []).filter((dataset) => dataset.type === 'eval')
|
||||
}
|
||||
if (results[2].status === 'fulfilled') gpus.value = results[2].value?.gpu || []
|
||||
if (results[2].status === 'fulfilled') {
|
||||
evalModels.value = (results[2].value || []).filter(
|
||||
(model) => model.purpose === 'evaluation' || (model.model_source === 'api' && !!model.api_url),
|
||||
)
|
||||
}
|
||||
if (results[3].status === 'fulfilled') {
|
||||
evalModels.value = (results[3].value || []).filter((model) => model.purpose === 'evaluation')
|
||||
gpus.value = ((results[3].value || []) as unknown as GpuInfo[]).filter((g) => g.status === 'idle')
|
||||
}
|
||||
|
||||
const failedCount = results.filter((result) => result.status === 'rejected').length
|
||||
@@ -139,11 +143,15 @@ async function handleSubmit() {
|
||||
submitting.value = true
|
||||
try {
|
||||
const dimensionId = await resolveDimensionId()
|
||||
await startEval({
|
||||
// GPU 选择为「节点:GPU序号」复合值,解析出节点与 GPU 序号,
|
||||
// 多算力节点时必须把节点信息传给后端,否则会派发到错误的算力节点
|
||||
const [gpuNodeId, gpuIndex] = String(taskForm.value.gpu_id).split(':')
|
||||
const evalResult: any = await startEval({
|
||||
eval_task_name: taskForm.value.eval_task_name,
|
||||
eval_type: 'custom',
|
||||
model_id: taskForm.value.model_id,
|
||||
gpu_id: taskForm.value.gpu_id,
|
||||
gpu_id: Number(gpuIndex) || 0,
|
||||
compute_node_id: gpuNodeId || '',
|
||||
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
||||
dimension_id: dimensionId,
|
||||
data_source: taskForm.value.data_source,
|
||||
@@ -163,6 +171,10 @@ async function handleSubmit() {
|
||||
output_precision: basicMetricForm.value.output_precision,
|
||||
},
|
||||
})
|
||||
if (evalResult?.status === 'failed' || evalResult?.error) {
|
||||
ElMessage.error(`评测启动失败:${evalResult?.error || '请检查算力节点与模型路径'}`)
|
||||
return
|
||||
}
|
||||
ElMessage.success('评测任务已创建并启动')
|
||||
router.push('/model-eval')
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import PageCard from '@/components/PageCard.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { getEvalDetail } from '@/api/modules/eval'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import type { EvalSampleResult, EvalTaskDetail } from '@/types'
|
||||
|
||||
const route = useRoute()
|
||||
@@ -16,6 +17,7 @@ const keyword = ref('')
|
||||
const judgementFilter = ref('')
|
||||
const currentPage = ref(1)
|
||||
const pageSize = ref(10)
|
||||
const ACTIVE_STATUSES = new Set(['pending', 'queued', 'running'])
|
||||
|
||||
const filteredSamples = computed(() => {
|
||||
const normalizedKeyword = keyword.value.trim().toLowerCase()
|
||||
@@ -48,6 +50,8 @@ const passRate = computed(() => {
|
||||
})
|
||||
|
||||
const overallScore = computed(() => formatScore(detail.value?.overall_score, detail.value?.overall_score_max))
|
||||
const displayModelName = computed(() => detail.value?.model_name || String(detail.value?.model_id || '-'))
|
||||
const displayMetric = computed(() => detail.value?.metric_label || detail.value?.metric || '-')
|
||||
|
||||
function formatDateTime(value?: string) {
|
||||
if (!value) return '-'
|
||||
@@ -74,8 +78,8 @@ function resetPage() {
|
||||
currentPage.value = 1
|
||||
}
|
||||
|
||||
async function loadDetail() {
|
||||
loading.value = true
|
||||
async function loadDetail(options: { silent?: boolean } = {}) {
|
||||
if (!options.silent) loading.value = true
|
||||
loadError.value = ''
|
||||
try {
|
||||
detail.value = await getEvalDetail(taskId)
|
||||
@@ -83,11 +87,29 @@ async function loadDetail() {
|
||||
detail.value = null
|
||||
loadError.value = '评测详情加载失败,请稍后重试。'
|
||||
} finally {
|
||||
loading.value = false
|
||||
if (!options.silent) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadDetail)
|
||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||
async () => {
|
||||
await loadDetail({ silent: true })
|
||||
if (!ACTIVE_STATUSES.has(String(detail.value?.status || ''))) {
|
||||
stopPolling()
|
||||
}
|
||||
},
|
||||
5000,
|
||||
{ immediate: false },
|
||||
)
|
||||
|
||||
onMounted(async () => {
|
||||
await loadDetail()
|
||||
if (ACTIVE_STATUSES.has(String(detail.value?.status || ''))) {
|
||||
startPolling()
|
||||
}
|
||||
})
|
||||
|
||||
onUnmounted(stopPolling)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -101,9 +123,9 @@ onMounted(loadDetail)
|
||||
</div>
|
||||
<dl class="task-meta">
|
||||
<div><dt>任务 ID</dt><dd>{{ detail?.id || taskId }}</dd></div>
|
||||
<div><dt>评测模型</dt><dd>{{ detail?.model_name || '-' }}</dd></div>
|
||||
<div><dt>评测模型</dt><dd>{{ displayModelName }}</dd></div>
|
||||
<div><dt>测试集</dt><dd>{{ detail?.dataset || '-' }}</dd></div>
|
||||
<div><dt>评测指标</dt><dd>{{ detail?.metric || '-' }}</dd></div>
|
||||
<div><dt>评测指标</dt><dd>{{ displayMetric }}</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
@@ -113,7 +135,7 @@ onMounted(loadDetail)
|
||||
<i class="fa fa-exclamation-circle" aria-hidden="true" />
|
||||
<h2>无法加载评测详情</h2>
|
||||
<p>{{ loadError }}</p>
|
||||
<el-button type="primary" @click="loadDetail">重新加载</el-button>
|
||||
<el-button type="primary" @click="() => loadDetail()">重新加载</el-button>
|
||||
</div>
|
||||
|
||||
<template v-else-if="detail">
|
||||
@@ -121,7 +143,7 @@ onMounted(loadDetail)
|
||||
<div class="overview-item score-hero">
|
||||
<span>综合得分</span>
|
||||
<strong>{{ overallScore }}</strong>
|
||||
<small>大模型综合评分</small>
|
||||
<small>模型综合评分</small>
|
||||
</div>
|
||||
<div class="overview-item">
|
||||
<span>样本通过率</span>
|
||||
@@ -144,7 +166,7 @@ onMounted(loadDetail)
|
||||
<div class="review-copy">
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<h2 id="overall-review-title">大模型综合评价</h2>
|
||||
<h2 id="overall-review-title">综合评价</h2>
|
||||
<p>基于全部已评测样本生成的总体结论</p>
|
||||
</div>
|
||||
<el-tag v-if="detail.evaluator_model" type="primary" size="small">
|
||||
@@ -152,7 +174,7 @@ onMounted(loadDetail)
|
||||
</el-tag>
|
||||
</div>
|
||||
<p class="review-text">
|
||||
{{ detail.overall_evaluation || (detail.status === 'running' ? '评测仍在进行,综合评价将在样本评分完成后生成。' : '暂无综合评价。') }}
|
||||
{{ detail.overall_evaluation || (detail.status === 'running' ? '评测正在进行,综合评价将在样本完成后生成。' : '暂无综合评价。') }}
|
||||
</p>
|
||||
|
||||
<div class="suggestion-block">
|
||||
@@ -170,8 +192,8 @@ onMounted(loadDetail)
|
||||
<section v-if="detail.dimension_summary?.length" class="dimension-summary" aria-labelledby="dimension-title">
|
||||
<div class="section-heading compact-heading">
|
||||
<div>
|
||||
<h2 id="dimension-title">维度表现</h2>
|
||||
<p>查看各评测维度的得分与样本通过率</p>
|
||||
<h2 id="dimension-title">指标表现</h2>
|
||||
<p>查看各评测指标的得分与通过率</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dimension-grid">
|
||||
@@ -192,22 +214,10 @@ onMounted(loadDetail)
|
||||
<p>共 {{ filteredSamples.length }} 条结果,展开行可查看评分依据与子维度分数</p>
|
||||
</div>
|
||||
<div class="sample-filters" aria-label="样本筛选">
|
||||
<el-input
|
||||
v-model="keyword"
|
||||
clearable
|
||||
placeholder="搜索问题、回答或评价"
|
||||
aria-label="搜索样本"
|
||||
@input="resetPage"
|
||||
>
|
||||
<el-input v-model="keyword" clearable placeholder="搜索问题、回答或评价" aria-label="搜索样本" @input="resetPage">
|
||||
<template #prefix><i class="fa fa-search" aria-hidden="true" /></template>
|
||||
</el-input>
|
||||
<el-select
|
||||
v-model="judgementFilter"
|
||||
clearable
|
||||
placeholder="全部判定"
|
||||
aria-label="按判定筛选"
|
||||
@change="resetPage"
|
||||
>
|
||||
<el-select v-model="judgementFilter" clearable placeholder="全部判定" aria-label="按判定筛选" @change="resetPage">
|
||||
<el-option label="正确" value="正确" />
|
||||
<el-option label="部分正确" value="部分正确" />
|
||||
<el-option label="错误" value="错误" />
|
||||
@@ -215,18 +225,12 @@ onMounted(loadDetail)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<el-table
|
||||
v-if="filteredSamples.length"
|
||||
class="sample-results-table"
|
||||
:data="paginatedSamples"
|
||||
row-key="id"
|
||||
table-layout="fixed"
|
||||
>
|
||||
<el-table v-if="filteredSamples.length" class="sample-results-table" :data="paginatedSamples" row-key="id" table-layout="fixed">
|
||||
<el-table-column type="expand" width="48">
|
||||
<template #default="{ row }">
|
||||
<div class="sample-detail-grid">
|
||||
<div class="evaluation-reason">
|
||||
<span>大模型评分依据</span>
|
||||
<span>评分依据</span>
|
||||
<p>{{ row.evaluation_reason || '暂无评分依据。' }}</p>
|
||||
</div>
|
||||
<div v-if="row.error_type" class="error-type">
|
||||
@@ -255,9 +259,7 @@ onMounted(loadDetail)
|
||||
<template #default="{ row }"><p class="cell-copy">{{ row.model_output || '等待生成' }}</p></template>
|
||||
</el-table-column>
|
||||
<el-table-column label="得分" width="90" align="center">
|
||||
<template #default="{ row }">
|
||||
<span class="sample-score">{{ formatScore(row.score, row.max_score) }}</span>
|
||||
</template>
|
||||
<template #default="{ row }"><span class="sample-score">{{ formatScore(row.score, row.max_score) }}</span></template>
|
||||
</el-table-column>
|
||||
<el-table-column label="判定" width="96" align="center">
|
||||
<template #default="{ row }">
|
||||
@@ -274,21 +276,11 @@ onMounted(loadDetail)
|
||||
<p>{{ detail.status === 'running' ? '任务正在运行,结果生成后会显示在这里。' : '请调整筛选条件或稍后重试。' }}</p>
|
||||
</div>
|
||||
|
||||
<el-pagination
|
||||
v-if="filteredSamples.length > pageSize"
|
||||
v-model:current-page="currentPage"
|
||||
v-model:page-size="pageSize"
|
||||
background
|
||||
layout="total, sizes, prev, pager, next"
|
||||
:page-sizes="[10, 20, 50]"
|
||||
:total="filteredSamples.length"
|
||||
aria-label="样本结果分页"
|
||||
/>
|
||||
<el-pagination v-if="filteredSamples.length > pageSize" v-model:current-page="currentPage" v-model:page-size="pageSize" background layout="total, sizes, prev, pager, next" :page-sizes="[10, 20, 50]" :total="filteredSamples.length" aria-label="样本结果分页" />
|
||||
</section>
|
||||
</template>
|
||||
</PageCard>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.eval-detail-page {
|
||||
min-width: 0;
|
||||
@@ -747,3 +739,6 @@ onMounted(loadDetail)
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted } from 'vue'
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted, onUnmounted } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import {
|
||||
getEvalList,
|
||||
deleteEval,
|
||||
@@ -23,14 +24,16 @@ const leaderboard = ref([
|
||||
{ rank: 3, name: 'Qwen-Max', score: 85.3 },
|
||||
])
|
||||
|
||||
async function loadEvalList() {
|
||||
evalLoading.value = true
|
||||
const ACTIVE_STATUSES = new Set(['pending', 'queued', 'running'])
|
||||
|
||||
async function loadEvalList(options: { silent?: boolean } = {}) {
|
||||
if (!options.silent) evalLoading.value = true
|
||||
try {
|
||||
evalList.value = (await getEvalList()) || []
|
||||
} catch {
|
||||
evalList.value = []
|
||||
} finally {
|
||||
evalLoading.value = false
|
||||
if (!options.silent) evalLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,8 +57,34 @@ function handleViewDetail(row: any) {
|
||||
router.push({ name: 'model-eval-detail', params: { id: row.id } })
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
loadEvalList()
|
||||
function displayModelName(row: Partial<EvalTask>) {
|
||||
return row.model_name || String(row.model_id || '-')
|
||||
}
|
||||
|
||||
function displayMetric(row: Partial<EvalTask>) {
|
||||
return row.metric_label || row.metric || '-'
|
||||
}
|
||||
|
||||
const { start: startPolling, stop: stopPolling } = usePolling(
|
||||
async () => {
|
||||
await loadEvalList({ silent: true })
|
||||
if (!evalList.value.some((item) => ACTIVE_STATUSES.has(String(item.status || '')))) {
|
||||
stopPolling()
|
||||
}
|
||||
},
|
||||
5000,
|
||||
{ immediate: false },
|
||||
)
|
||||
|
||||
onMounted(async () => {
|
||||
await loadEvalList()
|
||||
if (evalList.value.some((item) => ACTIVE_STATUSES.has(String(item.status || '')))) {
|
||||
startPolling()
|
||||
}
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
stopPolling()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -81,9 +110,21 @@ onMounted(() => {
|
||||
</template>
|
||||
<template #columns>
|
||||
<el-table-column label="任务名称" prop="eval_task_name" align="center" />
|
||||
<el-table-column label="评测模型" prop="model_name" align="center" />
|
||||
<el-table-column label="评测模型" align="center" min-width="160">
|
||||
<template #default="{ row }">
|
||||
<el-tooltip :content="displayModelName(row)" placement="top" :disabled="displayModelName(row).length < 18">
|
||||
<span class="cell-ellipsis">{{ displayModelName(row) }}</span>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="数据集" prop="dataset" align="center" />
|
||||
<el-table-column label="指标" prop="metric" align="center" />
|
||||
<el-table-column label="指标" align="center" min-width="220">
|
||||
<template #default="{ row }">
|
||||
<el-tooltip :content="displayMetric(row)" placement="top" :disabled="displayMetric(row).length < 24">
|
||||
<span class="cell-ellipsis">{{ displayMetric(row) }}</span>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="评分" prop="score" width="100" align="center" />
|
||||
<el-table-column label="状态" width="100" align="center">
|
||||
<template #default="{ row }">
|
||||
@@ -138,7 +179,7 @@ onMounted(() => {
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
/* 胶囊切换栏样式 */
|
||||
/* 胶囊切换栏 */
|
||||
.capsule-tabs {
|
||||
display: flex;
|
||||
background: #f1f5f9;
|
||||
@@ -172,4 +213,13 @@ onMounted(() => {
|
||||
}
|
||||
}
|
||||
|
||||
.cell-ellipsis {
|
||||
display: inline-block;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
vertical-align: middle;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
@@ -56,9 +56,9 @@ defineExpose({ validate })
|
||||
</el-form-item>
|
||||
<el-form-item v-if="form.rouge_enabled" label="ROUGE methods">
|
||||
<el-checkbox-group v-model="form.rouge_methods">
|
||||
<el-checkbox value="rouge_1">ROUGE-1</el-checkbox>
|
||||
<el-checkbox value="rouge_2">ROUGE-2</el-checkbox>
|
||||
<el-checkbox value="rouge_l">ROUGE-L</el-checkbox>
|
||||
<el-checkbox value="rouge1">ROUGE-1</el-checkbox>
|
||||
<el-checkbox value="rouge2">ROUGE-2</el-checkbox>
|
||||
<el-checkbox value="rougeL">ROUGE-L</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</el-form-item>
|
||||
|
||||
|
||||
@@ -103,10 +103,10 @@ defineExpose({ validate })
|
||||
<el-form-item label="选择 GPU" prop="gpu_id">
|
||||
<el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
|
||||
<el-option
|
||||
v-for="(gpu, index) in gpus"
|
||||
:key="index"
|
||||
:label="`${gpu.name} (GPU ${index})`"
|
||||
:value="index"
|
||||
v-for="gpu in gpus"
|
||||
:key="`${gpu.node_id || ''}:${gpu.id ?? 0}`"
|
||||
:label="`${gpu.node_name || gpu.node_code || '算力节点'} / ${gpu.name} (GPU ${gpu.id ?? 0})`"
|
||||
:value="`${gpu.node_id || ''}:${gpu.id ?? 0}`"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import type { DatasetItem, GpuInfo, ModelItem, TrainedModel } from '@/types'
|
||||
import type { BasicMetricSetupDraft } from './BasicMetricSetupStep.vue'
|
||||
import type { EvalRuleSetupDraft } from './EvalRuleSetupStep.vue'
|
||||
@@ -17,6 +18,15 @@ const props = defineProps<{
|
||||
function nameOf<T extends { id: string | number; name?: string }>(items: T[], id: string | number) {
|
||||
return items.find((item) => item.id === id)?.name || String(id || '-')
|
||||
}
|
||||
|
||||
/** GPU 选择为「节点:GPU序号」复合值,解析并展示为可读标签 */
|
||||
const gpuLabel = computed(() => {
|
||||
const key = String(props.task.gpu_id || '')
|
||||
const gpu = props.gpus.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === key)
|
||||
if (gpu) return `${gpu.node_name || gpu.node_code || '算力节点'} / GPU ${gpu.id ?? 0}`
|
||||
const [nodeId, idx] = key.split(':')
|
||||
return nodeId ? `节点 ${nodeId} / GPU ${idx || 0}` : `GPU ${key || 0}`
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -24,7 +34,7 @@ function nameOf<T extends { id: string | number; name?: string }>(items: T[], id
|
||||
<el-descriptions :column="2" border>
|
||||
<el-descriptions-item label="Task">{{ props.task.eval_task_name || '-' }}</el-descriptions-item>
|
||||
<el-descriptions-item label="Model">{{ nameOf(props.trainedModels, props.task.model_id) }}</el-descriptions-item>
|
||||
<el-descriptions-item label="GPU">GPU {{ props.task.gpu_id || 0 }}</el-descriptions-item>
|
||||
<el-descriptions-item label="GPU">{{ gpuLabel }}</el-descriptions-item>
|
||||
<el-descriptions-item label="Dataset">
|
||||
{{ props.task.data_source === 'dataset' ? nameOf(props.evalDatasets, props.task.dataset_id) : 'Inference results' }}
|
||||
</el-descriptions-item>
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { getModelList } from '@/api/modules/model'
|
||||
import { getDatasetList } from '@/api/modules/dataset'
|
||||
import { getSystemInfo } from '@/api/modules/system'
|
||||
import { getComputeNodes } from '@/api/modules/compute'
|
||||
import { TEMPLATE_GROUPS, LR_SCHEDULER_OPTIONS, QUANTIZATION_BIT_OPTIONS, QUANT_METHOD_OPTIONS, GGUF_FORMAT_OPTIONS } from '@/constants'
|
||||
import {
|
||||
DEFAULT_TRAINING_PARAMS,
|
||||
@@ -35,7 +36,18 @@ const preflightResult = ref<FineTunePreflightResult | null>(null)
|
||||
const models = ref<ModelItem[]>([])
|
||||
const datasets = ref<DatasetItem[]>([])
|
||||
const gpus = ref<GpuInfo[]>([])
|
||||
const selectedGpus = ref<number[]>([])
|
||||
const computeNodes = ref<Array<{ id: string; scheduler_status?: string }>>([])
|
||||
const selectedGpuKeys = ref<string[]>([])
|
||||
|
||||
/** Only show GPUs from nodes that are online or draining */
|
||||
const availableGpus = computed(() => {
|
||||
const onlineNodeIds = new Set(
|
||||
computeNodes.value
|
||||
.filter((n) => n.scheduler_status === 'online' || n.scheduler_status === 'draining')
|
||||
.map((n) => n.id),
|
||||
)
|
||||
return gpus.value.filter((gpu) => !gpu.node_id || onlineNodeIds.has(gpu.node_id))
|
||||
})
|
||||
const modelDialogVisible = ref(false)
|
||||
|
||||
const form = reactive(createDefaultFineTuneForm())
|
||||
@@ -62,7 +74,14 @@ const selectedModel = computed(() => models.value.find((model) => model.id === f
|
||||
const modelDialogTitle = computed(() => selectedModel.value?.name || '')
|
||||
|
||||
/** 训练命令与提交载荷共用同一份表单模型。 */
|
||||
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpus.value))
|
||||
const selectedGpus = computed(() =>
|
||||
selectedGpuKeys.value
|
||||
.map((key) => availableGpus.value.find((gpu) => gpuKey(gpu) === key))
|
||||
.filter((gpu): gpu is GpuInfo => Boolean(gpu)),
|
||||
)
|
||||
const selectedComputeNodeId = computed(() => selectedGpus.value[0]?.node_id)
|
||||
const selectedGpuIds = computed(() => selectedGpus.value.map((gpu) => Number(gpu.id)))
|
||||
const commandPreview = computed(() => buildFineTuneCommand(form, selectedGpuIds.value))
|
||||
|
||||
const remoteCommandPreview = computed(() => {
|
||||
const command = preflightResult.value?.preview?.command
|
||||
@@ -70,11 +89,32 @@ const remoteCommandPreview = computed(() => {
|
||||
return preflightResult.value?.preview?.command_text || ''
|
||||
})
|
||||
|
||||
/** GPU 多选切换 */
|
||||
function toggleGpu(index: number) {
|
||||
const idx = selectedGpus.value.indexOf(index)
|
||||
if (idx === -1) selectedGpus.value.push(index)
|
||||
else selectedGpus.value.splice(idx, 1)
|
||||
function gpuKey(gpu: GpuInfo) {
|
||||
return `${gpu.node_id || 'local'}:${gpu.id ?? gpu.uuid ?? gpu.name}`
|
||||
}
|
||||
|
||||
function isGpuUnavailable(gpu: GpuInfo) {
|
||||
return gpu.status === 'busy' || gpu.status === 'reserved' || gpu.status === 'offline'
|
||||
}
|
||||
|
||||
function isGpuSelected(gpu: GpuInfo) {
|
||||
return selectedGpuKeys.value.includes(gpuKey(gpu))
|
||||
}
|
||||
|
||||
/** GPU 多选切换:单个任务只允许选择同一算力节点内的空闲卡。 */
|
||||
function toggleGpu(gpu: GpuInfo) {
|
||||
if (isGpuUnavailable(gpu) || gpu.id == null) return
|
||||
const key = gpuKey(gpu)
|
||||
if (isGpuSelected(gpu)) {
|
||||
selectedGpuKeys.value = selectedGpuKeys.value.filter((item) => item !== key)
|
||||
return
|
||||
}
|
||||
if (selectedComputeNodeId.value && gpu.node_id && selectedComputeNodeId.value !== gpu.node_id) {
|
||||
selectedGpuKeys.value = [key]
|
||||
ElMessage.info('已切换到新的算力节点,之前选择的 GPU 已清空')
|
||||
return
|
||||
}
|
||||
selectedGpuKeys.value = [...selectedGpuKeys.value, key]
|
||||
}
|
||||
|
||||
function gpuUsageWidth(percent: number) {
|
||||
@@ -164,10 +204,11 @@ async function loadDatasets() {
|
||||
|
||||
async function loadGpus() {
|
||||
try {
|
||||
const sys = await getSystemInfo()
|
||||
const [sys, nodes] = await Promise.all([getSystemInfo(), getComputeNodes().catch(() => [])])
|
||||
gpus.value = sys?.gpu || []
|
||||
// 默认选中第一个
|
||||
if (gpus.value.length > 0) selectedGpus.value = [0]
|
||||
computeNodes.value = nodes || []
|
||||
const firstIdle = availableGpus.value.find((gpu) => !isGpuUnavailable(gpu) && gpu.id != null)
|
||||
if (firstIdle) selectedGpuKeys.value = [gpuKey(firstIdle)]
|
||||
} catch {
|
||||
gpus.value = []
|
||||
}
|
||||
@@ -177,8 +218,8 @@ async function handleSubmit() {
|
||||
if (!formRef.value) return
|
||||
await formRef.value.validate(async (valid) => {
|
||||
if (!valid) return
|
||||
if (selectedGpus.value.length === 0) {
|
||||
ElMessage.warning('请至少选择一个 GPU')
|
||||
if (!selectedGpuIds.value.length) {
|
||||
ElMessage.warning('请至少选择一张空闲 GPU')
|
||||
return
|
||||
}
|
||||
submitting.value = true
|
||||
@@ -195,7 +236,7 @@ async function handleSubmit() {
|
||||
return
|
||||
}
|
||||
|
||||
const payload = buildFineTunePayload(form, selectedGpus.value)
|
||||
const payload = buildFineTunePayload(form, selectedGpuIds.value, selectedComputeNodeId.value)
|
||||
const preflight = await runPreflight(payload)
|
||||
if (!preflight?.valid) {
|
||||
ElMessage.error('训练预检未通过,请先处理预检问题')
|
||||
@@ -220,7 +261,7 @@ async function handleSubmit() {
|
||||
})
|
||||
}
|
||||
|
||||
async function runPreflight(payload = buildFineTunePayload(form, selectedGpus.value)) {
|
||||
async function runPreflight(payload = buildFineTunePayload(form, selectedGpuIds.value, selectedComputeNodeId.value)) {
|
||||
preflightLoading.value = true
|
||||
try {
|
||||
const result = await preflightFineTune(payload)
|
||||
@@ -249,8 +290,8 @@ async function handlePreflightClick() {
|
||||
if (!formRef.value) return
|
||||
await formRef.value.validate(async (valid) => {
|
||||
if (!valid) return
|
||||
if (selectedGpus.value.length === 0) {
|
||||
ElMessage.warning('请至少选择一个 GPU')
|
||||
if (!selectedGpuIds.value.length) {
|
||||
ElMessage.warning('请至少选择一张空闲 GPU')
|
||||
return
|
||||
}
|
||||
await runPreflight()
|
||||
@@ -285,19 +326,26 @@ onMounted(() => {
|
||||
<el-divider content-position="left">训练配置</el-divider>
|
||||
<el-form-item label="GPU 硬件">
|
||||
<div class="gpu-list">
|
||||
<div class="gpu-selection-summary">
|
||||
已选择 {{ selectedGpuIds.length }} 张 GPU
|
||||
<template v-if="selectedGpus[0]?.node_code"> · {{ selectedGpus[0].node_code }}</template>
|
||||
</div>
|
||||
<div
|
||||
v-for="(gpu, idx) in gpus"
|
||||
:key="idx"
|
||||
v-for="gpu in availableGpus"
|
||||
:key="gpuKey(gpu)"
|
||||
class="gpu-card"
|
||||
:class="{ active: selectedGpus.includes(idx), 'is-busy': gpu.gpu_percent > 80 }"
|
||||
@click="toggleGpu(idx)"
|
||||
:class="{ active: isGpuSelected(gpu), 'is-busy': isGpuUnavailable(gpu), 'is-disabled': isGpuUnavailable(gpu) }"
|
||||
@click="toggleGpu(gpu)"
|
||||
>
|
||||
<div class="gpu-card-top">
|
||||
<div class="gpu-title">
|
||||
<span class="gpu-index">GPU-{{ idx }}</span>
|
||||
<span class="gpu-index">
|
||||
GPU-{{ gpu.id }}
|
||||
<template v-if="gpu.node_code"> · {{ gpu.node_code }}</template>
|
||||
</span>
|
||||
<span class="gpu-name">{{ gpu.name }}</span>
|
||||
</div>
|
||||
<span class="gpu-usage">{{ gpu.gpu_percent }}%</span>
|
||||
<span class="gpu-usage">{{ isGpuUnavailable(gpu) ? gpu.status : `${gpu.gpu_percent}%` }}</span>
|
||||
</div>
|
||||
<div class="gpu-usage-bar">
|
||||
<span :style="{ width: gpuUsageWidth(gpu.gpu_percent) }" />
|
||||
@@ -308,7 +356,7 @@ onMounted(() => {
|
||||
<span>{{ gpu.power_w }}W</span>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="!gpus.length" class="gpu-empty">暂无 GPU 信息</div>
|
||||
<div v-if="!availableGpus.length" class="gpu-empty">暂无可用 GPU(请检查算力节点是否在线)</div>
|
||||
</div>
|
||||
</el-form-item>
|
||||
|
||||
@@ -563,6 +611,13 @@ onMounted(() => {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.gpu-selection-summary {
|
||||
grid-column: 1 / -1;
|
||||
color: #64748b;
|
||||
font-size: 12px;
|
||||
line-height: 20px;
|
||||
}
|
||||
|
||||
.gpu-card {
|
||||
border: 1px solid #e5e7eb;
|
||||
border-radius: 6px;
|
||||
@@ -612,6 +667,11 @@ onMounted(() => {
|
||||
background: #dc2626;
|
||||
}
|
||||
}
|
||||
|
||||
&.is-disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.72;
|
||||
}
|
||||
}
|
||||
|
||||
.gpu-card-top {
|
||||
|
||||
@@ -65,6 +65,7 @@ export function createDefaultFineTuneForm(): FineTuneFormModel {
|
||||
export function buildFineTunePayload(
|
||||
form: FineTuneFormModel,
|
||||
gpus: number[],
|
||||
computeNodeId?: string,
|
||||
): Omit<FineTuneStartPayload, 'task_id'> {
|
||||
return {
|
||||
name: form.name,
|
||||
@@ -77,6 +78,7 @@ export function buildFineTunePayload(
|
||||
train_dataset_id: form.train_dataset_id,
|
||||
auto_merge: form.train_type === 'SFT' && form.auto_merge,
|
||||
output_model_name: form.name,
|
||||
compute_node_id: computeNodeId,
|
||||
batch_size: form.batch_size,
|
||||
learning_rate: form.learning_rate,
|
||||
n_epochs: form.n_epochs,
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, nextTick, onMounted, watch } from 'vue'
|
||||
import { ref, reactive, nextTick, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import MarkdownView from '@/components/MarkdownView.vue'
|
||||
import { useStreamChat } from '@/composables/useStreamChat'
|
||||
import { getCompare } from '@/api/modules/compare'
|
||||
import { getCompare, getLoadStatus } from '@/api/modules/compare'
|
||||
import type { CompareTask, LoadedModel } from '@/types'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const taskId = route.params.id as string
|
||||
/** 是否为 mock 直通模式(新建推理假数据进入,不走真实任务接口) */
|
||||
const isMock = taskId === 'mock'
|
||||
/** 是否为 mock 模式(新建推理无真实 taskId 或明确为 mock 时进入 mock 模式) */
|
||||
const isMock = taskId === 'mock' || !taskId || taskId === 'unknown'
|
||||
/** 当前对话使用的模型名 */
|
||||
const modelName = ref(route.query.model as string || '')
|
||||
|
||||
@@ -37,6 +37,10 @@ const contentRef = ref<HTMLElement>()
|
||||
let activeAssistant: ChatMessage | null = null
|
||||
/** 设置面板抽屉 */
|
||||
const showSettings = ref(false)
|
||||
/** 模型仍在加载中(直接 URL 进入 chat 时兜底轮询就绪状态) */
|
||||
const taskLoading = ref(false)
|
||||
const taskError = ref('')
|
||||
let statusTimer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
/** 获取任务信息,定位已启动的模型(mock 模式跳过) */
|
||||
async function loadTask() {
|
||||
@@ -45,6 +49,14 @@ async function loadTask() {
|
||||
task.value = await getCompare(taskId)
|
||||
const models = parseLoadedModels(task.value)
|
||||
if (models[0]?.model_name) modelName.value = models[0].model_name
|
||||
// 恢复本地保存的历史对话
|
||||
restoreHistory()
|
||||
// 模型仍在上次加载中:启动轮询等待就绪
|
||||
if (models.some((m) => m.status === 'starting')) {
|
||||
taskLoading.value = true
|
||||
await pollTaskStatus()
|
||||
statusTimer = setInterval(pollTaskStatus, 3000)
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
@@ -60,6 +72,80 @@ function parseLoadedModels(t: CompareTask | null): LoadedModel[] {
|
||||
}
|
||||
}
|
||||
|
||||
/** 对话历史本地持久化(按任务 id 存储,退出重进可恢复) */
|
||||
const STORAGE_PREFIX = 'ygft_chat_history_'
|
||||
|
||||
function historyKey(id: string | number): string {
|
||||
return `${STORAGE_PREFIX}${id}`
|
||||
}
|
||||
|
||||
function saveHistory() {
|
||||
if (isMock) return
|
||||
try {
|
||||
const snapshot = messages.value.map((m) => ({
|
||||
role: m.role,
|
||||
content: m.content,
|
||||
think: m.think,
|
||||
done: true,
|
||||
}))
|
||||
localStorage.setItem(historyKey(taskId), JSON.stringify(snapshot))
|
||||
} catch {
|
||||
// 存储失败忽略
|
||||
}
|
||||
}
|
||||
|
||||
function restoreHistory() {
|
||||
if (isMock) return
|
||||
try {
|
||||
const raw = localStorage.getItem(historyKey(taskId))
|
||||
if (!raw) return
|
||||
const parsed = JSON.parse(raw)
|
||||
if (Array.isArray(parsed)) {
|
||||
messages.value = parsed.map((m) => ({
|
||||
role: m.role === 'user' ? 'user' : 'assistant',
|
||||
content: m.content || '',
|
||||
think: m.think || '',
|
||||
isThinking: false,
|
||||
isStreaming: false,
|
||||
done: true,
|
||||
}))
|
||||
}
|
||||
} catch {
|
||||
// 恢复失败忽略
|
||||
}
|
||||
}
|
||||
|
||||
/** 停止就绪状态轮询 */
|
||||
function stopStatusPolling() {
|
||||
if (statusTimer) {
|
||||
clearInterval(statusTimer)
|
||||
statusTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
/** 轮询任务加载状态:starting → ready/error */
|
||||
async function pollTaskStatus() {
|
||||
try {
|
||||
const st = await getLoadStatus(taskId)
|
||||
const items = st.loaded_models || []
|
||||
const anyReady = items.some((m) => m.status === 'ready' || m.status === 'running')
|
||||
const anyError = items.some((m) => m.status === 'error')
|
||||
if (anyReady) {
|
||||
taskLoading.value = false
|
||||
taskError.value = ''
|
||||
stopStatusPolling()
|
||||
} else if (anyError) {
|
||||
taskLoading.value = false
|
||||
taskError.value = items.find((m) => m.status === 'error')?.error || '模型加载失败'
|
||||
stopStatusPolling()
|
||||
} else {
|
||||
taskLoading.value = true
|
||||
}
|
||||
} catch {
|
||||
// 轮询失败忽略,下次再试
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSend() {
|
||||
const question = inputQuestion.value.trim()
|
||||
if (!question || loading.value) return
|
||||
@@ -76,6 +162,7 @@ async function handleSend() {
|
||||
done: false,
|
||||
})
|
||||
messages.value.push(assistantMsg)
|
||||
saveHistory()
|
||||
|
||||
inputQuestion.value = ''
|
||||
await nextTick()
|
||||
@@ -85,33 +172,25 @@ async function handleSend() {
|
||||
// mock 模式:直接用假数据逐字填充
|
||||
if (isMock) {
|
||||
await mockReply(assistantMsg, question)
|
||||
saveHistory()
|
||||
return
|
||||
}
|
||||
|
||||
// 真实模式:获取已启动模型的端口/路径
|
||||
const models = parseLoadedModels(task.value)
|
||||
const target = models[0]
|
||||
if (!target) {
|
||||
ElMessage.error('未找到已启动的模型')
|
||||
assistantMsg.content = '未找到已启动的模型,请先返回列表加载模型'
|
||||
assistantMsg.done = true
|
||||
assistantMsg.isStreaming = false
|
||||
return
|
||||
}
|
||||
|
||||
// 流式状态变化时只同步当前回复,避免固定定时器空转。
|
||||
// 真实模式:通过后端 SSE 流式代理到算力节点进行推理
|
||||
activeAssistant = assistantMsg
|
||||
|
||||
await send({
|
||||
port: target.port,
|
||||
model_name: target.model_name,
|
||||
model_path: '',
|
||||
system_prompt: systemPrompt.value,
|
||||
user_question: question,
|
||||
temperature: temperature.value,
|
||||
top_p: top_p.value,
|
||||
max_tokens: maxTokens.value,
|
||||
})
|
||||
await send(
|
||||
{
|
||||
model_path: route.query.model_path as string || '',
|
||||
task_id: taskId,
|
||||
system_prompt: systemPrompt.value,
|
||||
user_question: question,
|
||||
temperature: temperature.value,
|
||||
top_p: top_p.value,
|
||||
max_tokens: maxTokens.value,
|
||||
},
|
||||
{ useMock: false },
|
||||
)
|
||||
|
||||
// 完成后同步最终内容
|
||||
assistantMsg.content = message.value.displayContent || message.value.error || '(无回复)'
|
||||
@@ -121,6 +200,7 @@ async function handleSend() {
|
||||
assistantMsg.done = true
|
||||
activeAssistant = null
|
||||
reset()
|
||||
saveHistory()
|
||||
await nextTick()
|
||||
scrollToBottom()
|
||||
}
|
||||
@@ -179,6 +259,11 @@ function handleNewChat() {
|
||||
activeAssistant = null
|
||||
messages.value = []
|
||||
reset()
|
||||
try {
|
||||
localStorage.removeItem(historyKey(taskId))
|
||||
} catch {
|
||||
// 忽略
|
||||
}
|
||||
}
|
||||
|
||||
/** 输入框自适应高度 */
|
||||
@@ -195,6 +280,7 @@ function resetInputHeight() {
|
||||
}
|
||||
|
||||
onMounted(loadTask)
|
||||
onUnmounted(stopStatusPolling)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -262,6 +348,12 @@ onMounted(loadTask)
|
||||
|
||||
<!-- 输入栏 -->
|
||||
<footer class="chat-input-container">
|
||||
<div v-if="taskLoading" class="loading-hint">
|
||||
<i class="fa fa-spinner fa-spin" style="margin-right: 6px" />模型加载中,就绪后即可对话...
|
||||
</div>
|
||||
<div v-else-if="taskError" class="loading-hint error">
|
||||
<i class="fa fa-exclamation-circle" style="margin-right: 6px" />{{ taskError }}
|
||||
</div>
|
||||
<div class="chat-input-inner">
|
||||
<button class="clear-btn" title="清空对话" @click="handleNewChat">
|
||||
<i class="fa fa-eraser" />
|
||||
@@ -271,22 +363,21 @@ onMounted(loadTask)
|
||||
v-model="inputQuestion"
|
||||
class="input-box"
|
||||
rows="1"
|
||||
:disabled="loading"
|
||||
:disabled="loading || taskLoading"
|
||||
placeholder="给模型发送消息..."
|
||||
@keydown.enter.exact.prevent="handleSend"
|
||||
@input="autoResize"
|
||||
/>
|
||||
<button
|
||||
class="send-btn"
|
||||
:class="{ active: inputQuestion.trim() && !loading }"
|
||||
:disabled="!inputQuestion.trim() || loading"
|
||||
:class="{ active: inputQuestion.trim() && !loading && !taskLoading }"
|
||||
:disabled="!inputQuestion.trim() || loading || taskLoading"
|
||||
@click="handleSend"
|
||||
>
|
||||
<i class="fa fa-arrow-up" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer-hint">内容由 AI 生成,请仔细甄别。</div>
|
||||
</footer>
|
||||
|
||||
<!-- 设置抽屉(系统提示词等) -->
|
||||
@@ -713,9 +804,18 @@ onMounted(loadTask)
|
||||
}
|
||||
}
|
||||
|
||||
.footer-hint {
|
||||
margin-top: 12px;
|
||||
font-size: 12px;
|
||||
color: #9ca3af;
|
||||
.loading-hint {
|
||||
margin-bottom: 10px;
|
||||
padding: 6px 14px;
|
||||
font-size: 13px;
|
||||
color: #b45309;
|
||||
background: #fef3c7;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
|
||||
&.error {
|
||||
color: #b91c1c;
|
||||
background: #fee2e2;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user