Compare commits
5 Commits
main
...
f04dc479bb
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f04dc479bb | ||
|
|
f453234057 | ||
|
|
a6868ec2e5 | ||
|
|
6cd1e46e86 | ||
|
|
836343b29e |
11
.gitignore
vendored
11
.gitignore
vendored
@@ -187,3 +187,14 @@ cython_debug/
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
docker/llamafactory-latest.tar.gz
|
||||
# Compute data - 保留目录结构和 README,忽略子目录内容(日志、模型、数据集等)
|
||||
!docker/compute/data/yg-ft/logs/
|
||||
docker/compute/data/yg-ft/datasets/*
|
||||
docker/compute/data/yg-ft/models/*
|
||||
docker/compute/data/yg-ft/outputs/*
|
||||
docker/compute/data/yg-ft/logs/**
|
||||
!docker/compute/data/yg-ft/logs/compute/
|
||||
!docker/compute/data/yg-ft/logs/training/
|
||||
!docker/compute/data/yg-ft/**/.gitkeep
|
||||
!docker/compute/data/yg-ft/**/README.md
|
||||
|
||||
36
README.md
36
README.md
@@ -48,6 +48,42 @@ YG_FT/
|
||||
- 前端新增 `/compute` 算力节点页面,展示节点地址、权重、标签、启用状态、GPU、队列和资源副本。
|
||||
- `compute/engines/llama_factory/adapter.py` 提供 LLaMA-Factory 参数校验、命令生成和日志解析基础能力。
|
||||
|
||||
## 前后端一键启动
|
||||
|
||||
首次使用前,请先按下方“后端启动”和“前端启动”说明安装依赖,并确保
|
||||
PostgreSQL 已可用。之后在项目根目录执行:
|
||||
|
||||
```bash
|
||||
bash ./start.sh
|
||||
```
|
||||
|
||||
脚本会同时启动前端 `http://localhost:16801` 和后端
|
||||
`http://127.0.0.1:17861`,按 `Ctrl+C` 会同时停止两个服务。脚本只负责
|
||||
启动前后端,不会自动安装依赖,也不会启动 PostgreSQL、Redis 或算力服务。
|
||||
|
||||
仅检查依赖和端口而不启动服务:
|
||||
|
||||
```bash
|
||||
bash ./start.sh --check
|
||||
```
|
||||
|
||||
本地启动推荐只配置数据库主机。脚本会复用 `docker/app/.env` 中已有的
|
||||
`POSTGRES_USER`、`POSTGRES_PASSWORD` 和 `POSTGRES_DB`,端口默认使用
|
||||
PostgreSQL 标准端口 `5432`:
|
||||
|
||||
```bash
|
||||
DATABASE_HOST='www.caoxiaozhu.com' bash ./start.sh
|
||||
```
|
||||
|
||||
也可以在 `docker/app/.env` 中增加:
|
||||
|
||||
```env
|
||||
DATABASE_HOST=www.caoxiaozhu.com
|
||||
```
|
||||
|
||||
需要使用非标准端口时再设置 `DATABASE_PORT`。`DATABASE_URL` 仍可作为完整连接串
|
||||
高级覆盖项;终端环境变量优先级最高。脚本不会输出数据库密码。
|
||||
|
||||
## 后端启动
|
||||
|
||||
```bash
|
||||
|
||||
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
1008
backend/app/api/v1/endpoints/data_process.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
import uuid
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import PlainTextResponse
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -19,6 +24,77 @@ def fail(status_code: int, message: str) -> HTTPException:
|
||||
return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None})
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
def _task_for_compute_job(job_id: str) -> dict[str, Any] | None:
|
||||
return next((task for task in get_platform_store().tasks() if task.get("compute_job_id") == job_id), None)
|
||||
|
||||
|
||||
async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
||||
if task_id and get_settings().compute_mode != "simulator":
|
||||
try:
|
||||
preflight = await _fine_tune_preflight(store, task_id, payload, validate=True)
|
||||
except Exception as exc: # noqa: BLE001 - task has not entered running state yet
|
||||
raise RuntimeError(f"preflight failed: {exc}") from exc
|
||||
if not preflight["valid"]:
|
||||
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
|
||||
raise RuntimeError(f"preflight failed: {errors}")
|
||||
task = store.start_task(payload)
|
||||
if get_settings().compute_mode == "simulator":
|
||||
return task
|
||||
node, job_payload = store.build_compute_job_payload(task["id"])
|
||||
job = await ComputeNodeClient(node["api_base_url"]).create_job(job_payload)
|
||||
return store.apply_compute_job(task["id"], job)
|
||||
|
||||
|
||||
async def _fine_tune_preflight(
|
||||
store: Any,
|
||||
task_id: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
validate: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
node, job_payload = store.prepare_compute_job_payload(task_id, payload or {})
|
||||
if get_settings().compute_mode == "simulator":
|
||||
preview = {
|
||||
"valid": True,
|
||||
"errors": [],
|
||||
"warnings": ["compute_mode=simulator skips remote compute validation"],
|
||||
"engine": job_payload.get("engine") or job_payload.get("training_engine") or "llama_factory",
|
||||
"command": [],
|
||||
"command_text": "",
|
||||
"work_dir": "",
|
||||
"env": {},
|
||||
"path_checks": [],
|
||||
}
|
||||
else:
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
preview = await (client.validate_job(job_payload) if validate else client.preview_job(job_payload))
|
||||
errors = list(preview.get("errors") or [])
|
||||
warnings = list(preview.get("warnings") or [])
|
||||
if not node.get("enabled"):
|
||||
errors.append(f"compute node disabled: {node.get('code')}")
|
||||
if node.get("scheduler_status") not in {"online", "draining"}:
|
||||
errors.append(f"compute node not schedulable: {node.get('code')} status={node.get('scheduler_status')}")
|
||||
return {
|
||||
"valid": bool(preview.get("valid", not errors)) and not errors,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
"node": {
|
||||
"id": node.get("id"),
|
||||
"code": node.get("code"),
|
||||
"name": node.get("name"),
|
||||
"api_base_url": node.get("api_base_url"),
|
||||
"scheduler_status": node.get("scheduler_status"),
|
||||
"gpu_count": node.get("gpu_count"),
|
||||
},
|
||||
"job_payload": job_payload,
|
||||
"preview": preview,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
user = get_platform_store().login(payload.get("username", ""), payload.get("password", ""))
|
||||
@@ -84,7 +160,29 @@ async def delete_user(user_id: str, current_username: str | None = Query(default
|
||||
|
||||
@router.get("/model-manage/local-models")
|
||||
async def local_models() -> dict[str, Any]:
|
||||
models = [{"path": item.get("path") or "", "name": item["name"]} for item in get_platform_store().models()]
|
||||
store = get_platform_store()
|
||||
models = [{"path": item.get("path") or "", "name": item["name"], "source": "registered"} for item in store.models()]
|
||||
seen = {item["path"] for item in models if item.get("path")}
|
||||
if get_settings().compute_mode != "simulator":
|
||||
for node in store.compute_nodes():
|
||||
if not node.get("enabled"):
|
||||
continue
|
||||
try:
|
||||
result = await ComputeNodeClient(node["api_base_url"]).list_files(root="models", directories_only=True)
|
||||
except Exception:
|
||||
continue
|
||||
for item in result.get("items") or []:
|
||||
path = str(item.get("path") or "")
|
||||
if not path or path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
models.append(
|
||||
{
|
||||
"path": path,
|
||||
"name": item.get("name") or path.rsplit("/", 1)[-1],
|
||||
"source": f"compute:{node.get('code')}",
|
||||
}
|
||||
)
|
||||
return ok({"models": models})
|
||||
|
||||
|
||||
@@ -95,6 +193,7 @@ async def trained_models() -> dict[str, Any]:
|
||||
|
||||
@router.delete("/model-manage/trained-models/{model_id}")
|
||||
async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]:
|
||||
get_platform_store().delete_trained_model(model_id)
|
||||
return ok({"deleted": model_id, "type": type})
|
||||
|
||||
|
||||
@@ -113,7 +212,14 @@ async def model_list() -> dict[str, Any]:
|
||||
|
||||
@router.post("/model-manage")
|
||||
async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().create_model(payload))
|
||||
except KeyError as exc:
|
||||
raise fail(400, f"missing field: {exc}")
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
except Exception as exc: # noqa: BLE001 - keep API errors visible to deployment smoke checks
|
||||
raise fail(500, f"create model failed: {exc}")
|
||||
|
||||
|
||||
@router.get("/model-manage/{model_id}")
|
||||
@@ -199,12 +305,73 @@ async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(
|
||||
|
||||
@router.delete("/dataset-manage/versions/{file_id}/{version_id}")
|
||||
async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]:
|
||||
return ok(get_platform_store().file_versions(file_id))
|
||||
try:
|
||||
return ok(get_platform_store().delete_file_version(file_id, version_id))
|
||||
except KeyError:
|
||||
raise fail(404, "dataset version not found")
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
|
||||
|
||||
async def _sync_dataset_file_to_compute_nodes(
|
||||
store: Any,
|
||||
dataset_id: str,
|
||||
file_id: str,
|
||||
filename: str,
|
||||
content: bytes,
|
||||
) -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
if get_settings().compute_mode == "simulator":
|
||||
return results
|
||||
target_name = Path(filename or f"{file_id}.jsonl").name
|
||||
target_relative_path = f"datasets/{dataset_id}/{target_name}"
|
||||
for node in store.compute_nodes():
|
||||
if not node.get("enabled"):
|
||||
continue
|
||||
try:
|
||||
result = await ComputeNodeClient(node["api_base_url"]).upload_file(
|
||||
target_name,
|
||||
content,
|
||||
target_relative_path,
|
||||
resource_type="dataset",
|
||||
resource_id=dataset_id,
|
||||
)
|
||||
store.upsert_resource_replica(
|
||||
node["id"],
|
||||
"dataset",
|
||||
dataset_id,
|
||||
str(result.get("local_path") or ""),
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"node_id": node["id"],
|
||||
"node_code": node.get("code"),
|
||||
"success": True,
|
||||
"local_path": result.get("local_path"),
|
||||
"byte_size": result.get("byte_size"),
|
||||
"checksum_sha256": result.get("checksum_sha256"),
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - keep upload usable while exposing sync failures
|
||||
results.append(
|
||||
{
|
||||
"node_id": node["id"],
|
||||
"node_code": node.get("code"),
|
||||
"success": False,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
@router.post("/dataset-manage/upload/{dataset_id}")
|
||||
async def upload_dataset_files(dataset_id: str, files: list[UploadFile] = File(default=[])) -> dict[str, Any]:
|
||||
async def upload_dataset_files(
|
||||
dataset_id: str,
|
||||
files: list[UploadFile] = File(default=[]),
|
||||
sync_to_compute: bool = Query(default=True),
|
||||
) -> dict[str, Any]:
|
||||
created: list[dict[str, Any]] = []
|
||||
compute_sync: list[dict[str, Any]] = []
|
||||
store = get_platform_store()
|
||||
try:
|
||||
store.dataset(dataset_id)
|
||||
@@ -214,8 +381,19 @@ async def upload_dataset_files(dataset_id: str, files: list[UploadFile] = File(d
|
||||
for file in files:
|
||||
raw = await file.read()
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
created.append(store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content))
|
||||
return ok({"files": created})
|
||||
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
|
||||
created.append(created_file)
|
||||
if sync_to_compute:
|
||||
compute_sync.extend(
|
||||
await _sync_dataset_file_to_compute_nodes(
|
||||
store,
|
||||
dataset_id,
|
||||
created_file["id"],
|
||||
created_file["name"],
|
||||
raw,
|
||||
)
|
||||
)
|
||||
return ok({"files": created, "compute_sync": compute_sync})
|
||||
|
||||
|
||||
@router.get("/dataset-manage/download/{dataset_id}")
|
||||
@@ -299,12 +477,45 @@ async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any
|
||||
|
||||
@router.post("/fine-tune/start")
|
||||
async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
return ok(get_platform_store().start_task(payload))
|
||||
return ok(await _submit_fine_tune_task(store, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
except RuntimeError as exc:
|
||||
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
||||
if task_id:
|
||||
store.mark_task_failed(task_id, str(exc))
|
||||
raise fail(409, str(exc))
|
||||
except Exception as exc: # noqa: BLE001 - mark task failed when remote submit fails
|
||||
task_id = str(payload.get("task_id") or payload.get("id") or "")
|
||||
if task_id:
|
||||
store.mark_task_failed(task_id, str(exc))
|
||||
raise fail(502, f"submit compute job failed: {exc}")
|
||||
|
||||
|
||||
@router.post("/fine-tune/{task_id}/preflight")
|
||||
async def fine_tune_preflight(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=True))
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
except RuntimeError as exc:
|
||||
raise fail(409, str(exc))
|
||||
except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page
|
||||
raise fail(502, f"compute preflight failed: {exc}")
|
||||
|
||||
|
||||
@router.post("/fine-tune/{task_id}/command-preview")
|
||||
async def fine_tune_command_preview(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(await _fine_tune_preflight(get_platform_store(), task_id, payload or {}, validate=False))
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
except RuntimeError as exc:
|
||||
raise fail(409, str(exc))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise fail(502, f"compute command preview failed: {exc}")
|
||||
|
||||
|
||||
@router.get("/fine-tune/{task_id}")
|
||||
@@ -315,6 +526,37 @@ async def fine_tune_detail(task_id: str) -> dict[str, Any]:
|
||||
raise fail(404, "fine tune task not found")
|
||||
|
||||
|
||||
@router.get("/fine-tune/{task_id}/logs")
|
||||
async def fine_tune_logs(
|
||||
task_id: str,
|
||||
tail_lines: int | None = Query(default=500, ge=1, le=5000),
|
||||
offset: int | None = Query(default=None, ge=0),
|
||||
limit: int | None = Query(default=None, ge=1, le=5000),
|
||||
) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
task = store.task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
if task.get("compute_job_id"):
|
||||
node = _node_for_task(task)
|
||||
if node:
|
||||
try:
|
||||
logs = await ComputeNodeClient(node["api_base_url"]).job_logs(task["compute_job_id"], tail_lines, offset, limit)
|
||||
if task.get("status") in {"queued", "running", "failed", "stopped", "completed"}:
|
||||
try:
|
||||
job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
|
||||
store.apply_compute_job(task_id, job)
|
||||
except Exception:
|
||||
pass
|
||||
return ok({"source": "compute", **logs})
|
||||
except Exception as exc: # noqa: BLE001 - keep failure reason visible even when log fetch fails
|
||||
content = task.get("failure_reason") or f"fetch compute log failed: {exc}"
|
||||
return ok({"job_id": task.get("compute_job_id"), "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
|
||||
content = task.get("failure_reason") or ""
|
||||
return ok({"job_id": task.get("compute_job_id") or "", "source": "task", "file": task.get("log_file") or "", "content": content, "size": f"{len(content.encode('utf-8'))} B"})
|
||||
|
||||
|
||||
@router.put("/fine-tune/{task_id}")
|
||||
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
@@ -325,8 +567,14 @@ async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) ->
|
||||
|
||||
@router.post("/fine-tune/stop/{task_id}")
|
||||
async def stop_fine_tune(task_id: str) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
try:
|
||||
return ok(get_platform_store().stop_task(task_id))
|
||||
task = store.task(task_id)
|
||||
node = _node_for_task(task)
|
||||
if task.get("compute_job_id") and node and get_settings().compute_mode != "simulator":
|
||||
job = await ComputeNodeClient(node["api_base_url"]).stop_job(task["compute_job_id"])
|
||||
return ok(store.apply_compute_job(task_id, job))
|
||||
return ok(store.stop_task(task_id))
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
|
||||
@@ -336,6 +584,27 @@ async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]:
|
||||
return await stop_fine_tune(task_id)
|
||||
|
||||
|
||||
@router.post("/fine-tune/{task_id}/retry")
|
||||
async def retry_fine_tune(task_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
payload = payload or {}
|
||||
try:
|
||||
task = store.task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "fine tune task not found")
|
||||
if task["status"] not in {"failed", "stopped"} and not payload.get("force"):
|
||||
raise fail(409, "only failed or stopped tasks can be retried without force=true")
|
||||
retry_payload = {**task, **payload, "task_id": task_id, "id": task_id}
|
||||
store.reset_task_for_retry(task_id, retry_payload)
|
||||
try:
|
||||
return ok(await _submit_fine_tune_task(store, retry_payload))
|
||||
except RuntimeError as exc:
|
||||
raise fail(409, str(exc))
|
||||
except Exception as exc: # noqa: BLE001 - mark retry failed when remote submit fails
|
||||
store.mark_task_failed(task_id, str(exc))
|
||||
raise fail(502, f"retry fine tune task failed: {exc}")
|
||||
|
||||
|
||||
@router.delete("/fine-tune/{task_id}")
|
||||
async def delete_fine_tune(task_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_task(task_id)
|
||||
@@ -358,17 +627,217 @@ async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]:
|
||||
return ok(checkpoints)
|
||||
|
||||
|
||||
@router.get("/model-eval")
|
||||
async def model_eval_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().eval_tasks())
|
||||
|
||||
|
||||
@router.get("/model-eval/{task_id}")
|
||||
async def model_eval_detail(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().eval_task(task_id))
|
||||
except KeyError:
|
||||
raise fail(404, "eval task not found")
|
||||
|
||||
|
||||
@router.post("/model-eval/start")
|
||||
async def model_eval_start(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
task = get_platform_store().create_eval_task(payload)
|
||||
return ok({"task_id": task["id"], **task})
|
||||
|
||||
|
||||
@router.delete("/model-eval/{task_id}")
|
||||
async def model_eval_delete(task_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_eval_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
|
||||
@router.get("/dimension")
|
||||
async def dimension_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().dimensions())
|
||||
|
||||
|
||||
@router.post("/dimension")
|
||||
async def dimension_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok(get_platform_store().create_dimension(payload))
|
||||
|
||||
|
||||
@router.get("/dimension/{dimension_id}")
|
||||
async def dimension_detail(dimension_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().dimension(dimension_id))
|
||||
except KeyError:
|
||||
raise fail(404, "dimension not found")
|
||||
|
||||
|
||||
@router.put("/dimension/{dimension_id}")
|
||||
async def dimension_update(dimension_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_dimension(dimension_id, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "dimension not found")
|
||||
|
||||
|
||||
@router.delete("/dimension/{dimension_id}")
|
||||
async def dimension_delete(dimension_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_dimension(dimension_id)
|
||||
return ok({"deleted": dimension_id})
|
||||
|
||||
|
||||
@router.get("/model-compare")
|
||||
async def model_compare_list() -> dict[str, Any]:
|
||||
return ok(get_platform_store().compare_tasks())
|
||||
|
||||
|
||||
@router.post("/model-compare")
|
||||
async def model_compare_create(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
task = get_platform_store().create_compare_task(payload)
|
||||
return ok({"id": task["id"]})
|
||||
|
||||
|
||||
@router.post("/model-compare/all/stop-all")
|
||||
async def model_compare_stop_all() -> dict[str, Any]:
|
||||
return ok({"stopped": True})
|
||||
|
||||
|
||||
@router.post("/model-compare/stop-by-pid")
|
||||
async def model_compare_stop_by_pid(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"stopped": True, "pid": payload.get("pid")})
|
||||
|
||||
|
||||
@router.get("/model-compare/{task_id}")
|
||||
async def model_compare_detail(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().compare_task(task_id))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.delete("/model-compare/{task_id}")
|
||||
async def model_compare_delete(task_id: str) -> dict[str, Any]:
|
||||
get_platform_store().delete_compare_task(task_id)
|
||||
return ok({"deleted": task_id})
|
||||
|
||||
|
||||
@router.get("/model-compare/{task_id}/load-status")
|
||||
async def model_compare_load_status(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
load_status = task.get("load_status") or {"loaded_models": []}
|
||||
if isinstance(load_status, str):
|
||||
try:
|
||||
load_status = json.loads(load_status)
|
||||
except json.JSONDecodeError:
|
||||
load_status = {"loaded_models": []}
|
||||
return ok({"all_ready": all(item.get("status") in {"ready", "running"} for item in load_status.get("loaded_models", [])), **load_status})
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/load-status")
|
||||
async def model_compare_update_load_status(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"load_status": payload.get("load_status") or {"loaded_models": []}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/load")
|
||||
async def model_compare_load(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
task = get_platform_store().compare_task(task_id)
|
||||
models = task.get("models") or []
|
||||
if isinstance(models, str):
|
||||
try:
|
||||
models = json.loads(models)
|
||||
except json.JSONDecodeError:
|
||||
models = []
|
||||
loaded_models = [
|
||||
{
|
||||
"model_id": item.get("model_id"),
|
||||
"model_name": item.get("model_name"),
|
||||
"status": "ready",
|
||||
"pid": 45000 + index,
|
||||
"port": item.get("port") or 18000 + index,
|
||||
}
|
||||
for index, item in enumerate(models)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"status": "loaded", "load_status": {"loaded_models": loaded_models}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/unload")
|
||||
async def model_compare_unload(task_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().update_compare_task(task_id, {"status": "pending", "load_status": {"loaded_models": []}}))
|
||||
except KeyError:
|
||||
raise fail(404, "compare task not found")
|
||||
|
||||
|
||||
@router.post("/model-compare/{task_id}/start-model")
|
||||
async def model_compare_start_model(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"pid": 45001, "port": payload.get("port") or 18001, "task_id": task_id})
|
||||
|
||||
|
||||
@router.post("/model-compare/chat-with-port")
|
||||
async def model_compare_chat_with_port(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
question = ""
|
||||
for message in payload.get("messages") or []:
|
||||
if message.get("role") == "user":
|
||||
question = str(message.get("content") or "")
|
||||
content = f"当前后端已收到推理请求:{question[:120]}"
|
||||
return ok({"response": content, "content": content})
|
||||
|
||||
|
||||
@router.post("/model-compare/stream-chat")
|
||||
async def model_compare_stream_chat(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
question = payload.get("user_question") or payload.get("question") or ""
|
||||
return ok({"response": f"当前后端已收到流式推理请求:{str(question)[:120]}"})
|
||||
|
||||
|
||||
@router.post("/model-chat/batch")
|
||||
async def model_chat_batch(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"responses": [], "request": payload})
|
||||
|
||||
|
||||
@router.post("/model-chat/local/chat")
|
||||
async def model_chat_local(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"response": "local chat adapter is not connected yet", "request": payload})
|
||||
|
||||
|
||||
@router.post("/model-chat/local/preload")
|
||||
async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"loaded": True, "request": payload})
|
||||
|
||||
|
||||
@router.post("/model-chat/trained/preload")
|
||||
async def model_chat_trained_preload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
return ok({"loaded": True, "request": payload})
|
||||
|
||||
|
||||
@router.get("/compute/nodes")
|
||||
async def compute_nodes() -> dict[str, Any]:
|
||||
return ok(get_platform_store().compute_nodes())
|
||||
|
||||
|
||||
@router.get("/compute/nodes/{node_id}")
|
||||
async def compute_node_detail(node_id: str) -> dict[str, Any]:
|
||||
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
return ok(node)
|
||||
|
||||
|
||||
@router.post("/compute/nodes")
|
||||
async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
try:
|
||||
return ok(get_platform_store().create_compute_node(payload))
|
||||
except KeyError as exc:
|
||||
raise fail(400, f"missing field: {exc}")
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
|
||||
|
||||
@router.put("/compute/nodes/{node_id}")
|
||||
@@ -377,11 +846,47 @@ async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...))
|
||||
return ok(get_platform_store().update_compute_node(node_id, payload))
|
||||
except KeyError:
|
||||
raise fail(404, "compute node not found")
|
||||
except ValueError as exc:
|
||||
raise fail(400, str(exc))
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/test-connection")
|
||||
async def test_compute_node(node_id: str) -> dict[str, Any]:
|
||||
return ok({"node_id": node_id, "success": True, "latency_ms": 12})
|
||||
store = get_platform_store()
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
client = ComputeNodeClient(node["api_base_url"])
|
||||
try:
|
||||
result = await client.test_connection()
|
||||
store.replace_node_gpus(node_id, result["gpus"])
|
||||
updated = store.update_compute_node_health(node_id, result["health"], True)
|
||||
return ok(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"success": True,
|
||||
"latency_ms": result["latency_ms"],
|
||||
"gpu_count": len(result["gpus"]),
|
||||
"health": updated["health_detail"],
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - return the connection error for node maintenance
|
||||
updated = store.update_compute_node_health(node_id, {}, False, str(exc))
|
||||
return ok(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"success": False,
|
||||
"latency_ms": 0,
|
||||
"gpu_count": updated.get("gpu_count", 0),
|
||||
"error": str(exc),
|
||||
"health": updated["health_detail"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/health-check")
|
||||
async def health_check_compute_node(node_id: str) -> dict[str, Any]:
|
||||
return await test_compute_node(node_id)
|
||||
|
||||
|
||||
@router.post("/compute/nodes/{node_id}/enable")
|
||||
@@ -404,6 +909,38 @@ async def compute_node_replicas(node_id: str) -> dict[str, Any]:
|
||||
return ok(get_platform_store().replicas(node_id))
|
||||
|
||||
|
||||
@router.get("/compute/nodes/{node_id}/engines")
|
||||
async def compute_node_engines(node_id: str) -> dict[str, Any]:
|
||||
node = next((item for item in get_platform_store().compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
health = node.get("health_detail") or {}
|
||||
live_error = ""
|
||||
try:
|
||||
health = await ComputeNodeClient(node["api_base_url"]).health()
|
||||
except Exception as exc: # noqa: BLE001 - stored health is enough for offline node detail
|
||||
live_error = str(exc)
|
||||
capabilities = health.get("capabilities") or node.get("capabilities") or []
|
||||
return ok(
|
||||
{
|
||||
"node_id": node_id,
|
||||
"items": [
|
||||
{
|
||||
"engine": "llama_factory",
|
||||
"display_name": "LLaMA-Factory",
|
||||
"status": "available" if "llama_factory" in capabilities else "unknown",
|
||||
"version": health.get("llama_factory_version") or "",
|
||||
"home": health.get("llama_factory_home") or "",
|
||||
"home_exists": bool(health.get("llama_factory_home_exists")),
|
||||
"capabilities": capabilities,
|
||||
"execution_mode": health.get("execution_mode") or "",
|
||||
"last_error": live_error,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/compute/gpus")
|
||||
async def compute_gpus() -> dict[str, Any]:
|
||||
return ok(get_platform_store().gpus())
|
||||
@@ -414,10 +951,107 @@ async def compute_queue() -> dict[str, Any]:
|
||||
return ok(get_platform_store().queue())
|
||||
|
||||
|
||||
@router.get("/compute/jobs/{job_id}")
|
||||
async def compute_job_detail(job_id: str) -> dict[str, Any]:
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
return ok(await ComputeNodeClient(node["api_base_url"]).get_job(job_id))
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/stop")
|
||||
async def compute_job_stop(job_id: str) -> dict[str, Any]:
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
job = await ComputeNodeClient(node["api_base_url"]).stop_job(job_id)
|
||||
get_platform_store().apply_compute_job(task["id"], job)
|
||||
return ok(job)
|
||||
|
||||
|
||||
@router.get("/compute/jobs/{job_id}/logs")
|
||||
async def compute_job_logs(
|
||||
job_id: str,
|
||||
tail_lines: int | None = Query(default=200, ge=1, le=5000),
|
||||
offset: int | None = Query(default=None, ge=0),
|
||||
limit: int | None = Query(default=None, ge=1, le=5000),
|
||||
) -> dict[str, Any]:
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
return ok(await ComputeNodeClient(node["api_base_url"]).job_logs(job_id, tail_lines, offset, limit))
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/retry")
|
||||
async def compute_job_retry(job_id: str, payload: dict[str, Any] | None = Body(default=None)) -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
payload = payload or {}
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
return await retry_fine_tune(task["id"], payload)
|
||||
|
||||
|
||||
@router.post("/compute/jobs/{job_id}/priority")
|
||||
async def compute_job_priority(job_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
task = _task_for_compute_job(job_id)
|
||||
if not task:
|
||||
raise fail(404, "compute job not found")
|
||||
priority = str(payload.get("priority") or "normal")
|
||||
return ok(get_platform_store().update_task_priority(task["id"], priority))
|
||||
|
||||
|
||||
@router.post("/internal/compute-sync/jobs/poll")
|
||||
async def poll_compute_jobs() -> dict[str, Any]:
|
||||
return ok(await poll_compute_jobs_once())
|
||||
|
||||
|
||||
@router.post("/internal/compute-sync/resources")
|
||||
async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
sync_id = get_platform_store().create_sync_job(payload.get("target_node_id", "node_01"), payload)
|
||||
return ok(get_platform_store().sync_job(sync_id))
|
||||
store = get_platform_store()
|
||||
node_id = payload.get("target_node_id") or payload.get("target_compute_node_id")
|
||||
if not node_id:
|
||||
raise fail(400, "target_node_id is required")
|
||||
node = next((item for item in store.compute_nodes() if item["id"] == node_id), None)
|
||||
if not node:
|
||||
raise fail(404, "compute node not found")
|
||||
sync_id = store.create_sync_job(node_id, payload)
|
||||
replicas = []
|
||||
failures = []
|
||||
resources = payload.get("resources") or []
|
||||
for resource in resources:
|
||||
if not resource.get("source_path"):
|
||||
continue
|
||||
try:
|
||||
result = await ComputeNodeClient(node["api_base_url"]).import_local_file(
|
||||
{
|
||||
"source_path": resource["source_path"],
|
||||
"target_relative_path": resource.get("target_relative_path"),
|
||||
"resource_type": resource.get("resource_type"),
|
||||
"resource_id": resource.get("resource_id"),
|
||||
}
|
||||
)
|
||||
replicas.append(
|
||||
store.upsert_resource_replica(
|
||||
node_id,
|
||||
str(resource.get("resource_type") or "file"),
|
||||
str(resource.get("resource_id") or result["id"]),
|
||||
result["local_path"],
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - collect per-resource failures
|
||||
failures.append({"resource_id": str(resource.get("resource_id")), "error": str(exc)})
|
||||
store.update_sync_job(sync_id, "failed" if failures else "completed", 100 if not failures else 99, completed=True)
|
||||
return ok({"sync": store.sync_job(sync_id), "replicas": replicas, "failed": failures})
|
||||
|
||||
|
||||
@router.get("/internal/compute-sync/resources/{sync_id}")
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.v1.endpoints.data_process import router as data_process_router
|
||||
from app.api.v1.endpoints.platform import router as platform_router
|
||||
from app.api.v1.endpoints.health import router as health_router
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(health_router, tags=["health"])
|
||||
api_router.include_router(data_process_router, tags=["data-process"])
|
||||
api_router.include_router(platform_router, tags=["platform"])
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ class Settings:
|
||||
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
|
||||
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
|
||||
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
|
||||
compute_request_timeout_seconds: int = _int_env("COMPUTE_REQUEST_TIMEOUT_SECONDS", 5)
|
||||
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
log_dir: str = os.getenv("LOG_DIR", "./logs")
|
||||
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")
|
||||
|
||||
@@ -53,6 +53,13 @@ def json_dumps(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def safe_float(value: Any, default: float = 0) -> float:
|
||||
try:
|
||||
return float(str(value).replace("[N/A]", "").strip() or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex[:12]}"
|
||||
|
||||
@@ -178,12 +185,33 @@ class PlatformStore:
|
||||
schema_path = Path(__file__).with_name("sql") / "001_platform_runtime.sql"
|
||||
with self.connect() as conn:
|
||||
conn.executescript(schema_path.read_text(encoding="utf-8"))
|
||||
columns = conn.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='users'"
|
||||
).fetchall()
|
||||
column_names = {row["column_name"] for row in columns}
|
||||
if "password" in column_names and "password_hash" not in column_names:
|
||||
user_columns = self._column_names(conn, "users")
|
||||
if "password" in user_columns and "password_hash" not in user_columns:
|
||||
conn.execute("ALTER TABLE users RENAME COLUMN password TO password_hash")
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"compute_nodes",
|
||||
{
|
||||
"api_version": "TEXT NOT NULL DEFAULT 'v1'",
|
||||
"capabilities": "TEXT NOT NULL DEFAULT '[]'",
|
||||
"description": "TEXT",
|
||||
},
|
||||
)
|
||||
self._ensure_columns(conn, "gpus", {"last_seen_at": "TEXT"})
|
||||
self._ensure_columns(conn, "fine_tune_tasks", {"compute_job_id": "TEXT"})
|
||||
|
||||
def _column_names(self, conn: PgConnection, table_name: str) -> set[str]:
|
||||
columns = conn.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name=?",
|
||||
(table_name,),
|
||||
).fetchall()
|
||||
return {row["column_name"] for row in columns}
|
||||
|
||||
def _ensure_columns(self, conn: PgConnection, table_name: str, columns: dict[str, str]) -> None:
|
||||
existing = self._column_names(conn, table_name)
|
||||
for column, definition in columns.items():
|
||||
if column not in existing:
|
||||
conn.execute(f"ALTER TABLE {table_name} ADD COLUMN {column} {definition}")
|
||||
|
||||
def ensure_seed_data(self) -> None:
|
||||
with self.connect() as conn:
|
||||
@@ -436,7 +464,7 @@ class PlatformStore:
|
||||
utcnow(),
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone())
|
||||
|
||||
def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.model(model_id)
|
||||
@@ -461,7 +489,7 @@ class PlatformStore:
|
||||
model_id,
|
||||
),
|
||||
)
|
||||
return self.model(model_id)
|
||||
return dict(conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone())
|
||||
|
||||
def delete_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
@@ -481,6 +509,10 @@ class PlatformStore:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def delete_trained_model(self, model_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM trained_models WHERE id=? OR name=?", (model_id, model_id))
|
||||
|
||||
def datasets(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM datasets ORDER BY create_time DESC").fetchall()
|
||||
@@ -641,6 +673,26 @@ class PlatformStore:
|
||||
conn.execute("UPDATE dataset_files SET active_version_id=? WHERE id=?", (version_id, file_id))
|
||||
return {"version": version, "content": row["content"]}
|
||||
|
||||
def delete_file_version(self, file_id: str, version_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(file_id)
|
||||
versions = json_loads(row["versions"], [])
|
||||
if row["active_version_id"] == version_id:
|
||||
raise ValueError("active dataset version cannot be deleted")
|
||||
if len(versions) <= 1:
|
||||
raise ValueError("last dataset version cannot be deleted")
|
||||
next_versions = [item for item in versions if item["id"] != version_id]
|
||||
if len(next_versions) == len(versions):
|
||||
raise KeyError(version_id)
|
||||
conn.execute("UPDATE dataset_files SET versions=? WHERE id=?", (json_dumps(next_versions), file_id))
|
||||
return {
|
||||
"versions": next_versions,
|
||||
"active_version_id": row["active_version_id"],
|
||||
"next_version_number": max(item.get("version", 0) for item in next_versions) + 1,
|
||||
}
|
||||
|
||||
def tasks(self) -> list[dict[str, Any]]:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
@@ -668,6 +720,8 @@ class PlatformStore:
|
||||
"train_duration": self._duration(row["start_time"], row["completed_at"]) if row["start_time"] else "",
|
||||
"compute_node_id": row["compute_node_id"],
|
||||
"sync_job_id": row["sync_job_id"],
|
||||
"compute_job_id": row.get("compute_job_id"),
|
||||
"completed_at": row.get("completed_at"),
|
||||
}
|
||||
)
|
||||
return payload
|
||||
@@ -689,6 +743,7 @@ class PlatformStore:
|
||||
"status": "pending",
|
||||
"train_type": payload.get("train_type", "SFT"),
|
||||
"train_method": payload.get("train_method", "lora"),
|
||||
"engine": payload.get("engine", payload.get("training_engine", "llama_factory")),
|
||||
"template": payload.get("template", "qwen"),
|
||||
"base_model": base_model,
|
||||
"train_dataset_id": train_dataset_id,
|
||||
@@ -751,7 +806,7 @@ class PlatformStore:
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status='syncing', progress=8, process_id=?, start_time=?,
|
||||
compute_node_id=?, gpus=?, sync_job_id=?
|
||||
compute_node_id=?, gpus=?, sync_job_id=?, compute_job_id=NULL
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
@@ -766,9 +821,140 @@ class PlatformStore:
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
def reset_task_for_retry(self, task_id: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
current = self.task(task_id)
|
||||
override = payload or {}
|
||||
merged = {
|
||||
**current,
|
||||
**override,
|
||||
"id": task_id,
|
||||
"status": "pending",
|
||||
"progress": 0,
|
||||
"process_id": None,
|
||||
"compute_job_id": None,
|
||||
}
|
||||
for runtime_key in ["failure_reason", "log_file", "artifacts"]:
|
||||
merged.pop(runtime_key, None)
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status='pending', progress=0, process_id=NULL, start_time=NULL,
|
||||
completed_at=NULL, compute_node_id=NULL, gpus=?, sync_job_id=NULL, compute_job_id=NULL
|
||||
WHERE id=?
|
||||
""",
|
||||
(json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def update_task_priority(self, task_id: str, priority: str) -> dict[str, Any]:
|
||||
current = self.task(task_id)
|
||||
priority = priority if priority in {"low", "normal", "high", "urgent"} else "normal"
|
||||
merged = {**current, "priority": priority}
|
||||
with self.connect() as conn:
|
||||
conn.execute("UPDATE fine_tune_tasks SET payload=? WHERE id=?", (json_dumps(merged), task_id))
|
||||
return self.task(task_id)
|
||||
|
||||
def prepare_compute_job_payload(self, task_id: str, payload: dict[str, Any] | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)})
|
||||
merged = {**task, **(payload or {}), "id": task_id}
|
||||
node = self.schedule_node(merged)
|
||||
selected_gpus = merged.get("gpus") or [0]
|
||||
return node, self._compute_job_payload_from_task_node(merged, node, selected_gpus)
|
||||
|
||||
def _compute_job_payload_from_task_node(
|
||||
self,
|
||||
task: dict[str, Any],
|
||||
node: dict[str, Any],
|
||||
selected_gpus: list[int] | list[Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT * FROM models WHERE id=?", (task.get("base_model"),)).fetchone()
|
||||
dataset = conn.execute("SELECT * FROM datasets WHERE id=?", (task.get("train_dataset_id"),)).fetchone()
|
||||
model_path = (model and model.get("path")) or task.get("base_model")
|
||||
dataset_name = task.get("dataset") or (dataset and dataset.get("name")) or task.get("train_dataset_id")
|
||||
health_detail = node.get("health_detail") or {}
|
||||
dataset_root = str(health_detail.get("dataset_root") or f"{node['data_root'].rstrip('/')}/datasets")
|
||||
output_root = str(health_detail.get("output_root") or f"{node['data_root'].rstrip('/')}/outputs")
|
||||
output_dir = task.get("output_dir") or f"{output_root.rstrip('/')}/{task['name']}"
|
||||
return {
|
||||
**task,
|
||||
"id": task["id"],
|
||||
"name": task["name"],
|
||||
"base_model": model_path,
|
||||
"model_name_or_path": model_path,
|
||||
"dataset": dataset_name,
|
||||
"dataset_dir": dataset_root,
|
||||
"output_dir": output_dir,
|
||||
"gpus": selected_gpus or task.get("gpus") or [0],
|
||||
"compute_node_id": node["id"],
|
||||
"compute_node_code": node["code"],
|
||||
}
|
||||
|
||||
def build_compute_job_payload(self, task_id: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
task = self.task(task_id)
|
||||
node = next((item for item in self.compute_nodes() if item["id"] == task.get("compute_node_id")), None)
|
||||
if not node:
|
||||
raise RuntimeError("compute node not found")
|
||||
return node, self._compute_job_payload_from_task_node(task, node, task.get("gpus") or [0])
|
||||
|
||||
def apply_compute_job(self, task_id: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
status_map = {
|
||||
"queued": "queued",
|
||||
"running": "running",
|
||||
"completed": "completed",
|
||||
"failed": "failed",
|
||||
"stopped": "stopped",
|
||||
}
|
||||
current = self.task(task_id)
|
||||
status = status_map.get(str(job.get("status")), str(job.get("status") or current["status"]))
|
||||
progress = int(job.get("progress", current.get("progress", 0)) or 0)
|
||||
payload = {
|
||||
**current,
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"process_id": job.get("pid") or current.get("process_id"),
|
||||
"compute_job_id": job.get("id") or current.get("compute_job_id"),
|
||||
"output_dir": job.get("output_dir") or current.get("output_dir"),
|
||||
"log_file": job.get("log_file") or current.get("log_file"),
|
||||
"artifacts": job.get("artifacts") or current.get("artifacts") or [],
|
||||
}
|
||||
if status == "failed":
|
||||
payload["failure_reason"] = job.get("error") or job.get("message") or current.get("failure_reason") or "compute job failed"
|
||||
elif status in {"queued", "running", "completed"}:
|
||||
payload.pop("failure_reason", None)
|
||||
completed_at = utcnow() if status in {"completed", "failed", "stopped"} and not current.get("completed_at") else None
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fine_tune_tasks
|
||||
SET payload=?, status=?, progress=?, process_id=?, compute_job_id=?, completed_at=COALESCE(?, completed_at)
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
json_dumps(payload),
|
||||
status,
|
||||
progress,
|
||||
payload.get("process_id"),
|
||||
payload.get("compute_job_id"),
|
||||
completed_at,
|
||||
task_id,
|
||||
),
|
||||
)
|
||||
if status == "completed":
|
||||
self._ensure_trained_model(conn, payload)
|
||||
return self.task(task_id)
|
||||
|
||||
def running_compute_tasks(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
task
|
||||
for task in self.tasks()
|
||||
if task.get("compute_job_id") and task.get("compute_node_id") and task["status"] in {"syncing", "queued", "running"}
|
||||
]
|
||||
|
||||
def mark_task_failed(self, task_id: str, reason: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": "failed", "progress": min(task.get("progress", 0), 99), "failure_reason": reason})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?",
|
||||
@@ -776,10 +962,179 @@ class PlatformStore:
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str, status: str = "stopped") -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
task.update({"status": status, "progress": min(task.get("progress", 0), 99)})
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE fine_tune_tasks SET status=?, payload=?, completed_at=? WHERE id=?",
|
||||
(status, json_dumps(task), utcnow(), task_id),
|
||||
)
|
||||
return self.task(task_id)
|
||||
|
||||
def delete_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def _json_payload_row(self, row: PgRow) -> dict[str, Any]:
|
||||
payload = json_loads(row["payload"], {})
|
||||
payload.update({"id": row["id"], "status": row.get("status"), "create_time": row["create_time"]})
|
||||
return payload
|
||||
|
||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
payload = self._json_payload_row(row)
|
||||
payload.setdefault("sample_count", 0)
|
||||
payload.setdefault("completed_count", 0)
|
||||
payload.setdefault("passed_count", 0)
|
||||
payload.setdefault("overall_score", payload.get("score") or 0)
|
||||
payload.setdefault("overall_score_max", 100)
|
||||
payload.setdefault("overall_evaluation", "")
|
||||
payload.setdefault("improvement_suggestions", [])
|
||||
payload.setdefault("dimension_summary", [])
|
||||
payload.setdefault("samples", [])
|
||||
return payload
|
||||
|
||||
def create_eval_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or payload.get("task_id") or new_id("eval"))
|
||||
name = str(payload.get("eval_task_name") or payload.get("name") or f"eval-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {
|
||||
**payload,
|
||||
"id": task_id,
|
||||
"eval_task_name": name,
|
||||
"status": status,
|
||||
"create_time": now,
|
||||
"metric": payload.get("metric") or "custom",
|
||||
}
|
||||
with self.connect() as conn:
|
||||
model = conn.execute("SELECT name FROM models WHERE id=?", (str(payload.get("model_id")),)).fetchone()
|
||||
dataset = conn.execute("SELECT name FROM datasets WHERE id=?", (str(payload.get("dataset_id")),)).fetchone()
|
||||
if model:
|
||||
data.setdefault("model_name", model["name"])
|
||||
if dataset:
|
||||
data.setdefault("dataset", dataset["name"])
|
||||
conn.execute(
|
||||
"INSERT INTO eval_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.eval_task(task_id)
|
||||
|
||||
def delete_eval_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def dimensions(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM eval_dimensions ORDER BY create_time DESC").fetchall()
|
||||
return [
|
||||
{
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def dimension(self, dimension_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM eval_dimensions WHERE id=?", (dimension_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(dimension_id)
|
||||
return {
|
||||
**json_loads(row["payload"], {}),
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"is_active": bool(row["is_active"]),
|
||||
"is_default": bool(row["is_default"]),
|
||||
"create_time": row["create_time"],
|
||||
}
|
||||
|
||||
def create_dimension(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
dimension_id = str(payload.get("id") or new_id("dim"))
|
||||
name = str(payload.get("name") or f"dimension-{dimension_id[-6:]}")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": dimension_id, "name": name, "create_time": now}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO eval_dimensions (id, name, payload, is_active, is_default, create_time) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(dimension_id, name, json_dumps(data), 1 if data.get("is_active", True) else 0, 1 if data.get("is_default") else 0, now),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def update_dimension(self, dimension_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.dimension(dimension_id)
|
||||
merged = {**current, **payload, "id": dimension_id}
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE eval_dimensions SET name=?, payload=?, is_active=?, is_default=? WHERE id=?",
|
||||
(
|
||||
merged["name"],
|
||||
json_dumps(merged),
|
||||
1 if merged.get("is_active", True) else 0,
|
||||
1 if merged.get("is_default") else 0,
|
||||
dimension_id,
|
||||
),
|
||||
)
|
||||
return self.dimension(dimension_id)
|
||||
|
||||
def delete_dimension(self, dimension_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM eval_dimensions WHERE id=?", (dimension_id,))
|
||||
|
||||
def compare_tasks(self) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM compare_tasks ORDER BY create_time DESC").fetchall()
|
||||
return [self._json_payload_row(row) for row in rows]
|
||||
|
||||
def compare_task(self, task_id: str) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute("SELECT * FROM compare_tasks WHERE id=?", (task_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(task_id)
|
||||
return self._json_payload_row(row)
|
||||
|
||||
def create_compare_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = str(payload.get("id") or new_id("cmp"))
|
||||
name = str(payload.get("name") or payload.get("model_name") or f"compare-{task_id[-6:]}")
|
||||
status = str(payload.get("status") or "pending")
|
||||
now = payload.get("create_time") or utcnow()
|
||||
data = {**payload, "id": task_id, "name": name, "model_name": payload.get("model_name") or name, "status": status, "create_time": now}
|
||||
data.setdefault("load_status", json_dumps({"loaded_models": []}))
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO compare_tasks (id, name, payload, status, create_time) VALUES (?, ?, ?, ?, ?)",
|
||||
(task_id, name, json_dumps(data), status, now),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def update_compare_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = self.compare_task(task_id)
|
||||
merged = {**current, **payload, "id": task_id}
|
||||
status = str(merged.get("status") or current.get("status") or "pending")
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE compare_tasks SET name=?, payload=?, status=? WHERE id=?",
|
||||
(merged.get("name") or merged.get("model_name") or task_id, json_dumps(merged), status, task_id),
|
||||
)
|
||||
return self.compare_task(task_id)
|
||||
|
||||
def delete_compare_task(self, task_id: str) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM compare_tasks WHERE id=?", (task_id,))
|
||||
|
||||
def schedule_node(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
requested = payload.get("requested_node_id") or payload.get("compute_node_id")
|
||||
nodes = self.compute_nodes()
|
||||
@@ -793,7 +1148,20 @@ class PlatformStore:
|
||||
if selected:
|
||||
return selected
|
||||
if not candidates:
|
||||
raise RuntimeError("no available compute node")
|
||||
if not nodes:
|
||||
raise RuntimeError("no available compute node: no compute node configured")
|
||||
reasons = []
|
||||
for node in nodes:
|
||||
if not node["enabled"]:
|
||||
reason = "disabled"
|
||||
elif node["scheduler_status"] != "online":
|
||||
reason = f"status={node['scheduler_status']}"
|
||||
elif node["current_running_jobs"] >= node["max_parallel_jobs"]:
|
||||
reason = f"capacity full {node['current_running_jobs']}/{node['max_parallel_jobs']}"
|
||||
else:
|
||||
reason = "not selected"
|
||||
reasons.append(f"{node['code']}({reason})")
|
||||
raise RuntimeError(f"no available compute node: {', '.join(reasons)}")
|
||||
return sorted(candidates, key=lambda n: (-n["scheduler_weight"], n["current_running_jobs"], n["code"]))[0]
|
||||
|
||||
def create_sync_job(self, node_id: str, task: dict[str, Any]) -> str:
|
||||
@@ -809,7 +1177,8 @@ class PlatformStore:
|
||||
sync_id,
|
||||
node_id,
|
||||
json_dumps(
|
||||
[
|
||||
task.get("resources")
|
||||
or [
|
||||
{"resource_type": "model", "resource_id": task.get("base_model")},
|
||||
{"resource_type": "dataset", "resource_id": task.get("train_dataset_id")},
|
||||
]
|
||||
@@ -819,6 +1188,46 @@ class PlatformStore:
|
||||
)
|
||||
return sync_id
|
||||
|
||||
def update_sync_job(self, sync_id: str, status: str, progress: int, completed: bool = False) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=COALESCE(?, completed_at) WHERE id=?",
|
||||
(status, progress, utcnow() if completed else None, sync_id),
|
||||
)
|
||||
return self.sync_job(sync_id)
|
||||
|
||||
def upsert_resource_replica(
|
||||
self,
|
||||
node_id: str,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
local_path: str,
|
||||
status: str = "available",
|
||||
sync_status: str = "synced",
|
||||
) -> dict[str, Any]:
|
||||
with self.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM resource_replicas WHERE node_id=? AND resource_type=? AND resource_id=?",
|
||||
(node_id, resource_type, resource_id),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE resource_replicas SET local_path=?, status=?, sync_status=? WHERE id=?",
|
||||
(local_path, status, sync_status, row["id"]),
|
||||
)
|
||||
replica_id = row["id"]
|
||||
else:
|
||||
replica_id = new_id("replica")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO resource_replicas
|
||||
(id, node_id, resource_type, resource_id, local_path, status, sync_status, create_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(replica_id, node_id, resource_type, resource_id, local_path, status, sync_status, utcnow()),
|
||||
)
|
||||
return dict(conn.execute("SELECT * FROM resource_replicas WHERE id=?", (replica_id,)).fetchone())
|
||||
|
||||
def progress(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.task(task_id)
|
||||
status = task.get("status", "pending")
|
||||
@@ -829,6 +1238,7 @@ class PlatformStore:
|
||||
"running": "training with LLaMA-Factory",
|
||||
"completed": "training completed",
|
||||
"failed": "training stopped",
|
||||
"stopped": "training stopped",
|
||||
}
|
||||
progress = int(task.get("progress", 0) or 0)
|
||||
eta = "--" if status in {"completed", "failed"} else f"{max(1, math.ceil((100 - progress) / 10))} min"
|
||||
@@ -853,23 +1263,62 @@ class PlatformStore:
|
||||
**dict(row),
|
||||
"enabled": bool(row["enabled"]),
|
||||
"tags": json_loads(row["tags"], []),
|
||||
"capabilities": json_loads(row.get("capabilities"), []),
|
||||
"health_detail": json_loads(row["health_detail"], {}),
|
||||
"current_running_jobs": running_map.get(row["id"], 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def _normalize_tags(self, value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
parts = value.replace(",", ",").split(",")
|
||||
return [item.strip() for item in parts if item.strip()]
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return []
|
||||
|
||||
def _normalize_compute_node_payload(self, payload: dict[str, Any], current: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
merged = {**(current or {}), **payload}
|
||||
api_base_url = str(merged.get("api_base_url") or "").rstrip("/")
|
||||
if not api_base_url:
|
||||
raise ValueError("api_base_url is required")
|
||||
file_gateway_url = str(merged.get("file_gateway_url") or api_base_url).rstrip("/")
|
||||
weight = max(0, min(1000, int(merged.get("scheduler_weight", 100))))
|
||||
max_jobs = max(1, int(merged.get("max_parallel_jobs", 1)))
|
||||
return {
|
||||
**merged,
|
||||
"code": str(merged.get("code") or "").strip(),
|
||||
"name": str(merged.get("name") or merged.get("code") or "").strip(),
|
||||
"api_base_url": api_base_url,
|
||||
"file_gateway_url": file_gateway_url,
|
||||
"enabled": bool(merged.get("enabled", True)),
|
||||
"scheduler_status": str(merged.get("scheduler_status") or "offline"),
|
||||
"scheduler_weight": weight,
|
||||
"tags": self._normalize_tags(merged.get("tags")),
|
||||
"gpu_count": max(0, int(merged.get("gpu_count", 0) or 0)),
|
||||
"max_parallel_jobs": max_jobs,
|
||||
"data_root": str(merged.get("data_root") or "/data/yg-ft"),
|
||||
"model_root": str(merged.get("model_root") or "/data/yg-ft/models"),
|
||||
"log_root": str(merged.get("log_root") or "/opt/yg-ft/logs/training"),
|
||||
"api_version": str(merged.get("api_version") or "v1"),
|
||||
"capabilities": merged.get("capabilities") or [],
|
||||
"description": merged.get("description") or "",
|
||||
"health_detail": merged.get("health_detail") or {"status": "registered"},
|
||||
}
|
||||
|
||||
def update_compute_node(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||
if not current:
|
||||
raise KeyError(node_id)
|
||||
merged = {**current, **payload}
|
||||
merged = self._normalize_compute_node_payload(payload, current)
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE compute_nodes
|
||||
SET name=?, api_base_url=?, file_gateway_url=?, enabled=?, scheduler_status=?,
|
||||
scheduler_weight=?, tags=?, max_parallel_jobs=?, last_health_check_at=?
|
||||
scheduler_weight=?, tags=?, max_parallel_jobs=?, data_root=?, model_root=?, log_root=?,
|
||||
api_version=?, capabilities=?, description=?, last_health_check_at=?, health_detail=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(
|
||||
@@ -881,46 +1330,116 @@ class PlatformStore:
|
||||
merged["scheduler_weight"],
|
||||
json_dumps(merged["tags"]),
|
||||
merged["max_parallel_jobs"],
|
||||
utcnow(),
|
||||
merged["data_root"],
|
||||
merged["model_root"],
|
||||
merged["log_root"],
|
||||
merged["api_version"],
|
||||
json_dumps(merged["capabilities"]),
|
||||
merged["description"],
|
||||
payload.get("last_health_check_at") or current.get("last_health_check_at"),
|
||||
json_dumps(merged["health_detail"]),
|
||||
node_id,
|
||||
),
|
||||
)
|
||||
return next(n for n in self.compute_nodes() if n["id"] == node_id)
|
||||
|
||||
def create_compute_node(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = self._normalize_compute_node_payload(payload)
|
||||
if not payload["code"]:
|
||||
raise ValueError("code is required")
|
||||
node_id = payload.get("id") or new_id("node")
|
||||
now = utcnow()
|
||||
tags = payload.get("tags") or []
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO compute_nodes
|
||||
(id, code, name, api_base_url, file_gateway_url, enabled, scheduler_status,
|
||||
scheduler_weight, tags, gpu_count, current_running_jobs, max_parallel_jobs,
|
||||
data_root, model_root, log_root, last_health_check_at, health_detail)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?)
|
||||
data_root, model_root, log_root, api_version, capabilities, description,
|
||||
last_health_check_at, health_detail)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
node_id,
|
||||
payload["code"],
|
||||
payload.get("name") or payload["code"],
|
||||
payload["name"] or payload["code"],
|
||||
payload["api_base_url"],
|
||||
payload.get("file_gateway_url") or payload["api_base_url"],
|
||||
1 if payload.get("enabled", True) else 0,
|
||||
payload.get("scheduler_status", "offline"),
|
||||
int(payload.get("scheduler_weight", 100)),
|
||||
json_dumps(tags),
|
||||
int(payload.get("gpu_count", 0)),
|
||||
int(payload.get("max_parallel_jobs", 1)),
|
||||
payload.get("data_root", "/data/yg-ft"),
|
||||
payload.get("model_root", "/models"),
|
||||
payload.get("log_root", "/data/yg-ft/training-logs"),
|
||||
payload["file_gateway_url"],
|
||||
1 if payload["enabled"] else 0,
|
||||
payload["scheduler_status"],
|
||||
payload["scheduler_weight"],
|
||||
json_dumps(payload["tags"]),
|
||||
payload["gpu_count"],
|
||||
payload["max_parallel_jobs"],
|
||||
payload["data_root"],
|
||||
payload["model_root"],
|
||||
payload["log_root"],
|
||||
payload["api_version"],
|
||||
json_dumps(payload["capabilities"]),
|
||||
payload["description"],
|
||||
now,
|
||||
json_dumps(payload.get("health_detail") or {"status": "registered"}),
|
||||
json_dumps(payload["health_detail"]),
|
||||
),
|
||||
)
|
||||
return next(node for node in self.compute_nodes() if node["id"] == node_id)
|
||||
|
||||
def update_compute_node_health(self, node_id: str, health: dict[str, Any], success: bool, error: str | None = None) -> dict[str, Any]:
|
||||
current = next((n for n in self.compute_nodes() if n["id"] == node_id), None)
|
||||
if not current:
|
||||
raise KeyError(node_id)
|
||||
status = "online" if success and current.get("enabled") else "offline"
|
||||
if current.get("scheduler_status") == "draining" and success:
|
||||
status = "draining"
|
||||
detail = {
|
||||
**(current.get("health_detail") or {}),
|
||||
**health,
|
||||
"status": "ok" if success else "failed",
|
||||
"last_error": error or "",
|
||||
"checked_at": utcnow(),
|
||||
}
|
||||
return self.update_compute_node(
|
||||
node_id,
|
||||
{
|
||||
"scheduler_status": status,
|
||||
"last_health_check_at": detail["checked_at"],
|
||||
"health_detail": detail,
|
||||
"data_root": health.get("data_root") or current.get("data_root"),
|
||||
"api_version": str(health.get("api_version") or current.get("api_version") or "v1"),
|
||||
"capabilities": health.get("capabilities") or current.get("capabilities") or [],
|
||||
},
|
||||
)
|
||||
|
||||
def replace_node_gpus(self, node_id: str, gpus: list[dict[str, Any]]) -> None:
|
||||
now = utcnow()
|
||||
with self.connect() as conn:
|
||||
conn.execute("DELETE FROM gpus WHERE node_id=?", (node_id,))
|
||||
for index, item in enumerate(gpus):
|
||||
gpu_index = int(item.get("gpu_index", item.get("id", index)) or 0)
|
||||
memory_total = safe_float(item.get("memory_total_gb") or item.get("memory_total"))
|
||||
if not memory_total and item.get("memory_total_mb") is not None:
|
||||
memory_total = round(safe_float(item.get("memory_total_mb")) / 1024, 2)
|
||||
power_limit = safe_float(item.get("power_limit_w") or item.get("power_limit"))
|
||||
temperature = int(safe_float(item.get("temperature") or item.get("base_temperature"), 35))
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO gpus
|
||||
(id, node_id, gpu_index, uuid, name, memory_total_gb, power_limit_w, base_temperature, last_seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
f"{node_id}_gpu_{gpu_index}",
|
||||
node_id,
|
||||
gpu_index,
|
||||
str(item.get("uuid") or f"{node_id}-GPU-{gpu_index}"),
|
||||
str(item.get("name") or "Unknown GPU"),
|
||||
memory_total or 0,
|
||||
power_limit or 0,
|
||||
temperature,
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute("UPDATE compute_nodes SET gpu_count=? WHERE id=?", (len(gpus), node_id))
|
||||
|
||||
def gpus(self) -> list[dict[str, Any]]:
|
||||
self.refresh_runtime_state()
|
||||
with self.connect() as conn:
|
||||
@@ -951,6 +1470,7 @@ class PlatformStore:
|
||||
reserved = task is not None and task.get("status") in {"syncing", "queued"}
|
||||
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
|
||||
gpu_percent = 86 if busy else 22 if reserved else 3
|
||||
memory_total = float(row["memory_total_gb"] or 0)
|
||||
items.append(
|
||||
{
|
||||
"id": row["gpu_index"],
|
||||
@@ -961,8 +1481,8 @@ class PlatformStore:
|
||||
"uuid": row["uuid"],
|
||||
"gpu_percent": gpu_percent,
|
||||
"memory_used_gb": memory_used,
|
||||
"memory_total_gb": row["memory_total_gb"],
|
||||
"memory_percent": round(memory_used / row["memory_total_gb"] * 100, 1),
|
||||
"memory_total_gb": memory_total,
|
||||
"memory_percent": round(memory_used / memory_total * 100, 1) if memory_total else 0,
|
||||
"temperature": row["base_temperature"] + (21 if busy else 6 if reserved else 0),
|
||||
"power_w": round(row["power_limit_w"] * (0.7 if busy else 0.25 if reserved else 0.08), 1),
|
||||
"power_limit_w": row["power_limit_w"],
|
||||
@@ -1034,12 +1554,14 @@ class PlatformStore:
|
||||
}
|
||||
|
||||
def queue(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
priority_score = {"urgent": 3, "high": 2, "normal": 1, "low": 0}
|
||||
items = [
|
||||
{
|
||||
"id": task["id"],
|
||||
"name": task["name"],
|
||||
"status": task["status"],
|
||||
"progress": task.get("progress", 0),
|
||||
"priority": task.get("priority", "normal"),
|
||||
"compute_node_id": task.get("compute_node_id"),
|
||||
"gpus": task.get("gpus", []),
|
||||
"create_time": task.get("create_time"),
|
||||
@@ -1047,6 +1569,7 @@ class PlatformStore:
|
||||
for task in self.tasks()
|
||||
if task["status"] in {"pending", "syncing", "queued", "running"}
|
||||
]
|
||||
return sorted(items, key=lambda item: (-priority_score.get(item["priority"], 1), item["create_time"]), reverse=False)
|
||||
|
||||
def replicas(self, node_id: str) -> list[dict[str, Any]]:
|
||||
with self.connect() as conn:
|
||||
|
||||
@@ -76,6 +76,9 @@ CREATE TABLE IF NOT EXISTS compute_nodes (
|
||||
data_root TEXT NOT NULL,
|
||||
model_root TEXT NOT NULL,
|
||||
log_root TEXT NOT NULL,
|
||||
api_version TEXT NOT NULL DEFAULT 'v1',
|
||||
capabilities TEXT NOT NULL DEFAULT '[]',
|
||||
description TEXT,
|
||||
last_health_check_at TEXT,
|
||||
health_detail TEXT NOT NULL
|
||||
);
|
||||
@@ -88,7 +91,8 @@ CREATE TABLE IF NOT EXISTS gpus (
|
||||
name TEXT NOT NULL,
|
||||
memory_total_gb DOUBLE PRECISION NOT NULL,
|
||||
power_limit_w DOUBLE PRECISION NOT NULL,
|
||||
base_temperature INTEGER NOT NULL
|
||||
base_temperature INTEGER NOT NULL,
|
||||
last_seen_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
||||
@@ -103,7 +107,8 @@ CREATE TABLE IF NOT EXISTS fine_tune_tasks (
|
||||
completed_at TEXT,
|
||||
compute_node_id TEXT REFERENCES compute_nodes(id) ON DELETE SET NULL,
|
||||
gpus TEXT NOT NULL,
|
||||
sync_job_id TEXT
|
||||
sync_job_id TEXT,
|
||||
compute_job_id TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resource_replicas (
|
||||
@@ -127,7 +132,40 @@ CREATE TABLE IF NOT EXISTS resource_sync_jobs (
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_dimensions (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 1,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compare_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
create_time TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_job ON fine_tune_tasks(compute_job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_compute_node_status ON fine_tune_tasks(compute_node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_gpus_node_index ON gpus(node_id, gpu_index);
|
||||
CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_replicas_node_resource ON resource_replicas(node_id, resource_type, resource_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sync_jobs_node_status ON resource_sync_jobs(target_node_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_tasks_status ON eval_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_eval_dimensions_active ON eval_dimensions(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_compare_tasks_status ON compare_tasks(status);
|
||||
|
||||
235
backend/app/db/sql/002_data_process.sql
Normal file
235
backend/app/db/sql/002_data_process.sql
Normal file
@@ -0,0 +1,235 @@
|
||||
-- Data processing migration.
|
||||
--
|
||||
-- IMPORTANT: This file is intentionally NOT wired into application startup.
|
||||
-- Apply it explicitly in a controlled deployment, or call
|
||||
-- DataProcessStore.ensure_schema() from an administrative command.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- This migration targets the current runtime schema created by
|
||||
-- 001_platform_runtime.sql. Refuse the UUID/JSONB target-design schema instead
|
||||
-- of partially altering it with incompatible TEXT foreign keys.
|
||||
DO $$
|
||||
DECLARE
|
||||
datasets_id_type TEXT;
|
||||
BEGIN
|
||||
SELECT format_type(a.atttypid, a.atttypmod)
|
||||
INTO datasets_id_type
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname = 'datasets'
|
||||
AND a.attname = 'id'
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped;
|
||||
IF datasets_id_type IS NULL THEN
|
||||
RAISE EXCEPTION '002_data_process.sql requires 001_platform_runtime.sql first';
|
||||
END IF;
|
||||
IF datasets_id_type <> 'text' THEN
|
||||
RAISE EXCEPTION
|
||||
'002_data_process.sql supports only the current TEXT runtime schema; found datasets.id type %',
|
||||
datasets_id_type;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS owner_id TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE datasets ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS current_version_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS size_bytes BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS record_count BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS file_format VARCHAR(40);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS checksum_sha256 CHAR(64);
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS version_no INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS tenant_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS project_id TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_by TEXT;
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
name VARCHAR(150) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'running', 'completed', 'failed', 'stopped')),
|
||||
process_type VARCHAR(20) NOT NULL
|
||||
CHECK (process_type IN ('structured', 'unstructured', 'external')),
|
||||
source_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
output_dataset_id TEXT REFERENCES datasets(id) ON DELETE SET NULL,
|
||||
config TEXT NOT NULL DEFAULT '{}',
|
||||
progress NUMERIC(5,2) NOT NULL DEFAULT 0 CHECK (progress >= 0 AND progress <= 100),
|
||||
input_count BIGINT NOT NULL DEFAULT 0 CHECK (input_count >= 0),
|
||||
output_count BIGINT NOT NULL DEFAULT 0 CHECK (output_count >= 0),
|
||||
filtered_count BIGINT NOT NULL DEFAULT 0 CHECK (filtered_count >= 0),
|
||||
duplicate_count BIGINT NOT NULL DEFAULT 0 CHECK (duplicate_count >= 0),
|
||||
error_count BIGINT NOT NULL DEFAULT 0 CHECK (error_count >= 0),
|
||||
failure_reason TEXT,
|
||||
generation_run_id TEXT,
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
owner_id TEXT,
|
||||
approval_status VARCHAR(30) NOT NULL DEFAULT 'not_required',
|
||||
created_by TEXT,
|
||||
updated_by TEXT,
|
||||
deleted_by TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
ALTER TABLE data_process_tasks ADD COLUMN IF NOT EXISTS generation_run_id TEXT;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_tasks_name_alive
|
||||
ON data_process_tasks(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_scope_status
|
||||
ON data_process_tasks(tenant_id, project_id, status, created_at DESC)
|
||||
WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_tasks_creator_created
|
||||
ON data_process_tasks(created_by, created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_source_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
storage_object_id TEXT,
|
||||
name TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
file_format VARCHAR(40),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
version_no INTEGER NOT NULL DEFAULT 1 CHECK (version_no > 0),
|
||||
content TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
tenant_id TEXT,
|
||||
project_id TEXT,
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_source_files_task
|
||||
ON data_process_source_files(task_id, created_at) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_data_process_source_checksum_alive
|
||||
ON data_process_source_files(task_id, checksum_sha256) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_preview_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
source_file_id TEXT REFERENCES data_process_source_files(id) ON DELETE CASCADE,
|
||||
original_content TEXT NOT NULL DEFAULT '',
|
||||
edited_content TEXT NOT NULL DEFAULT '',
|
||||
source_start INTEGER CHECK (source_start IS NULL OR source_start >= 0),
|
||||
source_end INTEGER CHECK (source_end IS NULL OR source_end >= 0),
|
||||
source_start_line INTEGER CHECK (source_start_line IS NULL OR source_start_line > 0),
|
||||
source_end_line INTEGER CHECK (source_end_line IS NULL OR source_end_line > 0),
|
||||
token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'original'
|
||||
CHECK (status IN ('original', 'modified', 'manual', 'invalid')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CHECK (source_start IS NULL OR source_end IS NULL OR source_end >= source_start),
|
||||
CHECK (source_start_line IS NULL OR source_end_line IS NULL OR source_end_line >= source_start_line)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_preview_task_file
|
||||
ON data_process_preview_items(task_id, source_file_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_process_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL REFERENCES data_process_tasks(id) ON DELETE CASCADE,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
instruction TEXT NOT NULL,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL,
|
||||
original_instruction TEXT,
|
||||
original_input TEXT,
|
||||
original_output TEXT,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
error TEXT,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
quality_score TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||
ON data_process_results(task_id, status, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||
ON data_process_results(task_id, split);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_file_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_file_id TEXT NOT NULL REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_no INTEGER NOT NULL CHECK (version_no > 0),
|
||||
storage_object_id TEXT NOT NULL,
|
||||
content_preview TEXT,
|
||||
description TEXT,
|
||||
base_version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE SET NULL,
|
||||
size_bytes BIGINT NOT NULL DEFAULT 0 CHECK (size_bytes >= 0),
|
||||
record_count BIGINT NOT NULL DEFAULT 0 CHECK (record_count >= 0),
|
||||
checksum_sha256 CHAR(64) NOT NULL,
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_file_versions ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_dataset_file_versions_no_002
|
||||
ON dataset_file_versions(dataset_file_id, version_no);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_file_versions_source_task_002
|
||||
ON dataset_file_versions(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dataset_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
dataset_id TEXT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
dataset_file_id TEXT REFERENCES dataset_files(id) ON DELETE CASCADE,
|
||||
version_id TEXT REFERENCES dataset_file_versions(id) ON DELETE CASCADE,
|
||||
line_no INTEGER,
|
||||
split VARCHAR(20) CHECK (split IS NULL OR split IN ('train', 'validation', 'test')),
|
||||
instruction TEXT,
|
||||
input TEXT,
|
||||
output TEXT,
|
||||
raw TEXT NOT NULL DEFAULT '{}',
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||
source_task_id TEXT REFERENCES data_process_tasks(id) ON DELETE SET NULL,
|
||||
source_result_id TEXT REFERENCES data_process_results(id) ON DELETE SET NULL,
|
||||
preview_item_id TEXT REFERENCES data_process_preview_items(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_task_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS source_result_id TEXT;
|
||||
ALTER TABLE dataset_records ADD COLUMN IF NOT EXISTS preview_item_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_dataset_002
|
||||
ON dataset_records(dataset_id, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_records_source_task_002
|
||||
ON dataset_records(source_task_id, source_result_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_datasets_source_task_002
|
||||
ON datasets(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_dataset_files_source_task_002
|
||||
ON dataset_files(source_task_id) WHERE source_task_id IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
@@ -1,9 +1,13 @@
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api.v1.router import api_router
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import configure_logging, setup_request_logging
|
||||
from app.workers.compute_poller import run_compute_poller
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@@ -20,6 +24,19 @@ def create_app() -> FastAPI:
|
||||
)
|
||||
setup_request_logging(app)
|
||||
app.include_router(api_router, prefix=settings.route_prefix)
|
||||
|
||||
@app.on_event("startup")
|
||||
async def start_workers() -> None:
|
||||
app.state.compute_poller_task = asyncio.create_task(run_compute_poller())
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def stop_workers() -> None:
|
||||
task = getattr(app.state, "compute_poller_task", None)
|
||||
if task:
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
206
backend/app/modules/compute_gateway/client.py
Normal file
206
backend/app/modules/compute_gateway/client.py
Normal file
@@ -0,0 +1,206 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
def _join_url(base_url: str, path: str) -> str:
|
||||
return urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
|
||||
|
||||
|
||||
def _unwrap_items(payload: Any) -> list[dict[str, Any]]:
|
||||
if isinstance(payload, list):
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
if isinstance(payload, dict):
|
||||
data = payload.get("data")
|
||||
if isinstance(data, dict) and isinstance(data.get("items"), list):
|
||||
return [item for item in data["items"] if isinstance(item, dict)]
|
||||
if isinstance(payload.get("items"), list):
|
||||
return [item for item in payload["items"] if isinstance(item, dict)]
|
||||
if isinstance(data, list):
|
||||
return [item for item in data if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _unwrap_dict(payload: Any) -> dict[str, Any]:
|
||||
if isinstance(payload, dict) and isinstance(payload.get("data"), dict):
|
||||
return payload["data"]
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
class ComputeNodeClient:
|
||||
"""Application-side client for one compute node.
|
||||
|
||||
The client accepts both current YG Compute API responses and common
|
||||
wrapper shapes such as `{code,message,data}` to make future engine/node
|
||||
adapters less brittle.
|
||||
"""
|
||||
|
||||
def __init__(self, api_base_url: str, token: str | None = None, timeout: float | None = None) -> None:
|
||||
settings = get_settings()
|
||||
self.api_base_url = api_base_url.rstrip("/")
|
||||
self.token = token or settings.compute_service_token
|
||||
self.timeout = timeout or settings.compute_request_timeout_seconds
|
||||
self.route_prefix = settings.route_prefix.rstrip("/") or "/modelTF"
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
if not self.token:
|
||||
return {}
|
||||
return {"X-Compute-Token": self.token}
|
||||
|
||||
async def test_connection(self) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
health = await self.health()
|
||||
gpus = await self.gpus()
|
||||
return {
|
||||
"success": True,
|
||||
"latency_ms": int((time.perf_counter() - started) * 1000),
|
||||
"health": health,
|
||||
"gpus": gpus,
|
||||
}
|
||||
|
||||
async def health(self) -> dict[str, Any]:
|
||||
paths = [f"{self.route_prefix}/v1/compute/health", f"{self.route_prefix}/health", "/health"]
|
||||
last_error = ""
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
for path in paths:
|
||||
try:
|
||||
response = await client.get(_join_url(self.api_base_url, path))
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
except Exception as exc: # noqa: BLE001 - keep endpoint compatibility fallback broad
|
||||
last_error = str(exc)
|
||||
raise RuntimeError(last_error or "compute health check failed")
|
||||
|
||||
async def gpus(self) -> list[dict[str, Any]]:
|
||||
paths = [
|
||||
f"{self.route_prefix}/compute/resources/gpus",
|
||||
f"{self.route_prefix}/v1/compute/resources/gpus",
|
||||
"/compute/resources/gpus",
|
||||
]
|
||||
last_error = ""
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
for path in paths:
|
||||
try:
|
||||
response = await client.get(_join_url(self.api_base_url, path))
|
||||
response.raise_for_status()
|
||||
return _unwrap_items(response.json())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = str(exc)
|
||||
raise RuntimeError(last_error or "compute gpu discovery failed")
|
||||
|
||||
async def create_job(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs"), json=payload)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def preview_job(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/preview"),
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def validate_job(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/validate"),
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def check_paths(self, paths: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/check-paths"),
|
||||
json={"paths": paths},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def list_files(
|
||||
self,
|
||||
root: str = "data",
|
||||
relative_path: str = "",
|
||||
directories_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.get(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/list"),
|
||||
params={"root": root, "relative_path": relative_path, "directories_only": directories_only},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.get(_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/{job_id}"))
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def stop_job(self, job_id: str) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/{job_id}/stop"))
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def job_logs(
|
||||
self,
|
||||
job_id: str,
|
||||
tail_lines: int | None = None,
|
||||
offset: int | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params = {
|
||||
key: value
|
||||
for key, value in {"tail_lines": tail_lines, "offset": offset, "limit": limit}.items()
|
||||
if value is not None
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.get(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/jobs/{job_id}/logs"),
|
||||
params=params,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def import_local_file(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/import-local"),
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
filename: str,
|
||||
content: bytes,
|
||||
target_relative_path: str,
|
||||
resource_type: str | None = None,
|
||||
resource_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
data = {
|
||||
"target_relative_path": target_relative_path,
|
||||
"resource_type": resource_type or "",
|
||||
"resource_id": resource_id or "",
|
||||
}
|
||||
files = {"file": (filename, content)}
|
||||
async with httpx.AsyncClient(timeout=max(self.timeout, 60), headers=self.headers()) as client:
|
||||
response = await client.post(
|
||||
_join_url(self.api_base_url, f"{self.route_prefix}/compute/files/upload"),
|
||||
data=data,
|
||||
files=files,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return _unwrap_dict(response.json())
|
||||
27
backend/app/modules/compute_gateway/sync.py
Normal file
27
backend/app/modules/compute_gateway/sync.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.db.platform_store import get_platform_store
|
||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||
|
||||
|
||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||
|
||||
|
||||
async def poll_compute_jobs_once() -> dict[str, Any]:
|
||||
store = get_platform_store()
|
||||
synced: list[dict[str, Any]] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for task in store.running_compute_tasks():
|
||||
node = _node_for_task(task)
|
||||
if not node:
|
||||
failed.append({"task_id": task["id"], "error": "compute node not found"})
|
||||
continue
|
||||
try:
|
||||
job = await ComputeNodeClient(node["api_base_url"]).get_job(task["compute_job_id"])
|
||||
synced.append(store.apply_compute_job(task["id"], job))
|
||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||
return {"synced": len(synced), "failed": failed, "items": synced}
|
||||
912
backend/app/modules/data_process/algorithms.py
Normal file
912
backend/app/modules/data_process/algorithms.py
Normal file
@@ -0,0 +1,912 @@
|
||||
"""数据处理模块使用的无副作用算法。
|
||||
|
||||
本模块不访问数据库、文件系统或网络,便于 API、后台任务和测试共同复用。
|
||||
所有偏移量均为 Python 字符串偏移量,``TextChunk.content`` 始终等于
|
||||
``source[chunk.start:chunk.end]``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from bisect import bisect_left
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
TextFormat = Literal["json", "jsonl", "csv", "markdown", "txt"]
|
||||
ChunkMethod = Literal["semantic", "heading", "fixed", "custom"]
|
||||
DatasetSplit = Literal["train", "validation", "test"]
|
||||
|
||||
SUPPORTED_TEXT_FORMATS: tuple[TextFormat, ...] = (
|
||||
"json",
|
||||
"jsonl",
|
||||
"csv",
|
||||
"markdown",
|
||||
"txt",
|
||||
)
|
||||
|
||||
_FORMAT_ALIASES: dict[str, TextFormat] = {
|
||||
"json": "json",
|
||||
"jsonl": "jsonl",
|
||||
"ndjson": "jsonl",
|
||||
"csv": "csv",
|
||||
"tsv": "csv",
|
||||
"md": "markdown",
|
||||
"markdown": "markdown",
|
||||
"txt": "txt",
|
||||
"text": "txt",
|
||||
}
|
||||
|
||||
_EMAIL_PATTERN = re.compile(
|
||||
r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+"
|
||||
r"@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
|
||||
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+(?![\w.-])"
|
||||
)
|
||||
_PHONE_PATTERN = re.compile(r"(?<!\d)(?:(?:\+|00)?86[-\s]?)?1[3-9]\d{9}(?!\d)")
|
||||
_ID_CARD_PATTERN = re.compile(r"(?<!\d)(?:\d{17}[\dXx]|\d{15})(?!\d)")
|
||||
_TOKEN_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]")
|
||||
_HEADING_PATTERN = re.compile(
|
||||
r"(?m)^(?:#{1,6}\s+|第[一二三四五六七八九十百千万0-9]+[章节篇部分]\s*|"
|
||||
r"\d+(?:\.\d+)*[、.\s]+)"
|
||||
)
|
||||
_SEMANTIC_BOUNDARY_PATTERN = re.compile(r"\n\s*\n|[。!?!?;;](?:[\"'”’)】》]*)|\.(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParsedText:
|
||||
"""UTF-8 文本的解析结果。"""
|
||||
|
||||
format: TextFormat
|
||||
text: str
|
||||
records: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TextChunk:
|
||||
"""带有可追溯来源位置的非结构化文本切片。"""
|
||||
|
||||
content: str
|
||||
start: int
|
||||
end: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
token_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class QualityScore:
|
||||
"""标准 instruction/input/output 记录的可解释质量分。"""
|
||||
|
||||
overall: float
|
||||
completeness: float
|
||||
length: float
|
||||
readability: float
|
||||
relevance: float
|
||||
duplicate: float
|
||||
is_valid: bool
|
||||
flags: tuple[str, ...]
|
||||
fingerprint: str
|
||||
|
||||
|
||||
def decode_utf8(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""严格解码 UTF-8 文本,并移除可选 BOM。
|
||||
|
||||
不使用 ``errors='replace'``,避免上传内容损坏后仍被静默接收。
|
||||
"""
|
||||
|
||||
if isinstance(raw, str):
|
||||
return raw.removeprefix("\ufeff")
|
||||
if not isinstance(raw, (bytes, bytearray, memoryview)):
|
||||
raise TypeError("raw must be bytes-like or str")
|
||||
try:
|
||||
return bytes(raw).decode("utf-8-sig")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"content is not valid UTF-8 at byte {exc.start}") from exc
|
||||
|
||||
|
||||
def parse_utf8_text(raw: bytes | bytearray | memoryview | str) -> str:
|
||||
"""``decode_utf8`` 的语义化别名,供上传服务直接调用。"""
|
||||
|
||||
return decode_utf8(raw)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""规范 Unicode、换行和行尾空白,同时保留段落结构。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
normalized = unicodedata.normalize("NFKC", text.removeprefix("\ufeff"))
|
||||
normalized = normalized.replace("\r\n", "\n").replace("\r", "\n")
|
||||
normalized = "".join(
|
||||
char
|
||||
for char in normalized
|
||||
if char in {"\n", "\t"} or not unicodedata.category(char).startswith("C")
|
||||
)
|
||||
lines = [re.sub(r"[\t \f\v]+$", "", line) for line in normalized.split("\n")]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _normalize_format(value: str | None) -> TextFormat | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip().lower().removeprefix(".")
|
||||
try:
|
||||
return _FORMAT_ALIASES[normalized]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unsupported text format: {value}") from exc
|
||||
|
||||
|
||||
def detect_text_format(
|
||||
*,
|
||||
filename: str | None = None,
|
||||
text: str = "",
|
||||
file_format: str | None = None,
|
||||
) -> TextFormat:
|
||||
"""按显式格式、扩展名和内容特征依次识别文本格式。"""
|
||||
|
||||
explicit = _normalize_format(file_format)
|
||||
if explicit:
|
||||
return explicit
|
||||
|
||||
if filename:
|
||||
suffix = Path(filename).suffix.lower().removeprefix(".")
|
||||
detected = _FORMAT_ALIASES.get(suffix)
|
||||
if detected:
|
||||
return detected
|
||||
|
||||
stripped = text.strip()
|
||||
if stripped:
|
||||
if stripped[0] in "[{":
|
||||
try:
|
||||
json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "json"
|
||||
|
||||
nonempty_lines = [line for line in stripped.splitlines() if line.strip()]
|
||||
if len(nonempty_lines) > 1:
|
||||
try:
|
||||
for line in nonempty_lines:
|
||||
json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return "jsonl"
|
||||
|
||||
if re.search(r"(?m)^(?:#{1,6}\s+|```|~~~)", stripped) or re.search(
|
||||
r"(?m)^\s*\|.+\|\s*$", stripped
|
||||
):
|
||||
return "markdown"
|
||||
|
||||
sample = stripped[:8192]
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(sample, delimiters=",\t;")
|
||||
rows = list(csv.reader(io.StringIO(sample), dialect))
|
||||
if len(rows) >= 2 and len(rows[0]) >= 2:
|
||||
return "csv"
|
||||
except csv.Error:
|
||||
pass
|
||||
|
||||
return "txt"
|
||||
|
||||
|
||||
def _normalize_value(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return normalize_text(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {normalize_text(str(key)): _normalize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_normalize_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _record_from_value(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return dict(_normalize_value(value))
|
||||
return {"value": _normalize_value(value)}
|
||||
|
||||
|
||||
def extract_structured_records(text: str, file_format: str) -> list[dict[str, Any]]:
|
||||
"""从 JSON、JSONL 或 CSV 中提取规范化记录。
|
||||
|
||||
JSON 顶层对象若包含 ``records/data/items/rows`` 数组,则提取该数组;
|
||||
其他顶层对象视为单条记录。标量会稳定包装为 ``{"value": ...}``。
|
||||
"""
|
||||
|
||||
normalized_format = _normalize_format(file_format)
|
||||
if normalized_format not in {"json", "jsonl", "csv"}:
|
||||
raise ValueError("structured record extraction only supports JSON, JSONL and CSV")
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
if not normalized_text:
|
||||
return []
|
||||
|
||||
if normalized_format == "json":
|
||||
try:
|
||||
payload = json.loads(normalized_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
||||
values: Sequence[Any]
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("records", "data", "items", "rows")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
values = [payload]
|
||||
return [_record_from_value(value) for value in values]
|
||||
|
||||
if normalized_format == "jsonl":
|
||||
records: list[dict[str, Any]] = []
|
||||
for line_number, line in enumerate(normalized_text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL at line {line_number}, "
|
||||
f"column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
records.append(_record_from_value(value))
|
||||
return records
|
||||
|
||||
try:
|
||||
dialect = csv.Sniffer().sniff(normalized_text[:8192], delimiters=",\t;")
|
||||
except csv.Error:
|
||||
dialect = csv.excel
|
||||
reader = csv.DictReader(io.StringIO(normalized_text), dialect=dialect)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError("CSV header is required")
|
||||
headers = [normalize_text(header or "") for header in reader.fieldnames]
|
||||
if any(not header for header in headers):
|
||||
raise ValueError("CSV header cannot be empty")
|
||||
if len(set(headers)) != len(headers):
|
||||
raise ValueError("CSV headers must be unique")
|
||||
reader.fieldnames = headers
|
||||
|
||||
records = []
|
||||
for row in reader:
|
||||
if None in row:
|
||||
raise ValueError("CSV row has more fields than the header")
|
||||
normalized_row = {
|
||||
key: normalize_text(value or "")
|
||||
for key, value in row.items()
|
||||
}
|
||||
if any(value for value in normalized_row.values()):
|
||||
records.append(normalized_row)
|
||||
return records
|
||||
|
||||
|
||||
def parse_text_content(
|
||||
raw: bytes | bytearray | memoryview | str,
|
||||
*,
|
||||
filename: str | None = None,
|
||||
file_format: str | None = None,
|
||||
) -> ParsedText:
|
||||
"""严格解码并解析支持的 UTF-8 文本格式。"""
|
||||
|
||||
text = normalize_text(decode_utf8(raw))
|
||||
detected_format = detect_text_format(filename=filename, text=text, file_format=file_format)
|
||||
records: list[dict[str, Any]] = []
|
||||
if detected_format in {"json", "jsonl", "csv"}:
|
||||
records = extract_structured_records(text, detected_format)
|
||||
return ParsedText(format=detected_format, text=text, records=tuple(records))
|
||||
|
||||
|
||||
def desensitize_pii(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""掩码邮箱、中国大陆手机号和 15/18 位身份证号,并返回命中统计。"""
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("text must be str")
|
||||
counts: dict[str, int] = {"email": 0, "phone": 0, "id_card": 0}
|
||||
|
||||
def replace(pattern: re.Pattern[str], replacement: str, kind: str, value: str) -> str:
|
||||
def replacer(_: re.Match[str]) -> str:
|
||||
counts[kind] += 1
|
||||
return replacement
|
||||
|
||||
return pattern.sub(replacer, value)
|
||||
|
||||
masked = replace(_EMAIL_PATTERN, "[EMAIL]", "email", text)
|
||||
masked = replace(_ID_CARD_PATTERN, "[ID_CARD]", "id_card", masked)
|
||||
masked = replace(_PHONE_PATTERN, "[PHONE]", "phone", masked)
|
||||
counts["total"] = sum(counts.values())
|
||||
return masked, counts
|
||||
|
||||
|
||||
def estimate_token_count(text: str) -> int:
|
||||
"""无分词器依赖的确定性 token 估算,用于预览与保护性限流。"""
|
||||
|
||||
return len(_TOKEN_PATTERN.findall(text))
|
||||
|
||||
|
||||
def _token_spans(text: str) -> list[tuple[int, int]]:
|
||||
return [match.span() for match in _TOKEN_PATTERN.finditer(text)]
|
||||
|
||||
|
||||
def _line_number(newline_offsets: list[int], offset: int) -> int:
|
||||
# 换行符本身仍属于上一行;只有严格位于 offset 之前的换行才推进行号。
|
||||
return bisect_left(newline_offsets, offset) + 1
|
||||
|
||||
|
||||
def _token_index_at_or_after(spans: list[tuple[int, int]], offset: int) -> int:
|
||||
starts = [span[0] for span in spans]
|
||||
return bisect_left(starts, offset)
|
||||
|
||||
|
||||
def _protected_markdown_ranges(
|
||||
text: str,
|
||||
*,
|
||||
preserve_code_blocks: bool,
|
||||
preserve_tables: bool,
|
||||
preserve_lists: bool,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""找出不应从中间切开的 Markdown 代码块、表格和连续列表。"""
|
||||
|
||||
lines: list[tuple[int, int, str]] = []
|
||||
cursor = 0
|
||||
for raw_line in text.splitlines(keepends=True):
|
||||
end = cursor + len(raw_line)
|
||||
lines.append((cursor, end, raw_line.rstrip("\r\n")))
|
||||
cursor = end
|
||||
if cursor < len(text) or not lines:
|
||||
lines.append((cursor, len(text), text[cursor:]))
|
||||
|
||||
ranges: list[tuple[int, int]] = []
|
||||
code_line_indexes: set[int] = set()
|
||||
if preserve_code_blocks:
|
||||
open_block: tuple[int, str, int] | None = None
|
||||
for index, (start, end, content) in enumerate(lines):
|
||||
fence = re.match(r"^\s*(`{3,}|~{3,})", content)
|
||||
if not fence:
|
||||
continue
|
||||
marker = fence.group(1)[0]
|
||||
length = len(fence.group(1))
|
||||
if open_block is None:
|
||||
open_block = (index, marker, length)
|
||||
continue
|
||||
first_index, open_marker, open_length = open_block
|
||||
if marker == open_marker and length >= open_length:
|
||||
ranges.append((lines[first_index][0], end))
|
||||
code_line_indexes.update(range(first_index, index + 1))
|
||||
open_block = None
|
||||
if open_block is not None:
|
||||
first_index = open_block[0]
|
||||
ranges.append((lines[first_index][0], len(text)))
|
||||
code_line_indexes.update(range(first_index, len(lines)))
|
||||
|
||||
if preserve_tables:
|
||||
index = 0
|
||||
while index + 1 < len(lines):
|
||||
if index in code_line_indexes:
|
||||
index += 1
|
||||
continue
|
||||
header = lines[index][2].strip()
|
||||
separator = lines[index + 1][2].strip().strip("|")
|
||||
cells = [cell.strip() for cell in separator.split("|")]
|
||||
if (
|
||||
"|" not in header
|
||||
or len(cells) < 2
|
||||
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index + 1
|
||||
while (
|
||||
end_index + 1 < len(lines)
|
||||
and end_index + 1 not in code_line_indexes
|
||||
and lines[end_index + 1][2].strip()
|
||||
and "|" in lines[end_index + 1][2]
|
||||
):
|
||||
end_index += 1
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
if preserve_lists:
|
||||
list_pattern = re.compile(r"^\s*(?:[-+*]|\d+[.)])\s+\S")
|
||||
continuation_pattern = re.compile(r"^\s{2,}\S")
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
if index in code_line_indexes or not list_pattern.match(lines[index][2]):
|
||||
index += 1
|
||||
continue
|
||||
end_index = index
|
||||
item_count = 1
|
||||
while end_index + 1 < len(lines) and end_index + 1 not in code_line_indexes:
|
||||
next_line = lines[end_index + 1][2]
|
||||
if list_pattern.match(next_line):
|
||||
item_count += 1
|
||||
end_index += 1
|
||||
elif continuation_pattern.match(next_line):
|
||||
end_index += 1
|
||||
else:
|
||||
break
|
||||
if item_count >= 2:
|
||||
ranges.append((lines[index][0], lines[end_index][1]))
|
||||
index = end_index + 1
|
||||
|
||||
merged: list[tuple[int, int]] = []
|
||||
for start, end in sorted(ranges):
|
||||
if merged and start < merged[-1][1]:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
|
||||
def _range_containing(
|
||||
ranges: Sequence[tuple[int, int]], offset: int
|
||||
) -> tuple[int, int] | None:
|
||||
return next((item for item in ranges if item[0] < offset < item[1]), None)
|
||||
|
||||
|
||||
def _boundary_for_method(
|
||||
text: str,
|
||||
spans: list[tuple[int, int]],
|
||||
start_index: int,
|
||||
ideal_end_index: int,
|
||||
minimum_end_index: int,
|
||||
method: ChunkMethod,
|
||||
custom_delimiter: str,
|
||||
) -> tuple[int, int | None]:
|
||||
if method == "fixed":
|
||||
return ideal_end_index, None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
ideal_end_offset = spans[ideal_end_index - 1][1]
|
||||
minimum_end_offset = spans[minimum_end_index - 1][1]
|
||||
search_text = text[start_offset:ideal_end_offset]
|
||||
|
||||
if method == "custom":
|
||||
delimiter = custom_delimiter.replace("\\n", "\n").replace("\\t", "\t")
|
||||
if not delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
relative_minimum = max(0, minimum_end_offset - start_offset)
|
||||
delimiter_start = search_text.rfind(delimiter, relative_minimum)
|
||||
if delimiter_start >= 0:
|
||||
boundary_offset = start_offset + delimiter_start + len(delimiter)
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
if method == "heading":
|
||||
heading_offsets = [
|
||||
start_offset + match.start()
|
||||
for match in _HEADING_PATTERN.finditer(search_text)
|
||||
if start_offset + match.start() >= minimum_end_offset
|
||||
]
|
||||
if heading_offsets:
|
||||
boundary_offset = heading_offsets[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if start_index < boundary_index <= ideal_end_index:
|
||||
return boundary_index, boundary_offset
|
||||
|
||||
semantic_boundaries = [
|
||||
start_offset + match.end()
|
||||
for match in _SEMANTIC_BOUNDARY_PATTERN.finditer(search_text)
|
||||
if start_offset + match.end() >= minimum_end_offset
|
||||
]
|
||||
if semantic_boundaries:
|
||||
boundary_offset = semantic_boundaries[-1]
|
||||
boundary_index = _token_index_at_or_after(spans, boundary_offset)
|
||||
if boundary_index > start_index:
|
||||
return min(boundary_index, ideal_end_index), boundary_offset
|
||||
return ideal_end_index, None
|
||||
|
||||
|
||||
def chunk_unstructured(
|
||||
text: str,
|
||||
*,
|
||||
method: ChunkMethod = "semantic",
|
||||
chunk_size: int = 800,
|
||||
chunk_overlap: int = 100,
|
||||
min_chunk_size: int = 100,
|
||||
custom_delimiter: str = "",
|
||||
preserve_code_blocks: bool = False,
|
||||
preserve_tables: bool = False,
|
||||
preserve_lists: bool = False,
|
||||
) -> list[TextChunk]:
|
||||
"""按估算 token 切分非结构化文本。
|
||||
|
||||
overlap 足够时精确保留配置数量;短边界下会自动收缩,并且每轮至少推进
|
||||
一个 token,避免异常配置或分隔符造成死循环。
|
||||
"""
|
||||
|
||||
if method not in {"semantic", "heading", "fixed", "custom"}:
|
||||
raise ValueError(f"unsupported chunk method: {method}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if min_chunk_size <= 0 or min_chunk_size > chunk_size:
|
||||
raise ValueError("min_chunk_size must be in [1, chunk_size]")
|
||||
if chunk_overlap + min_chunk_size > chunk_size:
|
||||
raise ValueError("chunk_overlap + min_chunk_size cannot exceed chunk_size")
|
||||
if method == "custom" and not custom_delimiter:
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
spans = _token_spans(normalized)
|
||||
if not spans:
|
||||
return []
|
||||
|
||||
newline_offsets = [index for index, char in enumerate(normalized) if char == "\n"]
|
||||
protected_ranges = _protected_markdown_ranges(
|
||||
normalized,
|
||||
preserve_code_blocks=preserve_code_blocks,
|
||||
preserve_tables=preserve_tables,
|
||||
preserve_lists=preserve_lists,
|
||||
)
|
||||
chunks: list[TextChunk] = []
|
||||
start_index = 0
|
||||
|
||||
while start_index < len(spans):
|
||||
ideal_end_index = min(len(spans), start_index + chunk_size)
|
||||
if ideal_end_index == len(spans):
|
||||
end_index, end_override = ideal_end_index, len(normalized)
|
||||
else:
|
||||
minimum_end_index = min(ideal_end_index, start_index + min_chunk_size)
|
||||
end_index, end_override = _boundary_for_method(
|
||||
normalized,
|
||||
spans,
|
||||
start_index,
|
||||
ideal_end_index,
|
||||
minimum_end_index,
|
||||
method,
|
||||
custom_delimiter,
|
||||
)
|
||||
if end_index <= start_index:
|
||||
end_index = min(len(spans), start_index + chunk_size)
|
||||
end_override = None
|
||||
|
||||
start_offset = spans[start_index][0]
|
||||
end_offset = end_override if end_override is not None else spans[end_index - 1][1]
|
||||
end_offset = max(spans[end_index - 1][1], min(len(normalized), end_offset))
|
||||
split_range = _range_containing(protected_ranges, end_offset)
|
||||
if split_range:
|
||||
before_index = _token_index_at_or_after(spans, split_range[0])
|
||||
if before_index - start_index >= min_chunk_size:
|
||||
end_index = before_index
|
||||
end_offset = split_range[0]
|
||||
else:
|
||||
end_index = min(
|
||||
len(spans),
|
||||
max(start_index + 1, _token_index_at_or_after(spans, split_range[1])),
|
||||
)
|
||||
end_offset = split_range[1]
|
||||
content = normalized[start_offset:end_offset]
|
||||
chunks.append(
|
||||
TextChunk(
|
||||
content=content,
|
||||
start=start_offset,
|
||||
end=end_offset,
|
||||
start_line=_line_number(newline_offsets, start_offset),
|
||||
end_line=_line_number(newline_offsets, max(start_offset, end_offset - 1)),
|
||||
token_count=end_index - start_index,
|
||||
)
|
||||
)
|
||||
|
||||
if end_index >= len(spans):
|
||||
break
|
||||
next_start = max(start_index + 1, end_index - chunk_overlap)
|
||||
overlap_range = _range_containing(protected_ranges, spans[next_start][0])
|
||||
if overlap_range:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[0])
|
||||
if candidate <= start_index:
|
||||
candidate = _token_index_at_or_after(spans, overlap_range[1])
|
||||
next_start = min(len(spans), max(start_index + 1, candidate))
|
||||
start_index = next_start
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def record_fingerprint(record: Mapping[str, Any]) -> str:
|
||||
"""计算与字典键顺序无关的稳定记录指纹。"""
|
||||
|
||||
canonical = {
|
||||
"instruction": normalize_text(str(record.get("instruction") or "")),
|
||||
"input": normalize_text(str(record.get("input") or "")),
|
||||
"output": normalize_text(str(record.get("output") or "")),
|
||||
}
|
||||
raw = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _readability_score(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
nonspace = [char for char in text if not char.isspace()]
|
||||
if not nonspace:
|
||||
return 0.0
|
||||
printable_ratio = sum(char.isprintable() for char in nonspace) / len(nonspace)
|
||||
useful_ratio = sum(
|
||||
char.isalnum() or "\u3400" <= char <= "\u9fff" or unicodedata.category(char).startswith("P")
|
||||
for char in nonspace
|
||||
) / len(nonspace)
|
||||
return round(100 * (0.65 * printable_ratio + 0.35 * useful_ratio), 2)
|
||||
|
||||
|
||||
def _internal_duplicate_score(text: str) -> float:
|
||||
units = [unit.strip().lower() for unit in re.split(r"[\n。!?!?;;]+", text) if unit.strip()]
|
||||
if len(units) <= 1:
|
||||
return 100.0
|
||||
return round(100 * len(set(units)) / len(units), 2)
|
||||
|
||||
|
||||
def _source_relevance_score(record: Mapping[str, Any], source_content: str) -> float:
|
||||
"""估算结果与来源文本的词元覆盖率。
|
||||
|
||||
这是无外部模型依赖、可重复的首版评分。没有来源文本(例如人工新增结果)
|
||||
时不扣分;存在来源时,以结果中的有效词元被来源覆盖的比例计分。
|
||||
"""
|
||||
|
||||
source = normalize_text(source_content)
|
||||
if not source:
|
||||
return 100.0
|
||||
candidate = normalize_text(
|
||||
"\n".join(
|
||||
str(record.get(field) or "") for field in ("instruction", "input", "output")
|
||||
)
|
||||
)
|
||||
|
||||
def semantic_tokens(text: str) -> set[str]:
|
||||
return {
|
||||
token.lower()
|
||||
for token in _TOKEN_PATTERN.findall(text)
|
||||
if token.isalnum() or "\u3400" <= token <= "\u9fff"
|
||||
}
|
||||
|
||||
source_tokens = semantic_tokens(source)
|
||||
candidate_tokens = semantic_tokens(candidate)
|
||||
if not candidate_tokens:
|
||||
return 0.0
|
||||
if not source_tokens:
|
||||
return 0.0
|
||||
return round(100 * len(candidate_tokens & source_tokens) / len(candidate_tokens), 2)
|
||||
|
||||
|
||||
def score_quality(
|
||||
record: Mapping[str, Any],
|
||||
*,
|
||||
min_output_length: int = 20,
|
||||
source_content: str = "",
|
||||
known_fingerprints: Iterable[str] = (),
|
||||
threshold: float = 60.0,
|
||||
) -> QualityScore:
|
||||
"""按完整性、长度、可读性、来源相关性和重复度计算质量分。"""
|
||||
|
||||
if min_output_length <= 0:
|
||||
raise ValueError("min_output_length must be greater than 0")
|
||||
if not 0 <= threshold <= 100:
|
||||
raise ValueError("threshold must be in [0, 100]")
|
||||
|
||||
instruction = normalize_text(str(record.get("instruction") or ""))
|
||||
input_text = normalize_text(str(record.get("input") or ""))
|
||||
output = normalize_text(str(record.get("output") or ""))
|
||||
flags: list[str] = []
|
||||
|
||||
completeness = 100.0
|
||||
if not instruction:
|
||||
completeness -= 50
|
||||
flags.append("missing_instruction")
|
||||
if not output:
|
||||
completeness -= 50
|
||||
flags.append("missing_output")
|
||||
|
||||
output_length = len(output)
|
||||
length_score = round(min(100.0, output_length / min_output_length * 100), 2)
|
||||
if output_length < min_output_length:
|
||||
flags.append("output_too_short")
|
||||
|
||||
readability = _readability_score("\n".join((instruction, input_text, output)))
|
||||
if readability < 70:
|
||||
flags.append("low_readability")
|
||||
|
||||
relevance = _source_relevance_score(record, source_content)
|
||||
if source_content and relevance < 30:
|
||||
flags.append("low_source_relevance")
|
||||
|
||||
fingerprint = record_fingerprint(record)
|
||||
known = set(known_fingerprints)
|
||||
duplicate = 0.0 if fingerprint in known else _internal_duplicate_score(output)
|
||||
if duplicate == 0:
|
||||
flags.append("duplicate_record")
|
||||
elif duplicate < 70:
|
||||
flags.append("repetitive_output")
|
||||
|
||||
overall = round(
|
||||
completeness * 0.35
|
||||
+ length_score * 0.20
|
||||
+ readability * 0.20
|
||||
+ relevance * 0.15
|
||||
+ duplicate * 0.10,
|
||||
2,
|
||||
)
|
||||
hard_valid = bool(instruction and output)
|
||||
return QualityScore(
|
||||
overall=overall,
|
||||
completeness=completeness,
|
||||
length=length_score,
|
||||
readability=readability,
|
||||
relevance=relevance,
|
||||
duplicate=duplicate,
|
||||
is_valid=hard_valid and overall >= threshold,
|
||||
flags=tuple(flags),
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def stable_split(
|
||||
value: str | int,
|
||||
split: Mapping[str, int] | None = None,
|
||||
*,
|
||||
seed: str = "",
|
||||
) -> DatasetSplit:
|
||||
"""按稳定哈希将记录划分到 train/validation/test。"""
|
||||
|
||||
ratios = dict(split or {"train": 80, "validation": 10, "test": 10})
|
||||
required = {"train", "validation", "test"}
|
||||
if set(ratios) != required:
|
||||
raise ValueError("split must contain exactly train, validation and test")
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in ratios.values()):
|
||||
raise ValueError("split ratios must be non-negative integers")
|
||||
if sum(ratios.values()) != 100:
|
||||
raise ValueError("split ratios must sum to 100")
|
||||
|
||||
digest = hashlib.sha256(f"{seed}:{value}".encode("utf-8")).digest()
|
||||
bucket = int.from_bytes(digest[:8], "big") % 10_000
|
||||
train_boundary = ratios["train"] * 100
|
||||
validation_boundary = train_boundary + ratios["validation"] * 100
|
||||
if bucket < train_boundary:
|
||||
return "train"
|
||||
if bucket < validation_boundary:
|
||||
return "validation"
|
||||
return "test"
|
||||
|
||||
|
||||
def _preview_content(item: Mapping[str, Any]) -> str:
|
||||
for field in ("edited_content", "editedContent", "original_content", "originalContent", "content"):
|
||||
value = item.get(field)
|
||||
if value is not None:
|
||||
return normalize_text(str(value))
|
||||
return ""
|
||||
|
||||
|
||||
def _standard_fields(content: str) -> tuple[str, str, str]:
|
||||
if not content:
|
||||
return "", "", ""
|
||||
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, Mapping):
|
||||
instruction = next(
|
||||
(
|
||||
str(payload[key])
|
||||
for key in ("instruction", "question", "prompt")
|
||||
if payload.get(key) is not None
|
||||
),
|
||||
"",
|
||||
)
|
||||
input_text = next(
|
||||
(str(payload[key]) for key in ("input", "context") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
output = next(
|
||||
(str(payload[key]) for key in ("output", "answer", "response") if payload.get(key) is not None),
|
||||
"",
|
||||
)
|
||||
if instruction or output:
|
||||
return normalize_text(instruction), normalize_text(input_text), normalize_text(output)
|
||||
|
||||
question_answer = re.match(
|
||||
r"^\s*(?:问|question)\s*[::]\s*(.+?)(?:\n|\r\n?)\s*(?:答|answer)\s*[::]\s*(.+)\s*$",
|
||||
content,
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if question_answer:
|
||||
return normalize_text(question_answer.group(1)), "", normalize_text(question_answer.group(2))
|
||||
|
||||
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
||||
first_line = re.sub(r"^(?:问|question)\s*[::]\s*", "", lines[0], flags=re.IGNORECASE)
|
||||
output = normalize_text("\n".join(lines[1:])) if len(lines) > 1 else normalize_text(content)
|
||||
return normalize_text(first_line), "", output
|
||||
|
||||
|
||||
def generate_standard_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
qa_pairs_per_item: int = 1,
|
||||
semantic_enrichment: bool = False,
|
||||
split: Mapping[str, int] | None = None,
|
||||
split_seed: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""把预览内容确定性转换为标准 instruction/input/output 记录。
|
||||
|
||||
该函数只负责本地标准化,不冒充 LLM;服务层可将其作为无模型模式或
|
||||
LLM 响应解析后的统一落库步骤。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ValueError("qa_pairs_per_item must be in [1, 5]")
|
||||
prefixes = (
|
||||
"请结合实际情况说明:",
|
||||
"请用通俗易懂的方式说明:",
|
||||
"请从实际应用角度说明:",
|
||||
"请简洁自然地说明:",
|
||||
"请详细解答:",
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for item_index, item in enumerate(preview_items):
|
||||
content = _preview_content(item)
|
||||
instruction, input_text, output = _standard_fields(content)
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
for variant_index in range(qa_pairs_per_item):
|
||||
variant_instruction = instruction
|
||||
if variant_index:
|
||||
if semantic_enrichment:
|
||||
variant_instruction = f"{prefixes[variant_index]}{instruction}"
|
||||
else:
|
||||
variant_instruction = f"{instruction}(问法 {variant_index + 1})"
|
||||
raw_id = f"{preview_id}:{variant_index + 1}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode('utf-8')).hexdigest()[:16]}"
|
||||
status = "valid" if variant_instruction and output else "invalid"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": variant_instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": variant_instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": status,
|
||||
"split": stable_split(result_id, split, seed=split_seed),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChunkMethod",
|
||||
"DatasetSplit",
|
||||
"ParsedText",
|
||||
"QualityScore",
|
||||
"SUPPORTED_TEXT_FORMATS",
|
||||
"TextChunk",
|
||||
"TextFormat",
|
||||
"chunk_unstructured",
|
||||
"decode_utf8",
|
||||
"desensitize_pii",
|
||||
"detect_text_format",
|
||||
"estimate_token_count",
|
||||
"extract_structured_records",
|
||||
"generate_standard_records",
|
||||
"normalize_text",
|
||||
"parse_text_content",
|
||||
"parse_utf8_text",
|
||||
"record_fingerprint",
|
||||
"score_quality",
|
||||
"stable_split",
|
||||
]
|
||||
250
backend/app/modules/data_process/generation.py
Normal file
250
backend/app/modules/data_process/generation.py
Normal file
@@ -0,0 +1,250 @@
|
||||
"""数据处理任务的大模型生成适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from app.modules.data_process.algorithms import normalize_text, stable_split
|
||||
|
||||
|
||||
class ModelGenerationError(ValueError):
|
||||
"""模型配置、响应或调用失败。"""
|
||||
|
||||
|
||||
def chat_completions_url(value: str) -> str:
|
||||
"""把域名、基础 URL 或完整地址统一为 chat completions 地址。"""
|
||||
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
raise ModelGenerationError("generation model api_url is required")
|
||||
if "://" not in raw:
|
||||
raw = f"https://{raw}"
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ModelGenerationError("generation model api_url must be an HTTP(S) host or URL")
|
||||
if parsed.username or parsed.password:
|
||||
raise ModelGenerationError("generation model api_url must not contain credentials")
|
||||
|
||||
path = parsed.path.rstrip("/")
|
||||
if path.endswith("/chat/completions"):
|
||||
target_path = path
|
||||
elif path.endswith("/v1"):
|
||||
target_path = f"{path}/chat/completions"
|
||||
elif not path:
|
||||
target_path = "/v1/chat/completions"
|
||||
else:
|
||||
target_path = f"{path}/v1/chat/completions"
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, target_path, "", ""))
|
||||
|
||||
|
||||
def _message_content(payload: Mapping[str, Any]) -> str:
|
||||
try:
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ModelGenerationError("model response does not contain choices[0].message.content") from exc
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [
|
||||
str(item.get("text") or "")
|
||||
for item in content
|
||||
if isinstance(item, Mapping) and item.get("type") in {None, "text", "output_text"}
|
||||
]
|
||||
if parts:
|
||||
return "".join(parts)
|
||||
raise ModelGenerationError("model response content must be text")
|
||||
|
||||
|
||||
def _json_payload(content: str) -> Any:
|
||||
cleaned = re.sub(r"<think>[\s\S]*?</think>", "", content, flags=re.IGNORECASE).strip()
|
||||
fenced = re.fullmatch(r"```(?:json)?\s*([\s\S]*?)\s*```", cleaned, flags=re.IGNORECASE)
|
||||
if fenced:
|
||||
cleaned = fenced.group(1).strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ModelGenerationError(
|
||||
f"model response is not valid JSON at line {exc.lineno}, column {exc.colno}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _result_items(payload: Any) -> list[Mapping[str, Any]]:
|
||||
if isinstance(payload, list):
|
||||
values = payload
|
||||
elif isinstance(payload, Mapping):
|
||||
nested = next(
|
||||
(
|
||||
payload[key]
|
||||
for key in ("items", "results", "data", "records")
|
||||
if isinstance(payload.get(key), list)
|
||||
),
|
||||
None,
|
||||
)
|
||||
values = nested if isinstance(nested, list) else [payload]
|
||||
else:
|
||||
raise ModelGenerationError("model JSON must be an object or array")
|
||||
items = [item for item in values if isinstance(item, Mapping)]
|
||||
if not items:
|
||||
raise ModelGenerationError("model JSON does not contain result objects")
|
||||
return items
|
||||
|
||||
|
||||
def _prompt_messages(prompt: str, content: str, count: int) -> list[dict[str, str]]:
|
||||
schema_instruction = (
|
||||
f"必须只返回 JSON 对象,格式为 {{\"items\":[{{\"instruction\":\"...\","
|
||||
f"\"input\":\"...\",\"output\":\"...\"}}]}};items 必须包含 {count} 条。"
|
||||
"instruction 和 output 不得为空,不要输出 Markdown 代码围栏或分析过程。"
|
||||
)
|
||||
base_prompt = (
|
||||
normalize_text(prompt)
|
||||
or "请根据来源内容生成可用于监督微调的问答数据。"
|
||||
)
|
||||
if "{{ content }}" in base_prompt:
|
||||
user_prompt = base_prompt.replace("{{ content }}", content)
|
||||
return [
|
||||
{"role": "system", "content": schema_instruction},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
return [
|
||||
{"role": "system", "content": f"{base_prompt}\n{schema_instruction}"},
|
||||
{"role": "user", "content": f"来源内容:\n{content}"},
|
||||
]
|
||||
|
||||
|
||||
def generate_model_records(
|
||||
preview_items: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
model: Mapping[str, Any],
|
||||
config: Mapping[str, Any],
|
||||
task_id: str,
|
||||
split: Mapping[str, int],
|
||||
qa_pairs_per_item: int,
|
||||
client: httpx.Client | None = None,
|
||||
on_progress: Callable[[int, int], None] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""调用 OpenAI 兼容接口,将预览切片生成标准训练记录。
|
||||
|
||||
单条调用失败会产生可人工修复的 invalid 结果,不会丢弃整批任务。
|
||||
"""
|
||||
|
||||
if not 1 <= qa_pairs_per_item <= 5:
|
||||
raise ModelGenerationError("qa_pairs_per_item must be in [1, 5]")
|
||||
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||
if not model_name:
|
||||
raise ModelGenerationError("generation model name is required")
|
||||
|
||||
temperature = float(config.get("temperature", 0.7))
|
||||
max_tokens = int(config.get("max_tokens", 1024))
|
||||
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60))))
|
||||
retries = max(0, min(5, int(config.get("generation_retries", 2))))
|
||||
headers = {"Content-Type": "application/json"}
|
||||
api_key = str(model.get("api_key") or "").strip()
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
owns_client = client is None
|
||||
http_client = client or httpx.Client(timeout=timeout)
|
||||
results: list[dict[str, Any]] = []
|
||||
try:
|
||||
preview_list = list(preview_items)
|
||||
total_items = len(preview_list)
|
||||
for item_index, item in enumerate(preview_list):
|
||||
preview_id = str(item.get("id") or f"preview-{item_index + 1}")
|
||||
content = normalize_text(
|
||||
str(item.get("edited_content") or item.get("original_content") or "")
|
||||
)
|
||||
request_payload: dict[str, Any] = {
|
||||
"model": model_name,
|
||||
"messages": _prompt_messages(
|
||||
str(config.get("generation_prompt") or ""),
|
||||
content,
|
||||
qa_pairs_per_item,
|
||||
),
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if bool(config.get("json_mode", False)):
|
||||
request_payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
last_error: Exception | None = None
|
||||
generated_items: list[Mapping[str, Any]] | None = None
|
||||
for _ in range(retries + 1):
|
||||
try:
|
||||
response = http_client.post(endpoint, headers=headers, json=request_payload)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if not isinstance(body, Mapping):
|
||||
raise ModelGenerationError("model response body must be a JSON object")
|
||||
generated_items = _result_items(_json_payload(_message_content(body)))
|
||||
break
|
||||
except (httpx.HTTPError, json.JSONDecodeError, ModelGenerationError) as exc:
|
||||
last_error = exc
|
||||
|
||||
if generated_items is None:
|
||||
error_message = str(last_error or "model generation failed")[:2000]
|
||||
result_id = f"result_{hashlib.sha256(f'{preview_id}:error'.encode()).hexdigest()[:16]}"
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": "模型生成失败,请人工补充",
|
||||
"input": content,
|
||||
"output": "",
|
||||
"original_instruction": "模型生成失败,请人工补充",
|
||||
"original_input": content,
|
||||
"original_output": "",
|
||||
"status": "invalid",
|
||||
"error": error_message,
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
continue
|
||||
|
||||
for variant_index, value in enumerate(generated_items[:qa_pairs_per_item]):
|
||||
instruction = normalize_text(str(value.get("instruction") or value.get("question") or ""))
|
||||
input_text = normalize_text(str(value.get("input") or value.get("context") or ""))
|
||||
output = normalize_text(
|
||||
str(
|
||||
value.get("output")
|
||||
or value.get("answer")
|
||||
or value.get("response")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
raw_id = f"{preview_id}:{variant_index + 1}:{instruction}:{output}"
|
||||
result_id = f"result_{hashlib.sha256(raw_id.encode()).hexdigest()[:16]}"
|
||||
valid = bool(instruction and output)
|
||||
results.append(
|
||||
{
|
||||
"id": result_id,
|
||||
"preview_item_id": preview_id,
|
||||
"instruction": instruction,
|
||||
"input": input_text,
|
||||
"output": output,
|
||||
"original_instruction": instruction,
|
||||
"original_input": input_text,
|
||||
"original_output": output,
|
||||
"status": "valid" if valid else "invalid",
|
||||
"error": None if valid else "model result is missing instruction or output",
|
||||
"split": stable_split(result_id, split, seed=task_id),
|
||||
}
|
||||
)
|
||||
if on_progress:
|
||||
on_progress(item_index + 1, total_items)
|
||||
finally:
|
||||
if owns_client:
|
||||
http_client.close()
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["ModelGenerationError", "chat_completions_url", "generate_model_records"]
|
||||
65
backend/app/modules/data_process/schema_cli.py
Normal file
65
backend/app/modules/data_process/schema_cli.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""数据处理运行表的显式检查与安装命令。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.modules.data_process.store import DataProcessStore
|
||||
|
||||
|
||||
def _target_label(database_url: str) -> str:
|
||||
parsed = urlsplit(database_url)
|
||||
database = parsed.path.strip("/") or "(unknown)"
|
||||
return f"{parsed.hostname or '(unknown)'}:{parsed.port or 5432}/{database}"
|
||||
|
||||
|
||||
def _schema_ready(store: DataProcessStore) -> bool:
|
||||
with store.connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema=current_schema()
|
||||
AND table_name='data_process_tasks'
|
||||
AND column_name='generation_run_id'
|
||||
) AS ready
|
||||
"""
|
||||
).fetchone()
|
||||
return bool(row and row["ready"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="检查或显式安装数据处理运行表(不会由应用启动自动执行)"
|
||||
)
|
||||
action = parser.add_mutually_exclusive_group(required=True)
|
||||
action.add_argument("--check", action="store_true", help="只读检查迁移是否已安装")
|
||||
action.add_argument("--apply", action="store_true", help="执行 002 数据处理迁移")
|
||||
parser.add_argument(
|
||||
"--yes",
|
||||
action="store_true",
|
||||
help="确认允许修改 DATABASE_URL 指向的数据库;与 --apply 同时使用",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
store = DataProcessStore()
|
||||
target = _target_label(store.database_url)
|
||||
if args.check:
|
||||
ready = _schema_ready(store)
|
||||
print(f"数据处理 schema:{'已安装' if ready else '未安装'};目标:{target}")
|
||||
return 0 if ready else 1
|
||||
if not args.yes:
|
||||
parser.error("--apply 必须同时提供 --yes,确认修改目标数据库")
|
||||
|
||||
print(f"正在安装数据处理 schema;目标:{target}")
|
||||
store.ensure_schema()
|
||||
if not _schema_ready(store):
|
||||
raise RuntimeError("迁移执行后仍未检测到 generation_run_id")
|
||||
print("数据处理 schema 安装完成")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1333
backend/app/modules/data_process/store.py
Normal file
1333
backend/app/modules/data_process/store.py
Normal file
File diff suppressed because it is too large
Load Diff
245
backend/app/schemas/data_process.py
Normal file
245
backend/app/schemas/data_process.py
Normal file
@@ -0,0 +1,245 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, default: Any) -> Any:
|
||||
if snake_name in config:
|
||||
return config[snake_name]
|
||||
return config.get(camel_name, default)
|
||||
|
||||
|
||||
def _validate_process_config(config: dict[str, Any]) -> None:
|
||||
split = _config_value(config, "dataset_split", "datasetSplit", None)
|
||||
if split is not None:
|
||||
if not isinstance(split, dict) or set(split) != {"train", "validation", "test"}:
|
||||
raise ValueError("dataset_split must contain train, validation and test")
|
||||
values = list(split.values())
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) for value in values):
|
||||
raise ValueError("dataset_split values must be integers")
|
||||
if any(value < 0 or value > 100 for value in values) or sum(values) != 100:
|
||||
raise ValueError("dataset_split values must be in [0, 100] and total 100")
|
||||
|
||||
chunk_fields = {
|
||||
"chunk_size",
|
||||
"chunkSize",
|
||||
"chunk_overlap",
|
||||
"chunkOverlap",
|
||||
"min_chunk_size",
|
||||
"minChunkSize",
|
||||
}
|
||||
if chunk_fields.intersection(config):
|
||||
chunk_size = _config_value(config, "chunk_size", "chunkSize", 800)
|
||||
overlap = _config_value(config, "chunk_overlap", "chunkOverlap", 100)
|
||||
minimum = _config_value(config, "min_chunk_size", "minChunkSize", 100)
|
||||
if any(
|
||||
isinstance(value, bool) or not isinstance(value, int)
|
||||
for value in (chunk_size, overlap, minimum)
|
||||
):
|
||||
raise ValueError("chunk_size, chunk_overlap and min_chunk_size must be integers")
|
||||
if not 16 <= chunk_size <= 32_768:
|
||||
raise ValueError("chunk_size must be in [16, 32768]")
|
||||
if overlap < 0 or overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be in [0, chunk_size)")
|
||||
if minimum <= 0 or minimum > chunk_size or overlap + minimum > chunk_size:
|
||||
raise ValueError("min_chunk_size and chunk_overlap exceed chunk_size")
|
||||
|
||||
temperature = _config_value(config, "temperature", "temperature", None)
|
||||
if temperature is not None:
|
||||
if isinstance(temperature, bool) or not isinstance(temperature, (int, float)):
|
||||
raise ValueError("temperature must be a number")
|
||||
if not 0 <= float(temperature) <= 2:
|
||||
raise ValueError("temperature must be in [0, 2]")
|
||||
|
||||
max_tokens = _config_value(config, "max_tokens", "maxTokens", None)
|
||||
if max_tokens is not None:
|
||||
if isinstance(max_tokens, bool) or not isinstance(max_tokens, int):
|
||||
raise ValueError("max_tokens must be an integer")
|
||||
if not 1 <= max_tokens <= 32_768:
|
||||
raise ValueError("max_tokens must be in [1, 32768]")
|
||||
|
||||
for snake_name, camel_name in (
|
||||
("qa_pairs_per_row", "qaPairsPerRow"),
|
||||
("qa_pairs_per_chunk", "qaPairsPerChunk"),
|
||||
):
|
||||
pairs = _config_value(config, snake_name, camel_name, None)
|
||||
if pairs is None:
|
||||
continue
|
||||
if isinstance(pairs, bool) or not isinstance(pairs, int) or not 1 <= pairs <= 5:
|
||||
raise ValueError(f"{snake_name} must be an integer in [1, 5]")
|
||||
|
||||
|
||||
class DataProcessStatus(StrEnum):
|
||||
pending = "pending"
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stopped = "stopped"
|
||||
|
||||
|
||||
class ProcessType(StrEnum):
|
||||
structured = "structured"
|
||||
unstructured = "unstructured"
|
||||
external = "external"
|
||||
|
||||
|
||||
class DataProcessTaskCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str = Field(min_length=1, max_length=150)
|
||||
description: str = ""
|
||||
process_type: ProcessType
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskCreate":
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class DataProcessTaskUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=150)
|
||||
description: str | None = None
|
||||
process_type: ProcessType | None = None
|
||||
source_dataset_id: str | None = None
|
||||
config: dict[str, Any] | None = None
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def normalize_name(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("task name cannot be empty")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_config(self) -> "DataProcessTaskUpdate":
|
||||
if self.config is not None:
|
||||
_validate_process_config(self.config)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewBuildRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
source_file_ids: list[str] | None = None
|
||||
|
||||
|
||||
class PreviewItemCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_file_id: str | None = None
|
||||
original_content: str = ""
|
||||
edited_content: str = ""
|
||||
source_start: int | None = Field(default=None, ge=0)
|
||||
source_end: int | None = Field(default=None, ge=0)
|
||||
source_start_line: int | None = Field(default=None, ge=1)
|
||||
source_end_line: int | None = Field(default=None, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_ranges(self) -> "PreviewItemCreate":
|
||||
if self.source_start is not None and self.source_end is not None:
|
||||
if self.source_end < self.source_start:
|
||||
raise ValueError("source_end must be greater than or equal to source_start")
|
||||
if self.source_start_line is not None and self.source_end_line is not None:
|
||||
if self.source_end_line < self.source_start_line:
|
||||
raise ValueError(
|
||||
"source_end_line must be greater than or equal to source_start_line"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class PreviewItemUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
edited_content: str
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
|
||||
|
||||
class ExternalSourceRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
type: str = Field(min_length=1, max_length=30)
|
||||
url: str = Field(min_length=1, max_length=2048)
|
||||
auth_mode: Literal["none", "basic"] = "none"
|
||||
username: str | None = Field(default=None, max_length=150)
|
||||
password: str | None = Field(default=None, max_length=500)
|
||||
limit: int = Field(default=1000, ge=1, le=100_000)
|
||||
|
||||
|
||||
class ExternalPullRequest(ExternalSourceRequest):
|
||||
query: str | None = Field(default=None, max_length=20_000)
|
||||
file_name: str = Field(default="external-data.jsonl", min_length=1, max_length=255)
|
||||
|
||||
@field_validator("file_name")
|
||||
@classmethod
|
||||
def validate_file_name(cls, value: str) -> str:
|
||||
name = value.strip()
|
||||
if not name.lower().endswith((".jsonl", ".ndjson")):
|
||||
raise ValueError("external pull file_name must end with .jsonl or .ndjson")
|
||||
return name
|
||||
|
||||
|
||||
class ResultUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
instruction: str | None = None
|
||||
input: str | None = None
|
||||
output: str | None = None
|
||||
expected_updated_at: str | None = None
|
||||
|
||||
|
||||
class DatasetSplit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
train: int = Field(default=80, ge=0, le=100)
|
||||
validation: int = Field(default=10, ge=0, le=100)
|
||||
test: int = Field(default=10, ge=0, le=100)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_total(self) -> "DatasetSplit":
|
||||
if self.train + self.validation + self.test != 100:
|
||||
raise ValueError("dataset split must total 100")
|
||||
return self
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dataset_name: str = Field(min_length=1, max_length=150)
|
||||
dataset_type: Literal["train", "test", "eval", "val", "other"] = "train"
|
||||
storage_type: Literal["local"] = "local"
|
||||
split: DatasetSplit = Field(default_factory=DatasetSplit)
|
||||
format: Literal["alpaca_jsonl", "jsonl"] = "alpaca_jsonl"
|
||||
description: str = ""
|
||||
|
||||
@field_validator("dataset_name")
|
||||
@classmethod
|
||||
def normalize_dataset_name(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise ValueError("dataset name cannot be empty")
|
||||
return value
|
||||
31
backend/app/workers/compute_poller.py
Normal file
31
backend/app/workers/compute_poller.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import get_logger
|
||||
from app.modules.compute_gateway.sync import poll_compute_jobs_once
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def run_compute_poller() -> None:
|
||||
settings = get_settings()
|
||||
if settings.compute_mode == "simulator" or settings.compute_status_sync_mode != "polling":
|
||||
logger.info("compute poller disabled", extra={"compute_mode": settings.compute_mode})
|
||||
return
|
||||
|
||||
interval = max(3, settings.compute_poll_interval_seconds)
|
||||
logger.info("compute poller started", extra={"interval_seconds": interval})
|
||||
while True:
|
||||
try:
|
||||
result = await poll_compute_jobs_once()
|
||||
if result["synced"] or result["failed"]:
|
||||
logger.info("compute jobs polled", extra={"result": result})
|
||||
except asyncio.CancelledError:
|
||||
logger.info("compute poller stopped")
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - keep background polling alive
|
||||
logger.exception("compute poller failed", extra={"error": str(exc)})
|
||||
await asyncio.sleep(interval)
|
||||
279
backend/tests/test_data_process_algorithms.py
Normal file
279
backend/tests/test_data_process_algorithms.py
Normal file
@@ -0,0 +1,279 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.modules.data_process.algorithms import (
|
||||
chunk_unstructured,
|
||||
desensitize_pii,
|
||||
detect_text_format,
|
||||
extract_structured_records,
|
||||
generate_standard_records,
|
||||
normalize_text,
|
||||
parse_text_content,
|
||||
record_fingerprint,
|
||||
score_quality,
|
||||
stable_split,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_utf8_json_jsonl_csv_markdown_and_txt() -> None:
|
||||
parsed_json = parse_text_content(
|
||||
b'\xef\xbb\xbf{"data":[{"name":"\xe5\xbc\xa0\xe4\xb8\x89"}]}',
|
||||
filename="records.json",
|
||||
)
|
||||
assert parsed_json.format == "json"
|
||||
assert parsed_json.records == ({"name": "张三"},)
|
||||
|
||||
parsed_jsonl = parse_text_content('{"id":1}\n\n{"id":2}\n', filename="records.jsonl")
|
||||
assert parsed_jsonl.format == "jsonl"
|
||||
assert parsed_jsonl.records == ({"id": 1}, {"id": 2})
|
||||
|
||||
parsed_csv = parse_text_content("name,answer\r\nAlice,yes\r\nBob,no", filename="records.csv")
|
||||
assert parsed_csv.format == "csv"
|
||||
assert parsed_csv.text == "name,answer\nAlice,yes\nBob,no"
|
||||
assert parsed_csv.records[1] == {"name": "Bob", "answer": "no"}
|
||||
|
||||
parsed_markdown = parse_text_content("# 标题\n\n正文", filename="README.md")
|
||||
assert parsed_markdown.format == "markdown"
|
||||
assert parsed_markdown.records == ()
|
||||
|
||||
parsed_txt = parse_text_content("普通文本", filename="note.txt")
|
||||
assert parsed_txt.format == "txt"
|
||||
assert parsed_txt.text == "普通文本"
|
||||
|
||||
|
||||
def test_invalid_utf8_and_malformed_structured_content_fail_loudly() -> None:
|
||||
with pytest.raises(ValueError, match="not valid UTF-8"):
|
||||
parse_text_content(b"\xff\xfe", filename="broken.txt")
|
||||
with pytest.raises(ValueError, match="invalid JSONL at line 2"):
|
||||
extract_structured_records('{"id":1}\nnot-json', "jsonl")
|
||||
with pytest.raises(ValueError, match="more fields"):
|
||||
extract_structured_records("a,b\n1,2,3", "csv")
|
||||
|
||||
|
||||
def test_detect_format_from_content_and_normalize() -> None:
|
||||
assert detect_text_format(text='{"id":1}\n{"id":2}') == "jsonl"
|
||||
assert detect_text_format(text="# Heading\ntext") == "markdown"
|
||||
assert detect_text_format(text="a,b\n1,2") == "csv"
|
||||
assert normalize_text("\ufeffABC \r\n第二\x00行\u200b\t \r\n") == "ABC\n第二行"
|
||||
|
||||
|
||||
def test_extract_json_scalar_and_nested_values_are_stable() -> None:
|
||||
assert extract_structured_records("[1, true, null]", "json") == [
|
||||
{"value": 1},
|
||||
{"value": True},
|
||||
{"value": None},
|
||||
]
|
||||
result = extract_structured_records(
|
||||
json.dumps({"items": [{"text": " 内容 "}], "ignored": 1}, ensure_ascii=False),
|
||||
"json",
|
||||
)
|
||||
assert result == [{"text": "内容"}]
|
||||
|
||||
|
||||
def test_desensitize_pii_returns_masked_text_and_counts() -> None:
|
||||
source = "邮箱 a.user+tag@example.com,手机 +86 13800138000,身份证 11010519491231002X。"
|
||||
masked, counts = desensitize_pii(source)
|
||||
assert masked == "邮箱 [EMAIL],手机 [PHONE],身份证 [ID_CARD]。"
|
||||
assert counts == {"email": 1, "phone": 1, "id_card": 1, "total": 3}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["semantic", "heading", "fixed", "custom"])
|
||||
def test_chunk_methods_preserve_offsets_and_always_advance(method: str) -> None:
|
||||
text = "# 第一章\n" + "甲。" * 18 + "\n# 第二章\n" + "乙。" * 18
|
||||
kwargs = {"custom_delimiter": "\\n"} if method == "custom" else {}
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method=method, # type: ignore[arg-type]
|
||||
chunk_size=12,
|
||||
chunk_overlap=2,
|
||||
min_chunk_size=4,
|
||||
**kwargs,
|
||||
)
|
||||
assert len(chunks) > 1
|
||||
assert all(chunk.content == normalize_text(text)[chunk.start : chunk.end] for chunk in chunks)
|
||||
assert all(chunk.end > chunk.start for chunk in chunks)
|
||||
assert all(left.start < right.start for left, right in zip(chunks, chunks[1:]))
|
||||
assert all(chunk.start_line <= chunk.end_line for chunk in chunks)
|
||||
|
||||
|
||||
def test_fixed_chunk_overlap_is_exact_when_chunks_are_large_enough() -> None:
|
||||
text = " ".join(f"token{i}" for i in range(30))
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method="fixed",
|
||||
chunk_size=10,
|
||||
chunk_overlap=3,
|
||||
min_chunk_size=4,
|
||||
)
|
||||
first_tokens = chunks[0].content.split()
|
||||
second_tokens = chunks[1].content.split()
|
||||
assert first_tokens[-3:] == second_tokens[:3]
|
||||
assert chunks[0].token_count == 10
|
||||
|
||||
|
||||
def test_chunk_line_numbers_treat_newline_as_previous_line_boundary() -> None:
|
||||
chunks = chunk_unstructured(
|
||||
"第一行。\n第二行。\n第三行。",
|
||||
method="custom",
|
||||
chunk_size=8,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=2,
|
||||
custom_delimiter="\\n",
|
||||
)
|
||||
assert chunks[0].content.endswith("\n")
|
||||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == 1
|
||||
assert chunks[1].start_line == 2
|
||||
|
||||
|
||||
def test_heading_and_custom_boundaries_are_respected() -> None:
|
||||
heading_text = "前言 " * 8 + "\n# 第二章\n" + "正文 " * 12
|
||||
heading_chunks = chunk_unstructured(
|
||||
heading_text,
|
||||
method="heading",
|
||||
chunk_size=20,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=4,
|
||||
)
|
||||
assert "# 第二章" not in heading_chunks[0].content
|
||||
assert heading_chunks[1].content.startswith("#")
|
||||
|
||||
custom_chunks = chunk_unstructured(
|
||||
"a b c d <CUT> e f g h i j",
|
||||
method="custom",
|
||||
chunk_size=8,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=2,
|
||||
custom_delimiter="<CUT>",
|
||||
)
|
||||
assert custom_chunks[0].content.endswith("<CUT>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "block"),
|
||||
[
|
||||
(
|
||||
"preserve_code_blocks",
|
||||
"```python\n" + "\n".join(f"value_{i} = {i}" for i in range(30)) + "\n```",
|
||||
),
|
||||
(
|
||||
"preserve_tables",
|
||||
"| 字段 | 说明 |\n| --- | --- |\n"
|
||||
+ "\n".join(f"| field_{i} | value_{i} |" for i in range(30)),
|
||||
),
|
||||
(
|
||||
"preserve_lists",
|
||||
"\n".join(f"- 第 {i} 项需要完整保留" for i in range(30)),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_markdown_protected_blocks_are_not_split(field: str, block: str) -> None:
|
||||
text = "前言。" * 15 + "\n" + block + "\n" + "结尾。" * 40
|
||||
chunks = chunk_unstructured(
|
||||
text,
|
||||
method="fixed",
|
||||
chunk_size=40,
|
||||
chunk_overlap=0,
|
||||
min_chunk_size=10,
|
||||
**{field: True},
|
||||
)
|
||||
assert any(block in chunk.content for chunk in chunks)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "message"),
|
||||
[
|
||||
({"chunk_size": 0}, "chunk_size"),
|
||||
({"chunk_size": 10, "chunk_overlap": 10}, "chunk_overlap"),
|
||||
({"chunk_size": 10, "chunk_overlap": 0, "min_chunk_size": 11}, "min_chunk_size"),
|
||||
(
|
||||
{"chunk_size": 10, "chunk_overlap": 5, "min_chunk_size": 6},
|
||||
"cannot exceed",
|
||||
),
|
||||
({"method": "custom", "custom_delimiter": ""}, "custom_delimiter"),
|
||||
],
|
||||
)
|
||||
def test_chunk_configuration_validation(kwargs: dict[str, object], message: str) -> None:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
chunk_unstructured("some text", **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_quality_scoring_covers_all_dimensions_and_duplicates() -> None:
|
||||
valid = {
|
||||
"instruction": "如何修改收货地址?",
|
||||
"input": "订单尚未发货",
|
||||
"output": "可以在订单详情页申请修改收货地址。",
|
||||
}
|
||||
source = "订单尚未发货时,可以在订单详情页申请修改收货地址。"
|
||||
first_score = score_quality(valid, min_output_length=10, source_content=source)
|
||||
assert first_score.is_valid
|
||||
assert first_score.completeness == 100
|
||||
assert first_score.length == 100
|
||||
assert first_score.readability >= 90
|
||||
assert first_score.relevance >= 70
|
||||
assert first_score.duplicate == 100
|
||||
|
||||
duplicate_score = score_quality(valid, known_fingerprints={first_score.fingerprint})
|
||||
assert duplicate_score.duplicate == 0
|
||||
assert "duplicate_record" in duplicate_score.flags
|
||||
|
||||
unrelated_score = score_quality(
|
||||
valid,
|
||||
min_output_length=10,
|
||||
source_content="量子计算使用量子比特处理信息。",
|
||||
)
|
||||
assert unrelated_score.relevance < first_score.relevance
|
||||
assert "low_source_relevance" in unrelated_score.flags
|
||||
|
||||
invalid_score = score_quality({"instruction": "", "output": "短"}, min_output_length=10)
|
||||
assert not invalid_score.is_valid
|
||||
assert {"missing_instruction", "output_too_short"}.issubset(invalid_score.flags)
|
||||
assert record_fingerprint(valid) == record_fingerprint(dict(reversed(list(valid.items()))))
|
||||
|
||||
|
||||
def test_stable_split_is_reproducible_and_validates_ratios() -> None:
|
||||
first = stable_split("record-42", seed="task-1")
|
||||
assert stable_split("record-42", seed="task-1") == first
|
||||
assert first in {"train", "validation", "test"}
|
||||
assert stable_split("record-42", {"train": 100, "validation": 0, "test": 0}) == "train"
|
||||
with pytest.raises(ValueError, match="sum to 100"):
|
||||
stable_split("record", {"train": 80, "validation": 10, "test": 9})
|
||||
|
||||
|
||||
def test_generate_standard_records_supports_json_qa_and_stable_variants() -> None:
|
||||
previews = [
|
||||
{
|
||||
"id": "preview-json",
|
||||
"edited_content": json.dumps(
|
||||
{"instruction": "问题", "input": "上下文", "output": "答案"},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
},
|
||||
{"id": "preview-qa", "editedContent": "问:如何操作?\n答:按步骤操作。"},
|
||||
]
|
||||
records = generate_standard_records(
|
||||
previews,
|
||||
qa_pairs_per_item=2,
|
||||
semantic_enrichment=True,
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-1",
|
||||
)
|
||||
assert len(records) == 4
|
||||
assert records[0]["instruction"] == "问题"
|
||||
assert records[0]["input"] == "上下文"
|
||||
assert records[0]["output"] == "答案"
|
||||
assert records[1]["instruction"].endswith("问题")
|
||||
assert records[2]["instruction"] == "如何操作?"
|
||||
assert records[2]["output"] == "按步骤操作。"
|
||||
assert all(record["status"] == "valid" for record in records)
|
||||
assert all(record["split"] == "train" for record in records)
|
||||
assert records == generate_standard_records(
|
||||
previews,
|
||||
qa_pairs_per_item=2,
|
||||
semantic_enrichment=True,
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
split_seed="task-1",
|
||||
)
|
||||
704
backend/tests/test_data_process_api.py
Normal file
704
backend/tests/test_data_process_api.py
Normal file
@@ -0,0 +1,704 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1.endpoints import data_process as data_process_endpoint
|
||||
from app.api.v1.endpoints.data_process import router
|
||||
from app.modules.data_process.store import InvalidStateError, NotFoundError, get_data_process_store
|
||||
|
||||
|
||||
class FakeDataProcessStore:
|
||||
"""接口测试专用内存实现,确保测试不会连接或迁移真实数据库。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.tasks: dict[str, dict[str, Any]] = {}
|
||||
self.sources: dict[str, list[dict[str, Any]]] = {}
|
||||
self.previews: dict[str, list[dict[str, Any]]] = {}
|
||||
self.results: dict[str, list[dict[str, Any]]] = {}
|
||||
self.datasets: dict[str, dict[str, Any]] = {}
|
||||
self.sequence = 0
|
||||
|
||||
def _id(self, prefix: str) -> str:
|
||||
self.sequence += 1
|
||||
return f"{prefix}_{self.sequence}"
|
||||
|
||||
def list_tasks(self, *, page: int, page_size: int, **filters: Any) -> dict[str, Any]:
|
||||
items = list(self.tasks.values())
|
||||
for field in ("status", "process_type", "tenant_id", "project_id"):
|
||||
if filters.get(field):
|
||||
items = [item for item in items if item.get(field) == filters[field]]
|
||||
keyword = filters.get("keyword")
|
||||
if keyword:
|
||||
items = [item for item in items if keyword in item["name"]]
|
||||
return {
|
||||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def create_task(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task_id = self._id("dpt")
|
||||
task = {
|
||||
"id": task_id,
|
||||
**deepcopy(payload),
|
||||
"status": "pending",
|
||||
"progress": 0,
|
||||
"input_count": 0,
|
||||
"output_count": 0,
|
||||
"filtered_count": 0,
|
||||
"duplicate_count": 0,
|
||||
"error_count": 0,
|
||||
"failure_reason": None,
|
||||
"output_dataset_id": None,
|
||||
}
|
||||
self.tasks[task_id] = task
|
||||
self.sources[task_id] = []
|
||||
self.previews[task_id] = []
|
||||
self.results[task_id] = []
|
||||
return deepcopy(task)
|
||||
|
||||
def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
if task_id not in self.tasks:
|
||||
raise NotFoundError("data process task not found")
|
||||
return deepcopy(self.tasks[task_id])
|
||||
|
||||
def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
self.get_task(task_id)
|
||||
self.tasks[task_id].update(deepcopy(payload))
|
||||
return self.get_task(task_id)
|
||||
|
||||
def delete_task(self, task_id: str, **_: Any) -> None:
|
||||
self.get_task(task_id)
|
||||
if self.tasks[task_id]["status"] == "running":
|
||||
raise InvalidStateError("running task must be stopped before deletion")
|
||||
del self.tasks[task_id]
|
||||
|
||||
def list_source_files(self, task_id: str) -> list[dict[str, Any]]:
|
||||
self.get_task(task_id)
|
||||
return [
|
||||
{key: value for key, value in item.items() if key != "content"}
|
||||
for item in self.sources[task_id]
|
||||
]
|
||||
|
||||
def add_source_file(self, task_id: str, **payload: Any) -> dict[str, Any]:
|
||||
self.get_task(task_id)
|
||||
source = {
|
||||
"id": self._id("dpsf"),
|
||||
"task_id": task_id,
|
||||
"version_no": 1,
|
||||
**deepcopy(payload),
|
||||
}
|
||||
self.sources[task_id].append(source)
|
||||
self.tasks[task_id]["input_count"] += payload["record_count"]
|
||||
return {key: value for key, value in deepcopy(source).items() if key != "content"}
|
||||
|
||||
def add_source_files(
|
||||
self, task_id: str, files: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
# 先验证整个批次,模拟数据库事务的 all-or-nothing 语义。
|
||||
checksums = {item["checksum_sha256"] for item in self.sources.get(task_id, [])}
|
||||
incoming: set[str] = set()
|
||||
for payload in files:
|
||||
checksum = payload["checksum_sha256"]
|
||||
if checksum in checksums or checksum in incoming:
|
||||
raise ValueError("the same source file content is already attached to this task")
|
||||
incoming.add(checksum)
|
||||
return [self.add_source_file(task_id, **payload) for payload in files]
|
||||
|
||||
def get_source_file(
|
||||
self, task_id: str, file_id: str, *, include_content: bool = True
|
||||
) -> dict[str, Any]:
|
||||
source = next(
|
||||
(item for item in self.sources.get(task_id, []) if item["id"] == file_id),
|
||||
None,
|
||||
)
|
||||
if not source:
|
||||
raise NotFoundError("source file not found")
|
||||
result = deepcopy(source)
|
||||
if not include_content:
|
||||
result.pop("content", None)
|
||||
return result
|
||||
|
||||
def source_content_window(
|
||||
self, task_id: str, file_id: str, offset: int, limit: int
|
||||
) -> dict[str, Any]:
|
||||
source = self.get_source_file(task_id, file_id)
|
||||
content = source.pop("content")
|
||||
return {
|
||||
"file": source,
|
||||
"content": content[offset : offset + limit],
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"total_chars": len(content),
|
||||
"has_more": offset + limit < len(content),
|
||||
}
|
||||
|
||||
def source_content_lines(
|
||||
self, task_id: str, file_id: str, start_line: int, line_count: int
|
||||
) -> dict[str, Any]:
|
||||
source = self.get_source_file(task_id, file_id)
|
||||
lines = source.pop("content").splitlines(keepends=True)
|
||||
selected = lines[start_line - 1 : start_line - 1 + line_count]
|
||||
return {
|
||||
"file": source,
|
||||
"content": "".join(selected),
|
||||
"start_line": start_line,
|
||||
"end_line": start_line - 1 + len(selected),
|
||||
"line_count": len(selected),
|
||||
"total_lines": len(lines),
|
||||
"has_more": start_line - 1 + len(selected) < len(lines),
|
||||
}
|
||||
|
||||
def delete_source_file(self, task_id: str, file_id: str) -> None:
|
||||
self.get_source_file(task_id, file_id)
|
||||
self.sources[task_id] = [item for item in self.sources[task_id] if item["id"] != file_id]
|
||||
self.previews[task_id] = [
|
||||
item for item in self.previews[task_id] if item["source_file_id"] != file_id
|
||||
]
|
||||
self.results[task_id] = []
|
||||
|
||||
def replace_preview_items(
|
||||
self, task_id: str, items: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
self.previews[task_id] = [
|
||||
{"id": self._id("dpp"), "task_id": task_id, **deepcopy(item)} for item in items
|
||||
]
|
||||
self.results[task_id] = []
|
||||
self.tasks[task_id]["progress"] = 20
|
||||
return deepcopy(self.previews[task_id])
|
||||
|
||||
def list_preview_items(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
source_file_id: str | None = None,
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
items = self.previews[task_id]
|
||||
if source_file_id:
|
||||
items = [item for item in items if item["source_file_id"] == source_file_id]
|
||||
if keyword:
|
||||
items = [item for item in items if keyword in item["edited_content"]]
|
||||
return {
|
||||
"items": deepcopy(items[(page - 1) * page_size : page * page_size]),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def get_preview_item(self, task_id: str, preview_id: str) -> dict[str, Any]:
|
||||
item = next(
|
||||
(item for item in self.previews.get(task_id, []) if item["id"] == preview_id),
|
||||
None,
|
||||
)
|
||||
if not item:
|
||||
raise NotFoundError("preview item not found")
|
||||
return deepcopy(item)
|
||||
|
||||
def create_preview_item(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
item = {"id": self._id("dpp"), "task_id": task_id, **deepcopy(payload)}
|
||||
self.previews[task_id].append(item)
|
||||
self.results[task_id] = []
|
||||
return deepcopy(item)
|
||||
|
||||
def update_preview_item(
|
||||
self, task_id: str, preview_id: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
item = next(
|
||||
(item for item in self.previews[task_id] if item["id"] == preview_id),
|
||||
None,
|
||||
)
|
||||
if not item:
|
||||
raise NotFoundError("preview item not found")
|
||||
item.update(deepcopy(payload))
|
||||
self.results[task_id] = []
|
||||
return deepcopy(item)
|
||||
|
||||
def delete_preview_item(self, task_id: str, preview_id: str) -> None:
|
||||
before = len(self.previews[task_id])
|
||||
self.previews[task_id] = [
|
||||
item for item in self.previews[task_id] if item["id"] != preview_id
|
||||
]
|
||||
if len(self.previews[task_id]) == before:
|
||||
raise NotFoundError("preview item not found")
|
||||
|
||||
def start_generation(self, task_id: str, *, replace_existing: bool) -> dict[str, Any]:
|
||||
if not self.previews[task_id]:
|
||||
raise InvalidStateError("preview must be built before generation")
|
||||
if replace_existing:
|
||||
self.results[task_id] = []
|
||||
self.tasks[task_id].update(
|
||||
status="running",
|
||||
progress=30,
|
||||
generation_run_id=self._id("dprun"),
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def generation_is_running(self, task_id: str, generation_run_id: str) -> bool:
|
||||
return (
|
||||
self.tasks[task_id]["status"] == "running"
|
||||
and self.tasks[task_id].get("generation_run_id") == generation_run_id
|
||||
)
|
||||
|
||||
def update_generation_progress(
|
||||
self,
|
||||
task_id: str,
|
||||
generation_run_id: str,
|
||||
processed_count: int,
|
||||
total_count: int,
|
||||
) -> bool:
|
||||
if not self.generation_is_running(task_id, generation_run_id):
|
||||
return False
|
||||
self.tasks[task_id]["progress"] = min(
|
||||
95,
|
||||
30 + processed_count / max(1, total_count) * 65,
|
||||
)
|
||||
return True
|
||||
|
||||
def complete_generation(
|
||||
self,
|
||||
task_id: str,
|
||||
results: list[dict[str, Any]],
|
||||
*,
|
||||
generation_run_id: str,
|
||||
**counts: Any,
|
||||
) -> dict[str, Any]:
|
||||
if not self.generation_is_running(task_id, generation_run_id):
|
||||
return self.get_task(task_id)
|
||||
self.results[task_id] = deepcopy(results)
|
||||
self.tasks[task_id].update(
|
||||
status="completed",
|
||||
progress=100,
|
||||
output_count=len(results),
|
||||
generation_run_id=None,
|
||||
**counts,
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def mark_failed(
|
||||
self, task_id: str, reason: str, *, generation_run_id: str
|
||||
) -> dict[str, Any]:
|
||||
if self.generation_is_running(task_id, generation_run_id):
|
||||
self.tasks[task_id].update(
|
||||
status="failed",
|
||||
failure_reason=reason,
|
||||
generation_run_id=None,
|
||||
)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def stop_task(self, task_id: str) -> dict[str, Any]:
|
||||
if self.tasks[task_id]["status"] != "running":
|
||||
raise InvalidStateError("only a running task can be stopped")
|
||||
self.tasks[task_id].update(status="stopped", generation_run_id=None)
|
||||
return self.get_task(task_id)
|
||||
|
||||
def progress(self, task_id: str) -> dict[str, Any]:
|
||||
task = self.get_task(task_id)
|
||||
result = {key: task.get(key) for key in (
|
||||
"status", "progress", "input_count", "output_count",
|
||||
"filtered_count", "duplicate_count", "error_count", "failure_reason",
|
||||
)}
|
||||
result["task_id"] = task["id"]
|
||||
return result
|
||||
|
||||
def list_results(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
status: str | None = None,
|
||||
split: str | None = None,
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
items = self.results[task_id]
|
||||
if status:
|
||||
items = [item for item in items if item["status"] == status]
|
||||
if split:
|
||||
items = [item for item in items if item["split"] == split]
|
||||
if keyword:
|
||||
items = [
|
||||
item
|
||||
for item in items
|
||||
if any(keyword in item[field] for field in ("instruction", "input", "output"))
|
||||
]
|
||||
return {
|
||||
"items": deepcopy(items),
|
||||
"total": len(items),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
def update_result(
|
||||
self, task_id: str, result_id: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
for field in ("instruction", "input", "output", "quality_score"):
|
||||
if field in payload:
|
||||
item[field] = deepcopy(payload[field])
|
||||
hard_valid = bool(item["instruction"].strip() and item["output"].strip())
|
||||
quality_valid = bool((item.get("quality_score") or {}).get("is_valid", hard_valid))
|
||||
changed = any(
|
||||
item[field] != item[f"original_{field}"]
|
||||
for field in ("instruction", "input", "output")
|
||||
)
|
||||
item["status"] = (
|
||||
"invalid"
|
||||
if not hard_valid or not quality_valid
|
||||
else "modified" if changed else "valid"
|
||||
)
|
||||
self.tasks[task_id]["error_count"] = sum(
|
||||
result["status"] == "invalid" for result in self.results[task_id]
|
||||
)
|
||||
return deepcopy(item)
|
||||
|
||||
def get_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
return deepcopy(item)
|
||||
|
||||
def restore_result(self, task_id: str, result_id: str) -> dict[str, Any]:
|
||||
item = next((item for item in self.results[task_id] if item["id"] == result_id), None)
|
||||
if not item:
|
||||
raise NotFoundError("data process result not found")
|
||||
for field in ("instruction", "input", "output"):
|
||||
item[field] = item[f"original_{field}"]
|
||||
item["status"] = "valid"
|
||||
return deepcopy(item)
|
||||
|
||||
def publish(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
task = self.tasks[task_id]
|
||||
if task.get("output_dataset_id"):
|
||||
return {"dataset": deepcopy(self.datasets[task["output_dataset_id"]]), "created": False}
|
||||
if task["status"] != "completed":
|
||||
raise InvalidStateError("only a completed task can be published")
|
||||
dataset_id = self._id("dataset")
|
||||
dataset = {"id": dataset_id, "name": payload["dataset_name"], "source_task_id": task_id}
|
||||
self.datasets[dataset_id] = dataset
|
||||
task["output_dataset_id"] = dataset_id
|
||||
return {"dataset": deepcopy(dataset), "created": True}
|
||||
|
||||
|
||||
def make_client() -> tuple[TestClient, FakeDataProcessStore]:
|
||||
store = FakeDataProcessStore()
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/modelTF")
|
||||
app.dependency_overrides[get_data_process_store] = lambda: store
|
||||
return TestClient(app), store
|
||||
|
||||
|
||||
def test_data_process_full_contract_without_database() -> None:
|
||||
client, store = make_client()
|
||||
created = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "客服问答处理",
|
||||
"process_type": "structured",
|
||||
"config": {"dataset_split": {"train": 80, "validation": 10, "test": 10}},
|
||||
},
|
||||
)
|
||||
assert created.status_code == 200
|
||||
task_id = created.json()["data"]["id"]
|
||||
|
||||
source_content = (
|
||||
'{"question":"如何修改地址?",'
|
||||
'"answer":"订单发货前可在订单详情申请修改收货地址。"}\n'
|
||||
'{"question":"如何申请退款?",'
|
||||
'"answer":"请在订单详情提交退款申请并等待审核处理。"}\n'
|
||||
)
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("customer.jsonl", source_content.encode(), "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
source = uploaded.json()["data"]["files"][0]
|
||||
assert len(source["checksum_sha256"]) == 64
|
||||
assert source["version_no"] == 1
|
||||
|
||||
window = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||||
params={"offset": 0, "limit": 20},
|
||||
)
|
||||
assert window.status_code == 200
|
||||
assert window.json()["data"]["has_more"] is True
|
||||
line_window = client.get(
|
||||
f"/modelTF/data-process/{task_id}/source-files/{source['id']}/content",
|
||||
params={"start_line": 2, "line_count": 1},
|
||||
)
|
||||
assert line_window.json()["data"]["start_line"] == 2
|
||||
assert line_window.json()["data"]["end_line"] == 2
|
||||
assert line_window.json()["data"]["total_lines"] == 2
|
||||
|
||||
preview = client.post(
|
||||
f"/modelTF/data-process/{task_id}/preview/build",
|
||||
json={"source_file_ids": [source["id"]]},
|
||||
)
|
||||
assert preview.status_code == 200
|
||||
assert preview.json()["data"]["total"] == 2
|
||||
listed_preview = client.get(f"/modelTF/data-process/{task_id}/preview")
|
||||
assert listed_preview.json()["data"]["total"] == 2
|
||||
preview_item = listed_preview.json()["data"]["items"][0]
|
||||
updated_preview = client.put(
|
||||
f"/modelTF/data-process/{task_id}/preview/{preview_item['id']}",
|
||||
json={
|
||||
"edited_content": preview_item["edited_content"],
|
||||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||||
},
|
||||
)
|
||||
assert "quality_score" in updated_preview.json()["data"]
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
progress = client.get(f"/modelTF/data-process/{task_id}/progress")
|
||||
assert progress.json()["data"]["status"] == "completed"
|
||||
result_page = client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]
|
||||
assert result_page["total"] == 2
|
||||
keyword_page = client.get(
|
||||
f"/modelTF/data-process/{task_id}/results", params={"keyword": "地址"}
|
||||
).json()["data"]
|
||||
assert keyword_page["total"] == 1
|
||||
|
||||
result = result_page["items"][0]
|
||||
edited = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||||
json={
|
||||
"output": "人工修改后的完整答案。",
|
||||
"expected_updated_at": "2026-07-23T00:00:00Z",
|
||||
},
|
||||
)
|
||||
assert edited.json()["data"]["status"] == "modified"
|
||||
assert "quality_score" in edited.json()["data"]
|
||||
invalid_edit = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}",
|
||||
json={"output": ""},
|
||||
)
|
||||
assert invalid_edit.json()["data"]["status"] == "invalid"
|
||||
assert store.tasks[task_id]["error_count"] == 1
|
||||
restored = client.post(
|
||||
f"/modelTF/data-process/{task_id}/results/{result['id']}/restore"
|
||||
)
|
||||
assert restored.json()["data"]["output"] == result["original_output"]
|
||||
assert restored.json()["data"]["status"] == "valid"
|
||||
assert store.tasks[task_id]["error_count"] == 0
|
||||
|
||||
publish_payload = {"dataset_name": "客服问答清洗集"}
|
||||
first_publish = client.post(
|
||||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||||
)
|
||||
second_publish = client.post(
|
||||
f"/modelTF/data-process/{task_id}/publish", json=publish_payload
|
||||
)
|
||||
assert first_publish.json()["data"]["created"] is True
|
||||
assert second_publish.json()["data"]["created"] is False
|
||||
assert (
|
||||
first_publish.json()["data"]["dataset"]["id"]
|
||||
== second_publish.json()["data"]["dataset"]["id"]
|
||||
)
|
||||
|
||||
|
||||
def test_external_source_never_returns_fake_success() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "外部数据", "process_type": "external", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/external/test",
|
||||
json={"type": "mysql", "url": "mysql://db.example/test"},
|
||||
)
|
||||
assert response.status_code == 501
|
||||
assert response.json()["detail"]["code"] == 501
|
||||
|
||||
|
||||
def test_config_validation_and_stop_state() -> None:
|
||||
client, store = make_client()
|
||||
invalid = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "错误切片配置",
|
||||
"process_type": "unstructured",
|
||||
"config": {
|
||||
"dataset_split": {"train": 80, "validation": 30, "test": 0},
|
||||
"chunk_size": 100,
|
||||
"chunk_overlap": 90,
|
||||
"min_chunk_size": 20,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert invalid.status_code == 422
|
||||
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "可停止任务", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
store.tasks[task_id]["status"] = "running"
|
||||
stopped = client.post(f"/modelTF/data-process/{task_id}/stop")
|
||||
assert stopped.status_code == 200
|
||||
assert stopped.json()["data"]["status"] == "stopped"
|
||||
|
||||
|
||||
def test_upload_batch_is_atomic_and_empty_files_are_rejected() -> None:
|
||||
client, store = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "批量上传", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
|
||||
duplicate_batch = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files=[
|
||||
("files", ("first.txt", b"same content", "text/plain")),
|
||||
("files", ("second.txt", b"same content", "text/plain")),
|
||||
],
|
||||
)
|
||||
assert duplicate_batch.status_code == 400
|
||||
assert store.sources[task_id] == []
|
||||
|
||||
empty = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("empty.txt", b"", "text/plain")},
|
||||
)
|
||||
assert empty.status_code == 400
|
||||
assert store.sources[task_id] == []
|
||||
|
||||
|
||||
def test_preprocess_deduplicates_and_quality_filter_removes_short_results() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={
|
||||
"name": "去重与质量筛选",
|
||||
"process_type": "structured",
|
||||
"config": {
|
||||
"preprocess_options": ["clean_invalid", "deduplicate"],
|
||||
"quality_filter_enabled": True,
|
||||
"filter_low_quality": False,
|
||||
"filter_short_content": True,
|
||||
"min_output_length": 100,
|
||||
},
|
||||
},
|
||||
).json()["data"]["id"]
|
||||
content = (
|
||||
'{"question":"问题","answer":"短答案"}\n'
|
||||
'{"question":"问题","answer":"短答案"}\n'
|
||||
).encode()
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("duplicates.jsonl", content, "application/jsonl")},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
preview = client.post(f"/modelTF/data-process/{task_id}/preview/build")
|
||||
assert preview.json()["data"]["total"] == 1
|
||||
|
||||
generated = client.post(f"/modelTF/data-process/{task_id}/generate")
|
||||
assert generated.status_code == 200
|
||||
progress = client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]
|
||||
assert progress["status"] == "completed"
|
||||
assert progress["filtered_count"] == 1
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 0
|
||||
|
||||
|
||||
def test_stale_generation_worker_cannot_overwrite_new_run(monkeypatch: Any) -> None:
|
||||
store = FakeDataProcessStore()
|
||||
task = store.create_task(
|
||||
{"name": "并发代次", "process_type": "structured", "config": {}}
|
||||
)
|
||||
task_id = task["id"]
|
||||
store.replace_preview_items(
|
||||
task_id,
|
||||
[
|
||||
{
|
||||
"source_file_id": None,
|
||||
"original_content": "来源内容",
|
||||
"edited_content": "来源内容",
|
||||
"status": "manual",
|
||||
}
|
||||
],
|
||||
)
|
||||
first = store.start_generation(task_id, replace_existing=True)
|
||||
first_run_id = first["generation_run_id"]
|
||||
second_run_id = ""
|
||||
|
||||
def restart_while_old_worker_runs(*_: Any, **__: Any) -> list[dict[str, Any]]:
|
||||
nonlocal second_run_id
|
||||
store.stop_task(task_id)
|
||||
second = store.start_generation(task_id, replace_existing=True)
|
||||
second_run_id = second["generation_run_id"]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
data_process_endpoint,
|
||||
"generate_standard_records",
|
||||
restart_while_old_worker_runs,
|
||||
)
|
||||
data_process_endpoint._run_generation(store, task_id, first_run_id)
|
||||
|
||||
assert second_run_id and second_run_id != first_run_id
|
||||
assert store.tasks[task_id]["status"] == "running"
|
||||
assert store.tasks[task_id]["generation_run_id"] == second_run_id
|
||||
assert store.results[task_id] == []
|
||||
store.mark_failed(task_id, "old failure", generation_run_id=first_run_id)
|
||||
assert store.tasks[task_id]["status"] == "running"
|
||||
|
||||
|
||||
def test_result_status_cannot_be_forged_by_client() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "状态保护", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.put(
|
||||
f"/modelTF/data-process/{task_id}/results/not-created",
|
||||
json={"instruction": "", "output": "", "status": "valid"},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_start_rebuilds_preview_and_generates_in_one_request() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "一键处理", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
uploaded = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={
|
||||
"files": (
|
||||
"one.jsonl",
|
||||
b'{"question":"What is one?","answer":"One."}\n',
|
||||
"application/jsonl",
|
||||
)
|
||||
},
|
||||
)
|
||||
assert uploaded.status_code == 200
|
||||
|
||||
started = client.post(f"/modelTF/data-process/{task_id}/start")
|
||||
assert started.status_code == 200
|
||||
assert started.json()["data"]["task_id"] == task_id
|
||||
assert started.json()["data"]["status"] == "running"
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/progress").json()["data"]["status"] == "completed"
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/preview").json()["data"]["total"] == 1
|
||||
assert client.get(f"/modelTF/data-process/{task_id}/results").json()["data"]["total"] == 1
|
||||
|
||||
|
||||
def test_unsupported_upload_format_returns_415() -> None:
|
||||
client, _ = make_client()
|
||||
task_id = client.post(
|
||||
"/modelTF/data-process",
|
||||
json={"name": "格式限制", "process_type": "structured", "config": {}},
|
||||
).json()["data"]["id"]
|
||||
response = client.post(
|
||||
f"/modelTF/data-process/{task_id}/source-files",
|
||||
files={"files": ("document.pdf", b"not a pdf", "application/pdf")},
|
||||
)
|
||||
assert response.status_code == 415
|
||||
102
backend/tests/test_data_process_generation.py
Normal file
102
backend/tests/test_data_process_generation.py
Normal file
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
from app.modules.data_process.generation import chat_completions_url, generate_model_records
|
||||
|
||||
|
||||
def test_chat_completions_url_accepts_host_base_and_complete_url() -> None:
|
||||
assert chat_completions_url("www.caoxiaozhu.com") == (
|
||||
"https://www.caoxiaozhu.com/v1/chat/completions"
|
||||
)
|
||||
assert chat_completions_url("https://model.example/v1") == (
|
||||
"https://model.example/v1/chat/completions"
|
||||
)
|
||||
complete = "https://model.example/openai/v1/chat/completions"
|
||||
assert chat_completions_url(complete) == complete
|
||||
|
||||
|
||||
def test_generate_model_records_uses_prompt_auth_and_stable_split() -> None:
|
||||
requests: list[httpx.Request] = []
|
||||
progress_updates: list[tuple[int, int]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
payload = json.loads(request.content)
|
||||
assert payload["model"] == "qwen-plus"
|
||||
assert payload["response_format"] == {"type": "json_object"}
|
||||
assert "客户反馈页面加载慢" in payload["messages"][1]["content"]
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps(
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"instruction": "请生成简洁客服回复",
|
||||
"input": "客户反馈页面加载慢",
|
||||
"output": "已收到反馈,我们正在排查。",
|
||||
}
|
||||
]
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-1", "edited_content": "客户反馈页面加载慢"}],
|
||||
model={
|
||||
"name": "Qwen",
|
||||
"online_model_name": "qwen-plus",
|
||||
"api_url": "model.example",
|
||||
"api_key": "test-secret",
|
||||
},
|
||||
config={
|
||||
"generation_prompt": "请处理:{{ content }}",
|
||||
"json_mode": True,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 512,
|
||||
},
|
||||
task_id="task-1",
|
||||
split={"train": 100, "validation": 0, "test": 0},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
on_progress=lambda processed, total: progress_updates.append((processed, total)),
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["status"] == "valid"
|
||||
assert records[0]["split"] == "train"
|
||||
assert requests[0].headers["Authorization"] == "Bearer test-secret"
|
||||
assert progress_updates == [(1, 1)]
|
||||
|
||||
|
||||
def test_generate_model_records_keeps_partial_failure_for_manual_repair() -> None:
|
||||
client = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda _: httpx.Response(200, json={"choices": [{"message": {"content": "not-json"}}]})
|
||||
)
|
||||
)
|
||||
records = generate_model_records(
|
||||
[{"id": "preview-1", "edited_content": "来源正文"}],
|
||||
model={"name": "model", "api_url": "https://model.example/v1"},
|
||||
config={"generation_retries": 1},
|
||||
task_id="task-1",
|
||||
split={"train": 80, "validation": 10, "test": 10},
|
||||
qa_pairs_per_item=1,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
assert records[0]["status"] == "invalid"
|
||||
assert records[0]["error"]
|
||||
29
backend/tests/test_data_process_migration.py
Normal file
29
backend/tests/test_data_process_migration.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.modules.data_process.schema_cli import _target_label
|
||||
|
||||
|
||||
def test_runtime_migration_fails_fast_on_incompatible_schema() -> None:
|
||||
sql_path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "app"
|
||||
/ "db"
|
||||
/ "sql"
|
||||
/ "002_data_process.sql"
|
||||
)
|
||||
sql = sql_path.read_text(encoding="utf-8")
|
||||
|
||||
assert "requires 001_platform_runtime.sql first" in sql
|
||||
assert "supports only the current TEXT runtime schema" in sql
|
||||
assert "generation_run_id" in sql
|
||||
assert "CREATE TABLE IF NOT EXISTS data_process_results" in sql
|
||||
assert sql.count("BEGIN;") == 1
|
||||
assert sql.rstrip().endswith("COMMIT;")
|
||||
|
||||
|
||||
def test_schema_cli_target_label_never_contains_credentials() -> None:
|
||||
label = _target_label("postgresql://secret-user:secret-password@db.example:5433/yg_ft")
|
||||
assert label == "db.example:5433/yg_ft"
|
||||
assert "secret" not in label
|
||||
@@ -25,5 +25,57 @@ compute/
|
||||
|
||||
## 运行模式
|
||||
|
||||
- 默认 `COMPUTE_EXECUTION_MODE=real`,Compute API 只暴露健康检查和接口契约;真实训练执行器完成前,创建作业会返回未实现错误。
|
||||
- 默认 `COMPUTE_EXECUTION_MODE=real`,Compute API 会通过 `compute.agent.process_manager.ProcessManager` 启动真实 `llamafactory-cli train` 子进程,并将日志写入 `TRAINING_LOG_ROOT`。
|
||||
- 真实模式下 GPU 发现优先使用宿主机 `nvidia-smi`。如果部署环境暂时无法调用 `nvidia-smi`,可通过 `COMPUTE_GPU_COUNT`、`COMPUTE_GPU_NAME`、`COMPUTE_GPU_MEMORY_GB`、`COMPUTE_GPU_POWER_LIMIT_W` 声明兼容 GPU 清单,便于应用侧先完成节点登记和联调。
|
||||
- 仅隔离联调时可设置 `COMPUTE_EXECUTION_MODE=simulator`,启用内存状态机和合成 GPU/日志数据。该模式不得作为生产运行路径。
|
||||
- 服务间鉴权默认开启:设置 `COMPUTE_AUTH_ENABLED=true` 和一致的 `COMPUTE_SERVICE_TOKEN`,应用侧会通过 `X-Compute-Token` 调用 Compute API。
|
||||
- 真实训练作业会登记到 `TRAINING_LOG_ROOT/compute-jobs.json`。Compute API 重启后会恢复作业索引,继续提供状态、停止和日志查询。
|
||||
- 同一算力节点内按 GPU ID 做轻量锁定;已有运行中作业占用的 GPU 不允许再次提交,避免同机多 GPU 场景下误复用。
|
||||
|
||||
真实执行前提:
|
||||
|
||||
- 镜像或宿主机环境中 `llamafactory-cli` 可执行。
|
||||
- `LLAMA_FACTORY_HOME` 指向 LLaMA-Factory 工作目录。
|
||||
- 基座模型路径和数据集名称/目录已经在算力服务器本地可访问。
|
||||
- 应用侧训练任务中的 GPU、模型、数据集配置能映射到当前节点本地路径。
|
||||
|
||||
## 应用侧接入
|
||||
|
||||
应用平台通过“算力节点”页面维护每台 GPU 服务器的 `Compute API` 和 `File Gateway` 地址。点击连接测试时,Backend API 会主动调用:
|
||||
|
||||
```text
|
||||
GET /modelTF/v1/compute/health
|
||||
GET /modelTF/compute/resources/gpus
|
||||
```
|
||||
|
||||
连接成功后,应用侧会同步节点健康信息、能力标签和 GPU 清单到 PostgreSQL。多节点阶段仍按“每台算力服务器 = 单机多 GPU 节点”管理,每台服务器都部署 Compute API、Agent、File Gateway 契约和 LLaMA-Factory。
|
||||
|
||||
训练闭环:
|
||||
|
||||
```text
|
||||
Frontend 创建/启动训练
|
||||
-> Backend API 选择 compute_nodes 节点
|
||||
-> Backend API POST /modelTF/compute/jobs 到目标 Compute API
|
||||
-> Compute API 启动 llamafactory-cli 子进程
|
||||
-> Backend Worker 定时 GET /modelTF/compute/jobs/{id}
|
||||
-> Backend API 同步 fine_tune_tasks 状态、进度、PID、日志路径和产物索引
|
||||
```
|
||||
|
||||
## 当前接口能力
|
||||
|
||||
日志接口:
|
||||
|
||||
```text
|
||||
GET /modelTF/compute/jobs/{job_id}/logs?tail_lines=200
|
||||
GET /modelTF/compute/jobs/{job_id}/logs?offset=0&limit=500
|
||||
```
|
||||
|
||||
返回 `content`、`metrics`、`total_lines`、`offset`、`limit`、`has_more`、`next_offset`,用于前端增量刷新和日志平台采集。
|
||||
|
||||
文件导入:
|
||||
|
||||
```text
|
||||
POST /modelTF/compute/files/import-local
|
||||
```
|
||||
|
||||
该接口用于应用侧调度前把算力服务器本地可访问的模型/数据集路径导入到 `YG_FT_DATA_ROOT` 内部。目标路径会校验不能逃逸出 `YG_FT_DATA_ROOT`,源路径必须已存在于算力服务器本地或挂载目录。
|
||||
|
||||
246
compute/agent/process_manager.py
Normal file
246
compute/agent/process_manager.py
Normal file
@@ -0,0 +1,246 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import contextlib
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
TERMINAL_STATUSES = {"completed", "failed", "stopped"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ManagedProcess:
|
||||
id: str
|
||||
name: str
|
||||
command: list[str]
|
||||
work_dir: str
|
||||
log_path: Path
|
||||
output_dir: str
|
||||
gpus: list[int]
|
||||
process: subprocess.Popen[Any] | None
|
||||
created_at: float
|
||||
pid: int | None = None
|
||||
status: str = "running"
|
||||
progress: int = 5
|
||||
artifacts: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
class ProcessManager:
|
||||
def __init__(self, log_root: str) -> None:
|
||||
self.log_root = Path(log_root)
|
||||
self.log_root.mkdir(parents=True, exist_ok=True)
|
||||
self.registry_path = self.log_root / "compute-jobs.json"
|
||||
self.jobs: dict[str, ManagedProcess] = {}
|
||||
self._load_registry()
|
||||
|
||||
def create_job(self, payload: dict[str, Any], command: list[str], work_dir: str) -> dict[str, Any]:
|
||||
job_id = str(payload.get("id") or f"job_{int(time.time() * 1000)}")
|
||||
if job_id in self.jobs and self.jobs[job_id].status not in TERMINAL_STATUSES:
|
||||
raise ValueError(f"job {job_id} is already running")
|
||||
|
||||
output_dir = str(payload.get("output_dir") or f"/data/yg-ft/outputs/{payload.get('name', job_id)}")
|
||||
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
||||
log_path = self.log_root / f"{job_id}.log"
|
||||
env = os.environ.copy()
|
||||
gpus = [int(item) for item in payload.get("gpus") or []]
|
||||
locked = self.locked_gpus()
|
||||
conflict = sorted(set(gpus).intersection(locked))
|
||||
if conflict:
|
||||
raise ValueError(f"gpu already locked: {conflict}")
|
||||
if gpus:
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in gpus)
|
||||
env.update({str(k): str(v) for k, v in payload.get("env", {}).items()})
|
||||
|
||||
cwd = work_dir if Path(work_dir).exists() else None
|
||||
with log_path.open("ab") as log_file:
|
||||
log_file.write(f"[INFO] starting job_id={job_id} command={' '.join(command)}\n".encode("utf-8"))
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
managed = ManagedProcess(
|
||||
id=job_id,
|
||||
name=str(payload.get("name") or job_id),
|
||||
command=command,
|
||||
work_dir=work_dir,
|
||||
log_path=log_path,
|
||||
output_dir=output_dir,
|
||||
gpus=gpus,
|
||||
process=process,
|
||||
created_at=time.time(),
|
||||
pid=process.pid,
|
||||
progress=10,
|
||||
)
|
||||
self.jobs[job_id] = managed
|
||||
data = self.serialize(managed)
|
||||
self._save_registry()
|
||||
return data
|
||||
|
||||
def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
job = self.jobs.get(job_id)
|
||||
if not job:
|
||||
return None
|
||||
return self.serialize(job)
|
||||
|
||||
def list_jobs(self) -> list[dict[str, Any]]:
|
||||
return [self.serialize(job) for job in self.jobs.values()]
|
||||
|
||||
def stop_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
job = self.jobs.get(job_id)
|
||||
if not job:
|
||||
return None
|
||||
if job.status not in TERMINAL_STATUSES:
|
||||
try:
|
||||
if job.process is not None and os.name == "nt":
|
||||
job.process.terminate()
|
||||
elif job.pid is not None:
|
||||
os.kill(job.pid, signal.SIGTERM)
|
||||
if job.process is not None:
|
||||
job.process.wait(timeout=10)
|
||||
except Exception:
|
||||
if job.process is not None:
|
||||
job.process.kill()
|
||||
elif job.pid is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
os.kill(job.pid, signal.SIGKILL)
|
||||
job.status = "stopped"
|
||||
job.progress = min(job.progress, 99)
|
||||
data = self.serialize(job)
|
||||
self._save_registry()
|
||||
return data
|
||||
|
||||
def logs(self, job_id: str) -> str:
|
||||
job = self.jobs.get(job_id)
|
||||
if not job or not job.log_path.exists():
|
||||
return ""
|
||||
return job.log_path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
def serialize(self, job: ManagedProcess) -> dict[str, Any]:
|
||||
code = job.process.poll() if job.process is not None else None
|
||||
if job.status not in TERMINAL_STATUSES:
|
||||
if job.process is None and job.pid is not None and not self._pid_alive(job.pid):
|
||||
job.status = "failed"
|
||||
job.progress = min(job.progress, 99)
|
||||
code = -1
|
||||
elif code is None:
|
||||
job.status = "running"
|
||||
elapsed = max(0, int(time.time() - job.created_at))
|
||||
job.progress = min(95, max(job.progress, 10 + elapsed // 6))
|
||||
elif code == 0:
|
||||
job.status = "completed"
|
||||
job.progress = 100
|
||||
job.artifacts = self._collect_artifacts(job.output_dir)
|
||||
else:
|
||||
job.status = "failed"
|
||||
job.progress = min(job.progress, 99)
|
||||
self._save_registry()
|
||||
return {
|
||||
"id": job.id,
|
||||
"name": job.name,
|
||||
"status": job.status,
|
||||
"progress": job.progress,
|
||||
"pid": job.pid,
|
||||
"gpus": job.gpus,
|
||||
"created_at": job.created_at,
|
||||
"command": job.command,
|
||||
"work_dir": job.work_dir,
|
||||
"output_dir": job.output_dir,
|
||||
"log_file": str(job.log_path),
|
||||
"artifacts": job.artifacts,
|
||||
"return_code": code,
|
||||
}
|
||||
|
||||
def locked_gpus(self) -> set[int]:
|
||||
locked: set[int] = set()
|
||||
for job in self.jobs.values():
|
||||
status = self.serialize(job)["status"]
|
||||
if status in {"queued", "running"}:
|
||||
locked.update(job.gpus)
|
||||
return locked
|
||||
|
||||
def _collect_artifacts(self, output_dir: str) -> list[dict[str, Any]]:
|
||||
root = Path(output_dir)
|
||||
if not root.exists():
|
||||
return []
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
for path in root.rglob("*"):
|
||||
if path.is_file():
|
||||
artifacts.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"name": path.name,
|
||||
"size": path.stat().st_size,
|
||||
}
|
||||
)
|
||||
return artifacts[:200]
|
||||
|
||||
def _save_registry(self) -> None:
|
||||
items = []
|
||||
for job in self.jobs.values():
|
||||
items.append(
|
||||
{
|
||||
"id": job.id,
|
||||
"name": job.name,
|
||||
"command": job.command,
|
||||
"work_dir": job.work_dir,
|
||||
"log_path": str(job.log_path),
|
||||
"output_dir": job.output_dir,
|
||||
"gpus": job.gpus,
|
||||
"pid": job.pid,
|
||||
"created_at": job.created_at,
|
||||
"status": job.status,
|
||||
"progress": job.progress,
|
||||
"artifacts": job.artifacts,
|
||||
}
|
||||
)
|
||||
self.registry_path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def _load_registry(self) -> None:
|
||||
if not self.registry_path.exists():
|
||||
return
|
||||
try:
|
||||
items = json.loads(self.registry_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return
|
||||
for item in items if isinstance(items, list) else []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
pid = item.get("pid")
|
||||
status = item.get("status", "failed")
|
||||
if status not in TERMINAL_STATUSES and pid and not self._pid_alive(int(pid)):
|
||||
status = "failed"
|
||||
job = ManagedProcess(
|
||||
id=str(item["id"]),
|
||||
name=str(item.get("name") or item["id"]),
|
||||
command=[str(part) for part in item.get("command") or []],
|
||||
work_dir=str(item.get("work_dir") or ""),
|
||||
log_path=Path(item.get("log_path") or self.log_root / f"{item['id']}.log"),
|
||||
output_dir=str(item.get("output_dir") or ""),
|
||||
gpus=[int(gpu) for gpu in item.get("gpus") or []],
|
||||
process=None,
|
||||
pid=int(pid) if pid else None,
|
||||
created_at=float(item.get("created_at") or time.time()),
|
||||
status=status,
|
||||
progress=int(item.get("progress") or 0),
|
||||
artifacts=item.get("artifacts") or [],
|
||||
)
|
||||
self.jobs[job.id] = job
|
||||
|
||||
def _pid_alive(self, pid: int) -> bool:
|
||||
if pid <= 0:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
@@ -2,12 +2,17 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import math
|
||||
import hashlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from compute.agent.process_manager import ProcessManager
|
||||
from compute.engines.llama_factory.adapter import build_command, parse_log_line
|
||||
|
||||
|
||||
@@ -15,6 +20,20 @@ def create_app() -> FastAPI:
|
||||
app = FastAPI(title="YG Fine-Tune Compute API")
|
||||
jobs: dict[str, dict[str, Any]] = {}
|
||||
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
|
||||
process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"))
|
||||
|
||||
@app.middleware("http")
|
||||
async def compute_token_auth(request: Request, call_next):
|
||||
token = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||
auth_enabled = os.getenv("COMPUTE_AUTH_ENABLED", "true").lower() == "true"
|
||||
public_paths = {f"{route_prefix}/health", "/health"}
|
||||
if auth_enabled and token and request.url.path not in public_paths:
|
||||
header_token = request.headers.get("x-compute-token", "")
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
bearer_token = auth_header.removeprefix("Bearer ").strip() if auth_header.startswith("Bearer ") else ""
|
||||
if header_token != token and bearer_token != token:
|
||||
return JSONResponse({"detail": "invalid compute service token"}, status_code=401)
|
||||
return await call_next(request)
|
||||
|
||||
def now() -> float:
|
||||
return time.time()
|
||||
@@ -25,6 +44,68 @@ def create_app() -> FastAPI:
|
||||
def execution_mode() -> str:
|
||||
return os.getenv("COMPUTE_EXECUTION_MODE", os.getenv("COMPUTE_MODE", "real")).lower()
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or raw == "":
|
||||
return default
|
||||
return int(raw)
|
||||
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or raw == "":
|
||||
return default
|
||||
return float(raw)
|
||||
|
||||
def _path_inside(root: Path, candidate: Path) -> bool:
|
||||
try:
|
||||
candidate.resolve().relative_to(root.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def _llama_factory_version() -> str:
|
||||
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=5)
|
||||
except Exception:
|
||||
continue
|
||||
output = (result.stdout or result.stderr).strip()
|
||||
if result.returncode == 0 and output:
|
||||
return output.splitlines()[0][:120]
|
||||
return ""
|
||||
|
||||
def _slice_log_content(
|
||||
content: str,
|
||||
tail_lines: int | None = None,
|
||||
offset: int | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
lines = content.splitlines()
|
||||
total = len(lines)
|
||||
if offset is not None or limit is not None:
|
||||
start = max(0, offset or 0)
|
||||
end = start + limit if limit else total
|
||||
selected = lines[start:end]
|
||||
else:
|
||||
tail = tail_lines or 200
|
||||
start = max(0, total - tail)
|
||||
selected = lines[start:]
|
||||
next_offset = start + len(selected)
|
||||
return {
|
||||
"content": "\n".join(selected),
|
||||
"total_lines": total,
|
||||
"offset": start,
|
||||
"limit": len(selected),
|
||||
"has_more": next_offset < total,
|
||||
"next_offset": next_offset if next_offset < total else None,
|
||||
}
|
||||
|
||||
def _safe_float(value: Any, default: float = 0) -> float:
|
||||
try:
|
||||
return float(str(value).replace("[N/A]", "").strip() or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def job_status(job: dict[str, Any]) -> dict[str, Any]:
|
||||
if execution_mode() != "simulator":
|
||||
return job
|
||||
@@ -74,9 +155,80 @@ def create_app() -> FastAPI:
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
def real_gpu_resources() -> list[dict[str, Any]]:
|
||||
query = (
|
||||
"index,uuid,name,memory.total,memory.used,utilization.gpu,"
|
||||
"temperature.gpu,power.draw,power.limit"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
except Exception:
|
||||
return fallback_gpu_resources()
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
for line in result.stdout.splitlines():
|
||||
parts = [part.strip() for part in line.split(",")]
|
||||
if len(parts) < 9:
|
||||
continue
|
||||
idx, uuid, name, mem_total, mem_used, util, temp, power, power_limit = parts[:9]
|
||||
total_gb = round(_safe_float(mem_total) / 1024, 2)
|
||||
used_gb = round(_safe_float(mem_used) / 1024, 2)
|
||||
memory_percent = round(used_gb / total_gb * 100, 1) if total_gb else 0
|
||||
gpu_percent = int(_safe_float(util))
|
||||
items.append(
|
||||
{
|
||||
"id": int(idx),
|
||||
"gpu_index": int(idx),
|
||||
"uuid": uuid,
|
||||
"name": name,
|
||||
"status": "busy" if gpu_percent >= 5 or used_gb > 1 else "idle",
|
||||
"gpu_percent": gpu_percent,
|
||||
"memory_used_gb": used_gb,
|
||||
"memory_total_gb": total_gb,
|
||||
"memory_percent": memory_percent,
|
||||
"temperature": int(_safe_float(temp)),
|
||||
"power_w": round(_safe_float(power), 1),
|
||||
"power_limit_w": round(_safe_float(power_limit), 1),
|
||||
"processes": [],
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
def fallback_gpu_resources() -> list[dict[str, Any]]:
|
||||
count = _int_env("COMPUTE_GPU_COUNT", 0)
|
||||
if count <= 0:
|
||||
return []
|
||||
name = os.getenv("COMPUTE_GPU_NAME", "Configured GPU")
|
||||
memory_total = _float_env("COMPUTE_GPU_MEMORY_GB", 80.0)
|
||||
power_limit = _float_env("COMPUTE_GPU_POWER_LIMIT_W", 300.0)
|
||||
return [
|
||||
{
|
||||
"id": idx,
|
||||
"gpu_index": idx,
|
||||
"uuid": f"GPU-{host_id().upper()}-{idx}",
|
||||
"name": name,
|
||||
"status": "idle",
|
||||
"gpu_percent": 0,
|
||||
"memory_used_gb": 0,
|
||||
"memory_total_gb": memory_total,
|
||||
"memory_percent": 0,
|
||||
"temperature": _int_env("COMPUTE_GPU_BASE_TEMPERATURE", 35),
|
||||
"power_w": 0,
|
||||
"power_limit_w": power_limit,
|
||||
"processes": [],
|
||||
}
|
||||
for idx in range(count)
|
||||
]
|
||||
|
||||
def gpu_resources() -> list[dict[str, Any]]:
|
||||
if execution_mode() != "simulator":
|
||||
return []
|
||||
return real_gpu_resources()
|
||||
active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}]
|
||||
gpus: list[dict[str, Any]] = []
|
||||
for idx in range(4):
|
||||
@@ -109,6 +261,101 @@ def create_app() -> FastAPI:
|
||||
)
|
||||
return gpus
|
||||
|
||||
def _check_path_item(item: dict[str, Any]) -> dict[str, Any]:
|
||||
path = Path(str(item.get("path") or ""))
|
||||
exists = path.exists()
|
||||
expected_type = str(item.get("type") or "any")
|
||||
ok = exists
|
||||
if exists and expected_type == "dir":
|
||||
ok = path.is_dir()
|
||||
if exists and expected_type == "file":
|
||||
ok = path.is_file()
|
||||
return {
|
||||
"name": item.get("name") or "",
|
||||
"path": str(path),
|
||||
"type": expected_type,
|
||||
"required": bool(item.get("required", True)),
|
||||
"exists": exists,
|
||||
"is_dir": path.is_dir() if exists else False,
|
||||
"is_file": path.is_file() if exists else False,
|
||||
"ok": ok or not item.get("required", True),
|
||||
}
|
||||
|
||||
def _job_preview(payload: dict[str, Any], check_paths: bool) -> dict[str, Any]:
|
||||
warnings: list[str] = []
|
||||
try:
|
||||
command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||
except ValueError as exc:
|
||||
return {
|
||||
"valid": False,
|
||||
"errors": [part.strip() for part in str(exc).split(";") if part.strip()],
|
||||
"warnings": warnings,
|
||||
"engine": str(payload.get("engine") or payload.get("training_engine") or "llama_factory"),
|
||||
"command": [],
|
||||
"command_text": "",
|
||||
"work_dir": os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"),
|
||||
"env": {},
|
||||
"path_checks": [],
|
||||
}
|
||||
|
||||
errors: list[str] = []
|
||||
engine = str(payload.get("engine") or payload.get("training_engine") or "llama_factory")
|
||||
path_checks: list[dict[str, Any]] = []
|
||||
if check_paths and engine != "smoke":
|
||||
path_checks = [
|
||||
_check_path_item(
|
||||
{
|
||||
"name": "model_name_or_path",
|
||||
"path": payload.get("model_name_or_path") or payload.get("base_model") or "",
|
||||
"type": "any",
|
||||
"required": True,
|
||||
}
|
||||
)
|
||||
]
|
||||
if payload.get("dataset_dir"):
|
||||
path_checks.append(
|
||||
_check_path_item(
|
||||
{
|
||||
"name": "dataset_dir",
|
||||
"path": payload.get("dataset_dir"),
|
||||
"type": "dir",
|
||||
"required": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
output_dir = Path(str(payload.get("output_dir") or "/data/yg-ft/outputs/training-job"))
|
||||
path_checks.append(
|
||||
_check_path_item(
|
||||
{
|
||||
"name": "output_parent",
|
||||
"path": str(output_dir.parent),
|
||||
"type": "dir",
|
||||
"required": False,
|
||||
}
|
||||
)
|
||||
)
|
||||
errors.extend(
|
||||
[f"{item['name']} path not available: {item['path']}" for item in path_checks if not item["ok"] and item["required"]]
|
||||
)
|
||||
if shutil.which(command.command[0]) is None:
|
||||
errors.append(f"training command not found: {command.command[0]}")
|
||||
if not Path(command.work_dir).exists():
|
||||
errors.append(f"llama_factory_home not found: {command.work_dir}")
|
||||
elif engine == "smoke":
|
||||
warnings.append("smoke engine skips model and dataset path checks")
|
||||
|
||||
return {
|
||||
"valid": not errors,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
"engine": engine,
|
||||
"command": command.command,
|
||||
"command_text": " ".join(command.command),
|
||||
"work_dir": command.work_dir,
|
||||
"env": command.env,
|
||||
"path_checks": path_checks,
|
||||
}
|
||||
|
||||
@app.get(f"{route_prefix}/health")
|
||||
async def health_check() -> dict[str, str]:
|
||||
return {
|
||||
@@ -116,41 +363,121 @@ def create_app() -> FastAPI:
|
||||
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
||||
}
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check_root() -> dict[str, str]:
|
||||
return await health_check()
|
||||
|
||||
@app.get(f"{route_prefix}/v1/compute/health")
|
||||
async def compute_health_check() -> dict[str, str | bool]:
|
||||
async def compute_health_check() -> dict[str, Any]:
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
dataset_root = Path(os.getenv("YG_FT_DATASET_ROOT", str(data_root / "datasets")))
|
||||
output_root = Path(os.getenv("YG_FT_OUTPUT_ROOT", str(data_root / "outputs")))
|
||||
llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||
return {
|
||||
"status": "ok",
|
||||
"api_version": "v1",
|
||||
"compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
|
||||
"app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
|
||||
"data_root": str(data_root),
|
||||
"data_root_exists": data_root.exists(),
|
||||
"model_root": os.getenv("YG_FT_MODEL_ROOT", str(data_root / "models")),
|
||||
"dataset_root": str(dataset_root),
|
||||
"dataset_root_exists": dataset_root.exists(),
|
||||
"output_root": str(output_root),
|
||||
"output_root_exists": output_root.exists(),
|
||||
"log_root": os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"),
|
||||
"llama_factory_home": str(llama_factory_home),
|
||||
"llama_factory_home_exists": llama_factory_home.exists(),
|
||||
"llama_factory_version": os.getenv("LLAMA_FACTORY_VERSION", ""),
|
||||
"execution_mode": execution_mode(),
|
||||
"gpu_count": _int_env("COMPUTE_GPU_COUNT", 0),
|
||||
"gpu_discovery_endpoint": f"{route_prefix}/compute/resources/gpus",
|
||||
"capabilities": ["gpu_discovery", "llama_factory", "file_gateway", "job_polling"],
|
||||
}
|
||||
|
||||
@app.get(f"{route_prefix}/v1/compute/jobs")
|
||||
async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]:
|
||||
return {"items": [job_status(job) for job in jobs.values()]}
|
||||
items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()]
|
||||
return {"items": items}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/resources/gpus")
|
||||
async def list_gpus() -> dict[str, Any]:
|
||||
return {"items": gpu_resources(), "compute_host_id": host_id()}
|
||||
|
||||
@app.get(f"{route_prefix}/v1/compute/resources/gpus")
|
||||
async def list_gpus_v1() -> dict[str, Any]:
|
||||
return {"items": gpu_resources(), "compute_host_id": host_id()}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/jobs/preview")
|
||||
async def preview_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _job_preview(payload, check_paths=False)
|
||||
|
||||
@app.post(f"{route_prefix}/compute/jobs/validate")
|
||||
async def validate_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return _job_preview(payload, check_paths=True)
|
||||
|
||||
@app.post(f"{route_prefix}/v1/compute/jobs/preview")
|
||||
async def preview_job_v1(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return await preview_job(payload)
|
||||
|
||||
@app.post(f"{route_prefix}/v1/compute/jobs/validate")
|
||||
async def validate_job_v1(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return await validate_job(payload)
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/check-paths")
|
||||
async def check_paths(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
items = [_check_path_item(item) for item in payload.get("paths", []) if isinstance(item, dict)]
|
||||
return {"valid": all(item["ok"] for item in items), "items": items}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/list")
|
||||
async def list_files(
|
||||
root: str = Query(default="data"),
|
||||
relative_path: str = Query(default=""),
|
||||
directories_only: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
roots = {
|
||||
"data": Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")),
|
||||
"models": Path(os.getenv("YG_FT_MODEL_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/models")),
|
||||
"datasets": Path(os.getenv("YG_FT_DATASET_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/datasets")),
|
||||
"outputs": Path(os.getenv("YG_FT_OUTPUT_ROOT", os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft") + "/outputs")),
|
||||
}
|
||||
base = roots.get(root)
|
||||
if base is None:
|
||||
raise HTTPException(status_code=400, detail="invalid root")
|
||||
target = (base / relative_path.lstrip("/\\")).resolve()
|
||||
if not _path_inside(base, target):
|
||||
raise HTTPException(status_code=400, detail="path must stay inside selected root")
|
||||
if not target.exists():
|
||||
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": []}
|
||||
items = []
|
||||
for child in sorted(target.iterdir(), key=lambda path: (not path.is_dir(), path.name.lower())):
|
||||
if directories_only and not child.is_dir():
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"name": child.name,
|
||||
"path": str(child),
|
||||
"relative_path": str(child.relative_to(base)).replace("\\", "/"),
|
||||
"type": "directory" if child.is_dir() else "file",
|
||||
"byte_size": child.stat().st_size if child.is_file() else 0,
|
||||
}
|
||||
)
|
||||
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/jobs")
|
||||
async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
try:
|
||||
command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
if execution_mode() != "simulator":
|
||||
raise HTTPException(
|
||||
status_code=501,
|
||||
detail="real compute executor is not implemented yet; set COMPUTE_EXECUTION_MODE=simulator only for isolated development",
|
||||
)
|
||||
job_id = str(payload.get("id") or f"job_{int(now() * 1000)}")
|
||||
if execution_mode() != "simulator":
|
||||
try:
|
||||
return process_manager.create_job({**payload, "id": job_id}, command.command, command.work_dir)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=500, detail=f"training command not found: {exc.filename}")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
job = {
|
||||
"id": job_id,
|
||||
"name": payload.get("name", job_id),
|
||||
@@ -169,17 +496,29 @@ def create_app() -> FastAPI:
|
||||
|
||||
@app.get(f"{route_prefix}/compute/jobs")
|
||||
async def list_jobs() -> dict[str, Any]:
|
||||
return {"items": [job_status(job) for job in jobs.values()]}
|
||||
items = process_manager.list_jobs() if execution_mode() != "simulator" else [job_status(job) for job in jobs.values()]
|
||||
return {"items": items}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}")
|
||||
async def get_job(job_id: str) -> dict[str, Any]:
|
||||
job = jobs.get(job_id)
|
||||
if execution_mode() != "simulator":
|
||||
job = process_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
return job
|
||||
job = jobs.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
return job_status(job)
|
||||
|
||||
@app.post(f"{route_prefix}/compute/jobs/{{job_id}}/stop")
|
||||
async def stop_job(job_id: str) -> dict[str, Any]:
|
||||
if execution_mode() != "simulator":
|
||||
job = process_manager.stop_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
return job
|
||||
job = jobs.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
@@ -188,22 +527,101 @@ def create_app() -> FastAPI:
|
||||
return job
|
||||
|
||||
@app.get(f"{route_prefix}/compute/jobs/{{job_id}}/logs")
|
||||
async def job_logs(job_id: str) -> dict[str, Any]:
|
||||
async def job_logs(
|
||||
job_id: str,
|
||||
tail_lines: int | None = Query(default=200, ge=1, le=5000),
|
||||
offset: int | None = Query(default=None, ge=0),
|
||||
limit: int | None = Query(default=None, ge=1, le=5000),
|
||||
) -> dict[str, Any]:
|
||||
if execution_mode() != "simulator":
|
||||
job = process_manager.get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
content = process_manager.logs(job_id)
|
||||
else:
|
||||
job = jobs.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
job = job_status(job)
|
||||
metrics = [parse_log_line(line) for line in job["logs"].splitlines()]
|
||||
return {"job_id": job_id, "content": job["logs"], "metrics": [m for m in metrics if m]}
|
||||
content = job["logs"]
|
||||
window = _slice_log_content(content, tail_lines, offset, limit)
|
||||
metrics = [parse_log_line(line) for line in window["content"].splitlines()]
|
||||
return {"job_id": job_id, **window, "metrics": [m for m in metrics if m]}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/upload")
|
||||
async def upload_file(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
file_id = str(payload.get("id") or f"file_{int(now() * 1000)}")
|
||||
return {"id": file_id, "status": "available", "local_path": f"/data/yg-ft/uploads/{file_id}"}
|
||||
async def upload_file(
|
||||
file: UploadFile | None = File(default=None),
|
||||
target_relative_path: str | None = Form(default=None),
|
||||
resource_type: str | None = Form(default=None),
|
||||
resource_id: str | None = Form(default=None),
|
||||
) -> dict[str, Any]:
|
||||
file_id = f"file_{int(now() * 1000)}"
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
data_root.mkdir(parents=True, exist_ok=True)
|
||||
filename = Path(file.filename if file else file_id).name
|
||||
if target_relative_path:
|
||||
target = (data_root / target_relative_path.lstrip("/\\")).resolve()
|
||||
if not _path_inside(data_root, target):
|
||||
raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT")
|
||||
else:
|
||||
target = data_root / "uploads" / f"{file_id}_{filename}"
|
||||
if file:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with target.open("wb") as output:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
output.write(chunk)
|
||||
else:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("", encoding="utf-8")
|
||||
return {
|
||||
"id": file_id,
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
"status": "available",
|
||||
"local_path": str(target),
|
||||
"byte_size": target.stat().st_size,
|
||||
"checksum_sha256": hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "",
|
||||
}
|
||||
|
||||
@app.post(f"{route_prefix}/compute/files/import-local")
|
||||
async def import_local_file(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
source = Path(str(payload.get("source_path") or ""))
|
||||
if not source.exists():
|
||||
raise HTTPException(status_code=404, detail="source path not found")
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
data_root.mkdir(parents=True, exist_ok=True)
|
||||
relative = str(payload.get("target_relative_path") or f"imports/{source.name}").lstrip("/\\")
|
||||
target = (data_root / relative).resolve()
|
||||
if not _path_inside(data_root, target):
|
||||
raise HTTPException(status_code=400, detail="target path must stay inside YG_FT_DATA_ROOT")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if source.is_dir():
|
||||
if target.exists():
|
||||
shutil.rmtree(target)
|
||||
shutil.copytree(source, target)
|
||||
byte_size = sum(path.stat().st_size for path in target.rglob("*") if path.is_file())
|
||||
checksum = ""
|
||||
else:
|
||||
shutil.copy2(source, target)
|
||||
byte_size = target.stat().st_size
|
||||
checksum = hashlib.sha256(target.read_bytes()).hexdigest()
|
||||
return {
|
||||
"id": str(payload.get("id") or f"file_{int(now() * 1000)}"),
|
||||
"resource_type": payload.get("resource_type"),
|
||||
"resource_id": payload.get("resource_id"),
|
||||
"status": "available",
|
||||
"local_path": str(target),
|
||||
"byte_size": byte_size,
|
||||
"checksum_sha256": checksum,
|
||||
}
|
||||
|
||||
@app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
|
||||
async def download_file(file_id: str) -> dict[str, Any]:
|
||||
return {"id": file_id, "status": "ready", "download_url": f"{route_prefix}/compute/files/{file_id}/download"}
|
||||
async def download_file(file_id: str) -> FileResponse:
|
||||
upload_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) / "uploads"
|
||||
matches = list(upload_root.glob(f"{file_id}_*"))
|
||||
if not matches:
|
||||
raise HTTPException(status_code=404, detail="file not found")
|
||||
return FileResponse(matches[0])
|
||||
|
||||
return app
|
||||
|
||||
|
||||
@@ -19,22 +19,57 @@ def validate_config(config: dict[str, Any]) -> list[str]:
|
||||
errors.append("base_model or model_name_or_path is required")
|
||||
if not config.get("dataset") and not config.get("dataset_dir"):
|
||||
errors.append("dataset or dataset_dir is required")
|
||||
try:
|
||||
learning_rate = float(config.get("learning_rate", 0.0002))
|
||||
except (TypeError, ValueError):
|
||||
learning_rate = 0
|
||||
if learning_rate <= 0:
|
||||
errors.append("learning_rate must be greater than zero")
|
||||
try:
|
||||
epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
|
||||
except (TypeError, ValueError):
|
||||
epochs = 0
|
||||
if epochs <= 0:
|
||||
errors.append("n_epochs must be greater than zero")
|
||||
return errors
|
||||
|
||||
|
||||
def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None:
|
||||
for key in keys:
|
||||
value = config.get(key)
|
||||
if value is not None and value != "":
|
||||
command.extend([option, str(value)])
|
||||
return
|
||||
|
||||
|
||||
def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
|
||||
errors = validate_config(config)
|
||||
if errors:
|
||||
raise ValueError("; ".join(errors))
|
||||
|
||||
engine = str(config.get("engine") or config.get("training_engine") or "llama_factory")
|
||||
if engine == "smoke":
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}"
|
||||
script = (
|
||||
"import json, os, time; "
|
||||
f"out={str(output_dir)!r}; "
|
||||
"os.makedirs(out, exist_ok=True); "
|
||||
"print('[INFO] smoke training started', flush=True); "
|
||||
"\nfor step in range(1, 7):\n"
|
||||
" loss=round(1.8/(step+1), 4)\n"
|
||||
" lr=round(0.0002*(1-step/10), 8)\n"
|
||||
" print({'loss': loss, 'grad_norm': round(0.4 + step*0.03, 4), 'learning_rate': lr, 'epoch': round(step/6, 4)}, flush=True)\n"
|
||||
" time.sleep(0.4)\n"
|
||||
"\nopen(os.path.join(out, 'adapter_config.json'), 'w', encoding='utf-8').write(json.dumps({'engine':'smoke','status':'completed'})); "
|
||||
"print('***** train metrics *****', flush=True); "
|
||||
"print('train_loss = 0.12', flush=True); "
|
||||
"print('***** train metrics end *****', flush=True)"
|
||||
)
|
||||
return LlamaFactoryCommand(command=["python", "-u", "-c", script], work_dir="/app", env={})
|
||||
|
||||
model_path = config.get("base_model") or config.get("model_name_or_path")
|
||||
dataset = config.get("dataset") or config.get("dataset_dir")
|
||||
dataset = config.get("dataset") or config.get("dataset_name")
|
||||
dataset_dir = config.get("dataset_dir")
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}"
|
||||
command = [
|
||||
"llamafactory-cli",
|
||||
@@ -46,7 +81,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
"--model_name_or_path",
|
||||
str(model_path),
|
||||
"--dataset",
|
||||
str(dataset),
|
||||
str(dataset or "default"),
|
||||
"--template",
|
||||
str(config.get("template", "qwen")),
|
||||
"--finetuning_type",
|
||||
@@ -61,7 +96,22 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
str(config.get("n_epochs", 3)),
|
||||
"--save_steps",
|
||||
str(config.get("save_steps", 50)),
|
||||
"--logging_steps",
|
||||
str(config.get("logging_steps", 10)),
|
||||
"--overwrite_output_dir",
|
||||
"true",
|
||||
"--plot_loss",
|
||||
"true",
|
||||
]
|
||||
if dataset_dir:
|
||||
command.extend(["--dataset_dir", str(dataset_dir)])
|
||||
_optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len")
|
||||
_optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type")
|
||||
_optional_arg(config, command, "--warmup_ratio", "warmup_ratio")
|
||||
_optional_arg(config, command, "--weight_decay", "weight_decay")
|
||||
_optional_arg(config, command, "--lora_rank", "lora_rank", "rank")
|
||||
_optional_arg(config, command, "--lora_alpha", "lora_alpha")
|
||||
_optional_arg(config, command, "--lora_dropout", "lora_dropout")
|
||||
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
||||
if quantization_bit in {4, 8}:
|
||||
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
fastapi>=0.111.0
|
||||
uvicorn[standard]>=0.30.0
|
||||
python-multipart>=0.0.9
|
||||
pydantic>=2.7.0
|
||||
python-dotenv>=1.0.1
|
||||
httpx>=0.27.0
|
||||
|
||||
@@ -56,7 +56,7 @@ $images | ForEach-Object { docker pull $_ }
|
||||
| PostgreSQL | `15432` | `5432` | 开发阶段内置数据库 |
|
||||
| Redis | `16379` | `6379` | 开发阶段内置缓存 |
|
||||
| Compute API | `19100` | `9100` | 算力服务器 API |
|
||||
| File Gateway | `19101` | 后续服务端口 | 当前预留,后续拆出文件网关服务时使用 |
|
||||
| File Gateway | `19101` | `9100` | 当前由 Compute API 暴露文件网关契约,后续可拆为独立服务 |
|
||||
|
||||
注意:`8000` 是后端容器内部端口,不作为宿主机对外访问端口。宿主机或浏览器应访问 `http://<app-server-ip>:17861/modelTF/health`;前端 Nginx 容器在 Docker 网络内部访问 `http://backend-api:8000/modelTF/...`。
|
||||
|
||||
@@ -142,6 +142,16 @@ docker compose logs --tail=80 frontend
|
||||
|
||||
如果使用企业统一 PostgreSQL/Redis,修改 `docker/app/.env`:
|
||||
|
||||
如果前端 Nginx 日志出现 `open() "/usr/share/nginx/html/modelTF/login" failed` 或 `open() "/usr/share/nginx/html/login" failed`,说明当前容器没有加载项目的 Nginx 代理配置,`/modelTF/*` 被当成静态文件查找。处理方式:
|
||||
|
||||
```bash
|
||||
cd <repo-root>/docker/app
|
||||
docker compose up -d --force-recreate frontend
|
||||
docker compose exec frontend nginx -T | grep -n "location.*modelTF" -A12
|
||||
```
|
||||
|
||||
正常配置中应存在 `location ^~ /modelTF/`,并代理到 `BACKEND_PROXY_PASS`,默认是 `http://backend-api:8000`。
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql+psycopg://<user>:<password>@<postgres-host>:15432/<db>
|
||||
REDIS_URL=redis://<redis-host>:16379/0
|
||||
@@ -192,10 +202,28 @@ GET http://<compute-server-ip>:19100/modelTF/v1/compute/health
|
||||
```text
|
||||
../../compute -> /app/compute
|
||||
${YG_FT_DATA_ROOT_HOST} -> /data/yg-ft
|
||||
../../runtime/compute/logs -> /opt/yg-ft/logs/compute
|
||||
../../runtime/compute/training-logs -> /opt/yg-ft/logs/training
|
||||
${YG_FT_MODEL_ROOT_HOST} -> /data/yg-ft/models
|
||||
${YG_FT_DATASET_ROOT_HOST} -> /data/yg-ft/datasets
|
||||
${YG_FT_OUTPUT_ROOT_HOST} -> /data/yg-ft/outputs
|
||||
${COMPUTE_LOG_ROOT_HOST} -> /opt/yg-ft/logs/compute
|
||||
${TRAINING_LOG_ROOT_HOST} -> /opt/yg-ft/logs/training
|
||||
```
|
||||
|
||||
算力服务器启动前必须先在宿主机创建持久化目录,基座模型、训练数据、训练产物和训练日志都应落在宿主机磁盘上,不能只写入容器层。推荐默认目录:
|
||||
|
||||
```bash
|
||||
cd <repo-root>/docker/compute
|
||||
mkdir -p data/yg-ft/models \
|
||||
data/yg-ft/datasets \
|
||||
data/yg-ft/outputs \
|
||||
data/yg-ft/logs/compute \
|
||||
data/yg-ft/logs/training
|
||||
```
|
||||
|
||||
默认 `docker/compute/.env.example` 使用 `./data/yg-ft`,该相对路径以 `docker/compute/docker-compose.yml` 所在目录为基准,因此实际宿主机目录是 `<repo-root>/docker/compute/data/yg-ft`。如企业环境模型盘、数据盘、产物盘分盘挂载,可在 `docker/compute/.env` 中分别调整 `YG_FT_MODEL_ROOT_HOST`、`YG_FT_DATASET_ROOT_HOST`、`YG_FT_OUTPUT_ROOT_HOST`、`COMPUTE_LOG_ROOT_HOST`、`TRAINING_LOG_ROOT_HOST`,容器内路径建议保持 `/data/yg-ft/models`、`/data/yg-ft/datasets`、`/data/yg-ft/outputs`,避免训练参数和节点配置复杂化。
|
||||
|
||||
页面上传数据集时,文件先进入 Backend API,再由 Backend API 调用目标算力节点的 `POST /modelTF/compute/files/upload`,写入容器内 `/data/yg-ft/datasets/{dataset_id}/`。在默认开发配置下,宿主机可在 `<repo-root>/docker/compute/data/yg-ft/datasets/{dataset_id}/` 看到对应文件。仅创建 bind mount 不会自动让应用侧上传文件出现在算力目录,必须通过这条 File Gateway 链路同步。
|
||||
|
||||
## 应用与算力分离部署
|
||||
|
||||
应用服务器只需要主动访问算力服务器,不要求算力服务器回调应用服务器。
|
||||
@@ -207,7 +235,7 @@ COMPUTE_API_BASE_URL=http://<compute-server-ip>:19100
|
||||
FILE_GATEWAY_BASE_URL=http://<compute-server-ip>:19101
|
||||
COMPUTE_SERVICE_TOKEN=change_me
|
||||
COMPUTE_STATUS_SYNC_MODE=polling
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=10
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=3
|
||||
COMPUTE_POLL_BATCH_SIZE=100
|
||||
```
|
||||
|
||||
@@ -222,6 +250,8 @@ Frontend
|
||||
<- Backend Worker 定时轮询 Compute API
|
||||
```
|
||||
|
||||
算力服务默认开启服务间鉴权。`docker/compute/.env` 中保持 `COMPUTE_AUTH_ENABLED=true`,并确保 `COMPUTE_SERVICE_TOKEN` 与 `docker/app/.env` 一致;健康检查路径仍可用于容器探活。
|
||||
|
||||
## 多算力节点部署
|
||||
|
||||
多算力节点仍按“单机多 GPU 节点”部署。每台 GPU 服务器都独立部署一套 `docker/compute`:
|
||||
@@ -234,6 +264,8 @@ gpu-node-03: docker/compute + /data/yg-ft + 19100/19101
|
||||
|
||||
节点之间默认不互访。应用平台主动访问每个节点的 Compute API/File Gateway,并通过 `compute_nodes`、`resource_replicas`、`resource_sync_jobs` 统一调度和同步。
|
||||
|
||||
节点地址、权重、标签、启用状态和本地路径在前端“算力节点”页面动态维护。新增或编辑节点后,点击“测试”会由 Backend API 主动访问该节点的 `GET /modelTF/v1/compute/health` 和 `GET /modelTF/compute/resources/gpus`,并把健康信息与 GPU 清单同步到 PostgreSQL。
|
||||
|
||||
## 常用命令
|
||||
|
||||
重新构建应用镜像:
|
||||
|
||||
@@ -41,5 +41,6 @@ FILE_GATEWAY_BASE_URL=http://compute-api:9101
|
||||
# The application side polls Compute API for job state to avoid opening reverse network access.
|
||||
COMPUTE_MODE=real
|
||||
COMPUTE_STATUS_SYNC_MODE=polling
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=10
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=3
|
||||
COMPUTE_POLL_BATCH_SIZE=100
|
||||
COMPUTE_REQUEST_TIMEOUT_SECONDS=5
|
||||
|
||||
@@ -21,6 +21,8 @@ services:
|
||||
ls -la /usr/share/nginx/html;
|
||||
exit 1;
|
||||
fi;
|
||||
envsubst '$$BACKEND_PROXY_PASS' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf;
|
||||
nginx -t;
|
||||
nginx -g 'daemon off;'
|
||||
networks:
|
||||
- yg-ft-app
|
||||
@@ -58,8 +60,9 @@ services:
|
||||
FILE_GATEWAY_BASE_URL: ${FILE_GATEWAY_BASE_URL:-http://compute-api:9101}
|
||||
COMPUTE_MODE: ${COMPUTE_MODE:-real}
|
||||
COMPUTE_STATUS_SYNC_MODE: ${COMPUTE_STATUS_SYNC_MODE:-polling}
|
||||
COMPUTE_POLL_INTERVAL_SECONDS: ${COMPUTE_POLL_INTERVAL_SECONDS:-10}
|
||||
COMPUTE_POLL_INTERVAL_SECONDS: ${COMPUTE_POLL_INTERVAL_SECONDS:-3}
|
||||
COMPUTE_POLL_BATCH_SIZE: ${COMPUTE_POLL_BATCH_SIZE:-100}
|
||||
COMPUTE_REQUEST_TIMEOUT_SECONDS: ${COMPUTE_REQUEST_TIMEOUT_SECONDS:-5}
|
||||
PYTHONPATH: /app
|
||||
volumes:
|
||||
- ../../backend:/app:ro
|
||||
|
||||
@@ -8,14 +8,35 @@ FILE_GATEWAY_PORT=19101
|
||||
COMPUTE_API_IMAGE=yg-ft-compute-api:latest
|
||||
|
||||
# The application server actively polls Compute API; compute server does not need reverse access.
|
||||
COMPUTE_AUTH_ENABLED=true
|
||||
COMPUTE_SERVICE_TOKEN=change_me
|
||||
ENABLE_APP_CALLBACK=false
|
||||
|
||||
# LLaMA-Factory is provided by the official hiyouga/llamafactory base image.
|
||||
LLAMA_FACTORY_HOME=/app/LLaMA-Factory
|
||||
|
||||
# Persistent host directories on the compute server.
|
||||
# Create these directories before starting docker compose. They are mounted into
|
||||
# the container so base models, datasets, training outputs and logs survive
|
||||
# container recreation or image upgrades.
|
||||
YG_FT_DATA_ROOT=/data/yg-ft
|
||||
YG_FT_DATA_ROOT_HOST=/data/yg-ft
|
||||
YG_FT_DATA_ROOT_HOST=./data/yg-ft
|
||||
YG_FT_MODEL_ROOT=/data/yg-ft/models
|
||||
YG_FT_MODEL_ROOT_HOST=./data/yg-ft/models
|
||||
YG_FT_DATASET_ROOT=/data/yg-ft/datasets
|
||||
YG_FT_DATASET_ROOT_HOST=./data/yg-ft/datasets
|
||||
YG_FT_OUTPUT_ROOT=/data/yg-ft/outputs
|
||||
YG_FT_OUTPUT_ROOT_HOST=./data/yg-ft/outputs
|
||||
TRAINING_LOG_ROOT=/opt/yg-ft/logs/training
|
||||
TRAINING_LOG_ROOT_HOST=./data/yg-ft/logs/training
|
||||
COMPUTE_LOG_ROOT_HOST=./data/yg-ft/logs/compute
|
||||
|
||||
# Optional fallback used when nvidia-smi is unavailable.
|
||||
# Leave COMPUTE_GPU_COUNT=0 on real GPU servers with working NVIDIA runtime.
|
||||
COMPUTE_GPU_COUNT=0
|
||||
COMPUTE_GPU_NAME=NVIDIA A800-SXM4-80GB
|
||||
COMPUTE_GPU_MEMORY_GB=80
|
||||
COMPUTE_GPU_POWER_LIMIT_W=300
|
||||
|
||||
LOG_DIR=/opt/yg-ft/logs/compute
|
||||
CUDA_VISIBLE_DEVICES=all
|
||||
|
||||
38
docker/compute/data/yg-ft/README.md
Normal file
38
docker/compute/data/yg-ft/README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# YG-FT Compute 数据目录说明
|
||||
|
||||
本目录挂载到 `yg-ft-compute-api` 容器的 `/data/yg-ft`,用于持久化存储训练相关的数据。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
data/yg-ft/
|
||||
├── datasets/ # 数据集存储目录
|
||||
├── models/ # 模型文件存储目录
|
||||
├── outputs/ # 训练/推理输出结果目录
|
||||
└── logs/
|
||||
├── compute/ # 计算服务运行日志
|
||||
└── training/ # 训练任务执行日志
|
||||
```
|
||||
|
||||
## 各目录说明
|
||||
|
||||
### datasets/
|
||||
训练和评估所使用的数据集文件,包括 JSON、JSONL、CSV 等格式。数据集由用户上传或通过平台创建,供 LLaMA-Factory 等训练引擎读取。
|
||||
|
||||
### models/
|
||||
存放模型文件,包括:
|
||||
- 预训练基座模型(如 LLaMA、Qwen 等)
|
||||
- 微调后的自定义模型权重
|
||||
- 合并后的部署模型
|
||||
|
||||
### outputs/
|
||||
训练任务和推理任务的输出结果,包括:
|
||||
- 训练过程中的 checkpoint 文件
|
||||
- 评估结果和指标报告
|
||||
- 推理生成的结果文本
|
||||
|
||||
### logs/compute/
|
||||
计算服务(compute-api)的运行时日志,用于排查服务启动、GPU 调度、健康检查等问题。
|
||||
|
||||
### logs/training/
|
||||
各训练任务的执行日志,记录训练过程状态、报错信息等,便于追踪单个任务的运行情况。
|
||||
0
docker/compute/data/yg-ft/datasets/.gitkeep
Normal file
0
docker/compute/data/yg-ft/datasets/.gitkeep
Normal file
0
docker/compute/data/yg-ft/logs/compute/.gitkeep
Normal file
0
docker/compute/data/yg-ft/logs/compute/.gitkeep
Normal file
0
docker/compute/data/yg-ft/logs/training/.gitkeep
Normal file
0
docker/compute/data/yg-ft/logs/training/.gitkeep
Normal file
0
docker/compute/data/yg-ft/models/.gitkeep
Normal file
0
docker/compute/data/yg-ft/models/.gitkeep
Normal file
0
docker/compute/data/yg-ft/outputs/.gitkeep
Normal file
0
docker/compute/data/yg-ft/outputs/.gitkeep
Normal file
@@ -5,15 +5,25 @@ services:
|
||||
gpus: all
|
||||
ports:
|
||||
- "${COMPUTE_API_PORT:-19100}:9100"
|
||||
- "${FILE_GATEWAY_PORT:-19101}:9100"
|
||||
environment:
|
||||
COMPUTE_ENV: ${COMPUTE_ENV:-prod}
|
||||
COMPUTE_HOST_ID: ${COMPUTE_HOST_ID:-gpu-node-01}
|
||||
COMPUTE_EXECUTION_MODE: ${COMPUTE_EXECUTION_MODE:-real}
|
||||
MODELTF_ROUTE_PREFIX: ${MODELTF_ROUTE_PREFIX:-/modelTF}
|
||||
COMPUTE_AUTH_ENABLED: ${COMPUTE_AUTH_ENABLED:-true}
|
||||
COMPUTE_SERVICE_TOKEN: ${COMPUTE_SERVICE_TOKEN:-change_me}
|
||||
ENABLE_APP_CALLBACK: ${ENABLE_APP_CALLBACK:-false}
|
||||
LLAMA_FACTORY_HOME: ${LLAMA_FACTORY_HOME:-/app/LLaMA-Factory}
|
||||
YG_FT_DATA_ROOT: ${YG_FT_DATA_ROOT:-/data/yg-ft}
|
||||
YG_FT_MODEL_ROOT: ${YG_FT_MODEL_ROOT:-/data/yg-ft/models}
|
||||
YG_FT_DATASET_ROOT: ${YG_FT_DATASET_ROOT:-/data/yg-ft/datasets}
|
||||
YG_FT_OUTPUT_ROOT: ${YG_FT_OUTPUT_ROOT:-/data/yg-ft/outputs}
|
||||
TRAINING_LOG_ROOT: ${TRAINING_LOG_ROOT:-/opt/yg-ft/logs/training}
|
||||
COMPUTE_GPU_COUNT: ${COMPUTE_GPU_COUNT:-0}
|
||||
COMPUTE_GPU_NAME: ${COMPUTE_GPU_NAME:-NVIDIA A800-SXM4-80GB}
|
||||
COMPUTE_GPU_MEMORY_GB: ${COMPUTE_GPU_MEMORY_GB:-80}
|
||||
COMPUTE_GPU_POWER_LIMIT_W: ${COMPUTE_GPU_POWER_LIMIT_W:-300}
|
||||
LOG_DIR: ${LOG_DIR:-/opt/yg-ft/logs/compute}
|
||||
CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-all}
|
||||
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
||||
@@ -21,9 +31,12 @@ services:
|
||||
PYTHONPATH: /app
|
||||
volumes:
|
||||
- ../../compute:/app/compute:ro
|
||||
- ${YG_FT_DATA_ROOT_HOST:-/data/yg-ft}:${YG_FT_DATA_ROOT:-/data/yg-ft}
|
||||
- ../../runtime/compute/logs:/opt/yg-ft/logs/compute
|
||||
- ../../runtime/compute/training-logs:/opt/yg-ft/logs/training
|
||||
- ${YG_FT_DATA_ROOT_HOST:-./data/yg-ft}:${YG_FT_DATA_ROOT:-/data/yg-ft}
|
||||
- ${YG_FT_MODEL_ROOT_HOST:-./data/yg-ft/models}:${YG_FT_MODEL_ROOT:-/data/yg-ft/models}
|
||||
- ${YG_FT_DATASET_ROOT_HOST:-./data/yg-ft/datasets}:${YG_FT_DATASET_ROOT:-/data/yg-ft/datasets}
|
||||
- ${YG_FT_OUTPUT_ROOT_HOST:-./data/yg-ft/outputs}:${YG_FT_OUTPUT_ROOT:-/data/yg-ft/outputs}
|
||||
- ${COMPUTE_LOG_ROOT_HOST:-./data/yg-ft/logs/compute}:${LOG_DIR:-/opt/yg-ft/logs/compute}
|
||||
- ${TRAINING_LOG_ROOT_HOST:-./data/yg-ft/logs/training}:${TRAINING_LOG_ROOT:-/opt/yg-ft/logs/training}
|
||||
networks:
|
||||
- yg-ft-compute
|
||||
healthcheck:
|
||||
|
||||
@@ -7,11 +7,18 @@ server {
|
||||
|
||||
client_max_body_size 200m;
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
location ^~ /modelTF/ {
|
||||
proxy_pass ${BACKEND_PROXY_PASS};
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
}
|
||||
|
||||
location /modelTF {
|
||||
location = /modelTF {
|
||||
proxy_pass ${BACKEND_PROXY_PASS};
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
@@ -27,4 +34,8 @@ server {
|
||||
expires 30d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,7 +271,7 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
| POST | `/modelTF/dataset-manage` | 创建数据集 |
|
||||
| PUT | `/modelTF/dataset-manage/{id}` | 更新数据集 |
|
||||
| DELETE | `/modelTF/dataset-manage/{id}` | 删除数据集 |
|
||||
| POST | `/modelTF/dataset-manage/upload/{dataset_id}` | 上传文件,字段名 `files` |
|
||||
| POST | `/modelTF/dataset-manage/upload/{dataset_id}` | 上传文件,字段名 `files`;默认同步到启用的算力节点 `/data/yg-ft/datasets/{dataset_id}/` |
|
||||
| GET | `/modelTF/dataset-manage/download/{dataset_id}` | 打包下载数据集 |
|
||||
| GET | `/modelTF/dataset-manage/download/{dataset_id}/{file_id}` | 下载单文件 |
|
||||
|
||||
@@ -451,7 +451,9 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
| GET | `/modelTF/fine-tune/{id}` | 训练任务详情 |
|
||||
| GET | `/modelTF/fine-tune/check-name?name=xxx` | 任务名查重 |
|
||||
| POST | `/modelTF/fine-tune` | 创建训练任务记录 |
|
||||
| POST | `/modelTF/fine-tune/start` | 启动训练 |
|
||||
| POST | `/modelTF/fine-tune/{id}/command-preview` | 训练创建页/详情页命令预览,返回目标节点、Compute Job payload 和 LLaMA-Factory 命令 |
|
||||
| POST | `/modelTF/fine-tune/{id}/preflight` | 训练创建页启动前预检,校验节点、模型路径、数据集路径、引擎命令和训练参数 |
|
||||
| POST | `/modelTF/fine-tune/start` | 启动训练,应用侧选择算力节点并提交 Compute Job |
|
||||
| PUT | `/modelTF/fine-tune/{id}` | 更新任务 |
|
||||
| POST | `/modelTF/fine-tune/stop/{id}` | 停止任务 |
|
||||
| DELETE | `/modelTF/fine-tune/{id}` | 删除任务 |
|
||||
@@ -494,6 +496,49 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
}
|
||||
```
|
||||
|
||||
训练启动前检查和命令预览:
|
||||
|
||||
- 页面模块:`/fine-tune/create` 创建训练任务的“参数确认/启动训练”区域;`/training-log/:id` 训练详情页的“任务配置/命令查看”区域。
|
||||
- `POST /modelTF/fine-tune/{id}/command-preview`:不做远端路径强校验,只返回应用侧调度出的算力节点、标准 Compute Job payload、训练引擎命令和工作目录,供前端展示最终 LLaMA-Factory 启动命令。
|
||||
- `POST /modelTF/fine-tune/{id}/preflight`:启动前强校验,真实 `llama_factory` 会检查目标节点连通性、模型路径、数据集目录、LLaMA-Factory HOME、训练命令是否可用;`smoke` 引擎用于自动化闭环验收,会跳过模型/数据集路径检查。
|
||||
- `POST /modelTF/fine-tune/start`:内部先执行 preflight,预检失败返回 `409` 且任务保持 `pending`,预检通过后再写入 `syncing/queued/running` 运行态并提交 Compute Job。
|
||||
|
||||
请求体可传启动覆盖参数:
|
||||
|
||||
```json
|
||||
{
|
||||
"requested_node_id": "node_xxx",
|
||||
"gpus": [0],
|
||||
"batch_size": 1,
|
||||
"learning_rate": 0.0002,
|
||||
"n_epochs": 1
|
||||
}
|
||||
```
|
||||
|
||||
响应结构:
|
||||
|
||||
```json
|
||||
{
|
||||
"valid": true,
|
||||
"errors": [],
|
||||
"warnings": [],
|
||||
"node": {
|
||||
"id": "node_xxx",
|
||||
"code": "gpu-node-01",
|
||||
"scheduler_status": "online",
|
||||
"gpu_count": 1
|
||||
},
|
||||
"job_payload": {},
|
||||
"preview": {
|
||||
"engine": "llama_factory",
|
||||
"command": ["llamafactory-cli", "train", "..."],
|
||||
"command_text": "llamafactory-cli train ...",
|
||||
"work_dir": "/app/LLaMA-Factory",
|
||||
"path_checks": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 7.2 训练日志详情页
|
||||
|
||||
训练日志页还会联合调用:
|
||||
@@ -841,7 +886,7 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
| POST | `/modelTF/compute/nodes` | 新增算力节点 |
|
||||
| GET | `/modelTF/compute/nodes/{id}` | 算力节点详情 |
|
||||
| PUT | `/modelTF/compute/nodes/{id}` | 编辑节点地址、权重、标签、路径和启用状态 |
|
||||
| POST | `/modelTF/compute/nodes/{id}/test-connection` | 测试 Compute API/File Gateway 连通性 |
|
||||
| POST | `/modelTF/compute/nodes/{id}/test-connection` | 测试 Compute API/File Gateway 连通性,并同步节点健康信息和 GPU 清单 |
|
||||
| POST | `/modelTF/compute/nodes/{id}/enable` | 启用节点 |
|
||||
| POST | `/modelTF/compute/nodes/{id}/disable` | 禁用节点,不接收新任务 |
|
||||
| POST | `/modelTF/compute/nodes/{id}/drain` | 进入维护模式,已有任务跑完后下线 |
|
||||
@@ -853,6 +898,7 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
| GET | `/modelTF/compute/jobs/{id}` | 算力任务详情 |
|
||||
| POST | `/modelTF/compute/jobs/{id}/retry` | 重试任务 |
|
||||
| POST | `/modelTF/compute/jobs/{id}/priority` | 调整优先级 |
|
||||
| GET | `/modelTF/compute/jobs/{id}/logs` | 拉取算力任务训练日志,支持 tail/分页 |
|
||||
| POST | `/modelTF/internal/compute-sync/jobs/poll` | 应用平台主动轮询并同步算力任务状态 |
|
||||
| POST | `/modelTF/internal/compute-sync/resources` | 调度前同步数据集/模型到目标节点 |
|
||||
|
||||
@@ -862,6 +908,82 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
- 每个可执行训练的节点都需要部署 `Compute API`、`Compute Agent`、`File Gateway` 和宿主机挂载的 LLaMA-Factory。
|
||||
- 节点之间默认不互相访问,应用平台主动访问所有节点的 Compute API/File Gateway。
|
||||
- 调度支持 `auto` 和 `manual`:普通用户默认自动调度,管理员或高级用户可手动指定节点。
|
||||
- 节点地址、权重、标签、启用状态、最大并发和本地路径都由 `/compute` 算力节点页面维护。
|
||||
- 连接测试由应用后端发起,依次探测算力侧 `GET /modelTF/v1/compute/health` 和 `GET /modelTF/compute/resources/gpus`;返回包可为裸 JSON,也可为 `{code,message,data}` 包装结构。
|
||||
|
||||
新增/编辑节点请求:
|
||||
|
||||
```json
|
||||
{
|
||||
"code": "gpu-node-01",
|
||||
"name": "A800 Node 01",
|
||||
"api_base_url": "http://10.10.20.31:19100",
|
||||
"file_gateway_url": "http://10.10.20.31:19101",
|
||||
"enabled": true,
|
||||
"scheduler_status": "offline",
|
||||
"scheduler_weight": 100,
|
||||
"tags": ["A800", "80GB", "llama_factory"],
|
||||
"max_parallel_jobs": 4,
|
||||
"data_root": "/data/yg-ft",
|
||||
"model_root": "/data/yg-ft/models",
|
||||
"log_root": "/opt/yg-ft/logs/training",
|
||||
"description": "北京机房训练节点"
|
||||
}
|
||||
```
|
||||
|
||||
启动成功后,响应中的训练任务会包含 `compute_node_id`、`compute_job_id`、`process_id`、`status`、`progress`、`output_dir`、`log_file` 等字段。应用侧后台 worker 会按 `COMPUTE_POLL_INTERVAL_SECONDS` 定时调用目标算力节点查询 Compute Job,并回写训练任务状态。
|
||||
|
||||
算力任务日志查询参数:
|
||||
|
||||
| 参数 | 类型 | 必填 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `tail_lines` | int | 否 | 默认 `200`,返回最后 N 行,范围 `1-5000` |
|
||||
| `offset` | int | 否 | 从第 N 行开始读取;当传入 `offset` 或 `limit` 时分页优先,忽略默认 tail 行数 |
|
||||
| `limit` | int | 否 | 分页读取行数,范围 `1-5000` |
|
||||
|
||||
响应字段包括 `content`、`metrics`、`total_lines`、`offset`、`limit`、`has_more`、`next_offset`。前端训练详情页、训练日志页和算力队列页可以用该接口增量读取日志,避免一次性拉取大文件。
|
||||
|
||||
任务维度实时日志接口:`GET /modelTF/fine-tune/{task_id}/logs?tail_lines=500`。该接口由应用后端按任务绑定的 `compute_node_id` 和 `compute_job_id` 转发到目标算力节点日志接口;如果训练尚未创建 Compute Job 或远端日志暂时不可达,则返回任务 `failure_reason`,用于页面展示启动失败、预检失败和远端训练失败原因。
|
||||
|
||||
算力任务重试:
|
||||
|
||||
```json
|
||||
{
|
||||
"force": false,
|
||||
"priority": "high",
|
||||
"requested_node_id": "node_xxx",
|
||||
"gpus": [0]
|
||||
}
|
||||
```
|
||||
|
||||
默认只允许 `failed`、`stopped` 任务重试;如确需重新执行已完成任务,需要显式传 `force=true`。重试会清空旧的运行时字段,重新调度节点并创建新的 Compute Job。
|
||||
|
||||
算力任务优先级:
|
||||
|
||||
```json
|
||||
{
|
||||
"priority": "low|normal|high|urgent"
|
||||
}
|
||||
```
|
||||
|
||||
第一版优先级写入任务 payload,并影响 `/modelTF/compute/queue` 的展示排序;后续如接入独立队列调度器,可保持接口不变,将该字段映射到调度器优先级。
|
||||
|
||||
连接测试响应:
|
||||
|
||||
```json
|
||||
{
|
||||
"node_id": "node_xxx",
|
||||
"success": true,
|
||||
"latency_ms": 35,
|
||||
"gpu_count": 8,
|
||||
"health": {
|
||||
"status": "ok",
|
||||
"api_version": "v1",
|
||||
"execution_mode": "real",
|
||||
"capabilities": ["gpu_discovery", "llama_factory", "file_gateway", "job_polling"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
算力节点响应字段:
|
||||
|
||||
@@ -882,6 +1004,9 @@ page=1&page_size=20&keyword=xxx&sort=-created_at
|
||||
"data_root": "/data/yg-ft",
|
||||
"model_root": "/data/yg-ft/models",
|
||||
"log_root": "/opt/yg-ft/logs/compute",
|
||||
"api_version": "v1",
|
||||
"capabilities": ["gpu_discovery", "llama_factory"],
|
||||
"description": "北京机房训练节点",
|
||||
"last_health_check_at": "2026-07-20T12:00:00+08:00",
|
||||
"health_detail": {
|
||||
"compute_api": "ok",
|
||||
@@ -916,11 +1041,15 @@ GPU 响应字段:
|
||||
| 方法 | 路径 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| POST | `/modelTF/compute/jobs` | 创建训练/评测/数据处理/推理任务 |
|
||||
| POST | `/modelTF/compute/jobs/preview` | 算力节点训练命令预览,不启动进程 |
|
||||
| POST | `/modelTF/compute/jobs/validate` | 算力节点训练启动前预检,校验参数、路径和引擎命令 |
|
||||
| GET | `/modelTF/compute/jobs/{id}` | 查询任务 |
|
||||
| POST | `/modelTF/compute/jobs/{id}/stop` | 停止任务 |
|
||||
| GET | `/modelTF/compute/jobs/{id}/logs` | 拉取日志 |
|
||||
| POST | `/modelTF/compute/files/check-paths` | 算力节点本地路径可用性检查 |
|
||||
| GET | `/modelTF/compute/resources/gpus` | 查询 GPU |
|
||||
| POST | `/modelTF/compute/files/upload` | 上传到算力本地磁盘 |
|
||||
| POST | `/modelTF/compute/files/import-local` | 从算力服务器本地路径导入到 `YG_FT_DATA_ROOT` |
|
||||
| GET | `/modelTF/compute/files/{id}/download` | 下载文件 |
|
||||
|
||||
创建算力任务:
|
||||
@@ -957,6 +1086,48 @@ GPU 响应字段:
|
||||
}
|
||||
```
|
||||
|
||||
当前 LLaMA-Factory 训练作业最小 payload:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "ft_xxx",
|
||||
"name": "finance-sft-001",
|
||||
"engine": "llama_factory",
|
||||
"base_model": "/data/yg-ft/models/Qwen2.5-7B",
|
||||
"model_name_or_path": "/data/yg-ft/models/Qwen2.5-7B",
|
||||
"dataset": "finance_train",
|
||||
"dataset_dir": "/data/yg-ft/datasets",
|
||||
"output_dir": "/data/yg-ft/outputs/finance-sft-001",
|
||||
"template": "qwen",
|
||||
"train_method": "lora",
|
||||
"gpus": [0],
|
||||
"batch_size": 2,
|
||||
"learning_rate": 0.0002,
|
||||
"n_epochs": 3,
|
||||
"save_steps": 50
|
||||
}
|
||||
```
|
||||
|
||||
应用侧轮询同步响应:
|
||||
|
||||
```json
|
||||
{
|
||||
"synced": 1,
|
||||
"failed": [],
|
||||
"items": [
|
||||
{
|
||||
"id": "ft_xxx",
|
||||
"status": "running",
|
||||
"progress": 35,
|
||||
"compute_job_id": "ft_xxx",
|
||||
"process_id": 52341,
|
||||
"output_dir": "/data/yg-ft/outputs/finance-sft-001",
|
||||
"log_file": "/opt/yg-ft/logs/training/ft_xxx.log"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
手动指定节点时:
|
||||
|
||||
```json
|
||||
|
||||
225
docs/data-process-design.md
Normal file
225
docs/data-process-design.md
Normal file
@@ -0,0 +1,225 @@
|
||||
# 数据处理接口与算法设计
|
||||
|
||||
本文是 `team-development-plan.md` 板块 C 的落地契约,约束
|
||||
`/modelTF/data-process/*`、前端数据处理向导以及 PostgreSQL 数据模型。
|
||||
|
||||
## 1. 处理闭环
|
||||
|
||||
```text
|
||||
创建草稿任务
|
||||
→ 上传并登记源文件(格式、SHA-256、版本)
|
||||
→ 预处理(标准化、无效过滤、去重、可选脱敏)
|
||||
→ 构建可编辑预览(来源偏移与行号)
|
||||
→ 生成标准训练记录
|
||||
→ 质量评分与稳定数据集划分
|
||||
→ 人工编辑/恢复
|
||||
→ 幂等发布为数据集(保留完整来源链路)
|
||||
```
|
||||
|
||||
任务只使用以下五种状态:
|
||||
|
||||
```text
|
||||
pending ──start/generate──> running ──success──> completed
|
||||
▲ │ ├──error───────> failed
|
||||
│ │ └──stop────────> stopped
|
||||
└────────retry────────────┴────────retry─────┘
|
||||
```
|
||||
|
||||
- `pending` 允许修改配置、增删源文件和重建预览。
|
||||
- `running` 拒绝重复启动、修改配置和删除任务。
|
||||
- `failed`、`stopped` 可重试;重试前清理上一次未完成结果。
|
||||
- `completed` 可编辑结果和发布;重复发布返回同一个数据集。
|
||||
- 非法状态转换返回 HTTP 409。
|
||||
- 每次生成分配独立 `generation_run_id`;停止或重试会使旧代次立即失效,
|
||||
旧后台任务不能覆盖新代次的结果或状态。
|
||||
|
||||
## 2. 接口契约
|
||||
|
||||
所有路径由请求层统一添加 `/modelTF`,响应统一为
|
||||
`{ "code": 0, "message": "ok", "data": ... }`。
|
||||
|
||||
### 任务与进度
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| GET | `/data-process` | 分页查询任务,支持 keyword/status/process_type |
|
||||
| POST | `/data-process` | 创建 `pending` 草稿 |
|
||||
| GET | `/data-process/{id}` | 查询任务详情,不内嵌全部结果 |
|
||||
| PUT | `/data-process/{id}` | 更新草稿配置 |
|
||||
| DELETE | `/data-process/{id}` | 软删除非运行任务 |
|
||||
| POST | `/data-process/{id}/start` | 重建预览并生成的一键编排入口 |
|
||||
| POST | `/data-process/{id}/generate` | 使用已确认预览生成结果 |
|
||||
| POST | `/data-process/{id}/stop` | 请求停止运行任务 |
|
||||
| GET | `/data-process/{id}/progress` | 查询阶段、进度与计数 |
|
||||
|
||||
### 源文件与预览
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| POST | `/data-process/{id}/source-files` | multipart 上传,字段名 `files` |
|
||||
| DELETE | `/data-process/{id}/source-files/{file_id}` | 删除源文件及其预览 |
|
||||
| GET | `/data-process/{id}/source-files/{file_id}/content` | 按行窗口读取源文 |
|
||||
| POST | `/data-process/{id}/preview/build` | 后端预处理并重建预览 |
|
||||
| GET | `/data-process/{id}/preview` | 分页查询预览 |
|
||||
| POST | `/data-process/{id}/preview` | 手工增加预览条目 |
|
||||
| PUT | `/data-process/{id}/preview/{preview_id}` | 保存人工编辑 |
|
||||
| DELETE | `/data-process/{id}/preview/{preview_id}` | 删除预览条目 |
|
||||
|
||||
上传批次先全部完成有界读取、UTF-8 解码和解析,再在单个事务中登记;任一文件
|
||||
为空、超限、重复或格式非法时整批不落库。响应不回传整个文件,只返回文件 ID、
|
||||
格式、字节数、记录数和 SHA-256。二进制文档必须由对应解析器显式处理;
|
||||
不支持的格式返回 415,绝不能静默替换成示例正文。
|
||||
|
||||
### 结果与发布
|
||||
|
||||
| 方法 | 路径 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| GET | `/data-process/{id}/results` | 分页查询,支持 keyword/status/split |
|
||||
| PUT | `/data-process/{id}/results/{result_id}` | 保存人工编辑并重评分 |
|
||||
| POST | `/data-process/{id}/results/{result_id}/restore` | 恢复生成时的原值 |
|
||||
| POST | `/data-process/{id}/publish` | 幂等发布为数据集 |
|
||||
|
||||
## 3. 配置校验
|
||||
|
||||
- `process_type`:`structured | unstructured | external`。
|
||||
- 数据集划分的 `train + validation + test` 必须等于 100,各项为 0~100。
|
||||
- `chunk_size` 为 16~32768 token;`chunk_overlap` 必须小于
|
||||
`chunk_size`;`min_chunk_size` 不得大于 `chunk_size`。
|
||||
- `temperature` 为 0~2,`max_tokens` 为 1~32768。
|
||||
- 任务名称在未删除任务中唯一。
|
||||
- 选择 `generation_model_id` 后,启动生成时校验模型是否存在,并保存不含密钥的
|
||||
模型版本快照。
|
||||
- 当前运行库沿用平台现有的单租户模式,不接受客户端提交 tenant/owner/operator
|
||||
字段,避免伪造隔离上下文;接入平台可信认证上下文后再启用数据库中预留的
|
||||
tenant/project 字段。
|
||||
|
||||
## 4. 格式解析与标准化
|
||||
|
||||
首版文本解析支持 UTF-8/UTF-8 BOM 的 TXT、Markdown、CSV、JSON、JSONL。
|
||||
后续 PDF、DOCX、XLSX 必须接入明确的解析器后再开放前端选择。
|
||||
|
||||
处理顺序固定为:
|
||||
|
||||
1. 严格解码并识别格式;非法字节或畸形 JSON/JSONL 返回可定位错误。
|
||||
2. Unicode NFKC 标准化,统一 CRLF,清理 NUL、零宽字符和不可读控制字符。
|
||||
3. 结构化数据转为 canonical JSON;非结构化数据保留 Markdown 语义块。
|
||||
4. 若启用脱敏,替换邮箱、手机号和身份证号,同时保存各类型命中计数。
|
||||
5. 使用标准化正文的 SHA-256 去重;重复条目不进入生成阶段并计入
|
||||
`duplicate_count`。
|
||||
|
||||
脱敏是不可逆掩码:
|
||||
|
||||
- 邮箱:`[EMAIL]`
|
||||
- 中国大陆手机号:`[PHONE]`
|
||||
- 18 位身份证号:`[ID_CARD]`
|
||||
|
||||
源文件原文与脱敏后的预览分开保存,结果不得反向覆盖源文件。
|
||||
|
||||
## 5. 切片算法
|
||||
|
||||
`fixed` 按目标 token 窗口切分;`semantic` 优先在空行、换行和中英文句末
|
||||
标点结束;`heading` 进一步优先在 Markdown/中文章节标题之前结束;
|
||||
`custom` 使用用户给定分隔符。
|
||||
|
||||
首版使用可替换的确定性 token 估算器,中文字符、标点和英文词分别计数;
|
||||
所有偏移以 Python/JavaScript 都能稳定表达的 Unicode 文本偏移为准。
|
||||
|
||||
算法必须满足:
|
||||
|
||||
- 每轮游标严格前进,异常分隔符不能产生死循环。
|
||||
- overlap 是最大重叠量,尾部过短切片合并到上一片。
|
||||
- 代码块、Markdown 表格和连续列表在启用保护时不从中间切开。
|
||||
- 每个预览条目记录 `source_file_id`、字符偏移、起止行、token 数和算法版本。
|
||||
|
||||
## 6. 生成与质量评分
|
||||
|
||||
结构化记录优先识别以下字段:
|
||||
|
||||
1. `instruction/input/output`
|
||||
2. `question/context/answer`
|
||||
3. `prompt/input/response`
|
||||
|
||||
已有标准字段时只做标准化;需要语义生成时调用所选模型的 OpenAI 兼容接口,
|
||||
并固化模型 ID、模型版本、prompt、temperature、max_tokens 和 JSON mode 快照。
|
||||
模型地址可输入域名、`/v1` 基础地址或完整地址:例如输入
|
||||
`www.caoxiaozhu.com` 会规范为
|
||||
`https://www.caoxiaozhu.com/v1/chat/completions`,无需用户手工拼接路径。
|
||||
单条失败记录为 `invalid`,有限重试耗尽后继续处理下一条,避免整批丢失。
|
||||
|
||||
每条结果总分为 0~100:
|
||||
|
||||
```text
|
||||
总分 = 完整性 35% + 长度合理性 20% + 可读性 20%
|
||||
+ 来源相关性 15% + 非重复性 10%
|
||||
```
|
||||
|
||||
- instruction 或 output 为空时格式硬失败并标记 `invalid`。
|
||||
- 开启短文本过滤且 output 低于 `min_output_length` 时标记过滤原因。
|
||||
- 评分详情、命中规则与过滤原因必须落库并返回前端,不只返回一个总分。
|
||||
|
||||
## 7. 稳定划分
|
||||
|
||||
划分不能依赖结果插入顺序。对每条记录计算:
|
||||
|
||||
```text
|
||||
bucket = SHA256(task_id + ":" + result_id) mod 10000
|
||||
```
|
||||
|
||||
按万分位阈值映射为 `train/validation/test`。同一任务重试、分页或进程重启后,
|
||||
同一结果仍落入相同 split。
|
||||
|
||||
## 8. 发布与来源链路
|
||||
|
||||
发布在一个数据库事务中完成:
|
||||
|
||||
```text
|
||||
source_file
|
||||
→ data_process_task
|
||||
→ data_process_result
|
||||
→ dataset
|
||||
→ dataset_file + dataset_file_version
|
||||
→ dataset_record
|
||||
```
|
||||
|
||||
只发布 `valid/modified` 且满足质量门槛的结果。输出 JSONL 先计算 checksum,
|
||||
再登记文件版本和记录。发布请求中的 split 会重新进行稳定划分。任务的
|
||||
`output_dataset_id` 是幂等键;重复调用返回已有数据集,目标数据集若已被外部
|
||||
删除则解除断链并重新发布。当前运行库只开放 `local` 存储类型,正文保存在
|
||||
当前平台的 `dataset_files.content`,不虚假宣称已上传 MinIO 或云存储。
|
||||
|
||||
## 9. 安全边界
|
||||
|
||||
- 文件名只保留 basename,响应不返回宿主机绝对路径。
|
||||
- 上传限制单文件、批次文件数与批次总大小,解析采用有界读取。
|
||||
- 外部数据源凭据不写日志、不进入 localStorage、不在详情接口回显。
|
||||
- 外部 PostgreSQL 只允许单条 `SELECT/WITH`、只读事务、5 秒连接超时、
|
||||
30 秒语句超时和 50 MiB 响应上限;默认阻止回环、链路本地及私网地址。
|
||||
可信内网部署必须显式设置 `DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB=true`。
|
||||
- SQL 迁移独立存放,应用启动不会隐式修改当前远程数据库。
|
||||
|
||||
## 10. 迁移边界
|
||||
|
||||
`backend/app/db/sql/002_data_process.sql` 只面向当前运行脚本
|
||||
`001_platform_runtime.sql` 的 TEXT/最小表模型。它会在执行前检查
|
||||
`datasets.id` 类型;若检测到 `docs/postgres-schema.sql` 的 UUID/JSONB 目标模型,
|
||||
会直接失败而不是进行一半成功、一半失败的危险迁移。目标模型后续应由独立
|
||||
Alembic 迁移和对应存储实现承接。
|
||||
|
||||
`DataProcessStore.ensure_schema()` 仅供受控管理命令显式调用,API 路由和应用启动
|
||||
均不会自动执行该迁移。本次开发和测试没有修改任何远程数据库。
|
||||
|
||||
在已加载 `DATABASE_URL` 的终端中可先只读检查:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
.venv/bin/python -m app.modules.data_process.schema_cli --check
|
||||
```
|
||||
|
||||
确认目标主机和数据库名称无误后,才显式执行:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
.venv/bin/python -m app.modules.data_process.schema_cli --apply --yes
|
||||
```
|
||||
|
||||
命令输出只显示主机、端口和数据库名,不显示用户名或密码。
|
||||
@@ -228,6 +228,20 @@ GPU 算力服务器部署:
|
||||
|
||||
多节点任务调度由应用平台统一完成。应用平台从 `compute_nodes` 读取节点地址、权重、标签、启用状态、维护状态和健康检查结果;从 `resource_replicas` 判断目标节点是否已有所需数据集/模型副本;缺失时创建 `resource_sync_jobs`,通过目标节点 File Gateway 同步资源。
|
||||
|
||||
当前实现已支持在 `/compute` 算力节点页面新增和编辑节点。运维人员维护 `Compute API` 地址、`File Gateway` 地址、权重、标签、启用状态、最大并发和本地路径后,点击连接测试会由应用后端主动访问目标节点健康检查和 GPU 清单接口,并将 `health_detail`、`gpu_count`、`gpu_devices/gpus` 同步到 PostgreSQL。真实 GPU 服务器优先通过 `nvidia-smi` 发现 GPU;特殊环境可用 `COMPUTE_GPU_COUNT` 等环境变量声明兼容清单。
|
||||
|
||||
训练运行闭环:
|
||||
|
||||
- 前端启动训练后,Backend API 按 `compute_nodes` 的启用状态、调度状态、权重和并行任务数选择节点。
|
||||
- Backend API 向目标节点 `POST /modelTF/compute/jobs` 提交 LLaMA-Factory 训练作业,并在 `fine_tune_tasks.compute_job_id` 记录算力任务 ID。
|
||||
- Compute API 在真实模式下启动 `llamafactory-cli train` 子进程,训练日志写入 `TRAINING_LOG_ROOT/{job_id}.log`。
|
||||
- Backend API 启动后会运行应用侧轮询 worker,按 `COMPUTE_POLL_INTERVAL_SECONDS` 主动查询目标节点 `GET /modelTF/compute/jobs/{id}`,同步任务状态、进度、PID、输出目录、日志路径和产物索引。
|
||||
- 停止训练时,Backend API 优先调用目标节点 `POST /modelTF/compute/jobs/{id}/stop`,再回写应用任务状态。
|
||||
- 失败或停止任务可以通过 `POST /modelTF/compute/jobs/{id}/retry` 重试;重试会清空旧运行态,重新调度节点并创建 Compute Job。
|
||||
- 训练日志通过 `GET /modelTF/compute/jobs/{id}/logs` 读取,支持 `tail_lines`、`offset`、`limit`,用于训练详情页、训练日志页和日志平台采集。
|
||||
- Compute API 使用 `COMPUTE_SERVICE_TOKEN` 做服务间鉴权,应用侧请求携带 `X-Compute-Token`;健康检查接口保持可公开探活。
|
||||
- Compute API 会把本机训练作业登记到 `TRAINING_LOG_ROOT/compute-jobs.json`,服务重启后可恢复任务索引并继续暴露状态和日志。
|
||||
|
||||
调度策略:
|
||||
|
||||
- 默认自动调度,按节点健康、标签、GPU 空闲、队列长度、节点权重和资源副本命中率排序。
|
||||
@@ -279,7 +293,7 @@ COMPUTE_API_BASE_URL=https://compute.internal:19100
|
||||
COMPUTE_SERVICE_TOKEN=***
|
||||
FILE_GATEWAY_BASE_URL=https://compute.internal:19101
|
||||
COMPUTE_STATUS_SYNC_MODE=polling
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=10
|
||||
COMPUTE_POLL_INTERVAL_SECONDS=3
|
||||
COMPUTE_POLL_BATCH_SIZE=100
|
||||
```
|
||||
|
||||
@@ -290,10 +304,21 @@ COMPUTE_ENV=prod
|
||||
COMPUTE_HOST_ID=gpu-node-01
|
||||
COMPUTE_API_PORT=19100
|
||||
FILE_GATEWAY_PORT=19101
|
||||
COMPUTE_AUTH_ENABLED=true
|
||||
COMPUTE_SERVICE_TOKEN=***
|
||||
ENABLE_APP_CALLBACK=false
|
||||
LLAMA_FACTORY_HOME=/app/LLaMA-Factory
|
||||
YG_FT_DATA_ROOT=/data/yg-ft
|
||||
YG_FT_DATA_ROOT_HOST=./data/yg-ft
|
||||
YG_FT_MODEL_ROOT=/data/yg-ft/models
|
||||
YG_FT_MODEL_ROOT_HOST=./data/yg-ft/models
|
||||
YG_FT_DATASET_ROOT=/data/yg-ft/datasets
|
||||
YG_FT_DATASET_ROOT_HOST=./data/yg-ft/datasets
|
||||
YG_FT_OUTPUT_ROOT=/data/yg-ft/outputs
|
||||
YG_FT_OUTPUT_ROOT_HOST=./data/yg-ft/outputs
|
||||
TRAINING_LOG_ROOT=/opt/yg-ft/logs/training
|
||||
TRAINING_LOG_ROOT_HOST=./data/yg-ft/logs/training
|
||||
COMPUTE_LOG_ROOT_HOST=./data/yg-ft/logs/compute
|
||||
LOG_DIR=/opt/yg-ft/logs/compute
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
```
|
||||
@@ -393,7 +418,7 @@ gpu-node-03 -> http://10.10.20.33:19100 / http://10.10.20.33:19101
|
||||
```env
|
||||
ENABLE_APP_CALLBACK=false
|
||||
COMPUTE_SERVICE_TOKEN=change_me
|
||||
YG_FT_DATA_ROOT_HOST=/data/yg-ft
|
||||
YG_FT_DATA_ROOT_HOST=./data/yg-ft
|
||||
```
|
||||
|
||||
## 12. 仍需确认的问题
|
||||
|
||||
@@ -1061,8 +1061,20 @@ CREATE TABLE IF NOT EXISTS compute_nodes (
|
||||
name varchar(150) NOT NULL,
|
||||
host varchar(200) NOT NULL,
|
||||
api_base_url text NOT NULL,
|
||||
file_gateway_url text NOT NULL DEFAULT '',
|
||||
storage_node_id uuid REFERENCES storage_nodes(id) ON DELETE SET NULL,
|
||||
status varchar(40) NOT NULL DEFAULT 'online',
|
||||
scheduler_status varchar(40) NOT NULL DEFAULT 'online',
|
||||
scheduler_weight integer NOT NULL DEFAULT 100,
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
max_parallel_jobs integer NOT NULL DEFAULT 1,
|
||||
data_root text NOT NULL DEFAULT '/data/yg-ft',
|
||||
model_root text NOT NULL DEFAULT '/data/yg-ft/models',
|
||||
log_root text NOT NULL DEFAULT '/opt/yg-ft/logs/training',
|
||||
api_version varchar(40) NOT NULL DEFAULT 'v1',
|
||||
capabilities jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
description text,
|
||||
health_detail jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
agent_version varchar(80),
|
||||
gpu_count integer NOT NULL DEFAULT 0,
|
||||
last_heartbeat_at timestamptz,
|
||||
@@ -1434,10 +1446,15 @@ ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS tenant_id uuid REFERENCES t
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS project_id uuid REFERENCES projects(id) ON DELETE SET NULL;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS owner_id uuid REFERENCES users(id) ON DELETE SET NULL;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS approval_status approval_status NOT NULL DEFAULT 'not_required';
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS compute_node_id uuid REFERENCES compute_nodes(id) ON DELETE SET NULL;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS compute_job_id uuid REFERENCES compute_jobs(id) ON DELETE SET NULL;
|
||||
ALTER TABLE fine_tune_tasks ADD COLUMN IF NOT EXISTS resume_checkpoint_id uuid REFERENCES fine_tune_checkpoints(id) ON DELETE SET NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_scope_status
|
||||
ON fine_tune_tasks(tenant_id, project_id, status, created_at DESC) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_compute_job
|
||||
ON fine_tune_tasks(compute_job_id) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_fine_tune_tasks_node_status
|
||||
ON fine_tune_tasks(compute_node_id, status, created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
ALTER TABLE inference_tasks ADD COLUMN IF NOT EXISTS tenant_id uuid REFERENCES tenants(id) ON DELETE SET NULL;
|
||||
ALTER TABLE inference_tasks ADD COLUMN IF NOT EXISTS project_id uuid REFERENCES projects(id) ON DELETE SET NULL;
|
||||
|
||||
@@ -1 +1 @@
|
||||
import{d as C,bn as g,D as E,H as O,o as _,e as B,s as D,Z as $,w as A,c as I,aL as K,q as a,n as v,aa as r,g as q,bm as M,y as u,z as N,P as k}from"./index-BKKvzUDD.js";import{_ as R}from"./_plugin-vue_export-helper-DlAUqK2U.js";const V={class:"app-confirm-header"},z={class:"app-confirm-heading"},H={class:"app-confirm-icon","aria-hidden":"true"},L={class:"app-confirm-body"},P={class:"app-confirm-actions"},S=C({__name:"AppConfirmDialog",setup(j,{expose:x}){const c=u(!1),m=u(),p=u(),d=`app-confirm-title-${g()}`,y=`app-confirm-message-${g()}`,n=N({title:"请确认操作",message:"",confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1});let o=null,i=null;function s(t){c.value=!1;const e=o;o=null,e==null||e(t)}function h(t){return o&&s(!1),Object.assign(n,{confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1,...t}),c.value=!0,new Promise(e=>{o=e})}function w(){n.closeOnOverlay&&s(!1)}function T(t){var b;if(t.key==="Escape"){t.preventDefault(),s(!1);return}if(t.key!=="Tab")return;const e=Array.from(((b=m.value)==null?void 0:b.querySelectorAll("button:not([disabled])"))??[]),l=e[0],f=e[e.length-1];!l||!f||(t.shiftKey&&document.activeElement===l?(t.preventDefault(),f.focus()):!t.shiftKey&&document.activeElement===f&&(t.preventDefault(),l.focus()))}return E(c,async t=>{var e;if(t){i=document.activeElement instanceof HTMLElement?document.activeElement:null,await k(),(e=p.value)==null||e.focus();return}await k(),i==null||i.focus(),i=null}),O(()=>{o==null||o(!1),o=null}),x({open:h}),(t,e)=>(_(),B(M,{to:"body"},[D($,{name:"app-confirm"},{default:A(()=>[c.value?(_(),I("div",{key:0,class:"app-confirm-overlay",onMousedown:K(w,["self"])},[a("section",{ref_key:"dialogRef",ref:m,class:v(["app-confirm-dialog",`is-${n.tone}`]),role:"alertdialog","aria-modal":!0,"aria-labelledby":d,"aria-describedby":y,onKeydown:T},[a("header",V,[a("div",z,[a("span",H,[a("i",{class:v(n.tone==="primary"?"fa fa-question-circle":"fa fa-exclamation-triangle")},null,2)]),a("h2",{id:d},r(n.title),1)]),a("button",{class:"app-confirm-close",type:"button","aria-label":"关闭确认弹窗",onClick:e[0]||(e[0]=l=>s(!1))},[...e[3]||(e[3]=[a("i",{class:"fa fa-times","aria-hidden":"true"},null,-1)])])]),a("div",L,[a("p",{id:y},r(n.message),1)]),a("footer",P,[a("button",{ref_key:"cancelButtonRef",ref:p,class:"app-confirm-button is-cancel",type:"button",onClick:e[1]||(e[1]=l=>s(!1))},r(n.cancelText),513),a("button",{class:"app-confirm-button is-confirm",type:"button",onClick:e[2]||(e[2]=l=>s(!0))},r(n.confirmText),1)])],34)],32)):q("",!0)]),_:1})]))}}),G=R(S,[["__scopeId","data-v-398df98e"]]);export{G as A};
|
||||
import{d as C,bf as g,D as E,H as O,o as _,e as B,s as D,Z as $,w as A,c as I,aL as K,q as a,n as v,aa as r,g as q,bg as M,y as u,z as N,P as k}from"./index-Ds9AETjS.js";import{_ as R}from"./_plugin-vue_export-helper-DlAUqK2U.js";const V={class:"app-confirm-header"},z={class:"app-confirm-heading"},H={class:"app-confirm-icon","aria-hidden":"true"},L={class:"app-confirm-body"},P={class:"app-confirm-actions"},S=C({__name:"AppConfirmDialog",setup(j,{expose:x}){const c=u(!1),m=u(),p=u(),d=`app-confirm-title-${g()}`,y=`app-confirm-message-${g()}`,n=N({title:"请确认操作",message:"",confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1});let o=null,i=null;function s(t){c.value=!1;const e=o;o=null,e==null||e(t)}function h(t){return o&&s(!1),Object.assign(n,{confirmText:"确定",cancelText:"取消",tone:"warning",closeOnOverlay:!1,...t}),c.value=!0,new Promise(e=>{o=e})}function w(){n.closeOnOverlay&&s(!1)}function T(t){var b;if(t.key==="Escape"){t.preventDefault(),s(!1);return}if(t.key!=="Tab")return;const e=Array.from(((b=m.value)==null?void 0:b.querySelectorAll("button:not([disabled])"))??[]),l=e[0],f=e[e.length-1];!l||!f||(t.shiftKey&&document.activeElement===l?(t.preventDefault(),f.focus()):!t.shiftKey&&document.activeElement===f&&(t.preventDefault(),l.focus()))}return E(c,async t=>{var e;if(t){i=document.activeElement instanceof HTMLElement?document.activeElement:null,await k(),(e=p.value)==null||e.focus();return}await k(),i==null||i.focus(),i=null}),O(()=>{o==null||o(!1),o=null}),x({open:h}),(t,e)=>(_(),B(M,{to:"body"},[D($,{name:"app-confirm"},{default:A(()=>[c.value?(_(),I("div",{key:0,class:"app-confirm-overlay",onMousedown:K(w,["self"])},[a("section",{ref_key:"dialogRef",ref:m,class:v(["app-confirm-dialog",`is-${n.tone}`]),role:"alertdialog","aria-modal":!0,"aria-labelledby":d,"aria-describedby":y,onKeydown:T},[a("header",V,[a("div",z,[a("span",H,[a("i",{class:v(n.tone==="primary"?"fa fa-question-circle":"fa fa-exclamation-triangle")},null,2)]),a("h2",{id:d},r(n.title),1)]),a("button",{class:"app-confirm-close",type:"button","aria-label":"关闭确认弹窗",onClick:e[0]||(e[0]=l=>s(!1))},[...e[3]||(e[3]=[a("i",{class:"fa fa-times","aria-hidden":"true"},null,-1)])])]),a("div",L,[a("p",{id:y},r(n.message),1)]),a("footer",P,[a("button",{ref_key:"cancelButtonRef",ref:p,class:"app-confirm-button is-cancel",type:"button",onClick:e[1]||(e[1]=l=>s(!1))},r(n.cancelText),513),a("button",{class:"app-confirm-button is-confirm",type:"button",onClick:e[2]||(e[2]=l=>s(!0))},r(n.confirmText),1)])],34)],32)):q("",!0)]),_:1})]))}}),G=R(S,[["__scopeId","data-v-398df98e"]]);export{G as A};
|
||||
@@ -1 +1 @@
|
||||
import{a as N,E as B}from"./el-form-item-D5kF3B90.js";import{E as K}from"./index-TFUf94PZ.js";import{E as M}from"./index-BjEW7-SA.js";import{E as h}from"./index-BDEF353-.js";import{E as I}from"./el-divider-DjByQoml.js";import{E as R}from"./el-slider-DYENF1-i.js";import{d as F,G as z,e as V,w as l,ac as D,y as L,o as f,s as a,x as p,q as $,c as k,ad as j,aa as E,M as A,g as G,f as J,v as O,z as H,j as _,A as P}from"./index-BKKvzUDD.js";import"./el-popper-D6_hxRbQ.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-input-number-BFR4pu1i.js";/* empty css */import{P as Q}from"./PageCard-BoKXOzst.js";import{u as W}from"./usePolling-C6448AR2.js";import{a as X}from"./compare-CZ4TIoIW.js";import{_ as Y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./debounce-ByXPNh5F.js";import"./toNumber-Dkj3QRv9.js";import"./clamp-CbbY8h6F.js";import"./index-CWUnzf90.js";import"./index-BENk7lZo.js";import"./el-card-CApHJ1Gj.js";const Z={class:"model-list"},tt={key:0,class:"empty-hint"},et=F({__name:"CompareChatView",setup(ot){const b=D(),y=O(),x=b.params.id,n=L(null),e=H({systemPrompt:"",question:"",temperature:.7,topP:.9,topK:40,maxTokens:2048}),m=_(()=>{var s;if(!((s=n.value)!=null&&s.load_status))return[];try{return(typeof n.value.load_status=="string"?JSON.parse(n.value.load_status):n.value.load_status).loaded_models||[]}catch{return[]}}),T=_(()=>m.value.length>0&&m.value.every(s=>s.status==="ready"||s.status==="running")),g=_(()=>m.value.some(s=>s.status==="starting"));async function c(){try{n.value=await X(x)}catch{}}function C(){var u,i;if(!e.question.trim()){P.warning("请输入问题");return}if(g.value){P.warning("模型仍在启动中,请稍候");return}const s=new URLSearchParams({taskId:x,taskName:((u=n.value)==null?void 0:u.model_name)||((i=n.value)==null?void 0:i.name)||"",question:e.question,systemPrompt:e.systemPrompt,temperature:String(e.temperature),topP:String(e.topP),topK:String(e.topK),maxTokens:String(e.maxTokens)}),t=y.resolve(`/model-compare/result?${s.toString()}`).href;window.open(t,"_blank")}const{start:S}=W(c,5e3,{immediate:!1});return z(async()=>{await c(),S()}),(s,t)=>{const u=I,i=h,v=K,r=N,d=R,w=M,q=B;return f(),V(Q,{title:"模型对比配置"},{default:l(()=>[a(u,{"content-position":"left"},{default:l(()=>[...t[7]||(t[7]=[p("已启动模型",-1)])]),_:1}),$("div",Z,[(f(!0),k(A,null,j(m.value,(o,U)=>(f(),V(i,{key:U,type:o.status==="ready"||o.status==="running"?"success":o.status==="starting"?"warning":"danger",size:"large"},{default:l(()=>[p(E(o.model_name)+" ("+E(o.status)+") ",1)]),_:2},1032,["type"]))),128)),m.value.length?G("",!0):(f(),k("span",tt,"暂无已启动模型"))]),a(u,{"content-position":"left"},{default:l(()=>[...t[8]||(t[8]=[p("对话配置",-1)])]),_:1}),a(q,{"label-width":"120px",style:{"max-width":"700px"}},{default:l(()=>[a(r,{label:"系统提示词"},{default:l(()=>[a(v,{modelValue:e.systemPrompt,"onUpdate:modelValue":t[0]||(t[0]=o=>e.systemPrompt=o),type:"textarea",rows:3,placeholder:"可选"},null,8,["modelValue"])]),_:1}),a(r,{label:"问题"},{default:l(()=>[a(v,{modelValue:e.question,"onUpdate:modelValue":t[1]||(t[1]=o=>e.question=o),type:"textarea",rows:4,placeholder:"请输入要对比的问题"},null,8,["modelValue"])]),_:1}),a(r,{label:"Temperature"},{default:l(()=>[a(d,{modelValue:e.temperature,"onUpdate:modelValue":t[2]||(t[2]=o=>e.temperature=o),min:0,max:2,step:.1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-p"},{default:l(()=>[a(d,{modelValue:e.topP,"onUpdate:modelValue":t[3]||(t[3]=o=>e.topP=o),min:0,max:1,step:.05,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-k"},{default:l(()=>[a(d,{modelValue:e.topK,"onUpdate:modelValue":t[4]||(t[4]=o=>e.topK=o),min:1,max:100,step:1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Max Tokens"},{default:l(()=>[a(d,{modelValue:e.maxTokens,"onUpdate:modelValue":t[5]||(t[5]=o=>e.maxTokens=o),min:256,max:4096,step:128,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,null,{default:l(()=>[a(w,{type:"primary",disabled:!T.value||g.value,onClick:C},{default:l(()=>[...t[9]||(t[9]=[p(" 开始对比 ",-1)])]),_:1},8,["disabled"]),a(w,{onClick:t[6]||(t[6]=o=>J(y).back())},{default:l(()=>[...t[10]||(t[10]=[p("返回",-1)])]),_:1})]),_:1})]),_:1})]),_:1})}}}),qt=Y(et,[["__scopeId","data-v-5da55d96"]]);export{qt as default};
|
||||
import{a as N,E as B}from"./el-form-item-Bq0n_p5a.js";import{E as K}from"./index-h5FkzAP_.js";import{E as M}from"./index-BNsCyG4A.js";import{E as h}from"./index-ByhS8A1d.js";import{E as I}from"./el-divider-a924dciw.js";import{E as R}from"./el-slider-mZJHOhL2.js";import{d as F,G as z,e as V,w as l,ac as D,y as L,o as f,s as a,x as p,q as $,c as k,ad as j,aa as E,M as A,g as G,f as J,v as O,z as H,j as _,A as P}from"./index-Ds9AETjS.js";import"./el-popper-D1tByNLb.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-input-number-Bg2C0S8w.js";/* empty css */import{P as Q}from"./PageCard-CV3p14-x.js";import{u as W}from"./usePolling-Bx2QCw4O.js";import{a as X}from"./compare-fmy9bbtE.js";import{_ as Y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./debounce-D6mJVDYC.js";import"./toNumber-DrayciwT.js";import"./clamp-CyhADbdq.js";import"./index-sWKLi7bH.js";import"./index-2LOSO4Pw.js";import"./el-card-D86TXgMv.js";const Z={class:"model-list"},tt={key:0,class:"empty-hint"},et=F({__name:"CompareChatView",setup(ot){const b=D(),y=O(),x=b.params.id,n=L(null),e=H({systemPrompt:"",question:"",temperature:.7,topP:.9,topK:40,maxTokens:2048}),m=_(()=>{var s;if(!((s=n.value)!=null&&s.load_status))return[];try{return(typeof n.value.load_status=="string"?JSON.parse(n.value.load_status):n.value.load_status).loaded_models||[]}catch{return[]}}),T=_(()=>m.value.length>0&&m.value.every(s=>s.status==="ready"||s.status==="running")),g=_(()=>m.value.some(s=>s.status==="starting"));async function c(){try{n.value=await X(x)}catch{}}function C(){var u,i;if(!e.question.trim()){P.warning("请输入问题");return}if(g.value){P.warning("模型仍在启动中,请稍候");return}const s=new URLSearchParams({taskId:x,taskName:((u=n.value)==null?void 0:u.model_name)||((i=n.value)==null?void 0:i.name)||"",question:e.question,systemPrompt:e.systemPrompt,temperature:String(e.temperature),topP:String(e.topP),topK:String(e.topK),maxTokens:String(e.maxTokens)}),t=y.resolve(`/model-compare/result?${s.toString()}`).href;window.open(t,"_blank")}const{start:S}=W(c,5e3,{immediate:!1});return z(async()=>{await c(),S()}),(s,t)=>{const u=I,i=h,v=K,r=N,d=R,w=M,q=B;return f(),V(Q,{title:"模型对比配置"},{default:l(()=>[a(u,{"content-position":"left"},{default:l(()=>[...t[7]||(t[7]=[p("已启动模型",-1)])]),_:1}),$("div",Z,[(f(!0),k(A,null,j(m.value,(o,U)=>(f(),V(i,{key:U,type:o.status==="ready"||o.status==="running"?"success":o.status==="starting"?"warning":"danger",size:"large"},{default:l(()=>[p(E(o.model_name)+" ("+E(o.status)+") ",1)]),_:2},1032,["type"]))),128)),m.value.length?G("",!0):(f(),k("span",tt,"暂无已启动模型"))]),a(u,{"content-position":"left"},{default:l(()=>[...t[8]||(t[8]=[p("对话配置",-1)])]),_:1}),a(q,{"label-width":"120px",style:{"max-width":"700px"}},{default:l(()=>[a(r,{label:"系统提示词"},{default:l(()=>[a(v,{modelValue:e.systemPrompt,"onUpdate:modelValue":t[0]||(t[0]=o=>e.systemPrompt=o),type:"textarea",rows:3,placeholder:"可选"},null,8,["modelValue"])]),_:1}),a(r,{label:"问题"},{default:l(()=>[a(v,{modelValue:e.question,"onUpdate:modelValue":t[1]||(t[1]=o=>e.question=o),type:"textarea",rows:4,placeholder:"请输入要对比的问题"},null,8,["modelValue"])]),_:1}),a(r,{label:"Temperature"},{default:l(()=>[a(d,{modelValue:e.temperature,"onUpdate:modelValue":t[2]||(t[2]=o=>e.temperature=o),min:0,max:2,step:.1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-p"},{default:l(()=>[a(d,{modelValue:e.topP,"onUpdate:modelValue":t[3]||(t[3]=o=>e.topP=o),min:0,max:1,step:.05,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Top-k"},{default:l(()=>[a(d,{modelValue:e.topK,"onUpdate:modelValue":t[4]||(t[4]=o=>e.topK=o),min:1,max:100,step:1,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,{label:"Max Tokens"},{default:l(()=>[a(d,{modelValue:e.maxTokens,"onUpdate:modelValue":t[5]||(t[5]=o=>e.maxTokens=o),min:256,max:4096,step:128,"show-input":"",style:{"max-width":"500px"}},null,8,["modelValue"])]),_:1}),a(r,null,{default:l(()=>[a(w,{type:"primary",disabled:!T.value||g.value,onClick:C},{default:l(()=>[...t[9]||(t[9]=[p(" 开始对比 ",-1)])]),_:1},8,["disabled"]),a(w,{onClick:t[6]||(t[6]=o=>J(y).back())},{default:l(()=>[...t[10]||(t[10]=[p("返回",-1)])]),_:1})]),_:1})]),_:1})]),_:1})}}}),qt=Y(et,[["__scopeId","data-v-5da55d96"]]);export{qt as default};
|
||||
@@ -1 +1 @@
|
||||
import{E as D}from"./el-alert-xVLeNUgJ.js";import{E as $}from"./index-BjEW7-SA.js";import{E as z}from"./index-BDEF353-.js";import{E as F}from"./el-card-CApHJ1Gj.js";import{d as O,G as U,H as J,c as d,q as i,aa as c,f as v,s as N,w as m,M as K,ad as A,ac as G,y as g,o as r,x as p,e as y,g as H}from"./index-BKKvzUDD.js";/* empty css */import{_ as L}from"./MarkdownView.vue_vue_type_style_index_0_lang-BPqmX54I.js";import{a as W,c as j}from"./compare-CZ4TIoIW.js";import{_ as Q}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./vnode-78qeDweP.js";const X={class:"compare-result"},Y={class:"result-header"},Z={class:"header-actions"},tt={class:"result-grid"},et={class:"card-header"},st={class:"model-name"},ot={key:0,class:"error-text"},at={key:1,class:"streaming-text"},nt={key:3,class:"loading-text"},lt={key:4,class:"result-stats"},rt=O({__name:"CompareResultView",setup(it){const u=G(),q=u.query.taskId,k=decodeURIComponent(u.query.question||""),h=decodeURIComponent(u.query.systemPrompt||""),E=Number(u.query.temperature||.7),P=Number(u.query.topP||.9),I=Number(u.query.topK||40),b=Number(u.query.maxTokens||2048),w=u.query.taskName,l=g([]),C=g(!1),x=g([]),f=new Set;async function B(){if(!C.value){C.value=!0;try{const s=await W(q);let t=[];s.load_status&&(t=(typeof s.load_status=="string"?JSON.parse(s.load_status):s.load_status).loaded_models||[]),x.value=t,l.value=t.map(o=>({name:o.model_name||"模型",content:"",displayContent:"",isTyping:!1,status:"loading"})),await Promise.all(t.map((o,e)=>S(o,e)))}catch{}}}async function S(s,t){const o=Date.now();try{const e=await V(j({port:s.port,model_name:s.model_name,messages:[...h?[{role:"system",content:h}]:[],{role:"user",content:k}],temperature:E,top_p:P,top_k:I,max_tokens:b}),3e5),a=(e==null?void 0:e.response)||(e==null?void 0:e.content)||(e==null?void 0:e.data)||JSON.stringify(e),_=(Date.now()-o)/1e3;l.value[t].content=a,l.value[t].status="done",l.value[t].stats={totalTime:_,charsPerSec:_>0?Number((a.length/_).toFixed(1)):0},M(t,a)}catch(e){l.value[t].content="推理失败: "+(e.message||""),l.value[t].status="error"}}async function V(s,t){let o=null;try{return await Promise.race([s,new Promise((e,a)=>{o=setTimeout(()=>a(new Error("推理超时")),t)})])}finally{o&&clearTimeout(o)}}function M(s,t){let o=0;l.value[s].isTyping=!0;const e=Math.max(2,Math.ceil(t.length/30)),a=setInterval(()=>{o+=e,l.value[s].displayContent=t.slice(0,o),o>=t.length&&(clearInterval(a),f.delete(a),l.value[s].displayContent=t,l.value[s].isTyping=!1)},50);f.add(a)}return U(B),J(()=>{f.forEach(clearInterval),f.clear()}),(s,t)=>{const o=$,e=D,a=z,_=F;return r(),d("div",X,[i("div",Y,[i("h2",null,"对比结果"+c(v(w)?` - ${v(w)}`:""),1),i("div",Z,[N(o,{onClick:t[0]||(t[0]=n=>s.$router.push("/model-inference"))},{default:m(()=>[...t[1]||(t[1]=[p("返回列表",-1)])]),_:1})])]),N(e,{type:"info",closable:!1,"show-icon":"",class:"question-box"},{title:m(()=>[t[2]||(t[2]=i("strong",null,"问题:",-1)),p(c(v(k)),1)]),_:1}),i("div",tt,[(r(!0),d(K,null,A(l.value,(n,R)=>(r(),y(_,{key:R,shadow:"hover",class:"result-card"},{header:m(()=>[i("div",et,[i("span",st,c(n.name),1),n.status==="loading"?(r(),y(a,{key:0,type:"warning",size:"small"},{default:m(()=>[...t[3]||(t[3]=[p("生成中...",-1)])]),_:1})):n.status==="done"?(r(),y(a,{key:1,type:"success",size:"small"},{default:m(()=>[...t[4]||(t[4]=[p("完成",-1)])]),_:1})):(r(),y(a,{key:2,type:"danger",size:"small"},{default:m(()=>[...t[5]||(t[5]=[p("失败",-1)])]),_:1}))])]),default:m(()=>{var T;return[n.status==="error"?(r(),d("div",ot,c(n.content),1)):n.isTyping?(r(),d("div",at,c(n.displayContent),1)):n.displayContent?(r(),y(L,{key:2,content:n.displayContent},null,8,["content"])):(r(),d("div",nt,[...t[6]||(t[6]=[i("i",{class:"fa fa-spinner fa-spin"},null,-1),p(" 正在生成回答... ",-1)])])),n.stats?(r(),d("div",lt,[i("span",null,"耗时 "+c((T=n.stats.totalTime)==null?void 0:T.toFixed(1))+"s",1),i("span",null,"速度 "+c(n.stats.charsPerSec)+" 字/秒",1)])):H("",!0)]}),_:2},1024))),128))])])}}}),kt=Q(rt,[["__scopeId","data-v-4c4e03d7"]]);export{kt as default};
|
||||
import{E as D}from"./el-alert-CoSdHkeK.js";import{E as $}from"./index-BNsCyG4A.js";import{E as z}from"./index-ByhS8A1d.js";import{E as F}from"./el-card-D86TXgMv.js";import{d as O,G as U,H as J,c as d,q as i,aa as c,f as v,s as N,w as m,M as K,ad as A,ac as G,y as g,o as r,x as p,e as y,g as H}from"./index-Ds9AETjS.js";/* empty css */import{_ as L}from"./MarkdownView.vue_vue_type_style_index_0_lang-CNORZ6uG.js";import{a as W,c as j}from"./compare-fmy9bbtE.js";import{_ as Q}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./vnode-CBiWHFw7.js";const X={class:"compare-result"},Y={class:"result-header"},Z={class:"header-actions"},tt={class:"result-grid"},et={class:"card-header"},st={class:"model-name"},ot={key:0,class:"error-text"},at={key:1,class:"streaming-text"},nt={key:3,class:"loading-text"},lt={key:4,class:"result-stats"},rt=O({__name:"CompareResultView",setup(it){const u=G(),q=u.query.taskId,k=decodeURIComponent(u.query.question||""),h=decodeURIComponent(u.query.systemPrompt||""),E=Number(u.query.temperature||.7),P=Number(u.query.topP||.9),I=Number(u.query.topK||40),b=Number(u.query.maxTokens||2048),w=u.query.taskName,l=g([]),C=g(!1),x=g([]),f=new Set;async function B(){if(!C.value){C.value=!0;try{const s=await W(q);let t=[];s.load_status&&(t=(typeof s.load_status=="string"?JSON.parse(s.load_status):s.load_status).loaded_models||[]),x.value=t,l.value=t.map(o=>({name:o.model_name||"模型",content:"",displayContent:"",isTyping:!1,status:"loading"})),await Promise.all(t.map((o,e)=>S(o,e)))}catch{}}}async function S(s,t){const o=Date.now();try{const e=await V(j({port:s.port,model_name:s.model_name,messages:[...h?[{role:"system",content:h}]:[],{role:"user",content:k}],temperature:E,top_p:P,top_k:I,max_tokens:b}),3e5),a=(e==null?void 0:e.response)||(e==null?void 0:e.content)||(e==null?void 0:e.data)||JSON.stringify(e),_=(Date.now()-o)/1e3;l.value[t].content=a,l.value[t].status="done",l.value[t].stats={totalTime:_,charsPerSec:_>0?Number((a.length/_).toFixed(1)):0},M(t,a)}catch(e){l.value[t].content="推理失败: "+(e.message||""),l.value[t].status="error"}}async function V(s,t){let o=null;try{return await Promise.race([s,new Promise((e,a)=>{o=setTimeout(()=>a(new Error("推理超时")),t)})])}finally{o&&clearTimeout(o)}}function M(s,t){let o=0;l.value[s].isTyping=!0;const e=Math.max(2,Math.ceil(t.length/30)),a=setInterval(()=>{o+=e,l.value[s].displayContent=t.slice(0,o),o>=t.length&&(clearInterval(a),f.delete(a),l.value[s].displayContent=t,l.value[s].isTyping=!1)},50);f.add(a)}return U(B),J(()=>{f.forEach(clearInterval),f.clear()}),(s,t)=>{const o=$,e=D,a=z,_=F;return r(),d("div",X,[i("div",Y,[i("h2",null,"对比结果"+c(v(w)?` - ${v(w)}`:""),1),i("div",Z,[N(o,{onClick:t[0]||(t[0]=n=>s.$router.push("/model-inference"))},{default:m(()=>[...t[1]||(t[1]=[p("返回列表",-1)])]),_:1})])]),N(e,{type:"info",closable:!1,"show-icon":"",class:"question-box"},{title:m(()=>[t[2]||(t[2]=i("strong",null,"问题:",-1)),p(c(v(k)),1)]),_:1}),i("div",tt,[(r(!0),d(K,null,A(l.value,(n,R)=>(r(),y(_,{key:R,shadow:"hover",class:"result-card"},{header:m(()=>[i("div",et,[i("span",st,c(n.name),1),n.status==="loading"?(r(),y(a,{key:0,type:"warning",size:"small"},{default:m(()=>[...t[3]||(t[3]=[p("生成中...",-1)])]),_:1})):n.status==="done"?(r(),y(a,{key:1,type:"success",size:"small"},{default:m(()=>[...t[4]||(t[4]=[p("完成",-1)])]),_:1})):(r(),y(a,{key:2,type:"danger",size:"small"},{default:m(()=>[...t[5]||(t[5]=[p("失败",-1)])]),_:1}))])]),default:m(()=>{var T;return[n.status==="error"?(r(),d("div",ot,c(n.content),1)):n.isTyping?(r(),d("div",at,c(n.displayContent),1)):n.displayContent?(r(),y(L,{key:2,content:n.displayContent},null,8,["content"])):(r(),d("div",nt,[...t[6]||(t[6]=[i("i",{class:"fa fa-spinner fa-spin"},null,-1),p(" 正在生成回答... ",-1)])])),n.stats?(r(),d("div",lt,[i("span",null,"耗时 "+c((T=n.stats.totalTime)==null?void 0:T.toFixed(1))+"s",1),i("span",null,"速度 "+c(n.stats.charsPerSec)+" 字/秒",1)])):H("",!0)]}),_:2},1024))),128))])])}}}),kt=Q(rt,[["__scopeId","data-v-4c4e03d7"]]);export{kt as default};
|
||||
1
frontend/dist/assets/ComputeNodesView-BGkUyXxx.js
vendored
Normal file
1
frontend/dist/assets/ComputeNodesView-BGkUyXxx.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
.compute-page[data-v-daf09639]{display:flex;flex-direction:column;gap:16px;min-height:0;height:100%;padding:24px;background:#fff}.compute-header[data-v-daf09639]{display:flex;justify-content:space-between;gap:16px;align-items:flex-start}.compute-header h1[data-v-daf09639]{margin:0;font-size:24px;font-weight:650;color:#111827}.compute-header p[data-v-daf09639]{margin:8px 0 0;color:#64748b}.header-actions[data-v-daf09639]{display:flex;align-items:center;gap:12px}.last-updated[data-v-daf09639],.muted[data-v-daf09639]{color:#64748b;font-size:12px}.summary-grid[data-v-daf09639]{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.summary-tile[data-v-daf09639]{border:1px solid #e5e7eb;border-radius:8px;padding:14px 16px;background:#f8fafc}.summary-tile span[data-v-daf09639]{display:block;color:#64748b;font-size:12px}.summary-tile strong[data-v-daf09639]{display:block;margin-top:8px;color:#111827;font-size:24px}.compute-tabs[data-v-daf09639]{flex:1;min-height:0}.compute-tabs[data-v-daf09639] .el-tabs__content{height:calc(100% - 56px)}.compute-tabs[data-v-daf09639] .el-tab-pane{height:100%}.mono[data-v-daf09639]{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px}.replica-toolbar[data-v-daf09639]{display:flex;gap:12px;align-items:center;margin-bottom:12px}.replica-toolbar .el-select[data-v-daf09639]{width:280px}@media(max-width:960px){.compute-header[data-v-daf09639],.header-actions[data-v-daf09639],.replica-toolbar[data-v-daf09639]{flex-direction:column;align-items:stretch}.summary-grid[data-v-daf09639]{grid-template-columns:repeat(2,minmax(0,1fr))}}
|
||||
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/ComputeNodesView-Czj9vH6H.css
vendored
Normal file
1
frontend/dist/assets/ComputeNodesView-Czj9vH6H.css
vendored
Normal file
@@ -0,0 +1 @@
|
||||
.compute-page[data-v-5ebf5cb5]{display:flex;flex-direction:column;gap:16px;min-height:0;height:100%;padding:24px;background:#fff}.compute-header[data-v-5ebf5cb5]{display:flex;justify-content:space-between;gap:16px;align-items:flex-start}.compute-header h1[data-v-5ebf5cb5]{margin:0;font-size:24px;font-weight:650;color:#111827}.compute-header p[data-v-5ebf5cb5]{margin:8px 0 0;color:#64748b}.header-actions[data-v-5ebf5cb5]{display:flex;align-items:center;gap:12px}.last-updated[data-v-5ebf5cb5],.muted[data-v-5ebf5cb5]{color:#64748b;font-size:12px}.last-updated[data-v-5ebf5cb5]{display:inline-block;min-width:92px;text-align:right}.summary-grid[data-v-5ebf5cb5]{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.summary-tile[data-v-5ebf5cb5]{border:1px solid #e5e7eb;border-radius:8px;padding:14px 16px;background:#f8fafc}.summary-tile span[data-v-5ebf5cb5]{display:block;color:#64748b;font-size:12px}.summary-tile strong[data-v-5ebf5cb5]{display:block;margin-top:8px;color:#111827;font-size:24px}.compute-tabs[data-v-5ebf5cb5]{flex:1;min-height:0}.compute-tabs[data-v-5ebf5cb5] .el-tabs__content{height:calc(100% - 56px)}.compute-tabs[data-v-5ebf5cb5] .el-tab-pane{height:100%}.mono[data-v-5ebf5cb5]{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px}.replica-toolbar[data-v-5ebf5cb5]{display:flex;gap:12px;align-items:center;margin-bottom:12px}.replica-toolbar .el-select[data-v-5ebf5cb5]{width:280px}.node-form-grid[data-v-5ebf5cb5]{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));column-gap:12px}.node-form-grid[data-v-5ebf5cb5] .el-input-number,.node-form-grid[data-v-5ebf5cb5] .el-select{width:100%}@media(max-width:960px){.compute-header[data-v-5ebf5cb5],.header-actions[data-v-5ebf5cb5],.replica-toolbar[data-v-5ebf5cb5]{flex-direction:column;align-items:stretch}.summary-grid[data-v-5ebf5cb5]{grid-template-columns:repeat(2,minmax(0,1fr))}.node-form-grid[data-v-5ebf5cb5]{grid-template-columns:1fr}}
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{a as N,E as v}from"./el-form-item-D5kF3B90.js";import{E as b}from"./el-popper-D6_hxRbQ.js";import{E as O}from"./index-TFUf94PZ.js";import{E as S}from"./index-BjEW7-SA.js";import{E as g,a as E}from"./el-select-Dl-FRZQd.js";import{d as J,e as V,w as e,o as w,q as t,s as l,x as i,z as x,A as C}from"./index-BKKvzUDD.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-scrollbar-ClJnvz-9.js";/* empty css */import{P as U}from"./PageCard-BoKXOzst.js";import{_ as y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./index-GAnQrJsQ.js";import"./index-CWUnzf90.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./vnode-78qeDweP.js";import"./index-D5ryD3I5.js";import"./index-BDEF353-.js";import"./scroll-BkzZKETR.js";import"./clamp-CbbY8h6F.js";import"./toNumber-Dkj3QRv9.js";import"./_baseIteratee-BAilevJ_.js";import"./el-card-CApHJ1Gj.js";const F={class:"converter-panel"},L={class:"form-row"},T={class:"panel-footer"},B={class:"actions"},k=J({__name:"DataConvertView",setup(I){const a=x({outputName:"converted-data",encoding:"UTF-8"});function p(){C.info("当前仅完成界面设计,转换功能将在后续接入")}function d(){Object.assign(a,{outputName:"converted-data",encoding:"UTF-8"})}return(j,o)=>{const n=N,m=O,u=E,f=g,c=v,r=S,_=b;return w(),V(U,{class:"data-convert-page",title:"数据类型转换",subtitle:"将 JSON 文件转换为便于训练和评测使用的 JSONL 格式"},{default:e(()=>[t("div",F,[o[9]||(o[9]=t("div",{class:"panel-header"},[t("div",{class:"tool-icon","aria-hidden":"true"},[t("i",{class:"fa fa-exchange"})]),t("div",null,[t("h3",null,"JSON 转 JSONL"),t("p",null,"每条 JSON 数据将输出为 JSONL 文件中的一行记录")])],-1)),l(c,{class:"converter-form","label-position":"top"},{default:e(()=>[l(n,{label:"转换类型"},{default:e(()=>[...o[2]||(o[2]=[t("div",{class:"format-field","aria-label":"JSON 转 JSONL"},[t("span",null,"JSON"),t("i",{class:"fa fa-long-arrow-right","aria-hidden":"true"}),t("span",null,"JSONL")],-1)])]),_:1}),l(n,{label:"源文件",required:""},{default:e(()=>[t("button",{class:"upload-zone",type:"button",onClick:p},[...o[3]||(o[3]=[t("i",{class:"fa fa-cloud-upload","aria-hidden":"true"},null,-1),t("span",{class:"upload-content"},[t("strong",null,"点击选择或拖拽 JSON 文件到此处"),t("small",null,"仅支持 .json 格式,单文件不超过 200 MB")],-1),t("span",{class:"select-button"},"选择文件",-1)])])]),_:1}),t("div",L,[l(n,{label:"输出文件名"},{default:e(()=>[l(m,{modelValue:a.outputName,"onUpdate:modelValue":o[0]||(o[0]=s=>a.outputName=s)},{append:e(()=>[...o[4]||(o[4]=[i(".jsonl",-1)])]),_:1},8,["modelValue"])]),_:1}),l(n,{label:"字符编码"},{default:e(()=>[l(f,{modelValue:a.encoding,"onUpdate:modelValue":o[1]||(o[1]=s=>a.encoding=s),style:{width:"100%"}},{default:e(()=>[l(u,{label:"UTF-8",value:"UTF-8"})]),_:1},8,["modelValue"])]),_:1})]),o[5]||(o[5]=t("div",{class:"format-tip"},[t("i",{class:"fa fa-info-circle","aria-hidden":"true"}),t("span",null,"支持由 JSON 数组转换为 JSONL,每个数组元素输出为一行。")],-1))]),_:1}),t("div",T,[o[8]||(o[8]=t("span",{class:"prototype-label"},"当前为 UI 原型,暂不执行实际转换",-1)),t("div",B,[l(r,{onClick:d},{default:e(()=>[...o[6]||(o[6]=[i("重置",-1)])]),_:1}),l(_,{content:"转换功能将在后续开发中接入",placement:"top"},{default:e(()=>[t("span",null,[l(r,{type:"primary",disabled:""},{default:e(()=>[...o[7]||(o[7]=[i("开始转换",-1)])]),_:1})])]),_:1})])])])]),_:1})}}}),rt=y(k,[["__scopeId","data-v-fcb69543"]]);export{rt as default};
|
||||
import{a as N,E as v}from"./el-form-item-Bq0n_p5a.js";import{E as b}from"./el-popper-D1tByNLb.js";import{E as O}from"./index-h5FkzAP_.js";import{E as S}from"./index-BNsCyG4A.js";import{E as g,a as E}from"./el-select-BD70by95.js";import{d as J,e as V,w as e,o as w,q as t,s as l,x as i,z as x,A as C}from"./index-Ds9AETjS.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-scrollbar-CF42LNzg.js";/* empty css */import{P as U}from"./PageCard-CV3p14-x.js";import{_ as y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./index-BSbCLbaz.js";import"./index-sWKLi7bH.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./vnode-CBiWHFw7.js";import"./index-CQlF36Lk.js";import"./index-ByhS8A1d.js";import"./scroll-DVmMv0d9.js";import"./clamp-CyhADbdq.js";import"./toNumber-DrayciwT.js";import"./_baseIteratee-82gB5Yov.js";import"./el-card-D86TXgMv.js";const F={class:"converter-panel"},L={class:"form-row"},T={class:"panel-footer"},B={class:"actions"},k=J({__name:"DataConvertView",setup(I){const a=x({outputName:"converted-data",encoding:"UTF-8"});function p(){C.info("当前仅完成界面设计,转换功能将在后续接入")}function d(){Object.assign(a,{outputName:"converted-data",encoding:"UTF-8"})}return(j,o)=>{const n=N,m=O,u=E,f=g,c=v,r=S,_=b;return w(),V(U,{class:"data-convert-page",title:"数据类型转换",subtitle:"将 JSON 文件转换为便于训练和评测使用的 JSONL 格式"},{default:e(()=>[t("div",F,[o[9]||(o[9]=t("div",{class:"panel-header"},[t("div",{class:"tool-icon","aria-hidden":"true"},[t("i",{class:"fa fa-exchange"})]),t("div",null,[t("h3",null,"JSON 转 JSONL"),t("p",null,"每条 JSON 数据将输出为 JSONL 文件中的一行记录")])],-1)),l(c,{class:"converter-form","label-position":"top"},{default:e(()=>[l(n,{label:"转换类型"},{default:e(()=>[...o[2]||(o[2]=[t("div",{class:"format-field","aria-label":"JSON 转 JSONL"},[t("span",null,"JSON"),t("i",{class:"fa fa-long-arrow-right","aria-hidden":"true"}),t("span",null,"JSONL")],-1)])]),_:1}),l(n,{label:"源文件",required:""},{default:e(()=>[t("button",{class:"upload-zone",type:"button",onClick:p},[...o[3]||(o[3]=[t("i",{class:"fa fa-cloud-upload","aria-hidden":"true"},null,-1),t("span",{class:"upload-content"},[t("strong",null,"点击选择或拖拽 JSON 文件到此处"),t("small",null,"仅支持 .json 格式,单文件不超过 200 MB")],-1),t("span",{class:"select-button"},"选择文件",-1)])])]),_:1}),t("div",L,[l(n,{label:"输出文件名"},{default:e(()=>[l(m,{modelValue:a.outputName,"onUpdate:modelValue":o[0]||(o[0]=s=>a.outputName=s)},{append:e(()=>[...o[4]||(o[4]=[i(".jsonl",-1)])]),_:1},8,["modelValue"])]),_:1}),l(n,{label:"字符编码"},{default:e(()=>[l(f,{modelValue:a.encoding,"onUpdate:modelValue":o[1]||(o[1]=s=>a.encoding=s),style:{width:"100%"}},{default:e(()=>[l(u,{label:"UTF-8",value:"UTF-8"})]),_:1},8,["modelValue"])]),_:1})]),o[5]||(o[5]=t("div",{class:"format-tip"},[t("i",{class:"fa fa-info-circle","aria-hidden":"true"}),t("span",null,"支持由 JSON 数组转换为 JSONL,每个数组元素输出为一行。")],-1))]),_:1}),t("div",T,[o[8]||(o[8]=t("span",{class:"prototype-label"},"当前为 UI 原型,暂不执行实际转换",-1)),t("div",B,[l(r,{onClick:d},{default:e(()=>[...o[6]||(o[6]=[i("重置",-1)])]),_:1}),l(_,{content:"转换功能将在后续开发中接入",placement:"top"},{default:e(()=>[t("span",null,[l(r,{type:"primary",disabled:""},{default:e(()=>[...o[7]||(o[7]=[i("开始转换",-1)])]),_:1})])]),_:1})])])])]),_:1})}}}),rt=y(k,[["__scopeId","data-v-fcb69543"]]);export{rt as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{E as b}from"./index-BjEW7-SA.js";import{E as k}from"./index-BDEF353-.js";import{E as x}from"./el-table-D8K5zLyz.js";import{d as v,c as m,s as e,w as a,y as D,o as l,q as p,x as i,e as C,aa as n,A as E,v as T}from"./index-BKKvzUDD.js";/* empty css */import"./el-checkbox-qGZ2-RM2.js";import{D as z}from"./DataTablePage-C-EM-7-s.js";import{_ as B}from"./ModelStatusTag.vue_vue_type_script_setup_true_lang-VFIQNBU2.js";import{_ as V}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-ClJnvz-9.js";import"./index-GAnQrJsQ.js";import"./el-popper-D6_hxRbQ.js";import"./index-CWUnzf90.js";import"./_baseClone-B5RBzbh5.js";import"./_baseIteratee-BAilevJ_.js";import"./castArray-5uErZEc3.js";import"./debounce-ByXPNh5F.js";import"./toNumber-Dkj3QRv9.js";import"./raf-C-x62Pcl.js";import"./omit-BTSq4AYh.js";import"./index-TFUf94PZ.js";import"./index-DLzof2Fz.js";import"./el-card-CApHJ1Gj.js";import"./el-pagination-DOPYW-qb.js";import"./el-select-Dl-FRZQd.js";import"./vnode-78qeDweP.js";import"./index-D5ryD3I5.js";import"./scroll-BkzZKETR.js";import"./clamp-CbbY8h6F.js";import"./directive-DsZckUTJ.js";import"./index-dNo5H1ie.js";import"./validator-Fn0bm4xU.js";import"./index-CQXVR1Ud.js";import"./index-DfyOU94W.js";import"./el-tooltip-l0sNRNKZ.js";const L={class:"data-process-page",style:{height:"100%"}},N={key:1},I={class:"action-buttons"},P=v({__name:"DataProcessListView",setup(S){const u={structured:"结构化数据",unstructured:"非结构化数据",external:"外来数据源拉取"},d=T(),_=D([{id:183921,name:"客服问答数据清洗",status:"completed",process_type:"structured",source_dataset:"客服对话原始集",output_dataset:"客服对话清洗集",create_time:"2026-07-08 14:23:00"},{id:492015,name:"指令微调数据构造",status:"running",process_type:"unstructured",source_dataset:"通用语料库",output_dataset:"SFT 指令集",create_time:"2026-07-09 09:10:00"},{id:731948,name:"敏感信息脱敏处理",status:"pending",process_type:"structured",source_dataset:"用户反馈数据",create_time:"2026-07-09 16:45:00"},{id:582012,name:"多轮对话拼接",status:"failed",process_type:"structured",source_dataset:"单轮问答集",create_time:"2026-07-10 08:30:00"}]);function f(s){const r=s.id;d.push({name:"data-process-detail",params:{id:r}})}function g(s){E.info("删除功能开发中...")}function y(s){return s?new Date(s).toLocaleString("zh-CN",{hour12:!1}):"-"}return(s,r)=>{const o=x,h=k,c=b;return l(),m("div",L,[e(z,{title:"",data:_.value,searchable:"","search-fields":["name"],"create-text":"新建数据处理","create-to":"/data-process/create","row-key":"id","page-size":10},{columns:a(()=>[e(o,{label:"任务ID",prop:"id",align:"center",width:"100"}),e(o,{label:"任务名称",prop:"name",align:"center","show-overflow-tooltip":""}),e(o,{label:"任务状态",align:"center",width:"110"},{default:a(({row:t})=>[e(B,{status:t.status},null,8,["status"])]),_:1}),e(o,{label:"处理类型",align:"center",width:"140"},{default:a(({row:t})=>[t.process_type?(l(),C(h,{key:0,size:"small",type:"info",effect:"plain"},{default:a(()=>[i(n(u[t.process_type]||t.process_type),1)]),_:2},1024)):(l(),m("span",N,"-"))]),_:1}),e(o,{label:"源数据集",align:"center","show-overflow-tooltip":""},{default:a(({row:t})=>[i(n(t.source_dataset||"-"),1)]),_:1}),e(o,{label:"输出数据集",align:"center","show-overflow-tooltip":""},{default:a(({row:t})=>[i(n(t.output_dataset||"-"),1)]),_:1}),e(o,{label:"创建时间",align:"center",width:"190"},{default:a(({row:t})=>[i(n(y(t.create_time)),1)]),_:1})]),actions:a(({row:t})=>[p("div",I,[e(c,{type:"primary",link:"",size:"small",onClick:w=>f(t)},{default:a(()=>[...r[0]||(r[0]=[p("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),i("详情 ",-1)])]),_:1},8,["onClick"]),e(c,{type:"danger",link:"",size:"small",onClick:w=>g(t)},{default:a(()=>[...r[1]||(r[1]=[p("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),i("删除 ",-1)])]),_:1},8,["onClick"])])]),_:1},8,["data"])])}}}),yt=V(P,[["__scopeId","data-v-c0da9336"]]);export{yt as default};
|
||||
import{E as b}from"./index-BNsCyG4A.js";import{E as k}from"./index-ByhS8A1d.js";import{E as x}from"./el-table-C7Zx0Bmo.js";import{d as v,c as m,s as e,w as a,y as D,o as l,q as p,x as i,e as C,aa as n,A as E,v as T}from"./index-Ds9AETjS.js";/* empty css */import"./el-checkbox-pV3o_8iy.js";import{D as z}from"./DataTablePage-DzEUb2tf.js";import{_ as B}from"./ModelStatusTag.vue_vue_type_script_setup_true_lang-CHgevs5i.js";import{_ as V}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-CF42LNzg.js";import"./index-BSbCLbaz.js";import"./el-popper-D1tByNLb.js";import"./index-sWKLi7bH.js";import"./_baseClone-CZxxnGCv.js";import"./_baseIteratee-82gB5Yov.js";import"./castArray-BWI4UxBy.js";import"./debounce-D6mJVDYC.js";import"./toNumber-DrayciwT.js";import"./raf-zQ00nuMI.js";import"./omit-BzI1MV_K.js";import"./index-h5FkzAP_.js";import"./index-B0if-AHo.js";import"./el-card-D86TXgMv.js";import"./el-pagination-C9R_oTJ1.js";import"./el-select-BD70by95.js";import"./vnode-CBiWHFw7.js";import"./index-CQlF36Lk.js";import"./scroll-DVmMv0d9.js";import"./clamp-CyhADbdq.js";import"./directive-B1cwmVOz.js";import"./index-DqJTjrY3.js";import"./validator-D1MwuZXj.js";import"./index-jBnYrPdD.js";import"./index-RtwaMutE.js";import"./el-tooltip-l0sNRNKZ.js";const L={class:"data-process-page",style:{height:"100%"}},N={key:1},I={class:"action-buttons"},P=v({__name:"DataProcessListView",setup(S){const u={structured:"结构化数据",unstructured:"非结构化数据",external:"外来数据源拉取"},d=T(),_=D([{id:183921,name:"客服问答数据清洗",status:"completed",process_type:"structured",source_dataset:"客服对话原始集",output_dataset:"客服对话清洗集",create_time:"2026-07-08 14:23:00"},{id:492015,name:"指令微调数据构造",status:"running",process_type:"unstructured",source_dataset:"通用语料库",output_dataset:"SFT 指令集",create_time:"2026-07-09 09:10:00"},{id:731948,name:"敏感信息脱敏处理",status:"pending",process_type:"structured",source_dataset:"用户反馈数据",create_time:"2026-07-09 16:45:00"},{id:582012,name:"多轮对话拼接",status:"failed",process_type:"structured",source_dataset:"单轮问答集",create_time:"2026-07-10 08:30:00"}]);function f(s){const r=s.id;d.push({name:"data-process-detail",params:{id:r}})}function g(s){E.info("删除功能开发中...")}function y(s){return s?new Date(s).toLocaleString("zh-CN",{hour12:!1}):"-"}return(s,r)=>{const o=x,h=k,c=b;return l(),m("div",L,[e(z,{title:"",data:_.value,searchable:"","search-fields":["name"],"create-text":"新建数据处理","create-to":"/data-process/create","row-key":"id","page-size":10},{columns:a(()=>[e(o,{label:"任务ID",prop:"id",align:"center",width:"100"}),e(o,{label:"任务名称",prop:"name",align:"center","show-overflow-tooltip":""}),e(o,{label:"任务状态",align:"center",width:"110"},{default:a(({row:t})=>[e(B,{status:t.status},null,8,["status"])]),_:1}),e(o,{label:"处理类型",align:"center",width:"140"},{default:a(({row:t})=>[t.process_type?(l(),C(h,{key:0,size:"small",type:"info",effect:"plain"},{default:a(()=>[i(n(u[t.process_type]||t.process_type),1)]),_:2},1024)):(l(),m("span",N,"-"))]),_:1}),e(o,{label:"源数据集",align:"center","show-overflow-tooltip":""},{default:a(({row:t})=>[i(n(t.source_dataset||"-"),1)]),_:1}),e(o,{label:"输出数据集",align:"center","show-overflow-tooltip":""},{default:a(({row:t})=>[i(n(t.output_dataset||"-"),1)]),_:1}),e(o,{label:"创建时间",align:"center",width:"190"},{default:a(({row:t})=>[i(n(y(t.create_time)),1)]),_:1})]),actions:a(({row:t})=>[p("div",I,[e(c,{type:"primary",link:"",size:"small",onClick:w=>f(t)},{default:a(()=>[...r[0]||(r[0]=[p("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),i("详情 ",-1)])]),_:1},8,["onClick"]),e(c,{type:"danger",link:"",size:"small",onClick:w=>g(t)},{default:a(()=>[...r[1]||(r[1]=[p("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),i("删除 ",-1)])]),_:1},8,["onClick"])])]),_:1},8,["data"])])}}}),yt=V(P,[["__scopeId","data-v-c0da9336"]]);export{yt as default};
|
||||
@@ -1 +1 @@
|
||||
import{E as M}from"./index-TFUf94PZ.js";import{E as Z}from"./index-BjEW7-SA.js";import{E as A}from"./el-card-CApHJ1Gj.js";import{E as G}from"./el-pagination-DOPYW-qb.js";import{E as H,a as J}from"./el-table-D8K5zLyz.js";import{v as O}from"./directive-DsZckUTJ.js";import{E as b}from"./index-dNo5H1ie.js";import{d as Q,y as f,D as _,c as D,s as p,w as r,o as u,q as s,h as w,e as g,g as v,x as y,aa as k,Z as W,K as X,j as $,v as Y}from"./index-BKKvzUDD.js";import"./el-scrollbar-ClJnvz-9.js";import"./el-popper-D6_hxRbQ.js";/* empty css */import"./el-select-Dl-FRZQd.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-checkbox-qGZ2-RM2.js";import{_ as ee}from"./_plugin-vue_export-helper-DlAUqK2U.js";const te={class:"data-table-page"},ae={class:"table-toolbar"},le={class:"table-title"},oe={class:"toolbar-actions"},se={key:0,class:"batch-bar"},ne={class:"batch-info"},re={class:"table-body"},ie={class:"table-pagination"},ce=Q({__name:"DataTablePage",props:{title:{},data:{},loading:{type:Boolean,default:!1},searchable:{type:Boolean,default:!1},searchFields:{},multiSelect:{type:Boolean,default:!1},createText:{},createTo:{},deleteFn:{},rowKey:{default:"id"},emptyText:{default:"暂无数据"},pageSize:{default:10},pageSizes:{default:()=>[10,20,50,100]}},emits:["create","refresh"],setup(l,{expose:F,emit:V}){const a=l,x=V,P=Y(),S=f(),o=f(new Set),d=f(""),m=f(1),h=f(a.pageSize);function z(t){return t[a.rowKey]??t.name??t.id}const T=$(()=>{var e;if(!d.value.trim()||!((e=a.searchFields)!=null&&e.length))return a.data;const t=d.value.toLowerCase().trim();return a.data.filter(i=>a.searchFields.some(n=>String(i[n]??"").toLowerCase().includes(t)))}),I=$(()=>{const t=(m.value-1)*h.value;return T.value.slice(t,t+h.value)});function K(t){o.value=new Set(t.map(z))}async function E(t,e="确定要删除这条记录吗?"){if(a.deleteFn)try{await b.confirm(e,"确认删除",{type:"warning",confirmButtonText:"确定",cancelButtonText:"取消"}),await a.deleteFn(t),x("refresh")}catch{}}async function N(){if(!(o.value.size===0||!a.deleteFn))try{await b.confirm(`确定要删除选中的 ${o.value.size} 条记录吗?`,"批量删除",{type:"warning"});const t=a.data.filter(n=>o.value.has(z(n)));let e=0,i=0;for(const n of t)try{await a.deleteFn(n),e++}catch{i++}o.value.clear(),x("refresh"),i===0?b.alert(`成功删除 ${e} 条记录`,"成功",{type:"success"}):b.alert(`成功 ${e} 条,失败 ${i} 条`,"部分失败",{type:"warning"})}catch{}}function C(){var t;o.value.clear(),(t=S.value)==null||t.clearSelection()}function L(){a.createTo?P.push(a.createTo):x("create")}return _(()=>a.data,()=>{var t;o.value.clear(),m.value=1,(t=S.value)==null||t.clearSelection()}),_(()=>d.value,()=>{m.value=1}),F({handleDelete:E,clearSelection:C}),(t,e)=>{const i=M,n=Z,B=H,R=J,U=G,j=A,q=O;return u(),D("div",te,[p(j,{shadow:"never",class:"table-card"},{default:r(()=>[s("div",ae,[w(t.$slots,"title",{},()=>[s("h2",le,k(l.title),1)],!0),s("div",oe,[l.searchable?(u(),g(i,{key:0,modelValue:d.value,"onUpdate:modelValue":e[0]||(e[0]=c=>d.value=c),placeholder:`搜索${l.title}...`,clearable:"",class:"search-input"},{prefix:r(()=>[...e[3]||(e[3]=[s("i",{class:"fa fa-search"},null,-1)])]),_:1},8,["modelValue","placeholder"])):v("",!0),l.createText?(u(),g(n,{key:1,type:"primary",onClick:L},{default:r(()=>[e[4]||(e[4]=s("i",{class:"fa fa-plus",style:{"margin-right":"4px"}},null,-1)),y(" "+k(l.createText),1)]),_:1})):v("",!0),w(t.$slots,"toolbar-extra",{},void 0,!0)])]),p(W,{name:"el-zoom-in-top"},{default:r(()=>[l.multiSelect&&o.value.size>0?(u(),D("div",se,[s("span",ne,[e[6]||(e[6]=y(" 已选择 ",-1)),s("strong",null,k(o.value.size),1),e[7]||(e[7]=y(" 项 ",-1)),p(n,{link:"",type:"primary",onClick:C},{default:r(()=>[...e[5]||(e[5]=[y("取消选择",-1)])]),_:1})]),p(n,{type:"danger",size:"small",onClick:N},{default:r(()=>[e[8]||(e[8]=s("i",{class:"fa fa-trash",style:{"margin-right":"4px"}},null,-1)),y(" 批量删除 ("+k(o.value.size)+") ",1)]),_:1})])):v("",!0)]),_:1}),s("div",re,[X((u(),g(R,{ref_key:"tableRef",ref:S,data:I.value,"row-key":l.rowKey,height:"100%",style:{width:"100%"},"empty-text":l.emptyText,onSelectionChange:K},{default:r(()=>[l.multiSelect?(u(),g(B,{key:0,type:"selection",width:"50",align:"center","reserve-selection":"",fixed:"left"})):v("",!0),w(t.$slots,"columns",{},void 0,!0),t.$slots.actions?(u(),g(B,{key:1,label:"操作",align:"center",width:"220",fixed:"right"},{default:r(({row:c})=>[w(t.$slots,"actions",{row:c,handleDelete:E},void 0,!0)]),_:3})):v("",!0)]),_:3},8,["data","row-key","empty-text"])),[[q,l.loading]])]),s("div",ie,[p(U,{"current-page":m.value,"onUpdate:currentPage":e[1]||(e[1]=c=>m.value=c),"page-size":h.value,"onUpdate:pageSize":e[2]||(e[2]=c=>h.value=c),background:"",layout:"total, sizes, prev, pager, next, jumper","page-sizes":l.pageSizes,total:T.value.length},null,8,["current-page","page-size","page-sizes","total"])])]),_:3})])}}}),Te=ee(ce,[["__scopeId","data-v-e6e41965"]]);export{Te as D};
|
||||
import{E as M}from"./index-h5FkzAP_.js";import{E as Z}from"./index-BNsCyG4A.js";import{E as A}from"./el-card-D86TXgMv.js";import{E as G}from"./el-pagination-C9R_oTJ1.js";import{E as H,a as J}from"./el-table-C7Zx0Bmo.js";import{v as O}from"./directive-B1cwmVOz.js";import{E as b}from"./index-DqJTjrY3.js";import{d as Q,y as f,D as _,c as D,s as p,w as r,o as u,q as s,h as w,e as g,g as v,x as y,aa as k,Z as W,K as X,j as $,v as Y}from"./index-Ds9AETjS.js";import"./el-scrollbar-CF42LNzg.js";import"./el-popper-D1tByNLb.js";/* empty css */import"./el-select-BD70by95.js";import"./el-tooltip-l0sNRNKZ.js";import"./el-checkbox-pV3o_8iy.js";import{_ as ee}from"./_plugin-vue_export-helper-DlAUqK2U.js";const te={class:"data-table-page"},ae={class:"table-toolbar"},le={class:"table-title"},oe={class:"toolbar-actions"},se={key:0,class:"batch-bar"},ne={class:"batch-info"},re={class:"table-body"},ie={class:"table-pagination"},ce=Q({__name:"DataTablePage",props:{title:{},data:{},loading:{type:Boolean,default:!1},searchable:{type:Boolean,default:!1},searchFields:{},multiSelect:{type:Boolean,default:!1},createText:{},createTo:{},deleteFn:{},rowKey:{default:"id"},emptyText:{default:"暂无数据"},pageSize:{default:10},pageSizes:{default:()=>[10,20,50,100]}},emits:["create","refresh"],setup(l,{expose:F,emit:V}){const a=l,x=V,P=Y(),S=f(),o=f(new Set),d=f(""),m=f(1),h=f(a.pageSize);function z(t){return t[a.rowKey]??t.name??t.id}const T=$(()=>{var e;if(!d.value.trim()||!((e=a.searchFields)!=null&&e.length))return a.data;const t=d.value.toLowerCase().trim();return a.data.filter(i=>a.searchFields.some(n=>String(i[n]??"").toLowerCase().includes(t)))}),I=$(()=>{const t=(m.value-1)*h.value;return T.value.slice(t,t+h.value)});function K(t){o.value=new Set(t.map(z))}async function E(t,e="确定要删除这条记录吗?"){if(a.deleteFn)try{await b.confirm(e,"确认删除",{type:"warning",confirmButtonText:"确定",cancelButtonText:"取消"}),await a.deleteFn(t),x("refresh")}catch{}}async function N(){if(!(o.value.size===0||!a.deleteFn))try{await b.confirm(`确定要删除选中的 ${o.value.size} 条记录吗?`,"批量删除",{type:"warning"});const t=a.data.filter(n=>o.value.has(z(n)));let e=0,i=0;for(const n of t)try{await a.deleteFn(n),e++}catch{i++}o.value.clear(),x("refresh"),i===0?b.alert(`成功删除 ${e} 条记录`,"成功",{type:"success"}):b.alert(`成功 ${e} 条,失败 ${i} 条`,"部分失败",{type:"warning"})}catch{}}function C(){var t;o.value.clear(),(t=S.value)==null||t.clearSelection()}function L(){a.createTo?P.push(a.createTo):x("create")}return _(()=>a.data,()=>{var t;o.value.clear(),m.value=1,(t=S.value)==null||t.clearSelection()}),_(()=>d.value,()=>{m.value=1}),F({handleDelete:E,clearSelection:C}),(t,e)=>{const i=M,n=Z,B=H,R=J,U=G,j=A,q=O;return u(),D("div",te,[p(j,{shadow:"never",class:"table-card"},{default:r(()=>[s("div",ae,[w(t.$slots,"title",{},()=>[s("h2",le,k(l.title),1)],!0),s("div",oe,[l.searchable?(u(),g(i,{key:0,modelValue:d.value,"onUpdate:modelValue":e[0]||(e[0]=c=>d.value=c),placeholder:`搜索${l.title}...`,clearable:"",class:"search-input"},{prefix:r(()=>[...e[3]||(e[3]=[s("i",{class:"fa fa-search"},null,-1)])]),_:1},8,["modelValue","placeholder"])):v("",!0),l.createText?(u(),g(n,{key:1,type:"primary",onClick:L},{default:r(()=>[e[4]||(e[4]=s("i",{class:"fa fa-plus",style:{"margin-right":"4px"}},null,-1)),y(" "+k(l.createText),1)]),_:1})):v("",!0),w(t.$slots,"toolbar-extra",{},void 0,!0)])]),p(W,{name:"el-zoom-in-top"},{default:r(()=>[l.multiSelect&&o.value.size>0?(u(),D("div",se,[s("span",ne,[e[6]||(e[6]=y(" 已选择 ",-1)),s("strong",null,k(o.value.size),1),e[7]||(e[7]=y(" 项 ",-1)),p(n,{link:"",type:"primary",onClick:C},{default:r(()=>[...e[5]||(e[5]=[y("取消选择",-1)])]),_:1})]),p(n,{type:"danger",size:"small",onClick:N},{default:r(()=>[e[8]||(e[8]=s("i",{class:"fa fa-trash",style:{"margin-right":"4px"}},null,-1)),y(" 批量删除 ("+k(o.value.size)+") ",1)]),_:1})])):v("",!0)]),_:1}),s("div",re,[X((u(),g(R,{ref_key:"tableRef",ref:S,data:I.value,"row-key":l.rowKey,height:"100%",style:{width:"100%"},"empty-text":l.emptyText,onSelectionChange:K},{default:r(()=>[l.multiSelect?(u(),g(B,{key:0,type:"selection",width:"50",align:"center","reserve-selection":"",fixed:"left"})):v("",!0),w(t.$slots,"columns",{},void 0,!0),t.$slots.actions?(u(),g(B,{key:1,label:"操作",align:"center",width:"220",fixed:"right"},{default:r(({row:c})=>[w(t.$slots,"actions",{row:c,handleDelete:E},void 0,!0)]),_:3})):v("",!0)]),_:3},8,["data","row-key","empty-text"])),[[q,l.loading]])]),s("div",ie,[p(U,{"current-page":m.value,"onUpdate:currentPage":e[1]||(e[1]=c=>m.value=c),"page-size":h.value,"onUpdate:pageSize":e[2]||(e[2]=c=>h.value=c),background:"",layout:"total, sizes, prev, pager, next, jumper","page-sizes":l.pageSizes,total:T.value.length},null,8,["current-page","page-size","page-sizes","total"])])]),_:3})])}}}),Te=ee(ce,[["__scopeId","data-v-e6e41965"]]);export{Te as D};
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
import{E as z}from"./index-BjEW7-SA.js";import{E}from"./index-BDEF353-.js";import{E as T}from"./el-table-D8K5zLyz.js";import{d as L,G as A,e as b,w as e,y as u,j as V,o as k,q as r,s as a,x as s,g as $,aa as p,f as y,b1 as B,b2 as N,n as w,v as P,A as S}from"./index-BKKvzUDD.js";/* empty css */import"./el-checkbox-qGZ2-RM2.js";import{D as M}from"./DataTablePage-C-EM-7-s.js";import{g as R,d as G,a as I}from"./dataset-BJ2ikehQ.js";import{_ as j}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-ClJnvz-9.js";import"./index-GAnQrJsQ.js";import"./el-popper-D6_hxRbQ.js";import"./index-CWUnzf90.js";import"./_baseClone-B5RBzbh5.js";import"./_baseIteratee-BAilevJ_.js";import"./castArray-5uErZEc3.js";import"./debounce-ByXPNh5F.js";import"./toNumber-Dkj3QRv9.js";import"./raf-C-x62Pcl.js";import"./omit-BTSq4AYh.js";import"./index-TFUf94PZ.js";import"./index-DLzof2Fz.js";import"./el-card-CApHJ1Gj.js";import"./el-pagination-DOPYW-qb.js";import"./el-select-Dl-FRZQd.js";import"./vnode-78qeDweP.js";import"./index-D5ryD3I5.js";import"./scroll-BkzZKETR.js";import"./clamp-CbbY8h6F.js";import"./directive-DsZckUTJ.js";import"./index-dNo5H1ie.js";import"./validator-Fn0bm4xU.js";import"./index-CQXVR1Ud.js";import"./index-DfyOU94W.js";import"./el-tooltip-l0sNRNKZ.js";const q={class:"capsule-tabs"},O={class:"action-buttons"},U=L({__name:"DatasetListView",setup(Y){const C=P(),c=u(!1),m=u([]),n=u("upload"),D=V(()=>n.value==="task"?m.value.filter(l=>l.source==="task"):m.value.filter(l=>l.source!=="task"));async function f(){c.value=!0;try{m.value=await R()||[]}catch{}finally{c.value=!1}}async function _(l){await G(l.id),S.success("删除成功")}function h(l){C.push(`/dataset/${l.id}/preview`)}function x(l){window.open(I(l.id),"_blank")}return A(f),(l,i)=>{const o=T,g=E,d=z;return k(),b(M,{title:"数据集管理",data:D.value,loading:c.value,searchable:"","search-fields":["name","description"],"create-text":n.value==="upload"?"上传数据集":"","create-to":"/dataset/create","delete-fn":_,"row-key":"id",onRefresh:f},{title:e(()=>[r("div",q,[r("button",{class:w(["capsule-tab-item",{active:n.value==="upload"}]),onClick:i[0]||(i[0]=t=>n.value="upload")}," 上传任务 ",2),r("button",{class:w(["capsule-tab-item",{active:n.value==="task"}]),onClick:i[1]||(i[1]=t=>n.value="task")}," 数据任务 ",2)])]),columns:e(()=>[n.value==="task"?(k(),b(o,{key:0,label:"任务ID",prop:"task_id",align:"center",width:"100"})):$("",!0),a(o,{label:"数据集名称",prop:"name",align:"center"}),a(o,{label:"数据类型",align:"center",width:"110"},{default:e(({row:t})=>[a(g,{type:"primary",size:"small"},{default:e(()=>[s(p(y(B)[String(t.type).toLowerCase()]||t.type||"-"),1)]),_:2},1024)]),_:1}),a(o,{label:"存储位置",align:"center",width:"110"},{default:e(({row:t})=>[a(g,{type:"success",size:"small"},{default:e(()=>[s(p(y(N)[t.storage_type]||t.storage_type||"-"),1)]),_:2},1024)]),_:1}),a(o,{label:"大小",align:"center",width:"100"},{default:e(({row:t})=>[s(p(t.size&&t.size!=="0 B"&&t.size!=="0"?t.size:"-"),1)]),_:1}),a(o,{label:"数据条数",align:"center",width:"100"},{default:e(({row:t})=>[s(p(t.count||0),1)]),_:1}),a(o,{label:"描述",align:"center","show-overflow-tooltip":""},{default:e(({row:t})=>[s(p(t.description||"-"),1)]),_:1}),a(o,{label:"创建时间",align:"center",width:"180"},{default:e(({row:t})=>[s(p(t.create_time?new Date(t.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:e(({row:t})=>[r("div",O,[a(d,{type:"primary",link:"",size:"small",onClick:v=>h(t)},{default:e(()=>[...i[2]||(i[2]=[r("i",{class:"fa fa-eye",style:{"margin-right":"4px"}},null,-1),s("详情 ",-1)])]),_:1},8,["onClick"]),a(d,{type:"success",link:"",size:"small",onClick:v=>x(t)},{default:e(()=>[...i[3]||(i[3]=[r("i",{class:"fa fa-download",style:{"margin-right":"4px"}},null,-1),s("下载 ",-1)])]),_:1},8,["onClick"]),a(d,{type:"danger",link:"",size:"small",onClick:v=>_(t)},{default:e(()=>[...i[4]||(i[4]=[r("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),s("删除 ",-1)])]),_:1},8,["onClick"])])]),_:1},8,["data","loading","create-text"])}}}),Et=j(U,[["__scopeId","data-v-a2aaedec"]]);export{Et as default};
|
||||
1
frontend/dist/assets/DatasetListView-CF1w9JzC.js
vendored
Normal file
1
frontend/dist/assets/DatasetListView-CF1w9JzC.js
vendored
Normal file
@@ -0,0 +1 @@
|
||||
import{E as z}from"./index-BNsCyG4A.js";import{E}from"./index-ByhS8A1d.js";import{E as T}from"./el-table-C7Zx0Bmo.js";import{d as L,G as A,e as k,w as e,y as f,j as V,o as y,q as r,s as l,x as s,g as $,aa as p,f as b,aY as B,aZ as N,n as w,v as P,A as S}from"./index-Ds9AETjS.js";/* empty css */import"./el-checkbox-pV3o_8iy.js";import{D as M}from"./DataTablePage-DzEUb2tf.js";import{g as R,d as G,a as I}from"./dataset-BWM6p-mU.js";import{_ as Y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-CF42LNzg.js";import"./index-BSbCLbaz.js";import"./el-popper-D1tByNLb.js";import"./index-sWKLi7bH.js";import"./_baseClone-CZxxnGCv.js";import"./_baseIteratee-82gB5Yov.js";import"./castArray-BWI4UxBy.js";import"./debounce-D6mJVDYC.js";import"./toNumber-DrayciwT.js";import"./raf-zQ00nuMI.js";import"./omit-BzI1MV_K.js";import"./index-h5FkzAP_.js";import"./index-B0if-AHo.js";import"./el-card-D86TXgMv.js";import"./el-pagination-C9R_oTJ1.js";import"./el-select-BD70by95.js";import"./vnode-CBiWHFw7.js";import"./index-CQlF36Lk.js";import"./scroll-DVmMv0d9.js";import"./clamp-CyhADbdq.js";import"./directive-B1cwmVOz.js";import"./index-DqJTjrY3.js";import"./validator-D1MwuZXj.js";import"./index-jBnYrPdD.js";import"./index-RtwaMutE.js";import"./el-tooltip-l0sNRNKZ.js";const j={class:"capsule-tabs"},q={class:"action-buttons"},O=L({__name:"DatasetListView",setup(U){const C=P(),m=f(!1),c=f([]),n=f("upload"),D=V(()=>n.value==="task"?c.value.filter(i=>i.source==="task"):c.value.filter(i=>i.source!=="task"));async function d(){m.value=!0;try{c.value=await R()||[]}catch{}finally{m.value=!1}}async function _(i){await G(i.id),c.value=c.value.filter(a=>a.id!==i.id),await d(),S.success("删除成功")}function h(i){C.push(`/dataset/${i.id}/preview`)}function x(i){window.open(I(i.id),"_blank")}return A(d),(i,a)=>{const o=T,v=E,u=z;return y(),k(M,{title:"数据集管理",data:D.value,loading:m.value,searchable:"","search-fields":["name","description"],"create-text":n.value==="upload"?"上传数据集":"","create-to":"/dataset/create","delete-fn":_,"row-key":"id",onRefresh:d},{title:e(()=>[r("div",j,[r("button",{class:w(["capsule-tab-item",{active:n.value==="upload"}]),onClick:a[0]||(a[0]=t=>n.value="upload")}," 上传任务 ",2),r("button",{class:w(["capsule-tab-item",{active:n.value==="task"}]),onClick:a[1]||(a[1]=t=>n.value="task")}," 数据任务 ",2)])]),columns:e(()=>[n.value==="task"?(y(),k(o,{key:0,label:"任务ID",prop:"task_id",align:"center",width:"100"})):$("",!0),l(o,{label:"数据集名称",prop:"name",align:"center"}),l(o,{label:"数据类型",align:"center",width:"110"},{default:e(({row:t})=>[l(v,{type:"primary",size:"small"},{default:e(()=>[s(p(b(B)[String(t.type).toLowerCase()]||t.type||"-"),1)]),_:2},1024)]),_:1}),l(o,{label:"存储位置",align:"center",width:"110"},{default:e(({row:t})=>[l(v,{type:"success",size:"small"},{default:e(()=>[s(p(b(N)[t.storage_type]||t.storage_type||"-"),1)]),_:2},1024)]),_:1}),l(o,{label:"大小",align:"center",width:"100"},{default:e(({row:t})=>[s(p(t.size&&t.size!=="0 B"&&t.size!=="0"?t.size:"-"),1)]),_:1}),l(o,{label:"数据条数",align:"center",width:"100"},{default:e(({row:t})=>[s(p(t.count||0),1)]),_:1}),l(o,{label:"描述",align:"center","show-overflow-tooltip":""},{default:e(({row:t})=>[s(p(t.description||"-"),1)]),_:1}),l(o,{label:"创建时间",align:"center",width:"180"},{default:e(({row:t})=>[s(p(t.create_time?new Date(t.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:e(({row:t})=>[r("div",q,[l(u,{type:"primary",link:"",size:"small",onClick:g=>h(t)},{default:e(()=>[...a[2]||(a[2]=[r("i",{class:"fa fa-eye",style:{"margin-right":"4px"}},null,-1),s("详情 ",-1)])]),_:1},8,["onClick"]),l(u,{type:"success",link:"",size:"small",onClick:g=>x(t)},{default:e(()=>[...a[3]||(a[3]=[r("i",{class:"fa fa-download",style:{"margin-right":"4px"}},null,-1),s("下载 ",-1)])]),_:1},8,["onClick"]),l(u,{type:"danger",link:"",size:"small",onClick:g=>_(t)},{default:e(()=>[...a[4]||(a[4]=[r("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),s("删除 ",-1)])]),_:1},8,["onClick"])])]),_:1},8,["data","loading","create-text"])}}}),Et=Y(O,[["__scopeId","data-v-e406ac6d"]]);export{Et as default};
|
||||
@@ -1 +1 @@
|
||||
@charset "UTF-8";.capsule-tabs[data-v-a2aaedec]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-a2aaedec]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-a2aaedec]:hover{color:#1e293b}.capsule-tab-item.active[data-v-a2aaedec]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
@charset "UTF-8";.capsule-tabs[data-v-e406ac6d]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-e406ac6d]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-e406ac6d]:hover{color:#1e293b}.capsule-tab-item.active[data-v-e406ac6d]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{a as k,E as M}from"./el-form-item-D5kF3B90.js";import{E as q}from"./index-TFUf94PZ.js";import{E as R}from"./index-BjEW7-SA.js";import{d as S,G as B,e as F,w as i,j as y,y as _,z as I,o as O,s as r,f as m,aB as P,x as h,aa as j,A as x,v as A,ac as L}from"./index-BKKvzUDD.js";import{P as N}from"./PageCard-BoKXOzst.js";import{E as T,D as U}from"./DimensionFormFields-Bhy2Lp-a.js";import{b as z,u as G,c as H}from"./eval-D3O1S_j5.js";import{g as J}from"./model-gCK34aBo.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./el-card-CApHJ1Gj.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-select-Dl-FRZQd.js";import"./index-CWUnzf90.js";import"./vnode-78qeDweP.js";import"./el-popper-D6_hxRbQ.js";import"./el-scrollbar-ClJnvz-9.js";import"./index-D5ryD3I5.js";import"./index-BDEF353-.js";import"./scroll-BkzZKETR.js";import"./clamp-CbbY8h6F.js";import"./toNumber-Dkj3QRv9.js";import"./_baseIteratee-BAilevJ_.js";import"./el-checkbox-qGZ2-RM2.js";import"./omit-BTSq4AYh.js";import"./el-radio-group-BkCoTeQL.js";import"./el-input-number-BFR4pu1i.js";import"./index-BENk7lZo.js";import"./el-slider-DYENF1-i.js";import"./debounce-ByXPNh5F.js";import"./el-switch-DbCDiaMc.js";import"./validator-Fn0bm4xU.js";import"./el-tooltip-l0sNRNKZ.js";/* empty css *//* empty css */const Be=S({__name:"DimensionCreateView",setup(K){const c=L(),f=A(),n=_(),p=_(!1),u=y(()=>!!c.params.id),s=y(()=>c.params.id),d=_([]);let e=I({name:"",type:"",description:"",eval_model:"",eval_method:"",eval_prompt:"",is_active:!0,is_default:!1,bleu_n:1,output_precision:3,score_min:0,score_max:5,pass_threshold:3});const b={name:[{required:!0,message:"请输入维度名称",trigger:"blur"},{max:50,message:"不超过 50 字符",trigger:"blur"}],type:[{required:!0,message:"请选择指标类型",trigger:"change"}],eval_model:[{required:!0,message:"请选择大模型",trigger:"change"}],eval_method:[{required:!0,message:"请选择评估方式",trigger:"change"}],eval_prompt:[{required:!0,message:"请填写评估 Prompt",trigger:"blur"}]};async function w(){var o,t;if(s.value)try{const a=await z(s.value);Object.assign(e,{name:a.name||"",type:a.type||"",description:a.description||"",eval_model:a.eval_model||"",eval_method:a.eval_method||(a.type?(t=(o=T[a.type])==null?void 0:o[0])==null?void 0:t.value:""),eval_prompt:a.eval_prompt||"",is_active:a.is_active!==!1,is_default:!!a.is_default,bleu_n:a.bleu_n??1,output_precision:a.output_precision??3,score_min:a.score_min??0,score_max:a.score_max??5,pass_threshold:a.pass_threshold??3})}catch{}}async function E(){try{const o=await J()||[];d.value=o.filter(t=>t.purpose==="evaluation")}catch{d.value=[]}}async function D(){n.value&&await n.value.validate(async o=>{if(o){p.value=!0;try{const t={name:e.name,type:e.type,description:e.description,eval_model:e.type==="text_similarity"?null:e.eval_model,eval_method:e.eval_method,eval_prompt:e.type==="text_similarity"?null:e.eval_prompt,is_active:e.is_active,is_default:e.is_default,create_time:new Date().toISOString()};e.type==="text_similarity"&&(t.bleu_n=e.bleu_n,t.output_precision=e.output_precision),e.type==="metric"&&(t.score_min=e.score_min,t.score_max=e.score_max,t.pass_threshold=e.pass_threshold),u.value&&s.value?(await G(s.value,t),x.success("更新成功")):(await H(t),x.success("创建成功")),f.push("/model-eval")}catch{}finally{p.value=!1}}})}function V(){f.back()}return B(()=>{E(),w()}),(o,t)=>{const a=q,v=k,g=R,C=M;return O(),F(N,{title:u.value?"编辑评测维度":"添加评测维度"},{default:i(()=>[r(C,{ref_key:"formRef",ref:n,model:m(e),rules:b,"label-width":"130px",style:{"max-width":"760px"}},{default:i(()=>[r(v,{label:"维度名称",prop:"name"},{default:i(()=>[r(a,{modelValue:m(e).name,"onUpdate:modelValue":t[0]||(t[0]=l=>m(e).name=l),placeholder:"请输入维度名称",maxlength:"50","show-word-limit":""},null,8,["modelValue"])]),_:1}),r(U,{modelValue:m(e),"onUpdate:modelValue":t[1]||(t[1]=l=>P(e)?e.value=l:e=l),"eval-models":d.value},null,8,["modelValue","eval-models"]),r(v,null,{default:i(()=>[r(g,{type:"primary",loading:p.value,onClick:D},{default:i(()=>[h(j(u.value?"保存":"创建"),1)]),_:1},8,["loading"]),r(g,{onClick:V},{default:i(()=>[...t[2]||(t[2]=[h("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1},8,["title"])}}});export{Be as default};
|
||||
import{a as k,E as M}from"./el-form-item-Bq0n_p5a.js";import{E as q}from"./index-h5FkzAP_.js";import{E as R}from"./index-BNsCyG4A.js";import{d as S,G as B,e as F,w as i,j as y,y as _,z as I,o as O,s as r,f as m,aB as P,x as h,aa as j,A as x,v as A,ac as L}from"./index-Ds9AETjS.js";import{P as N}from"./PageCard-CV3p14-x.js";import{E as T,D as U}from"./DimensionFormFields-Bvlt4Km3.js";import{b as z,u as G,c as H}from"./eval-RqHG_SzR.js";import{g as J}from"./model-cWd9CDtT.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./el-card-D86TXgMv.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-select-BD70by95.js";import"./index-sWKLi7bH.js";import"./vnode-CBiWHFw7.js";import"./el-popper-D1tByNLb.js";import"./el-scrollbar-CF42LNzg.js";import"./index-CQlF36Lk.js";import"./index-ByhS8A1d.js";import"./scroll-DVmMv0d9.js";import"./clamp-CyhADbdq.js";import"./toNumber-DrayciwT.js";import"./_baseIteratee-82gB5Yov.js";import"./el-checkbox-pV3o_8iy.js";import"./omit-BzI1MV_K.js";import"./el-radio-group-DxV9SE4w.js";import"./el-input-number-Bg2C0S8w.js";import"./index-2LOSO4Pw.js";import"./el-slider-mZJHOhL2.js";import"./debounce-D6mJVDYC.js";import"./el-switch-DflKYtBf.js";import"./validator-D1MwuZXj.js";import"./el-tooltip-l0sNRNKZ.js";/* empty css *//* empty css */const Be=S({__name:"DimensionCreateView",setup(K){const c=L(),f=A(),n=_(),p=_(!1),u=y(()=>!!c.params.id),s=y(()=>c.params.id),d=_([]);let e=I({name:"",type:"",description:"",eval_model:"",eval_method:"",eval_prompt:"",is_active:!0,is_default:!1,bleu_n:1,output_precision:3,score_min:0,score_max:5,pass_threshold:3});const b={name:[{required:!0,message:"请输入维度名称",trigger:"blur"},{max:50,message:"不超过 50 字符",trigger:"blur"}],type:[{required:!0,message:"请选择指标类型",trigger:"change"}],eval_model:[{required:!0,message:"请选择大模型",trigger:"change"}],eval_method:[{required:!0,message:"请选择评估方式",trigger:"change"}],eval_prompt:[{required:!0,message:"请填写评估 Prompt",trigger:"blur"}]};async function w(){var o,t;if(s.value)try{const a=await z(s.value);Object.assign(e,{name:a.name||"",type:a.type||"",description:a.description||"",eval_model:a.eval_model||"",eval_method:a.eval_method||(a.type?(t=(o=T[a.type])==null?void 0:o[0])==null?void 0:t.value:""),eval_prompt:a.eval_prompt||"",is_active:a.is_active!==!1,is_default:!!a.is_default,bleu_n:a.bleu_n??1,output_precision:a.output_precision??3,score_min:a.score_min??0,score_max:a.score_max??5,pass_threshold:a.pass_threshold??3})}catch{}}async function E(){try{const o=await J()||[];d.value=o.filter(t=>t.purpose==="evaluation")}catch{d.value=[]}}async function D(){n.value&&await n.value.validate(async o=>{if(o){p.value=!0;try{const t={name:e.name,type:e.type,description:e.description,eval_model:e.type==="text_similarity"?null:e.eval_model,eval_method:e.eval_method,eval_prompt:e.type==="text_similarity"?null:e.eval_prompt,is_active:e.is_active,is_default:e.is_default,create_time:new Date().toISOString()};e.type==="text_similarity"&&(t.bleu_n=e.bleu_n,t.output_precision=e.output_precision),e.type==="metric"&&(t.score_min=e.score_min,t.score_max=e.score_max,t.pass_threshold=e.pass_threshold),u.value&&s.value?(await G(s.value,t),x.success("更新成功")):(await H(t),x.success("创建成功")),f.push("/model-eval")}catch{}finally{p.value=!1}}})}function V(){f.back()}return B(()=>{E(),w()}),(o,t)=>{const a=q,v=k,g=R,C=M;return O(),F(N,{title:u.value?"编辑评测维度":"添加评测维度"},{default:i(()=>[r(C,{ref_key:"formRef",ref:n,model:m(e),rules:b,"label-width":"130px",style:{"max-width":"760px"}},{default:i(()=>[r(v,{label:"维度名称",prop:"name"},{default:i(()=>[r(a,{modelValue:m(e).name,"onUpdate:modelValue":t[0]||(t[0]=l=>m(e).name=l),placeholder:"请输入维度名称",maxlength:"50","show-word-limit":""},null,8,["modelValue"])]),_:1}),r(U,{modelValue:m(e),"onUpdate:modelValue":t[1]||(t[1]=l=>P(e)?e.value=l:e=l),"eval-models":d.value},null,8,["modelValue","eval-models"]),r(v,null,{default:i(()=>[r(g,{type:"primary",loading:p.value,onClick:D},{default:i(()=>[h(j(u.value?"保存":"创建"),1)]),_:1},8,["loading"]),r(g,{onClick:V},{default:i(()=>[...t[2]||(t[2]=[h("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1},8,["title"])}}});export{Be as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/EvalView--2TVFV5G.css
vendored
Normal file
1
frontend/dist/assets/EvalView--2TVFV5G.css
vendored
Normal file
@@ -0,0 +1 @@
|
||||
@charset "UTF-8";.eval-page[data-v-78ab2fa4]{width:100%;height:100%;display:flex;flex-direction:column;min-height:0}.capsule-tabs[data-v-78ab2fa4]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-78ab2fa4]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-78ab2fa4]:hover{color:#1e293b}.capsule-tab-item.active[data-v-78ab2fa4]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
1
frontend/dist/assets/EvalView-CfhhxItX.css
vendored
1
frontend/dist/assets/EvalView-CfhhxItX.css
vendored
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";.eval-page[data-v-8012e1f7]{width:100%;height:100%;display:flex;flex-direction:column;min-height:0}.capsule-tabs[data-v-8012e1f7]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-8012e1f7]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-8012e1f7]:hover{color:#1e293b}.capsule-tab-item.active[data-v-8012e1f7]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}
|
||||
1
frontend/dist/assets/EvalView-Eldo4kFi.js
vendored
Normal file
1
frontend/dist/assets/EvalView-Eldo4kFi.js
vendored
Normal file
@@ -0,0 +1 @@
|
||||
import{E as V}from"./index-BNsCyG4A.js";import{E as B}from"./el-table-C7Zx0Bmo.js";import{d as D,G as T,c as $,e as _,w as o,g as C,y as m,o as v,s as a,q as i,x as b,aa as L,n as c,v as N,A as z}from"./index-Ds9AETjS.js";import{E as M}from"./index-DqJTjrY3.js";/* empty css */import"./el-checkbox-pV3o_8iy.js";import{D as y}from"./DataTablePage-DzEUb2tf.js";import{g as A,d as G}from"./eval-RqHG_SzR.js";import{_ as P}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-CF42LNzg.js";import"./index-BSbCLbaz.js";import"./el-popper-D1tByNLb.js";import"./index-sWKLi7bH.js";import"./_baseClone-CZxxnGCv.js";import"./_baseIteratee-82gB5Yov.js";import"./castArray-BWI4UxBy.js";import"./debounce-D6mJVDYC.js";import"./toNumber-DrayciwT.js";import"./raf-zQ00nuMI.js";import"./validator-D1MwuZXj.js";import"./index-jBnYrPdD.js";import"./clamp-CyhADbdq.js";import"./index-RtwaMutE.js";import"./scroll-DVmMv0d9.js";import"./index-h5FkzAP_.js";import"./index-B0if-AHo.js";import"./omit-BzI1MV_K.js";import"./el-card-D86TXgMv.js";import"./el-pagination-C9R_oTJ1.js";import"./el-select-BD70by95.js";import"./vnode-CBiWHFw7.js";import"./index-CQlF36Lk.js";import"./index-ByhS8A1d.js";import"./directive-B1cwmVOz.js";import"./el-tooltip-l0sNRNKZ.js";const R={class:"eval-page"},S={class:"capsule-tabs"},q={class:"capsule-tabs"},I=D({__name:"EvalView",setup(Q){const f=N(),r=m("tasks"),d=m(!1),n=m([]),w=m([{rank:1,name:"GPT-4",score:92.5},{rank:2,name:"Claude-3",score:90.1},{rank:3,name:"Qwen-Max",score:85.3}]);async function u(){d.value=!0;try{n.value=await A()||[]}catch{n.value=[]}finally{d.value=!1}}async function k(s){await M.confirm("确定要删除这个评测任务吗?","确认删除",{type:"warning"}),await G(s.id),n.value=n.value.filter(e=>e.id!==s.id),z.success("删除成功"),await u()}function E(){f.push("/model-eval/create")}function p(s){return r.value===s}function x(s){f.push({name:"model-eval-detail",params:{id:s.id}})}return T(()=>{u()}),(s,e)=>{const t=B,g=V;return v(),$("div",R,[r.value==="tasks"?(v(),_(y,{key:0,title:"",data:n.value,loading:d.value,"delete-fn":k,"create-text":"创建评测任务",onCreate:E,"row-key":"id",onRefresh:u},{title:o(()=>[i("div",S,[i("button",{class:c(["capsule-tab-item",{active:p("tasks")}]),onClick:e[0]||(e[0]=l=>r.value="tasks")},"评测任务",2),i("button",{class:c(["capsule-tab-item",{active:p("leaderboard")}]),onClick:e[1]||(e[1]=l=>r.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(t,{label:"任务名称",prop:"eval_task_name",align:"center"}),a(t,{label:"评测模型",prop:"model_name",align:"center"}),a(t,{label:"数据集",prop:"dataset",align:"center"}),a(t,{label:"指标",prop:"metric",align:"center"}),a(t,{label:"评分",prop:"score",width:"100",align:"center"}),a(t,{label:"状态",prop:"status",width:"100",align:"center"}),a(t,{label:"创建时间",align:"center",width:"180"},{default:o(({row:l})=>[b(L(l.create_time?new Date(l.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:o(({row:l})=>[a(g,{link:"",type:"primary",size:"small",onClick:h=>x(l)},{default:o(()=>[...e[4]||(e[4]=[i("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),b("详情 ",-1)])]),_:1},8,["onClick"]),a(g,{link:"",type:"danger",size:"small",onClick:h=>k(l)},{default:o(()=>[...e[5]||(e[5]=[i("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),b("删除 ",-1)])]),_:1},8,["onClick"])]),_:1},8,["data","loading"])):C("",!0),r.value==="leaderboard"?(v(),_(y,{key:1,title:"",data:w.value,"row-key":"rank"},{title:o(()=>[i("div",q,[i("button",{class:c(["capsule-tab-item",{active:p("tasks")}]),onClick:e[2]||(e[2]=l=>r.value="tasks")},"评测任务",2),i("button",{class:c(["capsule-tab-item",{active:p("leaderboard")}]),onClick:e[3]||(e[3]=l=>r.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(t,{label:"排名",prop:"rank",width:"80",align:"center"}),a(t,{label:"模型名称",prop:"name",align:"center"}),a(t,{label:"综合评分",prop:"score",align:"center"})]),_:1},8,["data"])):C("",!0)])}}}),xe=P(I,[["__scopeId","data-v-78ab2fa4"]]);export{xe as default};
|
||||
1
frontend/dist/assets/EvalView-EyPQOCUX.js
vendored
1
frontend/dist/assets/EvalView-EyPQOCUX.js
vendored
@@ -1 +0,0 @@
|
||||
import{E as V}from"./index-BjEW7-SA.js";import{E as B}from"./el-table-D8K5zLyz.js";import{d as D,G as T,c as $,e as _,w as o,g as C,y as p,o as v,s as a,q as i,x as b,aa as L,n as m,v as N,A as z}from"./index-BKKvzUDD.js";import{E as M}from"./index-dNo5H1ie.js";/* empty css */import"./el-checkbox-qGZ2-RM2.js";import{D as y}from"./DataTablePage-C-EM-7-s.js";import{g as A,d as G}from"./eval-D3O1S_j5.js";import{_ as P}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./el-scrollbar-ClJnvz-9.js";import"./index-GAnQrJsQ.js";import"./el-popper-D6_hxRbQ.js";import"./index-CWUnzf90.js";import"./_baseClone-B5RBzbh5.js";import"./_baseIteratee-BAilevJ_.js";import"./castArray-5uErZEc3.js";import"./debounce-ByXPNh5F.js";import"./toNumber-Dkj3QRv9.js";import"./raf-C-x62Pcl.js";import"./validator-Fn0bm4xU.js";import"./index-CQXVR1Ud.js";import"./clamp-CbbY8h6F.js";import"./index-DfyOU94W.js";import"./scroll-BkzZKETR.js";import"./index-TFUf94PZ.js";import"./index-DLzof2Fz.js";import"./omit-BTSq4AYh.js";import"./el-card-CApHJ1Gj.js";import"./el-pagination-DOPYW-qb.js";import"./el-select-Dl-FRZQd.js";import"./vnode-78qeDweP.js";import"./index-D5ryD3I5.js";import"./index-BDEF353-.js";import"./directive-DsZckUTJ.js";import"./el-tooltip-l0sNRNKZ.js";const R={class:"eval-page"},S={class:"capsule-tabs"},q={class:"capsule-tabs"},I=D({__name:"EvalView",setup(Q){const f=N(),r=p("tasks"),c=p(!1),d=p([]),w=p([{rank:1,name:"GPT-4",score:92.5},{rank:2,name:"Claude-3",score:90.1},{rank:3,name:"Qwen-Max",score:85.3}]);async function u(){c.value=!0;try{d.value=await A()||[]}catch{d.value=[]}finally{c.value=!1}}async function k(s){await M.confirm("确定要删除这个评测任务吗?","确认删除",{type:"warning"}),await G(s.id),z.success("删除成功"),u()}function E(){f.push("/model-eval/create")}function n(s){return r.value===s}function x(s){f.push({name:"model-eval-detail",params:{id:s.id}})}return T(()=>{u()}),(s,e)=>{const t=B,g=V;return v(),$("div",R,[r.value==="tasks"?(v(),_(y,{key:0,title:"",data:d.value,loading:c.value,"delete-fn":k,"create-text":"创建评测任务",onCreate:E,"row-key":"id",onRefresh:u},{title:o(()=>[i("div",S,[i("button",{class:m(["capsule-tab-item",{active:n("tasks")}]),onClick:e[0]||(e[0]=l=>r.value="tasks")},"评测任务",2),i("button",{class:m(["capsule-tab-item",{active:n("leaderboard")}]),onClick:e[1]||(e[1]=l=>r.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(t,{label:"任务名称",prop:"eval_task_name",align:"center"}),a(t,{label:"评测模型",prop:"model_name",align:"center"}),a(t,{label:"数据集",prop:"dataset",align:"center"}),a(t,{label:"指标",prop:"metric",align:"center"}),a(t,{label:"评分",prop:"score",width:"100",align:"center"}),a(t,{label:"状态",prop:"status",width:"100",align:"center"}),a(t,{label:"创建时间",align:"center",width:"180"},{default:o(({row:l})=>[b(L(l.create_time?new Date(l.create_time).toLocaleString("zh-CN"):"-"),1)]),_:1})]),actions:o(({row:l})=>[a(g,{link:"",type:"primary",size:"small",onClick:h=>x(l)},{default:o(()=>[...e[4]||(e[4]=[i("i",{class:"fa fa-file-text-o",style:{"margin-right":"4px"}},null,-1),b("详情 ",-1)])]),_:1},8,["onClick"]),a(g,{link:"",type:"danger",size:"small",onClick:h=>k(l)},{default:o(()=>[...e[5]||(e[5]=[i("i",{class:"fa fa-trash-o",style:{"margin-right":"4px"}},null,-1),b("删除 ",-1)])]),_:1},8,["onClick"])]),_:1},8,["data","loading"])):C("",!0),r.value==="leaderboard"?(v(),_(y,{key:1,title:"",data:w.value,"row-key":"rank"},{title:o(()=>[i("div",q,[i("button",{class:m(["capsule-tab-item",{active:n("tasks")}]),onClick:e[2]||(e[2]=l=>r.value="tasks")},"评测任务",2),i("button",{class:m(["capsule-tab-item",{active:n("leaderboard")}]),onClick:e[3]||(e[3]=l=>r.value="leaderboard")},"排行榜",2)])]),columns:o(()=>[a(t,{label:"排名",prop:"rank",width:"80",align:"center"}),a(t,{label:"模型名称",prop:"name",align:"center"}),a(t,{label:"综合评分",prop:"score",align:"center"})]),_:1},8,["data"])):C("",!0)])}}}),xe=P(I,[["__scopeId","data-v-8012e1f7"]]);export{xe as default};
|
||||
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
.el-popover{--el-popover-bg-color:var(--el-bg-color-overlay);--el-popover-font-size:var(--el-font-size-base);--el-popover-border-color:var(--el-border-color-lighter);--el-popover-padding:12px;--el-popover-padding-large:18px 20px;--el-popover-title-font-size:16px;--el-popover-title-text-color:var(--el-text-color-primary);--el-popover-border-radius:4px}.el-popover.el-popper{background:var(--el-popover-bg-color);border-radius:var(--el-popover-border-radius);border:1px solid var(--el-popover-border-color);min-width:150px;padding:var(--el-popover-padding);z-index:var(--el-index-popper);color:var(--el-text-color-regular);line-height:1.4;font-size:var(--el-popover-font-size);box-shadow:var(--el-box-shadow-light);overflow-wrap:break-word;box-sizing:border-box}.el-popover.el-popper--plain{padding:var(--el-popover-padding-large)}.el-popover__title{color:var(--el-popover-title-text-color);font-size:var(--el-popover-title-font-size);margin-bottom:12px;line-height:1}.el-popover__reference:focus:not(.focusing),.el-popover__reference:focus:hover{outline-width:0}.el-popover.el-popper.is-dark{--el-popover-bg-color:var(--el-text-color-primary);--el-popover-border-color:var(--el-text-color-primary);--el-popover-title-text-color:var(--el-bg-color);color:var(--el-bg-color)}.el-popover.el-popper:focus:active,.el-popover.el-popper:focus{outline-width:0}.progress-value[data-v-5c76183f]{color:var(--primary-color);font-weight:600}.filter-header[data-v-5c76183f]{display:inline-flex;align-items:center;gap:6px}.filter-badge[data-v-5c76183f]{line-height:1}.filter-icon[data-v-5c76183f]{cursor:pointer;font-size:12px;color:#c0c4cc;transition:color .2s}.filter-icon[data-v-5c76183f]:hover,.filter-icon.active[data-v-5c76183f]{color:#1890ff}.filter-options[data-v-5c76183f]{display:flex;flex-direction:column;gap:8px;max-height:240px;overflow-y:auto}.filter-actions[data-v-5c76183f]{text-align:right;margin-top:8px;border-top:1px solid #ebeef5;padding-top:8px}
|
||||
.el-popover{--el-popover-bg-color:var(--el-bg-color-overlay);--el-popover-font-size:var(--el-font-size-base);--el-popover-border-color:var(--el-border-color-lighter);--el-popover-padding:12px;--el-popover-padding-large:18px 20px;--el-popover-title-font-size:16px;--el-popover-title-text-color:var(--el-text-color-primary);--el-popover-border-radius:4px}.el-popover.el-popper{background:var(--el-popover-bg-color);border-radius:var(--el-popover-border-radius);border:1px solid var(--el-popover-border-color);min-width:150px;padding:var(--el-popover-padding);z-index:var(--el-index-popper);color:var(--el-text-color-regular);line-height:1.4;font-size:var(--el-popover-font-size);box-shadow:var(--el-box-shadow-light);overflow-wrap:break-word;box-sizing:border-box}.el-popover.el-popper--plain{padding:var(--el-popover-padding-large)}.el-popover__title{color:var(--el-popover-title-text-color);font-size:var(--el-popover-title-font-size);margin-bottom:12px;line-height:1}.el-popover__reference:focus:not(.focusing),.el-popover__reference:focus:hover{outline-width:0}.el-popover.el-popper.is-dark{--el-popover-bg-color:var(--el-text-color-primary);--el-popover-border-color:var(--el-text-color-primary);--el-popover-title-text-color:var(--el-bg-color);color:var(--el-bg-color)}.el-popover.el-popper:focus:active,.el-popover.el-popper:focus{outline-width:0}.progress-value[data-v-d1d743ee]{color:var(--primary-color);font-weight:600}.filter-header[data-v-d1d743ee]{display:inline-flex;align-items:center;gap:6px}.filter-badge[data-v-d1d743ee]{line-height:1}.filter-icon[data-v-d1d743ee]{cursor:pointer;font-size:12px;color:#c0c4cc;transition:color .2s}.filter-icon[data-v-d1d743ee]:hover,.filter-icon.active[data-v-d1d743ee]{color:#1890ff}.filter-options[data-v-d1d743ee]{display:flex;flex-direction:column;gap:8px;max-height:240px;overflow-y:auto}.filter-actions[data-v-d1d743ee]{text-align:right;margin-top:8px;border-top:1px solid #ebeef5;padding-top:8px}
|
||||
1
frontend/dist/assets/FineTuneListView-CuiRAa28.js
vendored
Normal file
1
frontend/dist/assets/FineTuneListView-CuiRAa28.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{_ as s}from"./_plugin-vue_export-helper-DlAUqK2U.js";import{o as t,c,q as o}from"./index-BKKvzUDD.js";const n={},r={class:"simple-page"};function a(_,e){return t(),c("section",r,[...e[0]||(e[0]=[o("h1",null,"使用文档",-1),o("p",null,"第一版系统已接入后端、数据集、模型、微调任务、算力节点和训练日志主链路。",-1)])])}const p=s(n,[["render",a],["__scopeId","data-v-87551c13"]]);export{p as default};
|
||||
import{_ as s}from"./_plugin-vue_export-helper-DlAUqK2U.js";import{o as t,c,q as o}from"./index-Ds9AETjS.js";const n={},r={class:"simple-page"};function a(_,e){return t(),c("section",r,[...e[0]||(e[0]=[o("h1",null,"使用文档",-1),o("p",null,"第一版系统已接入后端、数据集、模型、微调任务、算力节点和训练日志主链路。",-1)])])}const p=s(n,[["render",a],["__scopeId","data-v-87551c13"]]);export{p as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{E as T}from"./el-alert-xVLeNUgJ.js";import{a as A,E as D}from"./el-form-item-D5kF3B90.js";import{E as L}from"./index-TFUf94PZ.js";import{E as j}from"./index-BjEW7-SA.js";import{E as z,b as H,a as J}from"./el-select-Dl-FRZQd.js";import{E as K}from"./el-divider-DjByQoml.js";import{d as Q,G as W,e as u,w as o,at as X,y as i,z as Y,o as s,s as a,x as b,c as v,ad as k,M as w,g as Z,j as p,A as B,v as ee}from"./index-BKKvzUDD.js";import"./el-scrollbar-ClJnvz-9.js";import"./el-popper-D6_hxRbQ.js";/* empty css */import{P as te}from"./PageCard-BoKXOzst.js";import{g as le,a as ae}from"./model-gCK34aBo.js";import"./vnode-78qeDweP.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./index-CWUnzf90.js";import"./index-D5ryD3I5.js";import"./index-BDEF353-.js";import"./scroll-BkzZKETR.js";import"./clamp-CbbY8h6F.js";import"./toNumber-Dkj3QRv9.js";import"./_baseIteratee-BAilevJ_.js";import"./el-card-CApHJ1Gj.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";const Ie=Q({__name:"InferenceCreateView",setup(oe){const E=ee(),f=i(),c=i(!1),m=i(""),V=i([]),h=i([]),_=i([]),x=p(()=>V.value.map(t=>({key:`db-${t.id}`,id:t.id,name:t.name,source:"database",model_path:t.path||""}))),M=p(()=>h.value.map(t=>({key:`trained-${t.id}`,id:t.id,name:t.name,source:"trained",model_path:t.merged_path||t.base_model_path||"",merged:t.merged,merging:t.merging,disabled:t.merged===!1}))),G=p(()=>{const t={};for(const e of[...x.value,...M.value])t[e.key]=e;return t}),n=Y({name:"",description:"",model_key:"",gpu_id:0}),I={name:[{required:!0,message:"请输入推理名称",trigger:"blur"}],model_key:[{required:!0,message:"请选择模型",trigger:"change"}]},O=p(()=>G.value[n.model_key]);async function S(){f.value&&await f.value.validate(async t=>{if(!t)return;const e=O.value;if(!e){B.warning("请选择模型");return}c.value=!0,m.value="正在启动模型服务...";try{await new Promise(r=>setTimeout(r,1200)),B.success("模型已启动"),E.push({path:"/model-inference/chat/mock",query:{model:e.name}})}finally{c.value=!1,m.value=""}})}function $(){E.back()}async function q(){try{const[t,e,r]=await Promise.all([le(),ae(),X()]);V.value=t||[],h.value=(e==null?void 0:e.models)||[],_.value=(r==null?void 0:r.gpu)||[],_.value.length>0&&(n.gpu_id=0)}catch{}}return W(q),(t,e)=>{const r=L,d=A,F=K,g=J,C=H,P=z,N=T,U=j,R=D;return s(),u(te,{title:"新建推理"},{default:o(()=>[a(R,{ref_key:"formRef",ref:f,model:n,rules:I,"label-width":"100px"},{default:o(()=>[a(d,{label:"推理名称",prop:"name"},{default:o(()=>[a(r,{modelValue:n.name,"onUpdate:modelValue":e[0]||(e[0]=l=>n.name=l),placeholder:"请输入推理名称",maxlength:"50","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(d,{label:"描述"},{default:o(()=>[a(r,{modelValue:n.description,"onUpdate:modelValue":e[1]||(e[1]=l=>n.description=l),type:"textarea",rows:2,maxlength:"200","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(F,{"content-position":"left"},{default:o(()=>[...e[4]||(e[4]=[b("选择模型",-1)])]),_:1}),a(d,{label:"选择模型",prop:"model_key"},{default:o(()=>[a(P,{modelValue:n.model_key,"onUpdate:modelValue":e[2]||(e[2]=l=>n.model_key=l),placeholder:"请选择模型",filterable:"",style:{width:"400px"}},{default:o(()=>[a(C,{label:"本地模型"},{default:o(()=>[(s(!0),v(w,null,k(x.value,l=>(s(),u(g,{key:l.key,label:l.name,value:l.key},null,8,["label","value"]))),128))]),_:1}),a(C,{label:"已训练模型"},{default:o(()=>[(s(!0),v(w,null,k(M.value,l=>(s(),u(g,{key:l.key,label:l.name+(l.disabled?"(未合并)":""),value:l.key,disabled:l.disabled},null,8,["label","value","disabled"]))),128))]),_:1})]),_:1},8,["modelValue"])]),_:1}),a(d,{label:"GPU"},{default:o(()=>[a(P,{modelValue:n.gpu_id,"onUpdate:modelValue":e[3]||(e[3]=l=>n.gpu_id=l),style:{width:"400px"}},{default:o(()=>[(s(!0),v(w,null,k(_.value,(l,y)=>(s(),u(g,{key:y,label:`${l.name} (GPU${y})`,value:y},null,8,["label","value"]))),128))]),_:1},8,["modelValue"])]),_:1}),m.value?(s(),u(d,{key:0,label:"启动状态"},{default:o(()=>[a(N,{title:m.value,type:"info",closable:!1,"show-icon":""},null,8,["title"])]),_:1})):Z("",!0),a(d,null,{default:o(()=>[a(U,{type:"primary",loading:c.value,onClick:S},{default:o(()=>[...e[5]||(e[5]=[b("开始推理",-1)])]),_:1},8,["loading"]),a(U,{onClick:$},{default:o(()=>[...e[6]||(e[6]=[b("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1})}}});export{Ie as default};
|
||||
import{E as T}from"./el-alert-CoSdHkeK.js";import{a as A,E as D}from"./el-form-item-Bq0n_p5a.js";import{E as L}from"./index-h5FkzAP_.js";import{E as j}from"./index-BNsCyG4A.js";import{E as z,b as H,a as J}from"./el-select-BD70by95.js";import{E as K}from"./el-divider-a924dciw.js";import{d as Q,G as W,e as u,w as o,at as X,y as i,z as Y,o as s,s as a,x as b,c as v,ad as k,M as w,g as Z,j as p,A as B,v as ee}from"./index-Ds9AETjS.js";import"./el-scrollbar-CF42LNzg.js";import"./el-popper-D1tByNLb.js";/* empty css */import{P as te}from"./PageCard-CV3p14-x.js";import{g as le,a as ae}from"./model-cWd9CDtT.js";import"./vnode-CBiWHFw7.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./index-sWKLi7bH.js";import"./index-CQlF36Lk.js";import"./index-ByhS8A1d.js";import"./scroll-DVmMv0d9.js";import"./clamp-CyhADbdq.js";import"./toNumber-DrayciwT.js";import"./_baseIteratee-82gB5Yov.js";import"./el-card-D86TXgMv.js";import"./_plugin-vue_export-helper-DlAUqK2U.js";const Ie=Q({__name:"InferenceCreateView",setup(oe){const E=ee(),f=i(),c=i(!1),m=i(""),V=i([]),h=i([]),_=i([]),x=p(()=>V.value.map(t=>({key:`db-${t.id}`,id:t.id,name:t.name,source:"database",model_path:t.path||""}))),M=p(()=>h.value.map(t=>({key:`trained-${t.id}`,id:t.id,name:t.name,source:"trained",model_path:t.merged_path||t.base_model_path||"",merged:t.merged,merging:t.merging,disabled:t.merged===!1}))),G=p(()=>{const t={};for(const e of[...x.value,...M.value])t[e.key]=e;return t}),n=Y({name:"",description:"",model_key:"",gpu_id:0}),I={name:[{required:!0,message:"请输入推理名称",trigger:"blur"}],model_key:[{required:!0,message:"请选择模型",trigger:"change"}]},O=p(()=>G.value[n.model_key]);async function S(){f.value&&await f.value.validate(async t=>{if(!t)return;const e=O.value;if(!e){B.warning("请选择模型");return}c.value=!0,m.value="正在启动模型服务...";try{await new Promise(r=>setTimeout(r,1200)),B.success("模型已启动"),E.push({path:"/model-inference/chat/mock",query:{model:e.name}})}finally{c.value=!1,m.value=""}})}function $(){E.back()}async function q(){try{const[t,e,r]=await Promise.all([le(),ae(),X()]);V.value=t||[],h.value=(e==null?void 0:e.models)||[],_.value=(r==null?void 0:r.gpu)||[],_.value.length>0&&(n.gpu_id=0)}catch{}}return W(q),(t,e)=>{const r=L,d=A,F=K,g=J,C=H,P=z,N=T,U=j,R=D;return s(),u(te,{title:"新建推理"},{default:o(()=>[a(R,{ref_key:"formRef",ref:f,model:n,rules:I,"label-width":"100px"},{default:o(()=>[a(d,{label:"推理名称",prop:"name"},{default:o(()=>[a(r,{modelValue:n.name,"onUpdate:modelValue":e[0]||(e[0]=l=>n.name=l),placeholder:"请输入推理名称",maxlength:"50","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(d,{label:"描述"},{default:o(()=>[a(r,{modelValue:n.description,"onUpdate:modelValue":e[1]||(e[1]=l=>n.description=l),type:"textarea",rows:2,maxlength:"200","show-word-limit":"",style:{"max-width":"400px"}},null,8,["modelValue"])]),_:1}),a(F,{"content-position":"left"},{default:o(()=>[...e[4]||(e[4]=[b("选择模型",-1)])]),_:1}),a(d,{label:"选择模型",prop:"model_key"},{default:o(()=>[a(P,{modelValue:n.model_key,"onUpdate:modelValue":e[2]||(e[2]=l=>n.model_key=l),placeholder:"请选择模型",filterable:"",style:{width:"400px"}},{default:o(()=>[a(C,{label:"本地模型"},{default:o(()=>[(s(!0),v(w,null,k(x.value,l=>(s(),u(g,{key:l.key,label:l.name,value:l.key},null,8,["label","value"]))),128))]),_:1}),a(C,{label:"已训练模型"},{default:o(()=>[(s(!0),v(w,null,k(M.value,l=>(s(),u(g,{key:l.key,label:l.name+(l.disabled?"(未合并)":""),value:l.key,disabled:l.disabled},null,8,["label","value","disabled"]))),128))]),_:1})]),_:1},8,["modelValue"])]),_:1}),a(d,{label:"GPU"},{default:o(()=>[a(P,{modelValue:n.gpu_id,"onUpdate:modelValue":e[3]||(e[3]=l=>n.gpu_id=l),style:{width:"400px"}},{default:o(()=>[(s(!0),v(w,null,k(_.value,(l,y)=>(s(),u(g,{key:y,label:`${l.name} (GPU${y})`,value:y},null,8,["label","value"]))),128))]),_:1},8,["modelValue"])]),_:1}),m.value?(s(),u(d,{key:0,label:"启动状态"},{default:o(()=>[a(N,{title:m.value,type:"info",closable:!1,"show-icon":""},null,8,["title"])]),_:1})):Z("",!0),a(d,null,{default:o(()=>[a(U,{type:"primary",loading:c.value,onClick:S},{default:o(()=>[...e[5]||(e[5]=[b("开始推理",-1)])]),_:1},8,["loading"]),a(U,{onClick:$},{default:o(()=>[...e[6]||(e[6]=[b("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1})}}});export{Ie as default};
|
||||
1
frontend/dist/assets/InferenceListView-B74C2RPL.js
vendored
Normal file
1
frontend/dist/assets/InferenceListView-B74C2RPL.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{E as $,a as q}from"./el-form-item-D5kF3B90.js";import{E as z}from"./index-TFUf94PZ.js";import{u as K,E as P}from"./index-BjEW7-SA.js";import{E as R}from"./el-checkbox-qGZ2-RM2.js";import{b as T,i as U,d as N,u as A,a as D,o as u,c as k,e as h,w as t,r as M,f as w,E as j,g,n as E,h as V,j as b,k as C,l as G,m as H,p as J,q as n,s,t as O,v as Q,x as y,y as B,z as W,A as X}from"./index-BKKvzUDD.js";import{_ as S}from"./logo-CxXS7KxG.js";import{_ as Y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./omit-BTSq4AYh.js";const Z=T({type:{type:String,values:["primary","success","warning","info","danger","default"],default:void 0},underline:{type:[Boolean,String],values:[!0,!1,"always","never","hover"],default:void 0},disabled:Boolean,href:{type:String,default:""},target:{type:String,default:"_self"},icon:{type:U}}),ee={click:l=>l instanceof MouseEvent},ae=["href","target"];var ne=N({name:"ElLink",__name:"link",props:Z,emits:ee,setup(l,{emit:m}){const o=l,i=m,r=A("link");K({scope:"el-link",from:"The underline option (boolean)",replacement:"'always' | 'hover' | 'never'",version:"3.0.0",ref:"https://element-plus.org/en-US/component/link.html#underline"},b(()=>C(o.underline)));const a=D("link"),c=b(()=>{var e;return[a.b(),a.m(o.type??((e=r.value)==null?void 0:e.type)??"default"),a.is("disabled",o.disabled),a.is("underline",d.value==="always"),a.is("hover-underline",d.value==="hover"&&!o.disabled)]}),d=b(()=>{var e;return C(o.underline)?o.underline?"hover":"never":o.underline??((e=r.value)==null?void 0:e.underline)??"hover"});function p(e){o.disabled||i("click",e)}return(e,f)=>(u(),k("a",{class:E(c.value),href:l.disabled||!l.href?void 0:l.href,target:l.disabled||!l.href?void 0:l.target,onClick:p},[l.icon?(u(),h(w(j),{key:0},{default:t(()=>[(u(),h(M(l.icon)))]),_:1})):g("v-if",!0),e.$slots.default?(u(),k("span",{key:1,class:E(w(a).e("inner"))},[V(e.$slots,"default")],2)):g("v-if",!0),e.$slots.icon?V(e.$slots,"icon",{key:2}):g("v-if",!0)],10,ae))}}),le=ne;const oe=G(le),se={class:"login-page"},te={class:"login-panel"},re={class:"login-form-wrap"},ie={class:"login-options"},de=N({__name:"LoginView",setup(l){const m=Q(),o=H(),i=B(),r=B(!1),a=W({username:"",password:""}),c={username:[{required:!0,message:"请输入账号",trigger:"blur"}],password:[{required:!0,message:"请输入密码",trigger:"blur"}]};async function d(){i.value&&await i.value.validate(async p=>{if(p){r.value=!0;try{await o.login(a.username,a.password),X.success("登录成功"),m.push("/dashboard")}catch{}finally{r.value=!1}}})}return(p,e)=>{const f=z,_=q,x=R,F=oe,I=P,L=$;return u(),k("div",se,[e[8]||(e[8]=J('<section class="login-visual" aria-labelledby="platform-title" data-v-713185bd><div class="login-visual-content" data-v-713185bd><div class="login-visual-copy" data-v-713185bd><h1 id="platform-title" data-v-713185bd>远光软件微调平台</h1><p data-v-713185bd>大模型微调、评测与推理的一体化工作台</p></div><div class="login-visual-footer" aria-label="平台核心能力" data-v-713185bd><span data-v-713185bd>数据准备</span><span aria-hidden="true" data-v-713185bd>·</span><span data-v-713185bd>模型训练</span><span aria-hidden="true" data-v-713185bd>·</span><span data-v-713185bd>效果评测</span></div></div></section>',1)),n("main",te,[e[6]||(e[6]=n("div",{class:"brand-lockup",role:"img","aria-label":"远光软件"},[n("span",{class:"brand-logo-crop brand-logo-crop-mark","aria-hidden":"true"},[n("img",{src:S,alt:""})]),n("span",{class:"brand-logo-crop brand-logo-crop-wordmark","aria-hidden":"true"},[n("img",{src:S,alt:""})])],-1)),n("div",re,[e[5]||(e[5]=n("div",{class:"login-heading"},[n("span",null,"账号登录"),n("h2",null,"欢迎回来"),n("p",null,"登录后继续使用微调平台")],-1)),s(L,{ref_key:"loginFormRef",ref:i,model:a,rules:c,"label-position":"top",size:"large",onKeyup:O(d,["enter"])},{default:t(()=>[s(_,{label:"账号",prop:"username"},{default:t(()=>[s(f,{modelValue:a.username,"onUpdate:modelValue":e[0]||(e[0]=v=>a.username=v),placeholder:"请输入账号",clearable:""},null,8,["modelValue"])]),_:1}),s(_,{label:"密码",prop:"password"},{default:t(()=>[s(f,{modelValue:a.password,"onUpdate:modelValue":e[1]||(e[1]=v=>a.password=v),type:"password",placeholder:"请输入密码","show-password":""},null,8,["modelValue"])]),_:1}),n("div",ie,[s(x,null,{default:t(()=>[...e[2]||(e[2]=[y("记住密码",-1)])]),_:1}),s(F,{type:"primary",underline:!1},{default:t(()=>[...e[3]||(e[3]=[y("忘记密码?",-1)])]),_:1})]),s(I,{type:"primary",class:"login-btn",loading:r.value,onClick:d},{default:t(()=>[...e[4]||(e[4]=[y(" 登录 ",-1)])]),_:1},8,["loading"])]),_:1},8,["model"])]),e[7]||(e[7]=n("footer",null,"© 2026 远光软件",-1))])])}}}),Ee=Y(de,[["__scopeId","data-v-713185bd"]]);export{Ee as default};
|
||||
import{E as $,a as q}from"./el-form-item-Bq0n_p5a.js";import{E as z}from"./index-h5FkzAP_.js";import{u as K,E as P}from"./index-BNsCyG4A.js";import{E as R}from"./el-checkbox-pV3o_8iy.js";import{b as T,i as U,d as N,u as A,a as D,o as u,c as k,e as h,w as t,r as M,f as w,E as j,g,n as E,h as V,j as b,k as C,l as G,m as H,p as J,q as n,s,t as O,v as Q,x as y,y as B,z as W,A as X}from"./index-Ds9AETjS.js";import{_ as S}from"./logo-CxXS7KxG.js";import{_ as Y}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./omit-BzI1MV_K.js";const Z=T({type:{type:String,values:["primary","success","warning","info","danger","default"],default:void 0},underline:{type:[Boolean,String],values:[!0,!1,"always","never","hover"],default:void 0},disabled:Boolean,href:{type:String,default:""},target:{type:String,default:"_self"},icon:{type:U}}),ee={click:l=>l instanceof MouseEvent},ae=["href","target"];var ne=N({name:"ElLink",__name:"link",props:Z,emits:ee,setup(l,{emit:m}){const o=l,i=m,r=A("link");K({scope:"el-link",from:"The underline option (boolean)",replacement:"'always' | 'hover' | 'never'",version:"3.0.0",ref:"https://element-plus.org/en-US/component/link.html#underline"},b(()=>C(o.underline)));const a=D("link"),c=b(()=>{var e;return[a.b(),a.m(o.type??((e=r.value)==null?void 0:e.type)??"default"),a.is("disabled",o.disabled),a.is("underline",d.value==="always"),a.is("hover-underline",d.value==="hover"&&!o.disabled)]}),d=b(()=>{var e;return C(o.underline)?o.underline?"hover":"never":o.underline??((e=r.value)==null?void 0:e.underline)??"hover"});function p(e){o.disabled||i("click",e)}return(e,f)=>(u(),k("a",{class:E(c.value),href:l.disabled||!l.href?void 0:l.href,target:l.disabled||!l.href?void 0:l.target,onClick:p},[l.icon?(u(),h(w(j),{key:0},{default:t(()=>[(u(),h(M(l.icon)))]),_:1})):g("v-if",!0),e.$slots.default?(u(),k("span",{key:1,class:E(w(a).e("inner"))},[V(e.$slots,"default")],2)):g("v-if",!0),e.$slots.icon?V(e.$slots,"icon",{key:2}):g("v-if",!0)],10,ae))}}),le=ne;const oe=G(le),se={class:"login-page"},te={class:"login-panel"},re={class:"login-form-wrap"},ie={class:"login-options"},de=N({__name:"LoginView",setup(l){const m=Q(),o=H(),i=B(),r=B(!1),a=W({username:"",password:""}),c={username:[{required:!0,message:"请输入账号",trigger:"blur"}],password:[{required:!0,message:"请输入密码",trigger:"blur"}]};async function d(){i.value&&await i.value.validate(async p=>{if(p){r.value=!0;try{await o.login(a.username,a.password),X.success("登录成功"),m.push("/dashboard")}catch{}finally{r.value=!1}}})}return(p,e)=>{const f=z,_=q,x=R,F=oe,I=P,L=$;return u(),k("div",se,[e[8]||(e[8]=J('<section class="login-visual" aria-labelledby="platform-title" data-v-713185bd><div class="login-visual-content" data-v-713185bd><div class="login-visual-copy" data-v-713185bd><h1 id="platform-title" data-v-713185bd>远光软件微调平台</h1><p data-v-713185bd>大模型微调、评测与推理的一体化工作台</p></div><div class="login-visual-footer" aria-label="平台核心能力" data-v-713185bd><span data-v-713185bd>数据准备</span><span aria-hidden="true" data-v-713185bd>·</span><span data-v-713185bd>模型训练</span><span aria-hidden="true" data-v-713185bd>·</span><span data-v-713185bd>效果评测</span></div></div></section>',1)),n("main",te,[e[6]||(e[6]=n("div",{class:"brand-lockup",role:"img","aria-label":"远光软件"},[n("span",{class:"brand-logo-crop brand-logo-crop-mark","aria-hidden":"true"},[n("img",{src:S,alt:""})]),n("span",{class:"brand-logo-crop brand-logo-crop-wordmark","aria-hidden":"true"},[n("img",{src:S,alt:""})])],-1)),n("div",re,[e[5]||(e[5]=n("div",{class:"login-heading"},[n("span",null,"账号登录"),n("h2",null,"欢迎回来"),n("p",null,"登录后继续使用微调平台")],-1)),s(L,{ref_key:"loginFormRef",ref:i,model:a,rules:c,"label-position":"top",size:"large",onKeyup:O(d,["enter"])},{default:t(()=>[s(_,{label:"账号",prop:"username"},{default:t(()=>[s(f,{modelValue:a.username,"onUpdate:modelValue":e[0]||(e[0]=v=>a.username=v),placeholder:"请输入账号",clearable:""},null,8,["modelValue"])]),_:1}),s(_,{label:"密码",prop:"password"},{default:t(()=>[s(f,{modelValue:a.password,"onUpdate:modelValue":e[1]||(e[1]=v=>a.password=v),type:"password",placeholder:"请输入密码","show-password":""},null,8,["modelValue"])]),_:1}),n("div",ie,[s(x,null,{default:t(()=>[...e[2]||(e[2]=[y("记住密码",-1)])]),_:1}),s(F,{type:"primary",underline:!1},{default:t(()=>[...e[3]||(e[3]=[y("忘记密码?",-1)])]),_:1})]),s(I,{type:"primary",class:"login-btn",loading:r.value,onClick:d},{default:t(()=>[...e[4]||(e[4]=[y(" 登录 ",-1)])]),_:1},8,["loading"])]),_:1},8,["model"])]),e[7]||(e[7]=n("footer",null,"© 2026 远光软件",-1))])])}}}),Ee=Y(de,[["__scopeId","data-v-713185bd"]]);export{Ee as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
||||
import{E as A}from"./el-alert-xVLeNUgJ.js";import{a as N,E as T}from"./el-form-item-D5kF3B90.js";import{E as B}from"./index-TFUf94PZ.js";import{E as I}from"./index-BjEW7-SA.js";import{E as R}from"./index-BDEF353-.js";import{d as P,z as q,G as D,e as d,w as o,j as y,y as E,o as n,s as t,f as F,ar as U,x as m,aa as W,c as j,g as z,A as M,v as G,ac as H}from"./index-BKKvzUDD.js";/* empty css */import{P as L}from"./PageCard-BoKXOzst.js";import{a as O,m as S}from"./model-gCK34aBo.js";import{_ as J}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./vnode-78qeDweP.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./el-card-CApHJ1Gj.js";const K={key:0,class:"model-status"},Q=P({__name:"MergeWeightsView",setup(X){const v=H(),b=G(),s=E(!1),u=y(()=>v.query.model||""),w=y(()=>v.query.method||"lora"),i=E([]),_=y(()=>i.value.find(l=>l.name===u.value)),a=q({model_name:u.value,train_method:w.value,base_model_path:""});async function V(){try{const l=await O();i.value=(l==null?void 0:l.models)||[];const e=i.value.find(p=>p.name===u.value);a.base_model_path=(e==null?void 0:e.base_model_path)||""}catch{}}async function k(){if(!a.model_name||!a.base_model_path){M.warning("缺少模型信息");return}s.value=!0;try{await S({model_name:a.model_name,train_method:a.train_method,base_model_path:a.base_model_path}),M.success("合并成功"),b.push("/model-manage")}catch{}finally{s.value=!1}}function x(){b.back()}return D(V),(l,e)=>{const p=A,f=B,r=N,h=I,C=T,c=R;return n(),d(L,{title:"合并权重"},{default:o(()=>[t(p,{type:"info",closable:!1,"show-icon":"",title:"将 LoRA 适配器权重合并到基座模型,合并后可直接用于推理部署。",style:{"margin-bottom":"20px"}}),t(C,{"label-width":"120px",style:{"max-width":"600px"}},{default:o(()=>[t(r,{label:"模型名称"},{default:o(()=>[t(f,{modelValue:a.model_name,"onUpdate:modelValue":e[0]||(e[0]=g=>a.model_name=g),disabled:""},null,8,["modelValue"])]),_:1}),t(r,{label:"训练方法"},{default:o(()=>[t(f,{"model-value":F(U)[a.train_method]||a.train_method,disabled:""},null,8,["model-value"])]),_:1}),t(r,{label:"基座模型路径"},{default:o(()=>[t(f,{modelValue:a.base_model_path,"onUpdate:modelValue":e[1]||(e[1]=g=>a.base_model_path=g),placeholder:"基座模型路径"},null,8,["modelValue"])]),_:1}),t(r,null,{default:o(()=>[t(h,{type:"primary",loading:s.value,onClick:k},{default:o(()=>[m(W(s.value?"合并中...":"开始合并"),1)]),_:1},8,["loading"]),t(h,{onClick:x},{default:o(()=>[...e[2]||(e[2]=[m("取消",-1)])]),_:1})]),_:1})]),_:1}),_.value?(n(),j("div",K,[_.value.merging?(n(),d(c,{key:0,type:"warning"},{default:o(()=>[...e[3]||(e[3]=[m("合并中",-1)])]),_:1})):_.value.merged?(n(),d(c,{key:1,type:"success"},{default:o(()=>[...e[4]||(e[4]=[m("已合并",-1)])]),_:1})):(n(),d(c,{key:2,type:"info"},{default:o(()=>[...e[5]||(e[5]=[m("未合并",-1)])]),_:1}))])):z("",!0)]),_:1})}}}),fe=J(Q,[["__scopeId","data-v-2e30aa99"]]);export{fe as default};
|
||||
import{E as A}from"./el-alert-CoSdHkeK.js";import{a as N,E as T}from"./el-form-item-Bq0n_p5a.js";import{E as B}from"./index-h5FkzAP_.js";import{E as I}from"./index-BNsCyG4A.js";import{E as R}from"./index-ByhS8A1d.js";import{d as P,z as q,G as D,e as d,w as o,j as y,y as E,o as n,s as t,f as F,ar as U,x as m,aa as W,c as j,g as z,A as M,v as G,ac as H}from"./index-Ds9AETjS.js";/* empty css */import{P as L}from"./PageCard-CV3p14-x.js";import{a as O,m as S}from"./model-cWd9CDtT.js";import{_ as J}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./vnode-CBiWHFw7.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./el-card-D86TXgMv.js";const K={key:0,class:"model-status"},Q=P({__name:"MergeWeightsView",setup(X){const v=H(),b=G(),s=E(!1),u=y(()=>v.query.model||""),w=y(()=>v.query.method||"lora"),i=E([]),_=y(()=>i.value.find(l=>l.name===u.value)),a=q({model_name:u.value,train_method:w.value,base_model_path:""});async function V(){try{const l=await O();i.value=(l==null?void 0:l.models)||[];const e=i.value.find(p=>p.name===u.value);a.base_model_path=(e==null?void 0:e.base_model_path)||""}catch{}}async function k(){if(!a.model_name||!a.base_model_path){M.warning("缺少模型信息");return}s.value=!0;try{await S({model_name:a.model_name,train_method:a.train_method,base_model_path:a.base_model_path}),M.success("合并成功"),b.push("/model-manage")}catch{}finally{s.value=!1}}function x(){b.back()}return D(V),(l,e)=>{const p=A,f=B,r=N,h=I,C=T,c=R;return n(),d(L,{title:"合并权重"},{default:o(()=>[t(p,{type:"info",closable:!1,"show-icon":"",title:"将 LoRA 适配器权重合并到基座模型,合并后可直接用于推理部署。",style:{"margin-bottom":"20px"}}),t(C,{"label-width":"120px",style:{"max-width":"600px"}},{default:o(()=>[t(r,{label:"模型名称"},{default:o(()=>[t(f,{modelValue:a.model_name,"onUpdate:modelValue":e[0]||(e[0]=g=>a.model_name=g),disabled:""},null,8,["modelValue"])]),_:1}),t(r,{label:"训练方法"},{default:o(()=>[t(f,{"model-value":F(U)[a.train_method]||a.train_method,disabled:""},null,8,["model-value"])]),_:1}),t(r,{label:"基座模型路径"},{default:o(()=>[t(f,{modelValue:a.base_model_path,"onUpdate:modelValue":e[1]||(e[1]=g=>a.base_model_path=g),placeholder:"基座模型路径"},null,8,["modelValue"])]),_:1}),t(r,null,{default:o(()=>[t(h,{type:"primary",loading:s.value,onClick:k},{default:o(()=>[m(W(s.value?"合并中...":"开始合并"),1)]),_:1},8,["loading"]),t(h,{onClick:x},{default:o(()=>[...e[2]||(e[2]=[m("取消",-1)])]),_:1})]),_:1})]),_:1}),_.value?(n(),j("div",K,[_.value.merging?(n(),d(c,{key:0,type:"warning"},{default:o(()=>[...e[3]||(e[3]=[m("合并中",-1)])]),_:1})):_.value.merged?(n(),d(c,{key:1,type:"success"},{default:o(()=>[...e[4]||(e[4]=[m("已合并",-1)])]),_:1})):(n(),d(c,{key:2,type:"info"},{default:o(()=>[...e[5]||(e[5]=[m("未合并",-1)])]),_:1}))])):z("",!0)]),_:1})}}}),fe=J(Q,[["__scopeId","data-v-2e30aa99"]]);export{fe as default};
|
||||
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/ModelCreateView-cLO_UPLT.css
vendored
Normal file
1
frontend/dist/assets/ModelCreateView-cLO_UPLT.css
vendored
Normal file
@@ -0,0 +1 @@
|
||||
.model-path-row[data-v-6de511b8]{display:grid;grid-template-columns:minmax(0,1fr) auto auto;align-items:center;gap:8px;width:100%}.model-path-help[data-v-6de511b8]{color:#64748b;cursor:help}
|
||||
1
frontend/dist/assets/ModelCreateView-rx4YrtfV.js
vendored
Normal file
1
frontend/dist/assets/ModelCreateView-rx4YrtfV.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
||||
@charset "UTF-8";.capsule-tabs[data-v-f06fa847]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-f06fa847]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-f06fa847]:hover{color:#1e293b}.capsule-tab-item.active[data-v-f06fa847]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}.action-buttons[data-v-f06fa847]{display:flex;justify-content:center;gap:8px}
|
||||
1
frontend/dist/assets/ModelManageView-CSQiXUqK.js
vendored
Normal file
1
frontend/dist/assets/ModelManageView-CSQiXUqK.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/ModelManageView-SqczPXXo.css
vendored
Normal file
1
frontend/dist/assets/ModelManageView-SqczPXXo.css
vendored
Normal file
@@ -0,0 +1 @@
|
||||
@charset "UTF-8";.capsule-tabs[data-v-4609d016]{display:flex;background:#f1f5f9;padding:3px;border-radius:8px;gap:2px;border:1px solid #e2e8f0}.capsule-tab-item[data-v-4609d016]{border:0;background:transparent;padding:6px 20px;font-size:13px;font-weight:500;color:#64748b;cursor:pointer;border-radius:6px;transition:all .2s ease;outline:none}.capsule-tab-item[data-v-4609d016]:hover{color:#1e293b}.capsule-tab-item.active[data-v-4609d016]{background:#fff;color:#4f46e5;box-shadow:0 1px 3px #0000000f,0 1px 2px #0000000a;font-weight:600}.action-buttons[data-v-4609d016]{display:flex;justify-content:center;gap:8px}
|
||||
@@ -1 +1 @@
|
||||
import{E as r}from"./index-BDEF353-.js";import{d as o,e as c,w as p,j as u,o as i,x as d,aa as l}from"./index-BKKvzUDD.js";/* empty css */const m=o({__name:"ModelStatusTag",props:{status:{}},setup(a){const s=a,t=u(()=>{const e=s.status||"";switch(e){case"running":case"ready":case"loaded":return{text:e==="ready"||e==="loaded"?"已就绪":"运行中",type:"success"};case"pending":return{text:"等待中",type:"info"};case"failed":return{text:"失败",type:"danger"};case"starting":case"loading":return{text:"加载中",type:"warning"};case"completed":return{text:"已完成",type:"success"};case"stopped":return{text:"已停止",type:"info"};default:return{text:e||"未知",type:"info"}}});return(e,f)=>{const n=r;return i(),c(n,{type:t.value.type,size:"small",effect:"light"},{default:p(()=>[d(l(t.value.text),1)]),_:1},8,["type"])}}});export{m as _};
|
||||
import{E as r}from"./index-ByhS8A1d.js";import{d as o,e as c,w as p,j as u,o as i,x as d,aa as l}from"./index-Ds9AETjS.js";/* empty css */const m=o({__name:"ModelStatusTag",props:{status:{}},setup(a){const s=a,t=u(()=>{const e=s.status||"";switch(e){case"running":case"ready":case"loaded":return{text:e==="ready"||e==="loaded"?"已就绪":"运行中",type:"success"};case"pending":return{text:"等待中",type:"info"};case"failed":return{text:"失败",type:"danger"};case"starting":case"loading":return{text:"加载中",type:"warning"};case"completed":return{text:"已完成",type:"success"};case"stopped":return{text:"已停止",type:"info"};default:return{text:e||"未知",type:"info"}}});return(e,f)=>{const n=r;return i(),c(n,{type:t.value.type,size:"small",effect:"light"},{default:p(()=>[d(l(t.value.text),1)]),_:1},8,["type"])}}});export{m as _};
|
||||
@@ -1 +1 @@
|
||||
import{E as i}from"./el-card-CApHJ1Gj.js";import{d as n,e as u,w as p,o as a,c as s,h as o,g as r,q as d,aa as l}from"./index-BKKvzUDD.js";import{_}from"./_plugin-vue_export-helper-DlAUqK2U.js";const h={key:0,class:"page-card-header"},m={class:"page-card-title"},f={key:0,class:"page-card-subtitle"},g={key:0,class:"page-card-extra"},v={class:"page-card-body"},b=n({__name:"PageCard",props:{title:{},subtitle:{},bordered:{type:Boolean,default:!1}},setup(t){return(e,k)=>{const c=i;return a(),u(c,{shadow:"never",class:"page-card"},{default:p(()=>[t.title||e.$slots.header?(a(),s("div",h,[o(e.$slots,"header",{},()=>[d("div",null,[d("h2",m,l(t.title),1),t.subtitle?(a(),s("p",f,l(t.subtitle),1)):r("",!0)])],!0),e.$slots.extra?(a(),s("div",g,[o(e.$slots,"extra",{},void 0,!0)])):r("",!0)])):r("",!0),d("div",v,[o(e.$slots,"default",{},void 0,!0)])]),_:3})}}}),$=_(b,[["__scopeId","data-v-05c37e8a"]]);export{$ as P};
|
||||
import{E as i}from"./el-card-D86TXgMv.js";import{d as n,e as u,w as p,o as a,c as s,h as o,g as r,q as d,aa as l}from"./index-Ds9AETjS.js";import{_}from"./_plugin-vue_export-helper-DlAUqK2U.js";const h={key:0,class:"page-card-header"},m={class:"page-card-title"},f={key:0,class:"page-card-subtitle"},g={key:0,class:"page-card-extra"},v={class:"page-card-body"},b=n({__name:"PageCard",props:{title:{},subtitle:{},bordered:{type:Boolean,default:!1}},setup(t){return(e,k)=>{const c=i;return a(),u(c,{shadow:"never",class:"page-card"},{default:p(()=>[t.title||e.$slots.header?(a(),s("div",h,[o(e.$slots,"header",{},()=>[d("div",null,[d("h2",m,l(t.title),1),t.subtitle?(a(),s("p",f,l(t.subtitle),1)):r("",!0)])],!0),e.$slots.extra?(a(),s("div",g,[o(e.$slots,"extra",{},void 0,!0)])):r("",!0)])):r("",!0),d("div",v,[o(e.$slots,"default",{},void 0,!0)])]),_:3})}}}),$=_(b,[["__scopeId","data-v-05c37e8a"]]);export{$ as P};
|
||||
@@ -1 +1 @@
|
||||
import{_ as o}from"./_plugin-vue_export-helper-DlAUqK2U.js";import{o as c,c as t,q as s}from"./index-BKKvzUDD.js";const n={},r={class:"simple-page"};function a(i,e){return c(),t("section",r,[...e[0]||(e[0]=[s("h1",null,"无权访问",-1),s("p",null,"当前账号没有访问该页面的权限,请联系管理员调整角色或页面权限。",-1)])])}const p=o(n,[["render",a],["__scopeId","data-v-cc370c43"]]);export{p as default};
|
||||
import{_ as o}from"./_plugin-vue_export-helper-DlAUqK2U.js";import{o as c,c as t,q as s}from"./index-Ds9AETjS.js";const n={},r={class:"simple-page"};function a(i,e){return c(),t("section",r,[...e[0]||(e[0]=[s("h1",null,"无权访问",-1),s("p",null,"当前账号没有访问该页面的权限,请联系管理员调整角色或页面权限。",-1)])])}const p=o(n,[["render",a],["__scopeId","data-v-cc370c43"]]);export{p as default};
|
||||
1
frontend/dist/assets/ToolCreateView-BGgtRypT.js
vendored
Normal file
1
frontend/dist/assets/ToolCreateView-BGgtRypT.js
vendored
Normal file
@@ -0,0 +1 @@
|
||||
import{a as y,E as h}from"./el-form-item-Bq0n_p5a.js";import{E as B}from"./index-h5FkzAP_.js";import{E as R}from"./index-BNsCyG4A.js";import{d as S,G as I,e as N,w as l,j as g,z as O,o as m,s as o,q as v,c as x,ad as U,n as V,f as q,bx as F,M as L,x as b,aa as M,A as w,y as j,v as z,ac as D}from"./index-Ds9AETjS.js";import{P}from"./PageCard-CV3p14-x.js";import{u as A}from"./tools-s3Et6p-2.js";import{_ as G}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-BWI4UxBy.js";import"./_baseClone-CZxxnGCv.js";import"./raf-zQ00nuMI.js";import"./index-B0if-AHo.js";import"./index-BSbCLbaz.js";import"./el-card-D86TXgMv.js";const $={class:"icon-grid"},H=["onClick"],J=S({__name:"ToolCreateView",setup(K){const d=D(),p=z(),n=A(),i=j(),s=g(()=>d.params.id!=null),c=g(()=>s.value?n.getTool(d.params.id):void 0),e=O({id:"",name:"",description:"",url:"",icon:"fa-cog"}),C={name:[{required:!0,message:"请输入工具名称",trigger:"blur"},{max:30,message:"不超过 30 字符",trigger:"blur"}],url:[{required:!0,message:"请输入跳转地址",trigger:"blur"}]};function k(){i.value&&i.value.validate(f=>{f&&(s.value?(n.updateTool(e.id,{...e}),w.success("更新成功")):(n.addTool({...e,id:"custom_"+Date.now()}),w.success("添加成功")),p.push("/tools"))})}function E(){p.back()}return I(()=>{c.value&&Object.assign(e,c.value)}),(f,a)=>{const u=B,r=y,_=R,T=h;return m(),N(P,{title:s.value?"编辑自定义工具":"添加自定义工具"},{default:l(()=>[o(T,{ref_key:"formRef",ref:i,model:e,rules:C,"label-width":"100px",style:{"max-width":"600px"}},{default:l(()=>[o(r,{label:"工具名称",prop:"name"},{default:l(()=>[o(u,{modelValue:e.name,"onUpdate:modelValue":a[0]||(a[0]=t=>e.name=t),placeholder:"请输入工具名称",maxlength:"30","show-word-limit":""},null,8,["modelValue"])]),_:1}),o(r,{label:"工具描述"},{default:l(()=>[o(u,{modelValue:e.description,"onUpdate:modelValue":a[1]||(a[1]=t=>e.description=t),type:"textarea",rows:2,maxlength:"100","show-word-limit":""},null,8,["modelValue"])]),_:1}),o(r,{label:"跳转地址",prop:"url"},{default:l(()=>[o(u,{modelValue:e.url,"onUpdate:modelValue":a[2]||(a[2]=t=>e.url=t),placeholder:"相对路径或完整 URL"},null,8,["modelValue"])]),_:1}),o(r,{label:"图标"},{default:l(()=>[v("div",$,[(m(!0),x(L,null,U(q(F),t=>(m(),x("div",{key:t,class:V(["icon-option",{selected:e.icon===t}]),onClick:Q=>e.icon=t},[v("i",{class:V(["fa",t])},null,2)],10,H))),128))])]),_:1}),o(r,null,{default:l(()=>[o(_,{type:"primary",onClick:k},{default:l(()=>[b(M(s.value?"保存":"添加"),1)]),_:1}),o(_,{onClick:E},{default:l(()=>[...a[3]||(a[3]=[b("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1},8,["title"])}}}),ue=G(J,[["__scopeId","data-v-9116c5de"]]);export{ue as default};
|
||||
@@ -1 +0,0 @@
|
||||
import{a as y,E as h}from"./el-form-item-D5kF3B90.js";import{E as B}from"./index-TFUf94PZ.js";import{E as R}from"./index-BjEW7-SA.js";import{d as S,G as I,e as N,w as l,j as g,z as O,o as m,s as o,q as v,c as V,ad as U,n as b,f as q,bE as F,M as L,x,aa as M,A as w,y as j,v as z,ac as D}from"./index-BKKvzUDD.js";import{P}from"./PageCard-BoKXOzst.js";import{u as A}from"./tools-BSjcVinb.js";import{_ as G}from"./_plugin-vue_export-helper-DlAUqK2U.js";import"./castArray-5uErZEc3.js";import"./_baseClone-B5RBzbh5.js";import"./raf-C-x62Pcl.js";import"./index-DLzof2Fz.js";import"./index-GAnQrJsQ.js";import"./el-card-CApHJ1Gj.js";const $={class:"icon-grid"},H=["onClick"],J=S({__name:"ToolCreateView",setup(K){const d=D(),p=z(),n=A(),i=j(),s=g(()=>d.params.id!=null),c=g(()=>s.value?n.getTool(d.params.id):void 0),e=O({id:"",name:"",description:"",url:"",icon:"fa-cog"}),C={name:[{required:!0,message:"请输入工具名称",trigger:"blur"},{max:30,message:"不超过 30 字符",trigger:"blur"}],url:[{required:!0,message:"请输入跳转地址",trigger:"blur"}]};function E(){i.value&&i.value.validate(f=>{f&&(s.value?(n.updateTool(e.id,{...e}),w.success("更新成功")):(n.addTool({...e,id:"custom_"+Date.now()}),w.success("添加成功")),p.push("/tools"))})}function k(){p.back()}return I(()=>{c.value&&Object.assign(e,c.value)}),(f,a)=>{const u=B,r=y,_=R,T=h;return m(),N(P,{title:s.value?"编辑自定义工具":"添加自定义工具"},{default:l(()=>[o(T,{ref_key:"formRef",ref:i,model:e,rules:C,"label-width":"100px",style:{"max-width":"600px"}},{default:l(()=>[o(r,{label:"工具名称",prop:"name"},{default:l(()=>[o(u,{modelValue:e.name,"onUpdate:modelValue":a[0]||(a[0]=t=>e.name=t),placeholder:"请输入工具名称",maxlength:"30","show-word-limit":""},null,8,["modelValue"])]),_:1}),o(r,{label:"工具描述"},{default:l(()=>[o(u,{modelValue:e.description,"onUpdate:modelValue":a[1]||(a[1]=t=>e.description=t),type:"textarea",rows:2,maxlength:"100","show-word-limit":""},null,8,["modelValue"])]),_:1}),o(r,{label:"跳转地址",prop:"url"},{default:l(()=>[o(u,{modelValue:e.url,"onUpdate:modelValue":a[2]||(a[2]=t=>e.url=t),placeholder:"相对路径或完整 URL"},null,8,["modelValue"])]),_:1}),o(r,{label:"图标"},{default:l(()=>[v("div",$,[(m(!0),V(L,null,U(q(F),t=>(m(),V("div",{key:t,class:b(["icon-option",{selected:e.icon===t}]),onClick:Q=>e.icon=t},[v("i",{class:b(["fa",t])},null,2)],10,H))),128))])]),_:1}),o(r,null,{default:l(()=>[o(_,{type:"primary",onClick:E},{default:l(()=>[x(M(s.value?"保存":"添加"),1)]),_:1}),o(_,{onClick:k},{default:l(()=>[...a[3]||(a[3]=[x("取消",-1)])]),_:1})]),_:1})]),_:1},8,["model"])]),_:1},8,["title"])}}}),ue=G(J,[["__scopeId","data-v-9116c5de"]]);export{ue as default};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
frontend/dist/assets/TrainingLogView-DGSpikbN.css
vendored
Normal file
1
frontend/dist/assets/TrainingLogView-DGSpikbN.css
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user