diff --git a/.gitignore b/.gitignore index 36b13f1..6462e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ __pycache__/ build/ develop-eggs/ dist/ +node_modules/ +*.tsbuildinfo downloads/ eggs/ .eggs/ @@ -37,6 +39,15 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt +# Runtime data and logs +runtime/ +backend/runtime/ +logs/ +backend/logs/ +*.db +*.sqlite +*.sqlite3 + # Unit test / coverage reports htmlcov/ .tox/ diff --git a/README.md b/README.md index 1614838..498e0e1 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # YG_FT 模型微调平台 -YG_FT 是一个面向企业治理场景的完整模型微调平台，覆盖用户中心、多租户、项目隔离、数据集管理、模型管理、训练任务、评测、推理、审批流、审计留存、算力调度和训练引擎适配。当前前端已存在基础页面，后端与算力平台已按多人协作开发方式建立工程骨架。 +YG_FT 是一个面向企业治理场景的模型微调平台，覆盖用户中心、多租户、项目隔离、数据集管理、模型管理、训练任务、评测、推理、审批流、审计留存、算力调度和训练引擎适配。 + +当前前端已有基础页面，后端与算力平台已按多人协作开发方式建立工程骨架，并开始实现第一版可演示闭环能力。第一版实现不是临时代码，工程模块、配置项、接口标签和数据库对象均按后续正式开发可延续的方式命名。 ## 总体架构 ```text YG_FT/ - frontend/ # 前端应用，承载训练平台控制台页面 + frontend/ # 前端控制台 backend/ # FastAPI 应用平台后端 app/ api/v1/ # 对前端暴露的 REST API @@ -23,7 +25,7 @@ YG_FT/ engines/llama_factory/ # LLaMA-Factory 适配器 file_gateway/ # 本地文件上传、下载、导入、产物管理 docs/ # 需求、接口、数据库、开发计划和部署文档 - docker/ # Nginx 等容器化配置 + docker/ # 容器化配置 ``` ## 平台分层 @@ -32,20 +34,19 @@ YG_FT/ | --- | --- | --- | | 前端控制台 | 用户操作入口、任务看板、项目/模型/数据集/训练/审批/审计页面 | `frontend/` | | 应用平台后端 | 用户中心、多租户、RBAC/ABAC、项目隔离、元数据、审批流、审计、API 编排 | `backend/` | -| 算力平台 | GPU 发现、资源锁定、训练进程管理、日志采集、产物归档、任务状态回传 | `compute/` | +| 算力平台 | GPU 发现、资源锁定、训练进程管理、日志采集、产物归档、任务状态同步 | `compute/` | | 训练引擎 | 当前固定接入 LLaMA-Factory，预留其他训练平台适配标准 | `compute/engines/` | | 数据层 | PostgreSQL、Redis、本地文件存储、日志归档 | `docs/postgres-schema.sql` | -## 关键能力 +## 第一版能力 -- 多租户：租户级数据隔离、租户配置、租户成员和角色。 -- 权限控制：支持项目、模型、数据集级隔离，后续可扩展到字段级和操作级策略。 -- 审批流：覆盖数据集发布、模型发布、训练资源申请、推理服务上线等企业流程。 -- 审计留存：操作审计、安全审计、审批审计、任务审计，支持留存周期策略。 -- 训练任务：训练参数管理、单机多 GPU 调度、任务状态同步、训练日志、产物管理。 -- 引擎适配：默认 LLaMA-Factory，预留统一 Engine Adapter 接口接入其他微调框架。 -- 文件存储：当前使用本地磁盘，按租户/项目/数据集/任务分区。 -- 日志采集：后端 JSON Lines 日志，主日志和错误日志拆分，便于 ELK/日志平台采集。 +- 使用 FastAPI 提供统一 API 响应结构 `{ code, message, data }`。 +- 本地运行阶段使用 SQLite 初始化样例用户、模型、数据集、算力节点、GPU 和微调任务。 +- 支持登录、模型管理、数据集管理、微调任务创建/启动/停止/进度轮询。 +- 支持训练日志、loss 指标、checkpoint 和训练产物的模拟生成。 +- 支持多算力节点、GPU、任务队列、资源副本和资源同步状态接口。 +- 前端新增 `/compute` 算力节点页面，展示节点地址、权重、标签、启用状态、GPU、队列和资源副本。 +- `compute/engines/llama_factory/adapter.py` 提供 LLaMA-Factory 参数校验、命令生成和日志解析基础能力。 ## 后端启动 @@ -54,15 +55,45 @@ cd backend python -m venv .venv .venv\Scripts\activate pip install -r requirements.txt -uvicorn app.main:app --reload +uvicorn app.main:app --reload --port 17861 ``` -默认健康检查： +默认 API 前缀为 `/api`，例如： ```text -GET /api/v1/health +GET /api/health +POST /api/login +GET /api/model-manage +GET /api/dataset-manage +GET /api/fine-tune +GET /api/compute/nodes ``` +本地运行时默认数据库路径： + +```text +LOCAL_DB_PATH=./runtime/platform.db +``` + +## 前端启动 + +```bash +cd frontend +npm install +npm run dev +``` + +前端开发服务默认运行在 `http://localhost:16801`，并通过 Vite proxy 将 `/api` 转发到 `http://localhost:17861`。 + +## 算力服务启动 + +```bash +cd compute +uvicorn api.main:app --reload --port 19100 +``` + +第一版默认 `COMPUTE_MODE=simulator`。后续真实 GPU 接入时，在算力服务器上部署 Compute API、Agent、File Gateway 和 LLaMA-Factory，应用平台通过 `compute_nodes.api_base_url` 和 `compute_nodes.file_gateway_url` 主动轮询。 + ## 日志 后端日志模块位于 `backend/app/core/logging.py`，说明文档见： @@ -84,22 +115,11 @@ logs/error-YYYY-MM-DD.log - `docs/backend-api-design.md`：FastAPI 接口分组、参数定义、权限说明。 - `docs/postgres-schema.sql`：PostgreSQL 数据库脚本，包含权限、用户中心、多租户、审批、审计等模型。 - `docs/system-development-plan.md`：多人协作开发计划，按前端、后端、DB、部署拆分。 +- `docs/first-version-development-plan.md`：第一版可演示闭环开发计划，覆盖前端、后端、DB、Compute Simulator、GPU 和 LLaMA-Factory 适配。 - `docs/backend-logging.md`：后端日志模块使用说明。 - `docs/deployment-plan.md`：后期部署方案，覆盖单机算力服务器部署与应用/算力分离部署。 -- `docs/demo-development-plan.md`：可演示 Demo 开发计划，覆盖前端、后端、DB、Compute Simulator、GPU 和 LLaMA-Factory 适配。 - `docker/README.md`：Docker 部署入口，包含应用服务器和算力服务器两套 Compose 使用方式。 -## 部署模式 - -平台支持两种主要部署模式： - -1. 所有服务部署在算力服务器：适合 PoC、内网试点、小团队单机多 GPU 使用。 -2. 应用服务和算力/训练服务独立部署：适合企业生产环境，应用平台部署在业务服务区，算力平台和 LLaMA-Factory 部署在 GPU 服务器。 - -生产环境建议采用第二种模式。算力平台与训练框架应部署在 GPU 算力服务器上，应用平台不直接控制 GPU 进程，而是通过内部 Compute API 调度训练任务。 - -详细方案见 `docs/deployment-plan.md`。 - ## Docker 部署入口 应用服务器： @@ -107,8 +127,7 @@ logs/error-YYYY-MM-DD.log ```bash cd docker/app cp .env.example .env -docker compose --profile build run --rm frontend-builder -docker compose up -d --build +docker compose up -d ``` 算力服务器： @@ -116,19 +135,10 @@ docker compose up -d --build ```bash cd docker/compute cp .env.example .env -docker compose up -d --build +docker compose up -d ``` -两套 Compose 均采用代码外挂方式运行，镜像只包含运行时环境和第三方依赖。详细说明见 `docker/README.md`。 - -项目根目录不再保留 `Dockerfile` 和 `docker-compose.yml`，部署时统一进入 `docker/app` 或 `docker/compute` 目录执行。 - -当前 Docker 约定： - -- PostgreSQL/Redis 开发阶段使用项目自带 Compose 服务，生产阶段保留切换企业统一基础设施的配置入口。 -- 算力服务器上的 LLaMA-Factory 使用宿主机目录挂载，默认 `/opt/LLaMA-Factory`。 -- 应用平台通过定时轮询 Compute API 同步训练状态，默认不要求算力服务器回调应用服务器。 -- 多算力节点阶段，每台单机多 GPU 服务器都独立部署 `docker/compute`、Compute API、Agent、File Gateway 和 LLaMA-Factory；节点之间默认不互访，由应用平台通过 `compute_nodes`、`resource_replicas` 和 `resource_sync_jobs` 统一调度与同步。 +两套 Compose 均采用代码外挂方式运行，镜像只包含运行时环境和第三方依赖。项目根目录不再保留 `Dockerfile` 和 `docker-compose.yml`，部署时统一进入 `docker/app` 或 `docker/compute` 目录执行。 ## 后续开发原则 diff --git a/backend/app/api/v1/endpoints/health.py b/backend/app/api/v1/endpoints/health.py index 72b27fe..c5676c1 100644 --- a/backend/app/api/v1/endpoints/health.py +++ b/backend/app/api/v1/endpoints/health.py @@ -1,12 +1,14 @@ -from fastapi import APIRouter +from fastapi import APIRouter from app.core.logging import get_logger +from app.db.platform_store import get_platform_store router = APIRouter() logger = get_logger(__name__) @router.get("/health") -async def health_check() -> dict[str, str]: +async def health_check() -> dict[str, object]: logger.info("health check requested") - return {"status": "ok"} + return {"code": 0, "message": "ok", "data": get_platform_store().health_metrics()} + diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py new file mode 100644 index 0000000..d7d260c --- /dev/null +++ b/backend/app/api/v1/endpoints/platform.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Body, File, HTTPException, Query, UploadFile +from fastapi.responses import PlainTextResponse + +from app.db.platform_store import get_platform_store + +router = APIRouter() + + +def ok(data: Any = None, message: str = "ok") -> dict[str, Any]: + return {"code": 0, "message": message, "data": data} + + +def fail(status_code: int, message: str) -> HTTPException: + return HTTPException(status_code=status_code, detail={"code": status_code, "message": message, "data": None}) + + +@router.post("/login") +async def login(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + user = get_platform_store().login(payload.get("username", ""), payload.get("password", "")) + if not user: + raise fail(401, "invalid username or password") + return ok({"token": f"platform-token-{user['id']}", "user": user}) + + +@router.get("/me") +async def me() -> dict[str, Any]: + return ok(get_platform_store().users()[0]) + + +@router.get("/dashboard/overview") +async def dashboard_overview() -> dict[str, Any]: + store = get_platform_store() + tasks = store.tasks() + return ok( + { + "models": len(store.models()), + "datasets": len(store.datasets()), + "fine_tune_tasks": len(tasks), + "running_tasks": len([t for t in tasks if t["status"] in {"syncing", "queued", "running"}]), + "compute_nodes": len(store.compute_nodes()), + "gpus": len(store.gpus()), + } + ) + + +@router.get("/system-info") +async def system_info() -> dict[str, Any]: + return ok(get_platform_store().system_info()) + + +@router.get("/users") +async def users() -> dict[str, Any]: + return ok(get_platform_store().users()) + + +@router.post("/users") +async def create_user(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok(get_platform_store().create_user(payload)) + + +@router.put("/users/{user_id}") +async def update_user(user_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_user(user_id, payload)) + except KeyError: + raise fail(404, "user not found") + + +@router.delete("/users/{user_id}") +async def delete_user(user_id: str, current_username: str | None = Query(default=None)) -> dict[str, Any]: + try: + get_platform_store().delete_user(user_id) + return ok({"deleted": user_id, "current_username": current_username}) + except KeyError: + raise fail(404, "user not found") + except ValueError as exc: + raise fail(400, str(exc)) + + +@router.get("/model-manage/local-models") +async def local_models() -> dict[str, Any]: + models = [{"path": item.get("path") or "", "name": item["name"]} for item in get_platform_store().models()] + return ok({"models": models}) + + +@router.get("/model-manage/trained-models") +async def trained_models() -> dict[str, Any]: + return ok({"models": get_platform_store().trained_models()}) + + +@router.delete("/model-manage/trained-models/{model_id}") +async def delete_trained_model(model_id: str, type: str = Query(default="merged")) -> dict[str, Any]: + return ok({"deleted": model_id, "type": type}) + + +@router.get("/model-manage/name/{name}") +async def model_by_name(name: str) -> dict[str, Any]: + try: + return ok(get_platform_store().model_by_name(name)) + except KeyError: + raise fail(404, "model not found") + + +@router.get("/model-manage") +async def model_list() -> dict[str, Any]: + return ok(get_platform_store().models()) + + +@router.post("/model-manage") +async def create_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok(get_platform_store().create_model(payload)) + + +@router.get("/model-manage/{model_id}") +async def model_detail(model_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().model(model_id)) + except KeyError: + raise fail(404, "model not found") + + +@router.put("/model-manage/{model_id}") +async def update_model(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_model(model_id, payload)) + except KeyError: + raise fail(404, "model not found") + + +@router.put("/model-manage/{model_id}/purpose") +async def update_model_purpose(model_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_model(model_id, {"purpose": payload.get("purpose", "training")})) + except KeyError: + raise fail(404, "model not found") + + +@router.delete("/model-manage/{model_id}") +async def delete_model(model_id: str) -> dict[str, Any]: + get_platform_store().delete_model(model_id) + return ok({"deleted": model_id}) + + +@router.post("/model-manage/merge") +async def merge_model(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"job_id": "merge-sample-job", "status": "queued", **payload}) + + +@router.get("/dataset-manage/preview/{file_id}") +async def dataset_preview(file_id: str) -> dict[str, Any]: + try: + row = get_platform_store().dataset_file(file_id) + return ok({"content": row["content"]}) + except KeyError: + raise fail(404, "dataset file not found") + + +@router.get("/dataset-manage/versions/{file_id}") +async def dataset_versions(file_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().file_versions(file_id)) + except KeyError: + raise fail(404, "dataset file not found") + + +@router.get("/dataset-manage/versions/{file_id}/{version_id}") +async def dataset_version_content(file_id: str, version_id: str) -> dict[str, Any]: + try: + row = get_platform_store().dataset_file(file_id) + versions = get_platform_store().file_versions(file_id)["versions"] + version = next((item for item in versions if item["id"] == version_id), None) + if not version: + raise KeyError(version_id) + return ok({"version": version, "content": row["content"]}) + except KeyError: + raise fail(404, "dataset version not found") + + +@router.post("/dataset-manage/versions/{file_id}") +async def create_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().create_file_version(file_id, payload)) + except KeyError: + raise fail(404, "dataset file not found") + + +@router.put("/dataset-manage/versions/{file_id}/active") +async def activate_dataset_version(file_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().activate_file_version(file_id, payload["version_id"])) + except KeyError: + raise fail(404, "dataset version not found") + + +@router.delete("/dataset-manage/versions/{file_id}/{version_id}") +async def delete_dataset_version(file_id: str, version_id: str) -> dict[str, Any]: + return ok(get_platform_store().file_versions(file_id)) + + +@router.post("/dataset-manage/upload/{dataset_id}") +async def upload_dataset_files(dataset_id: str, files: list[UploadFile] = File(default=[])) -> dict[str, Any]: + created: list[dict[str, Any]] = [] + store = get_platform_store() + try: + store.dataset(dataset_id) + except KeyError: + raise fail(404, "dataset not found") + with store.connect() as conn: + for file in files: + raw = await file.read() + content = raw.decode("utf-8", errors="replace") + created.append(store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)) + return ok({"files": created}) + + +@router.get("/dataset-manage/download/{dataset_id}") +async def download_dataset(dataset_id: str) -> PlainTextResponse: + dataset = get_platform_store().dataset(dataset_id) + content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])]) + return PlainTextResponse(content, media_type="text/plain") + + +@router.get("/dataset-manage/download/{dataset_id}/{file_id}") +async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse: + row = get_platform_store().dataset_file(file_id) + return PlainTextResponse(row["content"], media_type="text/plain") + + +@router.get("/dataset-manage") +async def dataset_list() -> dict[str, Any]: + return ok(get_platform_store().datasets()) + + +@router.post("/dataset-manage") +async def create_dataset(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + dataset = get_platform_store().create_dataset(payload) + return ok({"id": dataset["id"]}) + + +@router.get("/dataset-manage/{dataset_id}") +async def dataset_detail(dataset_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().dataset(dataset_id)) + except KeyError: + raise fail(404, "dataset not found") + + +@router.put("/dataset-manage/{dataset_id}") +async def update_dataset(dataset_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_dataset(dataset_id, payload)) + except KeyError: + raise fail(404, "dataset not found") + + +@router.delete("/dataset-manage/{dataset_id}") +async def delete_dataset(dataset_id: str) -> dict[str, Any]: + get_platform_store().delete_dataset(dataset_id) + return ok({"deleted": dataset_id}) + + +@router.get("/fine-tune/check-name") +async def check_fine_tune_name(name: str = Query(...)) -> dict[str, Any]: + exists = any(task["name"] == name for task in get_platform_store().tasks()) + return ok({"exists": exists}) + + +@router.get("/fine-tune/progress/{task_id}") +async def fine_tune_progress(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().progress(task_id)) + except KeyError: + raise fail(404, "fine tune task not found") + + +@router.post("/fine-tune/tensorboard/start") +async def tensorboard_start() -> dict[str, Any]: + return ok({"status": "running", "url": "http://localhost:6006"}) + + +@router.get("/fine-tune") +async def fine_tune_list() -> dict[str, Any]: + return ok(get_platform_store().tasks()) + + +@router.post("/fine-tune") +async def create_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + task = get_platform_store().create_task(payload) + return ok({"id": task["id"]}) + + +@router.post("/fine-tune/start") +async def start_fine_tune(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().start_task(payload)) + except KeyError: + raise fail(404, "fine tune task not found") + except RuntimeError as exc: + raise fail(409, str(exc)) + + +@router.get("/fine-tune/{task_id}") +async def fine_tune_detail(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().task(task_id)) + except KeyError: + raise fail(404, "fine tune task not found") + + +@router.put("/fine-tune/{task_id}") +async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_task(task_id, payload)) + except KeyError: + raise fail(404, "fine tune task not found") + + +@router.post("/fine-tune/stop/{task_id}") +async def stop_fine_tune(task_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().stop_task(task_id)) + except KeyError: + raise fail(404, "fine tune task not found") + + +@router.post("/fine-tune/{task_id}/stop") +async def stop_fine_tune_alt(task_id: str) -> dict[str, Any]: + return await stop_fine_tune(task_id) + + +@router.delete("/fine-tune/{task_id}") +async def delete_fine_tune(task_id: str) -> dict[str, Any]: + get_platform_store().delete_task(task_id) + return ok({"deleted": task_id}) + + +@router.get("/fine-tune/{task_id}/overview") +async def fine_tune_overview(task_id: str) -> dict[str, Any]: + task = get_platform_store().task(task_id) + return ok({"task": task, "progress": get_platform_store().progress(task_id)}) + + +@router.get("/fine-tune/{task_id}/checkpoints") +async def fine_tune_checkpoints(task_id: str) -> dict[str, Any]: + task = get_platform_store().task(task_id) + checkpoints = [] + for step in [50, 100, 150]: + if task.get("progress", 0) >= min(100, step // 2): + checkpoints.append({"step": step, "path": f"/data/yg-ft/outputs/{task['name']}/checkpoint-{step}"}) + return ok(checkpoints) + + +@router.get("/compute/nodes") +async def compute_nodes() -> dict[str, Any]: + return ok(get_platform_store().compute_nodes()) + + +@router.post("/compute/nodes") +async def create_compute_node(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"id": "node_sample_new", **payload, "status": "created"}) + + +@router.put("/compute/nodes/{node_id}") +async def update_compute_node(node_id: str, payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().update_compute_node(node_id, payload)) + except KeyError: + raise fail(404, "compute node not found") + + +@router.post("/compute/nodes/{node_id}/test-connection") +async def test_compute_node(node_id: str) -> dict[str, Any]: + return ok({"node_id": node_id, "success": True, "latency_ms": 12}) + + +@router.post("/compute/nodes/{node_id}/enable") +async def enable_compute_node(node_id: str) -> dict[str, Any]: + return ok(get_platform_store().update_compute_node(node_id, {"enabled": True, "scheduler_status": "online"})) + + +@router.post("/compute/nodes/{node_id}/disable") +async def disable_compute_node(node_id: str) -> dict[str, Any]: + return ok(get_platform_store().update_compute_node(node_id, {"enabled": False, "scheduler_status": "offline"})) + + +@router.post("/compute/nodes/{node_id}/drain") +async def drain_compute_node(node_id: str) -> dict[str, Any]: + return ok(get_platform_store().update_compute_node(node_id, {"scheduler_status": "draining"})) + + +@router.get("/compute/nodes/{node_id}/replicas") +async def compute_node_replicas(node_id: str) -> dict[str, Any]: + return ok(get_platform_store().replicas(node_id)) + + +@router.get("/compute/gpus") +async def compute_gpus() -> dict[str, Any]: + return ok(get_platform_store().gpus()) + + +@router.get("/compute/queue") +async def compute_queue() -> dict[str, Any]: + return ok(get_platform_store().queue()) + + +@router.post("/internal/compute-sync/resources") +async def create_compute_sync(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + sync_id = get_platform_store().create_sync_job(payload.get("target_node_id", "node_01"), payload) + return ok(get_platform_store().sync_job(sync_id)) + + +@router.get("/internal/compute-sync/resources/{sync_id}") +async def compute_sync_detail(sync_id: str) -> dict[str, Any]: + try: + return ok(get_platform_store().sync_job(sync_id)) + except KeyError: + raise fail(404, "sync job not found") + + +@router.get("/training-log-files") +async def training_log_files() -> dict[str, Any]: + return ok(get_platform_store().training_log_files()) + + +@router.get("/training-log-content") +async def training_log_content(file: str = Query(...)) -> dict[str, Any]: + try: + return ok(get_platform_store().training_log_content(file)) + except KeyError: + raise fail(404, "training log not found") + + +@router.get("/log-files") +async def log_files(date: str | None = Query(default=None)) -> dict[str, Any]: + return ok(get_platform_store().log_files(date)) + + +@router.get("/log-content") +async def log_content(file: str = Query(...)) -> dict[str, Any]: + return ok(get_platform_store().log_content(file)) + + +@router.post("/web-log") +async def web_log(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + return ok({"received": True, **payload}) + diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index 7d5f9d2..5a4dbcb 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -1,6 +1,9 @@ -from fastapi import APIRouter +from fastapi import APIRouter +from app.api.v1.endpoints.platform import router as platform_router from app.api.v1.endpoints.health import router as health_router api_router = APIRouter() api_router.include_router(health_router, tags=["health"]) +api_router.include_router(platform_router, tags=["platform"]) + diff --git a/backend/app/core/config.py b/backend/app/core/config.py index f5457ba..c6f6703 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass from functools import lru_cache import os @@ -15,6 +15,11 @@ class Settings: app_name: str = os.getenv("APP_NAME", "YG Fine-Tune Platform API") app_env: str = os.getenv("APP_ENV", "local") api_prefix: str = os.getenv("API_PREFIX", "/api") + app_mode: str = os.getenv("APP_MODE", "local") + local_db_path: str = os.getenv("LOCAL_DB_PATH", "./runtime/platform.db") + compute_mode: str = os.getenv("COMPUTE_MODE", "simulator") + compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling") + compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3) log_level: str = os.getenv("LOG_LEVEL", "INFO") log_dir: str = os.getenv("LOG_DIR", "./logs") log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend") @@ -26,3 +31,4 @@ class Settings: @lru_cache def get_settings() -> Settings: return Settings() + diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py new file mode 100644 index 0000000..092aa16 --- /dev/null +++ b/backend/app/db/platform_store.py @@ -0,0 +1,1486 @@ +from __future__ import annotations + +import json +import math +import sqlite3 +import time +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterator + +from app.core.config import get_settings + + +ALL_PERMISSIONS = [ + "dashboard", + "fine-tune", + "model-eval", + "model-inference", + "model-manage", + "dataset", + "data-process", + "data-convert", + "compute", + "hardware", + "logs", + "user-settings", +] + + +def utcnow() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def parse_time(value: str | None) -> datetime | None: + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def json_loads(value: str | None, default: Any) -> Any: + if not value: + return default + return json.loads(value) + + +def json_dumps(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def new_id(prefix: str) -> str: + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +class PlatformStore: + """Small SQLite-backed store for the first runnable platform version. + + The production model is PostgreSQL. This store mirrors the API-facing subset + needed by the first system iteration so developers can run the app without + provisioning enterprise infrastructure first. + """ + + def __init__(self, db_path: str | None = None) -> None: + settings = get_settings() + raw_path = db_path or settings.local_db_path + self.db_path = Path(raw_path) + if not self.db_path.is_absolute(): + self.db_path = Path.cwd() / self.db_path + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.ensure_schema() + self.ensure_seed_data() + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def ensure_schema(self) -> None: + with self.connect() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + password TEXT NOT NULL, + display_name TEXT NOT NULL, + role TEXT NOT NULL, + status TEXT NOT NULL, + permissions TEXT NOT NULL, + create_time TEXT NOT NULL, + last_login TEXT, + protected INTEGER NOT NULL DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS models ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + type TEXT NOT NULL, + purpose TEXT NOT NULL, + model_source TEXT NOT NULL, + description TEXT, + path TEXT, + api_url TEXT, + api_key TEXT, + online_model_name TEXT, + create_time TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS trained_models ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + train_methods TEXT NOT NULL, + base_model_path TEXT, + create_time TEXT NOT NULL, + merged INTEGER NOT NULL DEFAULT 0, + merging INTEGER NOT NULL DEFAULT 0, + merged_path TEXT + ); + + CREATE TABLE IF NOT EXISTS datasets ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + type TEXT NOT NULL, + storage_type TEXT NOT NULL, + source TEXT NOT NULL, + task_id TEXT, + size TEXT, + count INTEGER NOT NULL DEFAULT 0, + description TEXT, + create_time TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS dataset_files ( + id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL, + name TEXT NOT NULL, + size TEXT, + content TEXT NOT NULL, + active_version_id TEXT NOT NULL, + versions TEXT NOT NULL, + create_time TEXT NOT NULL, + FOREIGN KEY(dataset_id) REFERENCES datasets(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS compute_nodes ( + id TEXT PRIMARY KEY, + code TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + api_base_url TEXT NOT NULL, + file_gateway_url TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + scheduler_status TEXT NOT NULL, + scheduler_weight INTEGER NOT NULL DEFAULT 100, + tags TEXT NOT NULL, + gpu_count INTEGER NOT NULL DEFAULT 0, + current_running_jobs INTEGER NOT NULL DEFAULT 0, + max_parallel_jobs INTEGER NOT NULL DEFAULT 2, + data_root TEXT NOT NULL, + model_root TEXT NOT NULL, + log_root TEXT NOT NULL, + last_health_check_at TEXT, + health_detail TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS gpus ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + gpu_index INTEGER NOT NULL, + uuid TEXT NOT NULL, + name TEXT NOT NULL, + memory_total_gb REAL NOT NULL, + power_limit_w REAL NOT NULL, + base_temperature INTEGER NOT NULL, + FOREIGN KEY(node_id) REFERENCES compute_nodes(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS fine_tune_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + payload TEXT NOT NULL, + status TEXT NOT NULL, + progress INTEGER NOT NULL DEFAULT 0, + process_id INTEGER, + create_time TEXT NOT NULL, + start_time TEXT, + completed_at TEXT, + compute_node_id TEXT, + gpus TEXT NOT NULL, + sync_job_id TEXT + ); + + CREATE TABLE IF NOT EXISTS resource_replicas ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + local_path TEXT NOT NULL, + status TEXT NOT NULL, + sync_status TEXT NOT NULL, + create_time TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS resource_sync_jobs ( + id TEXT PRIMARY KEY, + target_node_id TEXT NOT NULL, + resources TEXT NOT NULL, + status TEXT NOT NULL, + progress INTEGER NOT NULL DEFAULT 0, + create_time TEXT NOT NULL, + completed_at TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_fine_tune_status ON fine_tune_tasks(status); + CREATE INDEX IF NOT EXISTS idx_dataset_files_dataset ON dataset_files(dataset_id); + CREATE INDEX IF NOT EXISTS idx_gpus_node ON gpus(node_id); + CREATE INDEX IF NOT EXISTS idx_replicas_resource ON resource_replicas(resource_type, resource_id); + """ + ) + + def ensure_seed_data(self) -> None: + with self.connect() as conn: + if conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] > 0: + return + + now = utcnow() + users = [ + ("u_admin", "admin", "admin123", "Platform Admin", "admin", "active", ALL_PERMISSIONS, 1), + ( + "u_operator", + "operator", + "operator123", + "Platform Operator", + "operator", + "active", + [p for p in ALL_PERMISSIONS if p != "user-settings"], + 0, + ), + ] + conn.executemany( + """ + INSERT INTO users + (id, username, password, display_name, role, status, permissions, create_time, protected) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [(u[0], u[1], u[2], u[3], u[4], u[5], json_dumps(u[6]), now, u[7]) for u in users], + ) + + models = [ + ( + "m_qwen25_7b", + "Qwen2.5-7B-Instruct", + "LLM", + "training", + "local", + "Sample base model for SFT and LoRA training.", + "/models/Qwen2.5-7B-Instruct", + ), + ( + "m_llama31_8b", + "Llama-3.1-8B-Instruct", + "LLM", + "training", + "local", + "Reserved base model path for LLaMA-Factory dry-run.", + "/models/Llama-3.1-8B-Instruct", + ), + ] + conn.executemany( + """ + INSERT INTO models + (id, name, type, purpose, model_source, description, path, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + [(*m, now) for m in models], + ) + + self._insert_dataset( + conn, + "ds_finance_sft", + "finance-sft-sample", + "train", + "Financial QA SFT samples for sample training.", + "finance-sft-sample.jsonl", + "\n".join( + [ + '{"instruction":"Summarize revenue growth.","input":"Revenue grew from 10M to 13M.","output":"Revenue increased 30% year over year."}', + '{"instruction":"Classify risk.","input":"Customer has overdue payment for 90 days.","output":"High credit risk."}', + '{"instruction":"Draft an analyst note.","input":"Gross margin improved by 4 points.","output":"Margin expansion indicates stronger operating leverage."}', + ] + ), + now, + ) + self._insert_dataset( + conn, + "ds_customer_service", + "customer-service-sample", + "train", + "Customer-service instruction tuning samples.", + "customer-service-sample.jsonl", + "\n".join( + [ + '{"instruction":"Respond politely.","input":"My package is late.","output":"I am sorry for the delay. I can help check the latest shipment status."}', + '{"instruction":"Escalate request.","input":"I need a refund for a defective item.","output":"I will create a refund case and share the next steps."}', + ] + ), + now, + ) + + node_rows = [ + ( + "node_01", + "gpu-node-01", + "GPU Node 01", + "http://gpu-node-01:19100", + "http://gpu-node-01:19101", + 1, + "online", + 100, + ["A800", "80GB", "llama_factory"], + 4, + 0, + 2, + "/data/yg-ft", + "/models", + "/data/yg-ft/training-logs", + now, + {"mode": "simulator", "heartbeat": "ok"}, + ), + ( + "node_02", + "gpu-node-02", + "GPU Node 02", + "http://gpu-node-02:19100", + "http://gpu-node-02:19101", + 1, + "online", + 60, + ["4090", "24GB", "llama_factory"], + 4, + 0, + 1, + "/data/yg-ft", + "/models", + "/data/yg-ft/training-logs", + now, + {"mode": "simulator", "heartbeat": "ok"}, + ), + ] + conn.executemany( + """ + INSERT INTO compute_nodes + (id, code, name, api_base_url, file_gateway_url, enabled, scheduler_status, + scheduler_weight, tags, gpu_count, current_running_jobs, max_parallel_jobs, + data_root, model_root, log_root, last_health_check_at, health_detail) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + r[0], + r[1], + r[2], + r[3], + r[4], + r[5], + r[6], + r[7], + json_dumps(r[8]), + r[9], + r[10], + r[11], + r[12], + r[13], + r[14], + r[15], + json_dumps(r[16]), + ) + for r in node_rows + ], + ) + for node_id, model_name, mem in [ + ("node_01", "NVIDIA A800-SXM4-80GB", 80), + ("node_02", "NVIDIA GeForce RTX 4090", 24), + ]: + for idx in range(4): + conn.execute( + """ + INSERT INTO gpus + (id, node_id, gpu_index, uuid, name, memory_total_gb, power_limit_w, base_temperature) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + f"{node_id}_gpu_{idx}", + node_id, + idx, + f"GPU-{node_id.upper()}-{idx}", + model_name, + mem, + 300 if mem >= 80 else 450, + 35 + idx, + ), + ) + + self._insert_task( + conn, + task_id="ft_pending_sample", + name="pending-sft-sample", + status="pending", + progress=0, + create_time=now, + start_time=None, + completed_at=None, + node_id=None, + gpus=[], + process_id=None, + ) + started = datetime.now(timezone.utc).timestamp() - 22 + self._insert_task( + conn, + task_id="ft_running_sample", + name="running-sft-sample", + status="running", + progress=45, + create_time=now, + start_time=datetime.fromtimestamp(started, timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + completed_at=None, + node_id="node_01", + gpus=[0, 1], + process_id=42001, + ) + self._insert_task( + conn, + task_id="ft_completed_sample", + name="completed-sft-sample", + status="completed", + progress=100, + create_time=now, + start_time=now, + completed_at=now, + node_id="node_01", + gpus=[2], + process_id=42002, + ) + conn.execute( + """ + INSERT INTO trained_models + (id, name, train_methods, base_model_path, create_time, merged, merging, merged_path) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "tm_completed_sample", + "completed-sft-sample-lora", + json_dumps([{"name": "lora"}]), + "/models/Qwen2.5-7B-Instruct", + now, + 0, + 0, + "/data/yg-ft/outputs/completed-sft-sample/adapter", + ), + ) + + def _insert_dataset( + self, + conn: sqlite3.Connection, + dataset_id: str, + name: str, + dataset_type: str, + description: str, + file_name: str, + content: str, + now: str, + ) -> None: + lines = [line for line in content.splitlines() if line.strip()] + size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB" + file_id = f"{dataset_id}_file_1" + version_id = f"{file_id}_v1" + conn.execute( + """ + INSERT INTO datasets + (id, name, type, storage_type, source, size, count, description, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (dataset_id, name, dataset_type, "local", "upload", size, len(lines), description, now), + ) + conn.execute( + """ + INSERT INTO dataset_files + (id, dataset_id, name, size, content, active_version_id, versions, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + file_id, + dataset_id, + file_name, + size, + content, + version_id, + json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "initial seed"}]), + now, + ), + ) + + def _insert_task( + self, + conn: sqlite3.Connection, + task_id: str, + name: str, + status: str, + progress: int, + create_time: str, + start_time: str | None, + completed_at: str | None, + node_id: str | None, + gpus: list[int], + process_id: int | None, + ) -> None: + payload = { + "id": task_id, + "name": name, + "description": "Seeded sample fine-tune task.", + "status": status, + "train_type": "SFT", + "train_method": "lora", + "template": "qwen", + "base_model": "m_qwen25_7b", + "train_dataset_id": "ds_finance_sft", + "auto_merge": False, + "output_model_name": f"{name}-lora", + "gpus": gpus, + "batch_size": 2, + "learning_rate": 0.0002, + "n_epochs": 3, + "save_steps": 50, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "lora_alpha": 16, + "lora_dropout": 0.05, + "lora_rank": 8, + "quantization_bit": 4, + "export_quantized": False, + "quant_method": "bnb", + "quant_bits": 4, + "quant_group_size": 128, + "export_format": "safetensors", + "progress": progress, + "process_id": process_id, + "train_duration": self._duration(start_time, completed_at), + "create_time": create_time, + } + conn.execute( + """ + INSERT INTO fine_tune_tasks + (id, name, payload, status, progress, process_id, create_time, start_time, completed_at, compute_node_id, gpus) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + task_id, + name, + json_dumps(payload), + status, + progress, + process_id, + create_time, + start_time, + completed_at, + node_id, + json_dumps(gpus), + ), + ) + + def _duration(self, start_time: str | None, end_time: str | None = None) -> str: + start = parse_time(start_time) + if not start: + return "" + end = parse_time(end_time) or datetime.now(timezone.utc) + seconds = max(0, int((end - start).total_seconds())) + minutes, sec = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + if hours: + return f"{hours}h {minutes}m {sec}s" + if minutes: + return f"{minutes}m {sec}s" + return f"{sec}s" + + def refresh_runtime_state(self) -> None: + with self.connect() as conn: + rows = conn.execute( + "SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')" + ).fetchall() + now_dt = datetime.now(timezone.utc) + for row in rows: + start = parse_time(row["start_time"]) + if not start: + continue + age = max(0, int((now_dt - start).total_seconds())) + if age < 4: + status, progress = "syncing", 8 + age + elif age < 8: + status, progress = "queued", 18 + age + elif age < 70: + status = "running" + progress = min(96, 25 + int((age - 8) / 62 * 70)) + else: + status, progress = "completed", 100 + + payload = json_loads(row["payload"], {}) + payload.update( + { + "status": status, + "progress": progress, + "train_duration": self._duration(row["start_time"], utcnow() if status == "completed" else None), + } + ) + completed_at = row["completed_at"] or (utcnow() if status == "completed" else None) + conn.execute( + """ + UPDATE fine_tune_tasks + SET status=?, progress=?, payload=?, completed_at=? + WHERE id=? + """, + (status, progress, json_dumps(payload), completed_at, row["id"]), + ) + if status == "completed": + self._ensure_trained_model(conn, payload) + + sync_rows = conn.execute( + "SELECT * FROM resource_sync_jobs WHERE status IN ('pending','running')" + ).fetchall() + for row in sync_rows: + created = parse_time(row["create_time"]) + age = int((now_dt - created).total_seconds()) if created else 0 + status = "completed" if age >= 6 else "running" + progress = 100 if status == "completed" else min(95, 15 + age * 12) + completed_at = row["completed_at"] or (utcnow() if status == "completed" else None) + conn.execute( + "UPDATE resource_sync_jobs SET status=?, progress=?, completed_at=? WHERE id=?", + (status, progress, completed_at, row["id"]), + ) + + def _ensure_trained_model(self, conn: sqlite3.Connection, task: dict[str, Any]) -> None: + name = task.get("output_model_name") or f"{task['name']}-lora" + exists = conn.execute("SELECT id FROM trained_models WHERE name=?", (name,)).fetchone() + if exists: + return + model = conn.execute("SELECT path FROM models WHERE id=?", (task.get("base_model"),)).fetchone() + conn.execute( + """ + INSERT INTO trained_models + (id, name, train_methods, base_model_path, create_time, merged, merging, merged_path) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + new_id("tm"), + name, + json_dumps([{"name": task.get("train_method", "lora")}]), + model["path"] if model else "", + utcnow(), + 0, + 0, + f"/data/yg-ft/outputs/{task['name']}/adapter", + ), + ) + + def users(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute("SELECT * FROM users ORDER BY create_time").fetchall() + return [self._user(row) for row in rows] + + def login(self, username: str, password: str) -> dict[str, Any] | None: + with self.connect() as conn: + row = conn.execute("SELECT * FROM users WHERE username=?", (username,)).fetchone() + if not row or row["password"] != password or row["status"] != "active": + return None + last_login = utcnow() + conn.execute("UPDATE users SET last_login=? WHERE id=?", (last_login, row["id"])) + data = self._user(row) + data["last_login"] = last_login + return data + + def create_user(self, payload: dict[str, Any]) -> dict[str, Any]: + user_id = new_id("u") + permissions = payload.get("permissions") or (ALL_PERMISSIONS if payload.get("role") == "admin" else ["dashboard"]) + with self.connect() as conn: + conn.execute( + """ + INSERT INTO users + (id, username, password, display_name, role, status, permissions, create_time, protected) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + user_id, + payload["username"], + payload.get("password", "platform123"), + payload.get("display_name") or payload["username"], + payload.get("role", "viewer"), + payload.get("status", "active"), + json_dumps(permissions), + utcnow(), + ), + ) + return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()) + + def update_user(self, user_id: str, payload: dict[str, Any]) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone() + if not row: + raise KeyError(user_id) + values = { + "role": payload.get("role", row["role"]), + "status": payload.get("status", row["status"]), + "permissions": json_dumps(payload.get("permissions", json_loads(row["permissions"], []))), + } + conn.execute( + "UPDATE users SET role=?, status=?, permissions=? WHERE id=?", + (values["role"], values["status"], values["permissions"], user_id), + ) + return self._user(conn.execute("SELECT * FROM users WHERE id=?", (user_id,)).fetchone()) + + def delete_user(self, user_id: str) -> None: + with self.connect() as conn: + row = conn.execute("SELECT protected FROM users WHERE id=?", (user_id,)).fetchone() + if not row: + raise KeyError(user_id) + if row["protected"]: + raise ValueError("protected user cannot be deleted") + conn.execute("DELETE FROM users WHERE id=?", (user_id,)) + + def _user(self, row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "role": row["role"], + "status": row["status"], + "permissions": json_loads(row["permissions"], []), + "create_time": row["create_time"], + "last_login": row["last_login"], + "protected": bool(row["protected"]), + } + + def models(self) -> list[dict[str, Any]]: + with self.connect() as conn: + return [dict(row) for row in conn.execute("SELECT * FROM models ORDER BY create_time DESC").fetchall()] + + def model(self, model_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM models WHERE id=?", (model_id,)).fetchone() + if not row: + raise KeyError(model_id) + return dict(row) + + def model_by_name(self, name: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM models WHERE name=?", (name,)).fetchone() + if not row: + raise KeyError(name) + return dict(row) + + def create_model(self, payload: dict[str, Any]) -> dict[str, Any]: + model_id = payload.get("id") or new_id("m") + with self.connect() as conn: + conn.execute( + """ + INSERT INTO models + (id, name, type, purpose, model_source, description, path, api_url, api_key, online_model_name, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + model_id, + payload["name"], + payload.get("type", "LLM"), + payload.get("purpose", "training"), + payload.get("model_source", "local"), + payload.get("description"), + payload.get("path"), + payload.get("api_url"), + payload.get("api_key"), + payload.get("online_model_name"), + utcnow(), + ), + ) + return self.model(model_id) + + def update_model(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = self.model(model_id) + merged = {**current, **payload} + with self.connect() as conn: + conn.execute( + """ + UPDATE models + SET name=?, type=?, purpose=?, model_source=?, description=?, path=?, api_url=?, api_key=?, online_model_name=? + WHERE id=? + """, + ( + merged["name"], + merged.get("type", "LLM"), + merged.get("purpose", "training"), + merged.get("model_source", "local"), + merged.get("description"), + merged.get("path"), + merged.get("api_url"), + merged.get("api_key"), + merged.get("online_model_name"), + model_id, + ), + ) + return self.model(model_id) + + def delete_model(self, model_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM models WHERE id=?", (model_id,)) + + def trained_models(self) -> list[dict[str, Any]]: + self.refresh_runtime_state() + with self.connect() as conn: + rows = conn.execute("SELECT * FROM trained_models ORDER BY create_time DESC").fetchall() + return [ + { + **dict(row), + "train_methods": json_loads(row["train_methods"], []), + "merged": bool(row["merged"]), + "merging": bool(row["merging"]), + } + for row in rows + ] + + def datasets(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute("SELECT * FROM datasets ORDER BY create_time DESC").fetchall() + return [self._dataset(conn, row) for row in rows] + + def dataset(self, dataset_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone() + if not row: + raise KeyError(dataset_id) + return self._dataset(conn, row) + + def _dataset(self, conn: sqlite3.Connection, row: sqlite3.Row) -> dict[str, Any]: + files = conn.execute( + "SELECT id, name, size, active_version_id, create_time FROM dataset_files WHERE dataset_id=? ORDER BY create_time", + (row["id"],), + ).fetchall() + return { + **dict(row), + "files": [ + { + "id": f["id"], + "name": f["name"], + "size": f["size"], + "active_version_id": f["active_version_id"], + "create_time": f["create_time"], + } + for f in files + ], + } + + def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]: + dataset_id = payload.get("id") or new_id("ds") + with self.connect() as conn: + conn.execute( + """ + INSERT INTO datasets + (id, name, type, storage_type, source, task_id, size, count, description, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + dataset_id, + payload["name"], + payload.get("type", "train"), + payload.get("storage_type", "local"), + payload.get("source", "upload"), + payload.get("task_id"), + payload.get("size", "0 KB"), + payload.get("count", 0), + payload.get("description"), + utcnow(), + ), + ) + if not payload.get("files"): + content = '{"instruction":"Sample instruction","input":"Sample input","output":"Sample output"}' + self.add_dataset_file(conn, dataset_id, "sample.jsonl", content) + return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()) + + def update_dataset(self, dataset_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = self.dataset(dataset_id) + merged = {**current, **payload} + with self.connect() as conn: + conn.execute( + """ + UPDATE datasets + SET name=?, type=?, storage_type=?, source=?, task_id=?, size=?, count=?, description=? + WHERE id=? + """, + ( + merged["name"], + merged.get("type", "train"), + merged.get("storage_type", "local"), + merged.get("source", "upload"), + merged.get("task_id"), + merged.get("size", "0 KB"), + merged.get("count", 0), + merged.get("description"), + dataset_id, + ), + ) + return self._dataset(conn, conn.execute("SELECT * FROM datasets WHERE id=?", (dataset_id,)).fetchone()) + + def delete_dataset(self, dataset_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM dataset_files WHERE dataset_id=?", (dataset_id,)) + conn.execute("DELETE FROM datasets WHERE id=?", (dataset_id,)) + + def add_dataset_file(self, conn: sqlite3.Connection, dataset_id: str, name: str, content: str) -> dict[str, Any]: + now = utcnow() + file_id = new_id("file") + version_id = f"{file_id}_v1" + size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB" + conn.execute( + """ + INSERT INTO dataset_files + (id, dataset_id, name, size, content, active_version_id, versions, create_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + file_id, + dataset_id, + name, + size, + content, + version_id, + json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]), + now, + ), + ) + count = len([line for line in content.splitlines() if line.strip()]) + conn.execute( + "UPDATE datasets SET count=count+?, size=? WHERE id=?", + (count, size, dataset_id), + ) + return {"id": file_id, "name": name, "size": size} + + def dataset_file(self, file_id: str) -> sqlite3.Row: + with self.connect() as conn: + row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone() + if not row: + raise KeyError(file_id) + return row + + def file_versions(self, file_id: str) -> dict[str, Any]: + row = self.dataset_file(file_id) + versions = json_loads(row["versions"], []) + return { + "versions": versions, + "active_version_id": row["active_version_id"], + "next_version_number": len(versions) + 1, + } + + def create_file_version(self, file_id: str, payload: dict[str, Any]) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone() + if not row: + raise KeyError(file_id) + versions = json_loads(row["versions"], []) + version = { + "id": f"{file_id}_v{len(versions) + 1}", + "version": len(versions) + 1, + "create_time": utcnow(), + "description": payload.get("description", "online edit"), + } + versions.append(version) + conn.execute( + "UPDATE dataset_files SET content=?, active_version_id=?, versions=? WHERE id=?", + (payload.get("content", ""), version["id"], json_dumps(versions), file_id), + ) + return {"version": version, "content": payload.get("content", "")} + + def activate_file_version(self, file_id: str, version_id: str) -> dict[str, Any]: + with self.connect() as conn: + row = conn.execute("SELECT * FROM dataset_files WHERE id=?", (file_id,)).fetchone() + if not row: + raise KeyError(file_id) + versions = json_loads(row["versions"], []) + version = next((item for item in versions if item["id"] == version_id), None) + if not version: + raise KeyError(version_id) + conn.execute("UPDATE dataset_files SET active_version_id=? WHERE id=?", (version_id, file_id)) + return {"version": version, "content": row["content"]} + + def tasks(self) -> list[dict[str, Any]]: + self.refresh_runtime_state() + with self.connect() as conn: + rows = conn.execute("SELECT * FROM fine_tune_tasks ORDER BY create_time DESC").fetchall() + return [self._task(row) for row in rows] + + def task(self, task_id: str) -> dict[str, Any]: + self.refresh_runtime_state() + with self.connect() as conn: + row = conn.execute("SELECT * FROM fine_tune_tasks WHERE id=?", (task_id,)).fetchone() + if not row: + raise KeyError(task_id) + return self._task(row) + + def _task(self, row: sqlite3.Row) -> dict[str, Any]: + payload = json_loads(row["payload"], {}) + payload.update( + { + "id": row["id"], + "status": row["status"], + "progress": row["progress"], + "process_id": row["process_id"], + "create_time": row["create_time"], + "gpus": json_loads(row["gpus"], payload.get("gpus", [])), + "train_duration": self._duration(row["start_time"], row["completed_at"]) if row["start_time"] else "", + "compute_node_id": row["compute_node_id"], + "sync_job_id": row["sync_job_id"], + } + ) + return payload + + def create_task(self, payload: dict[str, Any]) -> dict[str, Any]: + task_id = str(payload.get("task_id") or payload.get("id") or new_id("ft")) + name = payload.get("name") or f"fine-tune-{task_id[-6:]}" + now = utcnow() + task = { + "id": task_id, + "name": name, + "description": payload.get("description", ""), + "status": "pending", + "train_type": payload.get("train_type", "SFT"), + "train_method": payload.get("train_method", "lora"), + "template": payload.get("template", "qwen"), + "base_model": payload.get("base_model") or payload.get("base_model_id") or "m_qwen25_7b", + "train_dataset_id": payload.get("train_dataset_id") or "ds_finance_sft", + "auto_merge": bool(payload.get("auto_merge", False)), + "output_model_name": payload.get("output_model_name") or f"{name}-lora", + "gpus": payload.get("gpus") or [], + "batch_size": payload.get("batch_size", 2), + "learning_rate": payload.get("learning_rate", 0.0002), + "n_epochs": payload.get("n_epochs", 3), + "save_steps": payload.get("save_steps", 50), + "lr_scheduler_type": payload.get("lr_scheduler_type", "cosine"), + "max_length": payload.get("max_length", 2048), + "warmup_ratio": payload.get("warmup_ratio", 0.03), + "weight_decay": payload.get("weight_decay", 0.01), + "lora_alpha": payload.get("lora_alpha", 16), + "lora_dropout": payload.get("lora_dropout", 0.05), + "lora_rank": payload.get("lora_rank", 8), + "quantization_bit": payload.get("quantization_bit", 4), + "export_quantized": bool(payload.get("export_quantized", False)), + "quant_method": payload.get("quant_method", "bnb"), + "quant_bits": payload.get("quant_bits", 4), + "quant_group_size": payload.get("quant_group_size", 128), + "export_format": payload.get("export_format", "safetensors"), + "progress": 0, + "process_id": None, + "train_duration": "", + "create_time": now, + } + with self.connect() as conn: + conn.execute( + """ + INSERT INTO fine_tune_tasks + (id, name, payload, status, progress, process_id, create_time, gpus) + VALUES (?, ?, ?, 'pending', 0, NULL, ?, ?) + """, + (task_id, name, json_dumps(task), now, json_dumps(task["gpus"])), + ) + return task + + def update_task(self, task_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = self.task(task_id) + merged = {**current, **payload, "id": task_id} + with self.connect() as conn: + conn.execute( + "UPDATE fine_tune_tasks SET name=?, payload=?, gpus=? WHERE id=?", + (merged["name"], json_dumps(merged), json_dumps(merged.get("gpus", [])), task_id), + ) + return self.task(task_id) + + def start_task(self, payload: dict[str, Any]) -> dict[str, Any]: + task_id = str(payload.get("task_id") or payload.get("id")) + current = self.task(task_id) + merged = {**current, **payload, "id": task_id, "status": "syncing", "progress": 8} + node = self.schedule_node(payload) + selected_gpus = payload.get("gpus") or merged.get("gpus") or [0] + process_id = int(43000 + (time.time() % 10000)) + sync_job_id = self.create_sync_job(node["id"], current) + with self.connect() as conn: + conn.execute( + """ + UPDATE fine_tune_tasks + SET payload=?, status='syncing', progress=8, process_id=?, start_time=?, + compute_node_id=?, gpus=?, sync_job_id=? + WHERE id=? + """, + ( + json_dumps({**merged, "process_id": process_id, "gpus": selected_gpus}), + process_id, + utcnow(), + node["id"], + json_dumps(selected_gpus), + sync_job_id, + task_id, + ), + ) + return self.task(task_id) + + def stop_task(self, task_id: str) -> dict[str, Any]: + task = self.task(task_id) + task.update({"status": "failed", "progress": min(task.get("progress", 0), 99)}) + with self.connect() as conn: + conn.execute( + "UPDATE fine_tune_tasks SET status='failed', payload=?, completed_at=? WHERE id=?", + (json_dumps(task), utcnow(), task_id), + ) + return self.task(task_id) + + def delete_task(self, task_id: str) -> None: + with self.connect() as conn: + conn.execute("DELETE FROM fine_tune_tasks WHERE id=?", (task_id,)) + + def schedule_node(self, payload: dict[str, Any]) -> dict[str, Any]: + requested = payload.get("requested_node_id") or payload.get("compute_node_id") + nodes = self.compute_nodes() + candidates = [ + n + for n in nodes + if n["enabled"] and n["scheduler_status"] == "online" and n["current_running_jobs"] < n["max_parallel_jobs"] + ] + if requested: + selected = next((n for n in candidates if n["id"] == requested), None) + if selected: + return selected + if not candidates: + raise RuntimeError("no available compute node") + return sorted(candidates, key=lambda n: (-n["scheduler_weight"], n["current_running_jobs"], n["code"]))[0] + + def create_sync_job(self, node_id: str, task: dict[str, Any]) -> str: + sync_id = new_id("sync") + with self.connect() as conn: + conn.execute( + """ + INSERT INTO resource_sync_jobs + (id, target_node_id, resources, status, progress, create_time) + VALUES (?, ?, ?, 'pending', 0, ?) + """, + ( + sync_id, + node_id, + json_dumps( + [ + {"resource_type": "model", "resource_id": task.get("base_model")}, + {"resource_type": "dataset", "resource_id": task.get("train_dataset_id")}, + ] + ), + utcnow(), + ), + ) + return sync_id + + def progress(self, task_id: str) -> dict[str, Any]: + task = self.task(task_id) + status = task.get("status", "pending") + labels = { + "pending": "waiting for start", + "syncing": "syncing model and dataset to compute node", + "queued": "waiting for GPU slot", + "running": "training with LLaMA-Factory simulator", + "completed": "training completed", + "failed": "training stopped", + } + progress = int(task.get("progress", 0) or 0) + eta = "--" if status in {"completed", "failed"} else f"{max(1, math.ceil((100 - progress) / 10))} min" + return { + "status": status, + "progress": progress, + "step": labels.get(status, status), + "speed": "42.5 samples/s" if status == "running" else "--", + "eta": eta, + } + + def compute_nodes(self) -> list[dict[str, Any]]: + self.refresh_runtime_state() + with self.connect() as conn: + running = conn.execute( + "SELECT compute_node_id, COUNT(*) AS cnt FROM fine_tune_tasks WHERE status IN ('syncing','queued','running') GROUP BY compute_node_id" + ).fetchall() + running_map = {r["compute_node_id"]: r["cnt"] for r in running} + rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall() + return [ + { + **dict(row), + "enabled": bool(row["enabled"]), + "tags": json_loads(row["tags"], []), + "health_detail": json_loads(row["health_detail"], {}), + "current_running_jobs": running_map.get(row["id"], 0), + } + for row in rows + ] + + def update_compute_node(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]: + current = next((n for n in self.compute_nodes() if n["id"] == node_id), None) + if not current: + raise KeyError(node_id) + merged = {**current, **payload} + with self.connect() as conn: + conn.execute( + """ + UPDATE compute_nodes + SET name=?, api_base_url=?, file_gateway_url=?, enabled=?, scheduler_status=?, + scheduler_weight=?, tags=?, max_parallel_jobs=?, last_health_check_at=? + WHERE id=? + """, + ( + merged["name"], + merged["api_base_url"], + merged["file_gateway_url"], + 1 if merged["enabled"] else 0, + merged["scheduler_status"], + merged["scheduler_weight"], + json_dumps(merged["tags"]), + merged["max_parallel_jobs"], + utcnow(), + node_id, + ), + ) + return next(n for n in self.compute_nodes() if n["id"] == node_id) + + def gpus(self) -> list[dict[str, Any]]: + self.refresh_runtime_state() + with self.connect() as conn: + rows = conn.execute( + """ + SELECT g.*, n.code AS node_code, n.name AS node_name + FROM gpus g JOIN compute_nodes n ON n.id = g.node_id + ORDER BY n.code, g.gpu_index + """ + ).fetchall() + running_tasks = [ + self._task(row) + for row in conn.execute( + "SELECT * FROM fine_tune_tasks WHERE status IN ('syncing','queued','running')" + ).fetchall() + ] + items = [] + for row in rows: + task = next( + ( + t + for t in running_tasks + if t.get("compute_node_id") == row["node_id"] and row["gpu_index"] in (t.get("gpus") or []) + ), + None, + ) + busy = task is not None and task.get("status") == "running" + reserved = task is not None and task.get("status") in {"syncing", "queued"} + memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1) + gpu_percent = 86 if busy else 22 if reserved else 3 + items.append( + { + "id": row["gpu_index"], + "node_id": row["node_id"], + "node_code": row["node_code"], + "node_name": row["node_name"], + "name": row["name"], + "uuid": row["uuid"], + "gpu_percent": gpu_percent, + "memory_used_gb": memory_used, + "memory_total_gb": row["memory_total_gb"], + "memory_percent": round(memory_used / row["memory_total_gb"] * 100, 1), + "temperature": row["base_temperature"] + (21 if busy else 6 if reserved else 0), + "power_w": round(row["power_limit_w"] * (0.7 if busy else 0.25 if reserved else 0.08), 1), + "power_limit_w": row["power_limit_w"], + "status": "busy" if busy else "reserved" if reserved else "idle", + "processes": [ + { + "pid": task["process_id"], + "name": "llamafactory-cli", + "memory_used_gb": memory_used, + "task_name": task["name"], + "user": "admin", + } + ] + if task + else [], + } + ) + return items + + def system_info(self) -> dict[str, Any]: + gpus = self.gpus() + busy = len([g for g in gpus if g["status"] in {"busy", "reserved"}]) + cpu_percent = 18 + busy * 9 + memory_percent = 37 + busy * 4 + return { + "timestamp": utcnow(), + "cpu": { + "percent": min(cpu_percent, 95), + "cores": 32, + "percents": [min(cpu_percent + (i % 7) - 3, 99) for i in range(32)], + "model": "Platform x86_64 CPU", + "frequency_mhz": 2600, + "load_1m": round(cpu_percent / 10, 2), + }, + "memory": { + "used_gb": round(256 * memory_percent / 100, 1), + "total_gb": 256, + "percent": min(memory_percent, 95), + "available_gb": round(256 * (100 - memory_percent) / 100, 1), + "cached_gb": 32, + }, + "disk": { + "used_gb": 840, + "total_gb": 2048, + "percent": 41, + "read_mb_s": 120 if busy else 8, + "write_mb_s": 95 if busy else 5, + }, + "gpu": gpus, + "network": { + "download_mb_s": 12 if busy else 1.2, + "upload_mb_s": 7 if busy else 0.8, + "download_mb": 8024, + "upload_mb": 1732, + }, + "system": { + "uptime_seconds": int(time.time() % 100000), + "process_count": 248 + busy, + "os": "Linux platform compute image", + }, + } + + def health_metrics(self) -> dict[str, float]: + info = self.system_info() + return { + "cpu_percent": info["cpu"]["percent"], + "memory_percent": info["memory"]["percent"], + "disk_percent": info["disk"]["percent"], + } + + def queue(self) -> list[dict[str, Any]]: + return [ + { + "id": task["id"], + "name": task["name"], + "status": task["status"], + "progress": task.get("progress", 0), + "compute_node_id": task.get("compute_node_id"), + "gpus": task.get("gpus", []), + "create_time": task.get("create_time"), + } + for task in self.tasks() + if task["status"] in {"pending", "syncing", "queued", "running"} + ] + + def replicas(self, node_id: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + "SELECT * FROM resource_replicas WHERE node_id=? ORDER BY create_time DESC", + (node_id,), + ).fetchall() + if rows: + return [dict(row) for row in rows] + return [ + { + "id": f"rep_{node_id}_model_qwen", + "node_id": node_id, + "resource_type": "model", + "resource_id": "m_qwen25_7b", + "local_path": "/models/Qwen2.5-7B-Instruct", + "status": "available", + "sync_status": "completed", + "create_time": utcnow(), + }, + { + "id": f"rep_{node_id}_dataset_finance", + "node_id": node_id, + "resource_type": "dataset", + "resource_id": "ds_finance_sft", + "local_path": "/data/yg-ft/datasets/finance-sft-sample.jsonl", + "status": "available", + "sync_status": "completed", + "create_time": utcnow(), + }, + ] + + def sync_job(self, sync_id: str) -> dict[str, Any]: + self.refresh_runtime_state() + with self.connect() as conn: + row = conn.execute("SELECT * FROM resource_sync_jobs WHERE id=?", (sync_id,)).fetchone() + if not row: + raise KeyError(sync_id) + return {**dict(row), "resources": json_loads(row["resources"], [])} + + def training_log_files(self) -> list[dict[str, Any]]: + return [ + { + "file": f"train_{task['id']}_pid{task['process_id'] or 0}.log", + "name": task["name"], + "pid": task.get("process_id") or 0, + "size": f"{max(1, int((task.get('progress', 0) or 0) * 1.5))} KB", + "date": task.get("create_time", "")[:10], + } + for task in self.tasks() + ] + + def training_log_content(self, file_name: str) -> dict[str, Any]: + task = next((t for t in self.tasks() if t["id"] in file_name), None) + if not task: + raise KeyError(file_name) + content = self.generate_training_log(task) + return {"file": file_name, "content": content, "size": f"{max(1, len(content.encode('utf-8')) // 1024)} KB"} + + def generate_training_log(self, task: dict[str, Any]) -> str: + progress = int(task.get("progress", 0) or 0) + points = max(1, min(80, progress)) + lines = [ + f"[INFO] task={task['name']} engine=llama_factory mode=simulator status={task['status']}", + f"[INFO] base_model={task.get('base_model')} dataset={task.get('train_dataset_id')} gpus={task.get('gpus', [])}", + "[INFO] command=llamafactory-cli train --stage sft --finetuning_type lora --do_train true", + ] + for step in range(1, points + 1): + if step % 3 != 0 and step != points: + continue + loss = max(0.12, 2.4 * math.exp(-step / 42)) + grad_norm = 0.45 + (step % 8) * 0.03 + lr = float(task.get("learning_rate") or 0.0002) * max(0.05, 1 - step / 120) + epoch = round(step / max(1, points) * float(task.get("n_epochs") or 3), 4) + lines.append( + "{" + f"'loss': {loss:.4f}, 'grad_norm': {grad_norm:.4f}, " + f"'learning_rate': {lr:.8f}, 'epoch': {epoch:.4f}" + "}" + ) + if task.get("status") == "completed": + lines.extend( + [ + "***** train metrics *****", + f"epoch = {task.get('n_epochs', 3)}", + "train_loss = 0.1248", + f"train_runtime = {task.get('train_duration') or '1m 10s'}", + "***** train metrics end *****", + ] + ) + return "\n".join(lines) + + def log_files(self, date: str | None = None) -> list[dict[str, Any]]: + today = date or utcnow()[:10] + return [ + {"file": f"backend-{today}.log", "name": f"backend-{today}.log", "size": "32 KB", "date": today}, + {"file": f"error-{today}.log", "name": f"error-{today}.log", "size": "1 KB", "date": today}, + ] + + def log_content(self, file_name: str) -> dict[str, Any]: + lines = [ + json_dumps( + { + "timestamp": utcnow(), + "level": "INFO", + "logger": "platform", + "file": "backend/app/api/v1/endpoints/platform.py", + "line": 1, + "message": "Platform log stream is available.", + } + ) + ] + return {"file": file_name, "content": "\n".join(lines), "size": "1 KB"} + + +_store: PlatformStore | None = None + + +def get_platform_store() -> PlatformStore: + global _store + if _store is None: + _store = PlatformStore() + return _store + diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 5542d81..9aa8c33 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -1,4 +1,41 @@ -"""Database session factory placeholder. +from __future__ import annotations + +import os +from collections.abc import Generator +from contextlib import contextmanager + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + + +DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./runtime/platform.db") + +engine = create_engine( + DATABASE_URL, + pool_pre_ping=True, + future=True, + connect_args={"check_same_thread": False} if DATABASE_URL.startswith("sqlite") else {}, +) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False, future=True) + + +def get_db() -> Generator[Session, None, None]: + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@contextmanager +def session_scope() -> Generator[Session, None, None]: + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() -Implement SQLAlchemy/SQLModel session management here when database development starts. -""" diff --git a/compute/api/main.py b/compute/api/main.py index d79c1d7..60e286a 100644 --- a/compute/api/main.py +++ b/compute/api/main.py @@ -1,13 +1,105 @@ from __future__ import annotations import os +import math +import time from pathlib import Path +from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException + +from compute.engines.llama_factory.adapter import build_command, parse_log_line def create_app() -> FastAPI: app = FastAPI(title="YG Fine-Tune Compute API") + jobs: dict[str, dict[str, Any]] = {} + + def now() -> float: + return time.time() + + def host_id() -> str: + return os.getenv("COMPUTE_HOST_ID", "gpu-node-01") + + def job_status(job: dict[str, Any]) -> dict[str, Any]: + elapsed = max(0, int(now() - job["created_at"])) + if job["status"] not in {"stopped", "failed", "completed"}: + if elapsed < 5: + job["status"] = "queued" + job["progress"] = 12 + elapsed * 3 + elif elapsed < 60: + job["status"] = "running" + job["progress"] = min(96, 25 + int((elapsed - 5) / 55 * 70)) + else: + job["status"] = "completed" + job["progress"] = 100 + job["logs"] = generate_logs(job) + return job + + def generate_logs(job: dict[str, Any]) -> str: + progress = int(job.get("progress", 0) or 0) + points = max(1, min(80, progress)) + lines = [ + f"[INFO] compute_host_id={host_id()} job_id={job['id']} engine=llama_factory mode=simulator", + f"[INFO] command={' '.join(job['command'])}", + ] + for step in range(1, points + 1): + if step % 4 != 0 and step != points: + continue + loss = max(0.11, 2.5 * math.exp(-step / 40)) + grad_norm = 0.4 + (step % 5) * 0.04 + lr = 0.0002 * max(0.05, 1 - step / 100) + epoch = round(step / points * 3, 4) + lines.append( + "{" + f"'loss': {loss:.4f}, 'grad_norm': {grad_norm:.4f}, " + f"'learning_rate': {lr:.8f}, 'epoch': {epoch:.4f}" + "}" + ) + if job.get("status") == "completed": + lines.extend( + [ + "***** train metrics *****", + "epoch = 3", + "train_loss = 0.1181", + "train_runtime = 1m 0s", + "***** train metrics end *****", + ] + ) + return "\n".join(lines) + + def gpu_resources() -> list[dict[str, Any]]: + active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}] + gpus: list[dict[str, Any]] = [] + for idx in range(4): + task = next((job for job in active_jobs if idx in job.get("gpus", [])), None) + busy = task is not None and task["status"] == "running" + reserved = task is not None and task["status"] == "queued" + gpus.append( + { + "id": idx, + "uuid": f"GPU-{host_id().upper()}-{idx}", + "name": os.getenv("COMPUTE_GPU_NAME", "NVIDIA A800-SXM4-80GB"), + "status": "busy" if busy else "reserved" if reserved else "idle", + "gpu_percent": 88 if busy else 25 if reserved else 4, + "memory_used_gb": 58 if busy else 12 if reserved else 2, + "memory_total_gb": 80, + "temperature": 61 if busy else 45 if reserved else 36, + "power_w": 215 if busy else 80 if reserved else 25, + "power_limit_w": 300, + "processes": [ + { + "pid": task["pid"], + "name": "llamafactory-cli", + "task_name": task["name"], + "memory_used_gb": 58 if busy else 12, + } + ] + if task + else [], + } + ) + return gpus @app.get("/health") async def health_check() -> dict[str, str]: @@ -19,7 +111,7 @@ def create_app() -> FastAPI: @app.get("/api/v1/compute/health") async def compute_health_check() -> dict[str, str | bool]: data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")) - llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/opt/LLaMA-Factory")) + llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) return { "status": "ok", "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"), @@ -31,8 +123,73 @@ def create_app() -> FastAPI: } @app.get("/api/v1/compute/jobs") - async def list_jobs() -> dict[str, list[dict[str, str]]]: - return {"items": []} + async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]: + return {"items": [job_status(job) for job in jobs.values()]} + + @app.get("/compute/resources/gpus") + async def list_gpus() -> dict[str, Any]: + return {"items": gpu_resources(), "compute_host_id": host_id()} + + @app.post("/compute/jobs") + async def create_job(payload: dict[str, Any]) -> dict[str, Any]: + try: + command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory")) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + job_id = str(payload.get("id") or f"job_{int(now() * 1000)}") + job = { + "id": job_id, + "name": payload.get("name", job_id), + "status": "queued", + "progress": 10, + "pid": int(52000 + now() % 10000), + "gpus": payload.get("gpus") or [0], + "created_at": now(), + "command": command.command, + "work_dir": command.work_dir, + "artifacts": [], + "logs": "", + } + jobs[job_id] = job + return job_status(job) + + @app.get("/compute/jobs") + async def list_jobs() -> dict[str, Any]: + return {"items": [job_status(job) for job in jobs.values()]} + + @app.get("/compute/jobs/{job_id}") + async def get_job(job_id: str) -> dict[str, Any]: + job = jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + return job_status(job) + + @app.post("/compute/jobs/{job_id}/stop") + async def stop_job(job_id: str) -> dict[str, Any]: + job = jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + job["status"] = "stopped" + job["progress"] = min(job.get("progress", 0), 99) + return job + + @app.get("/compute/jobs/{job_id}/logs") + async def job_logs(job_id: str) -> dict[str, Any]: + job = jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="job not found") + job = job_status(job) + metrics = [parse_log_line(line) for line in job["logs"].splitlines()] + return {"job_id": job_id, "content": job["logs"], "metrics": [m for m in metrics if m]} + + @app.post("/compute/files/upload") + async def upload_file(payload: dict[str, Any]) -> dict[str, Any]: + file_id = str(payload.get("id") or f"file_{int(now() * 1000)}") + return {"id": file_id, "status": "available", "local_path": f"/data/yg-ft/uploads/{file_id}"} + + @app.get("/compute/files/{file_id}/download") + async def download_file(file_id: str) -> dict[str, Any]: + return {"id": file_id, "status": "ready", "download_url": f"/compute/files/{file_id}/download"} return app diff --git a/compute/engines/llama_factory/adapter.py b/compute/engines/llama_factory/adapter.py new file mode 100644 index 0000000..bb22029 --- /dev/null +++ b/compute/engines/llama_factory/adapter.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class LlamaFactoryCommand: + command: list[str] + work_dir: str + env: dict[str, str] + + +def validate_config(config: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if not config.get("base_model") and not config.get("model_name_or_path"): + errors.append("base_model or model_name_or_path is required") + if not config.get("dataset") and not config.get("dataset_dir"): + errors.append("dataset or dataset_dir is required") + learning_rate = float(config.get("learning_rate", 0.0002)) + if learning_rate <= 0: + errors.append("learning_rate must be greater than zero") + epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1))) + if epochs <= 0: + errors.append("n_epochs must be greater than zero") + return errors + + +def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand: + errors = validate_config(config) + if errors: + raise ValueError("; ".join(errors)) + + model_path = config.get("base_model") or config.get("model_name_or_path") + dataset = config.get("dataset") or config.get("dataset_dir") + output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'sample-job')}" + command = [ + "llamafactory-cli", + "train", + "--stage", + str(config.get("stage", "sft")).lower(), + "--do_train", + "true", + "--model_name_or_path", + str(model_path), + "--dataset", + str(dataset), + "--template", + str(config.get("template", "qwen")), + "--finetuning_type", + str(config.get("train_method", config.get("finetuning_type", "lora"))), + "--output_dir", + str(output_dir), + "--per_device_train_batch_size", + str(config.get("batch_size", 2)), + "--learning_rate", + str(config.get("learning_rate", 0.0002)), + "--num_train_epochs", + str(config.get("n_epochs", 3)), + "--save_steps", + str(config.get("save_steps", 50)), + ] + quantization_bit = int(config.get("quantization_bit", 0) or 0) + if quantization_bit in {4, 8}: + command.extend(["--quantization_bit", str(quantization_bit)]) + return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={}) + + +def parse_log_line(line: str) -> dict[str, float] | None: + if "loss" not in line or "learning_rate" not in line: + return None + result: dict[str, float] = {} + for key in ["loss", "grad_norm", "learning_rate", "epoch"]: + match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line) + if match: + result[key] = float(match.group(1)) + return result or None + diff --git a/docker/README.md b/docker/README.md index fc3a6ca..6362267 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,60 +1,106 @@ # Docker 部署说明 -本文档对应 `docs/deployment-plan.md`，按应用服务器和算力服务器拆分 Dockerfile 与 Docker Compose 文件。所有业务代码均通过 volume 外挂到容器内，镜像只包含运行时环境和第三方依赖。 +本目录按应用服务器和算力服务器拆分 Dockerfile 与 Docker Compose 文件。Compose 文件不包含 `build:`，不会在 `docker compose up` 时自动构建业务镜像。所有业务镜像需要先通过手动 `docker build` 构建，再由 Compose 启动。 -## 目录 +## 基础镜像 -```text -docker/ - app/ - Dockerfile.backend - Dockerfile.frontend - docker-compose.yml - .env.example - compute/ - Dockerfile.compute - docker-compose.yml - .env.example +| 镜像 | 用途 | +| --- | --- | +| `python:3.12-slim` | 应用后端基础镜像，后端运行环境要求 Python 3.12 及以上 | +| `nginx:1.27-alpine` | 前端静态资源与 `/api` 反向代理运行镜像 | +| `hiyouga/llamafactory:latest` | 算力服务基础镜像，基于 LLaMA-Factory 官方镜像扩展 Compute API | +| `postgres:16-alpine` | 开发阶段内置 PostgreSQL | +| `redis:7-alpine` | 开发阶段内置 Redis | + +一键拉取基础镜像： + +```bash +docker pull python:3.12-slim && \ +docker pull nginx:1.27-alpine && \ +docker pull hiyouga/llamafactory:latest && \ +docker pull postgres:16-alpine && \ +docker pull redis:7-alpine ``` -项目根目录不再保留 `Dockerfile` 和 `docker-compose.yml`，避免与应用服务器、算力服务器拆分部署入口混淆。 +Windows PowerShell： + +```powershell +$images = @( + "python:3.12-slim", + "nginx:1.27-alpine", + "hiyouga/llamafactory:latest", + "postgres:16-alpine", + "redis:7-alpine" +) +$images | ForEach-Object { docker pull $_ } +``` + +如果部署环境不能访问外网，需要提前在可联网环境执行上述拉取命令，再用 `docker save` / `docker load` 导出导入。 + +## 业务镜像 + +| 镜像 | Dockerfile | 构建命令 | +| --- | --- | --- | +| `yg-ft-backend-api:latest` | `docker/app/Dockerfile.backend` | `docker build -f docker/app/Dockerfile.backend -t yg-ft-backend-api:latest .` | +| `yg-ft-frontend-runtime:latest` | `docker/app/Dockerfile.frontend` | `docker build -f docker/app/Dockerfile.frontend -t yg-ft-frontend-runtime:latest .` | +| `yg-ft-compute-api:latest` | `docker/compute/Dockerfile.compute` | `docker build -f docker/compute/Dockerfile.compute -t yg-ft-compute-api:latest .` | + +## 对外端口 + +所有宿主机对外端口统一使用 5 位端口。容器内部端口保持镜像默认端口，便于容器内服务和健康检查稳定。 + +| 服务 | 宿主机对外端口 | 容器内部端口 | 说明 | +| --- | --- | --- | --- | +| 前端 Nginx | `16801` | `80` | 前端页面入口 | +| 后端 API | `17861` | `8000` | FastAPI 服务 | +| PostgreSQL | `15432` | `5432` | 开发阶段内置数据库 | +| Redis | `16379` | `6379` | 开发阶段内置缓存 | +| Compute API | `19100` | `9100` | 算力服务器 API | +| File Gateway | `19101` | 后续服务端口 | 当前预留，后续拆出文件网关服务时使用 | + +对应配置文件： + +- `docker/app/.env.example` + - `FRONTEND_PORT=16801` + - `BACKEND_API_PORT=17861` + - `POSTGRES_PORT=15432` + - `REDIS_PORT=16379` +- `docker/compute/.env.example` + - `COMPUTE_API_PORT=19100` + - `FILE_GATEWAY_PORT=19101` ## 应用服务器部署 应用服务器包含前端 Nginx、Backend API、PostgreSQL、Redis。 -开发阶段默认由项目自带 PostgreSQL/Redis： - -```env -USE_BUILTIN_POSTGRES=true -USE_BUILTIN_REDIS=true -DATABASE_URL=postgresql+asyncpg://yg_ft:change_me@postgres:5432/yg_ft -REDIS_URL=redis://redis:6379/0 -``` - -后续切换企业统一基础设施时，保留应用配置方式，只需要： - -- 修改 `DATABASE_URL` 指向企业 PostgreSQL。 -- 修改 `REDIS_URL` 指向企业 Redis。 -- 从 Compose 中移除或禁用 `postgres`、`redis` 服务。 -- 保留 `docs/postgres-schema.sql` 作为数据库初始化或迁移参考。 - -首次部署前先构建前端产物： +首次部署： ```bash +cd + +# 1. 使用当前 Windows/宿主机 npm 构建前端静态产物 +cd frontend +npm ci +npm run build +cd .. + +# 2. 手动构建业务镜像 +docker build -f docker/app/Dockerfile.backend -t yg-ft-backend-api:latest . +docker build -f docker/app/Dockerfile.frontend -t yg-ft-frontend-runtime:latest . + +# 3. 启动应用服务 cd docker/app cp .env.example .env -docker compose --profile build run --rm frontend-builder -docker compose up -d --build +docker compose up -d ``` 默认访问地址： ```text -http://:6801 +http://:16801 ``` -应用侧代码外挂： +应用侧代码和数据外挂： ```text ../../backend -> /app @@ -63,157 +109,128 @@ http://:6801 ../../runtime/app/data -> /data/yg-ft ``` -如果算力服务独立部署，需要在 `docker/app/.env` 中修改： +如果使用企业统一 PostgreSQL/Redis，修改 `docker/app/.env`： ```env -COMPUTE_API_BASE_URL=http://:9100 -FILE_GATEWAY_BASE_URL=http://:9101 -COMPUTE_SERVICE_TOKEN=change_me -COMPUTE_STATUS_SYNC_MODE=polling -COMPUTE_POLL_INTERVAL_SECONDS=10 -COMPUTE_POLL_BATCH_SIZE=100 +DATABASE_URL=postgresql+asyncpg://:@:15432/ +REDIS_URL=redis://:16379/0 +USE_BUILTIN_POSTGRES=false +USE_BUILTIN_REDIS=false ``` -状态同步采用应用侧定时轮询 Compute API 为主，避免算力服务器需要访问应用服务器，从而减少双向网络策略开通。 - -## 应用服务与算力服务交互 - -应用服务与算力服务之间只要求应用服务器主动访问算力服务器： - -```text -Frontend - -> Backend API - -> Compute API - -> Compute Agent / LLaMA-Factory - -> 本地数据盘 / 模型目录 / 训练产物 - <- Backend Worker 定时轮询 Compute API -``` - -默认交互流程： - -- `Backend API` 读取 `COMPUTE_API_BASE_URL`，向 `Compute API` 提交训练、评测、合并、导出等任务。 -- `Compute API` 在算力服务器上调度 `Compute Agent`。 -- `Compute Agent` 通过宿主机挂载目录调用 LLaMA-Factory，并读写 `/data/yg-ft` 下的数据集、模型和训练产物。 -- `Backend Worker` 按 `COMPUTE_POLL_INTERVAL_SECONDS` 定时轮询 Compute API，同步任务状态、训练指标、日志摘要和产物索引。 -- 前端只访问应用服务；文件下载和产物访问由应用服务完成权限校验后，再通过 `FILE_GATEWAY_BASE_URL` 获取受控资源。 - -当前支持通过环境变量动态配置算力服务地址： - -```env -COMPUTE_API_BASE_URL=http://:9100 -FILE_GATEWAY_BASE_URL=http://:9101 -COMPUTE_SERVICE_TOKEN=change_me -COMPUTE_STATUS_SYNC_MODE=polling -COMPUTE_POLL_INTERVAL_SECONDS=10 -COMPUTE_POLL_BATCH_SIZE=100 -``` - -后续多算力节点阶段建议升级为数据库配置：在 `compute_nodes` 表中维护节点地址、服务 token、启用状态、权重、标签和健康状态，并通过“算力节点管理”页面动态启停节点，避免每次调整地址都重启应用服务。 +生产环境如完全使用外部基础设施，可以删除或注释 Compose 中的 `postgres`、`redis` 服务及 `backend-api.depends_on` 中对应依赖。 ## 算力服务器部署 -算力服务器包含 Compute API、后续 Compute Agent、后续 File Gateway、GPU runtime、本地训练数据目录和宿主机挂载的 LLaMA-Factory。 +算力服务器包含 Compute API、后续 Compute Agent、File Gateway、GPU runtime、本地训练数据目录和 LLaMA-Factory。`Dockerfile.compute` 基于 LLaMA-Factory 官方镜像： + +```dockerfile +FROM hiyouga/llamafactory:latest +``` 部署前需要安装： -- NVIDIA Driver。 -- NVIDIA Container Toolkit。 -- Docker Engine 和 Docker Compose Plugin。 -- LLaMA-Factory 宿主机目录，默认 `/opt/LLaMA-Factory`。 -- 本地训练数据盘，默认 `/data/yg-ft`。 +- NVIDIA Driver +- NVIDIA Container Toolkit +- Docker Engine 和 Docker Compose Plugin +- 本地训练数据目录，默认 `/data/yg-ft` -算力服务器上的 LLaMA-Factory 使用宿主机挂载方式，不在当前 Compose 中重新构建 LLaMA-Factory 镜像： - -```env -LLAMA_FACTORY_HOME=/opt/LLaMA-Factory -LLAMA_FACTORY_HOST_PATH=/opt/LLaMA-Factory -``` - -启动： +首次部署： ```bash +cd + +# 手动构建算力业务镜像 +docker build -f docker/compute/Dockerfile.compute -t yg-ft-compute-api:latest . + +# 启动算力服务 cd docker/compute cp .env.example .env -docker compose up -d --build +docker compose up -d ``` 健康检查： ```text -GET http://:9100/health -GET http://:9100/api/v1/compute/health +GET http://:19100/health +GET http://:19100/api/v1/compute/health ``` 算力侧代码和数据外挂： ```text ../../compute -> /app/compute -${LLAMA_FACTORY_HOST_PATH} -> /opt/LLaMA-Factory ${YG_FT_DATA_ROOT_HOST} -> /data/yg-ft ../../runtime/compute/logs -> /opt/yg-ft/logs/compute ../../runtime/compute/training-logs -> /opt/yg-ft/logs/training ``` -算力侧只需要允许应用服务器访问 Compute API/File Gateway，不要求访问应用服务器： +## 应用与算力分离部署 + +应用服务器只需要主动访问算力服务器，不要求算力服务器回调应用服务器。 + +在 `docker/app/.env` 中配置： ```env -ENABLE_APP_CALLBACK=false +COMPUTE_API_BASE_URL=http://:19100 +FILE_GATEWAY_BASE_URL=http://:19101 COMPUTE_SERVICE_TOKEN=change_me +COMPUTE_STATUS_SYNC_MODE=polling +COMPUTE_POLL_INTERVAL_SECONDS=10 +COMPUTE_POLL_BATCH_SIZE=100 +``` + +交互链路： + +```text +Frontend + -> Backend API + -> Compute API + -> Compute Agent / LLaMA-Factory + -> 本地数据目录 / 模型目录 / 训练产物 + <- Backend Worker 定时轮询 Compute API ``` ## 多算力节点部署 -多算力节点仍按“单机多 GPU 节点”部署。每台 GPU 服务器都需要独立部署一套算力服务和宿主机挂载的 LLaMA-Factory： +多算力节点仍按“单机多 GPU 节点”部署。每台 GPU 服务器都独立部署一套 `docker/compute`： ```text -gpu-node-01: docker/compute + /opt/LLaMA-Factory + /data/yg-ft -gpu-node-02: docker/compute + /opt/LLaMA-Factory + /data/yg-ft -gpu-node-03: docker/compute + /opt/LLaMA-Factory + /data/yg-ft +gpu-node-01: docker/compute + /data/yg-ft + 19100/19101 +gpu-node-02: docker/compute + /data/yg-ft + 19100/19101 +gpu-node-03: docker/compute + /data/yg-ft + 19100/19101 ``` -节点之间默认不互相访问。应用服务器主动访问每个节点的 Compute API/File Gateway，并通过 `compute_nodes` 表或算力节点管理页面维护： +节点之间默认不互访。应用平台主动访问每个节点的 Compute API/File Gateway，并通过 `compute_nodes`、`resource_replicas`、`resource_sync_jobs` 统一调度和同步。 -- `api_base_url` -- `file_gateway_url` -- `enabled` -- `scheduler_status` -- `scheduler_weight` -- `tags` -- `data_root` -- `model_root` -- `log_root` +## 常用命令 -长期使用每台算力服务器本地磁盘时，需要由应用平台维护资源副本关系。调度前先检查目标节点是否已有数据集和模型副本；如果没有，应用平台通过目标节点 File Gateway 创建资源同步任务，同步完成后再提交训练任务。 +重新构建应用镜像： -## 单机所有服务部署在算力服务器 +```bash +docker build -f docker/app/Dockerfile.backend -t yg-ft-backend-api:latest . +docker build -f docker/app/Dockerfile.frontend -t yg-ft-frontend-runtime:latest . +``` -在同一台 GPU 服务器上分别启动两套 Compose： +重新构建算力镜像： + +```bash +docker build -f docker/compute/Dockerfile.compute -t yg-ft-compute-api:latest . +``` + +启动服务： ```bash cd docker/app -docker compose --profile build run --rm frontend-builder -docker compose up -d --build +docker compose up -d cd ../compute -docker compose up -d --build +docker compose up -d ``` -应用侧 `.env` 中可使用： +查看服务： -```env -COMPUTE_API_BASE_URL=http://host.docker.internal:9100 -FILE_GATEWAY_BASE_URL=http://host.docker.internal:9101 -COMPUTE_STATUS_SYNC_MODE=polling +```bash +docker compose ps +docker compose logs -f ``` - -Linux 环境如需容器访问宿主机地址，可在应用侧 Compose 中按需增加 `extra_hosts: ["host.docker.internal:host-gateway"]`，或直接配置算力服务器内网 IP。 - -## 生产注意事项 - -- 当前 Compose 是工程部署骨架，后续 Backend Worker、Compute Agent、File Gateway 有可运行入口后，再拆分为独立服务。 -- PostgreSQL 和 Redis 开发阶段采用项目自带部署，生产阶段保留切换企业统一基础设施的配置入口。 -- 生产环境请把 `change_me` 替换为强密码或密钥管理系统注入。 -- 当前镜像默认优先保证宿主机外挂日志和数据目录可写；生产环境如需非 root 运行，需要统一宿主机目录 UID/GID 后在 Compose 中增加 `user` 配置。 -- Compute API 不应暴露公网，建议通过防火墙限制只允许应用服务器访问。 -- GPU 容器需要 NVIDIA Container Toolkit，否则 `gpus: all` 无法生效。 -- 前端 Nginx 默认挂载 `frontend/dist`，发布前需要先运行 `frontend-builder` 或由 CI 构建产物。 diff --git a/docker/app/.env.example b/docker/app/.env.example index 5df8086..df3a898 100644 --- a/docker/app/.env.example +++ b/docker/app/.env.example @@ -2,6 +2,15 @@ APP_ENV=prod APP_NAME=YG Fine-Tune Platform API API_PREFIX=/api +FRONTEND_IMAGE=yg-ft-frontend-runtime:latest +BACKEND_API_IMAGE=yg-ft-backend-api:latest + +# Five-digit host ports exposed outside the application server. +FRONTEND_PORT=16801 +BACKEND_API_PORT=17861 +POSTGRES_PORT=15432 +REDIS_PORT=16379 + POSTGRES_DB=yg_ft POSTGRES_USER=yg_ft POSTGRES_PASSWORD=change_me @@ -23,7 +32,7 @@ LOG_RETENTION_DAYS=10 API_PROXY_PASS=http://backend-api:8000 -# Split deployment: set these to the compute server address, for example http://10.10.20.31:9100. +# Split deployment: set these to the compute server address, for example http://10.10.20.31:19100. COMPUTE_API_BASE_URL=http://compute-api:9100 COMPUTE_SERVICE_TOKEN=change_me FILE_GATEWAY_BASE_URL=http://compute-api:9101 diff --git a/docker/app/Dockerfile.backend b/docker/app/Dockerfile.backend index 5fd505d..57b0364 100644 --- a/docker/app/Dockerfile.backend +++ b/docker/app/Dockerfile.backend @@ -1,6 +1,4 @@ -# syntax=docker/dockerfile:1 - -FROM python:3.11-slim +FROM python:3.12-slim ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/docker/app/Dockerfile.frontend b/docker/app/Dockerfile.frontend index 341c147..740b827 100644 --- a/docker/app/Dockerfile.frontend +++ b/docker/app/Dockerfile.frontend @@ -1,4 +1,3 @@ -# syntax=docker/dockerfile:1 FROM nginx:1.27-alpine diff --git a/docker/app/docker-compose.yml b/docker/app/docker-compose.yml index 655dd3c..c6b29d0 100644 --- a/docker/app/docker-compose.yml +++ b/docker/app/docker-compose.yml @@ -1,25 +1,12 @@ services: - frontend-builder: - image: node:20-alpine - profiles: - - build - working_dir: /workspace - volumes: - - ../../frontend:/workspace - - frontend_node_modules:/workspace/node_modules - command: sh -c "npm ci && npm run build" - frontend: - build: - context: ../.. - dockerfile: docker/app/Dockerfile.frontend - image: yg-ft-frontend-runtime:latest + image: ${FRONTEND_IMAGE:-yg-ft-frontend-runtime:latest} container_name: yg-ft-frontend depends_on: backend-api: condition: service_started ports: - - "6801:80" + - "${FRONTEND_PORT:-16801}:80" environment: API_PROXY_PASS: ${API_PROXY_PASS:-http://backend-api:8000} volumes: @@ -30,10 +17,7 @@ services: restart: unless-stopped backend-api: - build: - context: ../.. - dockerfile: docker/app/Dockerfile.backend - image: yg-ft-backend-api:latest + image: ${BACKEND_API_IMAGE:-yg-ft-backend-api:latest} container_name: yg-ft-backend-api depends_on: postgres: @@ -42,6 +26,8 @@ services: condition: service_healthy expose: - "8000" + ports: + - "${BACKEND_API_PORT:-17861}:8000" environment: APP_ENV: ${APP_ENV:-prod} APP_NAME: ${APP_NAME:-YG Fine-Tune Platform API} @@ -88,6 +74,8 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - ../../docs/postgres-schema.sql:/docker-entrypoint-initdb.d/001-schema.sql:ro + ports: + - "${POSTGRES_PORT:-15432}:5432" networks: - yg-ft-app healthcheck: @@ -103,6 +91,8 @@ services: command: ["redis-server", "--appendonly", "yes"] volumes: - redis_data:/data + ports: + - "${REDIS_PORT:-16379}:6379" networks: - yg-ft-app healthcheck: @@ -117,6 +107,5 @@ networks: name: yg-ft-app volumes: - frontend_node_modules: postgres_data: redis_data: diff --git a/docker/compute/.env.example b/docker/compute/.env.example index 7a57bb0..866b3c8 100644 --- a/docker/compute/.env.example +++ b/docker/compute/.env.example @@ -1,14 +1,16 @@ COMPUTE_ENV=prod COMPUTE_HOST_ID=gpu-node-01 -COMPUTE_API_PORT=9100 -FILE_GATEWAY_PORT=9101 +# Five-digit host ports exposed outside the compute server. +COMPUTE_API_PORT=19100 +FILE_GATEWAY_PORT=19101 +COMPUTE_API_IMAGE=yg-ft-compute-api:latest # The application server actively polls Compute API; compute server does not need reverse access. COMPUTE_SERVICE_TOKEN=change_me ENABLE_APP_CALLBACK=false -LLAMA_FACTORY_HOME=/opt/LLaMA-Factory -LLAMA_FACTORY_HOST_PATH=/opt/LLaMA-Factory +# LLaMA-Factory is provided by the official hiyouga/llamafactory base image. +LLAMA_FACTORY_HOME=/app/LLaMA-Factory YG_FT_DATA_ROOT=/data/yg-ft YG_FT_DATA_ROOT_HOST=/data/yg-ft diff --git a/docker/compute/Dockerfile.compute b/docker/compute/Dockerfile.compute index 20d0b61..60af155 100644 --- a/docker/compute/Dockerfile.compute +++ b/docker/compute/Dockerfile.compute @@ -1,6 +1,5 @@ -# syntax=docker/dockerfile:1 -FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 +FROM hiyouga/llamafactory:latest ENV DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 \ @@ -10,25 +9,16 @@ ENV DEBIAN_FRONTEND=noninteractive \ WORKDIR /app RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 \ - python3-pip \ - python3-venv \ - git \ - curl \ - ca-certificates \ - tini \ - && ln -sf /usr/bin/python3 /usr/local/bin/python \ - && ln -sf /usr/bin/pip3 /usr/local/bin/pip \ + && apt-get install -y --no-install-recommends tini \ && rm -rf /var/lib/apt/lists/* COPY compute/requirements.txt /tmp/requirements.txt -RUN pip install --upgrade pip \ - && pip install -r /tmp/requirements.txt \ +RUN pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple \ && rm -f /tmp/requirements.txt -RUN mkdir -p /opt/yg-ft/logs/compute /opt/yg-ft/logs/training /data/yg-ft /opt/LLaMA-Factory \ - && chmod -R 0775 /opt/yg-ft /data/yg-ft /opt/LLaMA-Factory +RUN mkdir -p /opt/yg-ft/logs/compute /opt/yg-ft/logs/training /data/yg-ft /app/LLaMA-Factory \ + && chmod -R 0775 /opt/yg-ft /data/yg-ft /app/LLaMA-Factory ENTRYPOINT ["/usr/bin/tini", "--"] diff --git a/docker/compute/docker-compose.yml b/docker/compute/docker-compose.yml index d77a2d6..90dc3a5 100644 --- a/docker/compute/docker-compose.yml +++ b/docker/compute/docker-compose.yml @@ -1,19 +1,16 @@ services: compute-api: - build: - context: ../.. - dockerfile: docker/compute/Dockerfile.compute - image: yg-ft-compute-api:latest + image: ${COMPUTE_API_IMAGE:-yg-ft-compute-api:latest} container_name: yg-ft-compute-api gpus: all ports: - - "${COMPUTE_API_PORT:-9100}:9100" + - "${COMPUTE_API_PORT:-19100}:9100" environment: COMPUTE_ENV: ${COMPUTE_ENV:-prod} COMPUTE_HOST_ID: ${COMPUTE_HOST_ID:-gpu-node-01} COMPUTE_SERVICE_TOKEN: ${COMPUTE_SERVICE_TOKEN:-change_me} ENABLE_APP_CALLBACK: ${ENABLE_APP_CALLBACK:-false} - LLAMA_FACTORY_HOME: ${LLAMA_FACTORY_HOME:-/opt/LLaMA-Factory} + LLAMA_FACTORY_HOME: ${LLAMA_FACTORY_HOME:-/app/LLaMA-Factory} YG_FT_DATA_ROOT: ${YG_FT_DATA_ROOT:-/data/yg-ft} LOG_DIR: ${LOG_DIR:-/opt/yg-ft/logs/compute} CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-all} @@ -22,7 +19,6 @@ services: PYTHONPATH: /app volumes: - ../../compute:/app/compute:ro - - ${LLAMA_FACTORY_HOST_PATH:-/opt/LLaMA-Factory}:${LLAMA_FACTORY_HOME:-/opt/LLaMA-Factory} - ${YG_FT_DATA_ROOT_HOST:-/data/yg-ft}:${YG_FT_DATA_ROOT:-/data/yg-ft} - ../../runtime/compute/logs:/opt/yg-ft/logs/compute - ../../runtime/compute/training-logs:/opt/yg-ft/logs/training diff --git a/docs/backend-api-design.md b/docs/backend-api-design.md index 0d6b572..4ac2155 100644 --- a/docs/backend-api-design.md +++ b/docs/backend-api-design.md @@ -868,8 +868,8 @@ page=1&page_size=20&keyword=xxx&sort=-created_at "id": "uuid", "code": "gpu-node-01", "name": "A800 Node 01", - "api_base_url": "http://10.10.20.31:9100", - "file_gateway_url": "http://10.10.20.31:9101", + "api_base_url": "http://10.10.20.31:19100", + "file_gateway_url": "http://10.10.20.31:19101", "enabled": true, "scheduler_status": "online", "scheduler_weight": 100, diff --git a/docs/demo-development-plan.md b/docs/demo-development-plan.md deleted file mode 100644 index dca8aa4..0000000 --- a/docs/demo-development-plan.md +++ /dev/null @@ -1,545 +0,0 @@ -# Demo 开发计划 - -本文档用于指导后续开发一个可演示的模型微调平台 Demo。Demo 不是纯前端展示，而是包含前端、FastAPI 后端、PostgreSQL、算力服务、GPU 状态、LLaMA-Factory 适配器和训练任务主链路的工程化演示版本。 - -## 1. Demo 目标 - -Demo 目标是在没有完整生产环境的条件下，跑通一条可信的模型微调平台主链路： - -```text -登录 - -> 数据集管理 - -> 模型管理 - -> 创建微调任务 - -> 自动/手动选择算力节点 - -> 检查模型/数据集资源副本 - -> 缺失资源则同步到目标节点 - -> 启动训练任务 - -> 查看 GPU 占用、训练日志、loss 曲线、checkpoint - -> 训练完成后登记训练产物 - -> 可进入评测/推理演示 -``` - -Demo 支持两种算力运行模式： - -| 模式 | 说明 | 适用场景 | -| --- | --- | --- | -| `simulator` | 模拟 GPU、训练进程、训练日志、loss、checkpoint | 无 GPU、无 LLaMA-Factory 环境 | -| `real` | 调用 `nvidia-smi` 和 LLaMA-Factory 启动真实训练 | 有 GPU 和训练环境 | - -第一版优先实现 `simulator`，同时保留 `real` 模式接口和适配器边界。 - -## 2. 技术范围 - -### 2.1 前端 - -基于现有 `frontend/` 页面开发，逐步从 Mock 切换到 Demo 后端接口。 - -优先联调页面： - -- `/login` -- `/dashboard` -- `/dataset` -- `/dataset/create` -- `/dataset/:id/preview` -- `/model-manage` -- `/model-manage/create` -- `/fine-tune` -- `/fine-tune/create` -- `/training-log/:id` -- `/compute` -- `/compute/gpus` -- `/compute/queue` -- `/compute/nodes` - -### 2.2 后端 - -基于 `backend/` FastAPI 工程实现 Demo API： - -- 用户登录和当前用户。 -- 数据集、模型、训练任务。 -- 算力节点、GPU、队列。 -- 资源副本和同步任务。 -- 训练日志、指标、checkpoint。 -- 审计日志最小记录。 - -### 2.3 数据库 - -开发阶段使用项目自带 PostgreSQL。Demo 使用 `docs/postgres-schema.sql` 的核心子集，并通过 seed 数据初始化演示数据。 - -### 2.4 算力服务 - -基于 `compute/` 开发内部 Compute API： - -- `simulator` 模式：模拟 GPU 与训练生命周期。 -- `real` 模式：预留真实 GPU 和 LLaMA-Factory 调用。 - -每个算力节点仍按“单机多 GPU 节点”设计。多节点 Demo 可以通过多条 `compute_nodes` 记录模拟，也可以在多台机器上分别部署 `docker/compute`。 - -## 3. 模块开发清单 - -### 3.1 后端基础模块 - -目录建议： - -```text -backend/app/modules/ - auth/ - dataset/ - model/ - fine_tune/ - compute_gateway/ - audit/ -``` - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| POST | `/api/login` | 登录，Demo 可使用固定账号 | -| GET | `/api/me` | 当前用户 | -| GET | `/api/dashboard/overview` | Demo 看板 | -| GET | `/api/health` | 应用健康检查 | - -验收： - -- 能通过前端登录。 -- 能返回菜单权限。 -- 前端退出后可重新登录。 - -### 3.2 数据集 Demo - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/api/dataset-manage` | 数据集列表 | -| POST | `/api/dataset-manage` | 创建数据集 | -| GET | `/api/dataset-manage/{id}` | 数据集详情 | -| POST | `/api/dataset-manage/upload/{dataset_id}` | 上传或模拟上传文件 | -| GET | `/api/dataset-manage/preview/{file_id}` | 文件预览 | -| GET | `/api/dataset-manage/versions/{file_id}` | 文件版本 | - -Demo 行为： - -- 创建数据集后写入 PostgreSQL。 -- 文件内容可以存本地 demo 目录或数据库小样本字段。 -- 预览支持 JSONL 和文本。 -- 数据集创建后生成一条 `storage_objects` 记录。 - -### 3.3 模型 Demo - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/api/model-manage` | 模型列表 | -| POST | `/api/model-manage` | 新增模型 | -| GET | `/api/model-manage/{id}` | 模型详情 | -| GET | `/api/model-manage/local-models` | 本地模型路径列表 | -| GET | `/api/model-manage/trained-models` | 训练产物列表 | -| POST | `/api/model-manage/merge` | 模拟权重合并任务 | - -Demo 行为： - -- 新增本地模型时登记路径，不要求真实权重存在。 -- 训练完成后自动生成 `trained_models` 记录。 -- 权重合并可创建一个 `compute_jobs` 模拟任务。 - -### 3.4 算力节点与 GPU Demo - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/api/compute/nodes` | 算力节点列表 | -| POST | `/api/compute/nodes` | 新增节点 | -| PUT | `/api/compute/nodes/{id}` | 编辑节点 | -| POST | `/api/compute/nodes/{id}/test-connection` | 测试连接 | -| POST | `/api/compute/nodes/{id}/enable` | 启用 | -| POST | `/api/compute/nodes/{id}/disable` | 禁用 | -| POST | `/api/compute/nodes/{id}/drain` | 维护模式 | -| GET | `/api/compute/gpus` | GPU 状态 | -| GET | `/api/compute/queue` | 队列 | - -Demo 行为： - -- 默认 seed 两个算力节点： - - `gpu-node-01`，4 张模拟 GPU。 - - `gpu-node-02`，4 张模拟 GPU。 -- 支持节点标签：`A800`、`4090`、`80GB`、`llama_factory`。 -- 支持节点状态：`online`、`offline`、`draining`、`maintenance`。 -- GPU 状态随训练任务变化：`idle -> reserved -> running -> idle`。 - -### 3.5 资源副本与同步 Demo - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/api/compute/nodes/{id}/replicas` | 节点资源副本 | -| POST | `/api/internal/compute-sync/resources` | 创建资源同步任务 | -| GET | `/api/internal/compute-sync/resources/{id}` | 同步任务详情 | - -Demo 行为： - -- 训练任务启动前检查目标节点是否已有模型和数据集副本。 -- 如果缺失，创建 `resource_sync_jobs`。 -- 同步任务状态模拟：`pending -> running -> completed`。 -- 同步完成后写入 `resource_replicas`。 -- Demo 不需要真实复制大文件，可以创建本地占位文件或只写元数据。 - -### 3.6 微调任务 Demo - -接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/api/fine-tune` | 训练任务列表 | -| POST | `/api/fine-tune` | 创建训练任务 | -| POST | `/api/fine-tune/start` | 启动训练任务 | -| GET | `/api/fine-tune/{id}` | 任务详情 | -| GET | `/api/fine-tune/{id}/overview` | 训练概览 | -| GET | `/api/fine-tune/{id}/events` | 训练事件 | -| POST | `/api/fine-tune/{id}/stop` | 停止任务 | -| POST | `/api/fine-tune/{id}/retry` | 重试任务 | -| GET | `/api/fine-tune/{id}/checkpoints` | checkpoint 列表 | - -创建任务参数需要支持： - -```json -{ - "name": "demo-sft-task", - "project_id": "uuid", - "base_model_id": "uuid", - "train_dataset_id": "uuid", - "engine": "llama_factory", - "scheduler": { - "mode": "auto", - "requested_node_id": null, - "required_tags": ["llama_factory"], - "min_gpu_memory_mb": 24000 - }, - "training_args": { - "stage": "sft", - "finetuning_type": "lora", - "epochs": 3, - "learning_rate": 0.0002 - } -} -``` - -Demo 行为： - -- 自动调度：选择 `enabled + online` 节点，优先资源副本命中，按空闲 GPU、队列长度、权重排序。 -- 手动调度：使用用户指定的 `requested_node_id`。 -- 任务状态模拟：`pending -> syncing -> queued -> running -> completed`。 -- 训练期间每 2-5 秒追加日志和指标。 -- 完成后生成 checkpoint 和 trained model。 - -### 3.7 Compute API Demo - -算力服务内部接口： - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| GET | `/health` | 节点健康 | -| GET | `/api/v1/compute/health` | 节点详细健康 | -| GET | `/compute/resources/gpus` | GPU 状态 | -| POST | `/compute/jobs` | 创建任务 | -| GET | `/compute/jobs/{id}` | 查询任务 | -| POST | `/compute/jobs/{id}/stop` | 停止任务 | -| GET | `/compute/jobs/{id}/logs` | 拉取日志 | -| POST | `/compute/files/upload` | 文件网关上传 | -| GET | `/compute/files/{id}/download` | 文件网关下载 | - -`simulator` 模式行为： - -- 在内存或 SQLite/PostgreSQL 中维护任务状态。 -- 使用后台线程/async task 模拟训练进度。 -- 生成结构化日志、loss 曲线和 checkpoint。 - -`real` 模式预留： - -- `nvidia-smi` 采集 GPU。 -- `subprocess.Popen` 启动 LLaMA-Factory。 -- 解析真实训练日志。 -- 扫描真实 checkpoint 和 adapter。 - -### 3.8 LLaMA-Factory Adapter Demo - -目录建议： - -```text -compute/engines/llama_factory/ - adapter.py - schemas.py - command_builder.py - log_parser.py - simulator.py -``` - -能力： - -- `validate_config(config)`：校验训练参数。 -- `prepare_workspace(job)`：准备工作目录。 -- `build_command(job)`：生成 LLaMA-Factory 命令。 -- `start(job)`：启动真实或模拟训练。 -- `stop(job_id)`：停止任务。 -- `status(job_id)`：查询状态。 -- `parse_log(line)`：解析 loss、epoch、step、learning rate。 -- `collect_artifacts(job_id)`：收集 checkpoint、adapter、merged model。 - -Demo 阶段必须完成： - -- `dry_run` 命令生成。 -- `simulator` 训练。 -- 日志解析器单元测试。 - -真实训练阶段再完成： - -- `real` 进程启动。 -- 真实停止。 -- 真实产物扫描。 - -## 4. 数据库 Demo 子集 - -第一版 Demo 最小使用表： - -- `users` -- `tenants` -- `projects` -- `models` -- `trained_models` -- `datasets` -- `dataset_files` -- `storage_objects` -- `fine_tune_tasks` -- `fine_tune_metrics` -- `fine_tune_checkpoints` -- `compute_nodes` -- `compute_node_engines` -- `gpu_devices` -- `compute_jobs` -- `gpu_allocations` -- `resource_replicas` -- `resource_sync_jobs` -- `audit_logs` - -Seed 数据： - -- 管理员用户：`admin / admin123`。 -- 租户：`demo-tenant`。 -- 项目：`demo-project`。 -- 模型： - - `Qwen2.5-7B-Instruct` - - `Llama-3.1-8B-Instruct` -- 数据集： - - `finance-sft-demo` - - `customer-service-demo` -- 算力节点： - - `gpu-node-01` - - `gpu-node-02` -- GPU：每个节点 4 张模拟 GPU。 -- 训练任务：至少 3 条，分别处于 `pending`、`running`、`completed`。 - -## 5. 目录和配置建议 - -### 5.1 应用后端配置 - -```env -APP_ENV=demo -DATABASE_URL=postgresql+asyncpg://yg_ft:change_me@postgres:5432/yg_ft -REDIS_URL=redis://redis:6379/0 -COMPUTE_STATUS_SYNC_MODE=polling -COMPUTE_POLL_INTERVAL_SECONDS=3 -DEMO_MODE=true -``` - -### 5.2 算力服务配置 - -```env -COMPUTE_MODE=simulator -COMPUTE_HOST_ID=gpu-node-01 -LLAMA_FACTORY_HOME=/opt/LLaMA-Factory -YG_FT_DATA_ROOT=/data/yg-ft -ENABLE_APP_CALLBACK=false -``` - -### 5.3 本地文件目录 - -```text -runtime/ - app/ - data/ - logs/backend/ - compute/ - logs/ - training-logs/ - data/ - tenants/ -``` - -## 6. 开发阶段计划 - -### 阶段 1：后端和 DB 最小闭环 - -目标： - -- FastAPI 能启动。 -- PostgreSQL 能初始化。 -- Seed 数据可导入。 -- 前端能登录并读取真实接口。 - -任务： - -- 实现 DB session。 -- 建立 Alembic 或 SQL 初始化流程。 -- 实现 `auth`、`dashboard`、`dataset`、`model` 基础接口。 -- 完成 Docker app 启动说明。 - -验收： - -- `GET /api/health` 正常。 -- `POST /api/login` 成功。 -- 前端模型/数据集列表来自后端 DB。 - -### 阶段 2：Compute Simulator - -目标： - -- 算力节点、GPU、队列可演示。 -- 训练任务可以模拟运行。 - -任务： - -- 实现 `compute/api/main.py` 内部接口。 -- 实现模拟 GPU 状态。 -- 实现模拟任务生命周期。 -- 实现训练日志和 loss 生成。 -- 实现应用后端轮询同步。 - -验收： - -- 前端 `/compute/gpus` 可看到 GPU 动态状态。 -- 创建训练任务后 GPU 状态变化。 -- `/training-log/:id` 能看到日志和曲线。 - -### 阶段 3：微调主链路 - -目标： - -- 训练任务从创建到完成可完整演示。 - -任务： - -- 实现调度器。 -- 实现资源副本检查。 -- 实现资源同步任务模拟。 -- 实现 checkpoint 和 trained model 登记。 -- 前端微调创建页接入真实后端。 - -验收： - -- 自动调度能选择节点。 -- 手动指定节点能生效。 -- 缺资源时先同步再训练。 -- 训练完成后模型产物出现在模型管理页。 - -### 阶段 4：LLaMA-Factory Adapter Dry Run - -目标： - -- 即使没有真实 GPU，也能展示平台如何生成 LLaMA-Factory 命令和配置。 - -任务： - -- 实现参数校验。 -- 实现 YAML/命令生成。 -- 实现日志解析器。 -- 在训练详情页展示命令预览。 - -验收： - -- 创建训练任务后能查看 LLaMA-Factory 命令。 -- 参数错误能返回可读错误。 -- 日志解析器能从样例日志提取 loss。 - -### 阶段 5：真实 GPU/LLaMA-Factory 可选接入 - -目标： - -- 在有 GPU 环境时可切换为真实训练。 - -任务： - -- 接入 `nvidia-smi`。 -- 检查 CUDA/Driver/PyTorch。 -- 启动 LLaMA-Factory 训练进程。 -- 停止训练进程。 -- 扫描真实 checkpoint。 - -验收： - -- `COMPUTE_MODE=real` 时能读取真实 GPU。 -- 能启动一个最小 LLaMA-Factory 样例任务。 -- 真实日志能显示在训练日志页。 - -## 7. 前后端联调顺序 - -1. 登录。 -2. 看板。 -3. 模型列表。 -4. 数据集列表。 -5. 算力节点和 GPU。 -6. 微调创建。 -7. 训练任务列表。 -8. 训练日志详情。 -9. 模型产物列表。 -10. 推理对话 Mock/后端接口。 - -## 8. Demo 验收标准 - -必须满足： - -- 不依赖真实 GPU 时，Demo 仍可完整跑通。 -- 前端核心页面不再只依赖静态 Mock。 -- 数据写入 PostgreSQL，刷新页面后仍存在。 -- 训练任务状态会自动流转。 -- GPU 状态会随任务变化。 -- 训练日志会持续追加。 -- loss 曲线会随训练推进变化。 -- 训练完成后生成 checkpoint 和训练产物。 -- 多算力节点页面能展示节点权重、标签、启用状态和维护状态。 -- 资源副本页面能展示模型/数据集在哪些节点已有缓存。 - -可选满足： - -- 接入真实 `nvidia-smi`。 -- 接入真实 LLaMA-Factory。 -- 支持真实文件上传到算力节点本地磁盘。 - -## 9. 风险和约束 - -| 风险 | 影响 | Demo 处理 | -| --- | --- | --- | -| 无 GPU 环境 | 不能真实训练 | 使用 `COMPUTE_MODE=simulator` | -| 无 LLaMA-Factory | 不能启动训练框架 | 使用 adapter `dry_run` 和 simulator | -| 文件太大 | 本地 Demo 慢或失败 | Demo 只使用小样本和占位文件 | -| 前端页面仍有 Mock 依赖 | 联调不完整 | 逐页替换 API，不一次性重写 | -| 多节点真实网络不可用 | 无法多机演示 | 用多条 `compute_nodes` 模拟多节点 | - -## 10. 建议第一批开发任务 - -第一批建议只做 8 个任务： - -1. 后端 DB session 和配置。 -2. Seed 数据脚本。 -3. 登录、模型、数据集基础接口。 -4. Compute simulator 基础接口。 -5. GPU 动态状态模拟。 -6. 微调任务创建和状态机。 -7. 训练日志/指标模拟。 -8. 前端微调主链路接入后端。 - -完成这 8 个任务后，就可以形成第一版可演示 Demo。 diff --git a/docs/deployment-plan.md b/docs/deployment-plan.md index df8d693..38c1cf9 100644 --- a/docs/deployment-plan.md +++ b/docs/deployment-plan.md @@ -111,11 +111,11 @@ flowchart LR | 服务 | 端口 | 暴露范围 | | --- | --- | --- | | Nginx | 80/443 | 用户网段 | -| Backend API | 8000 | 仅 Nginx、本机 | -| Compute API | 9100 | 仅 Backend API、本机 | -| File Gateway | 9101 | 仅 Backend API、本机 | -| PostgreSQL | 5432 | 本机或内网 | -| Redis | 6379 | 本机或内网 | +| Backend API | 17861 | 仅 Nginx、本机 | +| Compute API | 19100 | 仅 Backend API、本机 | +| File Gateway | 19101 | 仅 Backend API、本机 | +| PostgreSQL | 15432 | 本机或内网 | +| Redis | 16379 | 本机或内网 | ## 5. 方案二：应用服务与算力/训练服务独立部署 @@ -275,9 +275,9 @@ API_PREFIX=/api DATABASE_URL=postgresql+asyncpg://yg_ft:***@postgres:5432/yg_ft REDIS_URL=redis://redis:6379/0 LOG_DIR=/opt/yg-ft/logs/backend -COMPUTE_API_BASE_URL=https://compute.internal:9100 +COMPUTE_API_BASE_URL=https://compute.internal:19100 COMPUTE_SERVICE_TOKEN=*** -FILE_GATEWAY_BASE_URL=https://compute.internal:9101 +FILE_GATEWAY_BASE_URL=https://compute.internal:19101 COMPUTE_STATUS_SYNC_MODE=polling COMPUTE_POLL_INTERVAL_SECONDS=10 COMPUTE_POLL_BATCH_SIZE=100 @@ -288,12 +288,11 @@ COMPUTE_POLL_BATCH_SIZE=100 ```env COMPUTE_ENV=prod COMPUTE_HOST_ID=gpu-node-01 -COMPUTE_API_PORT=9100 -FILE_GATEWAY_PORT=9101 +COMPUTE_API_PORT=19100 +FILE_GATEWAY_PORT=19101 COMPUTE_SERVICE_TOKEN=*** ENABLE_APP_CALLBACK=false -LLAMA_FACTORY_HOME=/opt/LLaMA-Factory -LLAMA_FACTORY_HOST_PATH=/opt/LLaMA-Factory +LLAMA_FACTORY_HOME=/app/LLaMA-Factory YG_FT_DATA_ROOT=/data/yg-ft LOG_DIR=/opt/yg-ft/logs/compute CUDA_VISIBLE_DEVICES=0,1,2,3 @@ -360,8 +359,7 @@ docker/ ```bash cd docker/app cp .env.example .env -docker compose --profile build run --rm frontend-builder -docker compose up -d --build +docker compose up -d ``` 算力服务器启动： @@ -369,14 +367,14 @@ docker compose up -d --build ```bash cd docker/compute cp .env.example .env -docker compose up -d --build +docker compose up -d ``` 应用服务器与算力服务器独立部署时，需要在 `docker/app/.env` 中配置： ```env -COMPUTE_API_BASE_URL=http://:9100 -FILE_GATEWAY_BASE_URL=http://:9101 +COMPUTE_API_BASE_URL=http://:19100 +FILE_GATEWAY_BASE_URL=http://:19101 COMPUTE_SERVICE_TOKEN=change_me ``` @@ -385,9 +383,9 @@ COMPUTE_SERVICE_TOKEN=change_me 多节点后，每台算力服务器各自进入 `docker/compute` 启动一套算力服务，并在应用平台中登记为一条 `compute_nodes` 记录： ```text -gpu-node-01 -> http://10.10.20.31:9100 / http://10.10.20.31:9101 -gpu-node-02 -> http://10.10.20.32:9100 / http://10.10.20.32:9101 -gpu-node-03 -> http://10.10.20.33:9100 / http://10.10.20.33:9101 +gpu-node-01 -> http://10.10.20.31:19100 / http://10.10.20.31:19101 +gpu-node-02 -> http://10.10.20.32:19100 / http://10.10.20.32:19101 +gpu-node-03 -> http://10.10.20.33:19100 / http://10.10.20.33:19101 ``` 算力服务器需要在 `docker/compute/.env` 中配置： @@ -395,7 +393,6 @@ gpu-node-03 -> http://10.10.20.33:9100 / http://10.10.20.33:9101 ```env ENABLE_APP_CALLBACK=false COMPUTE_SERVICE_TOKEN=change_me -LLAMA_FACTORY_HOST_PATH=/opt/LLaMA-Factory YG_FT_DATA_ROOT_HOST=/data/yg-ft ``` diff --git a/docs/first-version-development-plan.md b/docs/first-version-development-plan.md new file mode 100644 index 0000000..98054b1 --- /dev/null +++ b/docs/first-version-development-plan.md @@ -0,0 +1,135 @@ +# 第一版系统开发计划 + +> 说明：本计划虽然以“可演示闭环”为阶段目标，但实现内容按正式系统第一版开发，不在工程模块、配置项、接口标签、数据库对象中使用临时代码命名。初始化数据仅作为样例种子数据，后续可在此基础上继续扩展为生产能力。 + +## 1. 阶段目标 + +第一版系统需要在没有完整生产环境、没有真实 GPU 或没有 LLaMA-Factory 运行环境时，仍然可以跑通模型微调平台的主链路： + +```text +登录 + -> 模型管理 + -> 数据集管理 + -> 创建微调任务 + -> 调度算力节点与 GPU + -> 模拟资源副本检查和同步 + -> 启动训练任务 + -> 查看任务状态、GPU 占用、训练日志、loss 曲线 + -> 训练完成后登记训练产物 +``` + +该阶段不是一次性展示代码，而是后续系统继续开发的基础版本。算力、训练框架、资源同步、任务状态机均按真实平台边界设计，当前仅以 `simulator` 模式补齐无 GPU 环境下的可运行能力。 + +## 2. 运行模式 + +| 模式 | 说明 | 当前状态 | +| --- | --- | --- | +| `simulator` | 模拟 GPU、训练进程、日志、指标、checkpoint 和训练产物 | 第一版默认实现 | +| `real` | 预留真实 `nvidia-smi`、LLaMA-Factory 进程启动、真实日志解析和产物扫描 | 后续接入 | + +第一版实现必须保留 `real` 模式边界，不允许把训练命令直接写死在应用平台后端。训练相关能力统一沉淀在 `compute/engines/` 适配层。 + +## 3. 第一批开发范围 + +### 3.1 应用平台后端 + +对应目录： + +```text +backend/app/ + api/v1/endpoints/platform.py + db/platform_store.py + db/session.py +``` + +已覆盖能力： +- 统一 API 响应结构 `{ code, message, data }`。 +- 初始化样例用户、模型、数据集、算力节点、GPU 和微调任务。 +- 登录、当前用户、用户列表与权限。 +- 模型管理、训练产物列表、权重合并任务占位。 +- 数据集管理、文件上传、预览、版本管理和下载。 +- 微调任务创建、启动、停止、删除、进度查询、checkpoint 查询。 +- 系统健康指标、系统信息、训练日志、系统日志。 +- 算力节点、GPU、队列、资源副本、资源同步任务接口。 + +开发约定： +- 当前本地可运行存储使用 SQLite，配置项为 `LOCAL_DB_PATH`。 +- 生产数据库仍以 PostgreSQL 为目标，正式表结构以 `docs/postgres-schema.sql` 为准。 +- `backend/app/db/session.py` 已提供 SQLAlchemy session 入口，后续替换正式 ORM 时从此处收口。 + +### 3.2 算力平台服务 + +对应目录： + +```text +compute/ + api/main.py + engines/llama_factory/adapter.py +``` + +已覆盖能力： +- `/health` 与 `/api/v1/compute/health` 节点健康检查。 +- `/compute/resources/gpus` GPU 状态模拟。 +- `/compute/jobs` 创建、查询、停止训练任务。 +- `/compute/jobs/{id}/logs` 拉取训练日志与解析指标。 +- `/compute/files/upload` 与 `/compute/files/{id}/download` 文件网关占位。 +- LLaMA-Factory 参数校验、命令生成、日志指标解析。 + +设计约定： +- 多算力节点阶段，每台单机多 GPU 服务器都部署一套 Compute API、Agent、File Gateway 和训练框架。 +- 应用平台通过 `compute_nodes` 中维护的 `api_base_url`、`file_gateway_url` 主动轮询算力节点。 +- 算力节点之间默认不互访，资源副本和同步状态由应用平台统一编排。 + +### 3.3 前端页面 + +已接入页面： +- `/login`：登录接口。 +- `/model-manage`：模型列表与模型来源。 +- `/dataset`、`/dataset/:id/preview`：数据集列表、预览和版本。 +- `/fine-tune`、`/fine-tune/create`：微调任务创建、启动、状态轮询。 +- `/training-log/:id`：训练日志和 loss 曲线。 +- `/hardware`：平台 GPU 与系统性能。 +- `/compute`：算力节点、GPU、队列、资源副本。 + +新增前端能力： +- `frontend/src/api/modules/compute.ts`：算力节点 API 包装。 +- `frontend/src/views/compute/ComputeNodesView.vue`：节点、GPU、队列、资源副本四个视图。 +- 权限枚举增加 `compute`，菜单增加“算力节点”入口。 + +### 3.4 数据库 + +第一版运行时使用本地 SQLite 以降低开发环境门槛，但数据库模型仍按 PostgreSQL 正式设计推进。 + +核心表范围： +- 用户、租户、项目、角色、权限。 +- 模型、训练产物、数据集、数据集文件、存储对象。 +- 微调任务、训练指标、checkpoint、审计日志。 +- 算力节点、GPU 设备、算力任务、GPU 分配。 +- 资源副本、资源同步任务。 + +优化方向： +- 任务状态、租户/项目隔离字段、资源副本定位字段需要建立索引。 +- 审计日志、训练指标、日志归档后续建议按时间分区。 +- 大文件只落本地磁盘或对象存储，数据库保存元数据和版本关系。 + +## 4. 验收标准 + +第一版完成后应满足： +- 前端不启用静态 Mock 时，可以通过 FastAPI 获取主要页面数据。 +- 登录后可以看到模型、数据集、微调任务和算力节点。 +- 创建微调任务后可以启动任务，任务状态自动流转。 +- GPU 状态会随任务进入 `syncing`、`queued`、`running` 发生变化。 +- 训练日志持续生成，日志详情页能解析 loss、grad norm、learning rate、epoch。 +- 任务完成后训练产物出现在模型管理的训练产物列表中。 +- 算力节点页面可展示节点地址、权重、标签、启用状态、队列和资源副本。 +- 工程命名、配置和接口分组均可作为后续正式开发基础。 + +## 5. 后续开发计划 + +| 阶段 | 重点 | 说明 | +| --- | --- | --- | +| 第二阶段 | PostgreSQL ORM 与 Alembic | 将 `platform_store.py` 的能力迁移为正式 ORM、迁移脚本和 repository/service 分层 | +| 第三阶段 | 真实 Compute Agent | 接入 `nvidia-smi`、GPU 锁定、进程管理、训练日志文件采集 | +| 第四阶段 | LLaMA-Factory 真实训练 | 生成 YAML/命令、启动训练进程、停止任务、扫描 checkpoint 和 adapter | +| 第五阶段 | 企业治理 | 多租户、项目/模型/数据集隔离、审批流、审计留存、配额和资源申请 | +| 第六阶段 | 多节点调度 | 基于 `compute_nodes`、资源副本和节点标签实现自动/手动调度策略 | diff --git a/docs/superpowers/plans/2026-07-10-data-process-create-wizard.md b/docs/superpowers/plans/2026-07-10-data-process-create-wizard.md index 166d3f1..9a1f8e6 100644 --- a/docs/superpowers/plans/2026-07-10-data-process-create-wizard.md +++ b/docs/superpowers/plans/2026-07-10-data-process-create-wizard.md @@ -431,9 +431,9 @@ Expected: 三项检查全部 PASS。 - [ ] **Step 3: 启动页面并逐步验证四步交互** -Run: `cd frontend && npm run dev -- --host 0.0.0.0 --port 6801` +Run: `cd frontend && npm run dev -- --host 0.0.0.0 --port 16801` -Browser checks at `http://localhost:6801/data-process/create`: +Browser checks at `http://localhost:16801/data-process/create`: 1. 第一步上传文本并选择非结构化数据。 2. 第二步点击至少三个右侧切片，确认左侧滚动目标和高亮范围变化。 diff --git a/frontend/README.md b/frontend/README.md index 883fe95..a9a1f35 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -22,9 +22,9 @@ npm install npm run dev ``` -开发服务器默认运行在 `http://localhost:6801`。 +开发服务器默认运行在 `http://localhost:16801`。 -后端 API 默认通过 Vite 代理转发到 `http://localhost:7861`（见 `vite.config.ts`）。 +后端 API 默认通过 Vite 代理转发到 `http://localhost:17861`（见 `vite.config.ts`）。 开发环境默认启用前端 Mock。如需联调真实后端，使用： diff --git a/frontend/src/api/modules/compute.ts b/frontend/src/api/modules/compute.ts new file mode 100644 index 0000000..6d80d34 --- /dev/null +++ b/frontend/src/api/modules/compute.ts @@ -0,0 +1,85 @@ +import { get, post, put } from '../request' + +export interface ComputeNode { + id: string + code: string + name: string + api_base_url: string + file_gateway_url: string + enabled: boolean + scheduler_status: string + scheduler_weight: number + tags: string[] + gpu_count: number + current_running_jobs: number + max_parallel_jobs: number + data_root: string + model_root: string + log_root: string + last_health_check_at?: string +} + +export interface ComputeGpu { + id: number + node_id: string + node_code: string + node_name: string + name: string + uuid: string + status: string + gpu_percent: number + memory_used_gb: number + memory_total_gb: number + memory_percent: number + temperature: number + power_w: number + power_limit_w: number + processes?: Array<{ + pid: number + name: string + memory_used_gb: number + task_name?: string + user?: string + }> +} + +export interface ComputeQueueItem { + id: string + name: string + status: string + progress: number + compute_node_id?: string + gpus: number[] + create_time?: string +} + +export interface ResourceReplica { + id: string + node_id: string + resource_type: string + resource_id: string + local_path: string + status: string + sync_status: string + create_time: string +} + +export const getComputeNodes = () => get('/compute/nodes') + +export const updateComputeNode = (id: string, data: Partial) => + put(`/compute/nodes/${id}`, data) + +export const testComputeNode = (id: string) => + post<{ node_id: string; success: boolean; latency_ms: number }>(`/compute/nodes/${id}/test-connection`) + +export const enableComputeNode = (id: string) => post(`/compute/nodes/${id}/enable`) + +export const disableComputeNode = (id: string) => post(`/compute/nodes/${id}/disable`) + +export const drainComputeNode = (id: string) => post(`/compute/nodes/${id}/drain`) + +export const getComputeGpus = () => get('/compute/gpus') + +export const getComputeQueue = () => get('/compute/queue') + +export const getNodeReplicas = (id: string) => get(`/compute/nodes/${id}/replicas`) diff --git a/frontend/src/api/request.ts b/frontend/src/api/request.ts index 4c44b53..9bd26c0 100644 --- a/frontend/src/api/request.ts +++ b/frontend/src/api/request.ts @@ -12,7 +12,7 @@ export interface ApiResult { } const service: AxiosInstance = axios.create({ - // 统一走相对路径，由 Vite 代理转发到 http://localhost:7861 + // Use a relative path; Vite proxies /api to http://localhost:17861 in local development. baseURL: '/api', timeout: 30000, }) diff --git a/frontend/src/assets/login-hero-flow.jpg b/frontend/src/assets/login-hero-flow.jpg new file mode 100644 index 0000000..5c139c4 Binary files /dev/null and b/frontend/src/assets/login-hero-flow.jpg differ diff --git a/frontend/src/components/AppSidebar.vue b/frontend/src/components/AppSidebar.vue index 97372f5..86473a7 100644 --- a/frontend/src/components/AppSidebar.vue +++ b/frontend/src/components/AppSidebar.vue @@ -68,6 +68,12 @@ const menuGroups: MenuGroup[] = [ title: '其他工具', items: [{ key: 'data-convert', label: '数据类型转换', icon: 'fa-exchange', to: '/data-convert', permission: 'data-convert' }], }, + { + title: '算力资源', + items: [ + { key: 'compute', label: '算力节点', icon: 'fa-microchip', to: '/compute', permission: 'compute' }, + ], + }, { title: '系统设置', items: [ diff --git a/frontend/src/mock/users.ts b/frontend/src/mock/users.ts new file mode 100644 index 0000000..ff340b3 --- /dev/null +++ b/frontend/src/mock/users.ts @@ -0,0 +1,144 @@ +import type { + CreateUserPayload, + LoginResponse, + PermissionCode, + SystemUser, + UpdateUserAccessPayload, +} from '@/types' + +const STORAGE_KEY = 'mock:system-users' + +const allPermissions: PermissionCode[] = [ + 'dashboard', + 'fine-tune', + 'model-eval', + 'model-inference', + 'model-manage', + 'dataset', + 'data-process', + 'data-convert', + 'compute', + 'hardware', + 'logs', + 'user-settings', +] + +const defaultPasswords: Record = { + admin: 'admin123', + operator: 'operator123', +} + +export class UserMutationError extends Error { + status: number + + constructor(message: string, status = 400) { + super(message) + this.name = 'UserMutationError' + this.status = status + } +} + +function defaultUsers(): SystemUser[] { + return [ + { + id: 'u_admin', + username: 'admin', + display_name: 'Platform Admin', + role: 'admin', + status: 'active', + permissions: allPermissions, + create_time: '2026-01-01T00:00:00Z', + protected: true, + }, + { + id: 'u_operator', + username: 'operator', + display_name: 'Platform Operator', + role: 'operator', + status: 'active', + permissions: allPermissions.filter((item) => item !== 'user-settings'), + create_time: '2026-01-01T00:00:00Z', + protected: false, + }, + ] +} + +function readUsers(): SystemUser[] { + const raw = localStorage.getItem(STORAGE_KEY) + if (!raw) return defaultUsers() + try { + const users = JSON.parse(raw) as SystemUser[] + return users.length ? users : defaultUsers() + } catch { + return defaultUsers() + } +} + +function writeUsers(users: SystemUser[]) { + localStorage.setItem(STORAGE_KEY, JSON.stringify(users)) +} + +export function listMockUsers(): SystemUser[] { + return readUsers() +} + +export function authenticateMockUser(username: string, password: string): LoginResponse { + const users = readUsers() + const user = users.find((item) => item.username === username) + if (!user || user.status !== 'active') { + throw new UserMutationError('Invalid username or disabled account', 401) + } + const expected = defaultPasswords[username] || 'platform123' + if (password !== expected) { + throw new UserMutationError('Invalid username or password', 401) + } + user.last_login = new Date().toISOString() + writeUsers(users) + return { + token: `mock-token-${user.id}`, + user, + } +} + +export function createMockUser(payload: CreateUserPayload): SystemUser { + const users = readUsers() + if (users.some((item) => item.username === payload.username)) { + throw new UserMutationError('Username already exists', 409) + } + const user: SystemUser = { + id: `u_${Date.now()}`, + username: payload.username, + display_name: payload.display_name || payload.username, + role: payload.role, + status: payload.status || 'active', + permissions: payload.permissions || [], + create_time: new Date().toISOString(), + protected: false, + } + defaultPasswords[user.username] = payload.password || 'platform123' + users.push(user) + writeUsers(users) + return user +} + +export function updateMockUserAccess(id: string, payload: UpdateUserAccessPayload): SystemUser { + const users = readUsers() + const user = users.find((item) => item.id === id) + if (!user) throw new UserMutationError('User not found', 404) + if (payload.role) user.role = payload.role + if (payload.status) user.status = payload.status + if (payload.permissions) user.permissions = payload.permissions + writeUsers(users) + return user +} + +export function deleteMockUser(id: string, currentUsername: string): { deleted: string } { + const users = readUsers() + const user = users.find((item) => item.id === id) + if (!user) throw new UserMutationError('User not found', 404) + if (user.protected || user.username === currentUsername) { + throw new UserMutationError('Protected or current user cannot be deleted', 400) + } + writeUsers(users.filter((item) => item.id !== id)) + return { deleted: id } +} diff --git a/frontend/src/plugins/echarts-hardware.ts b/frontend/src/plugins/echarts-hardware.ts new file mode 100644 index 0000000..f7c1f9f --- /dev/null +++ b/frontend/src/plugins/echarts-hardware.ts @@ -0,0 +1,17 @@ +import { use } from 'echarts/core' +import { CanvasRenderer } from 'echarts/renderers' +import { GaugeChart, LineChart } from 'echarts/charts' +import { + GridComponent, + LegendComponent, + TooltipComponent, +} from 'echarts/components' + +use([ + CanvasRenderer, + LineChart, + GaugeChart, + GridComponent, + TooltipComponent, + LegendComponent, +]) diff --git a/frontend/src/plugins/echarts-training-log.ts b/frontend/src/plugins/echarts-training-log.ts new file mode 100644 index 0000000..4ce6041 --- /dev/null +++ b/frontend/src/plugins/echarts-training-log.ts @@ -0,0 +1,18 @@ +import { use } from 'echarts/core' +import { CanvasRenderer } from 'echarts/renderers' +import { LineChart } from 'echarts/charts' +import { + DataZoomComponent, + GridComponent, + LegendComponent, + TooltipComponent, +} from 'echarts/components' + +use([ + CanvasRenderer, + LineChart, + GridComponent, + TooltipComponent, + LegendComponent, + DataZoomComponent, +]) diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index a51b5f3..c02d992 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -197,6 +197,24 @@ const routes: RouteRecordRaw[] = [ component: () => import('@/views/data-convert/DataConvertView.vue'), meta: { title: '数据类型转换' }, }, + { + path: 'compute', + name: 'compute', + component: () => import('@/views/compute/ComputeNodesView.vue'), + meta: { title: '算力节点', pageSurface: 'self', permission: 'compute' }, + }, + { + path: 'compute/gpus', + redirect: '/compute?tab=gpus', + }, + { + path: 'compute/queue', + redirect: '/compute?tab=queue', + }, + { + path: 'compute/nodes', + redirect: '/compute?tab=nodes', + }, // 系统设置 { path: 'permission-denied', @@ -264,6 +282,7 @@ const permissionBySegment: Record = { dataset: 'dataset', 'data-process': 'data-process', 'data-convert': 'data-convert', + compute: 'compute', tools: 'data-convert', hardware: 'hardware', logs: 'logs', diff --git a/frontend/src/stores/auth.ts b/frontend/src/stores/auth.ts index a4949d9..5a3eb24 100644 --- a/frontend/src/stores/auth.ts +++ b/frontend/src/stores/auth.ts @@ -15,6 +15,7 @@ const allPermissions: PermissionCode[] = [ 'dataset', 'data-process', 'data-convert', + 'compute', 'hardware', 'logs', 'user-settings', diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index b664c6c..450bcf4 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -394,6 +394,7 @@ export type PermissionCode = | 'dataset' | 'data-process' | 'data-convert' + | 'compute' | 'hardware' | 'logs' | 'user-settings' diff --git a/frontend/src/views/compute/ComputeNodesView.vue b/frontend/src/views/compute/ComputeNodesView.vue new file mode 100644 index 0000000..835d4ca --- /dev/null +++ b/frontend/src/views/compute/ComputeNodesView.vue @@ -0,0 +1,376 @@ + + + + + diff --git a/frontend/src/views/eval/create/BasicMetricSetupStep.vue b/frontend/src/views/eval/create/BasicMetricSetupStep.vue new file mode 100644 index 0000000..962a96d --- /dev/null +++ b/frontend/src/views/eval/create/BasicMetricSetupStep.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/frontend/src/views/eval/create/StartEvalStep.vue b/frontend/src/views/eval/create/StartEvalStep.vue new file mode 100644 index 0000000..8c48e6d --- /dev/null +++ b/frontend/src/views/eval/create/StartEvalStep.vue @@ -0,0 +1,49 @@ + + + + + diff --git a/frontend/src/views/guide/GuideView.vue b/frontend/src/views/guide/GuideView.vue new file mode 100644 index 0000000..4e7a078 --- /dev/null +++ b/frontend/src/views/guide/GuideView.vue @@ -0,0 +1,12 @@ + + + diff --git a/frontend/src/views/system/PermissionDeniedView.vue b/frontend/src/views/system/PermissionDeniedView.vue new file mode 100644 index 0000000..c00917b --- /dev/null +++ b/frontend/src/views/system/PermissionDeniedView.vue @@ -0,0 +1,12 @@ + + + diff --git a/frontend/src/views/system/UserCreateView.vue b/frontend/src/views/system/UserCreateView.vue new file mode 100644 index 0000000..b6cfb4f --- /dev/null +++ b/frontend/src/views/system/UserCreateView.vue @@ -0,0 +1,94 @@ + + + + + diff --git a/frontend/src/views/system/UserPermissionView.vue b/frontend/src/views/system/UserPermissionView.vue new file mode 100644 index 0000000..a79c2b8 --- /dev/null +++ b/frontend/src/views/system/UserPermissionView.vue @@ -0,0 +1,12 @@ + + + diff --git a/frontend/src/views/system/UserSettingsView.vue b/frontend/src/views/system/UserSettingsView.vue new file mode 100644 index 0000000..ccf4534 --- /dev/null +++ b/frontend/src/views/system/UserSettingsView.vue @@ -0,0 +1,65 @@ + + + + + diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 4e77070..98143be 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -22,11 +22,11 @@ export default defineConfig({ }, }, server: { - port: 6801, + port: 16801, proxy: { - // 后端服务固定在 7861 端口，前端统一走 /api 相对路径 + // Frontend uses /api and proxies to the local five-digit backend port. '/api': { - target: 'http://localhost:7861', + target: 'http://localhost:17861', changeOrigin: true, }, },