第一次提交

compute/README.md

@@ -0,0 +1,29 @@
+# Compute Platform
+
+算力平台与应用平台分开部署，本目录用于后续实现单机多 GPU 调度、文件网关和训练引擎适配。
+
+## 目录结构
+
+```text
+compute/
+  api/                 # 只允许应用平台访问的内部 Compute API
+  agent/               # 单机 Agent，负责 GPU、进程、工作区管理
+  engines/
+    llama_factory/     # LLaMA-Factory 训练引擎适配器
+  file_gateway/        # 本地磁盘上传、下载、预览、离线导入
+  tests/
+```
+
+## 开发职责
+
+- GPU 发现、状态上报、锁定和释放。
+- 本地磁盘工作区管理。
+- 创建、停止、查询训练/评测/推理/合并任务。
+- LLaMA-Factory 命令生成、日志解析、产物收集。
+- 分片上传、短时下载、离线导入。
+- 通过服务间 token 接受应用平台调用。
+
+## 运行模式
+
+- 默认 `COMPUTE_EXECUTION_MODE=real`，Compute API 只暴露健康检查和接口契约；真实训练执行器完成前，创建作业会返回未实现错误。
+- 仅隔离联调时可设置 `COMPUTE_EXECUTION_MODE=simulator`，启用内存状态机和合成 GPU/日志数据。该模式不得作为生产运行路径。

compute/__init__.py

@@ -0,0 +1 @@
+"""Compute platform package."""

compute/agent/__init__.py

@@ -0,0 +1 @@
+"""Compute agent package."""

compute/api/__init__.py

@@ -0,0 +1 @@
+"""Compute API package."""

compute/api/main.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import os
+import math
+import time
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, HTTPException
+
+from compute.engines.llama_factory.adapter import build_command, parse_log_line
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="YG Fine-Tune Compute API")
+    jobs: dict[str, dict[str, Any]] = {}
+    route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
+
+    def now() -> float:
+        return time.time()
+
+    def host_id() -> str:
+        return os.getenv("COMPUTE_HOST_ID", "gpu-node-01")
+
+    def execution_mode() -> str:
+        return os.getenv("COMPUTE_EXECUTION_MODE", os.getenv("COMPUTE_MODE", "real")).lower()
+
+    def job_status(job: dict[str, Any]) -> dict[str, Any]:
+        if execution_mode() != "simulator":
+            return job
+        elapsed = max(0, int(now() - job["created_at"]))
+        if job["status"] not in {"stopped", "failed", "completed"}:
+            if elapsed < 5:
+                job["status"] = "queued"
+                job["progress"] = 12 + elapsed * 3
+            elif elapsed < 60:
+                job["status"] = "running"
+                job["progress"] = min(96, 25 + int((elapsed - 5) / 55 * 70))
+            else:
+                job["status"] = "completed"
+                job["progress"] = 100
+        job["logs"] = generate_logs(job)
+        return job
+
+    def generate_logs(job: dict[str, Any]) -> str:
+        progress = int(job.get("progress", 0) or 0)
+        points = max(1, min(80, progress))
+        lines = [
+            f"[INFO] compute_host_id={host_id()} job_id={job['id']} engine=llama_factory",
+            f"[INFO] command={' '.join(job['command'])}",
+        ]
+        for step in range(1, points + 1):
+            if step % 4 != 0 and step != points:
+                continue
+            loss = max(0.11, 2.5 * math.exp(-step / 40))
+            grad_norm = 0.4 + (step % 5) * 0.04
+            lr = 0.0002 * max(0.05, 1 - step / 100)
+            epoch = round(step / points * 3, 4)
+            lines.append(
+                "{"
+                f"'loss': {loss:.4f}, 'grad_norm': {grad_norm:.4f}, "
+                f"'learning_rate': {lr:.8f}, 'epoch': {epoch:.4f}"
+                "}"
+            )
+        if job.get("status") == "completed":
+            lines.extend(
+                [
+                    "***** train metrics *****",
+                    "epoch = 3",
+                    "train_loss = 0.1181",
+                    "train_runtime = 1m 0s",
+                    "***** train metrics end *****",
+                ]
+            )
+        return "\n".join(lines)
+
+    def gpu_resources() -> list[dict[str, Any]]:
+        if execution_mode() != "simulator":
+            return []
+        active_jobs = [job_status(job) for job in jobs.values() if job["status"] in {"queued", "running"}]
+        gpus: list[dict[str, Any]] = []
+        for idx in range(4):
+            task = next((job for job in active_jobs if idx in job.get("gpus", [])), None)
+            busy = task is not None and task["status"] == "running"
+            reserved = task is not None and task["status"] == "queued"
+            gpus.append(
+                {
+                    "id": idx,
+                    "uuid": f"GPU-{host_id().upper()}-{idx}",
+                    "name": os.getenv("COMPUTE_GPU_NAME", "NVIDIA A800-SXM4-80GB"),
+                    "status": "busy" if busy else "reserved" if reserved else "idle",
+                    "gpu_percent": 88 if busy else 25 if reserved else 4,
+                    "memory_used_gb": 58 if busy else 12 if reserved else 2,
+                    "memory_total_gb": 80,
+                    "temperature": 61 if busy else 45 if reserved else 36,
+                    "power_w": 215 if busy else 80 if reserved else 25,
+                    "power_limit_w": 300,
+                    "processes": [
+                        {
+                            "pid": task["pid"],
+                            "name": "llamafactory-cli",
+                            "task_name": task["name"],
+                            "memory_used_gb": 58 if busy else 12,
+                        }
+                    ]
+                    if task
+                    else [],
+                }
+            )
+        return gpus
+
+    @app.get(f"{route_prefix}/health")
+    async def health_check() -> dict[str, str]:
+        return {
+            "status": "ok",
+            "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
+        }
+
+    @app.get(f"{route_prefix}/v1/compute/health")
+    async def compute_health_check() -> dict[str, str | bool]:
+        data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
+        llama_factory_home = Path(os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
+        return {
+            "status": "ok",
+            "compute_host_id": os.getenv("COMPUTE_HOST_ID", "unknown"),
+            "app_callback_enabled": os.getenv("ENABLE_APP_CALLBACK", "false").lower() == "true",
+            "data_root": str(data_root),
+            "data_root_exists": data_root.exists(),
+            "llama_factory_home": str(llama_factory_home),
+            "llama_factory_home_exists": llama_factory_home.exists(),
+            "execution_mode": execution_mode(),
+        }
+
+    @app.get(f"{route_prefix}/v1/compute/jobs")
+    async def list_jobs_alias() -> dict[str, list[dict[str, Any]]]:
+        return {"items": [job_status(job) for job in jobs.values()]}
+
+    @app.get(f"{route_prefix}/compute/resources/gpus")
+    async def list_gpus() -> dict[str, Any]:
+        return {"items": gpu_resources(), "compute_host_id": host_id()}
+
+    @app.post(f"{route_prefix}/compute/jobs")
+    async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
+        try:
+            command = build_command(payload, os.getenv("LLAMA_FACTORY_HOME", "/app/LLaMA-Factory"))
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc))
+        if execution_mode() != "simulator":
+            raise HTTPException(
+                status_code=501,
+                detail="real compute executor is not implemented yet; set COMPUTE_EXECUTION_MODE=simulator only for isolated development",
+            )
+        job_id = str(payload.get("id") or f"job_{int(now() * 1000)}")
+        job = {
+            "id": job_id,
+            "name": payload.get("name", job_id),
+            "status": "queued",
+            "progress": 10,
+            "pid": int(52000 + now() % 10000),
+            "gpus": payload.get("gpus") or [0],
+            "created_at": now(),
+            "command": command.command,
+            "work_dir": command.work_dir,
+            "artifacts": [],
+            "logs": "",
+        }
+        jobs[job_id] = job
+        return job_status(job)
+
+    @app.get(f"{route_prefix}/compute/jobs")
+    async def list_jobs() -> dict[str, Any]:
+        return {"items": [job_status(job) for job in jobs.values()]}
+
+    @app.get(f"{route_prefix}/compute/jobs/{{job_id}}")
+    async def get_job(job_id: str) -> dict[str, Any]:
+        job = jobs.get(job_id)
+        if not job:
+            raise HTTPException(status_code=404, detail="job not found")
+        return job_status(job)
+
+    @app.post(f"{route_prefix}/compute/jobs/{{job_id}}/stop")
+    async def stop_job(job_id: str) -> dict[str, Any]:
+        job = jobs.get(job_id)
+        if not job:
+            raise HTTPException(status_code=404, detail="job not found")
+        job["status"] = "stopped"
+        job["progress"] = min(job.get("progress", 0), 99)
+        return job
+
+    @app.get(f"{route_prefix}/compute/jobs/{{job_id}}/logs")
+    async def job_logs(job_id: str) -> dict[str, Any]:
+        job = jobs.get(job_id)
+        if not job:
+            raise HTTPException(status_code=404, detail="job not found")
+        job = job_status(job)
+        metrics = [parse_log_line(line) for line in job["logs"].splitlines()]
+        return {"job_id": job_id, "content": job["logs"], "metrics": [m for m in metrics if m]}
+
+    @app.post(f"{route_prefix}/compute/files/upload")
+    async def upload_file(payload: dict[str, Any]) -> dict[str, Any]:
+        file_id = str(payload.get("id") or f"file_{int(now() * 1000)}")
+        return {"id": file_id, "status": "available", "local_path": f"/data/yg-ft/uploads/{file_id}"}
+
+    @app.get(f"{route_prefix}/compute/files/{{file_id}}/download")
+    async def download_file(file_id: str) -> dict[str, Any]:
+        return {"id": file_id, "status": "ready", "download_url": f"{route_prefix}/compute/files/{file_id}/download"}
+
+    return app
+
+
+app = create_app()

compute/engines/__init__.py

@@ -0,0 +1 @@
+"""Training engine adapters package."""

compute/engines/llama_factory/__init__.py

@@ -0,0 +1 @@
+"""LLaMA-Factory engine adapter package."""

compute/engines/llama_factory/adapter.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class LlamaFactoryCommand:
+    command: list[str]
+    work_dir: str
+    env: dict[str, str]
+
+
+def validate_config(config: dict[str, Any]) -> list[str]:
+    errors: list[str] = []
+    if not config.get("base_model") and not config.get("model_name_or_path"):
+        errors.append("base_model or model_name_or_path is required")
+    if not config.get("dataset") and not config.get("dataset_dir"):
+        errors.append("dataset or dataset_dir is required")
+    learning_rate = float(config.get("learning_rate", 0.0002))
+    if learning_rate <= 0:
+        errors.append("learning_rate must be greater than zero")
+    epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
+    if epochs <= 0:
+        errors.append("n_epochs must be greater than zero")
+    return errors
+
+
+def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
+    errors = validate_config(config)
+    if errors:
+        raise ValueError("; ".join(errors))
+
+    model_path = config.get("base_model") or config.get("model_name_or_path")
+    dataset = config.get("dataset") or config.get("dataset_dir")
+    output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}"
+    command = [
+        "llamafactory-cli",
+        "train",
+        "--stage",
+        str(config.get("stage", "sft")).lower(),
+        "--do_train",
+        "true",
+        "--model_name_or_path",
+        str(model_path),
+        "--dataset",
+        str(dataset),
+        "--template",
+        str(config.get("template", "qwen")),
+        "--finetuning_type",
+        str(config.get("train_method", config.get("finetuning_type", "lora"))),
+        "--output_dir",
+        str(output_dir),
+        "--per_device_train_batch_size",
+        str(config.get("batch_size", 2)),
+        "--learning_rate",
+        str(config.get("learning_rate", 0.0002)),
+        "--num_train_epochs",
+        str(config.get("n_epochs", 3)),
+        "--save_steps",
+        str(config.get("save_steps", 50)),
+    ]
+    quantization_bit = int(config.get("quantization_bit", 0) or 0)
+    if quantization_bit in {4, 8}:
+        command.extend(["--quantization_bit", str(quantization_bit)])
+    return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={})
+
+
+def parse_log_line(line: str) -> dict[str, float] | None:
+    if "loss" not in line or "learning_rate" not in line:
+        return None
+    result: dict[str, float] = {}
+    for key in ["loss", "grad_norm", "learning_rate", "epoch"]:
+        match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line)
+        if match:
+            result[key] = float(match.group(1))
+    return result or None
+

compute/file_gateway/__init__.py

@@ -0,0 +1 @@
+"""Local file gateway package."""

compute/requirements.txt

@@ -0,0 +1,5 @@
+fastapi>=0.111.0
+uvicorn[standard]>=0.30.0
+pydantic>=2.7.0
+python-dotenv>=1.0.1
+httpx>=0.27.0

compute/tests/__init__.py

@@ -0,0 +1 @@
+"""Compute platform tests package."""