feat: 新增 compute_gateway、compute_poller、agent 模块,重构前端 dist
- 新增 backend/app/modules/compute_gateway(client/sync)计算网关模块 - 新增 backend/app/workers/compute_poller 计算轮询 worker - 新增 compute/agent/process_manager 进程管理器 - 新增 scripts/ 脚本目录 - 更新 Docker 部署配置(app/compute/nginx) - 更新后端平台 API、数据库 SQL、core 配置 - 更新前端多个视图组件及 API 模块 - 重构 frontend/dist 构建产物(新 hash) - 更新多项文档 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -19,22 +19,57 @@ def validate_config(config: dict[str, Any]) -> list[str]:
|
||||
errors.append("base_model or model_name_or_path is required")
|
||||
if not config.get("dataset") and not config.get("dataset_dir"):
|
||||
errors.append("dataset or dataset_dir is required")
|
||||
learning_rate = float(config.get("learning_rate", 0.0002))
|
||||
try:
|
||||
learning_rate = float(config.get("learning_rate", 0.0002))
|
||||
except (TypeError, ValueError):
|
||||
learning_rate = 0
|
||||
if learning_rate <= 0:
|
||||
errors.append("learning_rate must be greater than zero")
|
||||
epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
|
||||
try:
|
||||
epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1)))
|
||||
except (TypeError, ValueError):
|
||||
epochs = 0
|
||||
if epochs <= 0:
|
||||
errors.append("n_epochs must be greater than zero")
|
||||
return errors
|
||||
|
||||
|
||||
def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None:
|
||||
for key in keys:
|
||||
value = config.get(key)
|
||||
if value is not None and value != "":
|
||||
command.extend([option, str(value)])
|
||||
return
|
||||
|
||||
|
||||
def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand:
|
||||
errors = validate_config(config)
|
||||
if errors:
|
||||
raise ValueError("; ".join(errors))
|
||||
|
||||
engine = str(config.get("engine") or config.get("training_engine") or "llama_factory")
|
||||
if engine == "smoke":
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}"
|
||||
script = (
|
||||
"import json, os, time; "
|
||||
f"out={str(output_dir)!r}; "
|
||||
"os.makedirs(out, exist_ok=True); "
|
||||
"print('[INFO] smoke training started', flush=True); "
|
||||
"\nfor step in range(1, 7):\n"
|
||||
" loss=round(1.8/(step+1), 4)\n"
|
||||
" lr=round(0.0002*(1-step/10), 8)\n"
|
||||
" print({'loss': loss, 'grad_norm': round(0.4 + step*0.03, 4), 'learning_rate': lr, 'epoch': round(step/6, 4)}, flush=True)\n"
|
||||
" time.sleep(0.4)\n"
|
||||
"\nopen(os.path.join(out, 'adapter_config.json'), 'w', encoding='utf-8').write(json.dumps({'engine':'smoke','status':'completed'})); "
|
||||
"print('***** train metrics *****', flush=True); "
|
||||
"print('train_loss = 0.12', flush=True); "
|
||||
"print('***** train metrics end *****', flush=True)"
|
||||
)
|
||||
return LlamaFactoryCommand(command=["python", "-u", "-c", script], work_dir="/app", env={})
|
||||
|
||||
model_path = config.get("base_model") or config.get("model_name_or_path")
|
||||
dataset = config.get("dataset") or config.get("dataset_dir")
|
||||
dataset = config.get("dataset") or config.get("dataset_name")
|
||||
dataset_dir = config.get("dataset_dir")
|
||||
output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}"
|
||||
command = [
|
||||
"llamafactory-cli",
|
||||
@@ -46,7 +81,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
"--model_name_or_path",
|
||||
str(model_path),
|
||||
"--dataset",
|
||||
str(dataset),
|
||||
str(dataset or "default"),
|
||||
"--template",
|
||||
str(config.get("template", "qwen")),
|
||||
"--finetuning_type",
|
||||
@@ -61,7 +96,22 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
str(config.get("n_epochs", 3)),
|
||||
"--save_steps",
|
||||
str(config.get("save_steps", 50)),
|
||||
"--logging_steps",
|
||||
str(config.get("logging_steps", 10)),
|
||||
"--overwrite_output_dir",
|
||||
"true",
|
||||
"--plot_loss",
|
||||
"true",
|
||||
]
|
||||
if dataset_dir:
|
||||
command.extend(["--dataset_dir", str(dataset_dir)])
|
||||
_optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len")
|
||||
_optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type")
|
||||
_optional_arg(config, command, "--warmup_ratio", "warmup_ratio")
|
||||
_optional_arg(config, command, "--weight_decay", "weight_decay")
|
||||
_optional_arg(config, command, "--lora_rank", "lora_rank", "rank")
|
||||
_optional_arg(config, command, "--lora_alpha", "lora_alpha")
|
||||
_optional_arg(config, command, "--lora_dropout", "lora_dropout")
|
||||
quantization_bit = int(config.get("quantization_bit", 0) or 0)
|
||||
if quantization_bit in {4, 8}:
|
||||
command.extend(["--quantization_bit", str(quantization_bit)])
|
||||
|
||||
Reference in New Issue
Block a user