from __future__ import annotations import re from dataclasses import dataclass from pathlib import Path from typing import Any @dataclass(frozen=True) class LlamaFactoryCommand: command: list[str] work_dir: str env: dict[str, str] def validate_config(config: dict[str, Any]) -> list[str]: errors: list[str] = [] if not config.get("base_model") and not config.get("model_name_or_path"): errors.append("base_model or model_name_or_path is required") if not config.get("dataset") and not config.get("dataset_dir"): errors.append("dataset or dataset_dir is required") try: learning_rate = float(config.get("learning_rate", 0.0002)) except (TypeError, ValueError): learning_rate = 0 if learning_rate <= 0: errors.append("learning_rate must be greater than zero") try: epochs = int(config.get("n_epochs", config.get("num_train_epochs", 1))) except (TypeError, ValueError): epochs = 0 if epochs <= 0: errors.append("n_epochs must be greater than zero") return errors def _optional_arg(config: dict[str, Any], command: list[str], option: str, *keys: str) -> None: for key in keys: value = config.get(key) if value is not None and value != "": command.extend([option, str(value)]) return def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-Factory") -> LlamaFactoryCommand: errors = validate_config(config) if errors: raise ValueError("; ".join(errors)) engine = str(config.get("engine") or config.get("training_engine") or "llama_factory") if engine == "smoke": output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-smoke')}" script = ( "import json, os, time; " f"out={str(output_dir)!r}; " "os.makedirs(out, exist_ok=True); " "print('[INFO] smoke training started', flush=True); " "\nfor step in range(1, 7):\n" " loss=round(1.8/(step+1), 4)\n" " lr=round(0.0002*(1-step/10), 8)\n" " print({'loss': loss, 'grad_norm': round(0.4 + step*0.03, 4), 'learning_rate': lr, 'epoch': round(step/6, 4)}, flush=True)\n" " time.sleep(0.4)\n" "\nopen(os.path.join(out, 'adapter_config.json'), 'w', encoding='utf-8').write(json.dumps({'engine':'smoke','status':'completed'})); " "print('***** train metrics *****', flush=True); " "print('train_loss = 0.12', flush=True); " "print('***** train metrics end *****', flush=True)" ) return LlamaFactoryCommand(command=["python", "-u", "-c", script], work_dir="/app", env={}) model_path = config.get("base_model") or config.get("model_name_or_path") dataset = config.get("dataset") or config.get("dataset_name") dataset_dir = config.get("dataset_dir") output_dir = config.get("output_dir") or f"/data/yg-ft/outputs/{config.get('name', 'training-job')}" command = [ "llamafactory-cli", "train", "--stage", str(config.get("stage", "sft")).lower(), "--do_train", "true", "--model_name_or_path", str(model_path), "--dataset", str(dataset or "default"), "--template", str(config.get("template", "qwen")), "--finetuning_type", str(config.get("train_method", config.get("finetuning_type", "lora"))), "--output_dir", str(output_dir), "--per_device_train_batch_size", str(config.get("batch_size", 2)), "--learning_rate", str(config.get("learning_rate", 0.0002)), "--num_train_epochs", str(config.get("n_epochs", 3)), "--save_steps", str(config.get("save_steps", 50)), "--logging_steps", str(config.get("logging_steps", 10)), "--overwrite_output_dir", "true", "--plot_loss", "true", ] if dataset_dir: command.extend(["--dataset_dir", str(dataset_dir)]) _optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len") _optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type") _optional_arg(config, command, "--warmup_ratio", "warmup_ratio") _optional_arg(config, command, "--weight_decay", "weight_decay") _optional_arg(config, command, "--lora_rank", "lora_rank", "rank") _optional_arg(config, command, "--lora_alpha", "lora_alpha") _optional_arg(config, command, "--lora_dropout", "lora_dropout") quantization_bit = int(config.get("quantization_bit", 0) or 0) if quantization_bit in {4, 8}: command.extend(["--quantization_bit", str(quantization_bit)]) return LlamaFactoryCommand(command=command, work_dir=str(Path(llama_factory_home)), env={}) def parse_log_line(line: str) -> dict[str, float] | None: if "loss" not in line or "learning_rate" not in line: return None result: dict[str, float] = {} for key in ["loss", "grad_norm", "learning_rate", "epoch"]: match = re.search(rf"['\"]?{key}['\"]?\s*:\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)", line) if match: result[key] = float(match.group(1)) return result or None