YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data-process): 发布精确三路数据切分

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-24 20:43:47 +08:00
parent e6a5a36bc0
commit 9cb77c251a
8 changed files with 405 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
compute/engines/llama_factory/adapter.py
View File

@@ -265,6 +265,9 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
]
if dataset_dir:
command.extend(["--dataset_dir", str(dataset_dir)])
eval_dataset = config.get("eval_dataset")
if eval_dataset:
command.extend(["--eval_dataset", str(eval_dataset), "--do_eval", "true"])
_optional_arg(config, command, "--cutoff_len", "max_length", "cutoff_len")
_optional_arg(config, command, "--lr_scheduler_type", "lr_scheduler_type")
_optional_arg(config, command, "--warmup_ratio", "warmup_ratio")
@@ -273,7 +276,8 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
_optional_arg(config, command, "--lora_alpha", "lora_alpha")
_optional_arg(config, command, "--lora_dropout", "lora_dropout")
_optional_arg(config, command, "--gradient_accumulation_steps", "gradient_accumulation_steps")
_optional_arg(config, command, "--val_size", "val_size")
if not eval_dataset:
_optional_arg(config, command, "--val_size", "val_size")
_optional_arg(config, command, "--max_samples", "max_samples")
_optional_arg(config, command, "--preprocessing_num_workers", "preprocessing_num_workers")
_optional_bool_arg(config, command, "--fp16", "fp16")
@@ -293,4 +297,3 @@ def parse_log_line(line: str) -> dict[str, float] | None:
if match:
result[key] = float(match.group(1))
return result or None