chore: 忽略离线部署包,提交安全加固、数据库初始化与文档
- .gitignore: 忽略 docker/offline 离线部署包(镜像/运行时等大文件) - 安全加固: 新增 compute/api/security.py 及各端安全测试,补充 docs/security-hardening.md - 数据库: 新增完整初始化 SQL 与 docs/database-config.md - 数据转换与评测: 修复类型检查、增强校验并补充测试 - Docker 配置与环境变量更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from compute.engines.llama_factory.adapter import build_command
|
||||
import json
|
||||
|
||||
from compute.engines.llama_factory.adapter import _validate_dataset_columns, build_command
|
||||
|
||||
|
||||
def test_build_command_uses_explicit_validation_dataset_without_resplitting() -> None:
|
||||
@@ -21,3 +23,70 @@ def test_build_command_uses_explicit_validation_dataset_without_resplitting() ->
|
||||
)
|
||||
assert "--do_eval" in result.command
|
||||
assert "--val_size" not in result.command
|
||||
|
||||
|
||||
def _write(tmp_path, name: str, lines: list[dict]) -> object:
|
||||
path = tmp_path / name
|
||||
path.write_text(
|
||||
"".join(json.dumps(line, ensure_ascii=False) + "\n" for line in lines),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def test_jsonl_alpaca_without_input_column_passes_validation(tmp_path) -> None:
|
||||
"""纯 jsonl Alpaca 数据缺省 input 字段(常见),不应被校验拦截。"""
|
||||
_write(tmp_path, "train.jsonl", [{"instruction": "hi", "output": "hello"}])
|
||||
errors = _validate_dataset_columns(
|
||||
{
|
||||
"dataset_dir": str(tmp_path),
|
||||
"dataset_info": {
|
||||
"ygft_a": {
|
||||
"file_name": "train.jsonl",
|
||||
"formatting": "alpaca",
|
||||
"columns": {"prompt": "instruction", "query": "input", "response": "output"},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
assert errors == []
|
||||
|
||||
|
||||
def test_jsonl_sharegpt_passes_validation(tmp_path) -> None:
|
||||
"""ShareGPT 格式 jsonl(messages)应通过校验。"""
|
||||
_write(
|
||||
tmp_path,
|
||||
"msg.jsonl",
|
||||
[{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}],
|
||||
)
|
||||
errors = _validate_dataset_columns(
|
||||
{
|
||||
"dataset_dir": str(tmp_path),
|
||||
"dataset_info": {
|
||||
"ygft_m": {
|
||||
"file_name": "msg.jsonl",
|
||||
"formatting": "sharegpt",
|
||||
"columns": {"messages": "messages"},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
assert errors == []
|
||||
|
||||
|
||||
def test_jsonl_missing_response_still_rejected(tmp_path) -> None:
|
||||
"""缺 output(response)仍应报错——没有答案无法做有监督微调。"""
|
||||
_write(tmp_path, "train.jsonl", [{"instruction": "hi"}])
|
||||
errors = _validate_dataset_columns(
|
||||
{
|
||||
"dataset_dir": str(tmp_path),
|
||||
"dataset_info": {
|
||||
"ygft_a": {
|
||||
"file_name": "train.jsonl",
|
||||
"formatting": "alpaca",
|
||||
"columns": {"prompt": "instruction", "query": "input", "response": "output"},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
assert errors and "output" in errors[0]
|
||||
|
||||
Reference in New Issue
Block a user