chore: 忽略离线部署包,提交安全加固、数据库初始化与文档

- .gitignore: 忽略 docker/offline 离线部署包(镜像/运行时等大文件)
- 安全加固: 新增 compute/api/security.py 及各端安全测试,补充 docs/security-hardening.md
- 数据库: 新增完整初始化 SQL 与 docs/database-config.md
- 数据转换与评测: 修复类型检查、增强校验并补充测试
- Docker 配置与环境变量更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-07 09:24:35 +08:00
parent e397bcc2ca
commit 75cc105ebc
24 changed files with 1850 additions and 50 deletions

View File

@@ -1,6 +1,8 @@
from __future__ import annotations
from compute.engines.llama_factory.adapter import build_command
import json
from compute.engines.llama_factory.adapter import _validate_dataset_columns, build_command
def test_build_command_uses_explicit_validation_dataset_without_resplitting() -> None:
@@ -21,3 +23,70 @@ def test_build_command_uses_explicit_validation_dataset_without_resplitting() ->
)
assert "--do_eval" in result.command
assert "--val_size" not in result.command
def _write(tmp_path, name: str, lines: list[dict]) -> object:
path = tmp_path / name
path.write_text(
"".join(json.dumps(line, ensure_ascii=False) + "\n" for line in lines),
encoding="utf-8",
)
return path
def test_jsonl_alpaca_without_input_column_passes_validation(tmp_path) -> None:
"""纯 jsonl Alpaca 数据缺省 input 字段(常见),不应被校验拦截。"""
_write(tmp_path, "train.jsonl", [{"instruction": "hi", "output": "hello"}])
errors = _validate_dataset_columns(
{
"dataset_dir": str(tmp_path),
"dataset_info": {
"ygft_a": {
"file_name": "train.jsonl",
"formatting": "alpaca",
"columns": {"prompt": "instruction", "query": "input", "response": "output"},
}
},
}
)
assert errors == []
def test_jsonl_sharegpt_passes_validation(tmp_path) -> None:
"""ShareGPT 格式 jsonlmessages应通过校验。"""
_write(
tmp_path,
"msg.jsonl",
[{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}],
)
errors = _validate_dataset_columns(
{
"dataset_dir": str(tmp_path),
"dataset_info": {
"ygft_m": {
"file_name": "msg.jsonl",
"formatting": "sharegpt",
"columns": {"messages": "messages"},
}
},
}
)
assert errors == []
def test_jsonl_missing_response_still_rejected(tmp_path) -> None:
"""缺 outputresponse仍应报错——没有答案无法做有监督微调。"""
_write(tmp_path, "train.jsonl", [{"instruction": "hi"}])
errors = _validate_dataset_columns(
{
"dataset_dir": str(tmp_path),
"dataset_info": {
"ygft_a": {
"file_name": "train.jsonl",
"formatting": "alpaca",
"columns": {"prompt": "instruction", "query": "input", "response": "output"},
}
},
}
)
assert errors and "output" in errors[0]