chore: 忽略离线部署包,提交安全加固、数据库初始化与文档

- .gitignore: 忽略 docker/offline 离线部署包(镜像/运行时等大文件)
- 安全加固: 新增 compute/api/security.py 及各端安全测试,补充 docs/security-hardening.md
- 数据库: 新增完整初始化 SQL 与 docs/database-config.md
- 数据转换与评测: 修复类型检查、增强校验并补充测试
- Docker 配置与环境变量更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-07 09:24:35 +08:00
parent e397bcc2ca
commit 75cc105ebc
24 changed files with 1850 additions and 50 deletions

View File

@@ -239,10 +239,18 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
fmt = str(formatting).lower()
for key, file_name in zip(llama_dataset_keys(dataset_key, file_names), file_names):
if fmt == "sharegpt":
# 平台校验按 OpenAI 风格消息role/content故 tags 用 role/content
# 与 LLaMA-Factory 默认的 from/value 不同,需显式声明避免解析失败。
result[key] = {
"file_name": file_name,
"formatting": "sharegpt",
"columns": {"messages": "messages"},
"tags": {
"role_tag": "role",
"content_tag": "content",
"user_tag": "user",
"assistant_tag": "assistant",
},
}
elif fmt == "dpo":
result[key] = {
@@ -273,6 +281,47 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
return result
def _sniff_dataset_format(sample_text: str, max_samples: int = 20) -> str:
"""嗅探数据集内容格式(兼容 jsonl返回 sharegpt / dpo / cpt / alpaca。
按内容而非文件名判断,纯 jsonl 数据集(如 ShareGPT messages、缺省 input 的
Alpaca都能被正确识别避免训练任务误按 alpaca 解析而失败。
"""
text = (sample_text or "").strip()
if not text:
return ""
records: list[dict[str, Any]] = []
try:
value = json.loads(text)
except (TypeError, ValueError, json.JSONDecodeError):
value = None
if isinstance(value, list):
records = [item for item in value[:max_samples] if isinstance(item, dict)]
elif isinstance(value, dict):
records = [value]
else:
for line in text.splitlines()[:max_samples]:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(obj, dict):
records.append(obj)
records = records[:max_samples]
if not records:
return ""
if all("messages" in record for record in records):
return "sharegpt"
if all(record.get("chosen") and record.get("rejected") for record in records):
return "dpo"
if all(record.get("text") and not (record.get("instruction") or record.get("output")) for record in records):
return "cpt"
return "alpaca"
PASSWORD_HASH_ITERATIONS = 390_000
@@ -2076,18 +2125,38 @@ class PlatformStore:
runtime_keys = llama_dataset_keys(dataset_key, runtime_file_names)
training_keys = runtime_keys[: len(training_files)]
validation_keys = runtime_keys[len(training_files) :]
dataset_format = str(task.get("dataset_format") or (dataset and dataset.get("formatting")) or "alpaca").lower()
# P0-2: Validate dataset content against declared format
# P0-2: 推导数据集格式并校验内容(兼容 jsonl按内容嗅探 ShareGPT/DPO/CPT/Alpaca
train_type = str(task.get("train_type", task.get("train_method", ""))).upper()
expected_format = {
"DPO": "dpo",
"CPT": "cpt",
}.get(train_type)
expected_format = {"DPO": "dpo", "CPT": "cpt"}.get(train_type)
raw_format = str(
task.get("dataset_format")
or dataset_metadata.get("format")
or (dataset and dataset.get("formatting"))
or "alpaca"
).lower()
sniffed_format = ""
content_samples: dict[str, str] = {}
if training_files:
with self.connect() as conn:
for file_entry in training_files:
sample_row = conn.execute(
"SELECT substr(content, 1, 400000) AS sample FROM dataset_files WHERE id=?",
(str(file_entry["id"]),),
).fetchone()
sample = (sample_row or {}).get("sample") or ""
content_samples[str(file_entry["id"])] = sample
if not sniffed_format:
sniffed_format = _sniff_dataset_format(sample)
known_formats = {"sharegpt", "dpo", "cpt", "pt", "pretrain"}
if expected_format:
dataset_format = expected_format
elif raw_format in known_formats:
dataset_format = raw_format
else:
dataset_format = sniffed_format or raw_format or "alpaca"
format_errors: list[str] = []
for file_entry in training_files:
content = file_entry.get("content") or ""
content = content_samples.get(str(file_entry["id"])) or ""
if content:
from app.modules.data_process.dataset_format import validate_dataset_format
file_errors = validate_dataset_format(dataset_format, content=content)