chore: 忽略离线部署包,提交安全加固、数据库初始化与文档
- .gitignore: 忽略 docker/offline 离线部署包(镜像/运行时等大文件) - 安全加固: 新增 compute/api/security.py 及各端安全测试,补充 docs/security-hardening.md - 数据库: 新增完整初始化 SQL 与 docs/database-config.md - 数据转换与评测: 修复类型检查、增强校验并补充测试 - Docker 配置与环境变量更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -239,10 +239,18 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
|
||||
fmt = str(formatting).lower()
|
||||
for key, file_name in zip(llama_dataset_keys(dataset_key, file_names), file_names):
|
||||
if fmt == "sharegpt":
|
||||
# 平台校验按 OpenAI 风格消息(role/content),故 tags 用 role/content
|
||||
# 与 LLaMA-Factory 默认的 from/value 不同,需显式声明避免解析失败。
|
||||
result[key] = {
|
||||
"file_name": file_name,
|
||||
"formatting": "sharegpt",
|
||||
"columns": {"messages": "messages"},
|
||||
"tags": {
|
||||
"role_tag": "role",
|
||||
"content_tag": "content",
|
||||
"user_tag": "user",
|
||||
"assistant_tag": "assistant",
|
||||
},
|
||||
}
|
||||
elif fmt == "dpo":
|
||||
result[key] = {
|
||||
@@ -273,6 +281,47 @@ def llama_dataset_info(dataset_key: str, file_names: list[str], formatting: str
|
||||
return result
|
||||
|
||||
|
||||
def _sniff_dataset_format(sample_text: str, max_samples: int = 20) -> str:
|
||||
"""嗅探数据集内容格式(兼容 jsonl),返回 sharegpt / dpo / cpt / alpaca。
|
||||
|
||||
按内容而非文件名判断,纯 jsonl 数据集(如 ShareGPT messages、缺省 input 的
|
||||
Alpaca)都能被正确识别,避免训练任务误按 alpaca 解析而失败。
|
||||
"""
|
||||
text = (sample_text or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
records: list[dict[str, Any]] = []
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
value = None
|
||||
if isinstance(value, list):
|
||||
records = [item for item in value[:max_samples] if isinstance(item, dict)]
|
||||
elif isinstance(value, dict):
|
||||
records = [value]
|
||||
else:
|
||||
for line in text.splitlines()[:max_samples]:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
records.append(obj)
|
||||
records = records[:max_samples]
|
||||
if not records:
|
||||
return ""
|
||||
if all("messages" in record for record in records):
|
||||
return "sharegpt"
|
||||
if all(record.get("chosen") and record.get("rejected") for record in records):
|
||||
return "dpo"
|
||||
if all(record.get("text") and not (record.get("instruction") or record.get("output")) for record in records):
|
||||
return "cpt"
|
||||
return "alpaca"
|
||||
|
||||
|
||||
PASSWORD_HASH_ITERATIONS = 390_000
|
||||
|
||||
|
||||
@@ -2076,18 +2125,38 @@ class PlatformStore:
|
||||
runtime_keys = llama_dataset_keys(dataset_key, runtime_file_names)
|
||||
training_keys = runtime_keys[: len(training_files)]
|
||||
validation_keys = runtime_keys[len(training_files) :]
|
||||
dataset_format = str(task.get("dataset_format") or (dataset and dataset.get("formatting")) or "alpaca").lower()
|
||||
# P0-2: Validate dataset content against declared format
|
||||
# P0-2: 推导数据集格式并校验内容(兼容 jsonl:按内容嗅探 ShareGPT/DPO/CPT/Alpaca)
|
||||
train_type = str(task.get("train_type", task.get("train_method", ""))).upper()
|
||||
expected_format = {
|
||||
"DPO": "dpo",
|
||||
"CPT": "cpt",
|
||||
}.get(train_type)
|
||||
expected_format = {"DPO": "dpo", "CPT": "cpt"}.get(train_type)
|
||||
raw_format = str(
|
||||
task.get("dataset_format")
|
||||
or dataset_metadata.get("format")
|
||||
or (dataset and dataset.get("formatting"))
|
||||
or "alpaca"
|
||||
).lower()
|
||||
sniffed_format = ""
|
||||
content_samples: dict[str, str] = {}
|
||||
if training_files:
|
||||
with self.connect() as conn:
|
||||
for file_entry in training_files:
|
||||
sample_row = conn.execute(
|
||||
"SELECT substr(content, 1, 400000) AS sample FROM dataset_files WHERE id=?",
|
||||
(str(file_entry["id"]),),
|
||||
).fetchone()
|
||||
sample = (sample_row or {}).get("sample") or ""
|
||||
content_samples[str(file_entry["id"])] = sample
|
||||
if not sniffed_format:
|
||||
sniffed_format = _sniff_dataset_format(sample)
|
||||
known_formats = {"sharegpt", "dpo", "cpt", "pt", "pretrain"}
|
||||
if expected_format:
|
||||
dataset_format = expected_format
|
||||
elif raw_format in known_formats:
|
||||
dataset_format = raw_format
|
||||
else:
|
||||
dataset_format = sniffed_format or raw_format or "alpaca"
|
||||
format_errors: list[str] = []
|
||||
for file_entry in training_files:
|
||||
content = file_entry.get("content") or ""
|
||||
content = content_samples.get(str(file_entry["id"])) or ""
|
||||
if content:
|
||||
from app.modules.data_process.dataset_format import validate_dataset_format
|
||||
file_errors = validate_dataset_format(dataset_format, content=content)
|
||||
|
||||
Reference in New Issue
Block a user