chore: 忽略离线部署包,提交安全加固、数据库初始化与文档
- .gitignore: 忽略 docker/offline 离线部署包(镜像/运行时等大文件) - 安全加固: 新增 compute/api/security.py 及各端安全测试,补充 docs/security-hardening.md - 数据库: 新增完整初始化 SQL 与 docs/database-config.md - 数据转换与评测: 修复类型检查、增强校验并补充测试 - Docker 配置与环境变量更新 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,10 @@ from typing import Any
|
||||
|
||||
|
||||
def _load_dataset(path: str) -> list[dict[str, Any]]:
|
||||
"""Load a JSON or JSONL dataset file.
|
||||
"""Load a JSON or JSONL dataset file (jsonl-compatible).
|
||||
|
||||
Content-sniffs instead of trusting the extension so jsonl files with a BOM,
|
||||
a single JSON array on one line, or mislabeled extensions all load correctly.
|
||||
|
||||
Supports common field names used across the platform:
|
||||
* ``instruction`` + ``input`` + ``output`` (Alpaca-style)
|
||||
@@ -30,14 +33,17 @@ def _load_dataset(path: str) -> list[dict[str, Any]]:
|
||||
* ``messages`` (ShareGPT-style – the last assistant message is treated as reference)
|
||||
"""
|
||||
file_path = Path(path)
|
||||
text = file_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
text = file_path.read_text(encoding="utf-8-sig", errors="replace").strip()
|
||||
if not text:
|
||||
return []
|
||||
if file_path.suffix.lower() == ".json":
|
||||
try:
|
||||
value = json.loads(text)
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
return [value] if isinstance(value, dict) else []
|
||||
except json.JSONDecodeError:
|
||||
value = None
|
||||
if isinstance(value, list):
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
if isinstance(value, dict):
|
||||
return [value]
|
||||
|
||||
samples: list[dict[str, Any]] = []
|
||||
for line in text.splitlines():
|
||||
|
||||
Reference in New Issue
Block a user