YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 忽略离线部署包，提交安全加固、数据库初始化与文档

Browse Source 
- .gitignore: 忽略 docker/offline 离线部署包（镜像/运行时等大文件）
- 安全加固: 新增 compute/api/security.py 及各端安全测试，补充 docs/security-hardening.md
- 数据库: 新增完整初始化 SQL 与 docs/database-config.md
- 数据转换与评测: 修复类型检查、增强校验并补充测试
- Docker 配置与环境变量更新

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-07 09:24:35 +08:00
parent e397bcc2ca
commit 75cc105ebc
24 changed files with 1850 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
backend/app/modules/data_convert/router.py
View File

@@ -5,10 +5,11 @@ import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, UploadFile, File
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi.responses import FileResponse
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user
from app.db.platform_store import get_platform_store, new_id
@@ -18,6 +19,30 @@ router = APIRouter(prefix="/data-convert", tags=["data-convert"])
STORAGE_ROOT = Path(__file__).resolve().parents[3] / "storage" / "data-convert"
def _safe_output_filename(value: Any) -> str:
"""输出文件名白名单校验：仅允许普通文件名，阻断 ``../``、``/``、``\\`` 等路径穿越。
转换结果始终写入 ``STORAGE_ROOT/<task_id>/output/<output_filename>``，
若文件名可被注入路径分隔符，将导致任意文件读写/删除。
"""
name = str(value or "converted-data.jsonl").strip()
if (
not name
or name in {".", ".."}
or name != Path(name).name
or "/" in name
or "\\" in name
or any(ord(character) < 32 or ord(character) == 127 for character in name)
):
raise fail(400, "output filename must be a plain file name")
return name
def _task_output_path(task: dict[str, Any]) -> Path:
"""返回经过白名单校验的转换输出文件路径（始终位于任务 output 目录内）。"""
return _output_dir(task["id"]) / _safe_output_filename(task.get("output_filename"))
def _task_dir(task_id: str) -> Path:
return STORAGE_ROOT / task_id
@@ -31,7 +56,11 @@ def _output_dir(task_id: str) -> Path:
@router.get("")
def list_tasks(page: int = 1, page_size: int = 20) -> dict[str, Any]:
def list_tasks(
page: int = 1,
page_size: int = 20,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
store = get_platform_store()
with store.connect() as conn:
rows = conn.execute(
@@ -46,12 +75,15 @@ def list_tasks(page: int = 1, page_size: int = 20) -> dict[str, Any]:
@router.post("")
def create_task(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
def create_task(
payload: dict[str, Any] = Body(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
name = str(payload.get("name") or "").strip()
if not name:
raise fail(400, "name is required")
task_id = new_id("dct")
output_filename = str(payload.get("output_filename") or "converted-data.jsonl").strip()
output_filename = _safe_output_filename(payload.get("output_filename"))
description = str(payload.get("description") or "").strip()
store = get_platform_store()
with store.connect() as conn:
@@ -67,7 +99,10 @@ def create_task(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
@router.get("/{task_id}")
def get_task(task_id: str) -> dict[str, Any]:
def get_task(
task_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
@@ -86,6 +121,7 @@ def get_task(task_id: str) -> dict[str, Any]:
async def upload_source_files(
task_id: str,
files: list[UploadFile] = File(...),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
task = _get_task(task_id)
if not task:
@@ -114,7 +150,7 @@ async def upload_source_files(
try:
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / (task["output_filename"] or "converted-data.jsonl")
output_path = _task_output_path(task)
# 清空旧输出（如果重新上传）
if output_path.exists():
output_path.unlink()
@@ -157,7 +193,7 @@ async def upload_source_files(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, task["output_filename"] or "converted-data.jsonl", content)
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
return ok({
"staged_files": staged,
"auto_converted": True,
@@ -175,7 +211,10 @@ async def upload_source_files(
@router.post("/{task_id}/run")
def run_convert(task_id: str) -> dict[str, Any]:
def run_convert(
task_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
@@ -192,7 +231,7 @@ def run_convert(task_id: str) -> dict[str, Any]:
input_dir = _input_dir(task_id)
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / (task["output_filename"] or "converted-data.jsonl")
output_path = _task_output_path(task)
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
@@ -229,19 +268,22 @@ def run_convert(task_id: str) -> dict[str, Any]:
@router.get("/{task_id}/download")
def download_result(task_id: str):
def download_result(
task_id: str,
current_user: dict = Depends(get_current_user),
):
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _output_dir(task_id) / (task["output_filename"] or "converted-data.jsonl")
output_path = _task_output_path(task)
if not output_path.exists():
raise fail(404, "output file not found")
return FileResponse(
str(output_path),
media_type="application/octet-stream",
filename=task["output_filename"] or "converted-data.jsonl",
filename=_safe_output_filename(task.get("output_filename")),
)
@@ -249,6 +291,7 @@ def download_result(task_id: str):
def import_as_dataset(
task_id: str,
payload: dict[str, Any] = Body(default={}),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
"""把已转换的 JSONL 文件导入为数据集管理中的上传任务记录（source='task'）。"""
task = _get_task(task_id)
@@ -256,7 +299,7 @@ def import_as_dataset(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _output_dir(task_id) / (task["output_filename"] or "converted-data.jsonl")
output_path = _task_output_path(task)
if not output_path.exists():
raise fail(404, "output file not found")
content = output_path.read_text(encoding="utf-8")
@@ -277,12 +320,15 @@ def import_as_dataset(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, task["output_filename"] or "converted-data.jsonl", content)
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
return ok({"dataset_id": dataset_id, "name": dataset_name})
@router.delete("/{task_id}")
def delete_task(task_id: str) -> dict[str, Any]:
def delete_task(
task_id: str,
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")

10
backend/app/modules/data_process/dataset_format.py
View File

@@ -26,6 +26,16 @@ def _load_sample(path: str | None, content: str | None = None, max_samples: int
if not text:
return []
# 先按整文件 JSON（数组/单对象）解析，兼容 .json；失败再按 jsonl 逐行解析
try:
value = json.loads(text)
except json.JSONDecodeError:
value = None
if isinstance(value, list):
return [item for item in value[:max_samples] if isinstance(item, dict)]
if isinstance(value, dict):
return [value]
lines = text.splitlines()[:max_samples]
records: list[dict[str, Any]] = []
for line in lines: