feat: 权限与日志治理完善,MinIO 独立部署与 tiktoken 离线打包适配

- 后端:强化平台/审批/资源/系统接口权限校验与操作日志,更新权限设计文档与测试用例
- 存储:新增 MinIO 独立部署适配(端口 19000/19001),外部端点与 host-gateway 互通
- 离线:打包 tiktoken cl100k_base 词表进镜像,避免无网环境联网下载
- 其他:算力节点接口微调,前端微调创建页小修,忽略 MinIO 运行时数据

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-12 15:21:23 +08:00
parent 2f64086177
commit 5ecca9f0bc
26 changed files with 101600 additions and 129 deletions

View File

@@ -25,6 +25,7 @@ from compute.engines.llama_factory.inference import get_inference_session
def create_app() -> FastAPI:
app = FastAPI(title="YG Fine-Tune Compute API", **docs_kwargs())
jobs: dict[str, dict[str, Any]] = {}
cache_locks: dict[str, asyncio.Lock] = {}
route_prefix = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF").rstrip("/") or "/modelTF"
process_manager = ProcessManager(os.getenv("TRAINING_LOG_ROOT", "/opt/yg-ft/logs/training"))
@@ -620,6 +621,30 @@ def create_app() -> FastAPI:
)
return {"root": root, "base_path": str(base), "relative_path": relative_path, "items": items}
@app.post(f"{route_prefix}/compute/files/upload-to-url")
async def upload_file_to_url(payload: dict[str, Any]) -> dict[str, Any]:
"""Upload one node-local artifact to a Backend-issued presigned URL."""
source = Path(str(payload.get("path") or "")).resolve()
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft")).resolve()
if not _path_inside(data_root, source) or not source.is_file():
raise HTTPException(status_code=400, detail="artifact path must be an existing file inside YG_FT_DATA_ROOT")
upload_url = str(payload.get("upload_url") or "")
if not upload_url:
raise HTTPException(status_code=400, detail="upload_url is required")
digest = hashlib.sha256()
byte_size = 0
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
with source.open("rb") as handle:
content = handle.read()
digest.update(content)
byte_size = len(content)
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
response.raise_for_status()
except Exception as exc:
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
return {"status": "available", "path": str(source), "byte_size": byte_size, "checksum_sha256": digest.hexdigest(), "object_key": payload.get("object_key")}
@app.post(f"{route_prefix}/compute/jobs")
async def create_job(payload: dict[str, Any]) -> dict[str, Any]:
payload = {**payload, "require_dataset_files": True}
@@ -645,7 +670,7 @@ def create_app() -> FastAPI:
"status": "queued",
"progress": 10,
"pid": int(52000 + now() % 10000),
"gpus": payload.get("gpus") or [0],
"gpus": payload.get("gpus") or [],
"created_at": now(),
"command": command.command,
"work_dir": command.work_dir,
@@ -843,21 +868,46 @@ def create_app() -> FastAPI:
target.parent.mkdir(parents=True, exist_ok=True)
temp_target = target.with_name(f".{target.name}.part")
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
expected_size = int(payload.get("byte_size") or 0)
lock = cache_locks.setdefault(str(target), asyncio.Lock())
async with lock:
if target.is_file() and expected_checksum:
existing_digest = hashlib.sha256()
with target.open("rb") as existing:
while chunk := existing.read(1024 * 1024):
existing_digest.update(chunk)
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
digest = hashlib.sha256()
byte_size = 0
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
async with client.stream("GET", download_url) as response:
response.raise_for_status()
with temp_target.open("wb") as output:
async for chunk in response.aiter_bytes(1024 * 1024):
output.write(chunk)
digest.update(chunk)
byte_size += len(chunk)
async with lock:
for attempt in range(3):
try:
digest = hashlib.sha256()
byte_size = 0
temp_target.unlink(missing_ok=True)
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30), follow_redirects=True) as client:
async with client.stream("GET", download_url) as response:
response.raise_for_status()
with temp_target.open("wb") as output:
async for chunk in response.aiter_bytes(1024 * 1024):
output.write(chunk)
digest.update(chunk)
byte_size += len(chunk)
break
except Exception:
temp_target.unlink(missing_ok=True)
if attempt == 2:
raise
await asyncio.sleep(2 ** attempt)
checksum = digest.hexdigest()
if expected_checksum and checksum != expected_checksum:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache checksum mismatch")
if expected_size and byte_size != expected_size:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache byte size mismatch")
temp_target.replace(target)
except HTTPException:
raise