YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合

Browse Source 
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请
- 存储: MinIO 存储进度迁移、对象存储安全加固与测试
- 计算: GPU 资源预留、compute 轮询与同步增强
- 权限: permission v2 迁移、权限安全验收测试
- 日志: 后端运行日志中文说明、操作日志整合
- 数据处理/评测: 数据转换与模型评测优化

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-21 09:49:48 +08:00
parent 080ef6ab00
commit 6f0e82f351
94 changed files with 9547 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
compute/api/main.py
View File

@@ -9,6 +9,7 @@ import shutil
import subprocess
import time
from pathlib import Path
from datetime import datetime
from typing import Any
import httpx
@@ -70,6 +71,76 @@ def create_app() -> FastAPI:
except ValueError:
return False
def cache_max_bytes() -> int:
return max(0, _int_env("COMPUTE_CACHE_MAX_BYTES", 0))
def cache_ttl_seconds() -> int:
return max(0, _int_env("COMPUTE_CACHE_TTL_SECONDS", 0))
def cache_meta_path(target: Path) -> Path:
return target.with_name(f".{target.name}.cache-meta.json")
def cache_protected_until(target: Path) -> float:
try:
value = json.loads(cache_meta_path(target).read_text(encoding="utf-8")).get("protected_until")
return float(value or 0)
except (OSError, TypeError, ValueError, json.JSONDecodeError):
return 0.0
def write_cache_meta(target: Path, resource_id: str, version_id: str, protected_until: float) -> None:
meta = cache_meta_path(target)
meta.write_text(json.dumps({
"resource_id": resource_id,
"version_id": version_id,
"protected_until": protected_until,
"last_accessed_at": now(),
}), encoding="utf-8")
def cache_usage(cache_root: Path) -> int:
total = 0
if not cache_root.exists():
return 0
for item in cache_root.rglob("*"):
try:
if item.is_file() and not item.name.endswith(".part"):
total += item.stat().st_size
except OSError:
continue
return total
def ensure_cache_capacity(cache_root: Path, required_bytes: int, protected: Path) -> None:
limit = cache_max_bytes()
if not limit or required_bytes <= 0:
return
usage = cache_usage(cache_root)
if usage + required_bytes <= limit:
return
candidates: list[tuple[float, int, Path]] = []
for item in cache_root.rglob("*"):
try:
if (
item.is_file()
and not item.name.endswith(".part")
and not item.name.endswith(".cache-meta.json")
and item.resolve() != protected.resolve()
and cache_protected_until(item) <= now()
):
stat = item.stat()
candidates.append((stat.st_atime, stat.st_size, item))
except OSError:
continue
candidates.sort(key=lambda value: value[0])
for _, size, item in candidates:
try:
item.unlink(missing_ok=True)
usage -= size
except OSError:
continue
if usage + required_bytes <= limit:
break
if usage + required_bytes > limit:
raise HTTPException(status_code=507, detail="compute cache capacity is insufficient")
def _llama_factory_version() -> str:
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
try:
@@ -662,13 +733,26 @@ def create_app() -> FastAPI:
raise HTTPException(status_code=400, detail="upload_url is required")
digest = hashlib.sha256()
byte_size = 0
content_length = source.stat().st_size
async def file_chunks():
nonlocal byte_size
with source.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
digest.update(chunk)
byte_size += len(chunk)
yield chunk
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
with source.open("rb") as handle:
content = handle.read()
digest.update(content)
byte_size = len(content)
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
response = await client.put(
upload_url,
content=file_chunks(),
headers={
"Content-Type": str(payload.get("content_type") or "application/octet-stream"),
"Content-Length": str(content_length),
},
)
response.raise_for_status()
except Exception as exc:
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
@@ -916,6 +1000,14 @@ def create_app() -> FastAPI:
temp_target = target.with_name(f".{target.name}.part")
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
expected_size = int(payload.get("byte_size") or 0)
requested_protected_until = str(payload.get("protected_until") or "")
try:
protected_until = float(requested_protected_until)
except ValueError:
try:
protected_until = datetime.fromisoformat(requested_protected_until.replace("Z", "+00:00")).timestamp()
except (ValueError, TypeError):
protected_until = now() + cache_ttl_seconds() if cache_ttl_seconds() else 0.0
lock = cache_locks.setdefault(str(target), asyncio.Lock())
async with lock:
if target.is_file() and expected_checksum:
@@ -924,7 +1016,10 @@ def create_app() -> FastAPI:
while chunk := existing.read(1024 * 1024):
existing_digest.update(chunk)
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
os.utime(target, None)
write_cache_meta(target, resource_id, version_id, protected_until)
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
ensure_cache_capacity(cache_root / "resources", expected_size, target)
digest = hashlib.sha256()
byte_size = 0
try:
@@ -956,6 +1051,7 @@ def create_app() -> FastAPI:
temp_target.unlink(missing_ok=True)
raise HTTPException(status_code=502, detail="cache byte size mismatch")
temp_target.replace(target)
write_cache_meta(target, resource_id, version_id, protected_until)
except HTTPException:
raise
except Exception as exc:
@@ -975,12 +1071,20 @@ def create_app() -> FastAPI:
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
target = cache_root / "resources" / resource_id / version_id / "resource"
ttl = cache_ttl_seconds()
if target.is_file() and ttl and target.stat().st_atime + ttl < now() and cache_protected_until(target) <= now():
target.unlink(missing_ok=True)
cache_meta_path(target).unlink(missing_ok=True)
return {
"resource_id": resource_id,
"version_id": version_id,
"status": "ready" if target.is_file() else "missing",
"local_path": str(target),
"byte_size": target.stat().st_size if target.is_file() else 0,
"cache_usage_bytes": cache_usage(cache_root / "resources"),
"cache_max_bytes": cache_max_bytes(),
"cache_ttl_seconds": ttl,
"protected_until": cache_protected_until(target) if target.is_file() else 0,
}
@app.delete(f"{route_prefix}/compute/cache")