feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请 - 存储: MinIO 存储进度迁移、对象存储安全加固与测试 - 计算: GPU 资源预留、compute 轮询与同步增强 - 权限: permission v2 迁移、权限安全验收测试 - 日志: 后端运行日志中文说明、操作日志整合 - 数据处理/评测: 数据转换与模型评测优化 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
@@ -9,6 +9,7 @@ import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -70,6 +71,76 @@ def create_app() -> FastAPI:
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def cache_max_bytes() -> int:
|
||||
return max(0, _int_env("COMPUTE_CACHE_MAX_BYTES", 0))
|
||||
|
||||
def cache_ttl_seconds() -> int:
|
||||
return max(0, _int_env("COMPUTE_CACHE_TTL_SECONDS", 0))
|
||||
|
||||
def cache_meta_path(target: Path) -> Path:
|
||||
return target.with_name(f".{target.name}.cache-meta.json")
|
||||
|
||||
def cache_protected_until(target: Path) -> float:
|
||||
try:
|
||||
value = json.loads(cache_meta_path(target).read_text(encoding="utf-8")).get("protected_until")
|
||||
return float(value or 0)
|
||||
except (OSError, TypeError, ValueError, json.JSONDecodeError):
|
||||
return 0.0
|
||||
|
||||
def write_cache_meta(target: Path, resource_id: str, version_id: str, protected_until: float) -> None:
|
||||
meta = cache_meta_path(target)
|
||||
meta.write_text(json.dumps({
|
||||
"resource_id": resource_id,
|
||||
"version_id": version_id,
|
||||
"protected_until": protected_until,
|
||||
"last_accessed_at": now(),
|
||||
}), encoding="utf-8")
|
||||
|
||||
def cache_usage(cache_root: Path) -> int:
|
||||
total = 0
|
||||
if not cache_root.exists():
|
||||
return 0
|
||||
for item in cache_root.rglob("*"):
|
||||
try:
|
||||
if item.is_file() and not item.name.endswith(".part"):
|
||||
total += item.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
def ensure_cache_capacity(cache_root: Path, required_bytes: int, protected: Path) -> None:
|
||||
limit = cache_max_bytes()
|
||||
if not limit or required_bytes <= 0:
|
||||
return
|
||||
usage = cache_usage(cache_root)
|
||||
if usage + required_bytes <= limit:
|
||||
return
|
||||
candidates: list[tuple[float, int, Path]] = []
|
||||
for item in cache_root.rglob("*"):
|
||||
try:
|
||||
if (
|
||||
item.is_file()
|
||||
and not item.name.endswith(".part")
|
||||
and not item.name.endswith(".cache-meta.json")
|
||||
and item.resolve() != protected.resolve()
|
||||
and cache_protected_until(item) <= now()
|
||||
):
|
||||
stat = item.stat()
|
||||
candidates.append((stat.st_atime, stat.st_size, item))
|
||||
except OSError:
|
||||
continue
|
||||
candidates.sort(key=lambda value: value[0])
|
||||
for _, size, item in candidates:
|
||||
try:
|
||||
item.unlink(missing_ok=True)
|
||||
usage -= size
|
||||
except OSError:
|
||||
continue
|
||||
if usage + required_bytes <= limit:
|
||||
break
|
||||
if usage + required_bytes > limit:
|
||||
raise HTTPException(status_code=507, detail="compute cache capacity is insufficient")
|
||||
|
||||
def _llama_factory_version() -> str:
|
||||
for command in (["llamafactory-cli", "version"], ["llamafactory-cli", "--version"]):
|
||||
try:
|
||||
@@ -662,13 +733,26 @@ def create_app() -> FastAPI:
|
||||
raise HTTPException(status_code=400, detail="upload_url is required")
|
||||
digest = hashlib.sha256()
|
||||
byte_size = 0
|
||||
content_length = source.stat().st_size
|
||||
|
||||
async def file_chunks():
|
||||
nonlocal byte_size
|
||||
with source.open("rb") as handle:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
byte_size += len(chunk)
|
||||
yield chunk
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(900, connect=30)) as client:
|
||||
with source.open("rb") as handle:
|
||||
content = handle.read()
|
||||
digest.update(content)
|
||||
byte_size = len(content)
|
||||
response = await client.put(upload_url, content=content, headers={"Content-Type": str(payload.get("content_type") or "application/octet-stream")})
|
||||
response = await client.put(
|
||||
upload_url,
|
||||
content=file_chunks(),
|
||||
headers={
|
||||
"Content-Type": str(payload.get("content_type") or "application/octet-stream"),
|
||||
"Content-Length": str(content_length),
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=f"artifact upload failed: {exc}") from exc
|
||||
@@ -916,6 +1000,14 @@ def create_app() -> FastAPI:
|
||||
temp_target = target.with_name(f".{target.name}.part")
|
||||
expected_checksum = str(payload.get("checksum_sha256") or "").lower()
|
||||
expected_size = int(payload.get("byte_size") or 0)
|
||||
requested_protected_until = str(payload.get("protected_until") or "")
|
||||
try:
|
||||
protected_until = float(requested_protected_until)
|
||||
except ValueError:
|
||||
try:
|
||||
protected_until = datetime.fromisoformat(requested_protected_until.replace("Z", "+00:00")).timestamp()
|
||||
except (ValueError, TypeError):
|
||||
protected_until = now() + cache_ttl_seconds() if cache_ttl_seconds() else 0.0
|
||||
lock = cache_locks.setdefault(str(target), asyncio.Lock())
|
||||
async with lock:
|
||||
if target.is_file() and expected_checksum:
|
||||
@@ -924,7 +1016,10 @@ def create_app() -> FastAPI:
|
||||
while chunk := existing.read(1024 * 1024):
|
||||
existing_digest.update(chunk)
|
||||
if existing_digest.hexdigest().lower() == expected_checksum and (not expected_size or target.stat().st_size == expected_size):
|
||||
os.utime(target, None)
|
||||
write_cache_meta(target, resource_id, version_id, protected_until)
|
||||
return {"resource_id": resource_id, "version_id": version_id, "status": "ready", "local_path": str(target), "byte_size": target.stat().st_size, "checksum_sha256": existing_digest.hexdigest(), "reused": True}
|
||||
ensure_cache_capacity(cache_root / "resources", expected_size, target)
|
||||
digest = hashlib.sha256()
|
||||
byte_size = 0
|
||||
try:
|
||||
@@ -956,6 +1051,7 @@ def create_app() -> FastAPI:
|
||||
temp_target.unlink(missing_ok=True)
|
||||
raise HTTPException(status_code=502, detail="cache byte size mismatch")
|
||||
temp_target.replace(target)
|
||||
write_cache_meta(target, resource_id, version_id, protected_until)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
@@ -975,12 +1071,20 @@ def create_app() -> FastAPI:
|
||||
data_root = Path(os.getenv("YG_FT_DATA_ROOT", "/data/yg-ft"))
|
||||
cache_root = Path(os.getenv("YG_FT_CACHE_ROOT", str(data_root)))
|
||||
target = cache_root / "resources" / resource_id / version_id / "resource"
|
||||
ttl = cache_ttl_seconds()
|
||||
if target.is_file() and ttl and target.stat().st_atime + ttl < now() and cache_protected_until(target) <= now():
|
||||
target.unlink(missing_ok=True)
|
||||
cache_meta_path(target).unlink(missing_ok=True)
|
||||
return {
|
||||
"resource_id": resource_id,
|
||||
"version_id": version_id,
|
||||
"status": "ready" if target.is_file() else "missing",
|
||||
"local_path": str(target),
|
||||
"byte_size": target.stat().st_size if target.is_file() else 0,
|
||||
"cache_usage_bytes": cache_usage(cache_root / "resources"),
|
||||
"cache_max_bytes": cache_max_bytes(),
|
||||
"cache_ttl_seconds": ttl,
|
||||
"protected_until": cache_protected_until(target) if target.is_file() else 0,
|
||||
}
|
||||
|
||||
@app.delete(f"{route_prefix}/compute/cache")
|
||||
|
||||
@@ -313,7 +313,7 @@ def build_command(config: dict[str, Any], llama_factory_home: str = "/app/LLaMA-
|
||||
"--save_steps",
|
||||
str(config.get("save_steps", 50)),
|
||||
"--logging_steps",
|
||||
str(config.get("logging_steps", 10)),
|
||||
str(max(1, int(config.get("logging_steps", 1) or 1))),
|
||||
"--overwrite_output_dir",
|
||||
"true",
|
||||
"--plot_loss",
|
||||
|
||||
@@ -16,6 +16,7 @@ import math
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -94,15 +95,13 @@ def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4)
|
||||
try:
|
||||
from sacrebleu.metrics import BLEU
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
|
||||
return _metric_record(None, len(predictions), "sacrebleu 未安装", available=False)
|
||||
if not predictions:
|
||||
return _metric_record(0, 0)
|
||||
bleu = BLEU(max_ngram_order=ngram)
|
||||
# sacrebleu expects list-of-strings; we have one reference per prediction
|
||||
score = bleu.corpus_score(predictions, [references])
|
||||
return {
|
||||
"enabled": True,
|
||||
"score": round(score.score, 2),
|
||||
"bleu": round(score.score, 2),
|
||||
}
|
||||
return _metric_record(score.score, len(predictions), bleu=round(score.score, 2))
|
||||
|
||||
|
||||
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
|
||||
@@ -110,20 +109,27 @@ def _compute_rouge(references: list[str], predictions: list[str], methods: list[
|
||||
try:
|
||||
from rouge_score import rouge_scorer
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "rouge-score not installed", "score": 0}
|
||||
rouge_scorer = None
|
||||
methods = methods or ["rouge1", "rouge2", "rougeL"]
|
||||
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
|
||||
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
|
||||
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
|
||||
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
|
||||
totals: dict[str, float] = {}
|
||||
n = max(len(predictions), 1)
|
||||
for ref, pred in zip(references, predictions):
|
||||
result = scorer.score(ref, pred)
|
||||
for key in methods:
|
||||
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
||||
avg = {k: round(v / n, 4) for k, v in totals.items()}
|
||||
return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
|
||||
if rouge_scorer is None:
|
||||
values = [_pair_rouge(ref, pred) for ref, pred in zip(references, predictions)]
|
||||
avg = {key: round(sum(item.get(key, 0.0) for item in values) / max(len(values), 1), 4) for key in methods}
|
||||
else:
|
||||
class _Tokenizer:
|
||||
def tokenize(self, value: str) -> list[str]:
|
||||
return value.split()
|
||||
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=False, tokenizer=_Tokenizer())
|
||||
totals: dict[str, float] = {}
|
||||
n = max(len(predictions), 1)
|
||||
for ref, pred in zip(references, predictions):
|
||||
result = scorer.score(_rouge_tokens(ref), _rouge_tokens(pred))
|
||||
for key in methods:
|
||||
totals[key] = totals.get(key, 0) + result[key].fmeasure
|
||||
avg = {key: round(value / n, 4) for key, value in totals.items()}
|
||||
return _metric_record(avg.get("rougeL", avg.get("rouge1", 0)) * 100, len(predictions), **avg)
|
||||
|
||||
|
||||
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
@@ -132,7 +138,9 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
||||
from sklearn.feature_extraction.text import TfidfVectorizer
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
except ImportError:
|
||||
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
|
||||
return _metric_record(None, len(predictions), "scikit-learn 未安装", available=False)
|
||||
if not predictions:
|
||||
return _metric_record(0, 0)
|
||||
try:
|
||||
vectorizer = TfidfVectorizer()
|
||||
tfidf = vectorizer.fit_transform(references + predictions)
|
||||
@@ -140,41 +148,145 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
|
||||
ref_vec = tfidf[:n]
|
||||
pred_vec = tfidf[n:]
|
||||
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
|
||||
return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
|
||||
except ValueError:
|
||||
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
|
||||
return _metric_record(float(sims.mean()) * 100, n)
|
||||
except ValueError as exc:
|
||||
return _metric_record(0, len(predictions), str(exc))
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(value or "").strip().lower())
|
||||
|
||||
|
||||
def _metric_record(score: float | None, sample_count: int, error: str = "", available: bool = True, **extra: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": True,
|
||||
"available": available,
|
||||
"score": None if score is None else round(max(0.0, min(100.0, float(score))), 2),
|
||||
"max_score": 100,
|
||||
"unit": "percent",
|
||||
"sample_count": sample_count,
|
||||
"error": error,
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _rouge_tokens(text: str) -> str:
|
||||
text = str(text or "").strip().lower()
|
||||
tokens: list[str] = []
|
||||
for segment in re.findall(r"[一-鿿]+|[^\s一-鿿]+", text):
|
||||
tokens.extend(segment if "一" <= segment[0] <= "鿿" else [segment])
|
||||
return " ".join(tokens)
|
||||
|
||||
|
||||
def _pair_rouge(reference: str, prediction: str) -> dict[str, float]:
|
||||
try:
|
||||
from rouge_score import rouge_scorer
|
||||
except ImportError:
|
||||
reference_tokens = _rouge_tokens(reference).split()
|
||||
prediction_tokens = _rouge_tokens(prediction).split()
|
||||
if not reference_tokens or not prediction_tokens:
|
||||
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
|
||||
|
||||
def f1(overlap: int, reference_count: int, prediction_count: int) -> float:
|
||||
if not reference_count or not prediction_count or not overlap:
|
||||
return 0.0
|
||||
precision = overlap / prediction_count
|
||||
recall = overlap / reference_count
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
from collections import Counter
|
||||
reference_unigrams = Counter(reference_tokens)
|
||||
prediction_unigrams = Counter(prediction_tokens)
|
||||
unigram_overlap = sum((reference_unigrams & prediction_unigrams).values())
|
||||
reference_bigrams = Counter(zip(reference_tokens, reference_tokens[1:]))
|
||||
prediction_bigrams = Counter(zip(prediction_tokens, prediction_tokens[1:]))
|
||||
bigram_overlap = sum((reference_bigrams & prediction_bigrams).values())
|
||||
matrix = [[0] * (len(prediction_tokens) + 1) for _ in range(len(reference_tokens) + 1)]
|
||||
for row, reference_token in enumerate(reference_tokens, start=1):
|
||||
for column, prediction_token in enumerate(prediction_tokens, start=1):
|
||||
matrix[row][column] = matrix[row - 1][column - 1] + 1 if reference_token == prediction_token else max(matrix[row - 1][column], matrix[row][column - 1])
|
||||
return {
|
||||
"rouge1": f1(unigram_overlap, len(reference_tokens), len(prediction_tokens)),
|
||||
"rouge2": f1(bigram_overlap, max(len(reference_tokens) - 1, 0), max(len(prediction_tokens) - 1, 0)),
|
||||
"rougeL": f1(matrix[-1][-1], len(reference_tokens), len(prediction_tokens)),
|
||||
}
|
||||
class _Tokenizer:
|
||||
def tokenize(self, value: str) -> list[str]:
|
||||
return value.split()
|
||||
if not reference.strip() or not prediction.strip():
|
||||
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
|
||||
scorer = rouge_scorer.RougeScorer(("rouge1", "rouge2", "rougeL"), use_stemmer=False, tokenizer=_Tokenizer())
|
||||
scores = scorer.score(_rouge_tokens(reference), _rouge_tokens(prediction))
|
||||
return {key: float(value.fmeasure) for key, value in scores.items()}
|
||||
|
||||
|
||||
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
total = len(predictions)
|
||||
if not total:
|
||||
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
|
||||
return _metric_record(0, 0, matched=0, total=0)
|
||||
matched = sum(
|
||||
1
|
||||
for ref, pred in zip(references, predictions)
|
||||
if _normalize_text(ref) == _normalize_text(pred)
|
||||
)
|
||||
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
|
||||
return _metric_record(matched / total * 100, total, matched=matched, total=total)
|
||||
|
||||
|
||||
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
|
||||
if not predictions:
|
||||
return {"enabled": True, "score": 0}
|
||||
return _metric_record(0, 0)
|
||||
scores = [
|
||||
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
|
||||
for ref, pred in zip(references, predictions)
|
||||
]
|
||||
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
|
||||
return _metric_record(sum(scores) / max(len(scores), 1) * 100, len(predictions))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM Judge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _normalise_api_url(value: str) -> str:
|
||||
url = str(value or "").strip().rstrip("/")
|
||||
return url + "/chat/completions" if url.endswith("/v1") else url + "/v1/chat/completions"
|
||||
|
||||
|
||||
def _parse_judge_reply(reply: str, score_min: float, score_max: float) -> tuple[float | None, dict[str, Any], str]:
|
||||
payload: dict[str, Any] = {}
|
||||
match = re.search(r"\{[\s\S]*\}", str(reply or ""))
|
||||
if match:
|
||||
try:
|
||||
value = json.loads(match.group(0))
|
||||
if isinstance(value, dict):
|
||||
payload = value
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
reason = str(payload.get("综合评价") or payload.get("reason") or reply or "")
|
||||
raw = payload.get("score", payload.get("overall_score"))
|
||||
if raw is None:
|
||||
matches = re.findall(r"(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)", reason, re.IGNORECASE)
|
||||
raw = matches[-1] if matches else None
|
||||
try:
|
||||
numeric = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
dimensions = payload.get("dimensions") if isinstance(payload.get("dimensions"), dict) else {}
|
||||
if not dimensions:
|
||||
dimensions = {
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||
}
|
||||
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
|
||||
numeric = sum(values) / len(values) if values else None
|
||||
if numeric is None:
|
||||
return None, payload, reason
|
||||
# The judge prompt uses the configured score range. Do not treat a
|
||||
# decimal such as 0.5/5 as a 0.5/1 score, otherwise low scores are
|
||||
# incorrectly inflated (0.5/5 would become 50 instead of 10).
|
||||
normalized = (numeric - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||
return max(0.0, min(100.0, normalized)), payload, reason
|
||||
|
||||
|
||||
def _judge_sample(
|
||||
question: str,
|
||||
reference: str,
|
||||
@@ -198,7 +310,7 @@ def _judge_sample(
|
||||
pass_threshold = float(config.get("pass_threshold", 3))
|
||||
|
||||
if not api_url or not eval_model:
|
||||
return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
|
||||
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "未配置",
|
||||
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
|
||||
|
||||
system_msg = (
|
||||
@@ -227,7 +339,7 @@ def _judge_sample(
|
||||
}).encode("utf-8")
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{api_url}/v1/chat/completions",
|
||||
_normalise_api_url(api_url),
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
@@ -238,39 +350,54 @@ def _judge_sample(
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
reply = data["choices"][0]["message"]["content"]
|
||||
except Exception as exc:
|
||||
return {"score": 0, "max_score": score_max, "passed": False,
|
||||
return {"available": False, "score": None, "max_score": 100, "passed": False,
|
||||
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
|
||||
"error_type": "其他"}
|
||||
|
||||
# Parse score from reply — look for patterns like "4分" or "Score: 4"
|
||||
score = 0
|
||||
import re
|
||||
score_patterns = [
|
||||
r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
|
||||
r'(\d+(?:\.\d+)?)\s*分',
|
||||
r'(\d+(?:\.\d+)?)\s*/\s*\d+',
|
||||
]
|
||||
for pat in score_patterns:
|
||||
m = re.search(pat, reply, re.IGNORECASE)
|
||||
if m:
|
||||
try:
|
||||
score = float(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
break
|
||||
score = max(score_min, min(score_max, score))
|
||||
passed = score >= pass_threshold
|
||||
parsed: dict[str, Any] = {}
|
||||
match = re.search(r"\{[\s\S]*\}", reply)
|
||||
if match:
|
||||
try:
|
||||
value = json.loads(match.group(0))
|
||||
if isinstance(value, dict):
|
||||
parsed = value
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
raw_score = parsed.get("score", parsed.get("overall_score"))
|
||||
reason = str(parsed.get("综合评价") or parsed.get("reason") or reply)
|
||||
if raw_score is None:
|
||||
matches = re.findall(r'(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)', reason, re.IGNORECASE)
|
||||
raw_score = matches[-1] if matches else None
|
||||
try:
|
||||
numeric_score = float(raw_score)
|
||||
except (TypeError, ValueError):
|
||||
dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {}
|
||||
if not dimensions:
|
||||
dimensions = {
|
||||
key: value
|
||||
for key, value in parsed.items()
|
||||
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||
}
|
||||
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
|
||||
numeric_score = sum(values) / len(values) if values else None
|
||||
if numeric_score is None:
|
||||
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "错误",
|
||||
"evaluation_reason": "评测模型未返回可解析分数：" + reason[:1800], "error_type": "其他"}
|
||||
normalized_score = (numeric_score - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||
normalized_score = max(0, min(100, normalized_score))
|
||||
normalized_threshold = (pass_threshold - score_min) / max(score_max - score_min, 1e-9) * 100
|
||||
passed = normalized_score >= normalized_threshold
|
||||
|
||||
# Determine judgement label
|
||||
if score >= pass_threshold + 1:
|
||||
if normalized_score >= min(100, normalized_threshold + 20):
|
||||
judgement = "正确"
|
||||
elif score >= pass_threshold:
|
||||
elif passed:
|
||||
judgement = "部分正确"
|
||||
else:
|
||||
judgement = "错误"
|
||||
|
||||
# Guess error type from reply
|
||||
reply_lower = reply.lower()
|
||||
reply_lower = (reply + " " + reason).lower()
|
||||
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
|
||||
error_type = "幻觉"
|
||||
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
|
||||
@@ -282,16 +409,84 @@ def _judge_sample(
|
||||
else:
|
||||
error_type = "其他"
|
||||
|
||||
parsed_dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {
|
||||
key: value
|
||||
for key, value in parsed.items()
|
||||
if key not in {"score", "overall_score", "综合评价", "reason"}
|
||||
}
|
||||
return {
|
||||
"score": score,
|
||||
"max_score": score_max,
|
||||
"available": True,
|
||||
"score": round(normalized_score, 2),
|
||||
"max_score": 100,
|
||||
"raw_score": numeric_score,
|
||||
"raw_max_score": score_max,
|
||||
"passed": passed,
|
||||
"judgement": judgement,
|
||||
"evaluation_reason": reply[:2000],
|
||||
"evaluation_reason": reason[:2000],
|
||||
"error_type": error_type,
|
||||
"dimension_scores": [
|
||||
{
|
||||
"name": str(name),
|
||||
"score": round(
|
||||
max(
|
||||
0,
|
||||
min(
|
||||
100,
|
||||
(float(value) - score_min)
|
||||
/ max(score_max - score_min, 1e-9)
|
||||
* 100,
|
||||
),
|
||||
),
|
||||
2,
|
||||
),
|
||||
"max_score": 100,
|
||||
}
|
||||
for name, value in parsed_dimensions.items()
|
||||
if isinstance(value, (int, float))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
temporary = path.with_suffix(path.suffix + ".part")
|
||||
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _write_eval_progress(
|
||||
output_dir: Path,
|
||||
status: str,
|
||||
stage: str,
|
||||
total: int,
|
||||
completed: int,
|
||||
message: str = "",
|
||||
current_index: int = 0,
|
||||
) -> None:
|
||||
_write_json(
|
||||
output_dir / "eval_progress.json",
|
||||
{
|
||||
"status": status,
|
||||
"stage": stage,
|
||||
"total": total,
|
||||
"completed": completed,
|
||||
"percentage": round(completed / total * 100, 1) if total else 100,
|
||||
"current_index": current_index,
|
||||
"message": message,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _deterministic_sample_score(reference: str, prediction: str) -> float:
|
||||
values = [SequenceMatcher(None, _normalize_text(reference), _normalize_text(prediction)).ratio()]
|
||||
if _normalize_text(reference) == _normalize_text(prediction):
|
||||
values.append(1.0)
|
||||
rouge = _pair_rouge(reference, prediction)
|
||||
if rouge.get("rougeL") is not None:
|
||||
values.append(float(rouge["rougeL"]))
|
||||
return round(sum(values) / len(values) * 100, 2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -312,11 +507,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
print(f"[eval] loading dataset: {dataset_path}")
|
||||
raw_samples = _load_dataset(dataset_path)
|
||||
print(f"[eval] loaded {len(raw_samples)} samples")
|
||||
_write_eval_progress(output_dir, "running", "dataset", len(raw_samples), 0, f"已加载 {len(raw_samples)} 条样本")
|
||||
|
||||
# ---- 2. Load model ----
|
||||
print(f"[eval] loading model: {model_path}")
|
||||
from compute.engines.llama_factory.inference import InferenceSession
|
||||
session = InferenceSession()
|
||||
_write_eval_progress(output_dir, "running", "model_loading", len(raw_samples), 0, "正在加载评测模型")
|
||||
session.load(
|
||||
model_name_or_path=model_path,
|
||||
adapter_name_or_path=adapter_path,
|
||||
@@ -329,6 +526,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
if not load_result.get("loaded"):
|
||||
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
|
||||
print(f"[eval] model loaded OK")
|
||||
_write_eval_progress(output_dir, "running", "inference", len(raw_samples), 0, "开始生成模型回答")
|
||||
|
||||
# ---- 3. Run inference on each sample ----
|
||||
samples: list[dict[str, Any]] = []
|
||||
@@ -384,12 +582,45 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
] if judge_result else [],
|
||||
"status": "completed",
|
||||
})
|
||||
if not judge_enabled:
|
||||
deterministic_score = _deterministic_sample_score(reference, prediction)
|
||||
samples[-1].update({
|
||||
"score": deterministic_score,
|
||||
"max_score": 100,
|
||||
"raw_score": deterministic_score,
|
||||
"raw_max_score": 100,
|
||||
"passed": deterministic_score >= 60,
|
||||
"judgement": "正确" if deterministic_score >= 80 else "部分正确" if deterministic_score >= 60 else "错误",
|
||||
"evaluation_reason": "基于精确匹配、文本相似度和 ROUGE-L 的确定性评分",
|
||||
})
|
||||
_write_json(
|
||||
output_dir / "eval_results.json",
|
||||
{
|
||||
"status": "running",
|
||||
"overall_score": round(
|
||||
sum(float(item["score"]) for item in samples if item.get("score") is not None)
|
||||
/ max(len([item for item in samples if item.get("score") is not None]), 1),
|
||||
output_precision,
|
||||
),
|
||||
"overall_score_max": 100,
|
||||
"overall_evaluation": f"已完成 {len(samples)}/{total} 条样本",
|
||||
"dimension_summary": [],
|
||||
"samples": samples,
|
||||
"sample_count": total,
|
||||
"completed_count": len(samples),
|
||||
"passed_count": sum(bool(item.get("passed")) for item in samples),
|
||||
"basic_metrics": {},
|
||||
"metric_summary_version": 2,
|
||||
},
|
||||
)
|
||||
|
||||
progress_pct = int(idx / max(total, 1) * 100)
|
||||
_write_eval_progress(output_dir, "running", "inference", total, len(samples), f"已完成第 {idx} 条样本", idx)
|
||||
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
|
||||
|
||||
# ---- 4. Compute basic metrics ----
|
||||
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
|
||||
_write_eval_progress(output_dir, "running", "metrics", total, len(samples), "正在计算评测指标", total)
|
||||
metrics_result: dict[str, Any] = {}
|
||||
|
||||
bleu_cfg = basic_cfg.get("bleu", {})
|
||||
@@ -397,11 +628,11 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
|
||||
|
||||
rouge_cfg = basic_cfg.get("rouge", {})
|
||||
if rouge_cfg.get("enabled"):
|
||||
if rouge_cfg.get("enabled") or not judge_enabled:
|
||||
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
|
||||
|
||||
cosine_cfg = basic_cfg.get("cosine", {})
|
||||
if cosine_cfg.get("enabled"):
|
||||
if cosine_cfg.get("enabled") or not judge_enabled:
|
||||
metrics_result["cosine"] = _compute_cosine(references, predictions)
|
||||
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
|
||||
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
|
||||
@@ -412,16 +643,15 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
scored = [s for s in samples if s.get("score") is not None]
|
||||
passed_count = len([s for s in scored if s.get("passed")])
|
||||
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
|
||||
max_score = dimension_cfg.get("score_max", 5)
|
||||
overall_score = round(avg_score / max_score * 100, output_precision)
|
||||
overall_score = avg_score
|
||||
overall_score_max = 100
|
||||
dimension_summary = [{
|
||||
"name": "综合评分",
|
||||
"name": "LLM Judge",
|
||||
"score": overall_score,
|
||||
"max_score": 100,
|
||||
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
|
||||
}]
|
||||
overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/{max_score} 分"
|
||||
overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/100 分"
|
||||
else:
|
||||
passed_count = 0
|
||||
enabled_scores = [
|
||||
@@ -444,6 +674,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
overall_evaluation = f"评测完成：{completed} 样本（未配置 LLM 评委）"
|
||||
|
||||
result = {
|
||||
"status": "completed",
|
||||
"overall_score": overall_score,
|
||||
"overall_score_max": overall_score_max,
|
||||
"overall_evaluation": overall_evaluation,
|
||||
@@ -454,11 +685,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"completed_count": completed,
|
||||
"passed_count": passed_count,
|
||||
"basic_metrics": metrics_result,
|
||||
"metric_summary_version": 2,
|
||||
}
|
||||
|
||||
# ---- 6. Write results ----
|
||||
result_path = output_dir / "eval_results.json"
|
||||
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
_write_eval_progress(output_dir, "completed", "completed", total, completed, "评测完成", total)
|
||||
print(f"[eval] results written to {result_path}")
|
||||
return result
|
||||
|
||||
|
||||
@@ -7,6 +7,28 @@ import uuid
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
def _ensure_peft_transformers_compat() -> None:
|
||||
"""Bridge a removed PEFT helper used by the bundled Transformers build.
|
||||
|
||||
The offline Compute image currently contains Transformers 5.8.0 and PEFT
|
||||
0.18.1. Transformers imports this private helper when a model directory
|
||||
contains PEFT metadata, but PEFT 0.18.1 does not expose it. Evaluation
|
||||
runs in single-process HuggingFace mode, so tensor-parallel sharding is
|
||||
not applicable and a no-op compatibility hook is the correct behavior.
|
||||
"""
|
||||
try:
|
||||
from peft.utils import save_and_load
|
||||
except Exception:
|
||||
return
|
||||
if hasattr(save_and_load, "_maybe_shard_state_dict_for_tp"):
|
||||
return
|
||||
|
||||
def _maybe_shard_state_dict_for_tp(_model: Any, _state_dict: dict[str, Any], _adapter_name: str) -> None:
|
||||
return None
|
||||
|
||||
save_and_load._maybe_shard_state_dict_for_tp = _maybe_shard_state_dict_for_tp
|
||||
|
||||
|
||||
class InferenceSession:
|
||||
"""Manages a loaded model for inference with LLaMA-Factory ChatModel.
|
||||
|
||||
@@ -143,6 +165,7 @@ class InferenceSession:
|
||||
|
||||
args = dict(self._load_args)
|
||||
infer_result = get_infer_args(args)
|
||||
_ensure_peft_transformers_compat()
|
||||
model = ChatModel(args)
|
||||
tokenizer = getattr(model, "tokenizer", None) or model.engine.tokenizer
|
||||
generating_args = infer_result[-1]
|
||||
@@ -182,6 +205,7 @@ class InferenceSession:
|
||||
self._loaded_at = time.time()
|
||||
self._status = "ready"
|
||||
|
||||
|
||||
def _release_model(self) -> None:
|
||||
with self._chat_lock:
|
||||
with self._state_lock:
|
||||
|
||||
@@ -2,7 +2,14 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from compute.engines.llama_factory.eval_runner import _load_dataset
|
||||
from compute.engines.llama_factory.eval_runner import (
|
||||
_compute_exact_match,
|
||||
_compute_rouge,
|
||||
_compute_text_similarity,
|
||||
_normalise_api_url,
|
||||
_parse_judge_reply,
|
||||
_load_dataset,
|
||||
)
|
||||
|
||||
|
||||
def _write(tmp_path, name: str, text: str) -> str:
|
||||
@@ -40,3 +47,48 @@ def test_load_jsonl_with_bom_and_embedded_array(tmp_path) -> None:
|
||||
"" + json.dumps([{"question": "a", "answer": "b"}, {"question": "c", "answer": "d"}]),
|
||||
)
|
||||
assert len(_load_dataset(path)) == 2
|
||||
|
||||
|
||||
def test_deterministic_metrics_use_percent_scale() -> None:
|
||||
references = ["北京是中国的首都"]
|
||||
predictions = ["北京是中国的首都"]
|
||||
assert _compute_exact_match(references, predictions)["score"] == 100
|
||||
assert _compute_text_similarity(references, predictions)["score"] == 100
|
||||
|
||||
|
||||
def test_rouge_supports_chinese_character_tokenization() -> None:
|
||||
import pytest
|
||||
pytest.importorskip("rouge_score")
|
||||
result = _compute_rouge(["北京是中国的首都"], ["北京是中国的首都"])
|
||||
assert result["available"] is True
|
||||
assert result["score"] == 100
|
||||
|
||||
|
||||
def test_judge_reply_accepts_json_and_normalises_score() -> None:
|
||||
score, payload, reason = _parse_judge_reply(
|
||||
'{"score": 4, "dimensions": {"正确性": 4}, "reason": "内容正确"}',
|
||||
0,
|
||||
5,
|
||||
)
|
||||
assert score == 80
|
||||
assert payload["dimensions"]["正确性"] == 4
|
||||
assert reason == "内容正确"
|
||||
|
||||
|
||||
def test_judge_reply_accepts_nlp_demo_dimension_format() -> None:
|
||||
score, _, _ = _parse_judge_reply(
|
||||
'{"语义一致性": 4, "信息完整性": 3, "事实准确性": 5, "语言流畅性": 4, "综合评价": "整体良好，0.8"}',
|
||||
0,
|
||||
5,
|
||||
)
|
||||
assert score == 80
|
||||
|
||||
|
||||
def test_judge_reply_keeps_decimal_scores_in_configured_range() -> None:
|
||||
score, _, _ = _parse_judge_reply('{"score": 0.5}', 0, 5)
|
||||
assert score == 10
|
||||
|
||||
|
||||
def test_openai_url_does_not_duplicate_v1() -> None:
|
||||
assert _normalise_api_url("https://example.test/v1") == "https://example.test/v1/chat/completions"
|
||||
assert _normalise_api_url("https://example.test") == "https://example.test/v1/chat/completions"
|
||||
|
||||
Reference in New Issue
Block a user