YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 平台治理与权限体系完善，存储进度/GPU预留/审批中心与日志整合

Browse Source 
- 平台治理: 租户用户权限层次、资源ACL、审批中心与审批模板、访问申请
- 存储: MinIO 存储进度迁移、对象存储安全加固与测试
- 计算: GPU 资源预留、compute 轮询与同步增强
- 权限: permission v2 迁移、权限安全验收测试
- 日志: 后端运行日志中文说明、操作日志整合
- 数据处理/评测: 数据转换与模型评测优化

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wuyongtao
2026-08-21 09:49:48 +08:00
parent 080ef6ab00
commit 6f0e82f351
94 changed files with 9547 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

347
compute/engines/llama_factory/eval_runner.py
View File

@@ -16,6 +16,7 @@ import math
import re
import sys
import time
from datetime import datetime, timezone
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any
@@ -94,15 +95,13 @@ def _compute_bleu(references: list[str], predictions: list[str], ngram: int = 4)
try:
from sacrebleu.metrics import BLEU
except ImportError:
return {"enabled": False, "error": "sacrebleu not installed", "score": 0}
return _metric_record(None, len(predictions), "sacrebleu 未安装", available=False)
if not predictions:
return _metric_record(0, 0)
bleu = BLEU(max_ngram_order=ngram)
# sacrebleu expects list-of-strings; we have one reference per prediction
score = bleu.corpus_score(predictions, [references])
return {
"enabled": True,
"score": round(score.score, 2),
"bleu": round(score.score, 2),
}
return _metric_record(score.score, len(predictions), bleu=round(score.score, 2))
def _compute_rouge(references: list[str], predictions: list[str], methods: list[str] | None = None) -> dict[str, Any]:
@@ -110,20 +109,27 @@ def _compute_rouge(references: list[str], predictions: list[str], methods: list[
try:
from rouge_score import rouge_scorer
except ImportError:
return {"enabled": False, "error": "rouge-score not installed", "score": 0}
rouge_scorer = None
methods = methods or ["rouge1", "rouge2", "rougeL"]
# Normalize: map "rouge_1"/"rouge1" → "rouge1", "rouge_l"/"rougeL" → "rougeL"
_rouge_aliases = {"rouge_1": "rouge1", "rouge_2": "rouge2", "rouge_l": "rougeL"}
methods = [_rouge_aliases.get(m, m.replace("_", "")) for m in methods]
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=True)
totals: dict[str, float] = {}
n = max(len(predictions), 1)
for ref, pred in zip(references, predictions):
result = scorer.score(ref, pred)
for key in methods:
totals[key] = totals.get(key, 0) + result[key].fmeasure
avg = {k: round(v / n, 4) for k, v in totals.items()}
return {"enabled": True, "score": round(avg.get("rougeL", avg.get("rouge1", 0)) * 100, 2), **avg}
if rouge_scorer is None:
values = [_pair_rouge(ref, pred) for ref, pred in zip(references, predictions)]
avg = {key: round(sum(item.get(key, 0.0) for item in values) / max(len(values), 1), 4) for key in methods}
else:
class _Tokenizer:
def tokenize(self, value: str) -> list[str]:
return value.split()
scorer = rouge_scorer.RougeScorer(methods, use_stemmer=False, tokenizer=_Tokenizer())
totals: dict[str, float] = {}
n = max(len(predictions), 1)
for ref, pred in zip(references, predictions):
result = scorer.score(_rouge_tokens(ref), _rouge_tokens(pred))
for key in methods:
totals[key] = totals.get(key, 0) + result[key].fmeasure
avg = {key: round(value / n, 4) for key, value in totals.items()}
return _metric_record(avg.get("rougeL", avg.get("rouge1", 0)) * 100, len(predictions), **avg)
def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str, Any]:
@@ -132,7 +138,9 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
except ImportError:
return {"enabled": False, "error": "scikit-learn not installed", "score": 0}
return _metric_record(None, len(predictions), "scikit-learn 未安装", available=False)
if not predictions:
return _metric_record(0, 0)
try:
vectorizer = TfidfVectorizer()
tfidf = vectorizer.fit_transform(references + predictions)
@@ -140,41 +148,145 @@ def _compute_cosine(references: list[str], predictions: list[str]) -> dict[str,
ref_vec = tfidf[:n]
pred_vec = tfidf[n:]
sims = cosine_similarity(ref_vec, pred_vec).diagonal()
return {"enabled": True, "score": round(float(sims.mean()) * 100, 2)}
except ValueError:
return {"enabled": True, "score": 0, "error": "insufficient text for vectorization"}
return _metric_record(float(sims.mean()) * 100, n)
except ValueError as exc:
return _metric_record(0, len(predictions), str(exc))
def _normalize_text(value: str) -> str:
return re.sub(r"\s+", " ", str(value or "").strip().lower())
def _metric_record(score: float | None, sample_count: int, error: str = "", available: bool = True, **extra: Any) -> dict[str, Any]:
return {
"enabled": True,
"available": available,
"score": None if score is None else round(max(0.0, min(100.0, float(score))), 2),
"max_score": 100,
"unit": "percent",
"sample_count": sample_count,
"error": error,
**extra,
}
def _rouge_tokens(text: str) -> str:
text = str(text or "").strip().lower()
tokens: list[str] = []
for segment in re.findall(r"[一-鿿]+|[^\s一-鿿]+", text):
tokens.extend(segment if "一" <= segment[0] <= "鿿" else [segment])
return " ".join(tokens)
def _pair_rouge(reference: str, prediction: str) -> dict[str, float]:
try:
from rouge_score import rouge_scorer
except ImportError:
reference_tokens = _rouge_tokens(reference).split()
prediction_tokens = _rouge_tokens(prediction).split()
if not reference_tokens or not prediction_tokens:
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
def f1(overlap: int, reference_count: int, prediction_count: int) -> float:
if not reference_count or not prediction_count or not overlap:
return 0.0
precision = overlap / prediction_count
recall = overlap / reference_count
return 2 * precision * recall / (precision + recall)
from collections import Counter
reference_unigrams = Counter(reference_tokens)
prediction_unigrams = Counter(prediction_tokens)
unigram_overlap = sum((reference_unigrams & prediction_unigrams).values())
reference_bigrams = Counter(zip(reference_tokens, reference_tokens[1:]))
prediction_bigrams = Counter(zip(prediction_tokens, prediction_tokens[1:]))
bigram_overlap = sum((reference_bigrams & prediction_bigrams).values())
matrix = [[0] * (len(prediction_tokens) + 1) for _ in range(len(reference_tokens) + 1)]
for row, reference_token in enumerate(reference_tokens, start=1):
for column, prediction_token in enumerate(prediction_tokens, start=1):
matrix[row][column] = matrix[row - 1][column - 1] + 1 if reference_token == prediction_token else max(matrix[row - 1][column], matrix[row][column - 1])
return {
"rouge1": f1(unigram_overlap, len(reference_tokens), len(prediction_tokens)),
"rouge2": f1(bigram_overlap, max(len(reference_tokens) - 1, 0), max(len(prediction_tokens) - 1, 0)),
"rougeL": f1(matrix[-1][-1], len(reference_tokens), len(prediction_tokens)),
}
class _Tokenizer:
def tokenize(self, value: str) -> list[str]:
return value.split()
if not reference.strip() or not prediction.strip():
return {"rouge1": 0.0, "rouge2": 0.0, "rougeL": 0.0}
scorer = rouge_scorer.RougeScorer(("rouge1", "rouge2", "rougeL"), use_stemmer=False, tokenizer=_Tokenizer())
scores = scorer.score(_rouge_tokens(reference), _rouge_tokens(prediction))
return {key: float(value.fmeasure) for key, value in scores.items()}
def _compute_exact_match(references: list[str], predictions: list[str]) -> dict[str, Any]:
total = len(predictions)
if not total:
return {"enabled": True, "score": 0, "matched": 0, "total": 0}
return _metric_record(0, 0, matched=0, total=0)
matched = sum(
1
for ref, pred in zip(references, predictions)
if _normalize_text(ref) == _normalize_text(pred)
)
return {"enabled": True, "score": round(matched / total * 100, 2), "matched": matched, "total": total}
return _metric_record(matched / total * 100, total, matched=matched, total=total)
def _compute_text_similarity(references: list[str], predictions: list[str]) -> dict[str, Any]:
if not predictions:
return {"enabled": True, "score": 0}
return _metric_record(0, 0)
scores = [
SequenceMatcher(None, _normalize_text(ref), _normalize_text(pred)).ratio()
for ref, pred in zip(references, predictions)
]
return {"enabled": True, "score": round(sum(scores) / max(len(scores), 1) * 100, 2)}
return _metric_record(sum(scores) / max(len(scores), 1) * 100, len(predictions))
# ---------------------------------------------------------------------------
# LLM Judge
# ---------------------------------------------------------------------------
def _normalise_api_url(value: str) -> str:
url = str(value or "").strip().rstrip("/")
return url + "/chat/completions" if url.endswith("/v1") else url + "/v1/chat/completions"
def _parse_judge_reply(reply: str, score_min: float, score_max: float) -> tuple[float | None, dict[str, Any], str]:
payload: dict[str, Any] = {}
match = re.search(r"\{[\s\S]*\}", str(reply or ""))
if match:
try:
value = json.loads(match.group(0))
if isinstance(value, dict):
payload = value
except json.JSONDecodeError:
pass
reason = str(payload.get("综合评价") or payload.get("reason") or reply or "")
raw = payload.get("score", payload.get("overall_score"))
if raw is None:
matches = re.findall(r"(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)", reason, re.IGNORECASE)
raw = matches[-1] if matches else None
try:
numeric = float(raw)
except (TypeError, ValueError):
dimensions = payload.get("dimensions") if isinstance(payload.get("dimensions"), dict) else {}
if not dimensions:
dimensions = {
key: value
for key, value in payload.items()
if key not in {"score", "overall_score", "综合评价", "reason"}
}
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
numeric = sum(values) / len(values) if values else None
if numeric is None:
return None, payload, reason
# The judge prompt uses the configured score range. Do not treat a
# decimal such as 0.5/5 as a 0.5/1 score, otherwise low scores are
# incorrectly inflated (0.5/5 would become 50 instead of 10).
normalized = (numeric - score_min) / max(score_max - score_min, 1e-9) * 100
return max(0.0, min(100.0, normalized)), payload, reason
def _judge_sample(
question: str,
reference: str,
@@ -198,7 +310,7 @@ def _judge_sample(
pass_threshold = float(config.get("pass_threshold", 3))
if not api_url or not eval_model:
return {"score": 0, "max_score": score_max, "passed": False, "judgement": "未配置",
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "未配置",
"evaluation_reason": "未配置评测模型", "error_type": "其他"}
system_msg = (
@@ -227,7 +339,7 @@ def _judge_sample(
}).encode("utf-8")
req = urllib.request.Request(
f"{api_url}/v1/chat/completions",
_normalise_api_url(api_url),
data=body,
headers={
"Content-Type": "application/json",
@@ -238,39 +350,54 @@ def _judge_sample(
data = json.loads(resp.read().decode("utf-8"))
reply = data["choices"][0]["message"]["content"]
except Exception as exc:
return {"score": 0, "max_score": score_max, "passed": False,
return {"available": False, "score": None, "max_score": 100, "passed": False,
"judgement": "错误", "evaluation_reason": f"评测模型调用失败: {exc}",
"error_type": "其他"}
# Parse score from reply — look for patterns like "4分" or "Score: 4"
score = 0
import re
score_patterns = [
r'(?:得分|分数|评分|score)[^\d]*(\d+(?:\.\d+)?)',
r'(\d+(?:\.\d+)?)\s*分',
r'(\d+(?:\.\d+)?)\s*/\s*\d+',
]
for pat in score_patterns:
m = re.search(pat, reply, re.IGNORECASE)
if m:
try:
score = float(m.group(1))
except ValueError:
continue
break
score = max(score_min, min(score_max, score))
passed = score >= pass_threshold
parsed: dict[str, Any] = {}
match = re.search(r"\{[\s\S]*\}", reply)
if match:
try:
value = json.loads(match.group(0))
if isinstance(value, dict):
parsed = value
except json.JSONDecodeError:
pass
raw_score = parsed.get("score", parsed.get("overall_score"))
reason = str(parsed.get("综合评价") or parsed.get("reason") or reply)
if raw_score is None:
matches = re.findall(r'(?:得分|分数|评分|score|分)[^\d]*(\d+(?:\.\d+)?)', reason, re.IGNORECASE)
raw_score = matches[-1] if matches else None
try:
numeric_score = float(raw_score)
except (TypeError, ValueError):
dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {}
if not dimensions:
dimensions = {
key: value
for key, value in parsed.items()
if key not in {"score", "overall_score", "综合评价", "reason"}
}
values = [float(value) for value in dimensions.values() if isinstance(value, (int, float))]
numeric_score = sum(values) / len(values) if values else None
if numeric_score is None:
return {"available": False, "score": None, "max_score": 100, "passed": False, "judgement": "错误",
"evaluation_reason": "评测模型未返回可解析分数：" + reason[:1800], "error_type": "其他"}
normalized_score = (numeric_score - score_min) / max(score_max - score_min, 1e-9) * 100
normalized_score = max(0, min(100, normalized_score))
normalized_threshold = (pass_threshold - score_min) / max(score_max - score_min, 1e-9) * 100
passed = normalized_score >= normalized_threshold
# Determine judgement label
if score >= pass_threshold + 1:
if normalized_score >= min(100, normalized_threshold + 20):
judgement = "正确"
elif score >= pass_threshold:
elif passed:
judgement = "部分正确"
else:
judgement = "错误"
# Guess error type from reply
reply_lower = reply.lower()
reply_lower = (reply + " " + reason).lower()
if any(w in reply_lower for w in ["幻觉", "hallucination", "编造"]):
error_type = "幻觉"
elif any(w in reply_lower for w in ["不完整", "incomplete", "遗漏"]):
@@ -282,16 +409,84 @@ def _judge_sample(
else:
error_type = "其他"
parsed_dimensions = parsed.get("dimensions") if isinstance(parsed.get("dimensions"), dict) else {
key: value
for key, value in parsed.items()
if key not in {"score", "overall_score", "综合评价", "reason"}
}
return {
"score": score,
"max_score": score_max,
"available": True,
"score": round(normalized_score, 2),
"max_score": 100,
"raw_score": numeric_score,
"raw_max_score": score_max,
"passed": passed,
"judgement": judgement,
"evaluation_reason": reply[:2000],
"evaluation_reason": reason[:2000],
"error_type": error_type,
"dimension_scores": [
{
"name": str(name),
"score": round(
max(
0,
min(
100,
(float(value) - score_min)
/ max(score_max - score_min, 1e-9)
* 100,
),
),
2,
),
"max_score": 100,
}
for name, value in parsed_dimensions.items()
if isinstance(value, (int, float))
],
}
def _write_json(path: Path, payload: dict[str, Any]) -> None:
temporary = path.with_suffix(path.suffix + ".part")
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
temporary.replace(path)
def _write_eval_progress(
output_dir: Path,
status: str,
stage: str,
total: int,
completed: int,
message: str = "",
current_index: int = 0,
) -> None:
_write_json(
output_dir / "eval_progress.json",
{
"status": status,
"stage": stage,
"total": total,
"completed": completed,
"percentage": round(completed / total * 100, 1) if total else 100,
"current_index": current_index,
"message": message,
"updated_at": datetime.now(timezone.utc).isoformat(),
},
)
def _deterministic_sample_score(reference: str, prediction: str) -> float:
values = [SequenceMatcher(None, _normalize_text(reference), _normalize_text(prediction)).ratio()]
if _normalize_text(reference) == _normalize_text(prediction):
values.append(1.0)
rouge = _pair_rouge(reference, prediction)
if rouge.get("rougeL") is not None:
values.append(float(rouge["rougeL"]))
return round(sum(values) / len(values) * 100, 2)
# ---------------------------------------------------------------------------
# Main entry point
# ---------------------------------------------------------------------------
@@ -312,11 +507,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
print(f"[eval] loading dataset: {dataset_path}")
raw_samples = _load_dataset(dataset_path)
print(f"[eval] loaded {len(raw_samples)} samples")
_write_eval_progress(output_dir, "running", "dataset", len(raw_samples), 0, f"已加载 {len(raw_samples)} 条样本")
# ---- 2. Load model ----
print(f"[eval] loading model: {model_path}")
from compute.engines.llama_factory.inference import InferenceSession
session = InferenceSession()
_write_eval_progress(output_dir, "running", "model_loading", len(raw_samples), 0, "正在加载评测模型")
session.load(
model_name_or_path=model_path,
adapter_name_or_path=adapter_path,
@@ -329,6 +526,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
if not load_result.get("loaded"):
raise RuntimeError(f"model load failed: {load_result.get('error', 'unknown')}")
print(f"[eval] model loaded OK")
_write_eval_progress(output_dir, "running", "inference", len(raw_samples), 0, "开始生成模型回答")
# ---- 3. Run inference on each sample ----
samples: list[dict[str, Any]] = []
@@ -384,12 +582,45 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
] if judge_result else [],
"status": "completed",
})
if not judge_enabled:
deterministic_score = _deterministic_sample_score(reference, prediction)
samples[-1].update({
"score": deterministic_score,
"max_score": 100,
"raw_score": deterministic_score,
"raw_max_score": 100,
"passed": deterministic_score >= 60,
"judgement": "正确" if deterministic_score >= 80 else "部分正确" if deterministic_score >= 60 else "错误",
"evaluation_reason": "基于精确匹配、文本相似度和 ROUGE-L 的确定性评分",
})
_write_json(
output_dir / "eval_results.json",
{
"status": "running",
"overall_score": round(
sum(float(item["score"]) for item in samples if item.get("score") is not None)
/ max(len([item for item in samples if item.get("score") is not None]), 1),
output_precision,
),
"overall_score_max": 100,
"overall_evaluation": f"已完成 {len(samples)}/{total} 条样本",
"dimension_summary": [],
"samples": samples,
"sample_count": total,
"completed_count": len(samples),
"passed_count": sum(bool(item.get("passed")) for item in samples),
"basic_metrics": {},
"metric_summary_version": 2,
},
)
progress_pct = int(idx / max(total, 1) * 100)
_write_eval_progress(output_dir, "running", "inference", total, len(samples), f"已完成第 {idx} 条样本", idx)
print(f"[eval] sample {idx}/{total} ({progress_pct}%) done")
# ---- 4. Compute basic metrics ----
print(f"[eval] computing basic metrics on {len(predictions)} predictions")
_write_eval_progress(output_dir, "running", "metrics", total, len(samples), "正在计算评测指标", total)
metrics_result: dict[str, Any] = {}
bleu_cfg = basic_cfg.get("bleu", {})
@@ -397,11 +628,11 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
metrics_result["bleu"] = _compute_bleu(references, predictions, int(bleu_cfg.get("ngram", 4)))
rouge_cfg = basic_cfg.get("rouge", {})
if rouge_cfg.get("enabled"):
if rouge_cfg.get("enabled") or not judge_enabled:
metrics_result["rouge"] = _compute_rouge(references, predictions, rouge_cfg.get("methods"))
cosine_cfg = basic_cfg.get("cosine", {})
if cosine_cfg.get("enabled"):
if cosine_cfg.get("enabled") or not judge_enabled:
metrics_result["cosine"] = _compute_cosine(references, predictions)
metrics_result["exact_match"] = _compute_exact_match(references, predictions)
metrics_result["text_similarity"] = _compute_text_similarity(references, predictions)
@@ -412,16 +643,15 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
scored = [s for s in samples if s.get("score") is not None]
passed_count = len([s for s in scored if s.get("passed")])
avg_score = round(sum(s["score"] for s in scored) / max(len(scored), 1), output_precision)
max_score = dimension_cfg.get("score_max", 5)
overall_score = round(avg_score / max_score * 100, output_precision)
overall_score = avg_score
overall_score_max = 100
dimension_summary = [{
"name": "综合评分",
"name": "LLM Judge",
"score": overall_score,
"max_score": 100,
"pass_rate": round(passed_count / max(completed, 1) * 100, 1),
}]
overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/{max_score} 分"
overall_evaluation = f"评测完成：{completed} 样本，{passed_count} 通过，平均 {avg_score}/100 分"
else:
passed_count = 0
enabled_scores = [
@@ -444,6 +674,7 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
overall_evaluation = f"评测完成：{completed} 样本（未配置 LLM 评委）"
result = {
"status": "completed",
"overall_score": overall_score,
"overall_score_max": overall_score_max,
"overall_evaluation": overall_evaluation,
@@ -454,11 +685,13 @@ def run_eval(config: dict[str, Any]) -> dict[str, Any]:
"completed_count": completed,
"passed_count": passed_count,
"basic_metrics": metrics_result,
"metric_summary_version": 2,
}
# ---- 6. Write results ----
result_path = output_dir / "eval_results.json"
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
_write_eval_progress(output_dir, "completed", "completed", total, completed, "评测完成", total)
print(f"[eval] results written to {result_path}")
return result