merge: 合并远程 ft_wyt 分支,解决冲突
This commit is contained in:
@@ -35,7 +35,12 @@ from fastapi.responses import StreamingResponse
|
|||||||
from psycopg.rows import dict_row
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
|
||||||
|
|
||||||
from app.core.logging import get_structured_logger
|
from app.core.logging import get_structured_logger
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.db.platform_store import get_platform_store
|
||||||
|
|
||||||
from app.modules.data_process.algorithms import (
|
from app.modules.data_process.algorithms import (
|
||||||
ParsedText,
|
ParsedText,
|
||||||
canonical_record_json,
|
canonical_record_json,
|
||||||
@@ -61,6 +66,10 @@ from app.modules.data_process.document_chunking import (
|
|||||||
chunk_semantic_text,
|
chunk_semantic_text,
|
||||||
merge_short_chunks,
|
merge_short_chunks,
|
||||||
)
|
)
|
||||||
|
from app.modules.data_process.evaluation import (
|
||||||
|
evaluate_result_record,
|
||||||
|
reevaluate_edited_record,
|
||||||
|
)
|
||||||
from app.modules.data_process.generation import generate_model_records
|
from app.modules.data_process.generation import generate_model_records
|
||||||
from app.modules.data_process.office_preview import (
|
from app.modules.data_process.office_preview import (
|
||||||
MAX_XLSX_PREVIEW_ROWS,
|
MAX_XLSX_PREVIEW_ROWS,
|
||||||
@@ -82,6 +91,8 @@ from app.modules.data_process.store import (
|
|||||||
new_id,
|
new_id,
|
||||||
repeat_task_id,
|
repeat_task_id,
|
||||||
)
|
)
|
||||||
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
from app.modules.storage.policy import should_store_in_minio
|
||||||
from app.schemas.data_process import (
|
from app.schemas.data_process import (
|
||||||
DataProcessRegenerateRequest,
|
DataProcessRegenerateRequest,
|
||||||
DataProcessRepeatRequest,
|
DataProcessRepeatRequest,
|
||||||
@@ -97,6 +108,7 @@ from app.schemas.data_process import (
|
|||||||
PreviewItemUpdate,
|
PreviewItemUpdate,
|
||||||
ProcessType,
|
ProcessType,
|
||||||
PublishRequest,
|
PublishRequest,
|
||||||
|
ResultBatchEvaluateRequest,
|
||||||
ResultBatchRegenerateRequest,
|
ResultBatchRegenerateRequest,
|
||||||
ResultRegenerateRequest,
|
ResultRegenerateRequest,
|
||||||
ResultUpdate,
|
ResultUpdate,
|
||||||
@@ -221,9 +233,38 @@ def _commit_source_batch(
|
|||||||
staged: list[StagedSourceObject],
|
staged: list[StagedSourceObject],
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
storage.publish(staged)
|
storage.publish(staged)
|
||||||
|
storage_object_ids: list[str] = []
|
||||||
try:
|
try:
|
||||||
|
# The source reference remains in the task schema for compatibility,
|
||||||
|
# while storage_objects provides the authoritative MinIO index.
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
for item in prepared:
|
||||||
|
reference = str(item.get("storage_object_id") or "")
|
||||||
|
if not reference.startswith("minio://"):
|
||||||
|
continue
|
||||||
|
object_key = storage.object_key(reference)
|
||||||
|
metadata = get_object_storage().stat(object_key)
|
||||||
|
object_row = get_platform_store().create_storage_object({
|
||||||
|
"resource_type": "data_process_source",
|
||||||
|
"resource_id": task_id,
|
||||||
|
"version_id": str(item.get("id") or new_id("dpsf")),
|
||||||
|
"bucket": get_object_storage().bucket,
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": item.get("name"),
|
||||||
|
"content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"),
|
||||||
|
"byte_size": metadata.get("byte_size") or item.get("raw_size") or 0,
|
||||||
|
"checksum_sha256": item.get("checksum_sha256"),
|
||||||
|
"status": "available",
|
||||||
|
"created_by": item.get("created_by"),
|
||||||
|
})
|
||||||
|
storage_object_ids.append(str(object_row["id"]))
|
||||||
return store.add_source_files(task_id, prepared)
|
return store.add_source_files(task_id, prepared)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
for object_id in storage_object_ids:
|
||||||
|
try:
|
||||||
|
get_platform_store().update_storage_object(object_id, {"status": "deleted"})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
for item in staged:
|
for item in staged:
|
||||||
try:
|
try:
|
||||||
storage.delete(item.reference)
|
storage.delete(item.reference)
|
||||||
@@ -771,6 +812,25 @@ def _run_generation(
|
|||||||
duplicate_count=duplicate_count,
|
duplicate_count=duplicate_count,
|
||||||
error_count=error_count,
|
error_count=error_count,
|
||||||
)
|
)
|
||||||
|
result_bytes = "".join(
|
||||||
|
structured_json_dumps(item) + "\n" for item in accepted
|
||||||
|
).encode("utf-8")
|
||||||
|
if should_store_in_minio(len(result_bytes)):
|
||||||
|
result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl"
|
||||||
|
uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl")
|
||||||
|
get_platform_store().create_storage_object({
|
||||||
|
"resource_type": "data_process_result",
|
||||||
|
"resource_id": task_id,
|
||||||
|
"version_id": generation_run_id,
|
||||||
|
"bucket": uploaded["bucket"],
|
||||||
|
"object_key": result_key,
|
||||||
|
"file_name": f"{generation_run_id}.jsonl",
|
||||||
|
"content_type": "application/jsonl",
|
||||||
|
"byte_size": len(result_bytes),
|
||||||
|
"checksum_sha256": hashlib.sha256(result_bytes).hexdigest(),
|
||||||
|
"status": "available",
|
||||||
|
"created_by": (store.get_task(task_id) or {}).get("created_by"),
|
||||||
|
})
|
||||||
logger.info(
|
logger.info(
|
||||||
"data process generation completed task_id=%s generation_run_id=%s "
|
"data process generation completed task_id=%s generation_run_id=%s "
|
||||||
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
|
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
|
||||||
@@ -930,7 +990,7 @@ def _repeat_file_copies(
|
|||||||
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
||||||
new_file_id = new_id("dpsf")
|
new_file_id = new_id("dpsf")
|
||||||
old_reference = str(source.get("storage_object_id") or "")
|
old_reference = str(source.get("storage_object_id") or "")
|
||||||
if old_reference.startswith("local://data-process/"):
|
if old_reference.startswith(("local://data-process/", "minio://data-process/")):
|
||||||
staged_object = storage.stage_copy(
|
staged_object = storage.stage_copy(
|
||||||
batch_id=batch_id,
|
batch_id=batch_id,
|
||||||
source_reference=old_reference,
|
source_reference=old_reference,
|
||||||
@@ -2044,12 +2104,13 @@ def update_result(
|
|||||||
or 20
|
or 20
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
quality = score_quality(
|
# 编辑后内容已变化:重算规则与语义层,旧的评审分不再可信直接丢弃。
|
||||||
|
update["quality_score"] = reevaluate_edited_record(
|
||||||
merged,
|
merged,
|
||||||
min_output_length=minimum,
|
|
||||||
source_content=source_content,
|
source_content=source_content,
|
||||||
|
previous_quality=current.get("quality_score"),
|
||||||
|
min_output_length=minimum,
|
||||||
)
|
)
|
||||||
update["quality_score"] = asdict(quality)
|
|
||||||
result = store.update_result(
|
result = store.update_result(
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -2094,11 +2155,6 @@ def restore_result(
|
|||||||
or 20
|
or 20
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
quality = score_quality(
|
|
||||||
restored,
|
|
||||||
min_output_length=minimum,
|
|
||||||
source_content=source_content,
|
|
||||||
)
|
|
||||||
restored = store.update_result(
|
restored = store.update_result(
|
||||||
task_id,
|
task_id,
|
||||||
result_id,
|
result_id,
|
||||||
@@ -2108,7 +2164,12 @@ def restore_result(
|
|||||||
"output": restored["output"],
|
"output": restored["output"],
|
||||||
"chosen": restored["chosen"],
|
"chosen": restored["chosen"],
|
||||||
"rejected": restored["rejected"],
|
"rejected": restored["rejected"],
|
||||||
"quality_score": asdict(quality),
|
"quality_score": reevaluate_edited_record(
|
||||||
|
restored,
|
||||||
|
source_content=source_content,
|
||||||
|
previous_quality=current.get("quality_score"),
|
||||||
|
min_output_length=minimum,
|
||||||
|
),
|
||||||
"expected_updated_at": current.get("updated_at"),
|
"expected_updated_at": current.get("updated_at"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -2271,6 +2332,46 @@ def _safe_regeneration_error(exc: Exception) -> str:
|
|||||||
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
|
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate_result_in_place(
|
||||||
|
task_id: str,
|
||||||
|
current: dict[str, Any],
|
||||||
|
source_content: str,
|
||||||
|
config: dict[str, Any],
|
||||||
|
evaluation_model: dict[str, Any] | None,
|
||||||
|
store: DataProcessStore,
|
||||||
|
*,
|
||||||
|
expected_updated_at: str,
|
||||||
|
model_client: httpx.Client | None = None,
|
||||||
|
minimum: int = 20,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""评测单条结果并落库;复用逐结果互斥锁避免与重生成并发写冲突。"""
|
||||||
|
|
||||||
|
result_id = str(current["id"])
|
||||||
|
with _claim_result_regeneration(task_id, result_id):
|
||||||
|
quality = evaluate_result_record(
|
||||||
|
{
|
||||||
|
"instruction": current.get("instruction"),
|
||||||
|
"input": current.get("input"),
|
||||||
|
"output": current.get("output"),
|
||||||
|
"chosen": current.get("chosen"),
|
||||||
|
"rejected": current.get("rejected"),
|
||||||
|
},
|
||||||
|
source_content=source_content,
|
||||||
|
model=evaluation_model,
|
||||||
|
config=config,
|
||||||
|
client=model_client,
|
||||||
|
min_output_length=minimum,
|
||||||
|
)
|
||||||
|
return store.update_result(
|
||||||
|
task_id,
|
||||||
|
result_id,
|
||||||
|
{
|
||||||
|
"quality_score": quality,
|
||||||
|
"expected_updated_at": expected_updated_at,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{task_id}/results/regenerate-batch")
|
@router.post("/{task_id}/results/regenerate-batch")
|
||||||
def regenerate_results_batch(
|
def regenerate_results_batch(
|
||||||
task_id: str,
|
task_id: str,
|
||||||
@@ -2444,6 +2545,186 @@ def regenerate_results_batch(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{task_id}/results/evaluate-batch")
|
||||||
|
def evaluate_results_batch(
|
||||||
|
task_id: str,
|
||||||
|
payload: ResultBatchEvaluateRequest,
|
||||||
|
store: DataProcessStore = Depends(get_data_process_store),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""对一批结果执行三层质量评测(规则+语义+评审),允许部分成功。"""
|
||||||
|
|
||||||
|
started_at = time.perf_counter()
|
||||||
|
batch_id = new_id("dpeb")
|
||||||
|
with api_errors():
|
||||||
|
task = store.get_task(task_id)
|
||||||
|
if task.get("status") == "running":
|
||||||
|
raise ConflictError("data process task is running")
|
||||||
|
if task.get("output_dataset_id"):
|
||||||
|
raise InvalidStateError("published results cannot be evaluated")
|
||||||
|
config = task.get("config") or {}
|
||||||
|
evaluation_model: dict[str, Any] | None = None
|
||||||
|
model_id = _value(config, "generation_model_id", "generationModelId", None)
|
||||||
|
if model_id:
|
||||||
|
try:
|
||||||
|
evaluation_model = store.get_generation_model(str(model_id))
|
||||||
|
except NotFoundError:
|
||||||
|
logger.warning(
|
||||||
|
"data process evaluation model unavailable, judge layer "
|
||||||
|
"skipped task_id=%s model_id=%s",
|
||||||
|
task_id,
|
||||||
|
model_id,
|
||||||
|
)
|
||||||
|
evaluation_config = {
|
||||||
|
**config,
|
||||||
|
"output_type": str(
|
||||||
|
_value(config, "output_type", "outputType", "standard")
|
||||||
|
).strip().lower(),
|
||||||
|
}
|
||||||
|
minimum = max(
|
||||||
|
1,
|
||||||
|
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared: list[tuple[int, dict[str, Any], str, str]] = []
|
||||||
|
failures: list[tuple[int, dict[str, str]]] = []
|
||||||
|
for index, requested in enumerate(payload.items):
|
||||||
|
try:
|
||||||
|
current = store.get_result(task_id, requested.result_id)
|
||||||
|
if requested.expected_updated_at != str(current.get("updated_at") or ""):
|
||||||
|
raise ConflictError("data process result was modified by another request")
|
||||||
|
source_content = ""
|
||||||
|
preview_id = current.get("preview_item_id")
|
||||||
|
if preview_id:
|
||||||
|
preview = store.get_preview_item(task_id, str(preview_id))
|
||||||
|
source_content = str(
|
||||||
|
preview.get("edited_content")
|
||||||
|
or preview.get("original_content")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
prepared.append(
|
||||||
|
(index, current, source_content, requested.expected_updated_at)
|
||||||
|
)
|
||||||
|
except ConflictError as exc:
|
||||||
|
failures.append((index, {
|
||||||
|
"result_id": requested.result_id,
|
||||||
|
"code": "conflict",
|
||||||
|
"message": _safe_regeneration_error(exc),
|
||||||
|
}))
|
||||||
|
except (NotFoundError, InvalidStateError) as exc:
|
||||||
|
failures.append((index, {
|
||||||
|
"result_id": requested.result_id,
|
||||||
|
"code": "skipped",
|
||||||
|
"message": _safe_regeneration_error(exc),
|
||||||
|
}))
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"data process result batch evaluation started batch_id=%s task_id=%s "
|
||||||
|
"requested=%s prepared=%s judge_enabled=%s",
|
||||||
|
batch_id,
|
||||||
|
task_id,
|
||||||
|
len(payload.items),
|
||||||
|
len(prepared),
|
||||||
|
evaluation_model is not None,
|
||||||
|
)
|
||||||
|
successes: list[tuple[int, dict[str, Any]]] = []
|
||||||
|
if prepared:
|
||||||
|
try:
|
||||||
|
from app.modules.data_process.algorithms.embedding import (
|
||||||
|
semantic_embedding_model,
|
||||||
|
)
|
||||||
|
|
||||||
|
semantic_embedding_model()
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"data process semantic embedding unavailable, semantic layer "
|
||||||
|
"will be skipped batch_id=%s",
|
||||||
|
batch_id,
|
||||||
|
)
|
||||||
|
request_timeout = _result_regeneration_timeout(config)
|
||||||
|
model_timeout = httpx.Timeout(
|
||||||
|
request_timeout,
|
||||||
|
connect=min(10.0, request_timeout),
|
||||||
|
)
|
||||||
|
model_limits = httpx.Limits(
|
||||||
|
max_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||||
|
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
|
||||||
|
)
|
||||||
|
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
|
||||||
|
ThreadPoolExecutor(
|
||||||
|
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
||||||
|
thread_name_prefix="data-result-evaluation",
|
||||||
|
) as executor:
|
||||||
|
futures = {
|
||||||
|
executor.submit(
|
||||||
|
_evaluate_result_in_place,
|
||||||
|
task_id,
|
||||||
|
current,
|
||||||
|
source_content,
|
||||||
|
evaluation_config,
|
||||||
|
evaluation_model,
|
||||||
|
store,
|
||||||
|
expected_updated_at=expected_updated_at,
|
||||||
|
model_client=model_client if evaluation_model else None,
|
||||||
|
minimum=minimum,
|
||||||
|
): (index, str(current["id"]), time.perf_counter())
|
||||||
|
for index, current, source_content, expected_updated_at in prepared
|
||||||
|
}
|
||||||
|
for future in as_completed(futures):
|
||||||
|
index, result_id, item_started_at = futures[future]
|
||||||
|
try:
|
||||||
|
evaluated = future.result()
|
||||||
|
successes.append((index, evaluated))
|
||||||
|
outcome = "succeeded"
|
||||||
|
except ConflictError as exc:
|
||||||
|
outcome = "conflict"
|
||||||
|
failures.append((index, {
|
||||||
|
"result_id": result_id,
|
||||||
|
"code": outcome,
|
||||||
|
"message": _safe_regeneration_error(exc),
|
||||||
|
}))
|
||||||
|
except Exception as exc:
|
||||||
|
outcome = "evaluation_failed"
|
||||||
|
failures.append((index, {
|
||||||
|
"result_id": result_id,
|
||||||
|
"code": outcome,
|
||||||
|
"message": _safe_regeneration_error(exc),
|
||||||
|
}))
|
||||||
|
logger.info(
|
||||||
|
"data process result batch evaluation item finished "
|
||||||
|
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
|
||||||
|
batch_id,
|
||||||
|
task_id,
|
||||||
|
result_id,
|
||||||
|
outcome,
|
||||||
|
(time.perf_counter() - item_started_at) * 1000,
|
||||||
|
)
|
||||||
|
|
||||||
|
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
|
||||||
|
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
|
||||||
|
duration_ms = (time.perf_counter() - started_at) * 1000
|
||||||
|
logger.info(
|
||||||
|
"data process result batch evaluation completed batch_id=%s task_id=%s "
|
||||||
|
"succeeded=%s failed=%s duration_ms=%.2f",
|
||||||
|
batch_id,
|
||||||
|
task_id,
|
||||||
|
len(success_items),
|
||||||
|
len(failure_items),
|
||||||
|
duration_ms,
|
||||||
|
)
|
||||||
|
return ok(
|
||||||
|
{
|
||||||
|
"batch_id": batch_id,
|
||||||
|
"total": len(payload.items),
|
||||||
|
"succeeded": len(success_items),
|
||||||
|
"failed": len(failure_items),
|
||||||
|
"duration_ms": round(duration_ms, 2),
|
||||||
|
"items": success_items,
|
||||||
|
"failures": failure_items,
|
||||||
|
},
|
||||||
|
"data process results evaluated",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{task_id}/results/{result_id}/regenerate")
|
@router.post("/{task_id}/results/{result_id}/regenerate")
|
||||||
def regenerate_result(
|
def regenerate_result(
|
||||||
task_id: str,
|
task_id: str,
|
||||||
|
|||||||
@@ -23,8 +23,9 @@ from app.core.audit import audit_log, AuditActions
|
|||||||
from app.core.op_log import op_log, OpModule, OpAction
|
from app.core.op_log import op_log, OpModule, OpAction
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
|
from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once
|
||||||
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
|
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
|
||||||
|
from app.modules.storage.policy import should_store_in_minio
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
_LOGIN_FAILURES: dict[str, list[float]] = {}
|
_LOGIN_FAILURES: dict[str, list[float]] = {}
|
||||||
@@ -57,6 +58,33 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_gpu_indices(payload: dict[str, Any], *, allow_primary: bool = True) -> list[int]:
|
||||||
|
"""Normalize all frontend GPU selection shapes to sorted integer indexes."""
|
||||||
|
raw = payload.get("gpu_indices")
|
||||||
|
if raw is None:
|
||||||
|
raw = payload.get("gpus")
|
||||||
|
if raw is None and allow_primary and payload.get("gpu_id") is not None:
|
||||||
|
raw = [payload.get("gpu_id")]
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return []
|
||||||
|
if isinstance(raw, str):
|
||||||
|
raw = [item.strip() for item in raw.split(",") if item.strip()]
|
||||||
|
if not isinstance(raw, (list, tuple, set)):
|
||||||
|
raw = [raw]
|
||||||
|
result: set[int] = set()
|
||||||
|
for item in raw:
|
||||||
|
if isinstance(item, str) and ":" in item:
|
||||||
|
item = item.rsplit(":", 1)[-1]
|
||||||
|
try:
|
||||||
|
index = int(item)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise ValueError(f"invalid GPU index: {item}") from exc
|
||||||
|
if index < 0:
|
||||||
|
raise ValueError("GPU index must be non-negative")
|
||||||
|
result.add(index)
|
||||||
|
return sorted(result)
|
||||||
|
|
||||||
|
|
||||||
async def _wait_for_object_storage() -> None:
|
async def _wait_for_object_storage() -> None:
|
||||||
"""Wait for MinIO before starting a resource task."""
|
"""Wait for MinIO before starting a resource task."""
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
@@ -73,6 +101,106 @@ async def _wait_for_object_storage() -> None:
|
|||||||
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
|
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
|
||||||
|
|
||||||
|
|
||||||
|
def _store_json_snapshot(
|
||||||
|
store: Any,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
version_id: str,
|
||||||
|
object_key: str,
|
||||||
|
payload: dict[str, Any],
|
||||||
|
created_by: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Persist non-secret task/model parameters as an auditable MinIO snapshot."""
|
||||||
|
sanitized = {
|
||||||
|
key: value
|
||||||
|
for key, value in payload.items()
|
||||||
|
if key not in {"api_key", "secret_key", "password", "token", "access_token"}
|
||||||
|
}
|
||||||
|
raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
|
||||||
|
# Task/evaluation payloads are already persisted in PostgreSQL. Avoid an
|
||||||
|
# extra MinIO round trip for small non-secret parameter snapshots.
|
||||||
|
if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"):
|
||||||
|
return {}
|
||||||
|
uploaded = get_object_storage().put_bytes(object_key, raw, "application/json")
|
||||||
|
return store.create_storage_object({
|
||||||
|
"resource_type": resource_type,
|
||||||
|
"resource_id": resource_id,
|
||||||
|
"version_id": version_id,
|
||||||
|
"bucket": uploaded["bucket"],
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": Path(object_key).name,
|
||||||
|
"content_type": "application/json",
|
||||||
|
"byte_size": len(raw),
|
||||||
|
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
||||||
|
"status": "available",
|
||||||
|
"created_by": created_by,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _dataset_file_bytes(store: Any, file_id: str) -> bytes:
|
||||||
|
"""Return the canonical dataset bytes, lazily indexing legacy DB content."""
|
||||||
|
row = store.dataset_file(file_id)
|
||||||
|
if not get_settings().minio_enabled:
|
||||||
|
return str(row.get("content") or "").encode("utf-8")
|
||||||
|
|
||||||
|
storage_object = None
|
||||||
|
object_id = str(row.get("storage_object_id") or "")
|
||||||
|
if object_id:
|
||||||
|
try:
|
||||||
|
storage_object = store.storage_object(object_id)
|
||||||
|
except KeyError:
|
||||||
|
storage_object = None
|
||||||
|
if storage_object and storage_object.get("status") == "available":
|
||||||
|
try:
|
||||||
|
return get_object_storage().get_bytes(storage_object["object_key"])
|
||||||
|
except Exception as exc: # noqa: BLE001 - expose storage outage to callers
|
||||||
|
raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc
|
||||||
|
|
||||||
|
# Compatibility migration for files created before MinIO was enabled.
|
||||||
|
raw = str(row.get("content") or "").encode("utf-8")
|
||||||
|
if not raw:
|
||||||
|
raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}")
|
||||||
|
# Small files intentionally remain database-backed. They can still be
|
||||||
|
# copied to a compute node directly when a task needs them.
|
||||||
|
if not should_store_in_minio(len(raw), file_format=row.get("file_format")):
|
||||||
|
return raw
|
||||||
|
object_key = (
|
||||||
|
f"datasets/{row['dataset_id']}/versions/"
|
||||||
|
f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}"
|
||||||
|
)
|
||||||
|
uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream")
|
||||||
|
created = store.create_storage_object({
|
||||||
|
"resource_type": "dataset",
|
||||||
|
"resource_id": str(row["dataset_id"]),
|
||||||
|
"version_id": str(row.get("active_version_id") or row["id"]),
|
||||||
|
"bucket": uploaded["bucket"],
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": row.get("name"),
|
||||||
|
"content_type": "application/octet-stream",
|
||||||
|
"byte_size": len(raw),
|
||||||
|
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
||||||
|
"status": "available",
|
||||||
|
})
|
||||||
|
store.link_dataset_file_storage_object(str(row["id"]), created["id"])
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes:
|
||||||
|
row = store.dataset_file(file_id)
|
||||||
|
try:
|
||||||
|
version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id)
|
||||||
|
except StopIteration as exc:
|
||||||
|
raise KeyError(version_id) from exc
|
||||||
|
object_id = str(version.get("storage_object_id") or "")
|
||||||
|
if get_settings().minio_enabled and object_id:
|
||||||
|
try:
|
||||||
|
obj = store.storage_object(object_id)
|
||||||
|
return get_object_storage().get_bytes(obj["object_key"])
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
return _dataset_file_bytes(store, file_id)
|
||||||
|
|
||||||
|
|
||||||
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
|
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
|
||||||
"""Select the compute node for an eval job.
|
"""Select the compute node for an eval job.
|
||||||
|
|
||||||
@@ -103,18 +231,63 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id:
|
|||||||
if not get_settings().minio_enabled or not resource_id:
|
if not get_settings().minio_enabled or not resource_id:
|
||||||
return None
|
return None
|
||||||
objects = store.storage_objects_for_resource(resource_type, resource_id)
|
objects = store.storage_objects_for_resource(resource_type, resource_id)
|
||||||
|
if not objects and resource_type in {"model", "trained_model"}:
|
||||||
|
resource = None
|
||||||
|
if resource_type == "model":
|
||||||
|
try:
|
||||||
|
resource = store.model(resource_id)
|
||||||
|
except KeyError:
|
||||||
|
try:
|
||||||
|
resource = store.model_by_name(resource_id)
|
||||||
|
except KeyError:
|
||||||
|
resource = next((item for item in store.models() if item.get("path") == resource_id), None)
|
||||||
|
else:
|
||||||
|
resource = next(
|
||||||
|
(item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
source_path = str(
|
||||||
|
(resource or {}).get("path")
|
||||||
|
or (resource or {}).get("merged_path")
|
||||||
|
or (resource or {}).get("artifact_dir")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
resolved_id = str((resource or {}).get("id") or resource_id)
|
||||||
|
if source_path and resolved_id:
|
||||||
|
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
||||||
|
await _archive_node_directory(
|
||||||
|
store,
|
||||||
|
client,
|
||||||
|
node,
|
||||||
|
source_path,
|
||||||
|
resource_type,
|
||||||
|
resolved_id,
|
||||||
|
"legacy-import",
|
||||||
|
f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}",
|
||||||
|
)
|
||||||
|
resource_id = resolved_id
|
||||||
|
objects = store.storage_objects_for_resource(resource_type, resource_id)
|
||||||
if not objects:
|
if not objects:
|
||||||
return None
|
return None
|
||||||
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
client = ComputeNodeClient(node["api_base_url"], timeout=900)
|
||||||
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
|
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
|
||||||
for obj in objects:
|
for obj in objects:
|
||||||
|
object_key = str(obj["object_key"])
|
||||||
|
marker = f"{root_name}/{resource_id}/versions/"
|
||||||
|
relative_name = Path(str(obj.get("file_name") or object_key)).name
|
||||||
|
if marker in object_key:
|
||||||
|
suffix = object_key.split(marker, 1)[1]
|
||||||
|
if "/" in suffix:
|
||||||
|
suffix = suffix.split("/", 1)[1]
|
||||||
|
if suffix:
|
||||||
|
relative_name = suffix
|
||||||
await client.prepare_cache({
|
await client.prepare_cache({
|
||||||
"resource_id": resource_id,
|
"resource_id": resource_id,
|
||||||
"version_id": obj["version_id"],
|
"version_id": obj["version_id"],
|
||||||
"download_url": get_object_storage().presigned_get(obj["object_key"]),
|
"download_url": get_object_storage().presigned_get(object_key),
|
||||||
"checksum_sha256": obj.get("checksum_sha256") or "",
|
"checksum_sha256": obj.get("checksum_sha256") or "",
|
||||||
"byte_size": obj.get("byte_size") or 0,
|
"byte_size": obj.get("byte_size") or 0,
|
||||||
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
|
"relative_path": f"{root_name}/{resource_id}/{relative_name}",
|
||||||
})
|
})
|
||||||
return f"/data/yg-ft/{root_name}/{resource_id}"
|
return f"/data/yg-ft/{root_name}/{resource_id}"
|
||||||
|
|
||||||
@@ -370,8 +543,23 @@ async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[st
|
|||||||
if not preflight["valid"]:
|
if not preflight["valid"]:
|
||||||
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
|
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
|
||||||
raise RuntimeError(f"preflight failed: {errors}")
|
raise RuntimeError(f"preflight failed: {errors}")
|
||||||
payload = {**payload, "compute_node_id": preflight["node"]["id"]}
|
prepared_job_payload = preflight.get("job_payload") or {}
|
||||||
|
payload = {
|
||||||
|
**payload,
|
||||||
|
"compute_node_id": preflight["node"]["id"],
|
||||||
|
"prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"),
|
||||||
|
}
|
||||||
task = store.start_task(payload)
|
task = store.start_task(payload)
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
_store_json_snapshot(
|
||||||
|
store,
|
||||||
|
"fine_tune",
|
||||||
|
str(task["id"]),
|
||||||
|
str(task["id"]),
|
||||||
|
f"training/{task['id']}/versions/{task['id']}/training-config.json",
|
||||||
|
task,
|
||||||
|
task.get("created_by"),
|
||||||
|
)
|
||||||
if get_settings().compute_mode == "simulator":
|
if get_settings().compute_mode == "simulator":
|
||||||
return task
|
return task
|
||||||
node, job_payload = store.build_compute_job_payload(task["id"])
|
node, job_payload = store.build_compute_job_payload(task["id"])
|
||||||
@@ -427,6 +615,20 @@ async def _fine_tune_preflight_with_job_payload(
|
|||||||
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
|
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
|
||||||
try:
|
try:
|
||||||
await _wait_for_object_storage()
|
await _wait_for_object_storage()
|
||||||
|
base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "")
|
||||||
|
base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "")
|
||||||
|
base_model = None
|
||||||
|
if base_model_id:
|
||||||
|
try:
|
||||||
|
base_model = store.model(base_model_id)
|
||||||
|
except KeyError:
|
||||||
|
base_model = None
|
||||||
|
if base_model is None:
|
||||||
|
base_model = next((item for item in store.models() if item.get("path") == base_model_path), None)
|
||||||
|
if base_model:
|
||||||
|
prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node)
|
||||||
|
if prepared_model:
|
||||||
|
job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model}
|
||||||
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
|
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
|
||||||
sync_errors.append(f"shared storage health check failed: {exc}")
|
sync_errors.append(f"shared storage health check failed: {exc}")
|
||||||
if get_settings().compute_mode == "simulator":
|
if get_settings().compute_mode == "simulator":
|
||||||
@@ -1076,8 +1278,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict =
|
|||||||
@router.get("/dataset-manage/preview/{file_id}")
|
@router.get("/dataset-manage/preview/{file_id}")
|
||||||
async def dataset_preview(file_id: str) -> dict[str, Any]:
|
async def dataset_preview(file_id: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
row = get_platform_store().dataset_file(file_id)
|
store = get_platform_store()
|
||||||
return ok({"content": row["content"]})
|
content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace")
|
||||||
|
return ok({"content": content})
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "dataset file not found")
|
raise fail(404, "dataset file not found")
|
||||||
|
|
||||||
@@ -1106,7 +1309,8 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An
|
|||||||
version = next((item for item in versions if item["id"] == version_id), None)
|
version = next((item for item in versions if item["id"] == version_id), None)
|
||||||
if not version:
|
if not version:
|
||||||
raise KeyError(version_id)
|
raise KeyError(version_id)
|
||||||
return ok({"version": version, "content": row["content"]})
|
content = _dataset_version_bytes(get_platform_store(), file_id, version_id)
|
||||||
|
return ok({"version": version, "content": content.decode("utf-8", errors="replace")})
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise fail(404, "dataset version not found")
|
raise fail(404, "dataset version not found")
|
||||||
|
|
||||||
@@ -1197,7 +1401,7 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
if get_settings().minio_enabled:
|
if get_settings().minio_enabled:
|
||||||
files = store.training_dataset_files(dataset_id)
|
files = store.training_dataset_files(dataset_id)
|
||||||
object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {}
|
object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||||
resource_ids = {str(dataset_id)} | {
|
resource_ids = {str(dataset_id)} | {
|
||||||
str(item.get("dataset_id"))
|
str(item.get("dataset_id"))
|
||||||
for item in files
|
for item in files
|
||||||
@@ -1206,18 +1410,19 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
for resource_id in resource_ids:
|
for resource_id in resource_ids:
|
||||||
for obj in store.storage_objects_for_resource("dataset", resource_id):
|
for obj in store.storage_objects_for_resource("dataset", resource_id):
|
||||||
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
|
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
|
||||||
object_by_resource_name[(resource_id, file_name)] = obj
|
object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj
|
||||||
results: list[dict[str, Any]] = []
|
results: list[dict[str, Any]] = []
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
for item in files:
|
for item in files:
|
||||||
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
|
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
|
||||||
item_dataset_id = str(item.get("dataset_id") or dataset_id)
|
item_dataset_id = str(item.get("dataset_id") or dataset_id)
|
||||||
obj = object_by_resource_name.get((item_dataset_id, target_name))
|
|
||||||
if not obj and item.get("content"):
|
|
||||||
# 兼容 MinIO 接入前已经发布的数据处理数据集:
|
|
||||||
# 预检时用数据库正文补建对象,避免要求用户重新处理数据集。
|
|
||||||
raw = str(item.get("content") or "").encode("utf-8")
|
|
||||||
version_id = str(item.get("active_version_id") or item["id"])
|
version_id = str(item.get("active_version_id") or item["id"])
|
||||||
|
obj = object_by_resource_name.get((item_dataset_id, target_name, version_id))
|
||||||
|
if not obj and item.get("content"):
|
||||||
|
# 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建
|
||||||
|
# MinIO 对象,小文件直接从数据库正文同步到目标节点。
|
||||||
|
raw = str(item.get("content") or "").encode("utf-8")
|
||||||
|
if should_store_in_minio(len(raw)):
|
||||||
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
|
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
|
||||||
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
|
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
|
||||||
obj = store.create_storage_object({
|
obj = store.create_storage_object({
|
||||||
@@ -1233,8 +1438,30 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
"status": "available",
|
"status": "available",
|
||||||
})
|
})
|
||||||
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
|
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
|
||||||
|
else:
|
||||||
|
result = await client.upload_file(
|
||||||
|
target_name,
|
||||||
|
raw,
|
||||||
|
f"datasets/{dataset_id}/{target_name}",
|
||||||
|
resource_type="dataset",
|
||||||
|
resource_id=dataset_id,
|
||||||
|
)
|
||||||
|
store.upsert_resource_replica(
|
||||||
|
node["id"], "dataset", dataset_id, str(result.get("local_path") or "")
|
||||||
|
)
|
||||||
|
results.append({
|
||||||
|
"node_id": node["id"],
|
||||||
|
"node_code": node.get("code"),
|
||||||
|
"file_id": item.get("id"),
|
||||||
|
"name": target_name,
|
||||||
|
"local_path": result.get("local_path"),
|
||||||
|
"byte_size": result.get("byte_size"),
|
||||||
|
"checksum_sha256": result.get("checksum_sha256"),
|
||||||
|
"storage_backend": "database",
|
||||||
|
})
|
||||||
|
continue
|
||||||
if not obj:
|
if not obj:
|
||||||
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
|
raise RuntimeError(f"dataset file is not available: {target_name}")
|
||||||
url = get_object_storage().presigned_get(obj["object_key"])
|
url = get_object_storage().presigned_get(obj["object_key"])
|
||||||
result = await client.prepare_cache({
|
result = await client.prepare_cache({
|
||||||
"resource_id": dataset_id,
|
"resource_id": dataset_id,
|
||||||
@@ -1250,7 +1477,13 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
dataset_id,
|
dataset_id,
|
||||||
str(result.get("local_path") or ""),
|
str(result.get("local_path") or ""),
|
||||||
)
|
)
|
||||||
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
|
results.append({
|
||||||
|
**result,
|
||||||
|
"file_id": item.get("id"),
|
||||||
|
"name": target_name,
|
||||||
|
"node_id": node["id"],
|
||||||
|
"storage_backend": "minio",
|
||||||
|
})
|
||||||
return results
|
return results
|
||||||
if not dataset_id:
|
if not dataset_id:
|
||||||
raise RuntimeError("train_dataset_id is required")
|
raise RuntimeError("train_dataset_id is required")
|
||||||
@@ -1316,7 +1549,11 @@ async def upload_dataset_files(
|
|||||||
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
|
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
|
||||||
created.append(created_file)
|
created.append(created_file)
|
||||||
pending_sync.append((created_file["id"], created_file["name"], raw))
|
pending_sync.append((created_file["id"], created_file["name"], raw))
|
||||||
if get_settings().minio_enabled:
|
if should_store_in_minio(
|
||||||
|
len(raw),
|
||||||
|
content_type=file.content_type,
|
||||||
|
file_format=Path(created_file["name"]).suffix,
|
||||||
|
):
|
||||||
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
|
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
|
||||||
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
|
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
|
||||||
storage_object = get_platform_store().create_storage_object({
|
storage_object = get_platform_store().create_storage_object({
|
||||||
@@ -1360,7 +1597,10 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
|
|||||||
full_file = store.dataset_file(str(item["id"]))
|
full_file = store.dataset_file(str(item["id"]))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
continue
|
continue
|
||||||
files.append({**item, "content": full_file.get("content") or ""})
|
files.append({
|
||||||
|
**item,
|
||||||
|
"content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"),
|
||||||
|
})
|
||||||
if not files:
|
if not files:
|
||||||
raise fail(404, "dataset has no downloadable files")
|
raise fail(404, "dataset has no downloadable files")
|
||||||
|
|
||||||
@@ -1401,8 +1641,12 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
|
|||||||
|
|
||||||
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
||||||
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
|
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
|
||||||
row = get_platform_store().dataset_file(file_id)
|
store = get_platform_store()
|
||||||
return PlainTextResponse(row["content"], media_type="text/plain")
|
row = store.dataset_file(file_id)
|
||||||
|
if str(row.get("dataset_id")) != str(dataset_id):
|
||||||
|
raise fail(404, "dataset file not found")
|
||||||
|
content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id)
|
||||||
|
return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain")
|
||||||
|
|
||||||
|
|
||||||
@router.get("/dataset-manage")
|
@router.get("/dataset-manage")
|
||||||
@@ -1545,10 +1789,10 @@ async def start_fine_tune(
|
|||||||
if node_id and gpu_indices:
|
if node_id and gpu_indices:
|
||||||
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
|
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
|
||||||
raise fail(403, "无权使用所选 GPU,请联系管理员分配")
|
raise fail(403, "无权使用所选 GPU,请联系管理员分配")
|
||||||
# 记录创建者
|
|
||||||
if node_id and not gpu_indices:
|
if node_id and not gpu_indices:
|
||||||
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
|
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
|
||||||
payload["strict_node_selection"] = bool(node_id)
|
# 页面明确选择节点时,调度器必须保持节点约束;否则可能落到其它节点。
|
||||||
|
payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id"))
|
||||||
payload.setdefault("created_by", current_user.get("id"))
|
payload.setdefault("created_by", current_user.get("id"))
|
||||||
try:
|
try:
|
||||||
return ok(await _submit_fine_tune_task(store, payload))
|
return ok(await _submit_fine_tune_task(store, payload))
|
||||||
@@ -1679,6 +1923,42 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/fine-tune/{task_id}/gpu-status")
|
||||||
|
async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
|
"""Return live GPU metrics for the task's selected node and cards."""
|
||||||
|
store = get_platform_store()
|
||||||
|
try:
|
||||||
|
task = store.task(task_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "fine tune task not found")
|
||||||
|
if not has_resource_access("fine-tune", task_id, current_user, "read"):
|
||||||
|
raise fail(403, "no permission to access this task")
|
||||||
|
node = _node_for_task(task)
|
||||||
|
selected = set(_normalize_gpu_indices({"gpus": task.get("gpus") or []}, allow_primary=False))
|
||||||
|
if not node:
|
||||||
|
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected)})
|
||||||
|
try:
|
||||||
|
if get_settings().compute_mode == "simulator":
|
||||||
|
live_items = store.gpus()
|
||||||
|
else:
|
||||||
|
live_items = await ComputeNodeClient(node["api_base_url"]).gpu_resources()
|
||||||
|
items = []
|
||||||
|
for item in live_items:
|
||||||
|
index = int(item.get("gpu_index", item.get("id", -1)))
|
||||||
|
if selected and index not in selected:
|
||||||
|
continue
|
||||||
|
items.append({
|
||||||
|
**item,
|
||||||
|
"id": index,
|
||||||
|
"node_id": node["id"],
|
||||||
|
"node_code": node.get("code"),
|
||||||
|
"node_name": node.get("name"),
|
||||||
|
})
|
||||||
|
return ok({"source": "compute", "items": items, "selected_gpus": sorted(selected)})
|
||||||
|
except Exception as exc: # noqa: BLE001 - let UI retain last good snapshot
|
||||||
|
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected), "error": str(exc)})
|
||||||
|
|
||||||
|
|
||||||
@router.put("/fine-tune/{task_id}")
|
@router.put("/fine-tune/{task_id}")
|
||||||
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
|
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
|
||||||
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
@@ -1828,15 +2108,35 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
|
|||||||
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
"""Start an evaluation task: submit eval job to compute node."""
|
"""Start an evaluation task: submit eval job to compute node."""
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
|
try:
|
||||||
|
gpu_indices = _normalize_gpu_indices(payload)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(400, str(exc))
|
||||||
|
if not gpu_indices:
|
||||||
|
raise fail(400, "请选择至少一张 GPU")
|
||||||
# 1. Create eval task record
|
# 1. Create eval task record
|
||||||
payload.setdefault("created_by", current_user.get("id"))
|
payload.setdefault("created_by", current_user.get("id"))
|
||||||
task = store.create_eval_task({**payload, "status": "pending"})
|
task = store.create_eval_task({**payload, "status": "pending"})
|
||||||
|
if get_settings().minio_enabled:
|
||||||
|
_store_json_snapshot(
|
||||||
|
store,
|
||||||
|
"eval",
|
||||||
|
str(task["id"]),
|
||||||
|
str(task["id"]),
|
||||||
|
f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json",
|
||||||
|
payload,
|
||||||
|
current_user.get("id"),
|
||||||
|
)
|
||||||
|
|
||||||
# 2. Resolve model path (supports both regular models and trained models)
|
# 2. Resolve model path (supports both regular models and trained models)
|
||||||
model_id = str(payload.get("model_id", ""))
|
model_id = str(payload.get("model_id", ""))
|
||||||
model_path = ""
|
model_path = ""
|
||||||
adapter_path = payload.get("adapter_path", "")
|
adapter_path = payload.get("adapter_path", "")
|
||||||
model_node_id = ""
|
model_node_id = ""
|
||||||
|
ds_files: list[dict[str, Any]] = []
|
||||||
|
model_resource_type = "model"
|
||||||
|
model_resource_id = model_id
|
||||||
|
adapter_resource_id = ""
|
||||||
try:
|
try:
|
||||||
db_model = store.model(model_id)
|
db_model = store.model(model_id)
|
||||||
model_path = db_model.get("path", "")
|
model_path = db_model.get("path", "")
|
||||||
@@ -1845,6 +2145,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
|
|||||||
# Try trained_models table (IDs prefixed with tm_)
|
# Try trained_models table (IDs prefixed with tm_)
|
||||||
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
|
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
|
||||||
if trained:
|
if trained:
|
||||||
|
model_resource_type = "trained_model" if trained.get("merged") else "model"
|
||||||
|
model_resource_id = trained.get("id") or model_id
|
||||||
|
adapter_resource_id = trained.get("id") or ""
|
||||||
model_node_id = trained.get("compute_node_id") or ""
|
model_node_id = trained.get("compute_node_id") or ""
|
||||||
merged_path = trained.get("merged_path", "")
|
merged_path = trained.get("merged_path", "")
|
||||||
base_path = trained.get("base_model_path", "")
|
base_path = trained.get("base_model_path", "")
|
||||||
@@ -1858,6 +2161,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
|
|||||||
adapter_path = merged_path
|
adapter_path = merged_path
|
||||||
else:
|
else:
|
||||||
model_path = merged_path or base_path
|
model_path = merged_path or base_path
|
||||||
|
if model_resource_type == "model":
|
||||||
|
base_model = next((item for item in store.models() if item.get("path") == base_path), None)
|
||||||
|
model_resource_id = str((base_model or {}).get("id") or base_path)
|
||||||
if not model_path:
|
if not model_path:
|
||||||
store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
|
store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
|
||||||
return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
|
return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
|
||||||
@@ -1930,6 +2236,33 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
|
|||||||
store.update_eval_task(task["id"], {"status": "failed", "error": message})
|
store.update_eval_task(task["id"], {"status": "failed", "error": message})
|
||||||
return ok({"task_id": task["id"], "status": "failed", "error": message})
|
return ok({"task_id": task["id"], "status": "failed", "error": message})
|
||||||
|
|
||||||
|
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
|
||||||
|
try:
|
||||||
|
prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node)
|
||||||
|
if prepared_model:
|
||||||
|
model_path = prepared_model
|
||||||
|
if adapter_resource_id and model_resource_type == "model":
|
||||||
|
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node)
|
||||||
|
if prepared_adapter:
|
||||||
|
adapter_path = prepared_adapter
|
||||||
|
dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id)
|
||||||
|
if dataset_sync:
|
||||||
|
dataset_path = str(dataset_sync[0].get("local_path") or dataset_path)
|
||||||
|
except Exception as exc:
|
||||||
|
store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"})
|
||||||
|
return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})
|
||||||
|
|
||||||
|
node_gpus = {
|
||||||
|
int(item.get("id", item.get("gpu_index", -1))): item
|
||||||
|
for item in store.gpus()
|
||||||
|
if item.get("node_id") == node["id"]
|
||||||
|
}
|
||||||
|
unavailable = [index for index in gpu_indices if node_gpus.get(index, {}).get("status") != "idle"]
|
||||||
|
if unavailable:
|
||||||
|
message = f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}"
|
||||||
|
store.update_eval_task(task["id"], {"status": "failed", "error": message})
|
||||||
|
return ok({"task_id": task["id"], "status": "failed", "error": message})
|
||||||
|
|
||||||
# 6. Build eval job payload
|
# 6. Build eval job payload
|
||||||
output_dir = f"/data/yg-ft/outputs/{task['id']}"
|
output_dir = f"/data/yg-ft/outputs/{task['id']}"
|
||||||
job_payload = {
|
job_payload = {
|
||||||
@@ -1943,7 +2276,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
|
|||||||
"output_dir": output_dir,
|
"output_dir": output_dir,
|
||||||
"basic_metrics": payload.get("basic_metrics", {}),
|
"basic_metrics": payload.get("basic_metrics", {}),
|
||||||
"dimension": dimension_cfg,
|
"dimension": dimension_cfg,
|
||||||
"gpus": [int(payload.get("gpu_id", 0))],
|
"gpu_id": gpu_indices[0],
|
||||||
|
"gpu_indices": gpu_indices,
|
||||||
|
"gpus": gpu_indices,
|
||||||
"temperature": payload.get("temperature", 0.1),
|
"temperature": payload.get("temperature", 0.1),
|
||||||
"max_new_tokens": payload.get("max_new_tokens", 512),
|
"max_new_tokens": payload.get("max_new_tokens", 512),
|
||||||
"compute_node_id": node["id"],
|
"compute_node_id": node["id"],
|
||||||
@@ -1991,7 +2326,10 @@ async def model_eval_delete(task_id: str, current_user: dict = Depends(get_curre
|
|||||||
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
|
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
|
||||||
if pending:
|
if pending:
|
||||||
return pending
|
return pending
|
||||||
|
try:
|
||||||
get_platform_store().delete_eval_task(task_id)
|
get_platform_store().delete_eval_task(task_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "eval task not found")
|
||||||
return ok({"deleted": task_id})
|
return ok({"deleted": task_id})
|
||||||
|
|
||||||
|
|
||||||
@@ -2247,6 +2585,15 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
|
|||||||
"model_name_or_path": model_path,
|
"model_name_or_path": model_path,
|
||||||
"template": item.get("template", "qwen"),
|
"template": item.get("template", "qwen"),
|
||||||
}
|
}
|
||||||
|
try:
|
||||||
|
item_gpu_indices = _normalize_gpu_indices(item)
|
||||||
|
except ValueError as exc:
|
||||||
|
loaded_models.append({**item, "status": "error", "error": str(exc)})
|
||||||
|
continue
|
||||||
|
if not item_gpu_indices:
|
||||||
|
loaded_models.append({**item, "status": "error", "error": "no GPU selected"})
|
||||||
|
continue
|
||||||
|
load_payload["gpu_indices"] = item_gpu_indices
|
||||||
if item.get("adapter_path"):
|
if item.get("adapter_path"):
|
||||||
load_payload["adapter_name_or_path"] = item["adapter_path"]
|
load_payload["adapter_name_or_path"] = item["adapter_path"]
|
||||||
if get_settings().compute_mode == "simulator":
|
if get_settings().compute_mode == "simulator":
|
||||||
@@ -2255,14 +2602,28 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
|
|||||||
# 只派发:HTTP 响应成功即视为已接受(节点会异步加载),loaded 字段忽略
|
# 只派发:HTTP 响应成功即视为已接受(节点会异步加载),loaded 字段忽略
|
||||||
item_dispatched = False
|
item_dispatched = False
|
||||||
errors = []
|
errors = []
|
||||||
for node in _candidate_online_nodes(store, preferred_node_id):
|
candidate_nodes = _candidate_online_nodes(store, preferred_node_id)
|
||||||
|
if preferred_node_id:
|
||||||
|
candidate_nodes = candidate_nodes[:1]
|
||||||
|
for node in candidate_nodes:
|
||||||
try:
|
try:
|
||||||
|
node_gpu_map = {
|
||||||
|
int(gpu.get("id", gpu.get("gpu_index", -1))): gpu
|
||||||
|
for gpu in store.gpus()
|
||||||
|
if gpu.get("node_id") == node["id"]
|
||||||
|
}
|
||||||
|
unavailable = [
|
||||||
|
index for index in item_gpu_indices
|
||||||
|
if node_gpu_map.get(index, {}).get("status") != "idle"
|
||||||
|
]
|
||||||
|
if unavailable:
|
||||||
|
raise RuntimeError(f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}")
|
||||||
if get_settings().minio_enabled:
|
if get_settings().minio_enabled:
|
||||||
await _wait_for_object_storage()
|
await _wait_for_object_storage()
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
await client.inference_load(load_payload)
|
await client.inference_load(load_payload)
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"], item_gpu_indices)
|
||||||
loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
|
loaded_models.append({**item, "gpu_indices": item_gpu_indices, "gpus": item_gpu_indices, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
|
||||||
item_dispatched = True
|
item_dispatched = True
|
||||||
break
|
break
|
||||||
except Exception as exc: # noqa: BLE001 - try next candidate node
|
except Exception as exc: # noqa: BLE001 - try next candidate node
|
||||||
@@ -2365,7 +2726,7 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[
|
|||||||
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
||||||
result = await client.inference_load(payload)
|
result = await client.inference_load(payload)
|
||||||
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
|
||||||
return ok(result)
|
return ok(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return ok({"loaded": False, "error": str(exc)})
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
@@ -2445,7 +2806,7 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), curren
|
|||||||
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
# 计算节点现在异步加载:HTTP 接受(loading/ready)即视为派发成功
|
||||||
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
|
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
|
||||||
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
|
||||||
store.mark_inference_loaded(node["id"])
|
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
|
||||||
return ok(result)
|
return ok(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return ok({"loaded": False, "error": str(exc)})
|
return ok({"loaded": False, "error": str(exc)})
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ def audit_log(
|
|||||||
detail = _build_detail(detail_template, kwargs)
|
detail = _build_detail(detail_template, kwargs)
|
||||||
_record_audit(
|
_record_audit(
|
||||||
action=action,
|
action=action,
|
||||||
|
actor_id=_extract_actor_id(kwargs),
|
||||||
target_type=target_type,
|
target_type=target_type,
|
||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
detail=detail,
|
detail=detail,
|
||||||
@@ -87,6 +88,7 @@ def audit_log(
|
|||||||
detail = _build_detail(detail_template, kwargs)
|
detail = _build_detail(detail_template, kwargs)
|
||||||
_record_audit(
|
_record_audit(
|
||||||
action=action,
|
action=action,
|
||||||
|
actor_id=_extract_actor_id(kwargs),
|
||||||
target_type=target_type,
|
target_type=target_type,
|
||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
detail=detail,
|
detail=detail,
|
||||||
@@ -136,6 +138,7 @@ def _build_detail(template: str, kwargs: dict) -> str:
|
|||||||
|
|
||||||
def _record_audit(
|
def _record_audit(
|
||||||
action: str,
|
action: str,
|
||||||
|
actor_id: Optional[str],
|
||||||
target_type: str,
|
target_type: str,
|
||||||
target_id: Optional[str],
|
target_id: Optional[str],
|
||||||
detail: str,
|
detail: str,
|
||||||
@@ -149,6 +152,7 @@ def _record_audit(
|
|||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
store.record_audit(
|
store.record_audit(
|
||||||
action=action,
|
action=action,
|
||||||
|
actor_id=actor_id,
|
||||||
target_type=target_type or None,
|
target_type=target_type or None,
|
||||||
target_id=target_id,
|
target_id=target_id,
|
||||||
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
|
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
|
||||||
@@ -157,6 +161,15 @@ def _record_audit(
|
|||||||
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
|
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_actor_id(kwargs: dict) -> Optional[str]:
|
||||||
|
"""从 FastAPI 注入的当前用户中提取操作人 ID。"""
|
||||||
|
for key in ("current_user", "user"):
|
||||||
|
value = kwargs.get(key)
|
||||||
|
if isinstance(value, dict) and value.get("id"):
|
||||||
|
return str(value["id"])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# ==================== 预定义的审计操作常量 ====================
|
# ==================== 预定义的审计操作常量 ====================
|
||||||
|
|
||||||
class AuditActions:
|
class AuditActions:
|
||||||
|
|||||||
@@ -68,6 +68,9 @@ class Settings:
|
|||||||
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
|
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
|
||||||
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
|
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
|
||||||
minio_secure: bool = _bool_env("MINIO_SECURE", False)
|
minio_secure: bool = _bool_env("MINIO_SECURE", False)
|
||||||
|
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
|
||||||
|
# MinIO round trips. Larger files remain the shared canonical objects.
|
||||||
|
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
|
||||||
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
|
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
|
||||||
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
|
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
|
||||||
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")
|
||||||
|
|||||||
@@ -454,15 +454,21 @@ class PlatformStore:
|
|||||||
self.ensure_seed_data()
|
self.ensure_seed_data()
|
||||||
# Track which compute nodes have an active inference model loaded
|
# Track which compute nodes have an active inference model loaded
|
||||||
self._inference_nodes: set[str] = set()
|
self._inference_nodes: set[str] = set()
|
||||||
|
self._inference_gpu_indexes: dict[str, set[int]] = {}
|
||||||
self._last_runtime_refresh = 0.0
|
self._last_runtime_refresh = 0.0
|
||||||
|
|
||||||
# ── inference node tracking ────────────────────────────────────
|
# ── inference node tracking ────────────────────────────────────
|
||||||
|
|
||||||
def mark_inference_loaded(self, node_id: str) -> None:
|
def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None:
|
||||||
self._inference_nodes.add(node_id)
|
self._inference_nodes.add(node_id)
|
||||||
|
if gpu_indexes is not None:
|
||||||
|
self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes}
|
||||||
|
else:
|
||||||
|
self._inference_gpu_indexes.pop(node_id, None)
|
||||||
|
|
||||||
def mark_inference_unloaded(self, node_id: str) -> None:
|
def mark_inference_unloaded(self, node_id: str) -> None:
|
||||||
self._inference_nodes.discard(node_id)
|
self._inference_nodes.discard(node_id)
|
||||||
|
self._inference_gpu_indexes.pop(node_id, None)
|
||||||
|
|
||||||
def is_inference_loaded(self, node_id: str) -> bool:
|
def is_inference_loaded(self, node_id: str) -> bool:
|
||||||
return node_id in self._inference_nodes
|
return node_id in self._inference_nodes
|
||||||
@@ -545,6 +551,16 @@ class PlatformStore:
|
|||||||
"last_error": "TEXT",
|
"last_error": "TEXT",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
self._ensure_columns(
|
||||||
|
conn,
|
||||||
|
"storage_objects",
|
||||||
|
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
|
||||||
|
)
|
||||||
|
self._ensure_columns(
|
||||||
|
conn,
|
||||||
|
"model_artifacts",
|
||||||
|
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
|
||||||
|
)
|
||||||
schema_dir = Path(__file__).with_name("sql")
|
schema_dir = Path(__file__).with_name("sql")
|
||||||
for extra in (
|
for extra in (
|
||||||
"002_governance.sql",
|
"002_governance.sql",
|
||||||
@@ -556,7 +572,17 @@ class PlatformStore:
|
|||||||
if extra_path.exists():
|
if extra_path.exists():
|
||||||
conn.executescript(extra_path.read_text(encoding="utf-8"))
|
conn.executescript(extra_path.read_text(encoding="utf-8"))
|
||||||
# data_convert_tasks 表补充 created_by 字段(用于数据隔离)
|
# data_convert_tasks 表补充 created_by 字段(用于数据隔离)
|
||||||
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
|
self._ensure_columns(
|
||||||
|
conn,
|
||||||
|
"data_convert_tasks",
|
||||||
|
{
|
||||||
|
"created_by": "TEXT",
|
||||||
|
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
|
||||||
|
"output_storage_object_id": "TEXT",
|
||||||
|
"output_content": "TEXT",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
|
||||||
# 修复历史数据:将 data_convert_tasks.created_by 回填到关联的 datasets 记录
|
# 修复历史数据:将 data_convert_tasks.created_by 回填到关联的 datasets 记录
|
||||||
try:
|
try:
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
@@ -1254,6 +1280,13 @@ class PlatformStore:
|
|||||||
raise KeyError(artifact_id)
|
raise KeyError(artifact_id)
|
||||||
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
|
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
|
||||||
|
|
||||||
|
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
|
||||||
|
(storage_object_id, artifact_id),
|
||||||
|
)
|
||||||
|
|
||||||
def model_lineage(self, model_id: str) -> dict[str, Any]:
|
def model_lineage(self, model_id: str) -> dict[str, Any]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
parents = conn.execute(
|
parents = conn.execute(
|
||||||
@@ -2221,7 +2254,7 @@ class PlatformStore:
|
|||||||
(str(validation_dataset_id),),
|
(str(validation_dataset_id),),
|
||||||
).fetchall(),
|
).fetchall(),
|
||||||
]
|
]
|
||||||
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
|
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
|
||||||
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
|
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
|
||||||
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
|
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
|
||||||
"dataset_split"
|
"dataset_split"
|
||||||
@@ -2482,12 +2515,18 @@ class PlatformStore:
|
|||||||
|
|
||||||
def eval_tasks(self) -> list[dict[str, Any]]:
|
def eval_tasks(self) -> list[dict[str, Any]]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
|
# 评测任务采用软删除,普通列表不得再次返回已删除记录。
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC"
|
||||||
|
).fetchall()
|
||||||
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
|
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
|
||||||
|
|
||||||
def eval_task(self, task_id: str) -> dict[str, Any]:
|
def eval_task(self, task_id: str) -> dict[str, Any]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
|
row = conn.execute(
|
||||||
|
"SELECT * FROM eval_tasks WHERE id=? AND deleted_at IS NULL",
|
||||||
|
(task_id,),
|
||||||
|
).fetchone()
|
||||||
if not row:
|
if not row:
|
||||||
raise KeyError(task_id)
|
raise KeyError(task_id)
|
||||||
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
|
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
|
||||||
@@ -2558,7 +2597,13 @@ class PlatformStore:
|
|||||||
|
|
||||||
def delete_eval_task(self, task_id: str) -> None:
|
def delete_eval_task(self, task_id: str) -> None:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
|
result = conn.execute(
|
||||||
|
"UPDATE eval_tasks SET deleted_at=?, deleted_by=? "
|
||||||
|
"WHERE id=? AND deleted_at IS NULL RETURNING id",
|
||||||
|
(utcnow(), "system", task_id),
|
||||||
|
)
|
||||||
|
if not result.fetchone():
|
||||||
|
raise KeyError(task_id)
|
||||||
|
|
||||||
def running_eval_tasks(self) -> list[dict[str, Any]]:
|
def running_eval_tasks(self) -> list[dict[str, Any]]:
|
||||||
"""Return eval tasks that have been submitted to a compute node and are still running."""
|
"""Return eval tasks that have been submitted to a compute node and are still running."""
|
||||||
@@ -2766,7 +2811,35 @@ class PlatformStore:
|
|||||||
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
|
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
|
||||||
(node_id,),
|
(node_id,),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return {int(row["gpu_index"]) for row in rows}
|
active = {int(row["gpu_index"]) for row in rows}
|
||||||
|
# Evaluation jobs use the same Compute ProcessManager GPU lock but do
|
||||||
|
# not have fine-tune allocation rows; derive their selected cards here
|
||||||
|
# so a training task cannot race onto an evaluation GPU.
|
||||||
|
for row in conn.execute(
|
||||||
|
"SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')"
|
||||||
|
).fetchall():
|
||||||
|
payload = json_loads(row["payload"], {})
|
||||||
|
if payload.get("compute_node_id") != node_id:
|
||||||
|
continue
|
||||||
|
selected = payload.get("gpu_indices") or payload.get("gpus")
|
||||||
|
if selected is None and payload.get("gpu_id") is not None:
|
||||||
|
selected = [payload.get("gpu_id")]
|
||||||
|
active.update(int(item) for item in selected or [])
|
||||||
|
# Loaded inference models also reserve only their selected cards.
|
||||||
|
active.update(self._inference_gpu_indexes.get(node_id, set()))
|
||||||
|
for row in conn.execute("SELECT payload FROM compare_tasks").fetchall():
|
||||||
|
payload = json_loads(row["payload"], {})
|
||||||
|
load_status = payload.get("load_status") or {}
|
||||||
|
if isinstance(load_status, str):
|
||||||
|
load_status = json_loads(load_status, {})
|
||||||
|
for item in load_status.get("loaded_models") or []:
|
||||||
|
if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}:
|
||||||
|
continue
|
||||||
|
selected = item.get("gpu_indices") or item.get("gpus")
|
||||||
|
if selected is None and item.get("gpu_id") is not None:
|
||||||
|
selected = [item.get("gpu_id")]
|
||||||
|
active.update(int(gpu) for gpu in selected or [])
|
||||||
|
return active
|
||||||
|
|
||||||
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
|
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
|
||||||
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
|
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
|
||||||
@@ -3000,17 +3073,18 @@ class PlatformStore:
|
|||||||
"""
|
"""
|
||||||
INSERT INTO storage_objects
|
INSERT INTO storage_objects
|
||||||
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
|
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
|
||||||
content_type, checksum_sha256, byte_size, status, created_by, create_time)
|
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT (resource_type, resource_id, version_id, object_key)
|
ON CONFLICT (resource_type, resource_id, version_id, object_key)
|
||||||
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
|
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
|
||||||
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
|
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
|
||||||
status=EXCLUDED.status, created_by=EXCLUDED.created_by
|
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
|
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
|
||||||
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
|
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
|
||||||
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
|
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
|
||||||
|
json_dumps(payload.get("metadata") or {}),
|
||||||
payload.get("created_by"), payload.get("create_time") or utcnow(),
|
payload.get("created_by"), payload.get("create_time") or utcnow(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -3049,6 +3123,13 @@ class PlatformStore:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
def storage_object(self, object_id: str) -> dict[str, Any]:
|
||||||
|
with self.connect() as conn:
|
||||||
|
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
raise KeyError(object_id)
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
|
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
|
||||||
fields = {key: value for key, value in payload.items() if key in allowed}
|
fields = {key: value for key, value in payload.items() if key in allowed}
|
||||||
@@ -3195,6 +3276,9 @@ class PlatformStore:
|
|||||||
# 推理模型占用算力节点同样计入:优先从 compare_tasks 持久化状态派生
|
# 推理模型占用算力节点同样计入:优先从 compare_tasks 持久化状态派生
|
||||||
# (重启后仍准确),并用内存标记兜底(直接 preload 的模型无 compare 记录)
|
# (重启后仍准确),并用内存标记兜底(直接 preload 的模型无 compare 记录)
|
||||||
inference_node_ids = set(self._inference_nodes)
|
inference_node_ids = set(self._inference_nodes)
|
||||||
|
inference_gpu_indexes: dict[str, set[int]] = {
|
||||||
|
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
|
||||||
|
}
|
||||||
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
|
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
|
||||||
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
|
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
|
||||||
if isinstance(ls, str):
|
if isinstance(ls, str):
|
||||||
@@ -3204,8 +3288,13 @@ class PlatformStore:
|
|||||||
ls = {}
|
ls = {}
|
||||||
for m in ls.get("loaded_models") or []:
|
for m in ls.get("loaded_models") or []:
|
||||||
if m.get("status") in {"ready", "running"} and m.get("node_id"):
|
if m.get("status") in {"ready", "running"} and m.get("node_id"):
|
||||||
inference_node_ids.add(m["node_id"])
|
node_id = m["node_id"]
|
||||||
for nid in inference_node_ids:
|
selected = m.get("gpu_indices") or m.get("gpus")
|
||||||
|
if selected:
|
||||||
|
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
|
||||||
|
else:
|
||||||
|
inference_node_ids.add(node_id)
|
||||||
|
for nid in set(inference_node_ids) | set(inference_gpu_indexes):
|
||||||
running_map[nid] = running_map.get(nid, 0) + 1
|
running_map[nid] = running_map.get(nid, 0) + 1
|
||||||
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
|
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
|
||||||
return [
|
return [
|
||||||
@@ -3434,6 +3523,9 @@ class PlatformStore:
|
|||||||
# 推理模型占用的节点:优先从 compare_tasks 持久化状态派生(重启后仍准确),
|
# 推理模型占用的节点:优先从 compare_tasks 持久化状态派生(重启后仍准确),
|
||||||
# 内存标记兜底(直接 preload 的模型无 compare 记录)
|
# 内存标记兜底(直接 preload 的模型无 compare 记录)
|
||||||
inference_node_ids = set(self._inference_nodes)
|
inference_node_ids = set(self._inference_nodes)
|
||||||
|
inference_gpu_indexes: dict[str, set[int]] = {
|
||||||
|
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
|
||||||
|
}
|
||||||
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
|
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
|
||||||
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
|
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
|
||||||
if isinstance(ls, str):
|
if isinstance(ls, str):
|
||||||
@@ -3443,7 +3535,12 @@ class PlatformStore:
|
|||||||
ls = {}
|
ls = {}
|
||||||
for m in ls.get("loaded_models") or []:
|
for m in ls.get("loaded_models") or []:
|
||||||
if m.get("status") in {"ready", "running"} and m.get("node_id"):
|
if m.get("status") in {"ready", "running"} and m.get("node_id"):
|
||||||
inference_node_ids.add(m["node_id"])
|
node_id = m["node_id"]
|
||||||
|
selected = m.get("gpu_indices") or m.get("gpus")
|
||||||
|
if selected:
|
||||||
|
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
|
||||||
|
else:
|
||||||
|
inference_node_ids.add(node_id)
|
||||||
items = []
|
items = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
task = next(
|
task = next(
|
||||||
@@ -3459,7 +3556,10 @@ class PlatformStore:
|
|||||||
t
|
t
|
||||||
for t in eval_running
|
for t in eval_running
|
||||||
if t.get("compute_node_id") == row["node_id"]
|
if t.get("compute_node_id") == row["node_id"]
|
||||||
and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1)
|
and row["gpu_index"] in {
|
||||||
|
int(item)
|
||||||
|
for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else []))
|
||||||
|
}
|
||||||
),
|
),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
@@ -3468,7 +3568,8 @@ class PlatformStore:
|
|||||||
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
|
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
|
||||||
)
|
)
|
||||||
# Also mark GPU as busy if an inference model is loaded on this node
|
# Also mark GPU as busy if an inference model is loaded on this node
|
||||||
if row["node_id"] in inference_node_ids and not busy:
|
inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set())
|
||||||
|
if inference_on_gpu and not busy:
|
||||||
busy = True
|
busy = True
|
||||||
reserved = False
|
reserved = False
|
||||||
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
|
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
|
||||||
@@ -3704,6 +3805,8 @@ class PlatformStore:
|
|||||||
actor_id: str | None = None,
|
actor_id: str | None = None,
|
||||||
action: str | None = None,
|
action: str | None = None,
|
||||||
target_type: str | None = None,
|
target_type: str | None = None,
|
||||||
|
target_id: str | None = None,
|
||||||
|
keyword: str | None = None,
|
||||||
start_time: str | None = None,
|
start_time: str | None = None,
|
||||||
end_time: str | None = None,
|
end_time: str | None = None,
|
||||||
limit: int = 50,
|
limit: int = 50,
|
||||||
@@ -3726,6 +3829,13 @@ class PlatformStore:
|
|||||||
if target_type:
|
if target_type:
|
||||||
clauses.append("target_type=?")
|
clauses.append("target_type=?")
|
||||||
params.append(target_type)
|
params.append(target_type)
|
||||||
|
if target_id:
|
||||||
|
clauses.append("target_id=?")
|
||||||
|
params.append(target_id)
|
||||||
|
if keyword:
|
||||||
|
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
|
||||||
|
pattern = f"%{keyword}%"
|
||||||
|
params.extend([pattern, pattern])
|
||||||
if start_time:
|
if start_time:
|
||||||
clauses.append("time>=?")
|
clauses.append("time>=?")
|
||||||
params.append(start_time)
|
params.append(start_time)
|
||||||
|
|||||||
@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
|
|||||||
path TEXT NOT NULL,
|
path TEXT NOT NULL,
|
||||||
size_bytes BIGINT NOT NULL DEFAULT 0,
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
||||||
checksum_sha256 TEXT,
|
checksum_sha256 TEXT,
|
||||||
|
storage_object_id TEXT,
|
||||||
|
storage_backend TEXT NOT NULL DEFAULT 'minio',
|
||||||
metadata TEXT NOT NULL,
|
metadata TEXT NOT NULL,
|
||||||
compute_job_id TEXT,
|
compute_job_id TEXT,
|
||||||
create_time TEXT NOT NULL
|
create_time TEXT NOT NULL
|
||||||
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
|
|||||||
checksum_sha256 TEXT,
|
checksum_sha256 TEXT,
|
||||||
byte_size BIGINT NOT NULL DEFAULT 0,
|
byte_size BIGINT NOT NULL DEFAULT 0,
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
metadata TEXT NOT NULL DEFAULT '{}',
|
||||||
created_by TEXT,
|
created_by TEXT,
|
||||||
create_time TEXT NOT NULL,
|
create_time TEXT NOT NULL,
|
||||||
UNIQUE (resource_type, resource_id, version_id, object_key)
|
UNIQUE (resource_type, resource_id, version_id, object_key)
|
||||||
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
|
|||||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||||
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||||
|
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
|
||||||
|
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
||||||
|
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
|
||||||
|
|
||||||
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
|
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
|
||||||
|
|
||||||
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
|
|||||||
input_count INTEGER NOT NULL DEFAULT 0,
|
input_count INTEGER NOT NULL DEFAULT 0,
|
||||||
output_count INTEGER NOT NULL DEFAULT 0,
|
output_count INTEGER NOT NULL DEFAULT 0,
|
||||||
error_message TEXT,
|
error_message TEXT,
|
||||||
|
output_content TEXT,
|
||||||
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||||
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||||
deleted_at TIMESTAMPTZ
|
deleted_at TIMESTAMPTZ
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
|
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
|
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
|
||||||
|
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
|
||||||
|
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
|
||||||
|
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- 七、种子数据:初始管理员 / 操作员
|
-- 七、种子数据:初始管理员 / 操作员
|
||||||
|
|||||||
@@ -240,6 +240,10 @@ class ComputeNodeClient:
|
|||||||
async def inference_status(self) -> dict[str, Any]:
|
async def inference_status(self) -> dict[str, Any]:
|
||||||
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
|
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
|
||||||
|
|
||||||
|
async def gpu_resources(self) -> list[dict[str, Any]]:
|
||||||
|
"""Read live per-GPU metrics from this compute node."""
|
||||||
|
return await self.gpus()
|
||||||
|
|
||||||
async def inference_unload(self) -> dict[str, Any]:
|
async def inference_unload(self) -> dict[str, Any]:
|
||||||
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
|
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)
|
||||||
|
|
||||||
|
|||||||
@@ -2,15 +2,75 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.db.platform_store import get_platform_store
|
from app.db.platform_store import get_platform_store
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.modules.compute_gateway.client import ComputeNodeClient
|
from app.modules.compute_gateway.client import ComputeNodeClient
|
||||||
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
|
||||||
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
# starting 状态允许的最大轮询次数(约 40 * 3s ≈ 2 分钟),超过即判定节点不可达
|
||||||
MAX_STARTING_ATTEMPTS = 40
|
MAX_STARTING_ATTEMPTS = 40
|
||||||
|
|
||||||
|
|
||||||
|
async def _archive_node_directory(
|
||||||
|
store: Any,
|
||||||
|
client: ComputeNodeClient,
|
||||||
|
node: dict[str, Any],
|
||||||
|
source_path: str,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
version_id: str,
|
||||||
|
object_prefix: str,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Archive a completed node directory to MinIO, preserving subdirectories."""
|
||||||
|
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
|
||||||
|
source = Path(source_path).resolve()
|
||||||
|
try:
|
||||||
|
relative_root = source.relative_to(data_root).as_posix()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
|
||||||
|
queue = [relative_root]
|
||||||
|
archived: list[dict[str, Any]] = []
|
||||||
|
while queue:
|
||||||
|
relative = queue.pop(0)
|
||||||
|
listing = await client.list_files(root="data", relative_path=relative)
|
||||||
|
for item in listing.get("items") or []:
|
||||||
|
item_relative = str(item.get("relative_path") or "")
|
||||||
|
if item.get("type") == "directory":
|
||||||
|
queue.append(item_relative)
|
||||||
|
continue
|
||||||
|
path = str(item.get("path") or "")
|
||||||
|
if not path:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
|
||||||
|
except ValueError:
|
||||||
|
relative_file = Path(str(item.get("name") or Path(path).name)).name
|
||||||
|
object_key = f"{object_prefix}/{version_id}/{relative_file}"
|
||||||
|
upload_url = get_object_storage().presigned_put(object_key)
|
||||||
|
result = await client.upload_file_to_url(path, upload_url, object_key)
|
||||||
|
metadata = get_object_storage().stat(object_key)
|
||||||
|
archived.append(
|
||||||
|
store.create_storage_object(
|
||||||
|
{
|
||||||
|
"resource_type": resource_type,
|
||||||
|
"resource_id": resource_id,
|
||||||
|
"version_id": version_id,
|
||||||
|
"bucket": get_object_storage().bucket,
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": relative_file,
|
||||||
|
"content_type": "application/octet-stream",
|
||||||
|
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
|
||||||
|
"checksum_sha256": result.get("checksum_sha256") or "",
|
||||||
|
"status": "available",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return archived
|
||||||
|
|
||||||
|
|
||||||
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
|
||||||
|
|
||||||
@@ -73,7 +133,8 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
|
|||||||
if node_status == "ready":
|
if node_status == "ready":
|
||||||
item["status"] = "ready"
|
item["status"] = "ready"
|
||||||
item.pop("error", None)
|
item.pop("error", None)
|
||||||
store.mark_inference_loaded(node["id"])
|
selected_gpus = item.get("gpu_indices") or item.get("gpus")
|
||||||
|
store.mark_inference_loaded(node["id"], selected_gpus)
|
||||||
elif node_status == "error":
|
elif node_status == "error":
|
||||||
item["status"] = "error"
|
item["status"] = "error"
|
||||||
item["error"] = status.get("error") or "model load failed on compute node"
|
item["error"] = status.get("error") or "model load failed on compute node"
|
||||||
@@ -138,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
synced.append(store.apply_compute_job(task["id"], job))
|
updated_task = store.apply_compute_job(task["id"], job)
|
||||||
|
if (
|
||||||
|
get_settings().minio_enabled
|
||||||
|
and job.get("status") == "completed"
|
||||||
|
and job.get("output_dir")
|
||||||
|
):
|
||||||
|
trained_model = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in store.trained_models()
|
||||||
|
if item.get("name")
|
||||||
|
== (task.get("output_model_name") or f"{task.get('name')}-lora")
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if trained_model:
|
||||||
|
archived = await _archive_node_directory(
|
||||||
|
store,
|
||||||
|
client,
|
||||||
|
node,
|
||||||
|
str(job["output_dir"]),
|
||||||
|
"trained_model",
|
||||||
|
str(trained_model["id"]),
|
||||||
|
str(job.get("id") or task.get("compute_job_id") or task["id"]),
|
||||||
|
f"trained_models/{trained_model['id']}",
|
||||||
|
)
|
||||||
|
artifacts = store.model_artifacts(str(trained_model["id"]))
|
||||||
|
if archived and artifacts:
|
||||||
|
store.link_model_artifact_storage_object(
|
||||||
|
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||||
|
)
|
||||||
|
synced.append(updated_task)
|
||||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
failed.append({"task_id": task["id"], "error": str(exc)})
|
failed.append({"task_id": task["id"], "error": str(exc)})
|
||||||
standalone_synced: list[dict[str, Any]] = []
|
standalone_synced: list[dict[str, Any]] = []
|
||||||
@@ -150,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
try:
|
try:
|
||||||
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
|
||||||
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
|
||||||
|
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
||||||
|
payload = (store.compute_job(record["id"]).get("payload") or {})
|
||||||
|
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
|
||||||
|
if trained_model_id:
|
||||||
|
trained_model = next(
|
||||||
|
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if trained_model:
|
||||||
|
archived = await _archive_node_directory(
|
||||||
|
store,
|
||||||
|
ComputeNodeClient(node["api_base_url"], timeout=900),
|
||||||
|
node,
|
||||||
|
str(job["output_dir"]),
|
||||||
|
"trained_model",
|
||||||
|
str(trained_model["id"]),
|
||||||
|
str(job.get("id") or record["id"]),
|
||||||
|
f"trained_models/{trained_model['id']}",
|
||||||
|
)
|
||||||
|
artifacts = store.model_artifacts(str(trained_model["id"]))
|
||||||
|
if archived and artifacts:
|
||||||
|
store.link_model_artifact_storage_object(
|
||||||
|
str(artifacts[0]["id"]), str(archived[0]["id"])
|
||||||
|
)
|
||||||
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
except Exception as exc: # noqa: BLE001 - keep polling other jobs
|
||||||
failed.append({"job_id": record["id"], "error": str(exc)})
|
failed.append({"job_id": record["id"], "error": str(exc)})
|
||||||
|
|
||||||
@@ -174,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
store.apply_eval_job_result(eval_task["id"], job, result_content)
|
||||||
|
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
|
||||||
|
await _archive_node_directory(
|
||||||
|
store,
|
||||||
|
client,
|
||||||
|
node,
|
||||||
|
str(job["output_dir"]),
|
||||||
|
"eval",
|
||||||
|
str(eval_task["id"]),
|
||||||
|
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
|
||||||
|
f"evaluations/{eval_task['id']}",
|
||||||
|
)
|
||||||
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
# 评测 GPU 占用由 eval_tasks 状态派生,无需维护推理内存标记
|
||||||
eval_synced += 1
|
eval_synced += 1
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc: # noqa: BLE001
|
||||||
|
|||||||
@@ -1,17 +1,21 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import hashlib
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Body, Depends, File, UploadFile
|
from fastapi import APIRouter, Body, Depends, File, UploadFile
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse, Response
|
||||||
|
|
||||||
from app.api.v1.endpoints.platform import ok, fail
|
from app.api.v1.endpoints.platform import ok, fail
|
||||||
from app.core.auth import get_current_user, is_admin
|
from app.core.auth import get_current_user, is_admin
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.core.op_log import op_log, OpModule, OpAction
|
from app.core.op_log import op_log, OpModule, OpAction
|
||||||
from app.db.platform_store import get_platform_store, new_id
|
from app.db.platform_store import get_platform_store, new_id
|
||||||
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
from app.modules.storage.policy import should_store_in_minio
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
|
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
|
||||||
@@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path:
|
|||||||
return _task_dir(task_id) / "output"
|
return _task_dir(task_id) / "output"
|
||||||
|
|
||||||
|
|
||||||
|
def _minio_enabled() -> bool:
|
||||||
|
return bool(get_settings().minio_enabled)
|
||||||
|
|
||||||
|
|
||||||
|
def _input_object_key(task_id: str, name: str) -> str:
|
||||||
|
return f"data-convert/{task_id}/input/{Path(name).name}"
|
||||||
|
|
||||||
|
|
||||||
|
def _output_object_key(task_id: str, name: str) -> str:
|
||||||
|
return f"data-convert/{task_id}/output/{Path(name).name}"
|
||||||
|
|
||||||
|
|
||||||
|
def _task_objects(task_id: str) -> list[dict[str, Any]]:
|
||||||
|
return get_platform_store().storage_objects_for_resource("data_convert", task_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _register_object(
|
||||||
|
task_id: str,
|
||||||
|
*,
|
||||||
|
version_id: str,
|
||||||
|
object_key: str,
|
||||||
|
file_name: str,
|
||||||
|
content_type: str,
|
||||||
|
content: bytes,
|
||||||
|
created_by: str | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
storage = get_object_storage()
|
||||||
|
uploaded = storage.put_bytes(object_key, content, content_type)
|
||||||
|
return get_platform_store().create_storage_object(
|
||||||
|
{
|
||||||
|
"resource_type": "data_convert",
|
||||||
|
"resource_id": task_id,
|
||||||
|
"version_id": version_id,
|
||||||
|
"bucket": uploaded["bucket"],
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": file_name,
|
||||||
|
"content_type": content_type,
|
||||||
|
"byte_size": len(content),
|
||||||
|
"checksum_sha256": hashlib.sha256(content).hexdigest(),
|
||||||
|
"status": "available",
|
||||||
|
"created_by": created_by,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _input_objects(task_id: str) -> list[dict[str, Any]]:
|
||||||
|
prefix = f"data-convert/{task_id}/input/"
|
||||||
|
return sorted(
|
||||||
|
[item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)],
|
||||||
|
key=lambda item: str(item.get("file_name") or item.get("object_key") or ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _output_object(task: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename")))
|
||||||
|
return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_output(task: dict[str, Any]) -> bytes | None:
|
||||||
|
if _minio_enabled():
|
||||||
|
item = _output_object(task)
|
||||||
|
if item:
|
||||||
|
return get_object_storage().get_bytes(item["object_key"])
|
||||||
|
inline = task.get("output_content")
|
||||||
|
return str(inline).encode("utf-8") if inline is not None else None
|
||||||
|
path = _task_output_path(task)
|
||||||
|
return path.read_bytes() if path.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]:
|
||||||
|
output_name = _safe_output_filename(task.get("output_filename"))
|
||||||
|
output_lines: list[str] = []
|
||||||
|
input_count = 0
|
||||||
|
output_count = 0
|
||||||
|
for item in _input_objects(task["id"]):
|
||||||
|
input_count += 1
|
||||||
|
raw = get_object_storage().get_bytes(item["object_key"])
|
||||||
|
data = json.loads(raw.decode("utf-8"))
|
||||||
|
if isinstance(data, list):
|
||||||
|
records = data
|
||||||
|
elif isinstance(data, dict):
|
||||||
|
records = [data]
|
||||||
|
else:
|
||||||
|
raise ValueError(f"JSON must be object or array: {item.get('file_name')}")
|
||||||
|
for record in records:
|
||||||
|
output_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
|
||||||
|
output_count += 1
|
||||||
|
output = "".join(output_lines).encode("utf-8")
|
||||||
|
store = get_platform_store()
|
||||||
|
with store.connect() as conn:
|
||||||
|
if should_store_in_minio(len(output)):
|
||||||
|
output_object = _register_object(
|
||||||
|
task["id"],
|
||||||
|
version_id="output",
|
||||||
|
object_key=_output_object_key(task["id"], output_name),
|
||||||
|
file_name=output_name,
|
||||||
|
content_type="application/jsonl",
|
||||||
|
content=output,
|
||||||
|
created_by=created_by,
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s",
|
||||||
|
(output_object["id"], task["id"]),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s",
|
||||||
|
(output.decode("utf-8"), task["id"]),
|
||||||
|
)
|
||||||
|
return input_count, output_count, output
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]:
|
||||||
|
input_dir = _input_dir(task["id"])
|
||||||
|
output_dir = _output_dir(task["id"])
|
||||||
|
input_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
output_path = _task_output_path(task)
|
||||||
|
output_path.unlink(missing_ok=True)
|
||||||
|
input_count = 0
|
||||||
|
output_count = 0
|
||||||
|
with output_path.open("w", encoding="utf-8") as output_file:
|
||||||
|
for json_file in sorted(input_dir.iterdir()):
|
||||||
|
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
|
||||||
|
continue
|
||||||
|
input_count += 1
|
||||||
|
data = json.loads(json_file.read_text(encoding="utf-8"))
|
||||||
|
records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None
|
||||||
|
if records is None:
|
||||||
|
raise ValueError(f"JSON must be object or array: {json_file.name}")
|
||||||
|
for record in records:
|
||||||
|
output_file.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||||
|
output_count += 1
|
||||||
|
return input_count, output_count, output_path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_tasks(
|
def list_tasks(
|
||||||
page: int = 1,
|
page: int = 1,
|
||||||
@@ -109,7 +249,8 @@ def create_task(
|
|||||||
"VALUES (%s, %s, %s, %s, %s)",
|
"VALUES (%s, %s, %s, %s, %s)",
|
||||||
(task_id, name, description, output_filename, user_id),
|
(task_id, name, description, output_filename, user_id),
|
||||||
)
|
)
|
||||||
# 创建目录
|
# MinIO 是正式存储;本地目录只在关闭 MinIO 的旧兼容模式下创建。
|
||||||
|
if not _minio_enabled():
|
||||||
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
|
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
|
||||||
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
|
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
|
||||||
return ok(_get_task(task_id))
|
return ok(_get_task(task_id))
|
||||||
@@ -123,9 +264,15 @@ def get_task(
|
|||||||
task = _get_task(task_id)
|
task = _get_task(task_id)
|
||||||
if not task:
|
if not task:
|
||||||
raise fail(404, "task not found")
|
raise fail(404, "task not found")
|
||||||
# 附加输入文件列表
|
# 附加输入文件列表;旧任务没有对象记录时继续读取本地兼容目录。
|
||||||
input_dir = _input_dir(task_id)
|
|
||||||
files = []
|
files = []
|
||||||
|
if _minio_enabled():
|
||||||
|
files = [
|
||||||
|
{"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0}
|
||||||
|
for item in _input_objects(task_id)
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
input_dir = _input_dir(task_id)
|
||||||
if input_dir.exists():
|
if input_dir.exists():
|
||||||
for f in sorted(input_dir.iterdir()):
|
for f in sorted(input_dir.iterdir()):
|
||||||
if f.is_file():
|
if f.is_file():
|
||||||
@@ -146,16 +293,26 @@ async def upload_source_files(
|
|||||||
raise fail(404, "task not found")
|
raise fail(404, "task not found")
|
||||||
if task["status"] not in ("pending", "uploaded"):
|
if task["status"] not in ("pending", "uploaded"):
|
||||||
raise fail(400, "task is not editable")
|
raise fail(400, "task is not editable")
|
||||||
input_dir = _input_dir(task_id)
|
|
||||||
input_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
staged = []
|
staged = []
|
||||||
for upload in files:
|
for upload in files:
|
||||||
name = Path(upload.filename or "input.json").name
|
name = Path(upload.filename or "input.json").name
|
||||||
if not name.lower().endswith(".json"):
|
if not name.lower().endswith(".json"):
|
||||||
raise fail(415, f"only JSON files are supported: {name}")
|
raise fail(415, f"only JSON files are supported: {name}")
|
||||||
target = input_dir / name
|
|
||||||
content = await upload.read()
|
content = await upload.read()
|
||||||
target.write_bytes(content)
|
if _minio_enabled():
|
||||||
|
_register_object(
|
||||||
|
task_id,
|
||||||
|
version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}",
|
||||||
|
object_key=_input_object_key(task_id, name),
|
||||||
|
file_name=name,
|
||||||
|
content_type=upload.content_type or "application/json",
|
||||||
|
content=content,
|
||||||
|
created_by=task.get("created_by") or current_user.get("id"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
input_dir = _input_dir(task_id)
|
||||||
|
input_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(input_dir / name).write_bytes(content)
|
||||||
staged.append({"name": name, "size": len(content)})
|
staged.append({"name": name, "size": len(content)})
|
||||||
store = get_platform_store()
|
store = get_platform_store()
|
||||||
# 标记上传完成
|
# 标记上传完成
|
||||||
@@ -166,30 +323,12 @@ async def upload_source_files(
|
|||||||
)
|
)
|
||||||
# 自动转换并导入数据集
|
# 自动转换并导入数据集
|
||||||
try:
|
try:
|
||||||
output_dir = _output_dir(task_id)
|
if _minio_enabled():
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
input_count, output_count, output = _convert_from_minio(
|
||||||
output_path = _task_output_path(task)
|
task, task.get("created_by") or current_user.get("id")
|
||||||
# 清空旧输出(如果重新上传)
|
)
|
||||||
if output_path.exists():
|
|
||||||
output_path.unlink()
|
|
||||||
input_count = 0
|
|
||||||
output_count = 0
|
|
||||||
for json_file in sorted(input_dir.iterdir()):
|
|
||||||
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
|
|
||||||
continue
|
|
||||||
input_count += 1
|
|
||||||
with open(json_file, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
if isinstance(data, list):
|
|
||||||
records = data
|
|
||||||
elif isinstance(data, dict):
|
|
||||||
records = [data]
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"JSON must be object or array: {json_file.name}")
|
input_count, output_count, output = _convert_from_local(task)
|
||||||
with open(output_path, "a", encoding="utf-8") as f:
|
|
||||||
for record in records:
|
|
||||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
||||||
output_count += 1
|
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE data_convert_tasks SET status='completed', "
|
"UPDATE data_convert_tasks SET status='completed', "
|
||||||
@@ -197,12 +336,12 @@ async def upload_source_files(
|
|||||||
(input_count, output_count, task_id),
|
(input_count, output_count, task_id),
|
||||||
)
|
)
|
||||||
# 自动导入数据集
|
# 自动导入数据集
|
||||||
content = output_path.read_text(encoding="utf-8")
|
content = output.decode("utf-8")
|
||||||
size_bytes = len(content.encode("utf-8"))
|
size_bytes = len(content.encode("utf-8"))
|
||||||
dataset = store.create_dataset({
|
dataset = store.create_dataset({
|
||||||
"name": task["name"],
|
"name": task["name"],
|
||||||
"type": "train",
|
"type": "train",
|
||||||
"storage_type": "local",
|
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
|
||||||
"source": "upload",
|
"source": "upload",
|
||||||
"task_id": task_id,
|
"task_id": task_id,
|
||||||
"size": f"{size_bytes} B",
|
"size": f"{size_bytes} B",
|
||||||
@@ -212,7 +351,20 @@ async def upload_source_files(
|
|||||||
})
|
})
|
||||||
dataset_id = dataset["id"]
|
dataset_id = dataset["id"]
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
||||||
|
if should_store_in_minio(len(output)):
|
||||||
|
output_name = _safe_output_filename(task.get("output_filename"))
|
||||||
|
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
|
||||||
|
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
|
||||||
|
storage_object = store.create_storage_object({
|
||||||
|
"resource_type": "dataset", "resource_id": dataset_id,
|
||||||
|
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
|
||||||
|
"bucket": uploaded["bucket"], "object_key": object_key,
|
||||||
|
"file_name": output_name, "content_type": "application/jsonl",
|
||||||
|
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
|
||||||
|
"status": "available", "created_by": task.get("created_by") or current_user.get("id"),
|
||||||
|
})
|
||||||
|
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
|
||||||
return ok({
|
return ok({
|
||||||
"staged_files": staged,
|
"staged_files": staged,
|
||||||
"auto_converted": True,
|
"auto_converted": True,
|
||||||
@@ -248,28 +400,10 @@ def run_convert(
|
|||||||
(task_id,),
|
(task_id,),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
input_dir = _input_dir(task_id)
|
if _minio_enabled():
|
||||||
output_dir = _output_dir(task_id)
|
input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id"))
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
output_path = _task_output_path(task)
|
|
||||||
input_count = 0
|
|
||||||
output_count = 0
|
|
||||||
for json_file in sorted(input_dir.iterdir()):
|
|
||||||
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
|
|
||||||
continue
|
|
||||||
input_count += 1
|
|
||||||
with open(json_file, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
if isinstance(data, list):
|
|
||||||
records = data
|
|
||||||
elif isinstance(data, dict):
|
|
||||||
records = [data]
|
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"JSON must be object or array: {json_file.name}")
|
input_count, output_count, _ = _convert_from_local(task)
|
||||||
with open(output_path, "a", encoding="utf-8") as f:
|
|
||||||
for record in records:
|
|
||||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
||||||
output_count += 1
|
|
||||||
# 更新任务状态
|
# 更新任务状态
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
@@ -297,11 +431,15 @@ def download_result(
|
|||||||
raise fail(404, "task not found")
|
raise fail(404, "task not found")
|
||||||
if task["status"] != "completed":
|
if task["status"] != "completed":
|
||||||
raise fail(400, "task is not completed")
|
raise fail(400, "task is not completed")
|
||||||
output_path = _task_output_path(task)
|
output = _read_output(task)
|
||||||
if not output_path.exists():
|
if output is None:
|
||||||
raise fail(404, "output file not found")
|
raise fail(404, "output file not found")
|
||||||
|
if _minio_enabled():
|
||||||
|
return Response(content=output, media_type="application/octet-stream", headers={
|
||||||
|
"Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}"
|
||||||
|
})
|
||||||
return FileResponse(
|
return FileResponse(
|
||||||
str(output_path),
|
str(_task_output_path(task)),
|
||||||
media_type="application/octet-stream",
|
media_type="application/octet-stream",
|
||||||
filename=_safe_output_filename(task.get("output_filename")),
|
filename=_safe_output_filename(task.get("output_filename")),
|
||||||
)
|
)
|
||||||
@@ -319,10 +457,10 @@ def import_as_dataset(
|
|||||||
raise fail(404, "task not found")
|
raise fail(404, "task not found")
|
||||||
if task["status"] != "completed":
|
if task["status"] != "completed":
|
||||||
raise fail(400, "task is not completed")
|
raise fail(400, "task is not completed")
|
||||||
output_path = _task_output_path(task)
|
output = _read_output(task)
|
||||||
if not output_path.exists():
|
if output is None:
|
||||||
raise fail(404, "output file not found")
|
raise fail(404, "output file not found")
|
||||||
content = output_path.read_text(encoding="utf-8")
|
content = output.decode("utf-8")
|
||||||
dataset_name = str(payload.get("name") or task["name"]).strip()
|
dataset_name = str(payload.get("name") or task["name"]).strip()
|
||||||
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
|
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
|
||||||
size_bytes = len(content.encode("utf-8"))
|
size_bytes = len(content.encode("utf-8"))
|
||||||
@@ -331,7 +469,7 @@ def import_as_dataset(
|
|||||||
dataset = store.create_dataset({
|
dataset = store.create_dataset({
|
||||||
"name": dataset_name,
|
"name": dataset_name,
|
||||||
"type": "train",
|
"type": "train",
|
||||||
"storage_type": "local",
|
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
|
||||||
"source": "upload",
|
"source": "upload",
|
||||||
"task_id": task_id,
|
"task_id": task_id,
|
||||||
"size": f"{size_bytes} B",
|
"size": f"{size_bytes} B",
|
||||||
@@ -341,7 +479,20 @@ def import_as_dataset(
|
|||||||
})
|
})
|
||||||
dataset_id = dataset["id"]
|
dataset_id = dataset["id"]
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
|
||||||
|
if should_store_in_minio(len(output)):
|
||||||
|
output_name = _safe_output_filename(task.get("output_filename"))
|
||||||
|
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
|
||||||
|
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
|
||||||
|
storage_object = store.create_storage_object({
|
||||||
|
"resource_type": "dataset", "resource_id": dataset_id,
|
||||||
|
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
|
||||||
|
"bucket": uploaded["bucket"], "object_key": object_key,
|
||||||
|
"file_name": output_name, "content_type": "application/jsonl",
|
||||||
|
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
|
||||||
|
"status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None),
|
||||||
|
})
|
||||||
|
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
|
||||||
return ok({"dataset_id": dataset_id, "name": dataset_name})
|
return ok({"dataset_id": dataset_id, "name": dataset_name})
|
||||||
|
|
||||||
|
|
||||||
@@ -360,7 +511,15 @@ def delete_task(
|
|||||||
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
|
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
|
||||||
(task_id,),
|
(task_id,),
|
||||||
)
|
)
|
||||||
# 清理文件
|
if _minio_enabled():
|
||||||
|
for item in _task_objects(task_id):
|
||||||
|
try:
|
||||||
|
get_object_storage().delete(item["object_key"])
|
||||||
|
store.update_storage_object(item["id"], {"status": "deleted"})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
# 旧兼容数据仍清理本地目录。
|
||||||
import shutil
|
import shutil
|
||||||
task_dir = _task_dir(task_id)
|
task_dir = _task_dir(task_id)
|
||||||
if task_dir.exists():
|
if task_dir.exists():
|
||||||
|
|||||||
24
backend/app/modules/data_process/algorithms/embedding.py
Normal file
24
backend/app/modules/data_process/algorithms/embedding.py
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
"""数据处理算法 - 本地语义嵌入模型共享单例。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def semantic_embedding_model() -> Any:
|
||||||
|
"""加载本地嵌入模型,供语义分块与语义质量评分共用。
|
||||||
|
|
||||||
|
模型可在部署环境覆盖;默认模型体积较小且适合中英文语义判断。
|
||||||
|
返回 LlamaIndex BaseEmbedding,通过 ``get_text_embedding`` 使用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||||
|
|
||||||
|
return HuggingFaceEmbedding(
|
||||||
|
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
|
||||||
|
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
|
||||||
|
trust_remote_code=False,
|
||||||
|
)
|
||||||
@@ -7,12 +7,13 @@ import re
|
|||||||
import unicodedata
|
import unicodedata
|
||||||
import zipfile
|
import zipfile
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Iterator, Mapping, Sequence
|
||||||
from pathlib import PurePosixPath
|
from pathlib import PurePosixPath
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import unquote, urlsplit
|
from urllib.parse import unquote, urlsplit
|
||||||
|
|
||||||
from docx import Document
|
from docx import Document
|
||||||
|
from docx.oxml.ns import qn
|
||||||
from docx.oxml.table import CT_Tbl
|
from docx.oxml.table import CT_Tbl
|
||||||
from docx.oxml.text.paragraph import CT_P
|
from docx.oxml.text.paragraph import CT_P
|
||||||
from docx.table import Table
|
from docx.table import Table
|
||||||
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
|
|||||||
except zipfile.BadZipFile as exc:
|
except zipfile.BadZipFile as exc:
|
||||||
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
|
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
|
||||||
|
|
||||||
|
def iter_document_blocks(parent: Any) -> Iterator[Any]:
|
||||||
|
"""按文档顺序产出正文段落与表格,并下钻 SDT 内容控件。
|
||||||
|
|
||||||
|
Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里,只遍历 body
|
||||||
|
直接子级会把这些段落整段丢掉。
|
||||||
|
"""
|
||||||
|
|
||||||
|
for child in parent.iterchildren():
|
||||||
|
if isinstance(child, (CT_P, CT_Tbl)):
|
||||||
|
yield child
|
||||||
|
elif child.tag == qn("w:sdt"):
|
||||||
|
content = child.find(qn("w:sdtContent"))
|
||||||
|
if content is not None:
|
||||||
|
yield from iter_document_blocks(content)
|
||||||
|
|
||||||
def _extract_docx_text(raw: bytes) -> str:
|
def _extract_docx_text(raw: bytes) -> str:
|
||||||
_validate_office_archive(raw, "docx")
|
_validate_office_archive(raw, "docx")
|
||||||
try:
|
try:
|
||||||
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
|
|||||||
|
|
||||||
parts: list[str] = []
|
parts: list[str] = []
|
||||||
total = 0
|
total = 0
|
||||||
for child in document.element.body.iterchildren():
|
for child in iter_document_blocks(document.element.body):
|
||||||
if isinstance(child, CT_P):
|
if isinstance(child, CT_P):
|
||||||
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
|
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
@@ -341,3 +342,87 @@ def score_quality(
|
|||||||
flags=tuple(flags),
|
flags=tuple(flags),
|
||||||
fingerprint=fingerprint,
|
fingerprint=fingerprint,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
|
||||||
|
if not left or not right or len(left) != len(right):
|
||||||
|
return 0.0
|
||||||
|
dot = math.fsum(a * b for a, b in zip(left, right))
|
||||||
|
norm_left = math.sqrt(math.fsum(a * a for a in left))
|
||||||
|
norm_right = math.sqrt(math.fsum(b * b for b in right))
|
||||||
|
if not norm_left or not norm_right:
|
||||||
|
return 0.0
|
||||||
|
return dot / (norm_left * norm_right)
|
||||||
|
|
||||||
|
|
||||||
|
def semantic_quality_scores(
|
||||||
|
record: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
source_content: str = "",
|
||||||
|
embed_model: Any = None,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
"""用本地嵌入向量计算语义相关性(0-100)。
|
||||||
|
|
||||||
|
返回 ``question_answer``(问题↔答案)、``answer_source``(答案↔来源,
|
||||||
|
无来源时缺省)与 ``overall``;嵌入模型不可用时返回 None 降级,不阻断流程。
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
if embed_model is None:
|
||||||
|
from .embedding import semantic_embedding_model
|
||||||
|
|
||||||
|
embed_model = semantic_embedding_model()
|
||||||
|
if embed_model is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
question = normalize_text(
|
||||||
|
" ".join(
|
||||||
|
str(record.get(field) or "")
|
||||||
|
for field in ("instruction", "input")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
answer = normalize_text(
|
||||||
|
str(record.get("output") or "") or str(record.get("chosen") or "")
|
||||||
|
)
|
||||||
|
source = normalize_text(source_content)
|
||||||
|
texts = [text for text in {question, answer, source} if text]
|
||||||
|
if not texts:
|
||||||
|
return None
|
||||||
|
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
scores: dict[str, Any] = {}
|
||||||
|
if question and answer:
|
||||||
|
scores["question_answer"] = round(
|
||||||
|
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
|
||||||
|
)
|
||||||
|
if answer and source:
|
||||||
|
scores["answer_source"] = round(
|
||||||
|
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
|
||||||
|
)
|
||||||
|
if not scores:
|
||||||
|
return None
|
||||||
|
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
|
||||||
|
return scores
|
||||||
|
|
||||||
|
|
||||||
|
def composite_overall(
|
||||||
|
*,
|
||||||
|
rule: float | None,
|
||||||
|
semantic: float | None = None,
|
||||||
|
judge: float | None = None,
|
||||||
|
) -> float:
|
||||||
|
"""三层加权组合:规则 35% + 语义 20% + 评审 45%,缺失层自动重归一。"""
|
||||||
|
|
||||||
|
if rule is None:
|
||||||
|
rule = 0.0
|
||||||
|
if judge is not None and semantic is not None:
|
||||||
|
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
|
||||||
|
elif semantic is not None:
|
||||||
|
overall = rule * 0.60 + semantic * 0.40
|
||||||
|
elif judge is not None:
|
||||||
|
overall = rule * 0.55 + judge * 0.45
|
||||||
|
else:
|
||||||
|
overall = rule
|
||||||
|
return round(max(0.0, min(100.0, overall)), 2)
|
||||||
|
|||||||
@@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding
|
|||||||
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
|
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
|
||||||
|
|
||||||
from app.modules.data_process.algorithms import normalize_text
|
from app.modules.data_process.algorithms import normalize_text
|
||||||
|
from app.modules.data_process.algorithms.embedding import semantic_embedding_model
|
||||||
|
|
||||||
ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
|
ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
|
||||||
|
|
||||||
_PAGE_FURNITURE = re.compile(
|
_PAGE_FURNITURE = re.compile(
|
||||||
r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[//]\s*\d+\s*[-—–]?)\s*$"
|
r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[//]\s*\d+\s*[-—–]?)\s*$"
|
||||||
)
|
)
|
||||||
|
# Docling 的 markdown 序列化会给列表项补上自动编号,而 Word 的编号存放在
|
||||||
|
# numbering.xml 中,python-docx 抽取的正文不含这些编号;紧凑匹配前剥掉
|
||||||
|
# 行首编号,否则带列表的切片会整体定位失败。
|
||||||
|
_LIST_MARKER_PREFIX = re.compile(
|
||||||
|
r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+"
|
||||||
|
)
|
||||||
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
|
||||||
_CONVERTER_LOCK = threading.Lock()
|
_CONVERTER_LOCK = threading.Lock()
|
||||||
|
|
||||||
@@ -149,18 +156,6 @@ def chunk_fixed_text(
|
|||||||
return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1)
|
|
||||||
def _semantic_embedding_model() -> BaseEmbedding:
|
|
||||||
# 模型可在部署环境覆盖;默认模型体积较小且适合中英文语义边界判断。
|
|
||||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
|
||||||
|
|
||||||
return HuggingFaceEmbedding(
|
|
||||||
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
|
|
||||||
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
|
|
||||||
trust_remote_code=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def chunk_semantic_text(
|
def chunk_semantic_text(
|
||||||
text: str,
|
text: str,
|
||||||
*,
|
*,
|
||||||
@@ -175,7 +170,7 @@ def chunk_semantic_text(
|
|||||||
if not normalized:
|
if not normalized:
|
||||||
return []
|
return []
|
||||||
splitter = SemanticSplitterNodeParser.from_defaults(
|
splitter = SemanticSplitterNodeParser.from_defaults(
|
||||||
embed_model=embed_model or _semantic_embedding_model(),
|
embed_model=embed_model or semantic_embedding_model(),
|
||||||
breakpoint_percentile_threshold=breakpoint_percentile_threshold,
|
breakpoint_percentile_threshold=breakpoint_percentile_threshold,
|
||||||
buffer_size=1,
|
buffer_size=1,
|
||||||
sentence_splitter=_sentence_chunks,
|
sentence_splitter=_sentence_chunks,
|
||||||
@@ -193,6 +188,7 @@ def chunk_semantic_text(
|
|||||||
if start is None:
|
if start is None:
|
||||||
start = _locate_text(normalized, content, 0)
|
start = _locate_text(normalized, content, 0)
|
||||||
if start is None:
|
if start is None:
|
||||||
|
result.append(_unlocated_chunk(content))
|
||||||
continue
|
continue
|
||||||
if len(_tokenizer().encode(content)) <= chunk_size:
|
if len(_tokenizer().encode(content)) <= chunk_size:
|
||||||
result.append(_make_text_chunk(normalized, start, start + len(content)))
|
result.append(_make_text_chunk(normalized, start, start + len(content)))
|
||||||
@@ -203,6 +199,7 @@ def chunk_semantic_text(
|
|||||||
chunk_overlap=chunk_overlap,
|
chunk_overlap=chunk_overlap,
|
||||||
):
|
):
|
||||||
if child.source_start is None or child.source_end is None:
|
if child.source_start is None or child.source_end is None:
|
||||||
|
result.append(_unlocated_chunk(child.original_content))
|
||||||
continue
|
continue
|
||||||
result.append(
|
result.append(
|
||||||
_make_text_chunk(
|
_make_text_chunk(
|
||||||
@@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
|
|||||||
if start is None:
|
if start is None:
|
||||||
start = _locate_text(source_text, content, 0)
|
start = _locate_text(source_text, content, 0)
|
||||||
if start is None:
|
if start is None:
|
||||||
|
chunks.append(_unlocated_chunk(content))
|
||||||
continue
|
continue
|
||||||
end = start + len(content)
|
end = start + len(content)
|
||||||
chunks.append(_make_text_chunk(source_text, start, end))
|
chunks.append(_make_text_chunk(source_text, start, end))
|
||||||
@@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None:
|
|||||||
return position if position >= 0 else None
|
return position if position >= 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _unlocated_chunk(content: str) -> DocumentChunk:
|
||||||
|
"""正文在源文本中定位失败时保底保留切片,只放弃行号信息。"""
|
||||||
|
|
||||||
|
return DocumentChunk(
|
||||||
|
original_content=content,
|
||||||
|
contextualized_content=content,
|
||||||
|
source_start=None,
|
||||||
|
source_end=None,
|
||||||
|
source_start_line=None,
|
||||||
|
source_end_line=None,
|
||||||
|
token_count=len(_tokenizer().encode(content)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
|
def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
|
||||||
content = source[start:end]
|
content = source[start:end]
|
||||||
return DocumentChunk(
|
return DocumentChunk(
|
||||||
@@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
|
|||||||
return "".join(compact), offsets
|
return "".join(compact), offsets
|
||||||
|
|
||||||
|
|
||||||
|
def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]:
|
||||||
|
while start > 0 and source_text[start - 1] not in "\r\n":
|
||||||
|
start -= 1
|
||||||
|
while end < len(source_text) and source_text[end] not in "\r\n":
|
||||||
|
end += 1
|
||||||
|
return start, end
|
||||||
|
|
||||||
|
|
||||||
def _project_layout_span(
|
def _project_layout_span(
|
||||||
source_text: str,
|
source_text: str,
|
||||||
content: str,
|
content: str,
|
||||||
@@ -324,21 +344,79 @@ def _project_layout_span(
|
|||||||
source_offsets: list[int],
|
source_offsets: list[int],
|
||||||
compact_start: int,
|
compact_start: int,
|
||||||
) -> tuple[int | None, int | None, int]:
|
) -> tuple[int | None, int | None, int]:
|
||||||
compact_content, _ = _compact_with_offsets(content)
|
for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)):
|
||||||
|
compact_content, _ = _compact_with_offsets(candidate)
|
||||||
if len(compact_content) < 4:
|
if len(compact_content) < 4:
|
||||||
return None, None, compact_start
|
continue
|
||||||
position = compact_source.find(compact_content, compact_start)
|
position = compact_source.find(compact_content, compact_start)
|
||||||
if position < 0:
|
if position < 0:
|
||||||
position = compact_source.find(compact_content)
|
position = compact_source.find(compact_content)
|
||||||
if position < 0:
|
if position < 0:
|
||||||
|
continue
|
||||||
|
start, end = _expand_to_line_boundaries(
|
||||||
|
source_text,
|
||||||
|
source_offsets[position],
|
||||||
|
source_offsets[position + len(compact_content) - 1] + 1,
|
||||||
|
)
|
||||||
|
# 重复内容回退匹配可能命中已消费的更早位置,游标只进不退,
|
||||||
|
# 避免后续切片跟着错位。
|
||||||
|
return start, end, max(compact_start, position + len(compact_content))
|
||||||
|
return _project_layout_span_by_anchors(
|
||||||
|
source_text,
|
||||||
|
content,
|
||||||
|
compact_source=compact_source,
|
||||||
|
source_offsets=source_offsets,
|
||||||
|
compact_start=compact_start,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _project_layout_span_by_anchors(
|
||||||
|
source_text: str,
|
||||||
|
content: str,
|
||||||
|
*,
|
||||||
|
compact_source: str,
|
||||||
|
source_offsets: list[int],
|
||||||
|
compact_start: int,
|
||||||
|
) -> tuple[int | None, int | None, int]:
|
||||||
|
"""按行锚点顺序匹配,容忍切片里插入的重复表头等非连续内容。"""
|
||||||
|
|
||||||
|
segments = [
|
||||||
|
compact
|
||||||
|
for compact in (
|
||||||
|
_compact_with_offsets(line)[0]
|
||||||
|
for line in _LIST_MARKER_PREFIX.sub("", content).split("\n")
|
||||||
|
)
|
||||||
|
if len(compact) >= 6
|
||||||
|
]
|
||||||
|
if not segments:
|
||||||
return None, None, compact_start
|
return None, None, compact_start
|
||||||
start = source_offsets[position]
|
total = sum(len(segment) for segment in segments)
|
||||||
end = source_offsets[position + len(compact_content) - 1] + 1
|
|
||||||
while start > 0 and source_text[start - 1] not in "\r\n":
|
def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]:
|
||||||
start -= 1
|
matched: list[tuple[int, int]] = []
|
||||||
while end < len(source_text) and source_text[end] not in "\r\n":
|
position = cursor
|
||||||
end += 1
|
for segment in segments:
|
||||||
return start, end, position + len(compact_content)
|
found = compact_source.find(segment, position)
|
||||||
|
if found < 0:
|
||||||
|
continue
|
||||||
|
matched.append((found, found + len(segment)))
|
||||||
|
position = found + len(segment)
|
||||||
|
return matched, sum(end - start for start, end in matched)
|
||||||
|
|
||||||
|
matched, covered = match_from(compact_start)
|
||||||
|
if covered * 2 < total:
|
||||||
|
retried, retry_covered = match_from(0)
|
||||||
|
if retry_covered > covered:
|
||||||
|
matched, covered = retried, retry_covered
|
||||||
|
# 覆盖不足一半时宁可不定位,也不能给出错误的行号。
|
||||||
|
if not matched or covered * 2 < total:
|
||||||
|
return None, None, compact_start
|
||||||
|
start, end = _expand_to_line_boundaries(
|
||||||
|
source_text,
|
||||||
|
source_offsets[matched[0][0]],
|
||||||
|
source_offsets[matched[-1][1] - 1] + 1,
|
||||||
|
)
|
||||||
|
return start, end, max(compact_start, matched[-1][1])
|
||||||
|
|
||||||
|
|
||||||
def chunk_layout_document(
|
def chunk_layout_document(
|
||||||
|
|||||||
313
backend/app/modules/data_process/evaluation.py
Normal file
313
backend/app/modules/data_process/evaluation.py
Normal file
@@ -0,0 +1,313 @@
|
|||||||
|
"""数据处理 - 生成结果的多层质量评测。
|
||||||
|
|
||||||
|
三层体系:规则层(确定性规则分)+ 语义层(本地嵌入向量)+ 评审层
|
||||||
|
(复用生成模型按 rubric 打分的 LLM-as-judge)。任一层失败自动降级,
|
||||||
|
评测永远返回可用结果,不阻断调用方流程。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import asdict
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .algorithms import normalize_text, score_quality
|
||||||
|
from .algorithms.quality import composite_overall, semantic_quality_scores
|
||||||
|
from .generation import (
|
||||||
|
ModelGenerationError,
|
||||||
|
_is_retryable_generation_error,
|
||||||
|
_json_payload,
|
||||||
|
_message_content,
|
||||||
|
chat_completions_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 送入评审提示词的来源正文上限,避免超长切片挤占评分输出空间。
|
||||||
|
_MAX_JUDGE_SOURCE_CHARS = 6000
|
||||||
|
|
||||||
|
_JUDGE_DIMENSIONS: dict[str, tuple[str, ...]] = {
|
||||||
|
"standard": (
|
||||||
|
"faithfulness",
|
||||||
|
"correctness",
|
||||||
|
"clarity",
|
||||||
|
"completeness",
|
||||||
|
"alignment",
|
||||||
|
),
|
||||||
|
"reasoning": (
|
||||||
|
"faithfulness",
|
||||||
|
"correctness",
|
||||||
|
"clarity",
|
||||||
|
"completeness",
|
||||||
|
"alignment",
|
||||||
|
"reasoning_validity",
|
||||||
|
),
|
||||||
|
"dpo": (
|
||||||
|
"clarity",
|
||||||
|
"chosen_quality",
|
||||||
|
"rejected_quality",
|
||||||
|
"preference_reasonableness",
|
||||||
|
"faithfulness",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
_DIMENSION_LABELS: dict[str, str] = {
|
||||||
|
"faithfulness": "忠实度",
|
||||||
|
"correctness": "正确性",
|
||||||
|
"clarity": "问题清晰度",
|
||||||
|
"completeness": "回答完整性",
|
||||||
|
"alignment": "指令对齐",
|
||||||
|
"reasoning_validity": "推理有效性",
|
||||||
|
"chosen_quality": "chosen 回答质量",
|
||||||
|
"rejected_quality": "rejected 回答质量",
|
||||||
|
"preference_reasonableness": "偏好区分合理性",
|
||||||
|
}
|
||||||
|
|
||||||
|
_DIMENSION_RULES: dict[str, str] = {
|
||||||
|
"faithfulness": "忠实度:答案的全部陈述是否被参考资料支持,没有编造、没有引入资料之外的信息;未提供参考资料时按答案内部自洽性评估",
|
||||||
|
"correctness": "正确性:答案中的事实、概念与计算是否正确",
|
||||||
|
"clarity": "问题清晰度:问题是否清晰、自包含、无歧义,脱离上下文也能理解",
|
||||||
|
"completeness": "回答完整性:答案是否充分、直接地回应了问题的全部要点",
|
||||||
|
"alignment": "指令对齐:答案的形式与范围是否符合问题的要求(如格式、语言、范围限定)",
|
||||||
|
"reasoning_validity": "推理有效性:思维链步骤是否逻辑连贯、无跳步或循环论证,结论是否由推理过程自然得出",
|
||||||
|
"chosen_quality": "chosen 回答质量:更优回答的正确性、完整性与表述质量",
|
||||||
|
"rejected_quality": "rejected 回答质量:较差回答是否仍具备基本可读性,使对比训练有意义",
|
||||||
|
"preference_reasonableness": "偏好区分合理性:chosen 是否明显优于 rejected,且优劣差异与问题直接相关",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _judge_system_prompt(output_type: str) -> str:
|
||||||
|
dimensions = _JUDGE_DIMENSIONS[output_type]
|
||||||
|
rules = "\n".join(f"- {_DIMENSION_RULES[name]}" for name in dimensions)
|
||||||
|
scores_schema = ", ".join(f'"{name}": 1-5' for name in dimensions)
|
||||||
|
return (
|
||||||
|
"你是大模型训练数据质量评审员。严格依据用户消息中的【参考资料】评审这条训练数据,逐维度按 1-5 分打分:\n"
|
||||||
|
f"{rules}\n"
|
||||||
|
"评分锚点:5 分=完全符合维度描述;3 分=基本符合但有明显不足;1 分=严重不符合。\n"
|
||||||
|
"忠实度只依据参考资料与公认常识判断,无法得到支持的陈述必须扣分;不要因为答案冗长而加分。\n"
|
||||||
|
"只输出一个 JSON 对象,不要输出 JSON 之外的任何文字。\n"
|
||||||
|
'输出格式:{"scores": {' + scores_schema + '}, "reason": "一句话总评", "issues": ["具体问题,没有则为空数组"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _judge_user_prompt(record: Mapping[str, Any], source_content: str) -> str:
|
||||||
|
source = normalize_text(source_content)[:_MAX_JUDGE_SOURCE_CHARS] or "(无参考资料)"
|
||||||
|
instruction = normalize_text(str(record.get("instruction") or "")) or "(空)"
|
||||||
|
input_text = normalize_text(str(record.get("input") or ""))
|
||||||
|
sections = [f"【参考资料】\n{source}", f"【问题】\n{instruction}"]
|
||||||
|
if input_text:
|
||||||
|
sections.append(f"【输入】\n{input_text}")
|
||||||
|
if record.get("chosen") or record.get("rejected"):
|
||||||
|
sections.append(f"【更优回答 chosen】\n{normalize_text(str(record.get('chosen') or '')) or '(空)'}")
|
||||||
|
sections.append(f"【较差回答 rejected】\n{normalize_text(str(record.get('rejected') or '')) or '(空)'}")
|
||||||
|
else:
|
||||||
|
output = normalize_text(str(record.get("output") or ""))
|
||||||
|
sections.append(f"【回答】\n{output or '(空)'}")
|
||||||
|
return "\n\n".join(sections)
|
||||||
|
|
||||||
|
|
||||||
|
def _validated_judge_payload(payload: Any, output_type: str) -> dict[str, Any]:
|
||||||
|
if not isinstance(payload, Mapping):
|
||||||
|
raise ModelGenerationError("评审响应不是 JSON 对象")
|
||||||
|
raw_scores = payload.get("scores")
|
||||||
|
if not isinstance(raw_scores, Mapping):
|
||||||
|
raise ModelGenerationError("评审响应缺少 scores 对象")
|
||||||
|
expected = _JUDGE_DIMENSIONS[output_type]
|
||||||
|
scores: dict[str, float] = {}
|
||||||
|
for name in expected:
|
||||||
|
value = raw_scores.get(name)
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||||
|
raise ModelGenerationError(f"评审响应缺少维度 {name} 的有效分数")
|
||||||
|
scores[name] = round(max(1.0, min(5.0, float(value))), 1)
|
||||||
|
issues = payload.get("issues")
|
||||||
|
if not isinstance(issues, list):
|
||||||
|
issues = []
|
||||||
|
issues = [str(item)[:200] for item in issues if str(item).strip()][:8]
|
||||||
|
reason = normalize_text(str(payload.get("reason") or ""))[:300]
|
||||||
|
return {
|
||||||
|
"scores": scores,
|
||||||
|
"overall": round(sum(scores.values()) / len(scores) * 20, 2),
|
||||||
|
"reason": reason,
|
||||||
|
"issues": issues,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _judge_record(
|
||||||
|
record: Mapping[str, Any],
|
||||||
|
source_content: str,
|
||||||
|
*,
|
||||||
|
model: Mapping[str, Any],
|
||||||
|
config: Mapping[str, Any],
|
||||||
|
client: httpx.Client | None,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
output_type = str(config.get("output_type") or "standard").strip().lower()
|
||||||
|
if output_type not in _JUDGE_DIMENSIONS:
|
||||||
|
output_type = "standard"
|
||||||
|
endpoint = chat_completions_url(str(model.get("api_url") or ""))
|
||||||
|
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
|
||||||
|
if not model_name:
|
||||||
|
raise ModelGenerationError("generation model name is required")
|
||||||
|
temperature = 0.1
|
||||||
|
max_tokens = max(256, min(2048, int(config.get("max_tokens", 1024) or 1024)))
|
||||||
|
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60) or 60)))
|
||||||
|
retries = max(0, min(5, int(config.get("generation_retries", 2) or 2)))
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
|
api_key = str(model.get("api_key") or "").strip()
|
||||||
|
if api_key:
|
||||||
|
headers["Authorization"] = f"Bearer {api_key}"
|
||||||
|
|
||||||
|
request_payload: dict[str, Any] = {
|
||||||
|
"model": model_name,
|
||||||
|
"messages": [
|
||||||
|
{"role": "system", "content": _judge_system_prompt(output_type)},
|
||||||
|
{"role": "user", "content": _judge_user_prompt(record, source_content)},
|
||||||
|
],
|
||||||
|
"temperature": temperature,
|
||||||
|
"max_tokens": max_tokens,
|
||||||
|
}
|
||||||
|
if bool(config.get("json_mode", False)):
|
||||||
|
request_payload["response_format"] = {"type": "json_object"}
|
||||||
|
|
||||||
|
owns_client = client is None
|
||||||
|
http_client = client or httpx.Client(timeout=timeout)
|
||||||
|
try:
|
||||||
|
last_error: Exception | None = None
|
||||||
|
for _ in range(retries + 1):
|
||||||
|
try:
|
||||||
|
response = http_client.post(endpoint, headers=headers, json=request_payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
body = response.json()
|
||||||
|
if not isinstance(body, Mapping):
|
||||||
|
raise ModelGenerationError("model response body must be a JSON object")
|
||||||
|
judged = _validated_judge_payload(
|
||||||
|
_json_payload(_message_content(body)),
|
||||||
|
output_type,
|
||||||
|
)
|
||||||
|
judged["model"] = model_name
|
||||||
|
judged["output_type"] = output_type
|
||||||
|
return judged
|
||||||
|
except Exception as exc:
|
||||||
|
last_error = exc
|
||||||
|
if not _is_retryable_generation_error(exc):
|
||||||
|
break
|
||||||
|
raise ModelGenerationError(f"质量评审调用失败: {last_error}")
|
||||||
|
finally:
|
||||||
|
if owns_client:
|
||||||
|
http_client.close()
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_result_record(
|
||||||
|
record: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
source_content: str = "",
|
||||||
|
model: Mapping[str, Any] | None = None,
|
||||||
|
config: Mapping[str, Any] | None = None,
|
||||||
|
client: httpx.Client | None = None,
|
||||||
|
embed_model: Any = None,
|
||||||
|
min_output_length: int = 20,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""对一条生成结果执行三层评测,返回可直接落库的 quality_score 字典。
|
||||||
|
|
||||||
|
规则层字段保持原样平铺(向后兼容既有读取方);新增 ``semantic``、
|
||||||
|
``judge``、``layers``、``evaluated`` 与组合 ``overall``。
|
||||||
|
"""
|
||||||
|
|
||||||
|
config_dict = dict(config or {})
|
||||||
|
rule = score_quality(
|
||||||
|
record,
|
||||||
|
min_output_length=min_output_length,
|
||||||
|
source_content=source_content,
|
||||||
|
)
|
||||||
|
quality: dict[str, Any] = asdict(rule)
|
||||||
|
|
||||||
|
semantic = semantic_quality_scores(
|
||||||
|
record,
|
||||||
|
source_content=source_content,
|
||||||
|
embed_model=embed_model,
|
||||||
|
)
|
||||||
|
judge: dict[str, Any] | None = None
|
||||||
|
if model is not None:
|
||||||
|
try:
|
||||||
|
judge = _judge_record(
|
||||||
|
record,
|
||||||
|
source_content,
|
||||||
|
model=model,
|
||||||
|
config=config_dict,
|
||||||
|
client=client,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"data process judge evaluation degraded: %s",
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
layers = {
|
||||||
|
"rule": rule.overall,
|
||||||
|
"semantic": semantic.get("overall") if semantic else None,
|
||||||
|
"judge": judge.get("overall") if judge else None,
|
||||||
|
}
|
||||||
|
quality.update(
|
||||||
|
semantic=semantic,
|
||||||
|
judge=judge,
|
||||||
|
layers=layers,
|
||||||
|
evaluated=True,
|
||||||
|
evaluated_at=datetime.now(UTC).isoformat(),
|
||||||
|
overall=composite_overall(
|
||||||
|
rule=layers["rule"],
|
||||||
|
semantic=layers["semantic"],
|
||||||
|
judge=layers["judge"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return quality
|
||||||
|
|
||||||
|
|
||||||
|
def reevaluate_edited_record(
|
||||||
|
record: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
source_content: str = "",
|
||||||
|
previous_quality: Mapping[str, Any] | None = None,
|
||||||
|
embed_model: Any = None,
|
||||||
|
min_output_length: int = 20,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""手动编辑/恢复后重算规则与语义层,丢弃已过期的评审层。
|
||||||
|
|
||||||
|
编辑会改变内容,旧的评审分不再可信;规则与语义层本地重算零成本。
|
||||||
|
``evaluated`` 标记沿用原值,保证已评测过的结果编辑后仍有可用分数。
|
||||||
|
"""
|
||||||
|
|
||||||
|
rule = score_quality(
|
||||||
|
record,
|
||||||
|
min_output_length=min_output_length,
|
||||||
|
source_content=source_content,
|
||||||
|
)
|
||||||
|
quality: dict[str, Any] = asdict(rule)
|
||||||
|
semantic = semantic_quality_scores(
|
||||||
|
record,
|
||||||
|
source_content=source_content,
|
||||||
|
embed_model=embed_model,
|
||||||
|
)
|
||||||
|
previous = dict(previous_quality or {})
|
||||||
|
evaluated = bool(previous.get("evaluated"))
|
||||||
|
layers = {
|
||||||
|
"rule": rule.overall,
|
||||||
|
"semantic": semantic.get("overall") if semantic else None,
|
||||||
|
"judge": None,
|
||||||
|
}
|
||||||
|
quality.update(
|
||||||
|
semantic=semantic,
|
||||||
|
judge=None,
|
||||||
|
layers=layers,
|
||||||
|
evaluated=evaluated,
|
||||||
|
evaluated_at=(
|
||||||
|
datetime.now(UTC).isoformat() if evaluated else None
|
||||||
|
),
|
||||||
|
overall=composite_overall(
|
||||||
|
rule=layers["rule"],
|
||||||
|
semantic=layers["semantic"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return quality
|
||||||
@@ -11,6 +11,7 @@ import re
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from docx import Document
|
from docx import Document
|
||||||
|
from docx.oxml.ns import qn
|
||||||
from docx.oxml.table import CT_Tbl
|
from docx.oxml.table import CT_Tbl
|
||||||
from docx.oxml.text.paragraph import CT_P
|
from docx.oxml.text.paragraph import CT_P
|
||||||
from docx.table import Table
|
from docx.table import Table
|
||||||
@@ -27,6 +28,7 @@ from app.modules.data_process.algorithms import (
|
|||||||
_xlsx_sheet_merge_ranges,
|
_xlsx_sheet_merge_ranges,
|
||||||
normalize_text,
|
normalize_text,
|
||||||
)
|
)
|
||||||
|
from app.modules.data_process.algorithms.parsers.office import iter_document_blocks
|
||||||
|
|
||||||
MAX_DOCX_PREVIEW_BLOCKS = 2_000
|
MAX_DOCX_PREVIEW_BLOCKS = 2_000
|
||||||
MAX_XLSX_PREVIEW_ROWS = 200
|
MAX_XLSX_PREVIEW_ROWS = 200
|
||||||
@@ -49,12 +51,21 @@ def _docx_alignment(paragraph: Paragraph) -> str:
|
|||||||
|
|
||||||
def _docx_heading_level(paragraph: Paragraph) -> int | None:
|
def _docx_heading_level(paragraph: Paragraph) -> int | None:
|
||||||
style = paragraph.style
|
style = paragraph.style
|
||||||
if style is None:
|
style_name = str(style.name or "") if style is not None else ""
|
||||||
return None
|
style_id = str(style.style_id or "") if style is not None else ""
|
||||||
style_name = str(style.name or "")
|
|
||||||
style_id = str(style.style_id or "")
|
|
||||||
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
|
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
|
||||||
return int(match.group(1)) if match else None
|
if match:
|
||||||
|
return int(match.group(1))
|
||||||
|
# Word 的目录和导航窗格依据大纲级别识别标题;未套标题样式但带
|
||||||
|
# outlineLvl 的段落(如手工排版的编号小节)同样是标题。
|
||||||
|
outline = paragraph._p.find(f"{qn('w:pPr')}/{qn('w:outlineLvl')}")
|
||||||
|
if outline is not None:
|
||||||
|
value = outline.get(qn("w:val"))
|
||||||
|
if value is not None and value.isdigit():
|
||||||
|
level = int(value)
|
||||||
|
if 0 <= level <= 5:
|
||||||
|
return level + 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def build_docx_preview(raw: bytes) -> dict[str, Any]:
|
def build_docx_preview(raw: bytes) -> dict[str, Any]:
|
||||||
@@ -84,7 +95,7 @@ def build_docx_preview(raw: bytes) -> dict[str, Any]:
|
|||||||
has_source_content = True
|
has_source_content = True
|
||||||
return text, start, source_cursor
|
return text, start, source_cursor
|
||||||
|
|
||||||
for child in document.element.body.iterchildren():
|
for child in iter_document_blocks(document.element.body):
|
||||||
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
|
||||||
truncated = True
|
truncated = True
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""数据处理原始源文件的受控本地对象存储。"""
|
"""数据处理源文件的受控暂存与分层对象存储。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath
|
|||||||
from typing import Iterable, Iterator
|
from typing import Iterable, Iterator
|
||||||
from urllib.parse import quote, unquote, urlsplit
|
from urllib.parse import quote, unquote, urlsplit
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
|
||||||
|
|
||||||
class DataProcessStorageError(ValueError):
|
class DataProcessStorageError(ValueError):
|
||||||
"""本地对象引用或文件系统状态不安全。"""
|
"""本地对象引用或文件系统状态不安全。"""
|
||||||
@@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
class LocalDataProcessStorage:
|
class LocalDataProcessStorage:
|
||||||
"""只允许访问配置根目录下的版本化原始文件。"""
|
"""Stage locally, but publish and read authoritative source files from MinIO."""
|
||||||
|
|
||||||
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
|
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
|
||||||
configured = Path(root) if root is not None else _configured_storage_root()
|
configured = Path(root) if root is not None else _configured_storage_root()
|
||||||
@@ -132,10 +135,7 @@ class LocalDataProcessStorage:
|
|||||||
f"v{version}",
|
f"v{version}",
|
||||||
basename,
|
basename,
|
||||||
)
|
)
|
||||||
reference = (
|
reference = self._reference(task_id, source_file_id, version, basename)
|
||||||
"local://data-process/"
|
|
||||||
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
|
||||||
)
|
|
||||||
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
||||||
self._issued_staged_objects[temporary_path] = staged
|
self._issued_staged_objects[temporary_path] = staged
|
||||||
return staged
|
return staged
|
||||||
@@ -168,6 +168,18 @@ class LocalDataProcessStorage:
|
|||||||
expected_task_id=expected_source_task_id,
|
expected_task_id=expected_source_task_id,
|
||||||
expected_source_file_id=expected_source_file_id,
|
expected_source_file_id=expected_source_file_id,
|
||||||
)
|
)
|
||||||
|
if self._is_minio_reference(source_reference):
|
||||||
|
content = self.read(source_reference)
|
||||||
|
if content is None:
|
||||||
|
raise DataProcessStorageError("original source object is not available")
|
||||||
|
return self.stage_bytes(
|
||||||
|
batch_id=batch_id,
|
||||||
|
task_id=task_id,
|
||||||
|
source_file_id=source_file_id,
|
||||||
|
version=version,
|
||||||
|
name=basename,
|
||||||
|
content=content,
|
||||||
|
)
|
||||||
descriptor, source_info = self._open_read_descriptor(source_relative)
|
descriptor, source_info = self._open_read_descriptor(source_relative)
|
||||||
os.close(descriptor)
|
os.close(descriptor)
|
||||||
|
|
||||||
@@ -193,10 +205,7 @@ class LocalDataProcessStorage:
|
|||||||
f"v{version}",
|
f"v{version}",
|
||||||
basename,
|
basename,
|
||||||
)
|
)
|
||||||
reference = (
|
reference = self._reference(task_id, source_file_id, version, basename)
|
||||||
"local://data-process/"
|
|
||||||
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
|
||||||
)
|
|
||||||
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
staged = StagedSourceObject(reference, temporary_path, relative_path)
|
||||||
self._issued_staged_objects[temporary_path] = staged
|
self._issued_staged_objects[temporary_path] = staged
|
||||||
return staged
|
return staged
|
||||||
@@ -212,14 +221,22 @@ class LocalDataProcessStorage:
|
|||||||
raise DataProcessStorageError("duplicate staged source object")
|
raise DataProcessStorageError("duplicate staged source object")
|
||||||
seen_temporary_paths.add(item._temporary_path)
|
seen_temporary_paths.add(item._temporary_path)
|
||||||
for item in staged:
|
for item in staged:
|
||||||
|
if self._is_minio_reference(item.reference):
|
||||||
|
content = item._temporary_path.read_bytes()
|
||||||
|
get_object_storage().put_bytes(
|
||||||
|
self.object_key(item.reference),
|
||||||
|
content,
|
||||||
|
"application/octet-stream",
|
||||||
|
)
|
||||||
|
elif not item.reference.startswith("db://data-process/"):
|
||||||
final_path = self._path_for_relative(item._relative_path)
|
final_path = self._path_for_relative(item._relative_path)
|
||||||
self._ensure_directory(final_path.parent)
|
self._ensure_directory(final_path.parent)
|
||||||
if final_path.exists() or final_path.is_symlink():
|
if final_path.exists() or final_path.is_symlink():
|
||||||
raise DataProcessStorageError("source storage object already exists")
|
raise DataProcessStorageError("source storage object already exists")
|
||||||
os.link(item._temporary_path, final_path, follow_symlinks=False)
|
os.link(item._temporary_path, final_path, follow_symlinks=False)
|
||||||
|
self._fsync_directory(final_path.parent)
|
||||||
published.append(item)
|
published.append(item)
|
||||||
item._temporary_path.unlink()
|
item._temporary_path.unlink()
|
||||||
self._fsync_directory(final_path.parent)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
for item in reversed(published):
|
for item in reversed(published):
|
||||||
try:
|
try:
|
||||||
@@ -258,7 +275,13 @@ class LocalDataProcessStorage:
|
|||||||
raise first_error
|
raise first_error
|
||||||
|
|
||||||
def read(self, reference: str) -> bytes | None:
|
def read(self, reference: str) -> bytes | None:
|
||||||
"""读取 local 引用;旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。"""
|
"""Read a MinIO object or legacy local reference."""
|
||||||
|
|
||||||
|
if self._is_minio_reference(reference):
|
||||||
|
try:
|
||||||
|
return get_object_storage().get_bytes(self.object_key(reference))
|
||||||
|
except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers
|
||||||
|
raise DataProcessStorageError("source storage object does not exist") from exc
|
||||||
|
|
||||||
relative_path = self._relative_from_reference(reference)
|
relative_path = self._relative_from_reference(reference)
|
||||||
if relative_path is None:
|
if relative_path is None:
|
||||||
@@ -276,6 +299,11 @@ class LocalDataProcessStorage:
|
|||||||
) -> int | None:
|
) -> int | None:
|
||||||
"""返回受控 local 对象大小;旧 ``db://`` 对象没有原始文件。"""
|
"""返回受控 local 对象大小;旧 ``db://`` 对象没有原始文件。"""
|
||||||
|
|
||||||
|
if self._is_minio_reference(reference):
|
||||||
|
try:
|
||||||
|
return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
raise DataProcessStorageError("source storage object does not exist") from exc
|
||||||
relative_path = self._relative_from_reference(reference)
|
relative_path = self._relative_from_reference(reference)
|
||||||
if relative_path is None:
|
if relative_path is None:
|
||||||
return None
|
return None
|
||||||
@@ -301,6 +329,15 @@ class LocalDataProcessStorage:
|
|||||||
) -> Iterator[bytes]:
|
) -> Iterator[bytes]:
|
||||||
"""按范围流式读取原始文件,避免 PDF 预览把大文件整体载入内存。"""
|
"""按范围流式读取原始文件,避免 PDF 预览把大文件整体载入内存。"""
|
||||||
|
|
||||||
|
if self._is_minio_reference(reference):
|
||||||
|
content = self.read(reference) or b""
|
||||||
|
if start < 0 or expected_size != len(content) or start > expected_size:
|
||||||
|
raise DataProcessStorageError("source object size does not match metadata")
|
||||||
|
remaining = expected_size - start if length is None else length
|
||||||
|
if remaining < 0 or start + remaining > expected_size:
|
||||||
|
raise DataProcessStorageError("invalid source byte range")
|
||||||
|
yield content[start : start + remaining]
|
||||||
|
return
|
||||||
relative_path = self._relative_from_reference(reference)
|
relative_path = self._relative_from_reference(reference)
|
||||||
if relative_path is None:
|
if relative_path is None:
|
||||||
raise DataProcessStorageError("original source object is not available")
|
raise DataProcessStorageError("original source object is not available")
|
||||||
@@ -337,6 +374,12 @@ class LocalDataProcessStorage:
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
"""校验 local 引用归属;旧 ``db://`` 引用无需文件系统处理。"""
|
"""校验 local 引用归属;旧 ``db://`` 引用无需文件系统处理。"""
|
||||||
|
|
||||||
|
if self._is_minio_reference(reference):
|
||||||
|
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
|
||||||
|
return True
|
||||||
|
if str(reference or "").startswith("db://data-process/"):
|
||||||
|
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
|
||||||
|
return True
|
||||||
relative_path = self._relative_from_reference(reference)
|
relative_path = self._relative_from_reference(reference)
|
||||||
if relative_path is None:
|
if relative_path is None:
|
||||||
return False
|
return False
|
||||||
@@ -382,6 +425,19 @@ class LocalDataProcessStorage:
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
"""删除受控 local 对象;旧 ``db://`` 引用保持不变。"""
|
"""删除受控 local 对象;旧 ``db://`` 引用保持不变。"""
|
||||||
|
|
||||||
|
if self._is_minio_reference(reference):
|
||||||
|
if (expected_task_id is None) != (expected_source_file_id is None):
|
||||||
|
raise DataProcessStorageError("both expected storage owner fields are required")
|
||||||
|
if expected_task_id is not None and expected_source_file_id is not None:
|
||||||
|
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
|
||||||
|
get_object_storage().delete(self.object_key(reference))
|
||||||
|
return True
|
||||||
|
if str(reference or "").startswith("db://data-process/"):
|
||||||
|
if (expected_task_id is None) != (expected_source_file_id is None):
|
||||||
|
raise DataProcessStorageError("both expected storage owner fields are required")
|
||||||
|
if expected_task_id is not None and expected_source_file_id is not None:
|
||||||
|
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
|
||||||
|
return False
|
||||||
relative_path = self._relative_from_reference(reference)
|
relative_path = self._relative_from_reference(reference)
|
||||||
if relative_path is None:
|
if relative_path is None:
|
||||||
return False
|
return False
|
||||||
@@ -426,7 +482,7 @@ class LocalDataProcessStorage:
|
|||||||
if reference.startswith("db://"):
|
if reference.startswith("db://"):
|
||||||
return None
|
return None
|
||||||
parsed = urlsplit(reference)
|
parsed = urlsplit(reference)
|
||||||
if parsed.scheme != "local" or parsed.netloc != "data-process":
|
if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process":
|
||||||
raise DataProcessStorageError("unsupported source storage reference")
|
raise DataProcessStorageError("unsupported source storage reference")
|
||||||
if parsed.query or parsed.fragment or "\\" in parsed.path:
|
if parsed.query or parsed.fragment or "\\" in parsed.path:
|
||||||
raise DataProcessStorageError("unsafe source storage reference")
|
raise DataProcessStorageError("unsafe source storage reference")
|
||||||
@@ -460,6 +516,39 @@ class LocalDataProcessStorage:
|
|||||||
basename = _safe_basename(decoded[3])
|
basename = _safe_basename(decoded[3])
|
||||||
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
|
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_minio_reference(reference: str) -> bool:
|
||||||
|
return str(reference or "").startswith("minio://data-process/")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str:
|
||||||
|
scheme = "minio" if get_settings().minio_enabled else "local"
|
||||||
|
return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def object_key(reference: str) -> str:
|
||||||
|
parsed = urlsplit(reference)
|
||||||
|
if parsed.scheme != "minio" or parsed.netloc != "data-process":
|
||||||
|
raise DataProcessStorageError("reference is not a MinIO source object")
|
||||||
|
return "data-process/" + parsed.path.lstrip("/")
|
||||||
|
|
||||||
|
def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None:
|
||||||
|
relative = self._relative_from_reference(reference)
|
||||||
|
if relative is None:
|
||||||
|
raise DataProcessStorageError("invalid MinIO source reference")
|
||||||
|
self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None:
|
||||||
|
parsed = urlsplit(reference)
|
||||||
|
parts = parsed.path.lstrip("/").split("/")
|
||||||
|
if parsed.netloc != "data-process" or len(parts) != 3:
|
||||||
|
raise DataProcessStorageError("invalid database source reference")
|
||||||
|
expected_task_id = _safe_component(task_id, "expected task id")
|
||||||
|
expected_source_file_id = _safe_component(source_file_id, "expected source file id")
|
||||||
|
if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1":
|
||||||
|
raise DataProcessStorageError("source storage object owner mismatch")
|
||||||
|
|
||||||
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
|
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
|
||||||
if relative_path.is_absolute() or any(
|
if relative_path.is_absolute() or any(
|
||||||
part in {"", ".", ".."} for part in relative_path.parts
|
part in {"", ".", ".."} for part in relative_path.parts
|
||||||
@@ -479,6 +568,19 @@ class LocalDataProcessStorage:
|
|||||||
raise DataProcessStorageError("invalid staged source object")
|
raise DataProcessStorageError("invalid staged source object")
|
||||||
if self._issued_staged_objects.get(item._temporary_path) is not item:
|
if self._issued_staged_objects.get(item._temporary_path) is not item:
|
||||||
raise DataProcessStorageError("staged source object was not issued by this storage")
|
raise DataProcessStorageError("staged source object was not issued by this storage")
|
||||||
|
if item.reference.startswith("db://data-process/"):
|
||||||
|
parsed = urlsplit(item.reference)
|
||||||
|
parts = parsed.path.lstrip("/").split("/")
|
||||||
|
expected = item._relative_path.parts[:3]
|
||||||
|
if (
|
||||||
|
parsed.netloc != "data-process"
|
||||||
|
or parsed.query
|
||||||
|
or parsed.fragment
|
||||||
|
or len(parts) != 3
|
||||||
|
or tuple(parts) != expected
|
||||||
|
):
|
||||||
|
raise DataProcessStorageError("staged source object reference mismatch")
|
||||||
|
else:
|
||||||
expected_relative = self._relative_from_reference(item.reference)
|
expected_relative = self._relative_from_reference(item.reference)
|
||||||
if expected_relative is None or expected_relative != item._relative_path:
|
if expected_relative is None or expected_relative != item._relative_path:
|
||||||
raise DataProcessStorageError("staged source object reference mismatch")
|
raise DataProcessStorageError("staged source object reference mismatch")
|
||||||
|
|||||||
@@ -247,14 +247,19 @@ def _source_storage_descriptor(
|
|||||||
or f"db://data-process/{task_id}/{file_id}/v1"
|
or f"db://data-process/{task_id}/{file_id}/v1"
|
||||||
)
|
)
|
||||||
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
|
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
|
||||||
|
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
|
||||||
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
|
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
|
||||||
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
|
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
|
||||||
expected_local_prefix
|
expected_local_prefix
|
||||||
):
|
):
|
||||||
storage_backend = "local"
|
storage_backend = "local"
|
||||||
|
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
|
||||||
|
expected_minio_prefix
|
||||||
|
):
|
||||||
|
storage_backend = "minio"
|
||||||
elif storage_object_id == expected_database_reference:
|
elif storage_object_id == expected_database_reference:
|
||||||
storage_backend = "database"
|
storage_backend = "database"
|
||||||
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
|
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
|
||||||
raise DataProcessStoreError("source storage object owner mismatch")
|
raise DataProcessStoreError("source storage object owner mismatch")
|
||||||
else:
|
else:
|
||||||
raise DataProcessStoreError("unsupported source storage object reference")
|
raise DataProcessStoreError("unsupported source storage object reference")
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import psycopg
|
|||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.modules.storage.minio_store import get_object_storage
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
from app.modules.storage.policy import should_store_in_minio
|
||||||
|
|
||||||
from .base import (
|
from .base import (
|
||||||
StoreBase,
|
StoreBase,
|
||||||
@@ -168,7 +169,11 @@ class DatasetsMixin:
|
|||||||
split_name: assignments.count(split_name) for split_name in split_order
|
split_name: assignments.count(split_name) for split_name in split_order
|
||||||
}
|
}
|
||||||
split_specs: list[dict[str, Any]] = []
|
split_specs: list[dict[str, Any]] = []
|
||||||
use_minio = bool(get_settings().minio_enabled)
|
# Explicit local is retained for old callers/tests that request the
|
||||||
|
# legacy backend; all normal platform requests default to MinIO.
|
||||||
|
allow_minio = bool(get_settings().minio_enabled) and str(
|
||||||
|
payload.get("storage_type") or "minio"
|
||||||
|
).lower() != "local"
|
||||||
for split_name in split_order:
|
for split_name in split_order:
|
||||||
split_records = [
|
split_records = [
|
||||||
(source_row, record)
|
(source_row, record)
|
||||||
@@ -193,12 +198,15 @@ class DatasetsMixin:
|
|||||||
"storage_object_id": (
|
"storage_object_id": (
|
||||||
f"db://data-process/{task_id}/{file_id}/v1"
|
f"db://data-process/{task_id}/{file_id}/v1"
|
||||||
),
|
),
|
||||||
|
"store_in_minio": allow_minio and should_store_in_minio(
|
||||||
|
len(raw), content_type="application/jsonl", file_format="jsonl"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
source_result_ids = [row["id"] for row in rows]
|
source_result_ids = [row["id"] for row in rows]
|
||||||
common_metadata = {
|
common_metadata = {
|
||||||
"source": "data_process",
|
"source": "data_process",
|
||||||
"storage_backend": "minio" if use_minio else "database",
|
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
|
||||||
"source_task_id": task_id,
|
"source_task_id": task_id,
|
||||||
"output_type": _task_output_type(task),
|
"output_type": _task_output_type(task),
|
||||||
"reasoning_detail": _task_reasoning_detail(task),
|
"reasoning_detail": _task_reasoning_detail(task),
|
||||||
@@ -273,7 +281,7 @@ class DatasetsMixin:
|
|||||||
split_name = str(spec["split"])
|
split_name = str(spec["split"])
|
||||||
dataset_id = dataset_ids[split_name]
|
dataset_id = dataset_ids[split_name]
|
||||||
storage_object_id = str(spec["storage_object_id"])
|
storage_object_id = str(spec["storage_object_id"])
|
||||||
if use_minio:
|
if spec["store_in_minio"]:
|
||||||
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
||||||
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
||||||
uploaded = get_object_storage().put_bytes(
|
uploaded = get_object_storage().put_bytes(
|
||||||
@@ -356,7 +364,7 @@ class DatasetsMixin:
|
|||||||
(
|
(
|
||||||
dataset_name,
|
dataset_name,
|
||||||
dataset_types[split_name],
|
dataset_types[split_name],
|
||||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
"minio" if spec["store_in_minio"] else "database",
|
||||||
f"{len(spec['raw'])} B",
|
f"{len(spec['raw'])} B",
|
||||||
len(spec["raw"]),
|
len(spec["raw"]),
|
||||||
len(spec["records"]),
|
len(spec["records"]),
|
||||||
@@ -386,7 +394,7 @@ class DatasetsMixin:
|
|||||||
dataset_id,
|
dataset_id,
|
||||||
dataset_name,
|
dataset_name,
|
||||||
dataset_types[split_name],
|
dataset_types[split_name],
|
||||||
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
"minio" if spec["store_in_minio"] else "database",
|
||||||
task_id,
|
task_id,
|
||||||
task_id,
|
task_id,
|
||||||
f"{len(spec['raw'])} B",
|
f"{len(spec['raw'])} B",
|
||||||
@@ -405,7 +413,11 @@ class DatasetsMixin:
|
|||||||
),
|
),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
|
|
||||||
file_metadata = {**dataset_metadata, "file_split": split_name}
|
file_metadata = {
|
||||||
|
**dataset_metadata,
|
||||||
|
"file_split": split_name,
|
||||||
|
"storage_backend": "minio" if spec["store_in_minio"] else "database",
|
||||||
|
}
|
||||||
version = {
|
version = {
|
||||||
"id": spec["version_id"],
|
"id": spec["version_id"],
|
||||||
"version_no": 1,
|
"version_no": 1,
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ class SourceFilesMixin:
|
|||||||
payload["record_count"],
|
payload["record_count"],
|
||||||
payload["file_format"],
|
payload["file_format"],
|
||||||
payload["checksum_sha256"],
|
payload["checksum_sha256"],
|
||||||
payload["content"],
|
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
|
||||||
str(payload["content"])[:2000],
|
str(payload["content"])[:2000],
|
||||||
json_dumps(metadata_payload),
|
json_dumps(metadata_payload),
|
||||||
task.get("tenant_id"),
|
task.get("tenant_id"),
|
||||||
@@ -223,7 +223,17 @@ class SourceFilesMixin:
|
|||||||
).fetchone()
|
).fetchone()
|
||||||
if not row:
|
if not row:
|
||||||
raise NotFoundError("source file not found")
|
raise NotFoundError("source file not found")
|
||||||
return _decode_row(row) or {}
|
decoded = _decode_row(row) or {}
|
||||||
|
# New source files keep only a preview in PostgreSQL. Load the
|
||||||
|
# authoritative body from MinIO on demand for existing processing code.
|
||||||
|
reference = str(decoded.get("storage_object_id") or "")
|
||||||
|
if include_content and not decoded.get("content") and reference.startswith("minio://"):
|
||||||
|
from app.modules.data_process.storage import get_data_process_storage
|
||||||
|
|
||||||
|
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
|
||||||
|
"utf-8", errors="replace"
|
||||||
|
)
|
||||||
|
return decoded
|
||||||
|
|
||||||
def source_content_window(
|
def source_content_window(
|
||||||
self, task_id: str, file_id: str, offset: int, limit: int
|
self, task_id: str, file_id: str, offset: int, limit: int
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
from collections.abc import Iterator
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
@@ -53,6 +54,64 @@ class MinioObjectStorage:
|
|||||||
except S3Error as exc:
|
except S3Error as exc:
|
||||||
raise ObjectStorageError(str(exc)) from exc
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
|
||||||
|
def get_bytes(self, object_key: str) -> bytes:
|
||||||
|
"""Read an object through the backend for small API responses and workers."""
|
||||||
|
self._ensure_enabled()
|
||||||
|
self.ensure_bucket()
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
response = self.client.get_object(self.bucket, object_key)
|
||||||
|
return response.read()
|
||||||
|
except S3Error as exc:
|
||||||
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
finally:
|
||||||
|
if response is not None:
|
||||||
|
response.close()
|
||||||
|
response.release_conn()
|
||||||
|
|
||||||
|
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
|
||||||
|
"""Stream an object without loading the complete file into memory."""
|
||||||
|
self._ensure_enabled()
|
||||||
|
self.ensure_bucket()
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
response = self.client.get_object(self.bucket, object_key)
|
||||||
|
while True:
|
||||||
|
chunk = response.read(chunk_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
yield chunk
|
||||||
|
except S3Error as exc:
|
||||||
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
finally:
|
||||||
|
if response is not None:
|
||||||
|
response.close()
|
||||||
|
response.release_conn()
|
||||||
|
|
||||||
|
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
|
||||||
|
self._ensure_enabled()
|
||||||
|
self.ensure_bucket()
|
||||||
|
try:
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"object_key": item.object_name,
|
||||||
|
"byte_size": item.size or 0,
|
||||||
|
"etag": item.etag,
|
||||||
|
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
|
||||||
|
}
|
||||||
|
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||||
|
]
|
||||||
|
except S3Error as exc:
|
||||||
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
|
||||||
|
def delete(self, object_key: str) -> None:
|
||||||
|
self._ensure_enabled()
|
||||||
|
self.ensure_bucket()
|
||||||
|
try:
|
||||||
|
self.client.remove_object(self.bucket, object_key)
|
||||||
|
except S3Error as exc:
|
||||||
|
raise ObjectStorageError(str(exc)) from exc
|
||||||
|
|
||||||
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
|
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
|
||||||
self._ensure_enabled()
|
self._ensure_enabled()
|
||||||
self.ensure_bucket()
|
self.ensure_bucket()
|
||||||
|
|||||||
54
backend/app/modules/storage/policy.py
Normal file
54
backend/app/modules/storage/policy.py
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
"""Storage placement rules shared by dataset and data-processing flows."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
|
||||||
|
|
||||||
|
_INLINE_TEXT_FORMATS = {
|
||||||
|
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
|
||||||
|
"yaml", "yml", "xml", "html", "text/plain", "application/json",
|
||||||
|
"application/jsonl", "text/csv",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def should_store_in_minio(
|
||||||
|
size_bytes: int | None,
|
||||||
|
*,
|
||||||
|
content_type: str | None = None,
|
||||||
|
file_format: str | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Return whether a file is large enough to use the shared object store.
|
||||||
|
|
||||||
|
Small files remain inline in PostgreSQL so page previews and metadata reads
|
||||||
|
do not pay an object-storage round trip. MinIO is still mandatory for
|
||||||
|
large files when it is enabled.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not get_settings().minio_enabled:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
size = max(0, int(size_bytes or 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
size = 0
|
||||||
|
if size > get_settings().minio_inline_max_bytes:
|
||||||
|
return True
|
||||||
|
# Binary office/document files remain in MinIO even when small because
|
||||||
|
# their original bytes cannot be safely represented by a text DB column.
|
||||||
|
normalized_format = str(file_format or "").strip().lower().lstrip(".")
|
||||||
|
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
|
||||||
|
if normalized_format or normalized_type:
|
||||||
|
return not (
|
||||||
|
normalized_format in _INLINE_TEXT_FORMATS
|
||||||
|
or normalized_type in _INLINE_TEXT_FORMATS
|
||||||
|
or normalized_type.startswith("text/")
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
|
||||||
|
"""Return ``minio`` or ``database`` for a managed file."""
|
||||||
|
|
||||||
|
if str(requested or "").strip().lower() == "local":
|
||||||
|
return "database"
|
||||||
|
return "minio" if should_store_in_minio(size_bytes) else "database"
|
||||||
@@ -56,13 +56,15 @@ def audit_logs(
|
|||||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||||
action: str | None = Query(default=None, description="动作类型"),
|
action: str | None = Query(default=None, description="动作类型"),
|
||||||
target_type: str | None = Query(default=None, description="目标类型"),
|
target_type: str | None = Query(default=None, description="目标类型"),
|
||||||
|
target_id: str | None = Query(default=None, description="目标 ID"),
|
||||||
|
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
|
||||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||||
limit: int = Query(default=50, ge=1, le=200),
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
offset: int = Query(default=0, ge=0),
|
offset: int = Query(default=0, ge=0),
|
||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""审计日志查询:按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
|
"""审计日志查询:按组织、操作人、动作、资源、关键字和时间范围分页过滤。"""
|
||||||
if not is_admin(current_user):
|
if not is_admin(current_user):
|
||||||
from app.api.v1.endpoints.platform import fail
|
from app.api.v1.endpoints.platform import fail
|
||||||
raise fail(403, "admin permission required")
|
raise fail(403, "admin permission required")
|
||||||
@@ -73,6 +75,8 @@ def audit_logs(
|
|||||||
actor_id=actor_id,
|
actor_id=actor_id,
|
||||||
action=action,
|
action=action,
|
||||||
target_type=target_type,
|
target_type=target_type,
|
||||||
|
target_id=target_id,
|
||||||
|
keyword=keyword,
|
||||||
start_time=start_time,
|
start_time=start_time,
|
||||||
end_time=end_time,
|
end_time=end_time,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
@@ -88,6 +92,8 @@ def audit_logs_export(
|
|||||||
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
actor_id: str | None = Query(default=None, description="操作人 ID"),
|
||||||
action: str | None = Query(default=None, description="动作类型"),
|
action: str | None = Query(default=None, description="动作类型"),
|
||||||
target_type: str | None = Query(default=None, description="目标类型"),
|
target_type: str | None = Query(default=None, description="目标类型"),
|
||||||
|
target_id: str | None = Query(default=None, description="目标 ID"),
|
||||||
|
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
|
||||||
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
|
||||||
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
|
||||||
current_user: dict = Depends(get_current_user),
|
current_user: dict = Depends(get_current_user),
|
||||||
@@ -103,6 +109,8 @@ def audit_logs_export(
|
|||||||
actor_id=actor_id,
|
actor_id=actor_id,
|
||||||
action=action,
|
action=action,
|
||||||
target_type=target_type,
|
target_type=target_type,
|
||||||
|
target_id=target_id,
|
||||||
|
keyword=keyword,
|
||||||
start_time=start_time,
|
start_time=start_time,
|
||||||
end_time=end_time,
|
end_time=end_time,
|
||||||
limit=10000,
|
limit=10000,
|
||||||
|
|||||||
@@ -384,6 +384,26 @@ class ResultBatchRegenerateRequest(BaseModel):
|
|||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ResultBatchEvaluateItem(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
result_id: str = Field(min_length=1, max_length=100)
|
||||||
|
expected_updated_at: str = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class ResultBatchEvaluateRequest(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
items: list[ResultBatchEvaluateItem] = Field(min_length=1, max_length=50)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_unique_results(self) -> ResultBatchEvaluateRequest:
|
||||||
|
result_ids = [item.result_id for item in self.items]
|
||||||
|
if len(result_ids) != len(set(result_ids)):
|
||||||
|
raise ValueError("result_id values must be unique")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
class DatasetSplit(BaseModel):
|
class DatasetSplit(BaseModel):
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ from decimal import Decimal
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from docx import Document
|
from docx import Document
|
||||||
|
from docx.oxml import parse_xml
|
||||||
|
from docx.oxml.ns import nsdecls, qn
|
||||||
from openpyxl import Workbook
|
from openpyxl import Workbook
|
||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
from pptx.util import Inches
|
from pptx.util import Inches
|
||||||
@@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import (
|
|||||||
stable_split_assignments,
|
stable_split_assignments,
|
||||||
structured_json_dumps,
|
structured_json_dumps,
|
||||||
)
|
)
|
||||||
|
from app.modules.data_process.office_preview import build_docx_preview
|
||||||
|
|
||||||
|
|
||||||
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
|
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
|
||||||
@@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
|
|||||||
assert parsed_pptx.records == ()
|
assert parsed_pptx.records == ()
|
||||||
|
|
||||||
|
|
||||||
|
def _docx_with_sdt_bytes() -> bytes:
|
||||||
|
"""构造带 SDT 目录内容控件的 docx,段落顺序为正文、SDT、正文。"""
|
||||||
|
|
||||||
|
document = Document()
|
||||||
|
document.add_paragraph("正文开头。")
|
||||||
|
sdt = parse_xml(
|
||||||
|
"<w:sdt %s><w:sdtPr><w:id w:val='1'/></w:sdtPr>"
|
||||||
|
"<w:sdtContent><w:p><w:r><w:t>目录条目 第一章 概述</w:t></w:r></w:p>"
|
||||||
|
"</w:sdtContent></w:sdt>" % nsdecls("w")
|
||||||
|
)
|
||||||
|
body = document.element.body
|
||||||
|
sect_pr = body.find(qn("w:sectPr"))
|
||||||
|
if sect_pr is not None:
|
||||||
|
sect_pr.addprevious(sdt)
|
||||||
|
else:
|
||||||
|
body.append(sdt)
|
||||||
|
document.add_paragraph("正文结尾。")
|
||||||
|
output = io.BytesIO()
|
||||||
|
document.save(output)
|
||||||
|
return output.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def test_docx_extraction_and_preview_include_sdt_content() -> None:
|
||||||
|
raw = _docx_with_sdt_bytes()
|
||||||
|
|
||||||
|
parsed = parse_text_content(raw, filename="toc.docx")
|
||||||
|
assert "目录条目 第一章 概述" in parsed.text
|
||||||
|
assert (
|
||||||
|
parsed.text.index("正文开头。")
|
||||||
|
< parsed.text.index("目录条目 第一章 概述")
|
||||||
|
< parsed.text.index("正文结尾。")
|
||||||
|
)
|
||||||
|
|
||||||
|
preview = build_docx_preview(raw)
|
||||||
|
paragraph_texts = [
|
||||||
|
block["text"] for block in preview["blocks"] if block["type"] == "paragraph"
|
||||||
|
]
|
||||||
|
assert "目录条目 第一章 概述" in paragraph_texts
|
||||||
|
# 预览偏移必须与正文抽取规则一致,否则前端定位会错位。
|
||||||
|
sdt_block = next(
|
||||||
|
block
|
||||||
|
for block in preview["blocks"]
|
||||||
|
if block.get("text") == "目录条目 第一章 概述"
|
||||||
|
)
|
||||||
|
assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == (
|
||||||
|
"目录条目 第一章 概述"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_docx_preview_detects_outline_level_headings() -> None:
|
||||||
|
"""未套标题样式但设了大纲级别的段落(Word 目录按此收录)也按标题渲染。"""
|
||||||
|
|
||||||
|
document = Document()
|
||||||
|
document.add_heading("一级标题", level=1)
|
||||||
|
plain = document.add_paragraph("4.2.1 数据管理")
|
||||||
|
p_pr = plain._p.get_or_add_pPr()
|
||||||
|
p_pr.append(parse_xml("<w:outlineLvl %s w:val='2'/>" % nsdecls("w")))
|
||||||
|
document.add_paragraph("普通正文段落。")
|
||||||
|
output = io.BytesIO()
|
||||||
|
document.save(output)
|
||||||
|
|
||||||
|
preview = build_docx_preview(output.getvalue())
|
||||||
|
blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"}
|
||||||
|
assert blocks["一级标题"]["heading_level"] == 1
|
||||||
|
assert blocks["4.2.1 数据管理"]["heading_level"] == 3
|
||||||
|
assert blocks["普通正文段落。"]["heading_level"] is None
|
||||||
|
|
||||||
|
|
||||||
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
|
||||||
workbook = Workbook()
|
workbook = Workbook()
|
||||||
first = workbook.active
|
first = workbook.active
|
||||||
|
|||||||
@@ -1691,6 +1691,223 @@ def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
|
|||||||
assert model_calls == 0
|
assert model_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_evaluation_task(
|
||||||
|
client: TestClient,
|
||||||
|
store: Any,
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
config: dict[str, Any] | None = None,
|
||||||
|
) -> str:
|
||||||
|
task_id = client.post(
|
||||||
|
"/modelTF/data-process",
|
||||||
|
json={
|
||||||
|
"name": "数据评测",
|
||||||
|
"process_type": "structured",
|
||||||
|
"config": config or {"generation_model_id": "model-1", "output_type": "standard"},
|
||||||
|
},
|
||||||
|
).json()["data"]["id"]
|
||||||
|
store.tasks[task_id].update(
|
||||||
|
status="completed",
|
||||||
|
progress=100,
|
||||||
|
workflow_step="results",
|
||||||
|
results_confirmed=False,
|
||||||
|
)
|
||||||
|
store.models["model-1"] = {
|
||||||
|
"id": "model-1",
|
||||||
|
"online_model_name": "test-model",
|
||||||
|
"api_url": "https://model.example/v1",
|
||||||
|
"api_key": "secret",
|
||||||
|
}
|
||||||
|
store.previews[task_id] = [
|
||||||
|
{
|
||||||
|
"id": "preview-1",
|
||||||
|
"status": "original",
|
||||||
|
"original_content": "申请编号用于唯一标识一笔报销申请。",
|
||||||
|
"edited_content": "申请编号用于唯一标识一笔报销申请。",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "preview-2",
|
||||||
|
"status": "original",
|
||||||
|
"original_content": "联系电话用于联系申请人。",
|
||||||
|
"edited_content": "联系电话用于联系申请人。",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
store.results[task_id] = [
|
||||||
|
{
|
||||||
|
"id": "result-1",
|
||||||
|
"preview_item_id": "preview-1",
|
||||||
|
"instruction": "申请编号有什么作用?",
|
||||||
|
"input": "",
|
||||||
|
"output": "申请编号用于唯一标识一笔报销申请。",
|
||||||
|
"original_instruction": "申请编号有什么作用?",
|
||||||
|
"original_input": "",
|
||||||
|
"original_output": "申请编号用于唯一标识一笔报销申请。",
|
||||||
|
"status": "valid",
|
||||||
|
"error": None,
|
||||||
|
"split": "train",
|
||||||
|
"quality_score": {},
|
||||||
|
"updated_at": "2026-08-19T09:00:00Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "result-2",
|
||||||
|
"preview_item_id": "preview-2",
|
||||||
|
"instruction": "联系电话有什么作用?",
|
||||||
|
"input": "",
|
||||||
|
"output": "联系电话用于联系申请人。",
|
||||||
|
"original_instruction": "联系电话有什么作用?",
|
||||||
|
"original_input": "",
|
||||||
|
"original_output": "联系电话用于联系申请人。",
|
||||||
|
"status": "valid",
|
||||||
|
"error": None,
|
||||||
|
"split": "train",
|
||||||
|
"quality_score": {},
|
||||||
|
"updated_at": "2026-08-19T09:00:01Z",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
return task_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_results_can_be_evaluated_in_batch_with_partial_success(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
client, store, _ = make_client(tmp_path)
|
||||||
|
task_id = _prepare_evaluation_task(client, store, tmp_path)
|
||||||
|
evaluation_calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
|
||||||
|
evaluation_calls.append({"record": deepcopy(record), "kwargs": {k: v for k, v in kwargs.items() if k != "client"}})
|
||||||
|
return {
|
||||||
|
"overall": 88.0,
|
||||||
|
"completeness": 100.0,
|
||||||
|
"length": 100.0,
|
||||||
|
"readability": 100.0,
|
||||||
|
"relevance": 90.0,
|
||||||
|
"duplicate": 100.0,
|
||||||
|
"is_valid": True,
|
||||||
|
"flags": [],
|
||||||
|
"fingerprint": "fp",
|
||||||
|
"semantic": {"question_answer": 80.0, "answer_source": 90.0, "overall": 85.0},
|
||||||
|
"judge": {"scores": {"faithfulness": 5}, "overall": 90.0},
|
||||||
|
"layers": {"rule": 92.0, "semantic": 85.0, "judge": 90.0},
|
||||||
|
"evaluated": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
|
||||||
|
response = client.post(
|
||||||
|
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
|
||||||
|
json={
|
||||||
|
"items": [
|
||||||
|
{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"},
|
||||||
|
# 乐观锁版本不匹配:该条应按冲突失败,另一条仍成功。
|
||||||
|
{"result_id": "result-2", "expected_updated_at": "2026-08-18T00:00:00Z"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()["data"]
|
||||||
|
assert data["total"] == 2
|
||||||
|
assert data["succeeded"] == 1
|
||||||
|
assert data["failed"] == 1
|
||||||
|
assert [item["id"] for item in data["items"]] == ["result-1"]
|
||||||
|
assert data["failures"][0]["result_id"] == "result-2"
|
||||||
|
assert data["failures"][0]["code"] == "conflict"
|
||||||
|
|
||||||
|
assert len(evaluation_calls) == 1
|
||||||
|
assert evaluation_calls[0]["record"]["instruction"] == "申请编号有什么作用?"
|
||||||
|
assert evaluation_calls[0]["kwargs"]["model"]["online_model_name"] == "test-model"
|
||||||
|
assert evaluation_calls[0]["kwargs"]["source_content"] == "申请编号用于唯一标识一笔报销申请。"
|
||||||
|
|
||||||
|
stored = store.results[task_id][0]["quality_score"]
|
||||||
|
assert stored["evaluated"] is True
|
||||||
|
assert stored["layers"]["judge"] == 90.0
|
||||||
|
assert store.results[task_id][1]["quality_score"] == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluation_without_generation_model_skips_judge_layer(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
client, store, _ = make_client(tmp_path)
|
||||||
|
task_id = _prepare_evaluation_task(client, store, tmp_path, config={"output_type": "standard"})
|
||||||
|
seen_models: list[Any] = []
|
||||||
|
|
||||||
|
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
|
||||||
|
seen_models.append(kwargs.get("model"))
|
||||||
|
return {
|
||||||
|
"overall": 70.0, "is_valid": True, "flags": [],
|
||||||
|
"semantic": None, "judge": None,
|
||||||
|
"layers": {"rule": 70.0, "semantic": None, "judge": None},
|
||||||
|
"evaluated": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
|
||||||
|
response = client.post(
|
||||||
|
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
|
||||||
|
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["data"]["succeeded"] == 1
|
||||||
|
# 任务未配置生成模型时,评审层收到的 model 必须是 None。
|
||||||
|
assert seen_models == [None]
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluation_rejects_running_task(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
client, store, _ = make_client(tmp_path)
|
||||||
|
task_id = _prepare_evaluation_task(client, store, tmp_path)
|
||||||
|
store.tasks[task_id]["status"] = "running"
|
||||||
|
evaluation_calls = 0
|
||||||
|
|
||||||
|
def fake_evaluate(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||||
|
nonlocal evaluation_calls
|
||||||
|
evaluation_calls += 1
|
||||||
|
return {"overall": 0, "is_valid": True, "flags": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
|
||||||
|
response = client.post(
|
||||||
|
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
|
||||||
|
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 409
|
||||||
|
assert evaluation_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_update_preserves_evaluation_layers_and_drops_stale_judge(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
client, store, _ = make_client(tmp_path)
|
||||||
|
task_id = _prepare_evaluation_task(client, store, tmp_path)
|
||||||
|
store.results[task_id][0]["quality_score"] = {
|
||||||
|
"overall": 90.0,
|
||||||
|
"is_valid": True,
|
||||||
|
"flags": [],
|
||||||
|
"semantic": {"overall": 85.0},
|
||||||
|
"judge": {"overall": 92.0},
|
||||||
|
"layers": {"rule": 90.0, "semantic": 85.0, "judge": 92.0},
|
||||||
|
"evaluated": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
response = client.put(
|
||||||
|
f"/modelTF/data-process/{task_id}/results/result-1",
|
||||||
|
json={"output": "人工修正后的答案:申请编号唯一标识一笔报销申请。"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
stored = store.results[task_id][0]["quality_score"]
|
||||||
|
# 手动编辑后:规则+语义重算,评审分丢弃,evaluated 标记保留。
|
||||||
|
assert stored["evaluated"] is True
|
||||||
|
assert stored["judge"] is None
|
||||||
|
assert stored["layers"]["judge"] is None
|
||||||
|
assert stored["layers"]["rule"] is not None
|
||||||
|
assert stored["overall"] >= 0
|
||||||
|
|
||||||
|
|
||||||
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
|
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
284
backend/tests/test_data_process_evaluation.py
Normal file
284
backend/tests/test_data_process_evaluation.py
Normal file
@@ -0,0 +1,284 @@
|
|||||||
|
"""数据评测模块(三层质量评分)的单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.data_process.algorithms.quality import (
|
||||||
|
composite_overall,
|
||||||
|
semantic_quality_scores,
|
||||||
|
)
|
||||||
|
from app.modules.data_process.evaluation import (
|
||||||
|
_JUDGE_DIMENSIONS,
|
||||||
|
_judge_system_prompt,
|
||||||
|
_validated_judge_payload,
|
||||||
|
evaluate_result_record,
|
||||||
|
reevaluate_edited_record,
|
||||||
|
)
|
||||||
|
from app.modules.data_process.generation import ModelGenerationError
|
||||||
|
|
||||||
|
RECORD = {
|
||||||
|
"instruction": "申请编号有什么作用?",
|
||||||
|
"input": "",
|
||||||
|
"output": "申请编号用于唯一标识一笔报销申请,便于跟踪审批状态。",
|
||||||
|
}
|
||||||
|
SOURCE = "报销系统中,申请编号用于唯一标识一笔报销申请,并支持跟踪审批状态。"
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeEmbedModel:
|
||||||
|
"""按关键词返回固定向量,模拟语义嵌入。"""
|
||||||
|
|
||||||
|
def get_text_embedding(self, text: str) -> list[float]:
|
||||||
|
if "作用" in text or "编号" in text and "?" in text:
|
||||||
|
return [0.9, 0.1, 0.0]
|
||||||
|
if "申请编号" in text:
|
||||||
|
return [0.85, 0.2, 0.0]
|
||||||
|
return [0.0, 0.1, 0.9]
|
||||||
|
|
||||||
|
|
||||||
|
class _FailingEmbedModel:
|
||||||
|
def get_text_embedding(self, text: str) -> list[float]:
|
||||||
|
raise RuntimeError("embedding unavailable")
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResponse:
|
||||||
|
def __init__(self, payload: dict[str, Any]):
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self) -> dict[str, Any]:
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeClient:
|
||||||
|
def __init__(self, content: str):
|
||||||
|
self._content = content
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||||||
|
self.calls.append({"endpoint": endpoint, "payload": json})
|
||||||
|
return _FakeResponse({
|
||||||
|
"choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
|
||||||
|
})
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _RaisingClient:
|
||||||
|
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
|
||||||
|
raise httpx.ConnectError("model endpoint unreachable")
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _judge_content(scores: dict[str, float], **extra: Any) -> str:
|
||||||
|
return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})
|
||||||
|
|
||||||
|
|
||||||
|
def test_judge_system_prompt_covers_rubric_dimensions() -> None:
|
||||||
|
standard = _judge_system_prompt("standard")
|
||||||
|
for name in _JUDGE_DIMENSIONS["standard"]:
|
||||||
|
assert name in standard
|
||||||
|
assert "1-5" in standard
|
||||||
|
|
||||||
|
dpo = _judge_system_prompt("dpo")
|
||||||
|
assert "chosen_quality" in dpo
|
||||||
|
assert "preference_reasonableness" in dpo
|
||||||
|
|
||||||
|
reasoning = _judge_system_prompt("reasoning")
|
||||||
|
assert "reasoning_validity" in reasoning
|
||||||
|
|
||||||
|
|
||||||
|
def test_validated_judge_payload_converts_scores_to_overall() -> None:
|
||||||
|
judged = _validated_judge_payload(
|
||||||
|
{
|
||||||
|
"scores": {
|
||||||
|
"faithfulness": 5,
|
||||||
|
"correctness": 4,
|
||||||
|
"clarity": 4,
|
||||||
|
"completeness": 3,
|
||||||
|
"alignment": 4,
|
||||||
|
},
|
||||||
|
"reason": "答案可靠",
|
||||||
|
"issues": ["回答略冗长"],
|
||||||
|
},
|
||||||
|
"standard",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
|
||||||
|
assert judged["issues"] == ["回答略冗长"]
|
||||||
|
assert judged["reason"] == "答案可靠"
|
||||||
|
|
||||||
|
|
||||||
|
def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
|
||||||
|
judged = _validated_judge_payload(
|
||||||
|
{
|
||||||
|
"scores": {
|
||||||
|
"faithfulness": 9,
|
||||||
|
"correctness": 4,
|
||||||
|
"clarity": 4,
|
||||||
|
"completeness": 0,
|
||||||
|
"alignment": 4,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"standard",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert judged["scores"]["faithfulness"] == 5.0
|
||||||
|
assert judged["scores"]["completeness"] == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"scores",
|
||||||
|
[
|
||||||
|
{"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
|
||||||
|
{
|
||||||
|
"faithfulness": 5,
|
||||||
|
"correctness": 4,
|
||||||
|
"clarity": "high",
|
||||||
|
"completeness": 3,
|
||||||
|
"alignment": 4,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
|
||||||
|
with pytest.raises(ModelGenerationError):
|
||||||
|
_validated_judge_payload({"scores": scores}, "standard")
|
||||||
|
|
||||||
|
|
||||||
|
def test_semantic_quality_scores_uses_cosine_similarity() -> None:
|
||||||
|
scores = semantic_quality_scores(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert scores is not None
|
||||||
|
assert 0 < scores["question_answer"] <= 100
|
||||||
|
assert 0 < scores["answer_source"] <= 100
|
||||||
|
assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
|
||||||
|
assert (
|
||||||
|
semantic_quality_scores(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
embed_model=_FailingEmbedModel(),
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_composite_overall_weights_available_layers() -> None:
|
||||||
|
assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
|
||||||
|
assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
|
||||||
|
assert composite_overall(rule=80) == 80.0
|
||||||
|
assert composite_overall(rule=None, judge=100) == 45.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_result_record_combines_three_layers() -> None:
|
||||||
|
client = _FakeClient(
|
||||||
|
_judge_content({
|
||||||
|
"faithfulness": 5,
|
||||||
|
"correctness": 4,
|
||||||
|
"clarity": 5,
|
||||||
|
"completeness": 4,
|
||||||
|
"alignment": 5,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
quality = evaluate_result_record(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||||||
|
config={"output_type": "standard", "generation_retries": 0},
|
||||||
|
client=client,
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert quality["evaluated"] is True
|
||||||
|
assert quality["judge"] is not None
|
||||||
|
assert quality["judge"]["model"] == "judge-model"
|
||||||
|
assert quality["semantic"] is not None
|
||||||
|
assert quality["layers"]["judge"] == quality["judge"]["overall"]
|
||||||
|
assert quality["overall"] == composite_overall(
|
||||||
|
rule=quality["layers"]["rule"],
|
||||||
|
semantic=quality["layers"]["semantic"],
|
||||||
|
judge=quality["layers"]["judge"],
|
||||||
|
)
|
||||||
|
# 评审提示词必须携带来源原文作为评分锚点(正文经 NFKC 归一化)。
|
||||||
|
user_message = client.calls[0]["payload"]["messages"][1]["content"]
|
||||||
|
assert "申请编号用于唯一标识一笔报销" in user_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_result_record_degrades_when_model_fails() -> None:
|
||||||
|
quality = evaluate_result_record(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
|
||||||
|
config={"output_type": "standard", "generation_retries": 0},
|
||||||
|
client=_RaisingClient(),
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert quality["judge"] is None
|
||||||
|
assert quality["layers"]["judge"] is None
|
||||||
|
assert quality["semantic"] is not None
|
||||||
|
assert quality["overall"] == composite_overall(
|
||||||
|
rule=quality["layers"]["rule"],
|
||||||
|
semantic=quality["layers"]["semantic"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_result_record_without_model_runs_two_layers() -> None:
|
||||||
|
quality = evaluate_result_record(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
model=None,
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert quality["judge"] is None
|
||||||
|
assert quality["evaluated"] is True
|
||||||
|
assert quality["overall"] == composite_overall(
|
||||||
|
rule=quality["layers"]["rule"],
|
||||||
|
semantic=quality["layers"]["semantic"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reevaluate_edited_record_drops_stale_judge() -> None:
|
||||||
|
previous = {
|
||||||
|
"evaluated": True,
|
||||||
|
"judge": {"overall": 90.0},
|
||||||
|
}
|
||||||
|
quality = reevaluate_edited_record(
|
||||||
|
{**RECORD, "output": "编辑后的新答案内容,用于验证重评逻辑。"},
|
||||||
|
source_content=SOURCE,
|
||||||
|
previous_quality=previous,
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert quality["evaluated"] is True
|
||||||
|
assert quality["judge"] is None
|
||||||
|
assert quality["layers"]["judge"] is None
|
||||||
|
assert quality["semantic"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
|
||||||
|
quality = reevaluate_edited_record(
|
||||||
|
RECORD,
|
||||||
|
source_content=SOURCE,
|
||||||
|
previous_quality={},
|
||||||
|
embed_model=_FakeEmbedModel(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert quality["evaluated"] is False
|
||||||
|
assert quality["evaluated_at"] is None
|
||||||
@@ -9,6 +9,7 @@ from app.modules.data_process.document_chunking import (
|
|||||||
DocumentChunk,
|
DocumentChunk,
|
||||||
_compact_with_offsets,
|
_compact_with_offsets,
|
||||||
_document_converter,
|
_document_converter,
|
||||||
|
_nodes_to_chunks,
|
||||||
_project_layout_span,
|
_project_layout_span,
|
||||||
chunk_fixed_text,
|
chunk_fixed_text,
|
||||||
chunk_semantic_text,
|
chunk_semantic_text,
|
||||||
@@ -103,6 +104,86 @@ def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() ->
|
|||||||
assert cursor > 0
|
assert cursor > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_projection_tolerates_list_numbers_inserted_by_serializer() -> None:
|
||||||
|
# Word 自动编号存放在 numbering.xml,python-docx 抽取的正文没有编号,
|
||||||
|
# 而 Docling 序列化切片时会补上 "1. " 前缀,投影不能因此失败。
|
||||||
|
source = "接入方式说明\n结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
|
||||||
|
compact_source, offsets = _compact_with_offsets(source)
|
||||||
|
start, end, cursor = _project_layout_span(
|
||||||
|
source,
|
||||||
|
"1. 结构化数据接入需要先配置连接地址。\n2. 非结构化接入需要上传文档。",
|
||||||
|
compact_source=compact_source,
|
||||||
|
source_offsets=offsets,
|
||||||
|
compact_start=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert start is not None and end is not None
|
||||||
|
assert source[start:end] == "结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
|
||||||
|
assert cursor > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_projection_never_moves_cursor_backwards() -> None:
|
||||||
|
source = "重复段落内容。\n中间正文。\n重复段落内容。"
|
||||||
|
compact_source, offsets = _compact_with_offsets(source)
|
||||||
|
# 重复内容回退匹配命中已消费的更早位置时,游标必须保持不退。
|
||||||
|
_, _, cursor = _project_layout_span(
|
||||||
|
source,
|
||||||
|
"重复段落内容。",
|
||||||
|
compact_source=compact_source,
|
||||||
|
source_offsets=offsets,
|
||||||
|
compact_start=compact_source.index("中间正文"),
|
||||||
|
)
|
||||||
|
assert cursor >= compact_source.index("中间正文")
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_projection_falls_back_to_line_anchors_for_inserted_content() -> None:
|
||||||
|
# 表格跨切片时 Docling 会在续片中重复表头,正文不再是连续子串;
|
||||||
|
# 按行锚点匹配仍应定位到表头所在行到末行数据之间的连续区间。
|
||||||
|
source = "表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
|
||||||
|
compact_source, offsets = _compact_with_offsets(source)
|
||||||
|
start, end, _ = _project_layout_span(
|
||||||
|
source,
|
||||||
|
"表头甲 表头乙\n第二行数据 说明二",
|
||||||
|
compact_source=compact_source,
|
||||||
|
source_offsets=offsets,
|
||||||
|
compact_start=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert start is not None and end is not None
|
||||||
|
assert source[start:end] == (
|
||||||
|
"表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_layout_projection_refuses_low_coverage_anchor_match() -> None:
|
||||||
|
source = "完全无关的正文内容甲。\n完全无关的正文内容乙。"
|
||||||
|
compact_source, offsets = _compact_with_offsets(source)
|
||||||
|
start, end, cursor = _project_layout_span(
|
||||||
|
source,
|
||||||
|
"找不到的数据行内容\n另一条找不到的数据行内容",
|
||||||
|
compact_source=compact_source,
|
||||||
|
source_offsets=offsets,
|
||||||
|
compact_start=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert start is None
|
||||||
|
assert end is None
|
||||||
|
assert cursor == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_text_splitter_keeps_chunks_that_cannot_be_located() -> None:
|
||||||
|
class FakeNode:
|
||||||
|
def get_content(self) -> str:
|
||||||
|
return "这段文本在源文本中不存在。"
|
||||||
|
|
||||||
|
chunks = _nodes_to_chunks([FakeNode()], "完全不同的源文本。")
|
||||||
|
|
||||||
|
assert len(chunks) == 1
|
||||||
|
assert chunks[0].original_content == "这段文本在源文本中不存在。"
|
||||||
|
assert chunks[0].source_start is None
|
||||||
|
assert chunks[0].source_start_line is None
|
||||||
|
|
||||||
|
|
||||||
def test_short_layout_chunk_merges_with_neighbor_and_keeps_page_provenance() -> None:
|
def test_short_layout_chunk_merges_with_neighbor_and_keeps_page_provenance() -> None:
|
||||||
source = "短标题\n这是一段足够长的正文内容,用于测试相邻切片合并。"
|
source = "短标题\n这是一段足够长的正文内容,用于测试相邻切片合并。"
|
||||||
chunks = [
|
chunks = [
|
||||||
|
|||||||
@@ -221,6 +221,35 @@ def create_app() -> FastAPI:
|
|||||||
return fallback_gpu_resources()
|
return fallback_gpu_resources()
|
||||||
|
|
||||||
items: list[dict[str, Any]] = []
|
items: list[dict[str, Any]] = []
|
||||||
|
processes_by_uuid: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
try:
|
||||||
|
process_result = subprocess.run(
|
||||||
|
[
|
||||||
|
"nvidia-smi",
|
||||||
|
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
|
||||||
|
"--format=csv,noheader,nounits",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=5,
|
||||||
|
)
|
||||||
|
for process_line in process_result.stdout.splitlines():
|
||||||
|
process_parts = [part.strip() for part in process_line.split(",")]
|
||||||
|
if len(process_parts) < 4:
|
||||||
|
continue
|
||||||
|
process_uuid, pid, process_name, used_memory = process_parts[:4]
|
||||||
|
processes_by_uuid.setdefault(process_uuid, []).append(
|
||||||
|
{
|
||||||
|
"pid": int(_safe_float(pid)),
|
||||||
|
"name": process_name,
|
||||||
|
"memory_used_gb": round(_safe_float(used_memory) / 1024, 2),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Some driver/runtime combinations do not expose compute-apps;
|
||||||
|
# utilization and memory metrics remain useful without processes.
|
||||||
|
pass
|
||||||
for line in result.stdout.splitlines():
|
for line in result.stdout.splitlines():
|
||||||
parts = [part.strip() for part in line.split(",")]
|
parts = [part.strip() for part in line.split(",")]
|
||||||
if len(parts) < 9:
|
if len(parts) < 9:
|
||||||
@@ -244,7 +273,7 @@ def create_app() -> FastAPI:
|
|||||||
"temperature": int(_safe_float(temp)),
|
"temperature": int(_safe_float(temp)),
|
||||||
"power_w": round(_safe_float(power), 1),
|
"power_w": round(_safe_float(power), 1),
|
||||||
"power_limit_w": round(_safe_float(power_limit), 1),
|
"power_limit_w": round(_safe_float(power_limit), 1),
|
||||||
"processes": [],
|
"processes": processes_by_uuid.get(uuid, []),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return items
|
return items
|
||||||
@@ -747,6 +776,23 @@ def create_app() -> FastAPI:
|
|||||||
infer_backend: str (default: "huggingface")
|
infer_backend: str (default: "huggingface")
|
||||||
infer_dtype: str (default: "auto")
|
infer_dtype: str (default: "auto")
|
||||||
"""
|
"""
|
||||||
|
requested_gpus = payload.get("gpu_indices")
|
||||||
|
if requested_gpus is None:
|
||||||
|
requested_gpus = payload.get("gpus") or []
|
||||||
|
try:
|
||||||
|
requested_gpus = sorted({int(item) for item in requested_gpus})
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"invalid GPU selection: {exc}") from exc
|
||||||
|
if any(item < 0 for item in requested_gpus):
|
||||||
|
raise HTTPException(status_code=400, detail="GPU index must be non-negative")
|
||||||
|
if requested_gpus:
|
||||||
|
known_gpus = {int(item.get("gpu_index", item.get("id", -1))) for item in gpu_resources()}
|
||||||
|
missing = sorted(set(requested_gpus) - known_gpus)
|
||||||
|
if missing:
|
||||||
|
raise HTTPException(status_code=409, detail=f"requested GPU not found: {missing}")
|
||||||
|
conflict = sorted(set(requested_gpus).intersection(process_manager.locked_gpus()))
|
||||||
|
if conflict:
|
||||||
|
raise HTTPException(status_code=409, detail=f"GPU already used by another compute job: {conflict}")
|
||||||
session = get_inference_session()
|
session = get_inference_session()
|
||||||
result = session.load(
|
result = session.load(
|
||||||
model_name_or_path=payload.get("model_name_or_path", ""),
|
model_name_or_path=payload.get("model_name_or_path", ""),
|
||||||
@@ -754,6 +800,7 @@ def create_app() -> FastAPI:
|
|||||||
template=payload.get("template", "qwen"),
|
template=payload.get("template", "qwen"),
|
||||||
infer_backend=payload.get("infer_backend", "huggingface"),
|
infer_backend=payload.get("infer_backend", "huggingface"),
|
||||||
infer_dtype=payload.get("infer_dtype", "auto"),
|
infer_dtype=payload.get("infer_dtype", "auto"),
|
||||||
|
gpu_indices=requested_gpus,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
@@ -36,6 +37,7 @@ class InferenceSession:
|
|||||||
self._generating_args: dict[str, Any] = {}
|
self._generating_args: dict[str, Any] = {}
|
||||||
self._model_name: str = ""
|
self._model_name: str = ""
|
||||||
self._adapter_path: str = ""
|
self._adapter_path: str = ""
|
||||||
|
self._gpu_indices: list[int] = []
|
||||||
self._loaded_at: float = 0.0
|
self._loaded_at: float = 0.0
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -53,6 +55,7 @@ class InferenceSession:
|
|||||||
"loaded_at": self._loaded_at,
|
"loaded_at": self._loaded_at,
|
||||||
"request_id": self._request_id,
|
"request_id": self._request_id,
|
||||||
"error": self._error,
|
"error": self._error,
|
||||||
|
"gpu_indices": list(self._gpu_indices),
|
||||||
}
|
}
|
||||||
|
|
||||||
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
|
def wait_until_loaded(self, timeout: float | None = None) -> dict[str, Any]:
|
||||||
@@ -87,8 +90,12 @@ class InferenceSession:
|
|||||||
template="qwen",
|
template="qwen",
|
||||||
infer_backend="huggingface",
|
infer_backend="huggingface",
|
||||||
infer_dtype="auto",
|
infer_dtype="auto",
|
||||||
|
gpu_indices=None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
requested_gpus = sorted({int(item) for item in (gpu_indices or [])})
|
||||||
|
if any(item < 0 for item in requested_gpus):
|
||||||
|
return {"loaded": False, "status": "error", "error": "GPU index must be non-negative"}
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
if self._status == "loading":
|
if self._status == "loading":
|
||||||
# A model is already loading — dedupe, reuse the same request id.
|
# A model is already loading — dedupe, reuse the same request id.
|
||||||
@@ -97,6 +104,7 @@ class InferenceSession:
|
|||||||
self._status = "loading"
|
self._status = "loading"
|
||||||
self._error = ""
|
self._error = ""
|
||||||
self._request_id = uuid.uuid4().hex[:12]
|
self._request_id = uuid.uuid4().hex[:12]
|
||||||
|
self._gpu_indices = requested_gpus
|
||||||
self._cancel_requested = False
|
self._cancel_requested = False
|
||||||
self._load_args = {
|
self._load_args = {
|
||||||
"model_name_or_path": model_name_or_path,
|
"model_name_or_path": model_name_or_path,
|
||||||
@@ -115,13 +123,21 @@ class InferenceSession:
|
|||||||
|
|
||||||
def _load_worker(self) -> None:
|
def _load_worker(self) -> None:
|
||||||
"""Build the ChatModel off the state lock so info() never blocks."""
|
"""Build the ChatModel off the state lock so info() never blocks."""
|
||||||
|
with self._state_lock:
|
||||||
|
requested_gpus = list(self._gpu_indices)
|
||||||
model = None
|
model = None
|
||||||
tokenizer = None
|
tokenizer = None
|
||||||
generating_args: dict[str, Any] = {}
|
generating_args: dict[str, Any] = {}
|
||||||
error = ""
|
error = ""
|
||||||
|
previous_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||||
try:
|
try:
|
||||||
|
# Set visibility before LLaMA-Factory/PyTorch initializes CUDA.
|
||||||
|
if requested_gpus:
|
||||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(item) for item in requested_gpus)
|
||||||
if self._teardown_old:
|
if self._teardown_old:
|
||||||
self._release_model()
|
self._release_model()
|
||||||
|
with self._state_lock:
|
||||||
|
self._gpu_indices = requested_gpus
|
||||||
from llamafactory.chat import ChatModel
|
from llamafactory.chat import ChatModel
|
||||||
from llamafactory.hparams import get_infer_args
|
from llamafactory.hparams import get_infer_args
|
||||||
|
|
||||||
@@ -138,6 +154,12 @@ class InferenceSession:
|
|||||||
generating_args = dict(generating_args)
|
generating_args = dict(generating_args)
|
||||||
except Exception as exc: # noqa: BLE001 - surface load failure via status
|
except Exception as exc: # noqa: BLE001 - surface load failure via status
|
||||||
error = str(exc)
|
error = str(exc)
|
||||||
|
finally:
|
||||||
|
if requested_gpus:
|
||||||
|
if previous_visible_devices is None:
|
||||||
|
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||||
|
else:
|
||||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = previous_visible_devices
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
if error:
|
if error:
|
||||||
self._model = None
|
self._model = None
|
||||||
@@ -152,6 +174,7 @@ class InferenceSession:
|
|||||||
self._model = None
|
self._model = None
|
||||||
self._tokenizer = None
|
self._tokenizer = None
|
||||||
self._status = "idle"
|
self._status = "idle"
|
||||||
|
self._gpu_indices = []
|
||||||
return
|
return
|
||||||
self._model = model
|
self._model = model
|
||||||
self._tokenizer = tokenizer
|
self._tokenizer = tokenizer
|
||||||
@@ -189,6 +212,7 @@ class InferenceSession:
|
|||||||
self._adapter_path = ""
|
self._adapter_path = ""
|
||||||
self._loaded_at = 0.0
|
self._loaded_at = 0.0
|
||||||
self._error = ""
|
self._error = ""
|
||||||
|
self._gpu_indices = []
|
||||||
|
|
||||||
def unload(self) -> dict[str, Any]:
|
def unload(self) -> dict[str, Any]:
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
@@ -208,6 +232,7 @@ class InferenceSession:
|
|||||||
self._adapter_path = ""
|
self._adapter_path = ""
|
||||||
self._loaded_at = 0.0
|
self._loaded_at = 0.0
|
||||||
self._error = ""
|
self._error = ""
|
||||||
|
self._gpu_indices = []
|
||||||
return {"unloaded": True, "status": "idle"}
|
return {"unloaded": True, "status": "idle"}
|
||||||
|
|
||||||
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:
|
def chat(self, messages, temperature=0.95, top_p=0.7, max_new_tokens=1024, do_sample=True, **kwargs) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -60,5 +60,6 @@ MINIO_ACCESS_KEY=minioadmin
|
|||||||
MINIO_SECRET_KEY=change_me_minio_secret
|
MINIO_SECRET_KEY=change_me_minio_secret
|
||||||
MINIO_BUCKET=yg-ft-resources
|
MINIO_BUCKET=yg-ft-resources
|
||||||
MINIO_SECURE=false
|
MINIO_SECURE=false
|
||||||
|
MINIO_INLINE_MAX_BYTES=262144
|
||||||
STORAGE_WAIT_SECONDS=300
|
STORAGE_WAIT_SECONDS=300
|
||||||
STORAGE_CHECK_INTERVAL_SECONDS=10
|
STORAGE_CHECK_INTERVAL_SECONDS=10
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ services:
|
|||||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
|
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
|
||||||
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
|
MINIO_BUCKET: ${MINIO_BUCKET:-yg-ft-resources}
|
||||||
MINIO_SECURE: ${MINIO_SECURE:-false}
|
MINIO_SECURE: ${MINIO_SECURE:-false}
|
||||||
|
MINIO_INLINE_MAX_BYTES: ${MINIO_INLINE_MAX_BYTES:-262144}
|
||||||
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
||||||
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
||||||
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}
|
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
# 平台治理功能使用指南
|
# 平台治理功能使用指南
|
||||||
|
|
||||||
> 版本:v1.1
|
|
||||||
> 日期:2026-08-13
|
> 版本:v1.3
|
||||||
> 适用版本:YG Zhilian v1.0+
|
> 日期:2026-08-19
|
||||||
> 更新说明:移除页面权限码设计,改为基于角色的简化权限模型
|
> 适用版本:YG Fine-Tune Platform v1.0+
|
||||||
|
> 更新说明:合并组织权限、审批和运行日志入口;取消项目空间菜单但保留旧接口兼容
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -40,28 +41,27 @@
|
|||||||
|
|
||||||
### 1.3 入口在哪里?
|
### 1.3 入口在哪里?
|
||||||
|
|
||||||
所有治理功能集中在左侧导航栏的 **「系统设置」** 和 **「平台治理」** 分组下:
|
治理、组织和运维功能按职责分布在左侧导航栏的 **「平台治理」**、**「系统设置」** 和 **「算力资源」** 分组下:
|
||||||
|
|
||||||
```
|
```
|
||||||
系统设置
|
|
||||||
├── 用户设置 ← 用户 CRUD + 角色权限 + 密码管理(仅 admin)
|
|
||||||
├── 平台性能 ← 系统监控
|
|
||||||
└── 查看日志 ← 日志查看
|
|
||||||
|
|
||||||
平台治理
|
平台治理
|
||||||
├── 租户管理 ← 组织/团队(仅 admin)
|
├── 组织与权限 ← 用户与角色、租户与配额(仅 admin)
|
||||||
├── 项目空间 ← 项目级资源隔离(仅 admin)
|
├── 资源授权 ← 数据集、模型等资源授权(仅 admin)
|
||||||
├── 审批模板 ← 定义哪些操作需要审批(仅 admin)
|
└── 审批中心 ← 待审批请求与审批策略(仅 admin)
|
||||||
├── 审批中心 ← 处理待审批请求(仅 admin)
|
|
||||||
└── 审计日志 ← 查看所有操作记录(仅 admin)
|
系统设置
|
||||||
|
├── 平台性能 ← 系统资源监控
|
||||||
|
└── 运行日志 ← 运行日志、审计记录、操作诊断
|
||||||
|
|
||||||
算力资源
|
算力资源
|
||||||
└── 算力节点 ← GPU 分配与管理(仅 admin)
|
└── 算力节点 ← GPU 分配与管理(仅 admin)
|
||||||
```
|
```
|
||||||
|
|
||||||
> ⚠️ 以上菜单**只有 admin 用户能看到**。普通用户登录后不会出现这些入口。
|
> ⚠️ 平台治理、资源授权、审批中心和算力节点菜单仅 admin 用户能看到。运行日志入口继续沿用原权限,普通用户可查看系统/训练日志;审计记录和操作诊断页签仅 admin 可见。
|
||||||
>
|
>
|
||||||
> **重要变更(v1.1)**:非 admin 用户**默认可以访问所有业务功能菜单**(模型训练、评测、推理、数据集、数据处理等),无需管理员单独分配权限。
|
> **重要变更(v1.2)**:非 admin 用户**默认可以访问所有业务功能菜单**(模型训练、评测、推理、数据集、数据处理等),无需管理员单独分配权限;治理和资源管理入口仍仅 admin 可见。
|
||||||
|
|
||||||
|
> **当前菜单调整(v1.3)**:平台不再提供项目空间菜单和项目级操作入口。历史项目表、接口和旧地址仅作为兼容层保留,当前资源访问以用户所有权、租户边界(如启用)和资源 ACL 为准;新建业务资源不再要求项目字段。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -73,24 +73,25 @@
|
|||||||
|
|
||||||
| 用户类型 | 可见菜单 | 说明 |
|
| 用户类型 | 可见菜单 | 说明 |
|
||||||
|---------|---------|------|
|
|---------|---------|------|
|
||||||
| **admin(管理员)** | **全部菜单** | 包括用户设置、平台治理、算力节点等管理功能 |
|
| **admin(管理员)** | **全部菜单** | 包括组织权限、审批、运行日志、平台治理和算力节点等管理功能 |
|
||||||
| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理、日志等 |
|
| **非 admin 用户** | **除管理功能外的所有业务菜单** | 模型训练/评测/推理、数据集、数据处理等 |
|
||||||
|
|
||||||
> **核心原则**:
|
> **核心原则**:
|
||||||
> - 非 admin 用户**默认拥有所有业务功能的访问权限**,无需单独分配
|
> - 非 admin 用户**默认拥有所有业务功能的访问权限**,无需单独分配
|
||||||
> - 仅以下功能**仅管理员可见**:
|
> - 仅以下功能**仅管理员可见**:
|
||||||
> - `用户设置`(用户 CRUD、角色管理)
|
> - `平台治理 - 组织与权限`(用户、角色、租户与配额)
|
||||||
> - `平台治理`(租户管理、项目空间、审批模板/中心、审计日志)
|
> - `平台治理`(资源授权、审批中心)
|
||||||
> - `算力节点`(GPU 分配)
|
> - `算力节点`(GPU 分配)
|
||||||
|
> - `运行日志`中的审计记录和操作诊断
|
||||||
>
|
>
|
||||||
> 资源级别的访问控制通过 **ACL(访问控制列表)** 实现,详见第 4 章。
|
> 资源级别的访问控制通过 **ACL(访问控制列表)** 实现,详见第 4 章。
|
||||||
|
|
||||||
### 2.2 创建用户
|
### 2.2 创建用户
|
||||||
|
|
||||||
**路径**:`用户设置` → `创建用户`
|
**路径**:`平台治理` → `组织与权限` → `用户与角色` → `创建用户`
|
||||||
|
|
||||||
1. 以 admin 身份登录平台
|
1. 以 admin 身份登录平台
|
||||||
2. 进入「用户设置」页面
|
2. 进入「组织与权限」页面的「用户与角色」页签
|
||||||
3. 点击右上角「创建用户」按钮
|
3. 点击右上角「创建用户」按钮
|
||||||
4. 填写信息:
|
4. 填写信息:
|
||||||
- **账号**:登录用户名(如 `zhangsan`)
|
- **账号**:登录用户名(如 `zhangsan`)
|
||||||
@@ -107,12 +108,10 @@
|
|||||||
|
|
||||||
| 功能分组 | 包含菜单 | 路由前缀 |
|
| 功能分组 | 包含菜单 | 路由前缀 |
|
||||||
|---------|---------|----------|
|
|---------|---------|----------|
|
||||||
| 系统设置 - 用户设置 | 用户列表、创建用户、重置密码 | `/user-settings` |
|
| 平台治理 - 组织与权限 | 用户、角色、租户与配额 | `/organization` |
|
||||||
| 平台治理 - 租户管理 | 租户列表、配额设置 | `/tenants` |
|
| 平台治理 - 资源授权 | 数据集、模型等资源 ACL | `/resource-acl` |
|
||||||
| 平台治理 - 项目空间 | 项目列表、成员管理、ACL | `/projects` |
|
| 平台治理 - 审批中心 | 待审批请求、审批历史与策略 | `/approval-instances` |
|
||||||
| 平台治理 - 审批模板 | 审批流程定义 | `/approval-templates` |
|
| 系统设置 - 运行日志 | 系统/训练日志;管理员可查看审计记录、操作诊断 | `/logs` |
|
||||||
| 平台治理 - 审批中心 | 待审批请求处理 | `/approval-instances` |
|
|
||||||
| 平台治理 - 审计日志 | 操作记录查询与导出 | `/audit-logs` |
|
|
||||||
| 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` |
|
| 算力资源 - 算力节点 | GPU 分配与管理 | `/compute` |
|
||||||
|
|
||||||
### 2.4 重置用户密码
|
### 2.4 重置用户密码
|
||||||
@@ -125,18 +124,18 @@
|
|||||||
3. 输入新密码,确认
|
3. 输入新密码,确认
|
||||||
|
|
||||||
**方式二:用户自行修改**
|
**方式二:用户自行修改**
|
||||||
1. 用户登录后在「用户设置」页面点击「修改密码」按钮
|
1. 用户登录后在「组织与权限」页面的「用户与角色」页签点击「修改密码」按钮
|
||||||
2. 输入旧密码 + 新密码(至少 6 位)
|
2. 输入旧密码 + 新密码(至少 6 位)
|
||||||
3. 确认修改
|
3. 确认修改
|
||||||
|
|
||||||
### 2.5 删除用户
|
### 2.5 删除用户
|
||||||
|
|
||||||
**路径**:`用户设置` → 用户列表 → 操作列「删除」
|
**路径**:`组织与权限` → `用户与角色` → 用户列表 → 操作列「删除」
|
||||||
|
|
||||||
> ⚠️ 删除用户时会**级联清理**其所有关联数据:
|
> ⚠️ 删除用户时会**级联清理**其所有关联数据:
|
||||||
> - 该用户创建的数据集、基座模型、微调产物、评测任务
|
> - 该用户创建的数据集、基座模型、微调产物、评测任务
|
||||||
> - 该用户的 ACL 授权记录、GPU 分配记录
|
> - 该用户的 ACL 授权记录、GPU 分配记录
|
||||||
> - 该用户的审批实例、审计日志、项目成员关系、登录会话
|
> - 该用户的审批实例、审计日志、历史项目成员关系、登录会话
|
||||||
> - **训练任务保留不删**(避免算力节点上的物理任务数据不一致)
|
> - **训练任务保留不删**(避免算力节点上的物理任务数据不一致)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -148,7 +147,7 @@
|
|||||||
当服务器有多张 GPU 卡(如 8×A800)时,需要指定**哪个用户能用哪张卡**:
|
当服务器有多张 GPU 卡(如 8×A800)时,需要指定**哪个用户能用哪张卡**:
|
||||||
|
|
||||||
- 避免两个人同时选同一张卡导致训练冲突
|
- 避免两个人同时选同一张卡导致训练冲突
|
||||||
- 按团队/项目隔离算力资源
|
- 按用户和租户边界隔离算力资源
|
||||||
- 控制每个用户的 GPU 配额
|
- 控制每个用户的 GPU 配额
|
||||||
|
|
||||||
### 3.2 分配 GPU(仅 admin)
|
### 3.2 分配 GPU(仅 admin)
|
||||||
@@ -259,7 +258,6 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
|
|||||||
| 删除他人的数据集 | 非 admin 删除别人创建的数据集 | 创建审批实例 或 admin 直接执行 |
|
| 删除他人的数据集 | 非 admin 删除别人创建的数据集 | 创建审批实例 或 admin 直接执行 |
|
||||||
| 删除他人的模型 | 非 admin 删除别人创建的模型 | 同上 |
|
| 删除他人的模型 | 非 admin 删除别人创建的模型 | 同上 |
|
||||||
| 停止他人的训练任务 | 非 admin 停止别人发起的任务 | 同上 |
|
| 停止他人的训练任务 | 非 admin 停止别人发起的任务 | 同上 |
|
||||||
| 归档/删除项目空间 | 存在待审批变更时 | 拒绝执行 |
|
|
||||||
|
|
||||||
**核心规则**:admin 做任何操作都直接执行(旁路);普通用户操作他人资源时进入审批流程。
|
**核心规则**:admin 做任何操作都直接执行(旁路);普通用户操作他人资源时进入审批流程。
|
||||||
|
|
||||||
@@ -305,7 +303,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
|
|||||||
3. 决策:「通过」或「拒绝」
|
3. 决策:「通过」或「拒绝」
|
||||||
4. 决策结果自动执行对应操作并记录审计日志
|
4. 决策结果自动执行对应操作并记录审计日志
|
||||||
|
|
||||||
**审批模板**(`平台治理` → `审批模板`):定义每种操作需要几步审批、每步谁来审。默认模板都是单步(admin 审批即可)。
|
**审批策略**(`平台治理` → `审批中心` → `审批策略`):定义每种操作需要几步审批、每步谁来审。默认模板都是单步(admin 审批即可)。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -324,24 +322,29 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
|
|||||||
|
|
||||||
### 6.2 查询审计日志
|
### 6.2 查询审计日志
|
||||||
|
|
||||||
**路径**:`平台治理` → `审计日志`
|
**路径**:`系统设置` → `运行日志` → `审计记录`
|
||||||
|
|
||||||
支持筛选条件:
|
支持筛选条件:
|
||||||
|
|
||||||
| 筛选项 | 说明 |
|
| 筛选项 | 说明 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| 操作人 | 按用户 ID 过滤 |
|
| 租户 | 按租户名称选择 |
|
||||||
| 动作类型 | 如 `user.create`, `dataset.delete`, `gpu.assign` 等 |
|
| 操作人 | 按用户名称选择,不需要手工填写用户 ID |
|
||||||
| 目标资源类型 | dataset / model / fine_tune_task 等 |
|
| 动作类型 | 使用中文动作选择,例如创建数据集、删除模型、授予资源权限 |
|
||||||
|
| 目标资源类型 | 使用中文资源类型选择,例如数据集、模型、训练任务 |
|
||||||
|
| 关键词 | 模糊搜索目标 ID 或审计详情 |
|
||||||
|
| 目标 ID | 对指定资源 ID 进行精确查询 |
|
||||||
| 时间范围 | 开始时间 ~ 结束时间 |
|
| 时间范围 | 开始时间 ~ 结束时间 |
|
||||||
|
|
||||||
|
项目筛选已移除。底层接口仍兼容历史 `project_id` 参数,但当前平台不再提供项目菜单。
|
||||||
|
|
||||||
### 6.3 导出审计日志
|
### 6.3 导出审计日志
|
||||||
|
|
||||||
审计日志页面底部有「导出 CSV」按钮,导出的文件包含当前筛选条件下的全部记录,可用于合规审计或问题追溯。
|
运行日志的「审计记录」页签提供「导出 CSV」按钮;「操作诊断」页签用于检索失败操作和接口耗时,可用于问题追溯。
|
||||||
|
|
||||||
### 6.4 日志保留策略
|
### 6.4 日志保留策略
|
||||||
|
|
||||||
审计日志受**留存策略**控制(`平台治理` → 租户管理 → 绑定留存策略)。默认保留 30 天,超期自动清理。
|
审计日志受**留存策略**控制(`平台治理` → `组织与权限` → `租户与配额`)。默认保留 30 天,超期自动清理。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -351,7 +354,7 @@ curl -X PUT /modelTF/resources/dataset/ds_alpaca_id/acl \
|
|||||||
|
|
||||||
根据 v1.1 权限模型:
|
根据 v1.1 权限模型:
|
||||||
1. **业务菜单**(训练、评测、推理、数据集等):普通用户**默认全部可见**,无需分配
|
1. **业务菜单**(训练、评测、推理、数据集等):普通用户**默认全部可见**,无需分配
|
||||||
2. **管理菜单**(用户设置、租户管理、算力节点等):**仅 admin 可见**,这是设计如此
|
2. **管理菜单**(组织与权限、资源授权、审批中心、算力节点,以及运行日志中的审计/诊断页签):**仅 admin 可见**,这是设计如此
|
||||||
|
|
||||||
如果普通用户看不到业务菜单,请检查:
|
如果普通用户看不到业务菜单,请检查:
|
||||||
- 用户是否正常登录(token 是否有效)
|
- 用户是否正常登录(token 是否有效)
|
||||||
@@ -402,7 +405,7 @@ curl -H "Authorization: Bearer platform-token-admin" \
|
|||||||
### Q6: 用户忘记密码怎么办?
|
### Q6: 用户忘记密码怎么办?
|
||||||
|
|
||||||
两种方案:
|
两种方案:
|
||||||
1. **admin 重置**:在「用户设置」→ 用户列表 →「重置密码」
|
1. **admin 重置**:在「组织与权限」→「用户与角色」→ 用户列表 →「重置密码」
|
||||||
2. **用户自助修改**:用户登录后点击「修改密码」(需知道旧密码)
|
2. **用户自助修改**:用户登录后点击「修改密码」(需知道旧密码)
|
||||||
|
|
||||||
如果是完全忘记且不是 admin,只能由 admin 重置。
|
如果是完全忘记且不是 admin,只能由 admin 重置。
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
# 菜单与功能需求总览
|
# 菜单与功能需求总览
|
||||||
|
|
||||||
> 本文根据当前前端侧边栏、路由、需求文档、接口文档、部署文档和 SQL 脚本整理。当前代码和 SQL 均按正式系统开发基线维护;Mock、Simulator 只能作为显式联调能力,不作为默认开发准则。
|
> 本文根据当前前端侧边栏、路由、需求文档、接口文档、部署文档和 SQL 脚本整理。当前代码和 SQL 均按正式系统开发基线维护;Mock、Simulator 只能作为显式联调能力,不作为默认开发准则。
|
||||||
|
>
|
||||||
|
> 当前治理版本取消“项目空间”菜单。历史项目表和接口仅保留兼容,不再作为前端业务入口或新资源的必填隔离层。
|
||||||
|
|
||||||
## 1. 菜单分层
|
## 1. 菜单分层
|
||||||
|
|
||||||
@@ -17,9 +19,11 @@
|
|||||||
| 数据治理 | 数据处理 | `/data-process` | `data-process` | 前端页面已有,后端待完整实现 | 文档上传、切片预览、LLM 生成、结果编辑、发布数据集 |
|
| 数据治理 | 数据处理 | `/data-process` | `data-process` | 前端页面已有,后端待完整实现 | 文档上传、切片预览、LLM 生成、结果编辑、发布数据集 |
|
||||||
| 其他工具 | 数据类型转换 | `/data-convert` | `data-convert` | 前端页面已有,后端待实现 | JSON/JSONL/Markdown 等格式转换任务 |
|
| 其他工具 | 数据类型转换 | `/data-convert` | `data-convert` | 前端页面已有,后端待实现 | JSON/JSONL/Markdown 等格式转换任务 |
|
||||||
| 算力资源 | 算力节点 | `/compute` | `compute` | 已接入节点管理接口 | 节点地址、权重、标签、启用状态、GPU、队列、资源副本 |
|
| 算力资源 | 算力节点 | `/compute` | `compute` | 已接入节点管理接口 | 节点地址、权重、标签、启用状态、GPU、队列、资源副本 |
|
||||||
| 系统设置 | 用户设置 | `/user-settings` | `user-settings` | 已接入基础用户接口 | 用户列表、创建用户、启停、页面权限 |
|
| 平台治理 | 组织与权限 | `/organization` | `user-settings` | 新增合并入口 | 用户与角色、租户与配额、密码管理 |
|
||||||
|
| 平台治理 | 资源授权 | `/resource-acl` | `user-settings` | 已接入 ACL 接口 | 数据集、模型等资源授权 |
|
||||||
|
| 平台治理 | 审批中心 | `/approval-instances` | `user-settings` | 新增合并入口 | 待审批请求、审批历史、审批策略 |
|
||||||
| 系统设置 | 平台性能 | `/hardware` | `hardware` | 已有接口,需接真实采集 | CPU、内存、磁盘、GPU、进程、网络监控 |
|
| 系统设置 | 平台性能 | `/hardware` | `hardware` | 已有接口,需接真实采集 | CPU、内存、磁盘、GPU、进程、网络监控 |
|
||||||
| 系统设置 | 查看日志 | `/logs` | `logs` | 已有接口,需接真实日志文件 | 后端日志、error 日志、训练日志索引、日志内容查看 |
|
| 系统设置 | 运行日志 | `/logs` | `logs` | 新增合并入口 | 运行日志、训练日志;管理员可查看审计记录、操作诊断 |
|
||||||
|
|
||||||
### 1.2 当前二级和隐藏路由
|
### 1.2 当前二级和隐藏路由
|
||||||
|
|
||||||
@@ -41,19 +45,17 @@
|
|||||||
| 数据集创建/编辑/预览 | `/dataset/create`、`/dataset/:id/edit`、`/dataset/:id/preview` | 数据集管理 | 数据集元数据、文件、版本与内容 |
|
| 数据集创建/编辑/预览 | `/dataset/create`、`/dataset/:id/edit`、`/dataset/:id/preview` | 数据集管理 | 数据集元数据、文件、版本与内容 |
|
||||||
| 自定义工具 | `/tools`、`/tools/create`、`/tools/:id/edit` | 规划入口 | 路由存在,当前侧边栏未展示,后续可归入“其他工具” |
|
| 自定义工具 | `/tools`、`/tools/create`、`/tools/:id/edit` | 规划入口 | 路由存在,当前侧边栏未展示,后续可归入“其他工具” |
|
||||||
| 算力子页 | `/compute/gpus`、`/compute/queue`、`/compute/nodes` | 算力节点 | 当前可作为页签或深链 |
|
| 算力子页 | `/compute/gpus`、`/compute/queue`、`/compute/nodes` | 算力节点 | 当前可作为页签或深链 |
|
||||||
| 创建用户/权限设置 | `/user-settings/create`、`/user-settings/:id/permission` | 用户设置 | 用户创建和页面权限 |
|
| 组织与权限内部页签 | `/user-settings`、`/tenants`、`/user-settings/create`、`/user-settings/:id/permission` | 平台治理 - 组织与权限 | 旧地址兼容,当前通过页签进入 |
|
||||||
|
| 项目旧地址 | `/projects`、`/projects/:id` | 兼容跳转 | 跳转到组织与权限,不再展示项目管理 |
|
||||||
|
| 审批策略旧地址 | `/approval-templates` | 兼容跳转 | 跳转到审批中心的策略页签 |
|
||||||
|
| 日志旧地址 | `/audit-logs`、`/operation-logs` | 兼容跳转 | 跳转到运行日志对应页签 |
|
||||||
| 无权限页 | `/permission-denied` | 系统页 | 路由守卫无权限跳转 |
|
| 无权限页 | `/permission-denied` | 系统页 | 路由守卫无权限跳转 |
|
||||||
|
|
||||||
### 1.3 企业治理待补菜单
|
### 1.3 后续治理扩展
|
||||||
|
|
||||||
| 建议菜单分组 | 菜单 | 建议路由 | 优先级 | 必要性 |
|
| 建议菜单分组 | 菜单 | 建议路由 | 优先级 | 必要性 |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| 组织与项目 | 租户管理 | `/tenants`、`/tenants/:id` | P0 | 多租户隔离、配额、留存策略入口 |
|
| 系统设置 | 运行日志扩展 | `/logs`、`/login-logs`、`/download-logs` | P1 | 增加登录审计、下载审计、导出审计维度 |
|
||||||
| 组织与项目 | 项目空间 | `/projects`、`/projects/:id`、`/projects/:id/members` | P0 | 项目级模型/数据集/任务隔离 |
|
|
||||||
| 组织与项目 | 资源授权 | `/projects/:id/permissions` 或资源详情弹窗 | P0 | 模型/数据集/任务级 ACL |
|
|
||||||
| 治理中心 | 审批中心 | `/approvals`、`/approvals/:id` | P0 | 删除、发布、导出、停止他人任务等高风险动作 |
|
|
||||||
| 治理中心 | 审批设置 | `/approval-settings` | P1 | 审批模板、审批人规则、超时策略 |
|
|
||||||
| 治理中心 | 审计中心 | `/audit-logs`、`/login-logs`、`/download-logs` | P1 | 操作审计、登录审计、下载审计、导出 |
|
|
||||||
| 运维中心 | 存储管理 | `/storage` | P1 | 本地磁盘占用、临时文件、checkpoint 清理、留存 |
|
| 运维中心 | 存储管理 | `/storage` | P1 | 本地磁盘占用、临时文件、checkpoint 清理、留存 |
|
||||||
| 运维中心 | 训练引擎管理 | `/training-engines` | P2 | LLaMA-Factory 和后续引擎能力 schema、健康检查 |
|
| 运维中心 | 训练引擎管理 | `/training-engines` | P2 | LLaMA-Factory 和后续引擎能力 schema、健康检查 |
|
||||||
| 模型服务 | 模型服务治理 | `/model-services`、`/model-services/:id` | P1 | 测试/生产服务发布、调用统计、下线审批 |
|
| 模型服务 | 模型服务治理 | `/model-services`、`/model-services/:id` | P1 | 测试/生产服务发布、调用统计、下线审批 |
|
||||||
@@ -62,7 +64,7 @@
|
|||||||
|
|
||||||
| 菜单/模块 | 主要接口 | 当前运行 SQL | 目标 SQL |
|
| 菜单/模块 | 主要接口 | 当前运行 SQL | 目标 SQL |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| 登录、用户设置 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users` | `users` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides` |
|
| 登录、组织与权限 | `/modelTF/login`、`/modelTF/me`、`/modelTF/users`、`/modelTF/tenants` | `users`、`tenants` | `users`、`login_sessions`、`permissions`、`role_permissions`、`user_permission_overrides`、`tenants` |
|
||||||
| 服务看板 | `/modelTF/dashboard/overview`、`/modelTF/health` | 复用模型/数据集/任务/算力表 | `system_metric_snapshots`、`web_logs`、各业务表聚合 |
|
| 服务看板 | `/modelTF/dashboard/overview`、`/modelTF/health` | 复用模型/数据集/任务/算力表 | `system_metric_snapshots`、`web_logs`、各业务表聚合 |
|
||||||
| 模型管理 | `/modelTF/model-manage`、`/modelTF/model-manage/trained-models`、`/modelTF/model-manage/merge` | `models`、`trained_models` | `models`、`trained_models`、`storage_objects`、`local_import_jobs`、`resource_acl` |
|
| 模型管理 | `/modelTF/model-manage`、`/modelTF/model-manage/trained-models`、`/modelTF/model-manage/merge` | `models`、`trained_models` | `models`、`trained_models`、`storage_objects`、`local_import_jobs`、`resource_acl` |
|
||||||
| 数据集管理 | `/modelTF/dataset-manage`、`/modelTF/dataset-manage/upload/{id}`、`/preview`、`/versions` | `datasets`、`dataset_files` | `datasets`、`dataset_files`、`dataset_file_versions`、`dataset_records`、`storage_objects` |
|
| 数据集管理 | `/modelTF/dataset-manage`、`/modelTF/dataset-manage/upload/{id}`、`/preview`、`/versions` | `datasets`、`dataset_files` | `datasets`、`dataset_files`、`dataset_file_versions`、`dataset_records`、`storage_objects` |
|
||||||
@@ -70,12 +72,12 @@
|
|||||||
| 训练日志 | `/modelTF/training-log-files`、`/modelTF/training-log-content` | 由任务表生成索引 | 日志文件元数据、`fine_tune_metrics`、`audit_logs` |
|
| 训练日志 | `/modelTF/training-log-files`、`/modelTF/training-log-content` | 由任务表生成索引 | 日志文件元数据、`fine_tune_metrics`、`audit_logs` |
|
||||||
| 算力节点 | `/modelTF/compute/nodes`、`/compute/gpus`、`/compute/queue`、`/compute/nodes/{id}/replicas` | `compute_nodes`、`gpus`、`resource_replicas`、`resource_sync_jobs` | `compute_nodes`、`gpu_devices`、`compute_node_engines`、`compute_jobs`、`resource_replicas`、`resource_sync_jobs` |
|
| 算力节点 | `/modelTF/compute/nodes`、`/compute/gpus`、`/compute/queue`、`/compute/nodes/{id}/replicas` | `compute_nodes`、`gpus`、`resource_replicas`、`resource_sync_jobs` | `compute_nodes`、`gpu_devices`、`compute_node_engines`、`compute_jobs`、`resource_replicas`、`resource_sync_jobs` |
|
||||||
| 平台性能 | `/modelTF/system-info`、`/modelTF/compute/gpus` | `gpus`、任务表 | `system_metric_snapshots`、`gpu_devices`、`compute_jobs` |
|
| 平台性能 | `/modelTF/system-info`、`/modelTF/compute/gpus` | `gpus`、任务表 | `system_metric_snapshots`、`gpu_devices`、`compute_jobs` |
|
||||||
| 查看日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log` | 文件日志 | `web_logs`、`audit_logs`,大日志进入日志平台 |
|
| 运行日志 | `/modelTF/log-files`、`/modelTF/log-content`、`/modelTF/web-log`、`/modelTF/audit-logs` | 文件日志 | `web_logs`、`audit_logs`,大日志进入日志平台 |
|
||||||
| 模型评测 | `/modelTF/model-eval`、`/modelTF/dimension` | 当前运行 SQL 未覆盖 | `eval_tasks`、`eval_dimensions`、`eval_sample_results`、`eval_dimension_summaries` |
|
| 模型评测 | `/modelTF/model-eval`、`/modelTF/dimension` | 当前运行 SQL 未覆盖 | `eval_tasks`、`eval_dimensions`、`eval_sample_results`、`eval_dimension_summaries` |
|
||||||
| 模型推理/对比 | `/modelTF/model-compare`、`/modelTF/model-chat/*` | 当前运行 SQL 未覆盖 | `inference_tasks`、`inference_task_models`、`chat_sessions`、`chat_messages` |
|
| 模型推理/对比 | `/modelTF/model-compare`、`/modelTF/model-chat/*` | 当前运行 SQL 未覆盖 | `inference_tasks`、`inference_task_models`、`chat_sessions`、`chat_messages` |
|
||||||
| 数据处理 | `/modelTF/data-process/*` | 当前运行 SQL 未覆盖 | `data_process_tasks`、`data_process_source_files`、`data_process_preview_items`、`data_process_results` |
|
| 数据处理 | `/modelTF/data-process/*` | 当前运行 SQL 未覆盖 | `data_process_tasks`、`data_process_source_files`、`data_process_preview_items`、`data_process_results` |
|
||||||
| 数据转换/自定义工具 | `/modelTF/data-convert/jobs`、`/modelTF/tools` | 当前运行 SQL 未覆盖 | `data_convert_jobs`、`custom_tools` |
|
| 数据转换/自定义工具 | `/modelTF/data-convert/jobs`、`/modelTF/tools` | 当前运行 SQL 未覆盖 | `data_convert_jobs`、`custom_tools` |
|
||||||
| 租户/项目/资源授权 | `/modelTF/tenants`、`/modelTF/projects`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`projects`、`project_members`、`resource_acl` |
|
| 租户/资源授权 | `/modelTF/tenants`、`/modelTF/resources/{type}/{id}/acl` | 当前运行 SQL 未覆盖 | `tenants`、`tenant_users`、`resource_acl`;`projects`、`project_members` 仅作兼容 |
|
||||||
| 审批/审计/留存/配额 | `/modelTF/approvals`、`/modelTF/audit-logs`、`/modelTF/retention-policies`、`/modelTF/quotas/usage` | 当前运行 SQL 未覆盖 | `approval_templates`、`approval_instances`、`approval_steps`、`audit_logs`、`retention_policies`、`quotas`、`quota_usage` |
|
| 审批/审计/留存/配额 | `/modelTF/approvals`、`/modelTF/audit-logs`、`/modelTF/retention-policies`、`/modelTF/quotas/usage` | 当前运行 SQL 未覆盖 | `approval_templates`、`approval_instances`、`approval_steps`、`audit_logs`、`retention_policies`、`quotas`、`quota_usage` |
|
||||||
|
|
||||||
## 3. 文档和脚本检查结论
|
## 3. 文档和脚本检查结论
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
> 日期:2026-08-02
|
> 日期:2026-08-02
|
||||||
> 状态:设计基线,供后端实现和前端联调参照
|
> 状态:设计基线,供后端实现和前端联调参照
|
||||||
|
|
||||||
|
> **当前菜单基线(2026-08-19)**:平台治理已取消“项目空间”作为用户可见菜单和新资源的业务隔离层。当前前端入口为“组织与权限、资源授权、审批中心”,系统设置下的“运行日志”承载运行日志、审计记录和操作诊断。`projects`、`project_members` 表及相关后端接口仅作历史兼容,不删除、不要求新建资源填写 `project_id`。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 目录
|
## 目录
|
||||||
@@ -129,7 +131,7 @@
|
|||||||
| `compute` | `/compute` | 算力节点 |
|
| `compute` | `/compute` | 算力节点 |
|
||||||
| `hardware` | `/hardware` | 平台性能 |
|
| `hardware` | `/hardware` | 平台性能 |
|
||||||
| `logs` | `/logs`, `/training-log/:id` | 查看日志 |
|
| `logs` | `/logs`, `/training-log/:id` | 查看日志 |
|
||||||
| `user-settings` | `/user-settings`, `/tenants`, `/projects`, `/approvals`, `/audit-logs` | 系统设置与平台治理 |
|
| `user-settings` | `/organization`, `/resource-acl`, `/approval-instances`, `/logs` | 平台治理和运行日志(管理员) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
120
docs/platform-governance-menu-design.md
Normal file
120
docs/platform-governance-menu-design.md
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
# 平台治理菜单设计与开发计划
|
||||||
|
|
||||||
|
> 版本:v1.0
|
||||||
|
> 日期:2026-08-19
|
||||||
|
> 状态:按本文档实施
|
||||||
|
|
||||||
|
## 1. 设计结论
|
||||||
|
|
||||||
|
当前项目已经有用户所有权、资源 ACL、租户接口和审计接口,但项目隔离尚未真正落地。核心资源的 `project_id` 当前没有有效业务数据,训练、评测、推理和数据集创建流程也没有统一的项目上下文。
|
||||||
|
|
||||||
|
因此当前版本取消项目层级设计,资源权限统一采用:
|
||||||
|
|
||||||
|
```text
|
||||||
|
用户所有权 + 资源 ACL + 租户边界(可选)
|
||||||
|
```
|
||||||
|
|
||||||
|
项目相关数据库表和后端接口暂不物理删除,仅作为历史兼容能力保留,后续不再新增项目数据,也不在前端提供项目入口。
|
||||||
|
|
||||||
|
## 2. 最终菜单
|
||||||
|
|
||||||
|
```text
|
||||||
|
平台治理
|
||||||
|
├── 组织与权限
|
||||||
|
├── 资源授权
|
||||||
|
└── 审批中心
|
||||||
|
|
||||||
|
系统设置
|
||||||
|
├── 平台性能
|
||||||
|
└── 运行日志
|
||||||
|
├── 系统日志
|
||||||
|
├── 训练日志
|
||||||
|
├── 审计记录
|
||||||
|
└── 操作诊断
|
||||||
|
|
||||||
|
算力资源
|
||||||
|
└── 算力节点
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.1 组织与权限
|
||||||
|
|
||||||
|
使用页签统一承载:
|
||||||
|
|
||||||
|
- 用户与角色:用户 CRUD、启停、密码、角色。
|
||||||
|
- 租户与配额:租户、GPU 配额、存储配额和资源数量配额。
|
||||||
|
|
||||||
|
用户、租户和配额仍使用独立表和接口,不把组织配额字段混入用户表。单租户部署时可默认停留在“用户与角色”页签。
|
||||||
|
|
||||||
|
### 2.2 资源授权
|
||||||
|
|
||||||
|
保留当前资源 ACL 能力,支持数据集、训练模型等资源的 `read/write/execute/download/delete/admin` 权限。项目不再作为授权前置条件。
|
||||||
|
|
||||||
|
### 2.3 审批中心
|
||||||
|
|
||||||
|
统一使用页签承载:
|
||||||
|
|
||||||
|
- 待审批/审批历史。
|
||||||
|
- 我的申请。
|
||||||
|
- 审批策略,仅管理员可见。
|
||||||
|
|
||||||
|
“审批策略”不再作为独立一级菜单。
|
||||||
|
|
||||||
|
### 2.4 运行日志
|
||||||
|
|
||||||
|
在现有系统日志、训练日志基础上增加:
|
||||||
|
|
||||||
|
- 审计记录:写操作、授权、审批、删除、导出等敏感操作。
|
||||||
|
- 操作诊断:失败操作、错误类型和接口耗时。
|
||||||
|
|
||||||
|
“审计中心”不再作为独立菜单。旧的 `/audit-logs` 和 `/operation-logs` 地址保留重定向。
|
||||||
|
|
||||||
|
## 3. 兼容策略
|
||||||
|
|
||||||
|
| 原入口 | 新入口/处理方式 |
|
||||||
|
|---|---|
|
||||||
|
| `/user-settings` | 重定向到 `/organization?tab=users` |
|
||||||
|
| `/tenants` | 重定向到 `/organization?tab=tenants` |
|
||||||
|
| `/approval-templates` | 重定向到 `/approval-instances?tab=strategies` |
|
||||||
|
| `/audit-logs` | 重定向到 `/logs?tab=audit` |
|
||||||
|
| `/operation-logs` | 重定向到 `/logs?tab=operations` |
|
||||||
|
| `/projects` | 移除前端入口;旧地址重定向到组织与权限 |
|
||||||
|
|
||||||
|
后端的租户、项目、审批、ACL、审计 API 暂不删除,保证已有脚本和历史客户端不立即失效。数据库不执行删表操作,也不新增项目字段迁移。
|
||||||
|
|
||||||
|
## 4. 开发计划
|
||||||
|
|
||||||
|
### 阶段一:导航和页面聚合
|
||||||
|
|
||||||
|
1. 新增“组织与权限”聚合页面。
|
||||||
|
2. 新增“审批中心”聚合页面。
|
||||||
|
3. 扩展“运行日志”页面,加入审计和操作诊断页签。
|
||||||
|
4. 调整侧边栏,只展示最终菜单。
|
||||||
|
|
||||||
|
### 阶段二:兼容旧入口
|
||||||
|
|
||||||
|
1. 旧用户、租户、审批策略、审计和操作日志路由改为重定向。
|
||||||
|
2. 保留原页面组件、API 和后端路由,避免历史调用失效。
|
||||||
|
3. 项目路由不再作为业务入口,不再新增项目数据。
|
||||||
|
|
||||||
|
### 阶段三:权限和功能检查
|
||||||
|
|
||||||
|
1. 管理员可以访问组织、租户、配额、审批、审计和 ACL。
|
||||||
|
2. 普通用户不能访问平台治理菜单;运行日志基础页签继续保持原有访问权限。
|
||||||
|
3. 审批策略页签仅管理员可见。
|
||||||
|
4. 审计和操作诊断仍保留管理员可见能力。
|
||||||
|
5. 数据集、模型、训练、评测、推理继续使用用户所有权和 ACL,不增加项目选择器。
|
||||||
|
|
||||||
|
### 阶段四:验证
|
||||||
|
|
||||||
|
- `npm run build`。
|
||||||
|
- 检查旧路由重定向。
|
||||||
|
- 检查管理员菜单显示。
|
||||||
|
- 检查非管理员权限拦截。
|
||||||
|
- 检查 Backend 健康接口和前端静态资源。
|
||||||
|
|
||||||
|
## 5. 暂不处理事项
|
||||||
|
|
||||||
|
- 不删除 `projects`、`project_members` 表。
|
||||||
|
- 不删除后端项目模块,避免历史数据和接口调用中断。
|
||||||
|
- 不把租户配额字段直接合并到 `users` 表。
|
||||||
|
- 不改变现有数据集、模型、训练、评测、推理的业务接口格式。
|
||||||
347
docs/当前项目开发进度.md
Normal file
347
docs/当前项目开发进度.md
Normal file
@@ -0,0 +1,347 @@
|
|||||||
|
# 当前项目开发进度
|
||||||
|
|
||||||
|
> 评估基线:2026-08-19 当前工作区代码、数据库初始化脚本、Docker 部署文件、前端页面和现有设计文档。
|
||||||
|
>
|
||||||
|
> 本文以代码实际情况为准。设计文档中已经提出但代码没有形成完整闭环的内容,统一标记为“部分完成”或“未完成”。
|
||||||
|
|
||||||
|
## 一、项目定位与总体结论
|
||||||
|
|
||||||
|
当前项目是一个面向多用户、多算力节点的模型训练与推理平台,主要链路为:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Vue 前端
|
||||||
|
|
|
||||||
|
FastAPI Backend API
|
||||||
|
|-- PostgreSQL:业务元数据、权限、任务状态、小型内容和预览数据
|
||||||
|
|-- Redis:会话、限流、短期缓存和任务辅助状态
|
||||||
|
|-- MinIO:模型、数据集、报告和大文件的统一对象存储
|
||||||
|
|-- Compute API / Agent:训练、推理、评测、模型合并和 GPU 执行
|
||||||
|
|
|
||||||
|
多台算力节点
|
||||||
|
```
|
||||||
|
|
||||||
|
整体判断:
|
||||||
|
|
||||||
|
| 范围 | 当前状态 | 结论 |
|
||||||
|
|---|---|---|
|
||||||
|
| 平台基础架构 | 基本完成 | 前后端、数据库、Redis、MinIO、Compute Agent 和 Docker 部署均已具备 |
|
||||||
|
| 核心业务闭环 | 基本可用 | 数据集、数据处理、数据转换、训练、模型、推理、评测均有页面和接口 |
|
||||||
|
| 多算力节点 | 部分完成 | 节点选择、GPU 分配和缓存准备已经接入,跨节点一致性和失败恢复仍需加强 |
|
||||||
|
| 权限治理 | 部分完成 | 登录、角色、权限码、ACL、审批、审计已实现,但完整的租户/项目隔离尚未闭环 |
|
||||||
|
| MinIO 统一存储 | 部分完成 | 大文件和模型已接入,仍存在兼容性的本地路径和部分数据双写/回退路径 |
|
||||||
|
| 生产可靠性 | 未完成 | 缓存容量治理、对象清理、流式上传、归档重试、备份和高可用尚未完成 |
|
||||||
|
| 前端体验 | 基本可用,需优化 | 构建问题已持续修复,但页面响应等待、首屏体积和部分错误提示仍需优化 |
|
||||||
|
|
||||||
|
## 二、已完成的功能
|
||||||
|
|
||||||
|
### 2.1 平台基础与部署
|
||||||
|
|
||||||
|
- 已建立 Vue 3 + TypeScript + Vite 前端工程。
|
||||||
|
- 已建立 FastAPI 后端服务,提供登录、平台管理和模型业务接口。
|
||||||
|
- 已建立 Compute API / Agent,用于连接算力节点并执行训练、推理、评测和模型处理任务。
|
||||||
|
- 已使用 PostgreSQL 保存核心业务数据,Redis 提供会话、限流和缓存能力。
|
||||||
|
- 已增加 MinIO 服务及 Backend 的 MinIO 配置,支持和 Compute Agent 分离部署。
|
||||||
|
- 已提供 `docker/app`、`docker/compute`、`docker/minio` 和 `docker/offline` 部署目录。
|
||||||
|
- 已考虑后端、算力服务、MinIO 分布在不同服务器时使用独立网络;Compute 节点访问 MinIO 需要配置所有节点都能访问的固定 IP 或 DNS。
|
||||||
|
- 离线部署目录已经同步后端、算力相关源码和初始化 SQL 的主要改造内容。
|
||||||
|
|
||||||
|
### 2.2 登录、用户和权限基础
|
||||||
|
|
||||||
|
- 用户登录、退出、当前用户信息和密码修改接口已经存在。
|
||||||
|
- 已有 Token 会话、Redis 会话记录和登录限流逻辑。
|
||||||
|
- 已建立用户、角色、权限码和角色权限关系。
|
||||||
|
- 已实现管理员、普通用户等基础角色分层。
|
||||||
|
- 已实现页面路由守卫、菜单过滤和前端按钮级权限的基础能力。
|
||||||
|
- 已建立资源 ACL 管理页面和相关接口,可对用户或角色授予资源级权限。
|
||||||
|
- 已建立审批模板、审批实例和审批步骤的基本数据模型与页面。
|
||||||
|
- 已建立运行日志、审计日志查询页面及审计记录写入机制。
|
||||||
|
- 已加入软删除相关字段和部分删除逻辑,避免直接物理删除业务资源。
|
||||||
|
|
||||||
|
### 2.3 算力节点与 GPU 资源
|
||||||
|
|
||||||
|
- 已实现算力节点的新增、编辑、启用、禁用、维护/删除、连通性测试和健康检查。
|
||||||
|
- 已实现节点列表、节点详情、节点副本/同步状态和 Compute Agent 连接。
|
||||||
|
- 已实现 GPU 信息发现、GPU 状态查询和队列查询。
|
||||||
|
- 已建立 `gpu_allocations`、`gpu_assignments`、调度锁等资源分配表。
|
||||||
|
- 训练任务已经支持选择调度节点和一张或多张 GPU,并在预检阶段校验资源可用性。
|
||||||
|
- Compute Agent 已支持训练、评测、推理和缓存准备等任务接口。
|
||||||
|
- 已存在资源副本和同步任务模型,用于记录节点侧资源同步状态。
|
||||||
|
|
||||||
|
### 2.4 数据集管理
|
||||||
|
|
||||||
|
- 已实现数据集创建、列表、详情、编辑、删除和文件上传。
|
||||||
|
- 已实现数据集文件下载、预览、记录列表和数据记录编辑入口。
|
||||||
|
- 已处理 JSON 与 JSONL 的记录数差异:JSON 数组按元素计数,JSONL 按有效行计数,避免把整个 JSON 文件误按行数统计。
|
||||||
|
- 已提供数据集版本列表、版本详情、创建版本、切换当前激活版本和删除版本接口。
|
||||||
|
- 已增加数据集文件、版本、数据记录等初始化表结构。
|
||||||
|
- 已支持数据集文件在数据库小内容和 MinIO 大文件之间按策略存储。
|
||||||
|
- 已在训练预检中检查数据集文件是否存在、是否可从 MinIO 获取以及是否能准备到目标算力节点。
|
||||||
|
|
||||||
|
### 2.5 数据处理与数据转换
|
||||||
|
|
||||||
|
- 已提供结构化数据、非结构化数据、外部数据源的处理创建流程。
|
||||||
|
- 已实现源文件上传、预览、分片/切分、生成、质量检查、去重和结果管理等数据处理流程。
|
||||||
|
- 已支持处理结果生成数据集或导入数据集版本。
|
||||||
|
- 已建立数据处理任务、源文件、预览项、结果等数据表。
|
||||||
|
- 已提供 JSON、JSONL 等数据格式转换页面和后端任务接口。
|
||||||
|
- 已将数据转换输出接入 MinIO/数据库分层存储:小型文本结果可存数据库,大文件存 MinIO。
|
||||||
|
- 已处理输出文件下载和转换结果元数据保存问题。
|
||||||
|
|
||||||
|
### 2.6 模型训练
|
||||||
|
|
||||||
|
- 已实现训练任务创建、配置预检、命令预览、启动、停止、重试和删除。
|
||||||
|
- 已接入 LLaMA-Factory 等训练适配逻辑。
|
||||||
|
- 已支持选择训练数据、基座模型、算力节点和 GPU。
|
||||||
|
- 已实现训练日志获取、训练任务概览、诊断信息、检查点和训练指标查询。
|
||||||
|
- 前端训练详情已经具备训练曲线解析和展示逻辑,日志轮询间隔已调整为 3 秒。
|
||||||
|
- 已增加 GPU 详情展示入口,包括显存和利用率等 Compute Agent 上报信息。
|
||||||
|
- 已支持训练任务的 MinIO 数据准备和目标算力节点缓存准备。
|
||||||
|
|
||||||
|
### 2.7 模型管理与权重合并
|
||||||
|
|
||||||
|
- 已实现在线模型/基座模型和训练模型的列表、创建、详情、用途修改和删除。
|
||||||
|
- 已建立模型、训练模型、模型血缘、模型产物和导出任务相关表。
|
||||||
|
- 已提供权重合并入口,能够根据训练任务准备基座模型和 Adapter,并提交 Compute Agent 执行合并。
|
||||||
|
- 已增加模型产物和 MinIO 对象关联字段。
|
||||||
|
- 合并结果能够在任务完成后归档到 MinIO 的设计和主要代码路径已经建立。
|
||||||
|
|
||||||
|
### 2.8 模型推理与模型对比
|
||||||
|
|
||||||
|
- 已实现推理模型列表、创建、详情和删除入口。
|
||||||
|
- 已实现模型加载、卸载、服务启动、服务状态查询和对话调用。
|
||||||
|
- 已实现模型对比任务及多模型聊天相关接口。
|
||||||
|
- 已增加推理失败重试、停止和资源释放的处理路径。
|
||||||
|
- 已支持根据页面选择的算力节点准备模型缓存,兼容训练时所选节点优先的业务要求。
|
||||||
|
- Compute Agent 已提供本地推理会话和模型缓存状态接口。
|
||||||
|
|
||||||
|
### 2.9 模型评测
|
||||||
|
|
||||||
|
- 已实现评测任务列表、创建、详情和删除。
|
||||||
|
- 已实现评测维度管理和评测规则配置页面。
|
||||||
|
- 已建立评测任务、评测维度、对比任务等数据库表。
|
||||||
|
- 已支持选择模型、数据集、评测维度、算力节点和 GPU 的基础流程。
|
||||||
|
- 已接入 Compute Agent 执行评测任务,并保存评测结果和报告相关元数据。
|
||||||
|
|
||||||
|
### 2.10 数据存储策略
|
||||||
|
|
||||||
|
- 已建立 `storage_objects`、`storage_cache_jobs` 等 MinIO 元数据和缓存任务表。
|
||||||
|
- 已建立 MinIO 对象上传、下载、预签名 URL 和节点缓存准备的主要接口。
|
||||||
|
- 已采用分层策略:
|
||||||
|
- 小型 JSON、JSONL、CSV、任务参数快照、预览数据保留在数据库,降低频繁预览的 MinIO 延迟。
|
||||||
|
- 模型权重、训练产物、评测报告和大文件使用 MinIO。
|
||||||
|
- 小型 PDF、DOCX、XLSX 仍优先存 MinIO,以保留原始二进制文件内容。
|
||||||
|
- 已增加 `data_convert_tasks.output_content`,用于保存小型转换结果,避免所有小结果都依赖 MinIO。
|
||||||
|
- Backend 和离线包中的 `000_full_init.sql` 已同步,当前两份初始化脚本内容一致。
|
||||||
|
|
||||||
|
## 三、部分完成、仍需完善的功能
|
||||||
|
|
||||||
|
### 3.1 MinIO 统一数据源尚未完全闭环
|
||||||
|
|
||||||
|
当前 MinIO 已成为模型、大文件和跨节点资源的主存储方向,但仍保留以下兼容路径:
|
||||||
|
|
||||||
|
- Compute Agent 仍有本地文件上传、导入本地模型和扫描本地模型目录的旧接口。
|
||||||
|
- 部分历史数据仍使用数据库中的 `content` 或 `output_content` 字段,这是当前已确认的小文件性能策略,不是错误,但必须统一记录来源、大小、校验值和版本。
|
||||||
|
- 大文件在部分代码路径中仍通过 `read()` 或 `put_bytes()` 一次性读入内存,未完成流式或分片上传。
|
||||||
|
- MinIO 对象和业务资源之间采用多态 `resource_type/resource_id` 关联,数据库没有直接外键,删除和数据一致性需要应用层保证。
|
||||||
|
- 删除业务资源后,对应 MinIO 对象的延迟清理、失败重试和孤儿对象扫描尚未形成完整闭环。
|
||||||
|
|
||||||
|
### 3.2 MinIO 预签名接口的权限边界需要加强
|
||||||
|
|
||||||
|
当前预签名接口已经存在,但 PUT 上传场景仍需要重点补强:
|
||||||
|
|
||||||
|
- 需要根据资源类型和资源 ID 校验当前用户的写权限,而不应只校验读取权限。
|
||||||
|
- 需要服务端生成并校验对象 Key,避免客户端任意写入其他用户或其他资源的对象路径。
|
||||||
|
- 需要增加上传完成确认接口,校验对象实际存在、大小和校验值后再写入业务表。
|
||||||
|
- 需要限制允许的 Bucket、Content-Type、大小和有效期。
|
||||||
|
- 需要记录预签名创建、上传完成、失败和过期事件,便于审计。
|
||||||
|
|
||||||
|
### 3.3 激活版本和跨节点资源版本仍需加强
|
||||||
|
|
||||||
|
数据集已经有 `active_version_id` 和版本表,但以下场景仍需补充:
|
||||||
|
|
||||||
|
- 训练、推理和评测必须只使用资源当前激活版本,并在任务创建时固化版本 ID。
|
||||||
|
- 同一文件名的不同版本不能只依靠文件名同步,应使用资源 ID、版本 ID 和对象 Key 组成唯一定位。
|
||||||
|
- 已创建任务在后续切换激活版本后,不能被意外切换到新版本。
|
||||||
|
- 需要为每个准备到算力节点的资源保存版本、对象 ETag/校验值和本地路径清单。
|
||||||
|
- 历史版本的数据库内容回退和 MinIO 对象回退逻辑还需要补全并增加测试。
|
||||||
|
|
||||||
|
### 3.4 权限 2.0 尚未完全落地
|
||||||
|
|
||||||
|
已有用户、角色、权限码、ACL、审批和审计基础,但仍存在以下差距:
|
||||||
|
|
||||||
|
- 租户、用户、资源、算力节点、模型、数据集之间的隔离规则没有全部在 SQL 查询层统一执行。
|
||||||
|
- 项目空间设计已经讨论过取消,但数据库中仍保留 `projects`、`project_members` 等历史结构,需要明确兼容策略和最终迁移方式。
|
||||||
|
- 训练创建的模型、数据集和训练任务之间的联合权限约束还没有完全统一。
|
||||||
|
- 评测、推理、模型合并、导出、缓存准备等动作需要逐一校验资源读权限和操作权限。
|
||||||
|
- 前端按钮权限已经有基础实现,但不能替代后端鉴权;仍需要对所有关键动作进行后端默认拒绝校验。
|
||||||
|
- 审批拦截范围、管理员豁免规则和跨租户资源访问规则需要形成可执行矩阵。
|
||||||
|
|
||||||
|
### 3.5 模型合并、导出和评测报告闭环不足
|
||||||
|
|
||||||
|
- 权重合并前自动准备 Base Model 和 Adapter 的主要路径已建立,但失败时的清理、重试和幂等性仍需加强。
|
||||||
|
- 合并结果归档到 MinIO 的逻辑主要依赖任务完成轮询,服务重启或轮询中断时可能需要补偿扫描。
|
||||||
|
- 模型导出任务目前有查询模型和表结构,但完整的创建、执行、进度、失败重试和下载闭环尚未完成。
|
||||||
|
- 评测结果和报告字段已经存在,但报告对象归档、报告下载、报告版本和报告与任务的稳定关联仍需验证。
|
||||||
|
- 评测指标配置和执行器返回指标之间仍需要强类型映射,避免前端显示为通用的 `custom`。
|
||||||
|
|
||||||
|
### 3.6 GPU 资源分配需要统一到所有任务类型
|
||||||
|
|
||||||
|
- 训练已经有较完整的节点/GPU 选择和预检流程。
|
||||||
|
- 推理和评测已经出现节点选择、缓存准备和 GPU 选择的接入代码,但还需要确认从页面选择到 Compute Agent 启动参数、进程环境变量和释放逻辑的全链路生效。
|
||||||
|
- 需要防止同一张 GPU 被多个任务绕过调度锁重复占用。
|
||||||
|
- 需要处理服务异常退出、Backend 重启、Compute Agent 重启后的分配回收和状态对账。
|
||||||
|
- 训练详情中的显存使用量、GPU 使用率等指标依赖 Compute Agent 上报,仍需要校验采样时间、单位、空值和任务对应关系。
|
||||||
|
|
||||||
|
## 四、尚未完成的功能
|
||||||
|
|
||||||
|
以下功能在当前代码中没有形成可验收的完整闭环,或仍处于设计/基础代码阶段:
|
||||||
|
|
||||||
|
1. **完整的租户隔离和资源继承模型**:所有列表、详情、下载、缓存、训练、推理、评测和导出接口都需要统一的租户范围过滤。
|
||||||
|
2. **项目取消后的正式数据迁移方案**:需要决定历史项目数据如何归属到用户或租户,并提供一次性迁移脚本和回滚方案。
|
||||||
|
3. **预签名上传完成确认和对象校验**:包括 Key 白名单、ACL、大小限制、哈希/ETag 和状态回写。
|
||||||
|
4. **MinIO 对象生命周期管理**:软删除后的延迟删除、失败重试、孤儿对象扫描、对象引用检查和管理员清理入口。
|
||||||
|
5. **Compute Agent 缓存治理**:容量上限、LRU/TTL、运行任务保护、磁盘占用监控、缓存清单和版本校验。
|
||||||
|
6. **统一的资源归档编排器**:训练、合并、评测和推理相关产物需要支持断点恢复、幂等重试和服务重启补偿。
|
||||||
|
7. **模型导出完整流程**:导出任务创建、格式/量化参数、进度、失败重试、MinIO 归档和下载权限。
|
||||||
|
8. **流式和分片文件传输**:避免大文件上传、下载和对象复制时将完整内容读入 Backend 或 Compute Agent 内存。
|
||||||
|
9. **生产级 MinIO 安全和高可用**:默认密钥替换、TLS、网络访问控制、管理员 Console 隔离、容量监控、备份和恢复。
|
||||||
|
10. **完整的端到端测试和持续集成**:至少覆盖单节点、多节点、多 GPU、跨用户、跨租户、版本切换、MinIO 不可用和服务重启恢复。
|
||||||
|
11. **统一数据库迁移体系**:当前初始化 SQL 适合新库初始化,但尚未替代正式的版本化迁移工具;已有数据库更新仍需要明确迁移脚本和执行记录。
|
||||||
|
|
||||||
|
## 五、需要优化的功能
|
||||||
|
|
||||||
|
### 5.1 后端响应性能
|
||||||
|
|
||||||
|
- 页面列表接口需要避免每条记录重复查询用户、资源、MinIO 元数据和 Compute 节点状态。
|
||||||
|
- MinIO 的 Bucket 检查、对象 Head 和预签名生成应使用连接复用、短期缓存和批量查询。
|
||||||
|
- 训练、推理、评测页面不应通过过短间隔轮询大量详情接口,应按任务状态动态退避,并在完成后停止轮询。
|
||||||
|
- 对 dashboard、节点健康、GPU 状态等高频数据应区分实时数据和缓存数据。
|
||||||
|
- 后端日志轮询和健康检查日志需要继续降噪,仅在状态变化、失败或达到较长周期时输出。
|
||||||
|
|
||||||
|
### 5.2 前端加载和交互
|
||||||
|
|
||||||
|
- 列表页面应区分首屏 loading、刷新 loading、操作 loading,避免整页长时间无反馈。
|
||||||
|
- 推理、评测、训练详情应使用统一的任务状态刷新策略和超时提示。
|
||||||
|
- 前端仍有 FontAwesome 在线资源解析警告,应清理对外部网络文件的依赖,保证离线环境打开速度。
|
||||||
|
- 应继续拆分首屏大体积 chunk,并减少一次性加载不相关页面组件。
|
||||||
|
- GPU 选择组件需要明确显示空闲、占用、不可达、预留和已分配状态。
|
||||||
|
- 错误提示应携带资源名称、节点名称、版本和下一步处理建议,减少只显示 500/404 的情况。
|
||||||
|
|
||||||
|
### 5.3 训练、推理和评测可靠性
|
||||||
|
|
||||||
|
- 所有任务创建前应执行同一套资源权限、版本存在性、MinIO 可用性和 GPU 原子分配校验。
|
||||||
|
- 任务创建接口应支持幂等键,避免前端重复点击造成重复任务。
|
||||||
|
- 节点不可达时应快速失败或进入可见的等待状态,不能让页面长时间无反馈。
|
||||||
|
- 失败重试应区分网络瞬时失败、资源不足、模型文件缺失、参数错误和执行器失败。
|
||||||
|
- 任务停止后必须释放 GPU 分配、推理端口、缓存锁和临时目录。
|
||||||
|
|
||||||
|
### 5.4 数据和模型一致性
|
||||||
|
|
||||||
|
- 每个对象都应保存大小、校验值、版本 ID、来源、创建者、租户和引用状态。
|
||||||
|
- 数据库中的小文件内容和 MinIO 对象不能同时被当作可独立修改的主副本;需要明确唯一写入入口。
|
||||||
|
- 数据集激活版本变更需要留下审计记录,并影响后续任务创建但不改变已创建任务。
|
||||||
|
- 模型权重、Adapter、合并结果和导出结果需要形成完整血缘关系。
|
||||||
|
|
||||||
|
## 六、数据库和初始化脚本状态
|
||||||
|
|
||||||
|
当前 `backend/app/db/sql/000_full_init.sql` 已包含以下主要类别:
|
||||||
|
|
||||||
|
- 用户、模型、训练模型、模型血缘、模型产物、模型导出任务。
|
||||||
|
- 数据集、数据集文件、数据集版本、数据集记录。
|
||||||
|
- 算力节点、GPU、GPU 分配、调度锁、Compute Job。
|
||||||
|
- 资源副本、资源同步任务、MinIO 对象、缓存任务。
|
||||||
|
- 评测任务、评测维度、模型对比任务。
|
||||||
|
- 租户、项目兼容表、项目成员、角色、会话、ACL。
|
||||||
|
- 审批模板、审批实例、审批步骤、审计日志、留存策略。
|
||||||
|
- 数据处理任务、源文件、预览项、处理结果、数据转换任务。
|
||||||
|
|
||||||
|
已确认的近期字段包括:
|
||||||
|
|
||||||
|
- `model_artifacts.storage_object_id`
|
||||||
|
- `model_artifacts.storage_backend`
|
||||||
|
- `dataset_files.storage_object_id`
|
||||||
|
- `data_convert_tasks.output_content`
|
||||||
|
- `data_convert_tasks.output_storage_object_id`
|
||||||
|
- `data_convert_tasks.storage_backend`
|
||||||
|
- `eval_tasks.report_storage_object_id`
|
||||||
|
|
||||||
|
离线包中的 `docker/offline/src/backend/app/db/sql/000_full_init.sql` 应与主工程初始化脚本保持同步。需要注意:
|
||||||
|
|
||||||
|
- 初始化 SQL 主要用于新数据库或新数据卷;已有数据库不能仅靠重启容器自动获得全部新字段。
|
||||||
|
- 生产/测试数据库需要执行可追踪的迁移脚本,并在迁移前备份或生成结构快照。
|
||||||
|
- `ensure_schema` 类运行时补字段逻辑只能作为兼容兜底,不能替代正式迁移。
|
||||||
|
- 后续如果正式移除项目设计,需要先完成数据归属迁移,再决定是否删除历史表,不能直接从初始化 SQL 中删除表。
|
||||||
|
|
||||||
|
## 七、当前验证结果
|
||||||
|
|
||||||
|
已完成的静态和局部验证:
|
||||||
|
|
||||||
|
- Backend 和离线 Backend 源码 `compileall` 检查通过。
|
||||||
|
- MinIO 分层策略冒烟验证通过:小型 JSON/JSONL 可落数据库,小型二进制和超过阈值的内容进入 MinIO。
|
||||||
|
- 主工程和离线包初始化 SQL 已做同步检查,内容一致。
|
||||||
|
- 前端此前已完成 `npm run build` 类型错误修复,构建剩余问题主要是非阻断的资源/分包警告。
|
||||||
|
- 已对训练日志、数据集 JSON/JSONL 统计、MinIO 资源准备等重点链路进行过问题修复。
|
||||||
|
|
||||||
|
当前不能据此宣称“全量功能测试通过”:
|
||||||
|
|
||||||
|
- 现有部分自动化测试仍保留旧的本地文件或旧 MinIO 行为假设,需要按当前分层存储策略更新。
|
||||||
|
- WSL Docker 运行时验证受当前环境的 `E_ACCESSDENIED` 影响,不能在本次文档生成时完成全部容器健康、数据库字段和跨节点测试。
|
||||||
|
- 多节点、多 GPU、MinIO 临时不可用、服务重启恢复和跨用户权限测试仍需要在可用运行环境中执行。
|
||||||
|
|
||||||
|
## 八、下一阶段开发计划
|
||||||
|
|
||||||
|
### P0:安全与数据正确性
|
||||||
|
|
||||||
|
1. 完善 MinIO 预签名 PUT 的资源写权限、对象 Key 白名单、大小/类型限制和上传完成确认。
|
||||||
|
2. 统一任务创建时的资源版本固化,训练、推理、评测只使用已授权的激活版本快照。
|
||||||
|
3. 逐一补齐评测、推理、模型合并、模型导出、缓存准备的后端权限校验和审计记录。
|
||||||
|
4. 完成租户隔离查询范围,清理或兼容历史项目字段,补充数据迁移脚本。
|
||||||
|
|
||||||
|
### P1:跨节点可靠性
|
||||||
|
|
||||||
|
1. 建立资源清单/manifest,记录 MinIO 对象版本、校验值、目标节点路径和缓存状态。
|
||||||
|
2. 完善训练、合并、评测和推理的准备、执行、归档、失败重试和服务重启补偿。
|
||||||
|
3. 完善 GPU 原子分配、异常回收、节点重连对账和任务释放。
|
||||||
|
4. 增加缓存容量、TTL/LRU、运行任务保护和磁盘占用监控。
|
||||||
|
5. 增加 MinIO 对象引用清理、孤儿对象扫描和软删除回收任务。
|
||||||
|
|
||||||
|
### P2:性能与用户体验
|
||||||
|
|
||||||
|
1. 优化页面列表接口和高频轮询,采用批量查询、短期缓存和动态退避。
|
||||||
|
2. 将大文件上传/下载/复制改为流式或分片传输。
|
||||||
|
3. 统一前端任务状态组件、loading、超时、重试和错误诊断信息。
|
||||||
|
4. 处理前端离线资源警告,继续拆分首屏 chunk。
|
||||||
|
5. 统一 GPU 状态展示及训练指标采样时间、单位和空值处理。
|
||||||
|
|
||||||
|
### P3:工程化和上线准备
|
||||||
|
|
||||||
|
1. 建立正式数据库版本迁移机制和离线升级脚本。
|
||||||
|
2. 增加 CI:前端类型检查/构建、Backend 单元测试、Compute Agent 测试、SQL 新库初始化测试。
|
||||||
|
3. 增加多节点端到端测试和 MinIO 故障注入测试。
|
||||||
|
4. 完善 MinIO TLS、密钥管理、网络隔离、监控、备份和恢复方案。
|
||||||
|
5. 建立生产运行手册,包括首次部署、升级、回滚、数据库迁移、对象清理和故障处理。
|
||||||
|
|
||||||
|
## 九、阶段验收标准
|
||||||
|
|
||||||
|
完成下一阶段后,至少应满足:
|
||||||
|
|
||||||
|
- 用户只能看到和操作其所属租户授权的模型、数据集、训练任务、推理服务和评测任务。
|
||||||
|
- 任何任务创建都能明确记录用户、租户、资源版本、算力节点、GPU 列表和 MinIO 对象版本。
|
||||||
|
- 同一个节点的同一张 GPU 不能被两个活动任务同时分配。
|
||||||
|
- MinIO 临时不可用时,任务进入可解释的等待/失败状态,并能按策略重试,页面不会无限等待。
|
||||||
|
- Backend 或 Compute Agent 重启后,任务、缓存、GPU 分配和归档状态可以对账恢复。
|
||||||
|
- 训练、合并、评测和推理产物都能在 MinIO 中找到,并且可以通过权限校验后的接口下载或使用。
|
||||||
|
- 删除资源后不会继续出现在普通列表中,关联对象能够按引用状态延迟清理并留下审计记录。
|
||||||
|
- 新数据库初始化和已有数据库迁移后,所有业务接口不再因为缺表或缺字段启动失败。
|
||||||
|
- 离线部署不依赖外部字体、图标或 CDN,前端首屏和核心业务操作在无网络环境下可用。
|
||||||
|
|
||||||
|
## 十、相关文件索引
|
||||||
|
|
||||||
|
- 平台架构:[platform-architecture-requirements.md](./platform-architecture-requirements.md)
|
||||||
|
- 权限设计:[permissions-design.md](./permissions-design.md)
|
||||||
|
- MinIO 与 Compute 缓存方案:[minio-compute-cache-plan.md](./minio-compute-cache-plan.md)
|
||||||
|
- 平台治理菜单设计:[platform-governance-menu-design.md](./platform-governance-menu-design.md)
|
||||||
|
- 数据处理设计:[data-process-design.md](./data-process-design.md)
|
||||||
|
- 数据库初始化脚本:[../backend/app/db/sql/000_full_init.sql](../backend/app/db/sql/000_full_init.sql)
|
||||||
|
- 离线部署目录:[../docker/offline](../docker/offline)
|
||||||
|
|
||||||
@@ -20,6 +20,8 @@ export interface AuditQuery {
|
|||||||
actor_id?: string
|
actor_id?: string
|
||||||
action?: string
|
action?: string
|
||||||
target_type?: string
|
target_type?: string
|
||||||
|
target_id?: string
|
||||||
|
keyword?: string
|
||||||
start_time?: string
|
start_time?: string
|
||||||
end_time?: string
|
end_time?: string
|
||||||
limit?: number
|
limit?: number
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ import type {
|
|||||||
DataProcessPublishResult,
|
DataProcessPublishResult,
|
||||||
DataProcessQualityScore,
|
DataProcessQualityScore,
|
||||||
DataProcessResult,
|
DataProcessResult,
|
||||||
|
DataProcessResultBatchEvaluatePayload,
|
||||||
|
DataProcessResultBatchEvaluateResult,
|
||||||
DataProcessResultBatchRegeneratePayload,
|
DataProcessResultBatchRegeneratePayload,
|
||||||
DataProcessResultBatchRegenerateResult,
|
DataProcessResultBatchRegenerateResult,
|
||||||
DataProcessResultRegeneratePayload,
|
DataProcessResultRegeneratePayload,
|
||||||
@@ -320,5 +322,14 @@ export const regenerateDataProcessResults = (
|
|||||||
{ timeout: 240_000 },
|
{ timeout: 240_000 },
|
||||||
)
|
)
|
||||||
|
|
||||||
|
export const evaluateDataProcessResults = (
|
||||||
|
taskId: string | number,
|
||||||
|
payload: DataProcessResultBatchEvaluatePayload,
|
||||||
|
) => post<DataProcessResultBatchEvaluateResult>(
|
||||||
|
`/data-process/${encodeURIComponent(taskId)}/results/evaluate-batch`,
|
||||||
|
payload,
|
||||||
|
{ timeout: 240_000 },
|
||||||
|
)
|
||||||
|
|
||||||
export const publishDataProcess = (taskId: string | number, payload: DataProcessPublishPayload) =>
|
export const publishDataProcess = (taskId: string | number, payload: DataProcessPublishPayload) =>
|
||||||
post<DataProcessPublishResult>(`/data-process/${encodeURIComponent(taskId)}/publish`, payload)
|
post<DataProcessPublishResult>(`/data-process/${encodeURIComponent(taskId)}/publish`, payload)
|
||||||
|
|||||||
@@ -42,12 +42,23 @@ export interface FineTunePreflightResult {
|
|||||||
sync_results?: Array<Record<string, unknown>>
|
sync_results?: Array<Record<string, unknown>>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FineTuneGpuStatus {
|
||||||
|
source: string
|
||||||
|
items: Array<Record<string, unknown>>
|
||||||
|
selected_gpus: number[]
|
||||||
|
error?: string
|
||||||
|
}
|
||||||
|
|
||||||
/** 训练任务列表 */
|
/** 训练任务列表 */
|
||||||
export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
|
export const getFineTuneList = () => get<FineTuneTask[]>('/fine-tune')
|
||||||
|
|
||||||
/** 训练任务详情 */
|
/** 训练任务详情 */
|
||||||
export const getFineTune = (id: string | number) => get<FineTuneTask>(`/fine-tune/${id}`)
|
export const getFineTune = (id: string | number) => get<FineTuneTask>(`/fine-tune/${id}`)
|
||||||
|
|
||||||
|
/** 获取任务所在 Compute 节点的实时 GPU 指标 */
|
||||||
|
export const getFineTuneGpuStatus = (id: string | number) =>
|
||||||
|
get<FineTuneGpuStatus>(`/fine-tune/${id}/gpu-status`)
|
||||||
|
|
||||||
/** 任务名查重 */
|
/** 任务名查重 */
|
||||||
export const checkFineTuneName = (name: string) =>
|
export const checkFineTuneName = (name: string) =>
|
||||||
get<{ exists: boolean }>('/fine-tune/check-name', { name })
|
get<{ exists: boolean }>('/fine-tune/check-name', { name })
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ const activeMenu = computed(() => {
|
|||||||
if (seg === 'training-log') return 'fine-tune'
|
if (seg === 'training-log') return 'fine-tune'
|
||||||
// 维度管理归到模型评测
|
// 维度管理归到模型评测
|
||||||
if (route.path.includes('model-eval/dimension')) return 'model-eval'
|
if (route.path.includes('model-eval/dimension')) return 'model-eval'
|
||||||
|
// 组织与权限承接用户、租户和历史治理入口
|
||||||
|
if (route.path.startsWith('/organization') || route.path.startsWith('/user-settings') || route.path.startsWith('/tenants')) return 'organization'
|
||||||
|
// 运行日志承接审计和操作诊断两个历史入口
|
||||||
|
if (route.path.startsWith('/logs') || route.path.startsWith('/audit-logs') || route.path.startsWith('/operation-logs')) return 'logs'
|
||||||
// 对比对话归到模型推理
|
// 对比对话归到模型推理
|
||||||
if (route.path.startsWith('/model-compare/chat')) return 'model-inference'
|
if (route.path.startsWith('/model-compare/chat')) return 'model-inference'
|
||||||
// 合并权重归到模型管理
|
// 合并权重归到模型管理
|
||||||
@@ -77,21 +81,16 @@ const menuGroups: MenuGroup[] = [
|
|||||||
{
|
{
|
||||||
title: '平台治理',
|
title: '平台治理',
|
||||||
items: [
|
items: [
|
||||||
{ key: 'tenants', label: '租户管理', icon: 'fa-building', to: '/tenants', permission: 'user-settings' },
|
{ key: 'organization', label: '组织与权限', icon: 'fa-users', to: '/organization', permission: 'user-settings' },
|
||||||
{ key: 'projects', label: '项目空间', icon: 'fa-folder', to: '/projects', permission: 'user-settings' },
|
|
||||||
{ key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' },
|
{ key: 'resource-acl', label: '资源授权', icon: 'fa-key', to: '/resource-acl', permission: 'user-settings' },
|
||||||
{ key: 'audit-logs', label: '审计日志', icon: 'fa-history', to: '/audit-logs', permission: 'user-settings' },
|
|
||||||
{ key: 'operation-logs', label: '操作日志', icon: 'fa-list', to: '/operation-logs', permission: 'user-settings' },
|
|
||||||
{ key: 'approval-templates', label: '审批模板', icon: 'fa-list-alt', to: '/approval-templates', permission: 'user-settings' },
|
|
||||||
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' },
|
{ key: 'approval-instances', label: '审批中心', icon: 'fa-check-square', to: '/approval-instances', permission: 'user-settings' },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '系统设置',
|
title: '系统设置',
|
||||||
items: [
|
items: [
|
||||||
{ key: 'user-settings', label: '用户设置', icon: 'fa-users', to: '/user-settings', permission: 'user-settings' },
|
|
||||||
{ key: 'hardware', label: '平台性能', icon: 'fa-bar-chart', to: '/hardware', permission: 'hardware' },
|
{ key: 'hardware', label: '平台性能', icon: 'fa-bar-chart', to: '/hardware', permission: 'hardware' },
|
||||||
{ key: 'logs', label: '查看日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' },
|
{ key: 'logs', label: '运行日志', icon: 'fa-file-text', to: '/logs', permission: 'logs' },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
@@ -101,9 +100,9 @@ const menuGroups: MenuGroup[] = [
|
|||||||
*
|
*
|
||||||
* 1. admin 用户:可以看到所有菜单
|
* 1. admin 用户:可以看到所有菜单
|
||||||
* 2. 非 admin 用户:
|
* 2. 非 admin 用户:
|
||||||
* - 默认可见所有业务菜单(模型训练、评测、推理、数据集、数据处理、转换、性能、日志等)
|
* - 默认可见所有业务菜单(模型训练、评测、推理、数据集、数据处理、转换、性能、运行日志等)
|
||||||
* - 仅以下菜单对非 admin 不可见:
|
* - 仅以下菜单对非 admin 不可见:
|
||||||
* - user-settings(用户设置、租户管理、项目空间、审批模板/中心、审计日志)
|
* - user-settings(组织与权限、资源授权、审批;运行日志中的审计/诊断页签)
|
||||||
* - compute(算力节点/GPU 分配)
|
* - compute(算力节点/GPU 分配)
|
||||||
*
|
*
|
||||||
* 注意:移除了旧的权限码(permission code)过滤逻辑,
|
* 注意:移除了旧的权限码(permission code)过滤逻辑,
|
||||||
|
|||||||
@@ -3,18 +3,21 @@
|
|||||||
*/
|
*/
|
||||||
import { use } from 'echarts/core'
|
import { use } from 'echarts/core'
|
||||||
import { CanvasRenderer } from 'echarts/renderers'
|
import { CanvasRenderer } from 'echarts/renderers'
|
||||||
import { BarChart, PieChart } from 'echarts/charts'
|
import { BarChart, PieChart, RadarChart } from 'echarts/charts'
|
||||||
import {
|
import {
|
||||||
GridComponent,
|
GridComponent,
|
||||||
TooltipComponent,
|
TooltipComponent,
|
||||||
LegendComponent,
|
LegendComponent,
|
||||||
|
RadarComponent,
|
||||||
} from 'echarts/components'
|
} from 'echarts/components'
|
||||||
|
|
||||||
use([
|
use([
|
||||||
CanvasRenderer,
|
CanvasRenderer,
|
||||||
BarChart,
|
BarChart,
|
||||||
PieChart,
|
PieChart,
|
||||||
|
RadarChart,
|
||||||
GridComponent,
|
GridComponent,
|
||||||
TooltipComponent,
|
TooltipComponent,
|
||||||
LegendComponent,
|
LegendComponent,
|
||||||
|
RadarComponent,
|
||||||
])
|
])
|
||||||
|
|||||||
@@ -32,11 +32,17 @@ const routes: RouteRecordRaw[] = [
|
|||||||
meta: { title: '服务看板' },
|
meta: { title: '服务看板' },
|
||||||
},
|
},
|
||||||
// 平台治理
|
// 平台治理
|
||||||
|
{
|
||||||
|
path: 'organization',
|
||||||
|
name: 'organization',
|
||||||
|
component: () => import('@/views/governance/OrganizationPermissionView.vue'),
|
||||||
|
meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' },
|
||||||
|
},
|
||||||
{
|
{
|
||||||
path: 'tenants',
|
path: 'tenants',
|
||||||
name: 'tenants',
|
name: 'tenants',
|
||||||
component: () => import('@/views/tenants/TenantListView.vue'),
|
redirect: '/organization?tab=tenants',
|
||||||
meta: { title: '租户管理', permission: 'user-settings' },
|
meta: { title: '租户与配额', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'tenants/:id',
|
path: 'tenants/:id',
|
||||||
@@ -47,37 +53,37 @@ const routes: RouteRecordRaw[] = [
|
|||||||
{
|
{
|
||||||
path: 'projects',
|
path: 'projects',
|
||||||
name: 'projects',
|
name: 'projects',
|
||||||
component: () => import('@/views/projects/ProjectListView.vue'),
|
redirect: '/organization?tab=users',
|
||||||
meta: { title: '项目空间', permission: 'user-settings' },
|
meta: { title: '组织与权限', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'projects/:id',
|
path: 'projects/:id',
|
||||||
name: 'project-detail',
|
name: 'project-detail',
|
||||||
component: () => import('@/views/projects/ProjectDetailView.vue'),
|
redirect: '/organization?tab=users',
|
||||||
meta: { title: '项目详情', permission: 'user-settings' },
|
meta: { title: '组织与权限', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'audit-logs',
|
path: 'audit-logs',
|
||||||
name: 'audit-logs',
|
name: 'audit-logs',
|
||||||
component: () => import('@/views/audit/AuditLogView.vue'),
|
redirect: '/logs?tab=audit',
|
||||||
meta: { title: '审计日志', permission: 'user-settings' },
|
meta: { title: '运行日志', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'operation-logs',
|
path: 'operation-logs',
|
||||||
name: 'operation-logs',
|
name: 'operation-logs',
|
||||||
component: () => import('@/views/audit/OperationLogView.vue'),
|
redirect: '/logs?tab=operations',
|
||||||
meta: { title: '操作日志', permission: 'user-settings' },
|
meta: { title: '运行日志', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'approval-templates',
|
path: 'approval-templates',
|
||||||
name: 'approval-templates',
|
name: 'approval-templates',
|
||||||
component: () => import('@/views/approvals/ApprovalTemplateView.vue'),
|
redirect: '/approval-instances?tab=strategies',
|
||||||
meta: { title: '审批模板', permission: 'user-settings' },
|
meta: { title: '审批中心', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'approval-instances',
|
path: 'approval-instances',
|
||||||
name: 'approval-instances',
|
name: 'approval-instances',
|
||||||
component: () => import('@/views/approvals/ApprovalInstanceView.vue'),
|
component: () => import('@/views/approvals/ApprovalCenterView.vue'),
|
||||||
meta: { title: '审批中心', permission: 'user-settings' },
|
meta: { title: '审批中心', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -302,14 +308,14 @@ const routes: RouteRecordRaw[] = [
|
|||||||
{
|
{
|
||||||
path: 'logs',
|
path: 'logs',
|
||||||
name: 'logs',
|
name: 'logs',
|
||||||
component: () => import('@/views/system/LogsView.vue'),
|
component: () => import('@/views/system/RuntimeLogsView.vue'),
|
||||||
meta: { title: '查看日志' },
|
meta: { title: '运行日志', pageSurface: 'self', permission: 'logs' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'user-settings',
|
path: 'user-settings',
|
||||||
name: 'user-settings',
|
name: 'user-settings',
|
||||||
component: () => import('@/views/system/UserSettingsView.vue'),
|
redirect: '/organization?tab=users',
|
||||||
meta: { title: '用户设置', pageSurface: 'self', permission: 'user-settings' },
|
meta: { title: '组织与权限', pageSurface: 'self', permission: 'user-settings' },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'user-settings/create',
|
path: 'user-settings/create',
|
||||||
@@ -357,6 +363,7 @@ const permissionBySegment: Record<string, PermissionCode> = {
|
|||||||
tools: 'data-convert',
|
tools: 'data-convert',
|
||||||
hardware: 'hardware',
|
hardware: 'hardware',
|
||||||
logs: 'logs',
|
logs: 'logs',
|
||||||
|
organization: 'user-settings',
|
||||||
'user-settings': 'user-settings',
|
'user-settings': 'user-settings',
|
||||||
tenants: 'user-settings',
|
tenants: 'user-settings',
|
||||||
projects: 'user-settings',
|
projects: 'user-settings',
|
||||||
@@ -379,7 +386,7 @@ function requiredPermission(path: string, explicit?: unknown) {
|
|||||||
// 权限控制规则(基于 governance-user-guide.md 设计):
|
// 权限控制规则(基于 governance-user-guide.md 设计):
|
||||||
// - admin 用户:可以访问所有页面
|
// - admin 用户:可以访问所有页面
|
||||||
// - 非 admin 用户:默认可访问所有业务页面(训练、评测、推理、数据等)
|
// - 非 admin 用户:默认可访问所有业务页面(训练、评测、推理、数据等)
|
||||||
// 仅以下页面限制 admin 访问:user-settings、compute(算力节点)
|
// 仅治理与资源管理页面限制 admin 访问:organization、user-settings、compute
|
||||||
router.beforeEach((to, _from, next) => {
|
router.beforeEach((to, _from, next) => {
|
||||||
if (!to.meta.public) routeLoading.value = true
|
if (!to.meta.public) routeLoading.value = true
|
||||||
const auth = useAuthStore()
|
const auth = useAuthStore()
|
||||||
@@ -404,7 +411,7 @@ router.beforeEach((to, _from, next) => {
|
|||||||
if (!to.meta.skipPermission) {
|
if (!to.meta.skipPermission) {
|
||||||
const permission = requiredPermission(to.path, to.meta.permission)
|
const permission = requiredPermission(to.path, to.meta.permission)
|
||||||
// 仅限制管理员专属页面的访问权限
|
// 仅限制管理员专属页面的访问权限
|
||||||
// user-settings(用户设置、租户管理、项目空间、审批、审计日志)仅 admin 可访问
|
// user-settings(组织与权限、资源授权、审批中心、运行日志)仅 admin 可访问
|
||||||
if (permission === 'user-settings' && !auth.isAdmin) {
|
if (permission === 'user-settings' && !auth.isAdmin) {
|
||||||
next({ name: 'permission-denied', replace: true })
|
next({ name: 'permission-denied', replace: true })
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -398,6 +398,52 @@ export interface DataProcessResultBatchRegenerateResult {
|
|||||||
failures: DataProcessResultBatchRegenerateFailure[]
|
failures: DataProcessResultBatchRegenerateFailure[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DataProcessResultBatchEvaluateItem {
|
||||||
|
result_id: string
|
||||||
|
expected_updated_at: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessResultBatchEvaluatePayload {
|
||||||
|
items: DataProcessResultBatchEvaluateItem[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessResultBatchEvaluateFailure {
|
||||||
|
result_id: string
|
||||||
|
code: 'conflict' | 'skipped' | 'evaluation_failed' | 'internal_error'
|
||||||
|
message: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessResultBatchEvaluateResult {
|
||||||
|
batch_id: string
|
||||||
|
total: number
|
||||||
|
succeeded: number
|
||||||
|
failed: number
|
||||||
|
duration_ms: number
|
||||||
|
items: DataProcessResult[]
|
||||||
|
failures: DataProcessResultBatchEvaluateFailure[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessQualitySemantic {
|
||||||
|
question_answer?: number
|
||||||
|
answer_source?: number
|
||||||
|
overall?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessQualityJudge {
|
||||||
|
scores?: Record<string, number>
|
||||||
|
overall?: number
|
||||||
|
reason?: string
|
||||||
|
issues?: string[]
|
||||||
|
model?: string
|
||||||
|
output_type?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataProcessQualityLayers {
|
||||||
|
rule?: number | null
|
||||||
|
semantic?: number | null
|
||||||
|
judge?: number | null
|
||||||
|
}
|
||||||
|
|
||||||
export interface DataProcessQualityScore {
|
export interface DataProcessQualityScore {
|
||||||
overall?: number
|
overall?: number
|
||||||
completeness?: number
|
completeness?: number
|
||||||
@@ -408,6 +454,11 @@ export interface DataProcessQualityScore {
|
|||||||
is_valid?: boolean
|
is_valid?: boolean
|
||||||
flags?: string[]
|
flags?: string[]
|
||||||
fingerprint?: string
|
fingerprint?: string
|
||||||
|
semantic?: DataProcessQualitySemantic | null
|
||||||
|
judge?: DataProcessQualityJudge | null
|
||||||
|
layers?: DataProcessQualityLayers | null
|
||||||
|
evaluated?: boolean
|
||||||
|
evaluated_at?: string | null
|
||||||
source_pages?: number[]
|
source_pages?: number[]
|
||||||
heading_path?: string[]
|
heading_path?: string[]
|
||||||
source_locator?: DataProcessSourceLocator
|
source_locator?: DataProcessSourceLocator
|
||||||
|
|||||||
@@ -218,6 +218,8 @@ export interface LoadedModel {
|
|||||||
port?: number
|
port?: number
|
||||||
node_id?: string
|
node_id?: string
|
||||||
node_name?: string
|
node_name?: string
|
||||||
|
gpu_indices?: number[]
|
||||||
|
gpus?: number[]
|
||||||
error?: string
|
error?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -239,6 +241,8 @@ export interface CompareModelRef {
|
|||||||
gpu_id: number
|
gpu_id: number
|
||||||
node_id?: string
|
node_id?: string
|
||||||
node_name?: string
|
node_name?: string
|
||||||
|
gpu_indices?: number[]
|
||||||
|
gpus?: number[]
|
||||||
source?: string
|
source?: string
|
||||||
port?: number
|
port?: number
|
||||||
}
|
}
|
||||||
@@ -282,7 +286,9 @@ export interface StartEvalPayload {
|
|||||||
eval_task_name: string
|
eval_task_name: string
|
||||||
eval_type: EvalType
|
eval_type: EvalType
|
||||||
model_id: string | number
|
model_id: string | number
|
||||||
gpu_id: string | number
|
gpu_id: string | number | string[]
|
||||||
|
gpu_indices?: number[]
|
||||||
|
gpus?: number[]
|
||||||
compute_node_id?: string
|
compute_node_id?: string
|
||||||
dataset_id: string | number
|
dataset_id: string | number
|
||||||
dimension_id: string | number
|
dimension_id: string | number
|
||||||
|
|||||||
66
frontend/src/views/approvals/ApprovalCenterView.vue
Normal file
66
frontend/src/views/approvals/ApprovalCenterView.vue
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import ApprovalInstanceView from './ApprovalInstanceView.vue'
|
||||||
|
import ApprovalTemplateView from './ApprovalTemplateView.vue'
|
||||||
|
|
||||||
|
const route = useRoute()
|
||||||
|
const router = useRouter()
|
||||||
|
|
||||||
|
const activeTab = computed<'instances' | 'mine' | 'strategies'>({
|
||||||
|
get: () => route.query.tab === 'strategies' ? 'strategies' : route.query.tab === 'mine' ? 'mine' : 'instances',
|
||||||
|
set: (value: string) => {
|
||||||
|
void router.replace({ query: value === 'instances' ? {} : { tab: value } })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="approval-center">
|
||||||
|
<header class="page-header">
|
||||||
|
<div>
|
||||||
|
<h2>审批中心</h2>
|
||||||
|
<p>集中处理审批申请、审批历史和审批策略。</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<el-tabs v-model="activeTab">
|
||||||
|
<el-tab-pane label="审批申请" name="instances">
|
||||||
|
<ApprovalInstanceView v-if="activeTab === 'instances'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="我的申请" name="mine">
|
||||||
|
<ApprovalInstanceView v-if="activeTab === 'mine'" mine />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="审批策略" name="strategies">
|
||||||
|
<ApprovalTemplateView v-if="activeTab === 'strategies'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
</el-tabs>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.approval-center {
|
||||||
|
min-height: 100%;
|
||||||
|
padding: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.page-header {
|
||||||
|
margin-bottom: 4px;
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0;
|
||||||
|
color: #1f2937;
|
||||||
|
font-size: 22px;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
margin: 6px 0 0;
|
||||||
|
color: #64748b;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.page) {
|
||||||
|
padding: 16px 0 0;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -1,11 +1,15 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, reactive, ref } from 'vue'
|
import { computed, onMounted, ref } from 'vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
import DataTablePage from '@/components/DataTablePage.vue'
|
import DataTablePage from '@/components/DataTablePage.vue'
|
||||||
import { getApprovalInstances, decideApproval, type ApprovalInstance } from '@/api/modules/approval'
|
import { getApprovalInstances, decideApproval, type ApprovalInstance } from '@/api/modules/approval'
|
||||||
import { getUsers } from '@/api/modules/system'
|
import { getUsers } from '@/api/modules/system'
|
||||||
|
import { useAuthStore } from '@/stores/auth'
|
||||||
import type { SystemUser } from '@/types'
|
import type { SystemUser } from '@/types'
|
||||||
|
|
||||||
|
const props = defineProps<{ mine?: boolean }>()
|
||||||
|
const auth = useAuthStore()
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const instances = ref<ApprovalInstance[]>([])
|
const instances = ref<ApprovalInstance[]>([])
|
||||||
const users = ref<SystemUser[]>([])
|
const users = ref<SystemUser[]>([])
|
||||||
@@ -14,6 +18,14 @@ const showDecide = ref(false)
|
|||||||
const current = ref<ApprovalInstance | null>(null)
|
const current = ref<ApprovalInstance | null>(null)
|
||||||
const decision = ref({ step_index: 0, approver_id: '', approved: true, comment: '' })
|
const decision = ref({ step_index: 0, approver_id: '', approved: true, comment: '' })
|
||||||
|
|
||||||
|
const visibleInstances = computed(() => {
|
||||||
|
if (!props.mine) return instances.value
|
||||||
|
const currentUserId = auth.currentUser?.id
|
||||||
|
return currentUserId
|
||||||
|
? instances.value.filter((item) => item.applicant_id === currentUserId)
|
||||||
|
: []
|
||||||
|
})
|
||||||
|
|
||||||
const statusOptions = [
|
const statusOptions = [
|
||||||
{ label: '待审批', value: 'pending' },
|
{ label: '待审批', value: 'pending' },
|
||||||
{ label: '已通过', value: 'approved' },
|
{ label: '已通过', value: 'approved' },
|
||||||
@@ -77,7 +89,7 @@ onMounted(() => {
|
|||||||
|
|
||||||
<template>
|
<template>
|
||||||
<div class="page">
|
<div class="page">
|
||||||
<DataTablePage title="审批实例" :data="instances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
|
<DataTablePage :title="props.mine ? '我的申请' : '审批申请'" :data="visibleInstances" :loading="loading" searchable :search-fields="['resource_type', 'resource_id']">
|
||||||
<template #toolbar-extra>
|
<template #toolbar-extra>
|
||||||
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
|
<el-select v-model="statusFilter" placeholder="状态" clearable style="width: 140px" @change="load">
|
||||||
<el-option v-for="s in statusOptions" :key="s.value" :label="s.label" :value="s.value" />
|
<el-option v-for="s in statusOptions" :key="s.value" :label="s.label" :value="s.value" />
|
||||||
|
|||||||
@@ -2,16 +2,21 @@
|
|||||||
import { onMounted, reactive, ref } from 'vue'
|
import { onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
import { getAuditLogs, exportAuditLogs, type AuditLog, type AuditQuery } from '@/api/modules/audit'
|
import { getAuditLogs, exportAuditLogs, type AuditLog, type AuditQuery } from '@/api/modules/audit'
|
||||||
|
import { getUsers, type SystemUser } from '@/api/modules/system'
|
||||||
|
import { getTenants, type Tenant } from '@/api/modules/tenant'
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const logs = ref<AuditLog[]>([])
|
const logs = ref<AuditLog[]>([])
|
||||||
const total = ref(0)
|
const total = ref(0)
|
||||||
|
const users = ref<SystemUser[]>([])
|
||||||
|
const tenants = ref<Tenant[]>([])
|
||||||
const query = reactive<AuditQuery>({
|
const query = reactive<AuditQuery>({
|
||||||
tenant_id: '',
|
tenant_id: '',
|
||||||
project_id: '',
|
|
||||||
actor_id: '',
|
actor_id: '',
|
||||||
action: '',
|
action: '',
|
||||||
target_type: '',
|
target_type: '',
|
||||||
|
target_id: '',
|
||||||
|
keyword: '',
|
||||||
start_time: '',
|
start_time: '',
|
||||||
end_time: '',
|
end_time: '',
|
||||||
limit: 50,
|
limit: 50,
|
||||||
@@ -21,6 +26,87 @@ const query = reactive<AuditQuery>({
|
|||||||
// 时间范围(el-date-picker 双向绑定数组 [start, end])
|
// 时间范围(el-date-picker 双向绑定数组 [start, end])
|
||||||
const timeRange = ref<[string, string] | null>(null)
|
const timeRange = ref<[string, string] | null>(null)
|
||||||
|
|
||||||
|
const actionOptions = [
|
||||||
|
{ value: 'create_dataset', label: '创建数据集' },
|
||||||
|
{ value: 'update_dataset', label: '修改数据集' },
|
||||||
|
{ value: 'delete_dataset', label: '删除数据集' },
|
||||||
|
{ value: 'create_model', label: '创建模型' },
|
||||||
|
{ value: 'update_model', label: '修改模型' },
|
||||||
|
{ value: 'delete_model', label: '删除模型' },
|
||||||
|
{ value: 'create_fine_tune', label: '创建训练任务' },
|
||||||
|
{ value: 'update_fine_tune', label: '修改训练任务' },
|
||||||
|
{ value: 'delete_fine_tune', label: '删除训练任务' },
|
||||||
|
{ value: 'create_inference', label: '创建推理任务' },
|
||||||
|
{ value: 'update_inference', label: '修改推理任务' },
|
||||||
|
{ value: 'delete_inference', label: '删除推理任务' },
|
||||||
|
{ value: 'create_user', label: '创建用户' },
|
||||||
|
{ value: 'update_user', label: '修改用户' },
|
||||||
|
{ value: 'delete_user', label: '删除用户' },
|
||||||
|
{ value: 'tenant.create', label: '创建租户' },
|
||||||
|
{ value: 'tenant.update', label: '修改租户' },
|
||||||
|
{ value: 'tenant.delete', label: '删除租户' },
|
||||||
|
{ value: 'tenant.quota.set', label: '设置租户配额' },
|
||||||
|
{ value: 'tenant.retention.set', label: '设置留存策略' },
|
||||||
|
{ value: 'grant_acl', label: '授予资源权限' },
|
||||||
|
{ value: 'revoke_acl', label: '撤销资源权限' },
|
||||||
|
{ value: 'gpu.assign', label: '分配算力卡' },
|
||||||
|
{ value: 'gpu.release', label: '释放算力卡' },
|
||||||
|
{ value: 'create', label: '创建' },
|
||||||
|
{ value: 'update', label: '修改' },
|
||||||
|
{ value: 'delete', label: '删除' },
|
||||||
|
{ value: 'start', label: '启动' },
|
||||||
|
{ value: 'stop', label: '停止' },
|
||||||
|
{ value: 'upload', label: '上传' },
|
||||||
|
{ value: 'download', label: '下载' },
|
||||||
|
{ value: 'convert', label: '转换' },
|
||||||
|
{ value: 'merge', label: '合并权重' },
|
||||||
|
{ value: 'publish', label: '发布' },
|
||||||
|
{ value: 'retry', label: '重试' },
|
||||||
|
{ value: 'login', label: '登录' },
|
||||||
|
{ value: 'logout', label: '退出登录' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const targetTypeOptions = [
|
||||||
|
{ value: 'dataset', label: '数据集' },
|
||||||
|
{ value: 'model', label: '模型' },
|
||||||
|
{ value: 'fine_tune_task', label: '训练任务' },
|
||||||
|
{ value: 'fine_tune', label: '训练任务' },
|
||||||
|
{ value: 'inference_task', label: '推理任务' },
|
||||||
|
{ value: 'inference', label: '推理任务' },
|
||||||
|
{ value: 'eval_task', label: '评测任务' },
|
||||||
|
{ value: 'trained_model', label: '训练模型' },
|
||||||
|
{ value: 'convert_task', label: '数据转换任务' },
|
||||||
|
{ value: 'tenant', label: '租户' },
|
||||||
|
{ value: 'user', label: '用户' },
|
||||||
|
{ value: 'resource_acl', label: '资源权限' },
|
||||||
|
{ value: 'gpu', label: '算力卡' },
|
||||||
|
{ value: 'retention_policy', label: '留存策略' },
|
||||||
|
{ value: 'module', label: '业务模块' },
|
||||||
|
{ value: 'api', label: '接口' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const actionLabels = Object.fromEntries(actionOptions.map((item) => [item.value, item.label]))
|
||||||
|
const targetTypeLabels = Object.fromEntries(targetTypeOptions.map((item) => [item.value, item.label]))
|
||||||
|
|
||||||
|
function userName(id?: string) {
|
||||||
|
if (!id) return '系统'
|
||||||
|
const user = users.value.find((item) => item.id === id)
|
||||||
|
return user ? `${user.display_name || user.username}(${user.username})` : id
|
||||||
|
}
|
||||||
|
|
||||||
|
function tenantName(id?: string) {
|
||||||
|
if (!id) return '未关联租户'
|
||||||
|
return tenants.value.find((item) => item.id === id)?.name || id
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionName(action?: string) {
|
||||||
|
return action ? actionLabels[action] || action : '未记录'
|
||||||
|
}
|
||||||
|
|
||||||
|
function targetTypeName(type?: string) {
|
||||||
|
return type ? targetTypeLabels[type] || type : '未指定'
|
||||||
|
}
|
||||||
|
|
||||||
function applyTimeRange() {
|
function applyTimeRange() {
|
||||||
if (timeRange.value && timeRange.value.length === 2) {
|
if (timeRange.value && timeRange.value.length === 2) {
|
||||||
query.start_time = timeRange.value[0]
|
query.start_time = timeRange.value[0]
|
||||||
@@ -42,6 +128,25 @@ async function load() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadFilterOptions() {
|
||||||
|
const [userResult, tenantResult] = await Promise.allSettled([getUsers(), getTenants()])
|
||||||
|
if (userResult.status === 'fulfilled') users.value = userResult.value
|
||||||
|
if (tenantResult.status === 'fulfilled') tenants.value = tenantResult.value
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetFilters() {
|
||||||
|
query.tenant_id = ''
|
||||||
|
query.actor_id = ''
|
||||||
|
query.action = ''
|
||||||
|
query.target_type = ''
|
||||||
|
query.target_id = ''
|
||||||
|
query.keyword = ''
|
||||||
|
query.start_time = ''
|
||||||
|
query.end_time = ''
|
||||||
|
timeRange.value = null
|
||||||
|
void load()
|
||||||
|
}
|
||||||
|
|
||||||
async function handleExport() {
|
async function handleExport() {
|
||||||
try {
|
try {
|
||||||
const blob = await exportAuditLogs({ ...query, limit: 10000, offset: 0 })
|
const blob = await exportAuditLogs({ ...query, limit: 10000, offset: 0 })
|
||||||
@@ -56,7 +161,9 @@ async function handleExport() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(load)
|
onMounted(() => {
|
||||||
|
void Promise.all([loadFilterOptions(), load()])
|
||||||
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
@@ -66,21 +173,32 @@ onMounted(load)
|
|||||||
<el-button @click="handleExport">导出 CSV</el-button>
|
<el-button @click="handleExport">导出 CSV</el-button>
|
||||||
</div>
|
</div>
|
||||||
<el-card class="filter-card">
|
<el-card class="filter-card">
|
||||||
<el-form :inline="true">
|
<el-form :inline="true" class="filter-form">
|
||||||
<el-form-item label="租户">
|
<el-form-item label="租户">
|
||||||
<el-input v-model="query.tenant_id" placeholder="tenant_id" clearable />
|
<el-select v-model="query.tenant_id" placeholder="全部租户" clearable filterable style="width: 190px">
|
||||||
</el-form-item>
|
<el-option v-for="tenant in tenants" :key="tenant.id" :label="tenant.name" :value="tenant.id" />
|
||||||
<el-form-item label="项目">
|
</el-select>
|
||||||
<el-input v-model="query.project_id" placeholder="project_id" clearable />
|
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="操作人">
|
<el-form-item label="操作人">
|
||||||
<el-input v-model="query.actor_id" placeholder="actor_id" clearable />
|
<el-select v-model="query.actor_id" placeholder="全部用户" clearable filterable style="width: 210px">
|
||||||
|
<el-option v-for="user in users" :key="user.id" :label="`${user.display_name || user.username}(${user.username})`" :value="user.id" />
|
||||||
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="动作">
|
<el-form-item label="动作">
|
||||||
<el-input v-model="query.action" placeholder="action" clearable />
|
<el-select v-model="query.action" placeholder="全部动作" clearable filterable style="width: 180px">
|
||||||
|
<el-option v-for="item in actionOptions" :key="item.value" :label="item.label" :value="item.value" />
|
||||||
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="目标类型">
|
<el-form-item label="目标类型">
|
||||||
<el-input v-model="query.target_type" placeholder="target_type" clearable />
|
<el-select v-model="query.target_type" placeholder="全部资源" clearable filterable style="width: 160px">
|
||||||
|
<el-option v-for="item in targetTypeOptions" :key="item.value" :label="item.label" :value="item.value" />
|
||||||
|
</el-select>
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="关键词">
|
||||||
|
<el-input v-model="query.keyword" placeholder="资源 ID 或详情" clearable style="width: 220px" />
|
||||||
|
</el-form-item>
|
||||||
|
<el-form-item label="目标 ID">
|
||||||
|
<el-input v-model="query.target_id" placeholder="精确查询,可选" clearable style="width: 180px" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="时间范围">
|
<el-form-item label="时间范围">
|
||||||
<el-date-picker
|
<el-date-picker
|
||||||
@@ -97,16 +215,24 @@ onMounted(load)
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item>
|
<el-form-item>
|
||||||
<el-button type="primary" @click="load">查询</el-button>
|
<el-button type="primary" @click="load">查询</el-button>
|
||||||
|
<el-button @click="resetFilters">重置</el-button>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
</el-card>
|
</el-card>
|
||||||
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
|
<el-table :data="logs" v-loading="loading" border stripe class="log-table">
|
||||||
<el-table-column prop="time" label="时间" min-width="180" />
|
<el-table-column prop="time" label="时间" min-width="180" />
|
||||||
<el-table-column prop="tenant_id" label="租户" min-width="120" />
|
<el-table-column label="租户" min-width="140">
|
||||||
<el-table-column prop="project_id" label="项目" min-width="120" />
|
<template #default="{ row }">{{ tenantName(row.tenant_id) }}</template>
|
||||||
<el-table-column prop="actor_id" label="操作人" min-width="120" />
|
</el-table-column>
|
||||||
<el-table-column prop="action" label="动作" min-width="140" />
|
<el-table-column label="操作人" min-width="180">
|
||||||
<el-table-column prop="target_type" label="目标类型" min-width="120" />
|
<template #default="{ row }">{{ userName(row.actor_id) }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="动作" min-width="140">
|
||||||
|
<template #default="{ row }">{{ actionName(row.action) }}</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="目标类型" min-width="120">
|
||||||
|
<template #default="{ row }">{{ targetTypeName(row.target_type) }}</template>
|
||||||
|
</el-table-column>
|
||||||
<el-table-column prop="target_id" label="目标 ID" min-width="140" show-overflow-tooltip />
|
<el-table-column prop="target_id" label="目标 ID" min-width="140" show-overflow-tooltip />
|
||||||
<el-table-column prop="detail" label="详情" min-width="200" show-overflow-tooltip />
|
<el-table-column prop="detail" label="详情" min-width="200" show-overflow-tooltip />
|
||||||
<el-table-column prop="client_ip" label="IP" min-width="120" />
|
<el-table-column prop="client_ip" label="IP" min-width="120" />
|
||||||
@@ -120,6 +246,7 @@ onMounted(load)
|
|||||||
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
|
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 16px; }
|
||||||
.page-title { margin: 0; font-size: 18px; }
|
.page-title { margin: 0; font-size: 18px; }
|
||||||
.filter-card { margin-bottom: 16px; }
|
.filter-card { margin-bottom: 16px; }
|
||||||
|
.filter-form { display: flex; flex-wrap: wrap; }
|
||||||
.log-table { margin-top: 8px; }
|
.log-table { margin-top: 8px; }
|
||||||
.pager { margin-top: 12px; text-align: right; color: #909399; }
|
.pager { margin-top: 12px; text-align: right; color: #909399; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ onMounted(() => {
|
|||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="操作" width="80" align="center" fixed="right">
|
<el-table-column label="操作" width="80" align="center" fixed="right">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-button link type="primary" size="small" @click="showDetail(row)">详情</el-button>
|
<el-button link type="primary" size="small" @click="showDetail(row as OperationLog)">详情</el-button>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
</el-table>
|
</el-table>
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref } from 'vue'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import { Plus, Delete, Refresh } from '@element-plus/icons-vue'
|
import { Plus, Delete, Refresh } from '@element-plus/icons-vue'
|
||||||
import type { TagProps, UploadRequestOptions, UploadFile } from 'element-plus'
|
import type { TagProps, UploadRequestOptions, UploadFile, UploadRawFile } from 'element-plus'
|
||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import {
|
import {
|
||||||
getDataConvertTasks,
|
getDataConvertTasks,
|
||||||
@@ -30,9 +30,9 @@ async function load() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 文件上传前的校验(仅校验文件格式)
|
// 文件上传前的校验(仅校验文件格式)
|
||||||
function beforeUpload(file: UploadFile) {
|
function beforeUpload(file: UploadRawFile) {
|
||||||
// 检查文件类型
|
// 检查文件类型
|
||||||
const isJson = file.name.endsWith('.json') || file.raw?.type === 'application/json'
|
const isJson = file.name.endsWith('.json') || file.type === 'application/json'
|
||||||
if (!isJson) {
|
if (!isJson) {
|
||||||
ElMessage.error('只能上传 .json 格式的文件')
|
ElMessage.error('只能上传 .json 格式的文件')
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
previewAffectingOptionsFor,
|
previewAffectingOptionsFor,
|
||||||
} from './create/dataProcessCreateState'
|
} from './create/dataProcessCreateState'
|
||||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||||
|
import { useDataProcessEvaluation } from './create/useDataProcessEvaluation'
|
||||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||||
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
|
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
|
||||||
@@ -126,6 +127,19 @@ const {
|
|||||||
outputType: activeOutputType,
|
outputType: activeOutputType,
|
||||||
beforeGenerate: beforeStartGeneration,
|
beforeGenerate: beforeStartGeneration,
|
||||||
})
|
})
|
||||||
|
const {
|
||||||
|
evaluation,
|
||||||
|
evaluateAllResults,
|
||||||
|
resetEvaluation,
|
||||||
|
} = useDataProcessEvaluation({
|
||||||
|
taskId,
|
||||||
|
results,
|
||||||
|
selectedResultId,
|
||||||
|
})
|
||||||
|
// 生成结果被重置(重新切分/上传/重新生成配置)时同步清空评测进度。
|
||||||
|
watch(results, (items) => {
|
||||||
|
if (!items.length) resetEvaluation()
|
||||||
|
})
|
||||||
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
|
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
|
||||||
taskId,
|
taskId,
|
||||||
uploadedFiles,
|
uploadedFiles,
|
||||||
@@ -1156,10 +1170,12 @@ onMounted(() => {
|
|||||||
:preview-items="previewItems"
|
:preview-items="previewItems"
|
||||||
:regenerating-result-id="regeneratingResultId"
|
:regenerating-result-id="regeneratingResultId"
|
||||||
:bulk-regeneration="bulkRegeneration"
|
:bulk-regeneration="bulkRegeneration"
|
||||||
|
:evaluation="evaluation"
|
||||||
:output-type="activeOutputType"
|
:output-type="activeOutputType"
|
||||||
@update:field="updateResultField"
|
@update:field="updateResultField"
|
||||||
@regenerate:all="regenerateAllResults"
|
@regenerate:all="regenerateAllResults"
|
||||||
@regenerate:item="regenerateResult"
|
@regenerate:item="regenerateResult"
|
||||||
|
@evaluate:all="evaluateAllResults"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { ElMessage, ElMessageBox } from 'element-plus'
|
|||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import { usePolling } from '@/composables/usePolling'
|
import { usePolling } from '@/composables/usePolling'
|
||||||
import {
|
import {
|
||||||
|
evaluateDataProcessResults,
|
||||||
getDataProcessProgress,
|
getDataProcessProgress,
|
||||||
getDataProcessResults,
|
getDataProcessResults,
|
||||||
getDataProcessTask,
|
getDataProcessTask,
|
||||||
@@ -13,6 +14,7 @@ import {
|
|||||||
restoreDataProcessResult,
|
restoreDataProcessResult,
|
||||||
updateDataProcessResult,
|
updateDataProcessResult,
|
||||||
} from '@/api/modules/dataProcess'
|
} from '@/api/modules/dataProcess'
|
||||||
|
import QualityRadarPopover from './create/QualityRadarPopover.vue'
|
||||||
import type {
|
import type {
|
||||||
DataProcessDatasetSplit,
|
DataProcessDatasetSplit,
|
||||||
DataProcessPublishPayload,
|
DataProcessPublishPayload,
|
||||||
@@ -456,13 +458,101 @@ function resultStatusType(status: DataProcessResultStatus) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function qualityScoreLabel(value: DataProcessResult['quality_score']) {
|
function qualityScoreLabel(value: DataProcessResult['quality_score']) {
|
||||||
if (value == null) return '-'
|
if (value == null || !value.evaluated) return '-'
|
||||||
const score = value.overall
|
const score = value.overall
|
||||||
return Number.isFinite(score) ? Number(score).toFixed(1) : '-'
|
return Number.isFinite(score) ? Number(score).toFixed(1) : '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
function qualityFlagsLabel(value: DataProcessResult['quality_score']) {
|
function qualityScoreTone(value: DataProcessResult['quality_score']) {
|
||||||
return value?.flags?.length ? value.flags.join('、') : '未命中质量规则'
|
const score = Number(value?.overall)
|
||||||
|
if (!value?.evaluated || !Number.isFinite(score)) return ''
|
||||||
|
return score >= 80 ? 'is-success' : score >= 60 ? 'is-warning' : 'is-danger'
|
||||||
|
}
|
||||||
|
|
||||||
|
function qualityScoreEvaluated(value: DataProcessResult['quality_score']) {
|
||||||
|
return Boolean(value?.evaluated && Number.isFinite(Number(value?.overall)))
|
||||||
|
}
|
||||||
|
|
||||||
|
const evaluationRunning = ref(false)
|
||||||
|
const evaluationProgress = reactive({
|
||||||
|
visible: false,
|
||||||
|
total: 0,
|
||||||
|
completed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
})
|
||||||
|
// 与批量重生成一致的分块大小,单批在接口 240 秒超时预算内。
|
||||||
|
const EVALUATION_CHUNK_SIZE = 12
|
||||||
|
const canEvaluate = computed(() => (
|
||||||
|
detail.value?.status === 'completed' && !hasCurrentPublishedDataset.value
|
||||||
|
))
|
||||||
|
const evaluationPercentage = computed(() => (
|
||||||
|
evaluationProgress.total
|
||||||
|
? Math.round((evaluationProgress.completed / evaluationProgress.total) * 100)
|
||||||
|
: 0
|
||||||
|
))
|
||||||
|
|
||||||
|
async function loadAllResultIds() {
|
||||||
|
const first = await getDataProcessResults(taskId.value, { page: 1, page_size: 500 })
|
||||||
|
const items = [...first.items]
|
||||||
|
const pages = Math.ceil(first.total / first.page_size)
|
||||||
|
for (let page = 2; page <= pages; page += 1) {
|
||||||
|
const next = await getDataProcessResults(taskId.value, { page, page_size: 500 })
|
||||||
|
items.push(...next.items)
|
||||||
|
}
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runResultEvaluation() {
|
||||||
|
if (evaluationRunning.value || !canEvaluate.value) return
|
||||||
|
evaluationRunning.value = true
|
||||||
|
Object.assign(evaluationProgress, {
|
||||||
|
visible: true,
|
||||||
|
total: 0,
|
||||||
|
completed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
const candidates = (await loadAllResultIds()).filter((item) => item.updated_at)
|
||||||
|
if (!candidates.length) {
|
||||||
|
ElMessage.info('当前没有可评测的结果')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
evaluationProgress.total = candidates.length
|
||||||
|
for (let offset = 0; offset < candidates.length; offset += EVALUATION_CHUNK_SIZE) {
|
||||||
|
const chunk = candidates.slice(offset, offset + EVALUATION_CHUNK_SIZE)
|
||||||
|
try {
|
||||||
|
const evaluated = await evaluateDataProcessResults(taskId.value, {
|
||||||
|
items: chunk.map((item) => ({
|
||||||
|
result_id: String(item.id),
|
||||||
|
expected_updated_at: item.updated_at as string,
|
||||||
|
})),
|
||||||
|
})
|
||||||
|
evaluationProgress.completed += evaluated.total
|
||||||
|
evaluationProgress.succeeded += evaluated.succeeded
|
||||||
|
evaluationProgress.failed += evaluated.failed
|
||||||
|
} catch {
|
||||||
|
evaluationProgress.completed = evaluationProgress.total
|
||||||
|
evaluationProgress.failed += candidates.length - offset
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await loadResults()
|
||||||
|
if (evaluationProgress.failed === 0) {
|
||||||
|
ElMessage.success(`数据评测完成:成功 ${evaluationProgress.succeeded} 条`)
|
||||||
|
} else if (evaluationProgress.succeeded > 0) {
|
||||||
|
ElMessage.warning(
|
||||||
|
`数据评测完成:成功 ${evaluationProgress.succeeded} 条,失败 ${evaluationProgress.failed} 条`,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
ElMessage.error(`数据评测失败:${evaluationProgress.failed} 条结果未完成评测`)
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
ElMessage.error('数据评测中断,已完成的评分保持不变')
|
||||||
|
} finally {
|
||||||
|
evaluationRunning.value = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function replaceResult(updated: DataProcessResult) {
|
function replaceResult(updated: DataProcessResult) {
|
||||||
@@ -824,10 +914,33 @@ onBeforeUnmount(() => {
|
|||||||
<el-option label="已修改" value="modified" />
|
<el-option label="已修改" value="modified" />
|
||||||
<el-option label="无效" value="invalid" />
|
<el-option label="无效" value="invalid" />
|
||||||
</el-select>
|
</el-select>
|
||||||
|
<el-button
|
||||||
|
v-if="canEvaluate"
|
||||||
|
type="primary"
|
||||||
|
plain
|
||||||
|
:loading="evaluationRunning"
|
||||||
|
:disabled="resultLoading"
|
||||||
|
@click="runResultEvaluation"
|
||||||
|
>
|
||||||
|
<i v-if="!evaluationRunning" class="fa fa-check-square-o" aria-hidden="true" /> 数据评测
|
||||||
|
</el-button>
|
||||||
<el-button :loading="resultLoading" @click="loadResults"><i class="fa fa-refresh" /></el-button>
|
<el-button :loading="resultLoading" @click="loadResults"><i class="fa fa-refresh" /></el-button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div v-if="evaluationProgress.visible" class="evaluation-progress">
|
||||||
|
<span>
|
||||||
|
数据评测 {{ evaluationProgress.completed }} / {{ evaluationProgress.total }}
|
||||||
|
· 成功 {{ evaluationProgress.succeeded }} · 失败 {{ evaluationProgress.failed }}
|
||||||
|
</span>
|
||||||
|
<el-progress
|
||||||
|
:percentage="evaluationPercentage"
|
||||||
|
:show-text="false"
|
||||||
|
:stroke-width="5"
|
||||||
|
:color="evaluationProgress.failed > 0 ? '#d97706' : '#5b50f2'"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<el-table
|
<el-table
|
||||||
v-if="results.length"
|
v-if="results.length"
|
||||||
:data="results"
|
:data="results"
|
||||||
@@ -845,9 +958,26 @@ onBeforeUnmount(() => {
|
|||||||
</template>
|
</template>
|
||||||
<el-table-column label="质量分" width="88" align="center">
|
<el-table-column label="质量分" width="88" align="center">
|
||||||
<template #default="{ row }">
|
<template #default="{ row }">
|
||||||
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
|
<el-popover
|
||||||
<span>{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
|
v-if="qualityScoreEvaluated((row as DataProcessResult).quality_score)"
|
||||||
</el-tooltip>
|
placement="top"
|
||||||
|
:width="296"
|
||||||
|
trigger="hover"
|
||||||
|
:show-after="150"
|
||||||
|
popper-class="quality-radar-popper"
|
||||||
|
>
|
||||||
|
<template #reference>
|
||||||
|
<span
|
||||||
|
class="detail-quality-score"
|
||||||
|
:class="qualityScoreTone((row as DataProcessResult).quality_score)"
|
||||||
|
>{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
|
||||||
|
</template>
|
||||||
|
<QualityRadarPopover
|
||||||
|
:quality="(row as DataProcessResult).quality_score!"
|
||||||
|
:score="Number((row as DataProcessResult).quality_score?.overall)"
|
||||||
|
/>
|
||||||
|
</el-popover>
|
||||||
|
<span v-else class="detail-quality-empty">{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
|
||||||
</template>
|
</template>
|
||||||
</el-table-column>
|
</el-table-column>
|
||||||
<el-table-column label="状态" width="90" align="center">
|
<el-table-column label="状态" width="90" align="center">
|
||||||
@@ -1099,6 +1229,35 @@ onBeforeUnmount(() => {
|
|||||||
.result-filters :deep(.el-select) { width: 120px; }
|
.result-filters :deep(.el-select) { width: 120px; }
|
||||||
.result-section :deep(.el-table) { border-radius: 0; }
|
.result-section :deep(.el-table) { border-radius: 0; }
|
||||||
|
|
||||||
|
.evaluation-progress {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 220px;
|
||||||
|
align-items: center;
|
||||||
|
gap: 14px;
|
||||||
|
padding: 10px 18px;
|
||||||
|
color: #667085;
|
||||||
|
background: #f8f9fc;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detail-quality-score {
|
||||||
|
display: inline-block;
|
||||||
|
min-width: 44px;
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 10px;
|
||||||
|
color: #475467;
|
||||||
|
background: #f2f4f7;
|
||||||
|
font-weight: 700;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
cursor: default;
|
||||||
|
|
||||||
|
&.is-success { color: #067647; background: #e6f4ee; }
|
||||||
|
&.is-warning { color: #b54708; background: #fef0c7; }
|
||||||
|
&.is-danger { color: #b42318; background: #fee4e2; }
|
||||||
|
}
|
||||||
|
|
||||||
|
.detail-quality-empty { color: #98a2b3; }
|
||||||
|
|
||||||
:global(.data-process-result-tooltip) {
|
:global(.data-process-result-tooltip) {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
max-width: min(520px, calc(100vw - 32px));
|
max-width: min(520px, calc(100vw - 32px));
|
||||||
|
|||||||
252
frontend/src/views/data-process/create/QualityRadarPopover.vue
Normal file
252
frontend/src/views/data-process/create/QualityRadarPopover.vue
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import VChart from 'vue-echarts'
|
||||||
|
import '@/plugins/echarts'
|
||||||
|
import type { EChartsOption } from 'echarts'
|
||||||
|
import type { ResultQualityDetails } from './types'
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
quality: ResultQualityDetails
|
||||||
|
score?: number
|
||||||
|
}>()
|
||||||
|
|
||||||
|
// 轴标签按词意预置断行,避免长中文标签把雷达网格挤偏或被机械切词。
|
||||||
|
const JUDGE_DIMENSION_LABELS: Record<string, string> = {
|
||||||
|
faithfulness: '忠实度',
|
||||||
|
correctness: '正确性',
|
||||||
|
clarity: '问题\n清晰度',
|
||||||
|
completeness: '回答\n完整性',
|
||||||
|
alignment: '指令\n对齐',
|
||||||
|
reasoning_validity: '推理\n有效性',
|
||||||
|
chosen_quality: 'chosen\n质量',
|
||||||
|
rejected_quality: 'rejected\n质量',
|
||||||
|
preference_reasonableness: '偏好\n区分',
|
||||||
|
}
|
||||||
|
|
||||||
|
const SEMANTIC_DIMENSION_LABELS: Record<string, string> = {
|
||||||
|
question_answer: '问答\n相关',
|
||||||
|
answer_source: '来源\n覆盖',
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RadarDimension {
|
||||||
|
name: string
|
||||||
|
value: number
|
||||||
|
}
|
||||||
|
|
||||||
|
const radarDimensions = computed<RadarDimension[]>(() => {
|
||||||
|
const dimensions: RadarDimension[] = []
|
||||||
|
for (const [key, value] of Object.entries(props.quality?.judge?.scores ?? {})) {
|
||||||
|
dimensions.push({
|
||||||
|
name: JUDGE_DIMENSION_LABELS[key] ?? key,
|
||||||
|
value: Math.round(value * 20),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (const [key, value] of Object.entries(props.quality?.semantic ?? {})) {
|
||||||
|
if (key === 'overall' || typeof value !== 'number') continue
|
||||||
|
dimensions.push({
|
||||||
|
name: SEMANTIC_DIMENSION_LABELS[key] ?? key,
|
||||||
|
value: Math.round(value),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return dimensions
|
||||||
|
})
|
||||||
|
|
||||||
|
// 可用维度太少时雷达图失去意义,降级为分层分数展示。
|
||||||
|
const showRadar = computed(() => radarDimensions.value.length >= 3)
|
||||||
|
|
||||||
|
const radarOption = computed<EChartsOption>(() => ({
|
||||||
|
radar: {
|
||||||
|
indicator: radarDimensions.value.map((dimension) => ({
|
||||||
|
name: dimension.name,
|
||||||
|
max: 100,
|
||||||
|
})),
|
||||||
|
radius: '56%',
|
||||||
|
center: ['50%', '50%'],
|
||||||
|
splitNumber: 4,
|
||||||
|
axisName: {
|
||||||
|
color: '#667085',
|
||||||
|
fontSize: 10,
|
||||||
|
lineHeight: 13,
|
||||||
|
},
|
||||||
|
splitArea: { areaStyle: { color: ['#fbfbfd', '#f2f4f8'] } },
|
||||||
|
splitLine: { lineStyle: { color: '#e4e7ec' } },
|
||||||
|
axisLine: { lineStyle: { color: '#e4e7ec' } },
|
||||||
|
},
|
||||||
|
series: [{
|
||||||
|
type: 'radar',
|
||||||
|
symbol: 'circle',
|
||||||
|
symbolSize: 3,
|
||||||
|
data: [{
|
||||||
|
value: radarDimensions.value.map((dimension) => dimension.value),
|
||||||
|
name: '质量维度',
|
||||||
|
areaStyle: { color: 'rgba(91, 80, 242, 0.18)' },
|
||||||
|
lineStyle: { color: '#5b50f2', width: 1.5 },
|
||||||
|
itemStyle: { color: '#5b50f2' },
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
}))
|
||||||
|
|
||||||
|
const layerScores = computed(() => {
|
||||||
|
const layers = props.quality?.layers ?? {}
|
||||||
|
return [
|
||||||
|
{ label: '规则层', value: layers.rule },
|
||||||
|
{ label: '语义层', value: layers.semantic },
|
||||||
|
{ label: '评审层', value: layers.judge },
|
||||||
|
].filter((layer): layer is { label: string; value: number } => (
|
||||||
|
typeof layer.value === 'number'
|
||||||
|
))
|
||||||
|
})
|
||||||
|
|
||||||
|
const displayScore = computed(() => {
|
||||||
|
if (typeof props.score === 'number' && !isNaN(props.score)) {
|
||||||
|
return props.score.toFixed(1)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
})
|
||||||
|
|
||||||
|
const scoreTone = computed(() => {
|
||||||
|
const numScore = props.score ?? 0
|
||||||
|
return numScore >= 80 ? 'is-success' : numScore >= 60 ? 'is-warning' : 'is-danger'
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="quality-popover">
|
||||||
|
<div v-if="displayScore !== null" class="popover-header">
|
||||||
|
<strong>质量评测</strong>
|
||||||
|
<span class="popover-score" :class="scoreTone">{{ displayScore }}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<VChart
|
||||||
|
v-if="showRadar"
|
||||||
|
class="quality-radar"
|
||||||
|
:option="radarOption"
|
||||||
|
autoresize
|
||||||
|
/>
|
||||||
|
<div v-else class="radar-fallback">
|
||||||
|
维度数据不足,已评测维度少于 3 个时以分层分数为准。
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="layer-scores">
|
||||||
|
<div v-for="layer in layerScores" :key="layer.label" class="layer-item">
|
||||||
|
<span>{{ layer.label }}</span>
|
||||||
|
<el-progress
|
||||||
|
:percentage="Math.round(layer.value)"
|
||||||
|
:stroke-width="6"
|
||||||
|
:show-text="false"
|
||||||
|
:color="layer.value >= 80 ? '#12b76a' : layer.value >= 60 ? '#f0b429' : '#d92d20'"
|
||||||
|
/>
|
||||||
|
<em>{{ layer.value.toFixed(0) }}</em>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p v-if="quality?.judge?.reason" class="judge-reason">{{ quality.judge.reason }}</p>
|
||||||
|
<div v-if="quality?.judge?.issues?.length" class="judge-issues">
|
||||||
|
<span v-for="issue in quality.judge.issues" :key="issue" class="issue-tag">{{ issue }}</span>
|
||||||
|
</div>
|
||||||
|
<div v-if="quality?.judge?.model" class="judge-model">评审模型:{{ quality.judge.model }}</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.quality-popover {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 10px;
|
||||||
|
width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
.popover-header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
padding-bottom: 6px;
|
||||||
|
border-bottom: 1px solid #f2f4f7;
|
||||||
|
|
||||||
|
strong {
|
||||||
|
color: #344054;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.popover-score {
|
||||||
|
color: #344054;
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 700;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
|
||||||
|
&.is-success { color: #12b76a; }
|
||||||
|
&.is-warning { color: #d99b0b; }
|
||||||
|
&.is-danger { color: #d92d20; }
|
||||||
|
}
|
||||||
|
|
||||||
|
.quality-radar {
|
||||||
|
width: 100%;
|
||||||
|
height: 220px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
.radar-fallback {
|
||||||
|
padding: 18px 10px;
|
||||||
|
color: #98a2b3;
|
||||||
|
font-size: 12px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.layer-scores {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.layer-item {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 44px 1fr 28px;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
color: #667085;
|
||||||
|
font-size: 11px;
|
||||||
|
|
||||||
|
em {
|
||||||
|
color: #344054;
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 600;
|
||||||
|
text-align: right;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.judge-reason {
|
||||||
|
margin: 0;
|
||||||
|
color: #475467;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.6;
|
||||||
|
}
|
||||||
|
|
||||||
|
.judge-issues {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.issue-tag {
|
||||||
|
padding: 2px 8px;
|
||||||
|
color: #b54708;
|
||||||
|
background: #fef0c7;
|
||||||
|
border-radius: 3px;
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.judge-model {
|
||||||
|
color: #98a2b3;
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
<style lang="scss">
|
||||||
|
.quality-radar-popper {
|
||||||
|
padding: 14px 16px !important;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
|
import QualityRadarPopover from './QualityRadarPopover.vue'
|
||||||
|
import type { BulkResultRegenerationState, PreviewItem, ResultEvaluationState, ResultItem } from './types'
|
||||||
import type { DataProcessOutputType } from '@/types/dataProcess'
|
import type { DataProcessOutputType } from '@/types/dataProcess'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
@@ -9,6 +10,7 @@ const props = defineProps<{
|
|||||||
selectedId: string | null
|
selectedId: string | null
|
||||||
regeneratingResultId: string | null
|
regeneratingResultId: string | null
|
||||||
bulkRegeneration: BulkResultRegenerationState
|
bulkRegeneration: BulkResultRegenerationState
|
||||||
|
evaluation: ResultEvaluationState
|
||||||
outputType: DataProcessOutputType
|
outputType: DataProcessOutputType
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
@@ -17,6 +19,7 @@ const emit = defineEmits<{
|
|||||||
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
|
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
|
||||||
'regenerate:item': [id: string]
|
'regenerate:item': [id: string]
|
||||||
'regenerate:all': []
|
'regenerate:all': []
|
||||||
|
'evaluate:all': []
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
const search = ref('')
|
const search = ref('')
|
||||||
@@ -30,6 +33,15 @@ const bulkRegenerationActive = computed(() => props.bulkRegeneration.status ===
|
|||||||
const bulkRegenerationVisible = computed(() => (
|
const bulkRegenerationVisible = computed(() => (
|
||||||
props.bulkRegeneration.status !== 'idle' && props.bulkRegeneration.total > 0
|
props.bulkRegeneration.status !== 'idle' && props.bulkRegeneration.total > 0
|
||||||
))
|
))
|
||||||
|
const evaluationActive = computed(() => props.evaluation.status === 'running')
|
||||||
|
const evaluationVisible = computed(() => (
|
||||||
|
props.evaluation.status !== 'idle' && props.evaluation.total > 0
|
||||||
|
))
|
||||||
|
const evaluationPercentage = computed(() => {
|
||||||
|
if (!props.evaluation.total) return 0
|
||||||
|
return Math.round((props.evaluation.completed / props.evaluation.total) * 100)
|
||||||
|
})
|
||||||
|
const evaluatedCount = computed(() => props.items.filter((item) => item.qualityDetails?.evaluated).length)
|
||||||
const bulkRegenerationPercentage = computed(() => (
|
const bulkRegenerationPercentage = computed(() => (
|
||||||
props.bulkRegeneration.total > 0
|
props.bulkRegeneration.total > 0
|
||||||
? Math.round((props.bulkRegeneration.completed / props.bulkRegeneration.total) * 100)
|
? Math.round((props.bulkRegeneration.completed / props.bulkRegeneration.total) * 100)
|
||||||
@@ -98,22 +110,48 @@ function selectRelative(offset: number) {
|
|||||||
<template>
|
<template>
|
||||||
<section class="result-step">
|
<section class="result-step">
|
||||||
<div class="result-workspace">
|
<div class="result-workspace">
|
||||||
<aside class="result-list-pane" :class="{ 'has-bulk-progress': bulkRegenerationVisible }">
|
<aside class="result-list-pane" :class="{ 'has-bulk-progress': bulkRegenerationVisible || evaluationVisible }">
|
||||||
<div class="pane-header result-list-header">
|
<div class="pane-header result-list-header">
|
||||||
<div class="result-list-title"><strong>生成结果</strong><span>共 {{ items.length }} 条</span></div>
|
<div class="result-list-title">
|
||||||
|
<strong>生成结果</strong><span>共 {{ items.length }} 条<template v-if="evaluatedCount"> · 已评测 {{ evaluatedCount }}</template></span>
|
||||||
|
</div>
|
||||||
|
<div class="result-list-actions">
|
||||||
|
<el-button
|
||||||
|
size="small"
|
||||||
|
plain
|
||||||
|
:loading="evaluationActive"
|
||||||
|
:disabled="!items.length || bulkRegenerationActive || Boolean(regeneratingResultId)"
|
||||||
|
@click="emit('evaluate:all')"
|
||||||
|
>
|
||||||
|
<i v-if="!evaluationActive" class="fa fa-check-square-o" style="margin-right: 4px;" />
|
||||||
|
数据评测
|
||||||
|
</el-button>
|
||||||
<el-button
|
<el-button
|
||||||
v-if="invalidCount > 0"
|
v-if="invalidCount > 0"
|
||||||
size="small"
|
size="small"
|
||||||
plain
|
plain
|
||||||
type="primary"
|
type="primary"
|
||||||
:loading="bulkRegenerationActive"
|
:loading="bulkRegenerationActive"
|
||||||
:disabled="Boolean(regeneratingResultId) || bulkRegenerationActive"
|
:disabled="Boolean(regeneratingResultId) || bulkRegenerationActive || evaluationActive"
|
||||||
@click="emit('regenerate:all')"
|
@click="emit('regenerate:all')"
|
||||||
>
|
>
|
||||||
<i v-if="!bulkRegenerationActive" class="fa fa-refresh" style="margin-right: 4px;" />
|
<i v-if="!bulkRegenerationActive" class="fa fa-refresh" style="margin-right: 4px;" />
|
||||||
{{ bulkRegenerationActive ? '重新生成中' : `全部重新生成(${invalidCount})` }}
|
{{ bulkRegenerationActive ? '重新生成中' : `全部重新生成(${invalidCount})` }}
|
||||||
</el-button>
|
</el-button>
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
<div v-if="evaluationVisible" class="bulk-regeneration-progress">
|
||||||
|
<div>
|
||||||
|
<span>数据评测 {{ evaluation.completed }} / {{ evaluation.total }}</span>
|
||||||
|
<span>成功 {{ evaluation.succeeded }} · 失败 {{ evaluation.failed }}</span>
|
||||||
|
</div>
|
||||||
|
<el-progress
|
||||||
|
:percentage="evaluationPercentage"
|
||||||
|
:show-text="false"
|
||||||
|
:stroke-width="5"
|
||||||
|
:color="evaluation.failed > 0 ? '#d97706' : '#5b50f2'"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
<div v-if="bulkRegenerationVisible" class="bulk-regeneration-progress">
|
<div v-if="bulkRegenerationVisible" class="bulk-regeneration-progress">
|
||||||
<div>
|
<div>
|
||||||
<span>已处理 {{ bulkRegeneration.completed }} / {{ bulkRegeneration.total }}</span>
|
<span>已处理 {{ bulkRegeneration.completed }} / {{ bulkRegeneration.total }}</span>
|
||||||
@@ -146,6 +184,23 @@ function selectRelative(offset: number) {
|
|||||||
<strong>{{ item.instruction || '未填写指令' }}</strong>
|
<strong>{{ item.instruction || '未填写指令' }}</strong>
|
||||||
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
|
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
|
||||||
</span>
|
</span>
|
||||||
|
<el-popover
|
||||||
|
v-if="item.qualityScore != null && item.qualityDetails"
|
||||||
|
placement="right"
|
||||||
|
:width="296"
|
||||||
|
trigger="hover"
|
||||||
|
:show-after="150"
|
||||||
|
popper-class="quality-radar-popper"
|
||||||
|
>
|
||||||
|
<template #reference>
|
||||||
|
<span
|
||||||
|
class="result-score"
|
||||||
|
:class="item.qualityScore >= 80 ? 'is-success' : item.qualityScore >= 60 ? 'is-warning' : 'is-danger'"
|
||||||
|
@click.stop
|
||||||
|
>{{ item.qualityScore.toFixed(0) }}</span>
|
||||||
|
</template>
|
||||||
|
<QualityRadarPopover :quality="item.qualityDetails" :score="item.qualityScore" />
|
||||||
|
</el-popover>
|
||||||
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
|
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
|
||||||
<i
|
<i
|
||||||
v-else
|
v-else
|
||||||
@@ -303,6 +358,42 @@ function selectRelative(offset: number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.result-list-actions {
|
||||||
|
display: flex;
|
||||||
|
flex: none;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-score {
|
||||||
|
flex: none;
|
||||||
|
min-width: 34px;
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 10px;
|
||||||
|
color: #475467;
|
||||||
|
background: #f2f4f7;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 700;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
text-align: center;
|
||||||
|
cursor: default;
|
||||||
|
|
||||||
|
&.is-success {
|
||||||
|
color: #067647;
|
||||||
|
background: #e6f4ee;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-warning {
|
||||||
|
color: #b54708;
|
||||||
|
background: #fef0c7;
|
||||||
|
}
|
||||||
|
|
||||||
|
&.is-danger {
|
||||||
|
color: #b42318;
|
||||||
|
background: #fee4e2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
.pane-header {
|
.pane-header {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
import type {
|
import type {
|
||||||
DataProcessOutputType,
|
DataProcessOutputType,
|
||||||
DataProcessPreviewFileStatus,
|
DataProcessPreviewFileStatus,
|
||||||
|
DataProcessQualityJudge,
|
||||||
|
DataProcessQualityLayers,
|
||||||
|
DataProcessQualitySemantic,
|
||||||
DataProcessReasoningDetail,
|
DataProcessReasoningDetail,
|
||||||
} from '@/types/dataProcess'
|
} from '@/types/dataProcess'
|
||||||
|
|
||||||
@@ -165,6 +168,14 @@ export interface GenerationState {
|
|||||||
message: string
|
message: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ResultQualityDetails {
|
||||||
|
semantic?: DataProcessQualitySemantic | null
|
||||||
|
judge?: DataProcessQualityJudge | null
|
||||||
|
layers?: DataProcessQualityLayers | null
|
||||||
|
evaluated?: boolean
|
||||||
|
flags?: string[]
|
||||||
|
}
|
||||||
|
|
||||||
export interface ResultItem {
|
export interface ResultItem {
|
||||||
id: string
|
id: string
|
||||||
previewItemId: string | null
|
previewItemId: string | null
|
||||||
@@ -188,7 +199,7 @@ export interface ResultItem {
|
|||||||
error?: string
|
error?: string
|
||||||
split?: 'train' | 'validation' | 'test'
|
split?: 'train' | 'validation' | 'test'
|
||||||
qualityScore?: number
|
qualityScore?: number
|
||||||
qualityDetails?: Record<string, number>
|
qualityDetails?: ResultQualityDetails
|
||||||
updatedAt?: string
|
updatedAt?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -201,3 +212,11 @@ export interface BulkResultRegenerationState {
|
|||||||
targetIds: string[]
|
targetIds: string[]
|
||||||
failedIds: string[]
|
failedIds: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ResultEvaluationState {
|
||||||
|
status: 'idle' | 'running' | 'completed' | 'partial' | 'failed'
|
||||||
|
total: number
|
||||||
|
completed: number
|
||||||
|
succeeded: number
|
||||||
|
failed: number
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
import { computed, reactive, type Ref } from 'vue'
|
||||||
|
import { ElMessage } from 'element-plus'
|
||||||
|
import { evaluateDataProcessResults } from '@/api/modules/dataProcess'
|
||||||
|
import { mapResult } from './useDataProcessGeneration'
|
||||||
|
import type { ResultEvaluationState, ResultItem } from './types'
|
||||||
|
|
||||||
|
interface EvaluationBindings {
|
||||||
|
taskId: Ref<string | null>
|
||||||
|
results: Ref<ResultItem[]>
|
||||||
|
selectedResultId: Ref<string | null>
|
||||||
|
}
|
||||||
|
|
||||||
|
// 与批量重新生成一致的分块大小:4 个后端 worker 消费三轮,
|
||||||
|
// 单条评测最长 60 秒,12 条在批量接口 240 秒超时预算内。
|
||||||
|
const EVALUATION_CHUNK_SIZE = 12
|
||||||
|
|
||||||
|
function hasUnsavedChanges(item: ResultItem) {
|
||||||
|
return item.instruction !== item.savedInstruction
|
||||||
|
|| item.input !== item.savedInput
|
||||||
|
|| item.output !== item.savedOutput
|
||||||
|
|| item.chosen !== item.savedChosen
|
||||||
|
|| item.rejected !== item.savedRejected
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useDataProcessEvaluation(bindings: EvaluationBindings) {
|
||||||
|
const evaluation = reactive<ResultEvaluationState>({
|
||||||
|
status: 'idle',
|
||||||
|
total: 0,
|
||||||
|
completed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
})
|
||||||
|
const evaluationBusy = computed(() => evaluation.status === 'running')
|
||||||
|
|
||||||
|
function resetEvaluation() {
|
||||||
|
Object.assign(evaluation, {
|
||||||
|
status: 'idle',
|
||||||
|
total: 0,
|
||||||
|
completed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateAllResults() {
|
||||||
|
const taskId = bindings.taskId.value
|
||||||
|
if (!taskId) return false
|
||||||
|
if (evaluationBusy.value) {
|
||||||
|
ElMessage.warning('请等待当前数据评测完成')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = bindings.results.value.filter((item) => item.updatedAt)
|
||||||
|
if (!candidates.length) {
|
||||||
|
ElMessage.info('当前没有可评测的结果')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
const unsaved = candidates.find(hasUnsavedChanges)
|
||||||
|
if (unsaved) {
|
||||||
|
bindings.selectedResultId.value = unsaved.id
|
||||||
|
ElMessage.warning('存在未保存的修改,请先保存后再进行数据评测')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
Object.assign(evaluation, {
|
||||||
|
status: 'running',
|
||||||
|
total: candidates.length,
|
||||||
|
completed: 0,
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
})
|
||||||
|
|
||||||
|
let interrupted = false
|
||||||
|
try {
|
||||||
|
for (let offset = 0; offset < candidates.length; offset += EVALUATION_CHUNK_SIZE) {
|
||||||
|
const chunk = candidates.slice(offset, offset + EVALUATION_CHUNK_SIZE)
|
||||||
|
try {
|
||||||
|
const evaluated = await evaluateDataProcessResults(taskId, {
|
||||||
|
items: chunk.map((item) => ({
|
||||||
|
result_id: item.id,
|
||||||
|
expected_updated_at: item.updatedAt as string,
|
||||||
|
})),
|
||||||
|
})
|
||||||
|
for (const item of evaluated.items) {
|
||||||
|
const index = bindings.results.value.findIndex((entry) => entry.id === String(item.id))
|
||||||
|
if (index >= 0) bindings.results.value[index] = mapResult(item)
|
||||||
|
}
|
||||||
|
evaluation.completed += evaluated.total
|
||||||
|
evaluation.succeeded += evaluated.succeeded
|
||||||
|
evaluation.failed += evaluated.failed
|
||||||
|
} catch {
|
||||||
|
const remaining = candidates.slice(offset)
|
||||||
|
evaluation.completed = evaluation.total
|
||||||
|
evaluation.failed += remaining.length
|
||||||
|
interrupted = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!interrupted && evaluation.failed === 0) {
|
||||||
|
evaluation.status = 'completed'
|
||||||
|
ElMessage.success(`数据评测完成:成功 ${evaluation.succeeded} 条`)
|
||||||
|
} else if (evaluation.succeeded > 0) {
|
||||||
|
evaluation.status = 'partial'
|
||||||
|
ElMessage.warning(
|
||||||
|
`数据评测完成:成功 ${evaluation.succeeded} 条,失败 ${evaluation.failed} 条`,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
evaluation.status = 'failed'
|
||||||
|
ElMessage.error(`数据评测失败:${evaluation.failed} 条结果未完成评测`)
|
||||||
|
}
|
||||||
|
return evaluation.failed === 0
|
||||||
|
} catch {
|
||||||
|
evaluation.status = 'failed'
|
||||||
|
ElMessage.error('批量数据评测意外中断,已完成的评分保持不变')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
evaluation,
|
||||||
|
evaluationBusy,
|
||||||
|
evaluateAllResults,
|
||||||
|
resetEvaluation,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,6 +28,7 @@ const POLL_INTERVAL_MS = 1500
|
|||||||
const BULK_REGENERATION_CHUNK_SIZE = 12
|
const BULK_REGENERATION_CHUNK_SIZE = 12
|
||||||
|
|
||||||
function mapResult(item: DataProcessResult): ResultItem {
|
function mapResult(item: DataProcessResult): ResultItem {
|
||||||
|
const quality = item.quality_score
|
||||||
return {
|
return {
|
||||||
id: String(item.id),
|
id: String(item.id),
|
||||||
previewItemId: item.preview_item_id == null ? null : String(item.preview_item_id),
|
previewItemId: item.preview_item_id == null ? null : String(item.preview_item_id),
|
||||||
@@ -50,11 +51,21 @@ function mapResult(item: DataProcessResult): ResultItem {
|
|||||||
status: item.status,
|
status: item.status,
|
||||||
error: item.error || undefined,
|
error: item.error || undefined,
|
||||||
split: item.split || undefined,
|
split: item.split || undefined,
|
||||||
qualityScore: item.quality_score?.overall,
|
// 生成阶段只有内部规则分,界面不展示;数据评测完成后才显示组合分。
|
||||||
|
qualityScore: quality?.evaluated ? quality.overall : undefined,
|
||||||
|
qualityDetails: {
|
||||||
|
semantic: quality?.semantic ?? null,
|
||||||
|
judge: quality?.judge ?? null,
|
||||||
|
layers: quality?.layers ?? null,
|
||||||
|
evaluated: Boolean(quality?.evaluated),
|
||||||
|
flags: quality?.flags ?? [],
|
||||||
|
},
|
||||||
updatedAt: item.updated_at,
|
updatedAt: item.updated_at,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export { mapResult }
|
||||||
|
|
||||||
export function useDataProcessGeneration(bindings: GenerationBindings) {
|
export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||||
const results = ref<ResultItem[]>([])
|
const results = ref<ResultItem[]>([])
|
||||||
const selectedResultId = ref<string | null>(null)
|
const selectedResultId = ref<string | null>(null)
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ const createdDimensionId = ref<string | number>('')
|
|||||||
const taskForm = ref<EvalTaskSetupDraft>({
|
const taskForm = ref<EvalTaskSetupDraft>({
|
||||||
eval_task_name: '',
|
eval_task_name: '',
|
||||||
model_id: '',
|
model_id: '',
|
||||||
gpu_id: '',
|
gpu_id: [],
|
||||||
data_source: 'dataset',
|
data_source: 'dataset',
|
||||||
dataset_id: '',
|
dataset_id: '',
|
||||||
leaderboard: false,
|
leaderboard: false,
|
||||||
@@ -145,12 +145,24 @@ async function handleSubmit() {
|
|||||||
const dimensionId = await resolveDimensionId()
|
const dimensionId = await resolveDimensionId()
|
||||||
// GPU 选择为「节点:GPU序号」复合值,解析出节点与 GPU 序号,
|
// GPU 选择为「节点:GPU序号」复合值,解析出节点与 GPU 序号,
|
||||||
// 多算力节点时必须把节点信息传给后端,否则会派发到错误的算力节点
|
// 多算力节点时必须把节点信息传给后端,否则会派发到错误的算力节点
|
||||||
const [gpuNodeId, gpuIndex] = String(taskForm.value.gpu_id).split(':')
|
const selectedGpuKeys = Array.isArray(taskForm.value.gpu_id)
|
||||||
|
? taskForm.value.gpu_id
|
||||||
|
: [String(taskForm.value.gpu_id)]
|
||||||
|
const gpuSelections = selectedGpuKeys
|
||||||
|
.map((key) => {
|
||||||
|
const [nodeId, gpuIndex] = String(key).split(':')
|
||||||
|
return { nodeId, gpuIndex: Number(gpuIndex) }
|
||||||
|
})
|
||||||
|
.filter((item) => item.nodeId && Number.isInteger(item.gpuIndex) && item.gpuIndex >= 0)
|
||||||
|
const gpuNodeId = gpuSelections[0]?.nodeId || ''
|
||||||
|
const gpuIndices = gpuSelections.map((item) => item.gpuIndex)
|
||||||
const evalResult: any = await startEval({
|
const evalResult: any = await startEval({
|
||||||
eval_task_name: taskForm.value.eval_task_name,
|
eval_task_name: taskForm.value.eval_task_name,
|
||||||
eval_type: 'custom',
|
eval_type: 'custom',
|
||||||
model_id: taskForm.value.model_id,
|
model_id: taskForm.value.model_id,
|
||||||
gpu_id: Number(gpuIndex) || 0,
|
gpu_id: gpuIndices[0] ?? 0,
|
||||||
|
gpu_indices: gpuIndices,
|
||||||
|
gpus: gpuIndices,
|
||||||
compute_node_id: gpuNodeId || '',
|
compute_node_id: gpuNodeId || '',
|
||||||
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
dataset_id: taskForm.value.data_source === 'dataset' ? taskForm.value.dataset_id : '',
|
||||||
dimension_id: dimensionId,
|
dimension_id: dimensionId,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import type { DatasetItem, GpuInfo, TrainedModel } from '@/types'
|
|||||||
export interface EvalTaskSetupDraft {
|
export interface EvalTaskSetupDraft {
|
||||||
eval_task_name: string
|
eval_task_name: string
|
||||||
model_id: string | number
|
model_id: string | number
|
||||||
gpu_id: string | number
|
gpu_id: string | number | string[]
|
||||||
data_source: 'dataset' | 'inference'
|
data_source: 'dataset' | 'inference'
|
||||||
dataset_id: string | number
|
dataset_id: string | number
|
||||||
leaderboard: boolean
|
leaderboard: boolean
|
||||||
@@ -23,6 +23,13 @@ defineProps<{
|
|||||||
const form = defineModel<EvalTaskSetupDraft>({ required: true })
|
const form = defineModel<EvalTaskSetupDraft>({ required: true })
|
||||||
const formRef = ref<FormInstance>()
|
const formRef = ref<FormInstance>()
|
||||||
|
|
||||||
|
function handleGpuChange(value: string | number | string[]) {
|
||||||
|
const keys = Array.isArray(value) ? value.map(String) : [String(value || '')]
|
||||||
|
const nodeId = keys[0]?.split(':', 1)[0]
|
||||||
|
if (!nodeId || !Array.isArray(form.value.gpu_id)) return
|
||||||
|
form.value.gpu_id = keys.filter((key) => key.split(':', 1)[0] === nodeId)
|
||||||
|
}
|
||||||
|
|
||||||
const rules: FormRules<EvalTaskSetupDraft> = {
|
const rules: FormRules<EvalTaskSetupDraft> = {
|
||||||
eval_task_name: [
|
eval_task_name: [
|
||||||
{ required: true, message: '请输入任务名称', trigger: 'blur' },
|
{ required: true, message: '请输入任务名称', trigger: 'blur' },
|
||||||
@@ -101,7 +108,16 @@ defineExpose({ validate })
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item label="选择 GPU" prop="gpu_id">
|
<el-form-item label="选择 GPU" prop="gpu_id">
|
||||||
<el-select v-model="form.gpu_id" placeholder="请选择 GPU" style="width: 100%" :loading="loading">
|
<el-select
|
||||||
|
v-model="form.gpu_id"
|
||||||
|
multiple
|
||||||
|
collapse-tags
|
||||||
|
collapse-tags-tooltip
|
||||||
|
placeholder="请选择同一算力节点内的一张或多张 GPU"
|
||||||
|
style="width: 100%"
|
||||||
|
:loading="loading"
|
||||||
|
@change="handleGpuChange"
|
||||||
|
>
|
||||||
<el-option
|
<el-option
|
||||||
v-for="gpu in gpus"
|
v-for="gpu in gpus"
|
||||||
:key="`${gpu.node_id || ''}:${gpu.id ?? 0}`"
|
:key="`${gpu.node_id || ''}:${gpu.id ?? 0}`"
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ function nameOf<T extends { id: string | number; name?: string }>(items: T[], id
|
|||||||
|
|
||||||
/** GPU 选择为「节点:GPU序号」复合值,解析并展示为可读标签 */
|
/** GPU 选择为「节点:GPU序号」复合值,解析并展示为可读标签 */
|
||||||
const gpuLabel = computed(() => {
|
const gpuLabel = computed(() => {
|
||||||
const key = String(props.task.gpu_id || '')
|
const key = Array.isArray(props.task.gpu_id) ? String(props.task.gpu_id[0] || '') : String(props.task.gpu_id || '')
|
||||||
const gpu = props.gpus.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === key)
|
const gpu = props.gpus.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === key)
|
||||||
if (gpu) return `${gpu.node_name || gpu.node_code || '算力节点'} / GPU ${gpu.id ?? 0}`
|
if (gpu) return `${gpu.node_name || gpu.node_code || '算力节点'} / GPU ${gpu.id ?? 0}`
|
||||||
const [nodeId, idx] = key.split(':')
|
const [nodeId, idx] = key.split(':')
|
||||||
|
|||||||
68
frontend/src/views/governance/OrganizationPermissionView.vue
Normal file
68
frontend/src/views/governance/OrganizationPermissionView.vue
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import UserSettingsView from '@/views/system/UserSettingsView.vue'
|
||||||
|
import TenantListView from '@/views/tenants/TenantListView.vue'
|
||||||
|
|
||||||
|
const route = useRoute()
|
||||||
|
const router = useRouter()
|
||||||
|
|
||||||
|
const activeTab = computed({
|
||||||
|
get: () => route.query.tab === 'tenants' ? 'tenants' : 'users',
|
||||||
|
set: (value: string) => {
|
||||||
|
void router.replace({ query: { tab: value === 'tenants' ? 'tenants' : 'users' } })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="organization-page">
|
||||||
|
<header class="page-header">
|
||||||
|
<div>
|
||||||
|
<h2>组织与权限</h2>
|
||||||
|
<p>统一管理平台用户、角色、租户和资源配额。</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<el-tabs v-model="activeTab">
|
||||||
|
<el-tab-pane label="用户与角色" name="users">
|
||||||
|
<UserSettingsView v-if="activeTab === 'users'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane label="租户与配额" name="tenants">
|
||||||
|
<TenantListView v-if="activeTab === 'tenants'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
</el-tabs>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.organization-page {
|
||||||
|
min-height: 100%;
|
||||||
|
padding: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.page-header {
|
||||||
|
margin-bottom: 4px;
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0;
|
||||||
|
color: #1f2937;
|
||||||
|
font-size: 22px;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
margin: 6px 0 0;
|
||||||
|
color: #64748b;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.user-settings),
|
||||||
|
:deep(.page) {
|
||||||
|
padding: 16px 0 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.user-settings .page-header > div:first-child) {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -4,8 +4,7 @@ import { useRouter } from 'vue-router'
|
|||||||
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
import { ElMessage, type FormInstance, type FormRules } from 'element-plus'
|
||||||
import PageCard from '@/components/PageCard.vue'
|
import PageCard from '@/components/PageCard.vue'
|
||||||
import { getModelList, getTrainedModels } from '@/api/modules/model'
|
import { getModelList, getTrainedModels } from '@/api/modules/model'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
import { getComputeGpus, getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
||||||
import { getComputeNodes, type ComputeNode } from '@/api/modules/compute'
|
|
||||||
import { createCompare, loadCompare } from '@/api/modules/compare'
|
import { createCompare, loadCompare } from '@/api/modules/compare'
|
||||||
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
|
import type { ModelItem, TrainedModel, GpuInfo } from '@/types'
|
||||||
|
|
||||||
@@ -83,8 +82,8 @@ const form = reactive({
|
|||||||
description: '',
|
description: '',
|
||||||
/** 选中的模型 key(单选) */
|
/** 选中的模型 key(单选) */
|
||||||
model_key: '',
|
model_key: '',
|
||||||
/** 使用的 GPU */
|
/** 使用的 GPU(同一节点内可多选) */
|
||||||
gpu_key: '',
|
gpu_keys: [] as string[],
|
||||||
})
|
})
|
||||||
|
|
||||||
const rules: FormRules = {
|
const rules: FormRules = {
|
||||||
@@ -94,12 +93,26 @@ const rules: FormRules = {
|
|||||||
|
|
||||||
/** 当前选中的模型对象 */
|
/** 当前选中的模型对象 */
|
||||||
const selectedModel = computed(() => modelMap.value[form.model_key])
|
const selectedModel = computed(() => modelMap.value[form.model_key])
|
||||||
const selectedGpu = computed(() => idleGpus.value.find((g) => `${g.node_id || ''}:${g.id ?? 0}` === form.gpu_key))
|
const selectedGpus = computed(() => idleGpus.value.filter((gpu) => form.gpu_keys.includes(gpuKey(gpu))))
|
||||||
|
|
||||||
|
function gpuKey(gpu: GpuInfo) {
|
||||||
|
return `${gpu.node_id || ''}:${gpu.id ?? 0}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleGpuChange(keys: string[]) {
|
||||||
|
const nodeId = keys[0]?.split(':', 1)[0]
|
||||||
|
if (!nodeId) return
|
||||||
|
const filtered = keys.filter((key) => key.split(':', 1)[0] === nodeId)
|
||||||
|
if (filtered.length !== keys.length) {
|
||||||
|
ElMessage.info('一次推理只能使用同一算力节点内的 GPU,已忽略其它节点的选择')
|
||||||
|
}
|
||||||
|
form.gpu_keys = filtered
|
||||||
|
}
|
||||||
|
|
||||||
watch(selectedModel, (model) => {
|
watch(selectedModel, (model) => {
|
||||||
if (!model?.compute_node_id) return
|
if (!model?.compute_node_id) return
|
||||||
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
|
const gpu = idleGpus.value.find((item) => item.node_id === model.compute_node_id)
|
||||||
if (gpu) form.gpu_key = `${gpu.node_id || ''}:${gpu.id ?? 0}`
|
if (gpu) form.gpu_keys = [gpuKey(gpu)]
|
||||||
})
|
})
|
||||||
|
|
||||||
async function handleSubmit() {
|
async function handleSubmit() {
|
||||||
@@ -111,6 +124,10 @@ async function handleSubmit() {
|
|||||||
ElMessage.warning('请选择模型')
|
ElMessage.warning('请选择模型')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (!selectedGpus.value.length) {
|
||||||
|
ElMessage.warning('请至少选择一张空闲 GPU')
|
||||||
|
return
|
||||||
|
}
|
||||||
submitting.value = true
|
submitting.value = true
|
||||||
startupStatus.value = '正在创建推理任务...'
|
startupStatus.value = '正在创建推理任务...'
|
||||||
try {
|
try {
|
||||||
@@ -130,9 +147,11 @@ async function handleSubmit() {
|
|||||||
model_name: m.name,
|
model_name: m.name,
|
||||||
model_path: m.model_path,
|
model_path: m.model_path,
|
||||||
source: m.source,
|
source: m.source,
|
||||||
gpu_id: selectedGpu.value?.id ?? 0,
|
gpu_id: selectedGpus.value[0]?.id ?? 0,
|
||||||
node_id: selectedGpu.value?.node_id || m.compute_node_id,
|
gpu_indices: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
|
||||||
node_name: selectedGpu.value?.node_name || m.compute_node_name,
|
gpus: selectedGpus.value.map((gpu) => Number(gpu.id ?? 0)),
|
||||||
|
node_id: selectedGpus.value[0]?.node_id || m.compute_node_id,
|
||||||
|
node_name: selectedGpus.value[0]?.node_name || m.compute_node_name,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
@@ -169,17 +188,17 @@ async function loadData() {
|
|||||||
const [db, trained, sys, nodes] = await Promise.all([
|
const [db, trained, sys, nodes] = await Promise.all([
|
||||||
getModelList(),
|
getModelList(),
|
||||||
getTrainedModels(),
|
getTrainedModels(),
|
||||||
getSystemInfo(),
|
getComputeGpus(),
|
||||||
getComputeNodes(),
|
getComputeNodes(),
|
||||||
])
|
])
|
||||||
dbModels.value = db || []
|
dbModels.value = db || []
|
||||||
trainedModels.value = trained?.models || []
|
trainedModels.value = trained?.models || []
|
||||||
gpus.value = sys?.gpu || []
|
gpus.value = (sys || []) as unknown as GpuInfo[]
|
||||||
computeNodes.value = nodes || []
|
computeNodes.value = nodes || []
|
||||||
// 默认选中第一个空闲 GPU
|
// 默认选中第一个空闲 GPU
|
||||||
if (idleGpus.value.length > 0) {
|
if (idleGpus.value.length > 0) {
|
||||||
const firstGpu = idleGpus.value[0]
|
const firstGpu = idleGpus.value[0]
|
||||||
form.gpu_key = `${firstGpu.node_id || ''}:${firstGpu.id ?? 0}`
|
form.gpu_keys = [gpuKey(firstGpu)]
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// ignore
|
||||||
@@ -228,12 +247,20 @@ onMounted(loadData)
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item label="GPU">
|
<el-form-item label="GPU">
|
||||||
<el-select v-model="form.gpu_key" style="width: 400px">
|
<el-select
|
||||||
|
v-model="form.gpu_keys"
|
||||||
|
multiple
|
||||||
|
collapse-tags
|
||||||
|
collapse-tags-tooltip
|
||||||
|
style="width: 400px"
|
||||||
|
placeholder="请选择同一算力节点内的一张或多张 GPU"
|
||||||
|
@change="handleGpuChange"
|
||||||
|
>
|
||||||
<el-option
|
<el-option
|
||||||
v-for="g in idleGpus"
|
v-for="g in idleGpus"
|
||||||
:key="`${g.node_id || ''}:${g.id ?? 0}`"
|
:key="gpuKey(g)"
|
||||||
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
:label="`${g.node_name || g.node_code || '算力节点'} / ${g.name} (GPU${g.id ?? 0}) [空闲]`"
|
||||||
:value="`${g.node_id || ''}:${g.id ?? 0}`"
|
:value="gpuKey(g)"
|
||||||
/>
|
/>
|
||||||
</el-select>
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|||||||
78
frontend/src/views/system/RuntimeLogsView.vue
Normal file
78
frontend/src/views/system/RuntimeLogsView.vue
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from 'vue'
|
||||||
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
|
import { useAuthStore } from '@/stores/auth'
|
||||||
|
import LogsView from './LogsView.vue'
|
||||||
|
import AuditLogView from '@/views/audit/AuditLogView.vue'
|
||||||
|
import OperationLogView from '@/views/audit/OperationLogView.vue'
|
||||||
|
|
||||||
|
const route = useRoute()
|
||||||
|
const router = useRouter()
|
||||||
|
const auth = useAuthStore()
|
||||||
|
|
||||||
|
const activeTab = computed({
|
||||||
|
get: () => {
|
||||||
|
if (!auth.isAdmin) return 'runtime'
|
||||||
|
if (route.query.tab === 'audit') return 'audit'
|
||||||
|
if (route.query.tab === 'operations') return 'operations'
|
||||||
|
return 'runtime'
|
||||||
|
},
|
||||||
|
set: (value: string) => {
|
||||||
|
void router.replace({ query: value === 'runtime' ? {} : { tab: value } })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="runtime-logs">
|
||||||
|
<header class="page-header">
|
||||||
|
<div>
|
||||||
|
<h2>运行日志</h2>
|
||||||
|
<p>查看系统运行、训练任务、审计记录和操作诊断信息。</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<el-tabs v-model="activeTab">
|
||||||
|
<el-tab-pane label="运行日志" name="runtime">
|
||||||
|
<LogsView v-if="activeTab === 'runtime'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane v-if="auth.isAdmin" label="审计记录" name="audit">
|
||||||
|
<AuditLogView v-if="activeTab === 'audit'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
<el-tab-pane v-if="auth.isAdmin" label="操作诊断" name="operations">
|
||||||
|
<OperationLogView v-if="activeTab === 'operations'" />
|
||||||
|
</el-tab-pane>
|
||||||
|
</el-tabs>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped lang="scss">
|
||||||
|
.runtime-logs {
|
||||||
|
min-height: 100%;
|
||||||
|
padding: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.page-header {
|
||||||
|
margin-bottom: 4px;
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin: 0;
|
||||||
|
color: #1f2937;
|
||||||
|
font-size: 22px;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
margin: 6px 0 0;
|
||||||
|
color: #64748b;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.page) {
|
||||||
|
padding: 16px 0 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
:deep(.page-header .page-title) {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -8,10 +8,9 @@ import TrainingTaskOverview from './training-log/TrainingTaskOverview.vue'
|
|||||||
import { usePolling } from '@/composables/usePolling'
|
import { usePolling } from '@/composables/usePolling'
|
||||||
import '@/plugins/echarts-training-log'
|
import '@/plugins/echarts-training-log'
|
||||||
import { useModelsStore } from '@/stores/models'
|
import { useModelsStore } from '@/stores/models'
|
||||||
import { getFineTune, getFineTuneDiagnostics, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
import { getFineTune, getFineTuneDiagnostics, getFineTuneGpuStatus, getFineTuneLogs, getFineTuneMetrics, type TrainingDiagnostic } from '@/api/modules/fineTune'
|
||||||
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
|
import { getTrainingLogFiles, getTrainingLogContent } from '@/api/modules/log'
|
||||||
import { getDataset } from '@/api/modules/dataset'
|
import { getDataset } from '@/api/modules/dataset'
|
||||||
import { getSystemInfo } from '@/api/modules/system'
|
|
||||||
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
|
import { TRAIN_TYPE_MAP, TRAIN_METHOD_MAP } from '@/constants'
|
||||||
import {
|
import {
|
||||||
buildMetricChartOption,
|
buildMetricChartOption,
|
||||||
@@ -205,15 +204,20 @@ async function loadDataset(datasetId: string | number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadGpuStatus() {
|
async function loadGpuStatus(currentTask: FineTuneTask) {
|
||||||
try {
|
try {
|
||||||
const systemInfo = await getSystemInfo()
|
const live = await getFineTuneGpuStatus(currentTask.id)
|
||||||
gpuPool.value = systemInfo.gpu ?? []
|
if (live.source === 'compute' && live.items.length) {
|
||||||
|
gpuPool.value = live.items as unknown as GpuInfo[]
|
||||||
gpuUpdatedAt.value = new Date()
|
gpuUpdatedAt.value = new Date()
|
||||||
gpuLoadError.value = ''
|
gpuLoadError.value = ''
|
||||||
|
return
|
||||||
|
}
|
||||||
|
gpuPool.value = []
|
||||||
|
gpuLoadError.value = live.error || 'Compute 节点暂未返回实时 GPU 指标'
|
||||||
} catch {
|
} catch {
|
||||||
gpuLoadError.value = 'GPU 监控数据暂时不可用'
|
gpuLoadError.value = 'GPU 监控数据暂时不可用'
|
||||||
if (!gpuUpdatedAt.value) gpuPool.value = []
|
gpuPool.value = []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,7 +310,7 @@ async function refreshAll() {
|
|||||||
const datasetPromise = currentTask.train_dataset_id
|
const datasetPromise = currentTask.train_dataset_id
|
||||||
? loadDataset(currentTask.train_dataset_id)
|
? loadDataset(currentTask.train_dataset_id)
|
||||||
: Promise.resolve()
|
: Promise.resolve()
|
||||||
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(), loadDiagnostics(currentTask)])
|
await Promise.all([datasetPromise, loadLog(currentTask), loadGpuStatus(currentTask), loadDiagnostics(currentTask)])
|
||||||
await loadMetrics(currentTask)
|
await loadMetrics(currentTask)
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
loading.value = false
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ const form = reactive<CreateUserPayload>({
|
|||||||
username: '',
|
username: '',
|
||||||
display_name: '',
|
display_name: '',
|
||||||
password: 'platform123',
|
password: 'platform123',
|
||||||
role: 'user',
|
role: 'operator',
|
||||||
status: 'active',
|
status: 'active',
|
||||||
permissions: [],
|
permissions: [],
|
||||||
})
|
})
|
||||||
@@ -45,7 +45,7 @@ async function submit() {
|
|||||||
<el-form-item label="角色">
|
<el-form-item label="角色">
|
||||||
<el-select v-model="form.role" style="width: 100%">
|
<el-select v-model="form.role" style="width: 100%">
|
||||||
<el-option label="管理员" value="admin" />
|
<el-option label="管理员" value="admin" />
|
||||||
<el-option label="普通用户" value="user" />
|
<el-option label="普通用户" value="operator" />
|
||||||
</el-select>
|
</el-select>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
<el-form-item label="状态">
|
<el-form-item label="状态">
|
||||||
|
|||||||
Reference in New Issue
Block a user