YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: 合并远程 ft_wyt 分支，解决冲突

Browse Source
This commit is contained in:
wangjiming
2026-08-19 17:39:18 +08:00
parent 78680a5c5c 080ef6ab00
commit 8455d99d49
64 changed files with 4616 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
backend/app/api/v1/endpoints/data_process.py
View File

@@ -35,7 +35,12 @@ from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.core.logging import get_structured_logger
from app.core.config import get_settings
from app.db.platform_store import get_platform_store
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -61,6 +66,10 @@ from app.modules.data_process.document_chunking import (
chunk_semantic_text,
merge_short_chunks,
)
from app.modules.data_process.evaluation import (
evaluate_result_record,
reevaluate_edited_record,
)
from app.modules.data_process.generation import generate_model_records
from app.modules.data_process.office_preview import (
MAX_XLSX_PREVIEW_ROWS,
@@ -82,6 +91,8 @@ from app.modules.data_process.store import (
new_id,
repeat_task_id,
)
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from app.schemas.data_process import (
DataProcessRegenerateRequest,
DataProcessRepeatRequest,
@@ -97,6 +108,7 @@ from app.schemas.data_process import (
PreviewItemUpdate,
ProcessType,
PublishRequest,
ResultBatchEvaluateRequest,
ResultBatchRegenerateRequest,
ResultRegenerateRequest,
ResultUpdate,
@@ -221,9 +233,38 @@ def _commit_source_batch(
staged: list[StagedSourceObject],
) -> list[dict[str, Any]]:
storage.publish(staged)
storage_object_ids: list[str] = []
try:
# The source reference remains in the task schema for compatibility,
# while storage_objects provides the authoritative MinIO index.
if get_settings().minio_enabled:
for item in prepared:
reference = str(item.get("storage_object_id") or "")
if not reference.startswith("minio://"):
continue
object_key = storage.object_key(reference)
metadata = get_object_storage().stat(object_key)
object_row = get_platform_store().create_storage_object({
"resource_type": "data_process_source",
"resource_id": task_id,
"version_id": str(item.get("id") or new_id("dpsf")),
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": item.get("name"),
"content_type": (item.get("metadata") or {}).get("content_type", "application/octet-stream"),
"byte_size": metadata.get("byte_size") or item.get("raw_size") or 0,
"checksum_sha256": item.get("checksum_sha256"),
"status": "available",
"created_by": item.get("created_by"),
})
storage_object_ids.append(str(object_row["id"]))
return store.add_source_files(task_id, prepared)
except Exception:
for object_id in storage_object_ids:
try:
get_platform_store().update_storage_object(object_id, {"status": "deleted"})
except Exception:
pass
for item in staged:
try:
storage.delete(item.reference)
@@ -771,6 +812,25 @@ def _run_generation(
duplicate_count=duplicate_count,
error_count=error_count,
)
result_bytes = "".join(
structured_json_dumps(item) + "\n" for item in accepted
).encode("utf-8")
if should_store_in_minio(len(result_bytes)):
result_key = f"data-process/{task_id}/results/{generation_run_id}.jsonl"
uploaded = get_object_storage().put_bytes(result_key, result_bytes, "application/jsonl")
get_platform_store().create_storage_object({
"resource_type": "data_process_result",
"resource_id": task_id,
"version_id": generation_run_id,
"bucket": uploaded["bucket"],
"object_key": result_key,
"file_name": f"{generation_run_id}.jsonl",
"content_type": "application/jsonl",
"byte_size": len(result_bytes),
"checksum_sha256": hashlib.sha256(result_bytes).hexdigest(),
"status": "available",
"created_by": (store.get_task(task_id) or {}).get("created_by"),
})
logger.info(
"data process generation completed task_id=%s generation_run_id=%s "
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
@@ -930,7 +990,7 @@ def _repeat_file_copies(
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
new_file_id = new_id("dpsf")
old_reference = str(source.get("storage_object_id") or "")
if old_reference.startswith("local://data-process/"):
if old_reference.startswith(("local://data-process/", "minio://data-process/")):
staged_object = storage.stage_copy(
batch_id=batch_id,
source_reference=old_reference,
@@ -2044,12 +2104,13 @@ def update_result(
or 20
),
)
quality = score_quality(
# 编辑后内容已变化：重算规则与语义层，旧的评审分不再可信直接丢弃。
update["quality_score"] = reevaluate_edited_record(
merged,
min_output_length=minimum,
source_content=source_content,
previous_quality=current.get("quality_score"),
min_output_length=minimum,
)
update["quality_score"] = asdict(quality)
result = store.update_result(
task_id,
result_id,
@@ -2094,11 +2155,6 @@ def restore_result(
or 20
),
)
quality = score_quality(
restored,
min_output_length=minimum,
source_content=source_content,
)
restored = store.update_result(
task_id,
result_id,
@@ -2108,7 +2164,12 @@ def restore_result(
"output": restored["output"],
"chosen": restored["chosen"],
"rejected": restored["rejected"],
"quality_score": asdict(quality),
"quality_score": reevaluate_edited_record(
restored,
source_content=source_content,
previous_quality=current.get("quality_score"),
min_output_length=minimum,
),
"expected_updated_at": current.get("updated_at"),
},
)
@@ -2271,6 +2332,46 @@ def _safe_regeneration_error(exc: Exception) -> str:
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
def _evaluate_result_in_place(
task_id: str,
current: dict[str, Any],
source_content: str,
config: dict[str, Any],
evaluation_model: dict[str, Any] | None,
store: DataProcessStore,
*,
expected_updated_at: str,
model_client: httpx.Client | None = None,
minimum: int = 20,
) -> dict[str, Any]:
"""评测单条结果并落库；复用逐结果互斥锁避免与重生成并发写冲突。"""
result_id = str(current["id"])
with _claim_result_regeneration(task_id, result_id):
quality = evaluate_result_record(
{
"instruction": current.get("instruction"),
"input": current.get("input"),
"output": current.get("output"),
"chosen": current.get("chosen"),
"rejected": current.get("rejected"),
},
source_content=source_content,
model=evaluation_model,
config=config,
client=model_client,
min_output_length=minimum,
)
return store.update_result(
task_id,
result_id,
{
"quality_score": quality,
"expected_updated_at": expected_updated_at,
},
)
@router.post("/{task_id}/results/regenerate-batch")
def regenerate_results_batch(
task_id: str,
@@ -2444,6 +2545,186 @@ def regenerate_results_batch(
)
@router.post("/{task_id}/results/evaluate-batch")
def evaluate_results_batch(
task_id: str,
payload: ResultBatchEvaluateRequest,
store: DataProcessStore = Depends(get_data_process_store),
) -> dict[str, Any]:
"""对一批结果执行三层质量评测（规则+语义+评审），允许部分成功。"""
started_at = time.perf_counter()
batch_id = new_id("dpeb")
with api_errors():
task = store.get_task(task_id)
if task.get("status") == "running":
raise ConflictError("data process task is running")
if task.get("output_dataset_id"):
raise InvalidStateError("published results cannot be evaluated")
config = task.get("config") or {}
evaluation_model: dict[str, Any] | None = None
model_id = _value(config, "generation_model_id", "generationModelId", None)
if model_id:
try:
evaluation_model = store.get_generation_model(str(model_id))
except NotFoundError:
logger.warning(
"data process evaluation model unavailable, judge layer "
"skipped task_id=%s model_id=%s",
task_id,
model_id,
)
evaluation_config = {
**config,
"output_type": str(
_value(config, "output_type", "outputType", "standard")
).strip().lower(),
}
minimum = max(
1,
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
)
prepared: list[tuple[int, dict[str, Any], str, str]] = []
failures: list[tuple[int, dict[str, str]]] = []
for index, requested in enumerate(payload.items):
try:
current = store.get_result(task_id, requested.result_id)
if requested.expected_updated_at != str(current.get("updated_at") or ""):
raise ConflictError("data process result was modified by another request")
source_content = ""
preview_id = current.get("preview_item_id")
if preview_id:
preview = store.get_preview_item(task_id, str(preview_id))
source_content = str(
preview.get("edited_content")
or preview.get("original_content")
or ""
)
prepared.append(
(index, current, source_content, requested.expected_updated_at)
)
except ConflictError as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "conflict",
"message": _safe_regeneration_error(exc),
}))
except (NotFoundError, InvalidStateError) as exc:
failures.append((index, {
"result_id": requested.result_id,
"code": "skipped",
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation started batch_id=%s task_id=%s "
"requested=%s prepared=%s judge_enabled=%s",
batch_id,
task_id,
len(payload.items),
len(prepared),
evaluation_model is not None,
)
successes: list[tuple[int, dict[str, Any]]] = []
if prepared:
try:
from app.modules.data_process.algorithms.embedding import (
semantic_embedding_model,
)
semantic_embedding_model()
except Exception:
logger.warning(
"data process semantic embedding unavailable, semantic layer "
"will be skipped batch_id=%s",
batch_id,
)
request_timeout = _result_regeneration_timeout(config)
model_timeout = httpx.Timeout(
request_timeout,
connect=min(10.0, request_timeout),
)
model_limits = httpx.Limits(
max_connections=RESULT_REGENERATION_CONCURRENCY,
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-evaluation",
) as executor:
futures = {
executor.submit(
_evaluate_result_in_place,
task_id,
current,
source_content,
evaluation_config,
evaluation_model,
store,
expected_updated_at=expected_updated_at,
model_client=model_client if evaluation_model else None,
minimum=minimum,
): (index, str(current["id"]), time.perf_counter())
for index, current, source_content, expected_updated_at in prepared
}
for future in as_completed(futures):
index, result_id, item_started_at = futures[future]
try:
evaluated = future.result()
successes.append((index, evaluated))
outcome = "succeeded"
except ConflictError as exc:
outcome = "conflict"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
except Exception as exc:
outcome = "evaluation_failed"
failures.append((index, {
"result_id": result_id,
"code": outcome,
"message": _safe_regeneration_error(exc),
}))
logger.info(
"data process result batch evaluation item finished "
"batch_id=%s task_id=%s result_id=%s outcome=%s duration_ms=%.2f",
batch_id,
task_id,
result_id,
outcome,
(time.perf_counter() - item_started_at) * 1000,
)
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
duration_ms = (time.perf_counter() - started_at) * 1000
logger.info(
"data process result batch evaluation completed batch_id=%s task_id=%s "
"succeeded=%s failed=%s duration_ms=%.2f",
batch_id,
task_id,
len(success_items),
len(failure_items),
duration_ms,
)
return ok(
{
"batch_id": batch_id,
"total": len(payload.items),
"succeeded": len(success_items),
"failed": len(failure_items),
"duration_ms": round(duration_ms, 2),
"items": success_items,
"failures": failure_items,
},
"data process results evaluated",
)
@router.post("/{task_id}/results/{result_id}/regenerate")
def regenerate_result(
task_id: str,

447
backend/app/api/v1/endpoints/platform.py
View File

@@ -23,8 +23,9 @@ from app.core.audit import audit_log, AuditActions
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.compute_gateway.sync import fetch_eval_result_content, poll_compute_jobs_once
from app.modules.compute_gateway.sync import _archive_node_directory, fetch_eval_result_content, poll_compute_jobs_once
from app.modules.storage.minio_store import ObjectStorageError, get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter()
_LOGIN_FAILURES: dict[str, list[float]] = {}
@@ -57,6 +58,33 @@ def _select_first_online_node(store: Any) -> dict[str, Any] | None:
return None
def _normalize_gpu_indices(payload: dict[str, Any], *, allow_primary: bool = True) -> list[int]:
"""Normalize all frontend GPU selection shapes to sorted integer indexes."""
raw = payload.get("gpu_indices")
if raw is None:
raw = payload.get("gpus")
if raw is None and allow_primary and payload.get("gpu_id") is not None:
raw = [payload.get("gpu_id")]
if raw is None or raw == "":
return []
if isinstance(raw, str):
raw = [item.strip() for item in raw.split(",") if item.strip()]
if not isinstance(raw, (list, tuple, set)):
raw = [raw]
result: set[int] = set()
for item in raw:
if isinstance(item, str) and ":" in item:
item = item.rsplit(":", 1)[-1]
try:
index = int(item)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid GPU index: {item}") from exc
if index < 0:
raise ValueError("GPU index must be non-negative")
result.add(index)
return sorted(result)
async def _wait_for_object_storage() -> None:
"""Wait for MinIO before starting a resource task."""
settings = get_settings()
@@ -73,6 +101,106 @@ async def _wait_for_object_storage() -> None:
await asyncio.sleep(max(1, settings.storage_check_interval_seconds))
def _store_json_snapshot(
store: Any,
resource_type: str,
resource_id: str,
version_id: str,
object_key: str,
payload: dict[str, Any],
created_by: str | None = None,
) -> dict[str, Any]:
"""Persist non-secret task/model parameters as an auditable MinIO snapshot."""
sanitized = {
key: value
for key, value in payload.items()
if key not in {"api_key", "secret_key", "password", "token", "access_token"}
}
raw = json.dumps(sanitized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
# Task/evaluation payloads are already persisted in PostgreSQL. Avoid an
# extra MinIO round trip for small non-secret parameter snapshots.
if not should_store_in_minio(len(raw), content_type="application/json", file_format="json"):
return {}
uploaded = get_object_storage().put_bytes(object_key, raw, "application/json")
return store.create_storage_object({
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": Path(object_key).name,
"content_type": "application/json",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
"created_by": created_by,
})
def _dataset_file_bytes(store: Any, file_id: str) -> bytes:
"""Return the canonical dataset bytes, lazily indexing legacy DB content."""
row = store.dataset_file(file_id)
if not get_settings().minio_enabled:
return str(row.get("content") or "").encode("utf-8")
storage_object = None
object_id = str(row.get("storage_object_id") or "")
if object_id:
try:
storage_object = store.storage_object(object_id)
except KeyError:
storage_object = None
if storage_object and storage_object.get("status") == "available":
try:
return get_object_storage().get_bytes(storage_object["object_key"])
except Exception as exc: # noqa: BLE001 - expose storage outage to callers
raise RuntimeError(f"dataset object is unavailable in MinIO: {exc}") from exc
# Compatibility migration for files created before MinIO was enabled.
raw = str(row.get("content") or "").encode("utf-8")
if not raw:
raise RuntimeError(f"dataset file has no MinIO object or legacy content: {file_id}")
# Small files intentionally remain database-backed. They can still be
# copied to a compute node directly when a task needs them.
if not should_store_in_minio(len(raw), file_format=row.get("file_format")):
return raw
object_key = (
f"datasets/{row['dataset_id']}/versions/"
f"{row.get('active_version_id') or row['id']}/{Path(str(row.get('name') or row['id'])).name}"
)
uploaded = get_object_storage().put_bytes(object_key, raw, "application/octet-stream")
created = store.create_storage_object({
"resource_type": "dataset",
"resource_id": str(row["dataset_id"]),
"version_id": str(row.get("active_version_id") or row["id"]),
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": row.get("name"),
"content_type": "application/octet-stream",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(row["id"]), created["id"])
return raw
def _dataset_version_bytes(store: Any, file_id: str, version_id: str) -> bytes:
row = store.dataset_file(file_id)
try:
version = next(item for item in store.file_versions(file_id)["versions"] if item["id"] == version_id)
except StopIteration as exc:
raise KeyError(version_id) from exc
object_id = str(version.get("storage_object_id") or "")
if get_settings().minio_enabled and object_id:
try:
obj = store.storage_object(object_id)
return get_object_storage().get_bytes(obj["object_key"])
except KeyError:
pass
return _dataset_file_bytes(store, file_id)
def _select_eval_node(store: Any, preferred_node_id: str | None = None) -> dict[str, Any] | None:
"""Select the compute node for an eval job.
@@ -103,18 +231,63 @@ async def _prepare_resource_on_node(store: Any, resource_type: str, resource_id:
if not get_settings().minio_enabled or not resource_id:
return None
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects and resource_type in {"model", "trained_model"}:
resource = None
if resource_type == "model":
try:
resource = store.model(resource_id)
except KeyError:
try:
resource = store.model_by_name(resource_id)
except KeyError:
resource = next((item for item in store.models() if item.get("path") == resource_id), None)
else:
resource = next(
(item for item in store.trained_models() if item.get("id") == resource_id or item.get("name") == resource_id),
None,
)
source_path = str(
(resource or {}).get("path")
or (resource or {}).get("merged_path")
or (resource or {}).get("artifact_dir")
or ""
)
resolved_id = str((resource or {}).get("id") or resource_id)
if source_path and resolved_id:
client = ComputeNodeClient(node["api_base_url"], timeout=900)
await _archive_node_directory(
store,
client,
node,
source_path,
resource_type,
resolved_id,
"legacy-import",
f"models/{resolved_id}" if resource_type == "model" else f"trained_models/{resolved_id}",
)
resource_id = resolved_id
objects = store.storage_objects_for_resource(resource_type, resource_id)
if not objects:
return None
client = ComputeNodeClient(node["api_base_url"], timeout=900)
root_name = "trained_models" if resource_type in {"trained_model", "model_artifact"} else f"{resource_type}s"
for obj in objects:
object_key = str(obj["object_key"])
marker = f"{root_name}/{resource_id}/versions/"
relative_name = Path(str(obj.get("file_name") or object_key)).name
if marker in object_key:
suffix = object_key.split(marker, 1)[1]
if "/" in suffix:
suffix = suffix.split("/", 1)[1]
if suffix:
relative_name = suffix
await client.prepare_cache({
"resource_id": resource_id,
"version_id": obj["version_id"],
"download_url": get_object_storage().presigned_get(obj["object_key"]),
"download_url": get_object_storage().presigned_get(object_key),
"checksum_sha256": obj.get("checksum_sha256") or "",
"byte_size": obj.get("byte_size") or 0,
"relative_path": f"{root_name}/{resource_id}/{Path(str(obj.get('file_name') or obj['object_key'])).name}",
"relative_path": f"{root_name}/{resource_id}/{relative_name}",
})
return f"/data/yg-ft/{root_name}/{resource_id}"
@@ -370,8 +543,23 @@ async def _submit_fine_tune_task(store: Any, payload: dict[str, Any]) -> dict[st
if not preflight["valid"]:
errors = "; ".join(preflight.get("errors") or ["preflight failed"])
raise RuntimeError(f"preflight failed: {errors}")
payload = {**payload, "compute_node_id": preflight["node"]["id"]}
prepared_job_payload = preflight.get("job_payload") or {}
payload = {
**payload,
"compute_node_id": preflight["node"]["id"],
"prepared_base_model_path": prepared_job_payload.get("model_name_or_path") or payload.get("prepared_base_model_path"),
}
task = store.start_task(payload)
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"fine_tune",
str(task["id"]),
str(task["id"]),
f"training/{task['id']}/versions/{task['id']}/training-config.json",
task,
task.get("created_by"),
)
if get_settings().compute_mode == "simulator":
return task
node, job_payload = store.build_compute_job_payload(task["id"])
@@ -427,6 +615,20 @@ async def _fine_tune_preflight_with_job_payload(
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
await _wait_for_object_storage()
base_model_path = str(job_payload.get("model_name_or_path") or job_payload.get("base_model") or "")
base_model_id = str(job_payload.get("base_model_id") or job_payload.get("model_id") or "")
base_model = None
if base_model_id:
try:
base_model = store.model(base_model_id)
except KeyError:
base_model = None
if base_model is None:
base_model = next((item for item in store.models() if item.get("path") == base_model_path), None)
if base_model:
prepared_model = await _prepare_resource_on_node(store, "model", str(base_model["id"]), node)
if prepared_model:
job_payload = {**job_payload, "base_model": prepared_model, "model_name_or_path": prepared_model}
except Exception as exc: # noqa: BLE001 - preflight exposes node storage failure
sync_errors.append(f"shared storage health check failed: {exc}")
if get_settings().compute_mode == "simulator":
@@ -1076,8 +1278,9 @@ async def merge_model(payload: dict[str, Any] = Body(...), current_user: dict =
@router.get("/dataset-manage/preview/{file_id}")
async def dataset_preview(file_id: str) -> dict[str, Any]:
try:
row = get_platform_store().dataset_file(file_id)
return ok({"content": row["content"]})
store = get_platform_store()
content = _dataset_file_bytes(store, file_id).decode("utf-8", errors="replace")
return ok({"content": content})
except KeyError:
raise fail(404, "dataset file not found")
@@ -1106,7 +1309,8 @@ async def dataset_version_content(file_id: str, version_id: str) -> dict[str, An
version = next((item for item in versions if item["id"] == version_id), None)
if not version:
raise KeyError(version_id)
return ok({"version": version, "content": row["content"]})
content = _dataset_version_bytes(get_platform_store(), file_id, version_id)
return ok({"version": version, "content": content.decode("utf-8", errors="replace")})
except KeyError:
raise fail(404, "dataset version not found")
@@ -1197,7 +1401,7 @@ async def _sync_training_dataset_to_compute_node(
) -> list[dict[str, Any]]:
if get_settings().minio_enabled:
files = store.training_dataset_files(dataset_id)
object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {}
object_by_resource_name: dict[tuple[str, str, str], dict[str, Any]] = {}
resource_ids = {str(dataset_id)} | {
str(item.get("dataset_id"))
for item in files
@@ -1206,35 +1410,58 @@ async def _sync_training_dataset_to_compute_node(
for resource_id in resource_ids:
for obj in store.storage_objects_for_resource("dataset", resource_id):
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
object_by_resource_name[(resource_id, file_name)] = obj
object_by_resource_name[(resource_id, file_name, str(obj.get("version_id") or ""))] = obj
results: list[dict[str, Any]] = []
client = ComputeNodeClient(node["api_base_url"])
for item in files:
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
item_dataset_id = str(item.get("dataset_id") or dataset_id)
obj = object_by_resource_name.get((item_dataset_id, target_name))
version_id = str(item.get("active_version_id") or item["id"])
obj = object_by_resource_name.get((item_dataset_id, target_name, version_id))
if not obj and item.get("content"):
# 兼容 MinIO 接入前已经发布的数据处理数据集：
# 预检时用数据库正文补建对象，避免要求用户重新处理数据集。
# 兼容 MinIO 接入前已经发布的数据处理数据集。大文件补建
# MinIO 对象，小文件直接从数据库正文同步到目标节点。
raw = str(item.get("content") or "").encode("utf-8")
version_id = str(item.get("active_version_id") or item["id"])
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
obj = store.create_storage_object({
"resource_type": "dataset",
"resource_id": item_dataset_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": target_name,
"content_type": "application/jsonl",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
if should_store_in_minio(len(raw)):
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
obj = store.create_storage_object({
"resource_type": "dataset",
"resource_id": item_dataset_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": target_name,
"content_type": "application/jsonl",
"byte_size": len(raw),
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
"status": "available",
})
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
else:
result = await client.upload_file(
target_name,
raw,
f"datasets/{dataset_id}/{target_name}",
resource_type="dataset",
resource_id=dataset_id,
)
store.upsert_resource_replica(
node["id"], "dataset", dataset_id, str(result.get("local_path") or "")
)
results.append({
"node_id": node["id"],
"node_code": node.get("code"),
"file_id": item.get("id"),
"name": target_name,
"local_path": result.get("local_path"),
"byte_size": result.get("byte_size"),
"checksum_sha256": result.get("checksum_sha256"),
"storage_backend": "database",
})
continue
if not obj:
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
raise RuntimeError(f"dataset file is not available: {target_name}")
url = get_object_storage().presigned_get(obj["object_key"])
result = await client.prepare_cache({
"resource_id": dataset_id,
@@ -1250,7 +1477,13 @@ async def _sync_training_dataset_to_compute_node(
dataset_id,
str(result.get("local_path") or ""),
)
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
results.append({
**result,
"file_id": item.get("id"),
"name": target_name,
"node_id": node["id"],
"storage_backend": "minio",
})
return results
if not dataset_id:
raise RuntimeError("train_dataset_id is required")
@@ -1316,7 +1549,11 @@ async def upload_dataset_files(
created_file = store.add_dataset_file(conn, dataset_id, file.filename or "upload.jsonl", content)
created.append(created_file)
pending_sync.append((created_file["id"], created_file["name"], raw))
if get_settings().minio_enabled:
if should_store_in_minio(
len(raw),
content_type=file.content_type,
file_format=Path(created_file["name"]).suffix,
):
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
storage_object = get_platform_store().create_storage_object({
@@ -1360,7 +1597,10 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
full_file = store.dataset_file(str(item["id"]))
except KeyError:
continue
files.append({**item, "content": full_file.get("content") or ""})
files.append({
**item,
"content": _dataset_file_bytes(store, str(item["id"])).decode("utf-8", errors="replace"),
})
if not files:
raise fail(404, "dataset has no downloadable files")
@@ -1401,8 +1641,12 @@ async def download_dataset(dataset_id: str, current_user: dict = Depends(get_cur
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
async def download_dataset_file(dataset_id: str, file_id: str, version_id: str | None = Query(default=None)) -> PlainTextResponse:
row = get_platform_store().dataset_file(file_id)
return PlainTextResponse(row["content"], media_type="text/plain")
store = get_platform_store()
row = store.dataset_file(file_id)
if str(row.get("dataset_id")) != str(dataset_id):
raise fail(404, "dataset file not found")
content = _dataset_version_bytes(store, file_id, version_id) if version_id else _dataset_file_bytes(store, file_id)
return PlainTextResponse(content.decode("utf-8", errors="replace"), media_type="text/plain")
@router.get("/dataset-manage")
@@ -1545,10 +1789,10 @@ async def start_fine_tune(
if node_id and gpu_indices:
if not store.check_gpu_access(current_user["id"], node_id, gpu_indices):
raise fail(403, "无权使用所选 GPU，请联系管理员分配")
# 记录创建者
if node_id and not gpu_indices:
payload["allowed_gpu_indices"] = store.assigned_gpu_indexes(current_user["id"], node_id)
payload["strict_node_selection"] = bool(node_id)
# 页面明确选择节点时，调度器必须保持节点约束；否则可能落到其它节点。
payload["strict_node_selection"] = bool(payload.get("compute_node_id") or payload.get("node_id"))
payload.setdefault("created_by", current_user.get("id"))
try:
return ok(await _submit_fine_tune_task(store, payload))
@@ -1679,6 +1923,42 @@ async def fine_tune_diagnostics(task_id: str) -> dict[str, Any]:
)
@router.get("/fine-tune/{task_id}/gpu-status")
async def fine_tune_gpu_status(task_id: str, current_user: dict[str, Any] = Depends(get_current_user)) -> dict[str, Any]:
"""Return live GPU metrics for the task's selected node and cards."""
store = get_platform_store()
try:
task = store.task(task_id)
except KeyError:
raise fail(404, "fine tune task not found")
if not has_resource_access("fine-tune", task_id, current_user, "read"):
raise fail(403, "no permission to access this task")
node = _node_for_task(task)
selected = set(_normalize_gpu_indices({"gpus": task.get("gpus") or []}, allow_primary=False))
if not node:
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected)})
try:
if get_settings().compute_mode == "simulator":
live_items = store.gpus()
else:
live_items = await ComputeNodeClient(node["api_base_url"]).gpu_resources()
items = []
for item in live_items:
index = int(item.get("gpu_index", item.get("id", -1)))
if selected and index not in selected:
continue
items.append({
**item,
"id": index,
"node_id": node["id"],
"node_code": node.get("code"),
"node_name": node.get("name"),
})
return ok({"source": "compute", "items": items, "selected_gpus": sorted(selected)})
except Exception as exc: # noqa: BLE001 - let UI retain last good snapshot
return ok({"source": "unavailable", "items": [], "selected_gpus": sorted(selected), "error": str(exc)})
@router.put("/fine-tune/{task_id}")
@op_log(module=OpModule.FINE_TUNE, action=OpAction.UPDATE, target_type="fine_tune", target_name_param="task_id")
async def update_fine_tune(task_id: str, payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
@@ -1828,15 +2108,35 @@ async def model_eval_detail(task_id: str, current_user: dict = Depends(get_curre
async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
"""Start an evaluation task: submit eval job to compute node."""
store = get_platform_store()
try:
gpu_indices = _normalize_gpu_indices(payload)
except ValueError as exc:
raise fail(400, str(exc))
if not gpu_indices:
raise fail(400, "请选择至少一张 GPU")
# 1. Create eval task record
payload.setdefault("created_by", current_user.get("id"))
task = store.create_eval_task({**payload, "status": "pending"})
if get_settings().minio_enabled:
_store_json_snapshot(
store,
"eval",
str(task["id"]),
str(task["id"]),
f"evaluations/{task['id']}/versions/{task['id']}/evaluation-config.json",
payload,
current_user.get("id"),
)
# 2. Resolve model path (supports both regular models and trained models)
model_id = str(payload.get("model_id", ""))
model_path = ""
adapter_path = payload.get("adapter_path", "")
model_node_id = ""
ds_files: list[dict[str, Any]] = []
model_resource_type = "model"
model_resource_id = model_id
adapter_resource_id = ""
try:
db_model = store.model(model_id)
model_path = db_model.get("path", "")
@@ -1845,6 +2145,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
# Try trained_models table (IDs prefixed with tm_)
trained = next((m for m in store.trained_models() if m["id"] == model_id), None)
if trained:
model_resource_type = "trained_model" if trained.get("merged") else "model"
model_resource_id = trained.get("id") or model_id
adapter_resource_id = trained.get("id") or ""
model_node_id = trained.get("compute_node_id") or ""
merged_path = trained.get("merged_path", "")
base_path = trained.get("base_model_path", "")
@@ -1858,6 +2161,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
adapter_path = merged_path
else:
model_path = merged_path or base_path
if model_resource_type == "model":
base_model = next((item for item in store.models() if item.get("path") == base_path), None)
model_resource_id = str((base_model or {}).get("id") or base_path)
if not model_path:
store.update_eval_task(task["id"], {"status": "failed", "error": "model not found or no path"})
return ok({"task_id": task["id"], "status": "failed", "error": "model not found or no path"})
@@ -1930,6 +2236,33 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
if get_settings().minio_enabled and get_settings().compute_mode != "simulator":
try:
prepared_model = await _prepare_resource_on_node(store, model_resource_type, model_resource_id, node)
if prepared_model:
model_path = prepared_model
if adapter_resource_id and model_resource_type == "model":
prepared_adapter = await _prepare_resource_on_node(store, "trained_model", adapter_resource_id, node)
if prepared_adapter:
adapter_path = prepared_adapter
dataset_sync = await _sync_training_dataset_to_compute_node(store, node, dataset_id)
if dataset_sync:
dataset_path = str(dataset_sync[0].get("local_path") or dataset_path)
except Exception as exc:
store.update_eval_task(task["id"], {"status": "failed", "error": f"MinIO resource preparation failed: {exc}"})
return ok({"task_id": task["id"], "status": "failed", "error": str(exc)})
node_gpus = {
int(item.get("id", item.get("gpu_index", -1))): item
for item in store.gpus()
if item.get("node_id") == node["id"]
}
unavailable = [index for index in gpu_indices if node_gpus.get(index, {}).get("status") != "idle"]
if unavailable:
message = f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}"
store.update_eval_task(task["id"], {"status": "failed", "error": message})
return ok({"task_id": task["id"], "status": "failed", "error": message})
# 6. Build eval job payload
output_dir = f"/data/yg-ft/outputs/{task['id']}"
job_payload = {
@@ -1943,7 +2276,9 @@ async def model_eval_start(payload: dict[str, Any] = Body(...), current_user: di
"output_dir": output_dir,
"basic_metrics": payload.get("basic_metrics", {}),
"dimension": dimension_cfg,
"gpus": [int(payload.get("gpu_id", 0))],
"gpu_id": gpu_indices[0],
"gpu_indices": gpu_indices,
"gpus": gpu_indices,
"temperature": payload.get("temperature", 0.1),
"max_new_tokens": payload.get("max_new_tokens", 512),
"compute_node_id": node["id"],
@@ -1991,7 +2326,10 @@ async def model_eval_delete(task_id: str, current_user: dict = Depends(get_curre
pending = _require_approval_or_admin("eval", task_id, current_user, f"删除评测任务 {task_id}")
if pending:
return pending
get_platform_store().delete_eval_task(task_id)
try:
get_platform_store().delete_eval_task(task_id)
except KeyError:
raise fail(404, "eval task not found")
return ok({"deleted": task_id})
@@ -2247,6 +2585,15 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
"model_name_or_path": model_path,
"template": item.get("template", "qwen"),
}
try:
item_gpu_indices = _normalize_gpu_indices(item)
except ValueError as exc:
loaded_models.append({**item, "status": "error", "error": str(exc)})
continue
if not item_gpu_indices:
loaded_models.append({**item, "status": "error", "error": "no GPU selected"})
continue
load_payload["gpu_indices"] = item_gpu_indices
if item.get("adapter_path"):
load_payload["adapter_name_or_path"] = item["adapter_path"]
if get_settings().compute_mode == "simulator":
@@ -2255,14 +2602,28 @@ async def model_compare_load(task_id: str, current_user: dict = Depends(get_curr
# 只派发：HTTP 响应成功即视为已接受（节点会异步加载），loaded 字段忽略
item_dispatched = False
errors = []
for node in _candidate_online_nodes(store, preferred_node_id):
candidate_nodes = _candidate_online_nodes(store, preferred_node_id)
if preferred_node_id:
candidate_nodes = candidate_nodes[:1]
for node in candidate_nodes:
try:
node_gpu_map = {
int(gpu.get("id", gpu.get("gpu_index", -1))): gpu
for gpu in store.gpus()
if gpu.get("node_id") == node["id"]
}
unavailable = [
index for index in item_gpu_indices
if node_gpu_map.get(index, {}).get("status") != "idle"
]
if unavailable:
raise RuntimeError(f"selected GPU is not idle on compute node {node.get('code')}: {unavailable}")
if get_settings().minio_enabled:
await _wait_for_object_storage()
client = ComputeNodeClient(node["api_base_url"])
await client.inference_load(load_payload)
store.mark_inference_loaded(node["id"])
loaded_models.append({**item, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
store.mark_inference_loaded(node["id"], item_gpu_indices)
loaded_models.append({**item, "gpu_indices": item_gpu_indices, "gpus": item_gpu_indices, "status": "starting", "node_id": node["id"], "node_name": node.get("name")})
item_dispatched = True
break
except Exception as exc: # noqa: BLE001 - try next candidate node
@@ -2365,7 +2726,7 @@ async def model_chat_local_preload(payload: dict[str, Any] = Body(...)) -> dict[
# 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
result = await client.inference_load(payload)
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"])
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})
@@ -2445,7 +2806,7 @@ async def model_chat_trained_preload(payload: dict[str, Any] = Body(...), curren
# 计算节点现在异步加载：HTTP 接受（loading/ready）即视为派发成功
result = await client.inference_load({**payload, "compute_node_id": node["id"]})
if result.get("loaded") or result.get("status") in {"loading", "ready"}:
store.mark_inference_loaded(node["id"])
store.mark_inference_loaded(node["id"], _normalize_gpu_indices(payload))
return ok(result)
except Exception as exc:
return ok({"loaded": False, "error": str(exc)})

13
backend/app/core/audit.py
View File

@@ -61,6 +61,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -87,6 +88,7 @@ def audit_log(
detail = _build_detail(detail_template, kwargs)
_record_audit(
action=action,
actor_id=_extract_actor_id(kwargs),
target_type=target_type,
target_id=target_id,
detail=detail,
@@ -136,6 +138,7 @@ def _build_detail(template: str, kwargs: dict) -> str:
def _record_audit(
action: str,
actor_id: Optional[str],
target_type: str,
target_id: Optional[str],
detail: str,
@@ -149,6 +152,7 @@ def _record_audit(
store = get_platform_store()
store.record_audit(
action=action,
actor_id=actor_id,
target_type=target_type or None,
target_id=target_id,
detail=f"{detail} trace_id={trace_id} duration_ms={duration_ms:.1f}" if detail else f"trace_id={trace_id} duration_ms={duration_ms:.1f}",
@@ -157,6 +161,15 @@ def _record_audit(
logger.error("写入审计日志失败 action=%s", action, exc_info=True)
def _extract_actor_id(kwargs: dict) -> Optional[str]:
"""从 FastAPI 注入的当前用户中提取操作人 ID。"""
for key in ("current_user", "user"):
value = kwargs.get(key)
if isinstance(value, dict) and value.get("id"):
return str(value["id"])
return None
# ==================== 预定义的审计操作常量 ====================
class AuditActions:

3
backend/app/core/config.py
View File

@@ -68,6 +68,9 @@ class Settings:
minio_secret_key: str = os.getenv("MINIO_SECRET_KEY", "minioadmin")
minio_bucket: str = os.getenv("MINIO_BUCKET", "yg-ft-resources")
minio_secure: bool = _bool_env("MINIO_SECURE", False)
# Small text/data files stay inline in PostgreSQL to avoid unnecessary
# MinIO round trips. Larger files remain the shared canonical objects.
minio_inline_max_bytes: int = _int_env("MINIO_INLINE_MAX_BYTES", 256 * 1024)
storage_wait_seconds: int = _int_env("STORAGE_WAIT_SECONDS", 300)
storage_check_interval_seconds: int = _int_env("STORAGE_CHECK_INTERVAL_SECONDS", 10)
compute_service_token: str = os.getenv("COMPUTE_SERVICE_TOKEN", "")

140
backend/app/db/platform_store.py
View File

@@ -454,15 +454,21 @@ class PlatformStore:
self.ensure_seed_data()
# Track which compute nodes have an active inference model loaded
self._inference_nodes: set[str] = set()
self._inference_gpu_indexes: dict[str, set[int]] = {}
self._last_runtime_refresh = 0.0
# ── inference node tracking ────────────────────────────────────
def mark_inference_loaded(self, node_id: str) -> None:
def mark_inference_loaded(self, node_id: str, gpu_indexes: list[int] | None = None) -> None:
self._inference_nodes.add(node_id)
if gpu_indexes is not None:
self._inference_gpu_indexes[node_id] = {int(item) for item in gpu_indexes}
else:
self._inference_gpu_indexes.pop(node_id, None)
def mark_inference_unloaded(self, node_id: str) -> None:
self._inference_nodes.discard(node_id)
self._inference_gpu_indexes.pop(node_id, None)
def is_inference_loaded(self, node_id: str) -> bool:
return node_id in self._inference_nodes
@@ -545,6 +551,16 @@ class PlatformStore:
"last_error": "TEXT",
},
)
self._ensure_columns(
conn,
"storage_objects",
{"metadata": "TEXT NOT NULL DEFAULT '{}'"},
)
self._ensure_columns(
conn,
"model_artifacts",
{"storage_object_id": "TEXT", "storage_backend": "TEXT NOT NULL DEFAULT 'minio'"},
)
schema_dir = Path(__file__).with_name("sql")
for extra in (
"002_governance.sql",
@@ -556,7 +572,17 @@ class PlatformStore:
if extra_path.exists():
conn.executescript(extra_path.read_text(encoding="utf-8"))
# data_convert_tasks 表补充 created_by 字段（用于数据隔离）
self._ensure_columns(conn, "data_convert_tasks", {"created_by": "TEXT"})
self._ensure_columns(
conn,
"data_convert_tasks",
{
"created_by": "TEXT",
"storage_backend": "TEXT NOT NULL DEFAULT 'minio'",
"output_storage_object_id": "TEXT",
"output_content": "TEXT",
},
)
self._ensure_columns(conn, "eval_tasks", {"report_storage_object_id": "TEXT"})
# 修复历史数据：将 data_convert_tasks.created_by 回填到关联的 datasets 记录
try:
conn.execute("""
@@ -1254,6 +1280,13 @@ class PlatformStore:
raise KeyError(artifact_id)
return {**dict(row), "metadata": json_loads(row["metadata"], {})}
def link_model_artifact_storage_object(self, artifact_id: str, storage_object_id: str) -> None:
with self.connect() as conn:
conn.execute(
"UPDATE model_artifacts SET storage_object_id=?, storage_backend='minio' WHERE id=?",
(storage_object_id, artifact_id),
)
def model_lineage(self, model_id: str) -> dict[str, Any]:
with self.connect() as conn:
parents = conn.execute(
@@ -2221,7 +2254,7 @@ class PlatformStore:
(str(validation_dataset_id),),
).fetchall(),
]
model_path = (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
model_path = task.get("prepared_base_model_path") or (model and model.get("path")) or task.get("model_name_or_path") or base_model_id
dataset_metadata = json_loads(dataset.get("metadata"), {}) if dataset else {}
if not dataset or dataset.get("type") != "train" or dataset_metadata.get(
"dataset_split"
@@ -2482,12 +2515,18 @@ class PlatformStore:
def eval_tasks(self) -> list[dict[str, Any]]:
with self.connect() as conn:
rows = conn.execute("SELECT * FROM eval_tasks ORDER BY create_time DESC").fetchall()
# 评测任务采用软删除，普通列表不得再次返回已删除记录。
rows = conn.execute(
"SELECT * FROM eval_tasks WHERE deleted_at IS NULL ORDER BY create_time DESC"
).fetchall()
return [self._enrich_eval_payload(conn, self._json_payload_row(row)) for row in rows]
def eval_task(self, task_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM eval_tasks WHERE id=?", (task_id,)).fetchone()
row = conn.execute(
"SELECT * FROM eval_tasks WHERE id=? AND deleted_at IS NULL",
(task_id,),
).fetchone()
if not row:
raise KeyError(task_id)
payload = self._enrich_eval_payload(conn, self._json_payload_row(row))
@@ -2558,7 +2597,13 @@ class PlatformStore:
def delete_eval_task(self, task_id: str) -> None:
with self.connect() as conn:
conn.execute("UPDATE eval_tasks SET deleted_at=?, deleted_by=? WHERE id=?", (utcnow(), "system", task_id))
result = conn.execute(
"UPDATE eval_tasks SET deleted_at=?, deleted_by=? "
"WHERE id=? AND deleted_at IS NULL RETURNING id",
(utcnow(), "system", task_id),
)
if not result.fetchone():
raise KeyError(task_id)
def running_eval_tasks(self) -> list[dict[str, Any]]:
"""Return eval tasks that have been submitted to a compute node and are still running."""
@@ -2766,7 +2811,35 @@ class PlatformStore:
"SELECT gpu_index FROM gpu_allocations WHERE node_id=? AND status IN ('allocated','running')",
(node_id,),
).fetchall()
return {int(row["gpu_index"]) for row in rows}
active = {int(row["gpu_index"]) for row in rows}
# Evaluation jobs use the same Compute ProcessManager GPU lock but do
# not have fine-tune allocation rows; derive their selected cards here
# so a training task cannot race onto an evaluation GPU.
for row in conn.execute(
"SELECT payload FROM eval_tasks WHERE status IN ('syncing','queued','running')"
).fetchall():
payload = json_loads(row["payload"], {})
if payload.get("compute_node_id") != node_id:
continue
selected = payload.get("gpu_indices") or payload.get("gpus")
if selected is None and payload.get("gpu_id") is not None:
selected = [payload.get("gpu_id")]
active.update(int(item) for item in selected or [])
# Loaded inference models also reserve only their selected cards.
active.update(self._inference_gpu_indexes.get(node_id, set()))
for row in conn.execute("SELECT payload FROM compare_tasks").fetchall():
payload = json_loads(row["payload"], {})
load_status = payload.get("load_status") or {}
if isinstance(load_status, str):
load_status = json_loads(load_status, {})
for item in load_status.get("loaded_models") or []:
if item.get("node_id") != node_id or item.get("status") not in {"starting", "ready", "running"}:
continue
selected = item.get("gpu_indices") or item.get("gpus")
if selected is None and item.get("gpu_id") is not None:
selected = [item.get("gpu_id")]
active.update(int(gpu) for gpu in selected or [])
return active
def _node_gpu_indexes(self, conn: PgConnection, node: dict[str, Any]) -> set[int]:
rows = conn.execute("SELECT gpu_index FROM gpus WHERE node_id=?", (node["id"],)).fetchall()
@@ -3000,17 +3073,18 @@ class PlatformStore:
"""
INSERT INTO storage_objects
(id, resource_type, resource_id, version_id, bucket, object_key, file_name,
content_type, checksum_sha256, byte_size, status, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
content_type, checksum_sha256, byte_size, status, metadata, created_by, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (resource_type, resource_id, version_id, object_key)
DO UPDATE SET file_name=EXCLUDED.file_name, content_type=EXCLUDED.content_type,
checksum_sha256=EXCLUDED.checksum_sha256, byte_size=EXCLUDED.byte_size,
status=EXCLUDED.status, created_by=EXCLUDED.created_by
status=EXCLUDED.status, metadata=EXCLUDED.metadata, created_by=EXCLUDED.created_by
""",
(
object_id, payload["resource_type"], payload["resource_id"], payload["version_id"],
payload["bucket"], payload["object_key"], payload.get("file_name"), payload.get("content_type"),
payload.get("checksum_sha256"), int(payload.get("byte_size") or 0), payload.get("status", "pending"),
json_dumps(payload.get("metadata") or {}),
payload.get("created_by"), payload.get("create_time") or utcnow(),
),
)
@@ -3049,6 +3123,13 @@ class PlatformStore:
).fetchall()
return [dict(row) for row in rows]
def storage_object(self, object_id: str) -> dict[str, Any]:
with self.connect() as conn:
row = conn.execute("SELECT * FROM storage_objects WHERE id=?", (object_id,)).fetchone()
if not row:
raise KeyError(object_id)
return dict(row)
def update_storage_object(self, object_id: str, payload: dict[str, Any]) -> dict[str, Any]:
allowed = {"status", "checksum_sha256", "byte_size", "content_type"}
fields = {key: value for key, value in payload.items() if key in allowed}
@@ -3195,6 +3276,9 @@ class PlatformStore:
# 推理模型占用算力节点同样计入：优先从 compare_tasks 持久化状态派生
# （重启后仍准确），并用内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3204,8 +3288,13 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
for nid in inference_node_ids:
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
for nid in set(inference_node_ids) | set(inference_gpu_indexes):
running_map[nid] = running_map.get(nid, 0) + 1
rows = conn.execute("SELECT * FROM compute_nodes ORDER BY scheduler_weight DESC, code").fetchall()
return [
@@ -3434,6 +3523,9 @@ class PlatformStore:
# 推理模型占用的节点：优先从 compare_tasks 持久化状态派生（重启后仍准确），
# 内存标记兜底（直接 preload 的模型无 compare 记录）
inference_node_ids = set(self._inference_nodes)
inference_gpu_indexes: dict[str, set[int]] = {
node_id: set(indexes) for node_id, indexes in self._inference_gpu_indexes.items()
}
for ctr in conn.execute("SELECT payload FROM compare_tasks").fetchall():
ls = json_loads(ctr["payload"], {}).get("load_status") or {}
if isinstance(ls, str):
@@ -3443,7 +3535,12 @@ class PlatformStore:
ls = {}
for m in ls.get("loaded_models") or []:
if m.get("status") in {"ready", "running"} and m.get("node_id"):
inference_node_ids.add(m["node_id"])
node_id = m["node_id"]
selected = m.get("gpu_indices") or m.get("gpus")
if selected:
inference_gpu_indexes.setdefault(node_id, set()).update(int(item) for item in selected)
else:
inference_node_ids.add(node_id)
items = []
for row in rows:
task = next(
@@ -3459,7 +3556,10 @@ class PlatformStore:
t
for t in eval_running
if t.get("compute_node_id") == row["node_id"]
and row["gpu_index"] == (int(t["gpu_id"]) if t.get("gpu_id") is not None else -1)
and row["gpu_index"] in {
int(item)
for item in (t.get("gpu_indices") or t.get("gpus") or ([t["gpu_id"]] if t.get("gpu_id") is not None else []))
}
),
None,
)
@@ -3468,7 +3568,8 @@ class PlatformStore:
eval_task is not None and eval_task.get("status") in {"syncing", "queued"}
)
# Also mark GPU as busy if an inference model is loaded on this node
if row["node_id"] in inference_node_ids and not busy:
inference_on_gpu = row["node_id"] in inference_node_ids or row["gpu_index"] in inference_gpu_indexes.get(row["node_id"], set())
if inference_on_gpu and not busy:
busy = True
reserved = False
memory_used = round(row["memory_total_gb"] * (0.72 if busy else 0.18 if reserved else 0.04), 1)
@@ -3704,6 +3805,8 @@ class PlatformStore:
actor_id: str | None = None,
action: str | None = None,
target_type: str | None = None,
target_id: str | None = None,
keyword: str | None = None,
start_time: str | None = None,
end_time: str | None = None,
limit: int = 50,
@@ -3726,6 +3829,13 @@ class PlatformStore:
if target_type:
clauses.append("target_type=?")
params.append(target_type)
if target_id:
clauses.append("target_id=?")
params.append(target_id)
if keyword:
clauses.append("(target_id LIKE ? OR detail LIKE ?)")
pattern = f"%{keyword}%"
params.extend([pattern, pattern])
if start_time:
clauses.append("time>=?")
params.append(start_time)

11
backend/app/db/sql/000_full_init.sql
View File

@@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS model_artifacts (
path TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum_sha256 TEXT,
storage_object_id TEXT,
storage_backend TEXT NOT NULL DEFAULT 'minio',
metadata TEXT NOT NULL,
compute_job_id TEXT,
create_time TEXT NOT NULL
@@ -303,6 +305,7 @@ CREATE TABLE IF NOT EXISTS storage_objects (
checksum_sha256 TEXT,
byte_size BIGINT NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending',
metadata TEXT NOT NULL DEFAULT '{}',
created_by TEXT,
create_time TEXT NOT NULL,
UNIQUE (resource_type, resource_id, version_id, object_key)
@@ -630,6 +633,9 @@ ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAUL
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now();
ALTER TABLE dataset_files ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_object_id TEXT;
ALTER TABLE model_artifacts ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE storage_objects ADD COLUMN IF NOT EXISTS metadata TEXT NOT NULL DEFAULT '{}';
-- ---- 数据处理任务 / 源文件 / 预览 / 结果 ----
@@ -860,12 +866,17 @@ CREATE TABLE IF NOT EXISTS data_convert_tasks (
input_count INTEGER NOT NULL DEFAULT 0,
output_count INTEGER NOT NULL DEFAULT 0,
error_message TEXT,
output_content TEXT,
create_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
update_time TEXT NOT NULL DEFAULT (to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
deleted_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_status ON data_convert_tasks(status);
CREATE INDEX IF NOT EXISTS idx_data_convert_tasks_create_time ON data_convert_tasks(create_time DESC);
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'minio';
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_storage_object_id TEXT;
ALTER TABLE data_convert_tasks ADD COLUMN IF NOT EXISTS output_content TEXT;
ALTER TABLE eval_tasks ADD COLUMN IF NOT EXISTS report_storage_object_id TEXT;
-- ============================================================================
-- 七、种子数据：初始管理员 / 操作员

4
backend/app/modules/compute_gateway/client.py
View File

@@ -240,6 +240,10 @@ class ComputeNodeClient:
async def inference_status(self) -> dict[str, Any]:
return await self._request("GET", "/inference/status", timeout=INFERENCE_STATUS_TIMEOUT)
async def gpu_resources(self) -> list[dict[str, Any]]:
"""Read live per-GPU metrics from this compute node."""
return await self.gpus()
async def inference_unload(self) -> dict[str, Any]:
return await self._request("POST", "/inference/unload", json_data={}, timeout=INFERENCE_UNLOAD_TIMEOUT)

131
backend/app/modules/compute_gateway/sync.py
View File

@@ -2,15 +2,75 @@ from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from app.db.platform_store import get_platform_store
from app.core.config import get_settings
from app.modules.compute_gateway.client import ComputeNodeClient
from app.modules.storage.minio_store import get_object_storage
# starting 状态允许的最大轮询次数（约 40 * 3s ≈ 2 分钟），超过即判定节点不可达
MAX_STARTING_ATTEMPTS = 40
async def _archive_node_directory(
store: Any,
client: ComputeNodeClient,
node: dict[str, Any],
source_path: str,
resource_type: str,
resource_id: str,
version_id: str,
object_prefix: str,
) -> list[dict[str, Any]]:
"""Archive a completed node directory to MinIO, preserving subdirectories."""
data_root = Path(str(node.get("data_root") or "/data/yg-ft")).resolve()
source = Path(source_path).resolve()
try:
relative_root = source.relative_to(data_root).as_posix()
except ValueError as exc:
raise RuntimeError(f"artifact path is outside compute data root: {source_path}") from exc
queue = [relative_root]
archived: list[dict[str, Any]] = []
while queue:
relative = queue.pop(0)
listing = await client.list_files(root="data", relative_path=relative)
for item in listing.get("items") or []:
item_relative = str(item.get("relative_path") or "")
if item.get("type") == "directory":
queue.append(item_relative)
continue
path = str(item.get("path") or "")
if not path:
continue
try:
relative_file = Path(item_relative).relative_to(Path(relative_root)).as_posix()
except ValueError:
relative_file = Path(str(item.get("name") or Path(path).name)).name
object_key = f"{object_prefix}/{version_id}/{relative_file}"
upload_url = get_object_storage().presigned_put(object_key)
result = await client.upload_file_to_url(path, upload_url, object_key)
metadata = get_object_storage().stat(object_key)
archived.append(
store.create_storage_object(
{
"resource_type": resource_type,
"resource_id": resource_id,
"version_id": version_id,
"bucket": get_object_storage().bucket,
"object_key": object_key,
"file_name": relative_file,
"content_type": "application/octet-stream",
"byte_size": metadata.get("byte_size") or result.get("byte_size") or 0,
"checksum_sha256": result.get("checksum_sha256") or "",
"status": "available",
}
)
)
return archived
def _node_for_task(task: dict[str, Any]) -> dict[str, Any] | None:
return next((node for node in get_platform_store().compute_nodes() if node["id"] == task.get("compute_node_id")), None)
@@ -73,7 +133,8 @@ async def reconcile_inference_loads(store: Any) -> list[dict[str, Any]]:
if node_status == "ready":
item["status"] = "ready"
item.pop("error", None)
store.mark_inference_loaded(node["id"])
selected_gpus = item.get("gpu_indices") or item.get("gpus")
store.mark_inference_loaded(node["id"], selected_gpus)
elif node_status == "error":
item["status"] = "error"
item["error"] = status.get("error") or "model load failed on compute node"
@@ -138,7 +199,38 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
job["log_snippet"] = str(last_logs.get("content") or "")[:8192]
except Exception:
pass
synced.append(store.apply_compute_job(task["id"], job))
updated_task = store.apply_compute_job(task["id"], job)
if (
get_settings().minio_enabled
and job.get("status") == "completed"
and job.get("output_dir")
):
trained_model = next(
(
item
for item in store.trained_models()
if item.get("name")
== (task.get("output_model_name") or f"{task.get('name')}-lora")
),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or task.get("compute_job_id") or task["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
synced.append(updated_task)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"task_id": task["id"], "error": str(exc)})
standalone_synced: list[dict[str, Any]] = []
@@ -150,6 +242,30 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
try:
job = await ComputeNodeClient(node["api_base_url"]).get_job(record["id"])
standalone_synced.append(store.sync_model_merge_job(record["id"], job))
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
payload = (store.compute_job(record["id"]).get("payload") or {})
trained_model_id = str(payload.get("trained_model_id") or payload.get("model_name") or "")
if trained_model_id:
trained_model = next(
(item for item in store.trained_models() if item.get("id") == trained_model_id or item.get("name") == trained_model_id),
None,
)
if trained_model:
archived = await _archive_node_directory(
store,
ComputeNodeClient(node["api_base_url"], timeout=900),
node,
str(job["output_dir"]),
"trained_model",
str(trained_model["id"]),
str(job.get("id") or record["id"]),
f"trained_models/{trained_model['id']}",
)
artifacts = store.model_artifacts(str(trained_model["id"]))
if archived and artifacts:
store.link_model_artifact_storage_object(
str(artifacts[0]["id"]), str(archived[0]["id"])
)
except Exception as exc: # noqa: BLE001 - keep polling other jobs
failed.append({"job_id": record["id"], "error": str(exc)})
@@ -174,6 +290,17 @@ async def poll_compute_jobs_once() -> dict[str, Any]:
except Exception:
pass
store.apply_eval_job_result(eval_task["id"], job, result_content)
if get_settings().minio_enabled and job.get("status") == "completed" and job.get("output_dir"):
await _archive_node_directory(
store,
client,
node,
str(job["output_dir"]),
"eval",
str(eval_task["id"]),
str(job.get("id") or eval_task.get("compute_job_id") or eval_task["id"]),
f"evaluations/{eval_task['id']}",
)
# 评测 GPU 占用由 eval_tasks 状态派生，无需维护推理内存标记
eval_synced += 1
except Exception as exc: # noqa: BLE001

311
backend/app/modules/data_convert/router.py
View File

@@ -1,17 +1,21 @@
from __future__ import annotations
import json
import hashlib
import os
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi.responses import FileResponse
from fastapi.responses import FileResponse, Response
from app.api.v1.endpoints.platform import ok, fail
from app.core.auth import get_current_user, is_admin
from app.core.config import get_settings
from app.core.op_log import op_log, OpModule, OpAction
from app.db.platform_store import get_platform_store, new_id
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
router = APIRouter(prefix="/data-convert", tags=["data-convert"])
@@ -56,6 +60,142 @@ def _output_dir(task_id: str) -> Path:
return _task_dir(task_id) / "output"
def _minio_enabled() -> bool:
return bool(get_settings().minio_enabled)
def _input_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/input/{Path(name).name}"
def _output_object_key(task_id: str, name: str) -> str:
return f"data-convert/{task_id}/output/{Path(name).name}"
def _task_objects(task_id: str) -> list[dict[str, Any]]:
return get_platform_store().storage_objects_for_resource("data_convert", task_id)
def _register_object(
task_id: str,
*,
version_id: str,
object_key: str,
file_name: str,
content_type: str,
content: bytes,
created_by: str | None,
) -> dict[str, Any]:
storage = get_object_storage()
uploaded = storage.put_bytes(object_key, content, content_type)
return get_platform_store().create_storage_object(
{
"resource_type": "data_convert",
"resource_id": task_id,
"version_id": version_id,
"bucket": uploaded["bucket"],
"object_key": object_key,
"file_name": file_name,
"content_type": content_type,
"byte_size": len(content),
"checksum_sha256": hashlib.sha256(content).hexdigest(),
"status": "available",
"created_by": created_by,
}
)
def _input_objects(task_id: str) -> list[dict[str, Any]]:
prefix = f"data-convert/{task_id}/input/"
return sorted(
[item for item in _task_objects(task_id) if str(item.get("object_key") or "").startswith(prefix)],
key=lambda item: str(item.get("file_name") or item.get("object_key") or ""),
)
def _output_object(task: dict[str, Any]) -> dict[str, Any] | None:
key = _output_object_key(task["id"], _safe_output_filename(task.get("output_filename")))
return next((item for item in _task_objects(task["id"]) if item.get("object_key") == key), None)
def _read_output(task: dict[str, Any]) -> bytes | None:
if _minio_enabled():
item = _output_object(task)
if item:
return get_object_storage().get_bytes(item["object_key"])
inline = task.get("output_content")
return str(inline).encode("utf-8") if inline is not None else None
path = _task_output_path(task)
return path.read_bytes() if path.exists() else None
def _convert_from_minio(task: dict[str, Any], created_by: str | None) -> tuple[int, int, bytes]:
output_name = _safe_output_filename(task.get("output_filename"))
output_lines: list[str] = []
input_count = 0
output_count = 0
for item in _input_objects(task["id"]):
input_count += 1
raw = get_object_storage().get_bytes(item["object_key"])
data = json.loads(raw.decode("utf-8"))
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {item.get('file_name')}")
for record in records:
output_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
output = "".join(output_lines).encode("utf-8")
store = get_platform_store()
with store.connect() as conn:
if should_store_in_minio(len(output)):
output_object = _register_object(
task["id"],
version_id="output",
object_key=_output_object_key(task["id"], output_name),
file_name=output_name,
content_type="application/jsonl",
content=output,
created_by=created_by,
)
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=%s, output_content=NULL, storage_backend='minio' WHERE id=%s",
(output_object["id"], task["id"]),
)
else:
conn.execute(
"UPDATE data_convert_tasks SET output_storage_object_id=NULL, output_content=%s, storage_backend='database' WHERE id=%s",
(output.decode("utf-8"), task["id"]),
)
return input_count, output_count, output
def _convert_from_local(task: dict[str, Any]) -> tuple[int, int, bytes]:
input_dir = _input_dir(task["id"])
output_dir = _output_dir(task["id"])
input_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
output_path.unlink(missing_ok=True)
input_count = 0
output_count = 0
with output_path.open("w", encoding="utf-8") as output_file:
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
data = json.loads(json_file.read_text(encoding="utf-8"))
records = data if isinstance(data, list) else [data] if isinstance(data, dict) else None
if records is None:
raise ValueError(f"JSON must be object or array: {json_file.name}")
for record in records:
output_file.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
return input_count, output_count, output_path.read_bytes()
@router.get("")
def list_tasks(
page: int = 1,
@@ -109,9 +249,10 @@ def create_task(
"VALUES (%s, %s, %s, %s, %s)",
(task_id, name, description, output_filename, user_id),
)
# 创建目录
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
# MinIO 是正式存储；本地目录只在关闭 MinIO 的旧兼容模式下创建。
if not _minio_enabled():
_input_dir(task_id).mkdir(parents=True, exist_ok=True)
_output_dir(task_id).mkdir(parents=True, exist_ok=True)
return ok(_get_task(task_id))
@@ -123,13 +264,19 @@ def get_task(
task = _get_task(task_id)
if not task:
raise fail(404, "task not found")
# 附加输入文件列表
input_dir = _input_dir(task_id)
# 附加输入文件列表；旧任务没有对象记录时继续读取本地兼容目录。
files = []
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
if _minio_enabled():
files = [
{"name": item.get("file_name") or Path(item["object_key"]).name, "size": item.get("byte_size") or 0}
for item in _input_objects(task_id)
]
else:
input_dir = _input_dir(task_id)
if input_dir.exists():
for f in sorted(input_dir.iterdir()):
if f.is_file():
files.append({"name": f.name, "size": f.stat().st_size})
task["input_files"] = files
return ok(task)
@@ -146,16 +293,26 @@ async def upload_source_files(
raise fail(404, "task not found")
if task["status"] not in ("pending", "uploaded"):
raise fail(400, "task is not editable")
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
staged = []
for upload in files:
name = Path(upload.filename or "input.json").name
if not name.lower().endswith(".json"):
raise fail(415, f"only JSON files are supported: {name}")
target = input_dir / name
content = await upload.read()
target.write_bytes(content)
if _minio_enabled():
_register_object(
task_id,
version_id=f"input-{hashlib.sha256(name.encode('utf-8')).hexdigest()[:16]}",
object_key=_input_object_key(task_id, name),
file_name=name,
content_type=upload.content_type or "application/json",
content=content,
created_by=task.get("created_by") or current_user.get("id"),
)
else:
input_dir = _input_dir(task_id)
input_dir.mkdir(parents=True, exist_ok=True)
(input_dir / name).write_bytes(content)
staged.append({"name": name, "size": len(content)})
store = get_platform_store()
# 标记上传完成
@@ -166,30 +323,12 @@ async def upload_source_files(
)
# 自动转换并导入数据集
try:
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
# 清空旧输出（如果重新上传）
if output_path.exists():
output_path.unlink()
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, output = _convert_from_minio(
task, task.get("created_by") or current_user.get("id")
)
else:
input_count, output_count, output = _convert_from_local(task)
with store.connect() as conn:
conn.execute(
"UPDATE data_convert_tasks SET status='completed', "
@@ -197,12 +336,12 @@ async def upload_source_files(
(input_count, output_count, task_id),
)
# 自动导入数据集
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
size_bytes = len(content.encode("utf-8"))
dataset = store.create_dataset({
"name": task["name"],
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -212,7 +351,20 @@ async def upload_source_files(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or current_user.get("id"),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({
"staged_files": staged,
"auto_converted": True,
@@ -248,28 +400,10 @@ def run_convert(
(task_id,),
)
try:
input_dir = _input_dir(task_id)
output_dir = _output_dir(task_id)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = _task_output_path(task)
input_count = 0
output_count = 0
for json_file in sorted(input_dir.iterdir()):
if not json_file.is_file() or not json_file.name.lower().endswith(".json"):
continue
input_count += 1
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
records = data
elif isinstance(data, dict):
records = [data]
else:
raise ValueError(f"JSON must be object or array: {json_file.name}")
with open(output_path, "a", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
output_count += 1
if _minio_enabled():
input_count, output_count, _ = _convert_from_minio(task, task.get("created_by") or current_user.get("id"))
else:
input_count, output_count, _ = _convert_from_local(task)
# 更新任务状态
with store.connect() as conn:
conn.execute(
@@ -297,11 +431,15 @@ def download_result(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
if _minio_enabled():
return Response(content=output, media_type="application/octet-stream", headers={
"Content-Disposition": f"attachment; filename={_safe_output_filename(task.get('output_filename'))}"
})
return FileResponse(
str(output_path),
str(_task_output_path(task)),
media_type="application/octet-stream",
filename=_safe_output_filename(task.get("output_filename")),
)
@@ -319,10 +457,10 @@ def import_as_dataset(
raise fail(404, "task not found")
if task["status"] != "completed":
raise fail(400, "task is not completed")
output_path = _task_output_path(task)
if not output_path.exists():
output = _read_output(task)
if output is None:
raise fail(404, "output file not found")
content = output_path.read_text(encoding="utf-8")
content = output.decode("utf-8")
dataset_name = str(payload.get("name") or task["name"]).strip()
description = str(payload.get("description") or f"由数据类型转换任务 {task_id} 导入").strip()
size_bytes = len(content.encode("utf-8"))
@@ -331,7 +469,7 @@ def import_as_dataset(
dataset = store.create_dataset({
"name": dataset_name,
"type": "train",
"storage_type": "local",
"storage_type": "minio" if should_store_in_minio(size_bytes) else ("database" if _minio_enabled() else "local"),
"source": "upload",
"task_id": task_id,
"size": f"{size_bytes} B",
@@ -341,7 +479,20 @@ def import_as_dataset(
})
dataset_id = dataset["id"]
with store.connect() as conn:
store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
dataset_file = store.add_dataset_file(conn, dataset_id, _safe_output_filename(task.get("output_filename")), content)
if should_store_in_minio(len(output)):
output_name = _safe_output_filename(task.get("output_filename"))
object_key = f"datasets/{dataset_id}/versions/{dataset_file.get('active_version_id') or dataset_file['id']}/{output_name}"
uploaded = get_object_storage().put_bytes(object_key, output, "application/jsonl")
storage_object = store.create_storage_object({
"resource_type": "dataset", "resource_id": dataset_id,
"version_id": dataset_file.get("active_version_id") or dataset_file["id"],
"bucket": uploaded["bucket"], "object_key": object_key,
"file_name": output_name, "content_type": "application/jsonl",
"byte_size": len(output), "checksum_sha256": hashlib.sha256(output).hexdigest(),
"status": "available", "created_by": task.get("created_by") or (current_user.get("id") if current_user else None),
})
store.link_dataset_file_storage_object(dataset_file["id"], storage_object["id"])
return ok({"dataset_id": dataset_id, "name": dataset_name})
@@ -360,11 +511,19 @@ def delete_task(
"UPDATE data_convert_tasks SET deleted_at=NOW() WHERE id=%s",
(task_id,),
)
# 清理文件
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
if _minio_enabled():
for item in _task_objects(task_id):
try:
get_object_storage().delete(item["object_key"])
store.update_storage_object(item["id"], {"status": "deleted"})
except Exception:
pass
else:
# 旧兼容数据仍清理本地目录。
import shutil
task_dir = _task_dir(task_id)
if task_dir.exists():
shutil.rmtree(task_dir, ignore_errors=True)
return ok({"deleted": task_id})

24
backend/app/modules/data_process/algorithms/embedding.py Normal file
View File

@@ -0,0 +1,24 @@
"""数据处理算法 - 本地语义嵌入模型共享单例。"""
from __future__ import annotations
import os
from functools import lru_cache
from typing import Any
@lru_cache(maxsize=1)
def semantic_embedding_model() -> Any:
"""加载本地嵌入模型，供语义分块与语义质量评分共用。
模型可在部署环境覆盖；默认模型体积较小且适合中英文语义判断。
返回 LlamaIndex BaseEmbedding，通过 ``get_text_embedding`` 使用。
"""
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
return HuggingFaceEmbedding(
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
trust_remote_code=False,
)

20
backend/app/modules/data_process/algorithms/parsers/office.py
View File

@@ -7,12 +7,13 @@ import re
import unicodedata
import zipfile
import xml.etree.ElementTree as ET
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from pathlib import PurePosixPath
from typing import Any
from urllib.parse import unquote, urlsplit
from docx import Document
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
@@ -121,6 +122,21 @@ def _validate_office_archive(raw: bytes, file_format: TextFormat) -> None:
except zipfile.BadZipFile as exc:
raise ValueError(f"invalid {file_format.upper()} file: not an Office ZIP package") from exc
def iter_document_blocks(parent: Any) -> Iterator[Any]:
"""按文档顺序产出正文段落与表格，并下钻 SDT 内容控件。
Word 的目录、复选框等内容控件包在 ``w:sdt`` 元素里，只遍历 body
直接子级会把这些段落整段丢掉。
"""
for child in parent.iterchildren():
if isinstance(child, (CT_P, CT_Tbl)):
yield child
elif child.tag == qn("w:sdt"):
content = child.find(qn("w:sdtContent"))
if content is not None:
yield from iter_document_blocks(content)
def _extract_docx_text(raw: bytes) -> str:
_validate_office_archive(raw, "docx")
try:
@@ -130,7 +146,7 @@ def _extract_docx_text(raw: bytes) -> str:
parts: list[str] = []
total = 0
for child in document.element.body.iterchildren():
for child in iter_document_blocks(document.element.body):
if isinstance(child, CT_P):
total = _append_bounded_text(parts, Paragraph(child, document).text, total)
continue

85
backend/app/modules/data_process/algorithms/quality.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import math
import re
import unicodedata
from collections import Counter
@@ -341,3 +342,87 @@ def score_quality(
flags=tuple(flags),
fingerprint=fingerprint,
)
def _cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
if not left or not right or len(left) != len(right):
return 0.0
dot = math.fsum(a * b for a, b in zip(left, right))
norm_left = math.sqrt(math.fsum(a * a for a in left))
norm_right = math.sqrt(math.fsum(b * b for b in right))
if not norm_left or not norm_right:
return 0.0
return dot / (norm_left * norm_right)
def semantic_quality_scores(
record: Mapping[str, Any],
*,
source_content: str = "",
embed_model: Any = None,
) -> dict[str, Any] | None:
"""用本地嵌入向量计算语义相关性（0-100）。
返回 ``question_answer``（问题↔答案）、``answer_source``（答案↔来源，
无来源时缺省）与 ``overall``；嵌入模型不可用时返回 None 降级，不阻断流程。
"""
try:
if embed_model is None:
from .embedding import semantic_embedding_model
embed_model = semantic_embedding_model()
if embed_model is None:
return None
question = normalize_text(
" ".join(
str(record.get(field) or "")
for field in ("instruction", "input")
)
)
answer = normalize_text(
str(record.get("output") or "") or str(record.get("chosen") or "")
)
source = normalize_text(source_content)
texts = [text for text in {question, answer, source} if text]
if not texts:
return None
vectors = {text: embed_model.get_text_embedding(text) for text in texts}
except Exception:
return None
scores: dict[str, Any] = {}
if question and answer:
scores["question_answer"] = round(
100 * max(0.0, _cosine_similarity(vectors[question], vectors[answer])), 2
)
if answer and source:
scores["answer_source"] = round(
100 * max(0.0, _cosine_similarity(vectors[answer], vectors[source])), 2
)
if not scores:
return None
scores["overall"] = round(sum(scores.values()) / len(scores), 2)
return scores
def composite_overall(
*,
rule: float | None,
semantic: float | None = None,
judge: float | None = None,
) -> float:
"""三层加权组合：规则 35% + 语义 20% + 评审 45%，缺失层自动重归一。"""
if rule is None:
rule = 0.0
if judge is not None and semantic is not None:
overall = rule * 0.35 + semantic * 0.20 + judge * 0.45
elif semantic is not None:
overall = rule * 0.60 + semantic * 0.40
elif judge is not None:
overall = rule * 0.55 + judge * 0.45
else:
overall = rule
return round(max(0.0, min(100.0, overall)), 2)

130
backend/app/modules/data_process/document_chunking.py
View File

@@ -18,12 +18,19 @@ from llama_index.core.base.embeddings.base import BaseEmbedding
from llama_index.core.node_parser import SemanticSplitterNodeParser, SentenceSplitter
from app.modules.data_process.algorithms import normalize_text
from app.modules.data_process.algorithms.embedding import semantic_embedding_model
ChunkMethod = Literal["layout_hybrid", "semantic", "fixed"]
_PAGE_FURNITURE = re.compile(
r"(?m)^\s*(?:第\s*\d+\s*页\s*共\s*\d+\s*页|[-—–]?\s*\d+\s*[/／]\s*\d+\s*[-—–]?)\s*$"
)
# Docling 的 markdown 序列化会给列表项补上自动编号，而 Word 的编号存放在
# numbering.xml 中，python-docx 抽取的正文不含这些编号；紧凑匹配前剥掉
# 行首编号，否则带列表的切片会整体定位失败。
_LIST_MARKER_PREFIX = re.compile(
r"(?m)^[ \t>]*(?:(?:\d{1,3}[.)])+|\([a-zA-Z0-9]{1,3}\)|[a-zA-Z][.)]|[-*+•·])[ \t]+"
)
_COMPACT_CHARACTER = re.compile(r"[\w\u3400-\u4dbf\u4e00-\u9fff]", re.UNICODE)
_CONVERTER_LOCK = threading.Lock()
@@ -149,18 +156,6 @@ def chunk_fixed_text(
return _text_chunks(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
@lru_cache(maxsize=1)
def _semantic_embedding_model() -> BaseEmbedding:
# 模型可在部署环境覆盖；默认模型体积较小且适合中英文语义边界判断。
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
return HuggingFaceEmbedding(
model_name=os.getenv("DATA_PROCESS_EMBEDDING_MODEL", "BAAI/bge-small-zh-v1.5"),
device=os.getenv("DATA_PROCESS_EMBEDDING_DEVICE", "cpu"),
trust_remote_code=False,
)
def chunk_semantic_text(
text: str,
*,
@@ -175,7 +170,7 @@ def chunk_semantic_text(
if not normalized:
return []
splitter = SemanticSplitterNodeParser.from_defaults(
embed_model=embed_model or _semantic_embedding_model(),
embed_model=embed_model or semantic_embedding_model(),
breakpoint_percentile_threshold=breakpoint_percentile_threshold,
buffer_size=1,
sentence_splitter=_sentence_chunks,
@@ -193,6 +188,7 @@ def chunk_semantic_text(
if start is None:
start = _locate_text(normalized, content, 0)
if start is None:
result.append(_unlocated_chunk(content))
continue
if len(_tokenizer().encode(content)) <= chunk_size:
result.append(_make_text_chunk(normalized, start, start + len(content)))
@@ -203,6 +199,7 @@ def chunk_semantic_text(
chunk_overlap=chunk_overlap,
):
if child.source_start is None or child.source_end is None:
result.append(_unlocated_chunk(child.original_content))
continue
result.append(
_make_text_chunk(
@@ -235,6 +232,7 @@ def _nodes_to_chunks(nodes: list[Any], source_text: str) -> list[DocumentChunk]:
if start is None:
start = _locate_text(source_text, content, 0)
if start is None:
chunks.append(_unlocated_chunk(content))
continue
end = start + len(content)
chunks.append(_make_text_chunk(source_text, start, end))
@@ -247,6 +245,20 @@ def _locate_text(source: str, content: str, start: int) -> int | None:
return position if position >= 0 else None
def _unlocated_chunk(content: str) -> DocumentChunk:
"""正文在源文本中定位失败时保底保留切片，只放弃行号信息。"""
return DocumentChunk(
original_content=content,
contextualized_content=content,
source_start=None,
source_end=None,
source_start_line=None,
source_end_line=None,
token_count=len(_tokenizer().encode(content)),
)
def _make_text_chunk(source: str, start: int, end: int) -> DocumentChunk:
content = source[start:end]
return DocumentChunk(
@@ -316,6 +328,14 @@ def _compact_with_offsets(value: str) -> tuple[str, list[int]]:
return "".join(compact), offsets
def _expand_to_line_boundaries(source_text: str, start: int, end: int) -> tuple[int, int]:
while start > 0 and source_text[start - 1] not in "\r\n":
start -= 1
while end < len(source_text) and source_text[end] not in "\r\n":
end += 1
return start, end
def _project_layout_span(
source_text: str,
content: str,
@@ -324,21 +344,79 @@ def _project_layout_span(
source_offsets: list[int],
compact_start: int,
) -> tuple[int | None, int | None, int]:
compact_content, _ = _compact_with_offsets(content)
if len(compact_content) < 4:
for candidate in (content, _LIST_MARKER_PREFIX.sub("", content)):
compact_content, _ = _compact_with_offsets(candidate)
if len(compact_content) < 4:
continue
position = compact_source.find(compact_content, compact_start)
if position < 0:
position = compact_source.find(compact_content)
if position < 0:
continue
start, end = _expand_to_line_boundaries(
source_text,
source_offsets[position],
source_offsets[position + len(compact_content) - 1] + 1,
)
# 重复内容回退匹配可能命中已消费的更早位置，游标只进不退，
# 避免后续切片跟着错位。
return start, end, max(compact_start, position + len(compact_content))
return _project_layout_span_by_anchors(
source_text,
content,
compact_source=compact_source,
source_offsets=source_offsets,
compact_start=compact_start,
)
def _project_layout_span_by_anchors(
source_text: str,
content: str,
*,
compact_source: str,
source_offsets: list[int],
compact_start: int,
) -> tuple[int | None, int | None, int]:
"""按行锚点顺序匹配，容忍切片里插入的重复表头等非连续内容。"""
segments = [
compact
for compact in (
_compact_with_offsets(line)[0]
for line in _LIST_MARKER_PREFIX.sub("", content).split("\n")
)
if len(compact) >= 6
]
if not segments:
return None, None, compact_start
position = compact_source.find(compact_content, compact_start)
if position < 0:
position = compact_source.find(compact_content)
if position < 0:
total = sum(len(segment) for segment in segments)
def match_from(cursor: int) -> tuple[list[tuple[int, int]], int]:
matched: list[tuple[int, int]] = []
position = cursor
for segment in segments:
found = compact_source.find(segment, position)
if found < 0:
continue
matched.append((found, found + len(segment)))
position = found + len(segment)
return matched, sum(end - start for start, end in matched)
matched, covered = match_from(compact_start)
if covered * 2 < total:
retried, retry_covered = match_from(0)
if retry_covered > covered:
matched, covered = retried, retry_covered
# 覆盖不足一半时宁可不定位，也不能给出错误的行号。
if not matched or covered * 2 < total:
return None, None, compact_start
start = source_offsets[position]
end = source_offsets[position + len(compact_content) - 1] + 1
while start > 0 and source_text[start - 1] not in "\r\n":
start -= 1
while end < len(source_text) and source_text[end] not in "\r\n":
end += 1
return start, end, position + len(compact_content)
start, end = _expand_to_line_boundaries(
source_text,
source_offsets[matched[0][0]],
source_offsets[matched[-1][1] - 1] + 1,
)
return start, end, max(compact_start, matched[-1][1])
def chunk_layout_document(

313
backend/app/modules/data_process/evaluation.py Normal file
View File

@@ -0,0 +1,313 @@
"""数据处理 - 生成结果的多层质量评测。
三层体系：规则层（确定性规则分）+ 语义层（本地嵌入向量）+ 评审层
（复用生成模型按 rubric 打分的 LLM-as-judge）。任一层失败自动降级，
评测永远返回可用结果，不阻断调用方流程。
"""
from __future__ import annotations
import logging
from collections.abc import Mapping
from dataclasses import asdict
from datetime import UTC, datetime
from typing import Any
import httpx
from .algorithms import normalize_text, score_quality
from .algorithms.quality import composite_overall, semantic_quality_scores
from .generation import (
ModelGenerationError,
_is_retryable_generation_error,
_json_payload,
_message_content,
chat_completions_url,
)
logger = logging.getLogger(__name__)
# 送入评审提示词的来源正文上限，避免超长切片挤占评分输出空间。
_MAX_JUDGE_SOURCE_CHARS = 6000
_JUDGE_DIMENSIONS: dict[str, tuple[str, ...]] = {
"standard": (
"faithfulness",
"correctness",
"clarity",
"completeness",
"alignment",
),
"reasoning": (
"faithfulness",
"correctness",
"clarity",
"completeness",
"alignment",
"reasoning_validity",
),
"dpo": (
"clarity",
"chosen_quality",
"rejected_quality",
"preference_reasonableness",
"faithfulness",
),
}
_DIMENSION_LABELS: dict[str, str] = {
"faithfulness": "忠实度",
"correctness": "正确性",
"clarity": "问题清晰度",
"completeness": "回答完整性",
"alignment": "指令对齐",
"reasoning_validity": "推理有效性",
"chosen_quality": "chosen 回答质量",
"rejected_quality": "rejected 回答质量",
"preference_reasonableness": "偏好区分合理性",
}
_DIMENSION_RULES: dict[str, str] = {
"faithfulness": "忠实度：答案的全部陈述是否被参考资料支持，没有编造、没有引入资料之外的信息；未提供参考资料时按答案内部自洽性评估",
"correctness": "正确性：答案中的事实、概念与计算是否正确",
"clarity": "问题清晰度：问题是否清晰、自包含、无歧义，脱离上下文也能理解",
"completeness": "回答完整性：答案是否充分、直接地回应了问题的全部要点",
"alignment": "指令对齐：答案的形式与范围是否符合问题的要求（如格式、语言、范围限定）",
"reasoning_validity": "推理有效性：思维链步骤是否逻辑连贯、无跳步或循环论证，结论是否由推理过程自然得出",
"chosen_quality": "chosen 回答质量：更优回答的正确性、完整性与表述质量",
"rejected_quality": "rejected 回答质量：较差回答是否仍具备基本可读性，使对比训练有意义",
"preference_reasonableness": "偏好区分合理性：chosen 是否明显优于 rejected，且优劣差异与问题直接相关",
}
def _judge_system_prompt(output_type: str) -> str:
dimensions = _JUDGE_DIMENSIONS[output_type]
rules = "\n".join(f"- {_DIMENSION_RULES[name]}" for name in dimensions)
scores_schema = ", ".join(f'"{name}": 1-5' for name in dimensions)
return (
"你是大模型训练数据质量评审员。严格依据用户消息中的【参考资料】评审这条训练数据，逐维度按 1-5 分打分：\n"
f"{rules}\n"
"评分锚点：5 分=完全符合维度描述；3 分=基本符合但有明显不足；1 分=严重不符合。\n"
"忠实度只依据参考资料与公认常识判断，无法得到支持的陈述必须扣分；不要因为答案冗长而加分。\n"
"只输出一个 JSON 对象，不要输出 JSON 之外的任何文字。\n"
'输出格式：{"scores": {' + scores_schema + '}, "reason": "一句话总评", "issues": ["具体问题，没有则为空数组"]}'
)
def _judge_user_prompt(record: Mapping[str, Any], source_content: str) -> str:
source = normalize_text(source_content)[:_MAX_JUDGE_SOURCE_CHARS] or "（无参考资料）"
instruction = normalize_text(str(record.get("instruction") or "")) or "（空）"
input_text = normalize_text(str(record.get("input") or ""))
sections = [f"【参考资料】\n{source}", f"【问题】\n{instruction}"]
if input_text:
sections.append(f"【输入】\n{input_text}")
if record.get("chosen") or record.get("rejected"):
sections.append(f"【更优回答 chosen】\n{normalize_text(str(record.get('chosen') or '')) or '（空）'}")
sections.append(f"【较差回答 rejected】\n{normalize_text(str(record.get('rejected') or '')) or '（空）'}")
else:
output = normalize_text(str(record.get("output") or ""))
sections.append(f"【回答】\n{output or '（空）'}")
return "\n\n".join(sections)
def _validated_judge_payload(payload: Any, output_type: str) -> dict[str, Any]:
if not isinstance(payload, Mapping):
raise ModelGenerationError("评审响应不是 JSON 对象")
raw_scores = payload.get("scores")
if not isinstance(raw_scores, Mapping):
raise ModelGenerationError("评审响应缺少 scores 对象")
expected = _JUDGE_DIMENSIONS[output_type]
scores: dict[str, float] = {}
for name in expected:
value = raw_scores.get(name)
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ModelGenerationError(f"评审响应缺少维度 {name} 的有效分数")
scores[name] = round(max(1.0, min(5.0, float(value))), 1)
issues = payload.get("issues")
if not isinstance(issues, list):
issues = []
issues = [str(item)[:200] for item in issues if str(item).strip()][:8]
reason = normalize_text(str(payload.get("reason") or ""))[:300]
return {
"scores": scores,
"overall": round(sum(scores.values()) / len(scores) * 20, 2),
"reason": reason,
"issues": issues,
}
def _judge_record(
record: Mapping[str, Any],
source_content: str,
*,
model: Mapping[str, Any],
config: Mapping[str, Any],
client: httpx.Client | None,
) -> dict[str, Any] | None:
output_type = str(config.get("output_type") or "standard").strip().lower()
if output_type not in _JUDGE_DIMENSIONS:
output_type = "standard"
endpoint = chat_completions_url(str(model.get("api_url") or ""))
model_name = str(model.get("online_model_name") or model.get("name") or "").strip()
if not model_name:
raise ModelGenerationError("generation model name is required")
temperature = 0.1
max_tokens = max(256, min(2048, int(config.get("max_tokens", 1024) or 1024)))
timeout = max(1.0, min(120.0, float(config.get("request_timeout_seconds", 60) or 60)))
retries = max(0, min(5, int(config.get("generation_retries", 2) or 2)))
headers = {"Content-Type": "application/json"}
api_key = str(model.get("api_key") or "").strip()
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
request_payload: dict[str, Any] = {
"model": model_name,
"messages": [
{"role": "system", "content": _judge_system_prompt(output_type)},
{"role": "user", "content": _judge_user_prompt(record, source_content)},
],
"temperature": temperature,
"max_tokens": max_tokens,
}
if bool(config.get("json_mode", False)):
request_payload["response_format"] = {"type": "json_object"}
owns_client = client is None
http_client = client or httpx.Client(timeout=timeout)
try:
last_error: Exception | None = None
for _ in range(retries + 1):
try:
response = http_client.post(endpoint, headers=headers, json=request_payload)
response.raise_for_status()
body = response.json()
if not isinstance(body, Mapping):
raise ModelGenerationError("model response body must be a JSON object")
judged = _validated_judge_payload(
_json_payload(_message_content(body)),
output_type,
)
judged["model"] = model_name
judged["output_type"] = output_type
return judged
except Exception as exc:
last_error = exc
if not _is_retryable_generation_error(exc):
break
raise ModelGenerationError(f"质量评审调用失败: {last_error}")
finally:
if owns_client:
http_client.close()
def evaluate_result_record(
record: Mapping[str, Any],
*,
source_content: str = "",
model: Mapping[str, Any] | None = None,
config: Mapping[str, Any] | None = None,
client: httpx.Client | None = None,
embed_model: Any = None,
min_output_length: int = 20,
) -> dict[str, Any]:
"""对一条生成结果执行三层评测，返回可直接落库的 quality_score 字典。
规则层字段保持原样平铺（向后兼容既有读取方）；新增 ``semantic``、
``judge``、``layers``、``evaluated`` 与组合 ``overall``。
"""
config_dict = dict(config or {})
rule = score_quality(
record,
min_output_length=min_output_length,
source_content=source_content,
)
quality: dict[str, Any] = asdict(rule)
semantic = semantic_quality_scores(
record,
source_content=source_content,
embed_model=embed_model,
)
judge: dict[str, Any] | None = None
if model is not None:
try:
judge = _judge_record(
record,
source_content,
model=model,
config=config_dict,
client=client,
)
except Exception as exc:
logger.warning(
"data process judge evaluation degraded: %s",
exc,
)
layers = {
"rule": rule.overall,
"semantic": semantic.get("overall") if semantic else None,
"judge": judge.get("overall") if judge else None,
}
quality.update(
semantic=semantic,
judge=judge,
layers=layers,
evaluated=True,
evaluated_at=datetime.now(UTC).isoformat(),
overall=composite_overall(
rule=layers["rule"],
semantic=layers["semantic"],
judge=layers["judge"],
),
)
return quality
def reevaluate_edited_record(
record: Mapping[str, Any],
*,
source_content: str = "",
previous_quality: Mapping[str, Any] | None = None,
embed_model: Any = None,
min_output_length: int = 20,
) -> dict[str, Any]:
"""手动编辑/恢复后重算规则与语义层，丢弃已过期的评审层。
编辑会改变内容，旧的评审分不再可信；规则与语义层本地重算零成本。
``evaluated`` 标记沿用原值，保证已评测过的结果编辑后仍有可用分数。
"""
rule = score_quality(
record,
min_output_length=min_output_length,
source_content=source_content,
)
quality: dict[str, Any] = asdict(rule)
semantic = semantic_quality_scores(
record,
source_content=source_content,
embed_model=embed_model,
)
previous = dict(previous_quality or {})
evaluated = bool(previous.get("evaluated"))
layers = {
"rule": rule.overall,
"semantic": semantic.get("overall") if semantic else None,
"judge": None,
}
quality.update(
semantic=semantic,
judge=None,
layers=layers,
evaluated=evaluated,
evaluated_at=(
datetime.now(UTC).isoformat() if evaluated else None
),
overall=composite_overall(
rule=layers["rule"],
semantic=layers["semantic"],
),
)
return quality

23
backend/app/modules/data_process/office_preview.py
View File

@@ -11,6 +11,7 @@ import re
from typing import Any
from docx import Document
from docx.oxml.ns import qn
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
@@ -27,6 +28,7 @@ from app.modules.data_process.algorithms import (
_xlsx_sheet_merge_ranges,
normalize_text,
)
from app.modules.data_process.algorithms.parsers.office import iter_document_blocks
MAX_DOCX_PREVIEW_BLOCKS = 2_000
MAX_XLSX_PREVIEW_ROWS = 200
@@ -49,12 +51,21 @@ def _docx_alignment(paragraph: Paragraph) -> str:
def _docx_heading_level(paragraph: Paragraph) -> int | None:
style = paragraph.style
if style is None:
return None
style_name = str(style.name or "")
style_id = str(style.style_id or "")
style_name = str(style.name or "") if style is not None else ""
style_id = str(style.style_id or "") if style is not None else ""
match = re.search(r"(?:heading|标题)\s*([1-6])", f"{style_name} {style_id}", re.IGNORECASE)
return int(match.group(1)) if match else None
if match:
return int(match.group(1))
# Word 的目录和导航窗格依据大纲级别识别标题；未套标题样式但带
# outlineLvl 的段落（如手工排版的编号小节）同样是标题。
outline = paragraph._p.find(f"{qn('w:pPr')}/{qn('w:outlineLvl')}")
if outline is not None:
value = outline.get(qn("w:val"))
if value is not None and value.isdigit():
level = int(value)
if 0 <= level <= 5:
return level + 1
return None
def build_docx_preview(raw: bytes) -> dict[str, Any]:
@@ -84,7 +95,7 @@ def build_docx_preview(raw: bytes) -> dict[str, Any]:
has_source_content = True
return text, start, source_cursor
for child in document.element.body.iterchildren():
for child in iter_document_blocks(document.element.body):
if rendered_blocks >= MAX_DOCX_PREVIEW_BLOCKS:
truncated = True
break

144
backend/app/modules/data_process/storage.py
View File

@@ -1,4 +1,4 @@
"""数据处理原始源文件的受控本地对象存储。"""
"""数据处理源文件的受控暂存与分层对象存储。"""
from __future__ import annotations
@@ -13,6 +13,9 @@ from pathlib import Path, PurePosixPath
from typing import Iterable, Iterator
from urllib.parse import quote, unquote, urlsplit
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
class DataProcessStorageError(ValueError):
"""本地对象引用或文件系统状态不安全。"""
@@ -68,7 +71,7 @@ def _safe_basename(value: str) -> str:
class LocalDataProcessStorage:
"""只允许访问配置根目录下的版本化原始文件。"""
"""Stage locally, but publish and read authoritative source files from MinIO."""
def __init__(self, root: str | os.PathLike[str] | Path | None = None) -> None:
configured = Path(root) if root is not None else _configured_storage_root()
@@ -132,10 +135,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -168,6 +168,18 @@ class LocalDataProcessStorage:
expected_task_id=expected_source_task_id,
expected_source_file_id=expected_source_file_id,
)
if self._is_minio_reference(source_reference):
content = self.read(source_reference)
if content is None:
raise DataProcessStorageError("original source object is not available")
return self.stage_bytes(
batch_id=batch_id,
task_id=task_id,
source_file_id=source_file_id,
version=version,
name=basename,
content=content,
)
descriptor, source_info = self._open_read_descriptor(source_relative)
os.close(descriptor)
@@ -193,10 +205,7 @@ class LocalDataProcessStorage:
f"v{version}",
basename,
)
reference = (
"local://data-process/"
f"{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
)
reference = self._reference(task_id, source_file_id, version, basename)
staged = StagedSourceObject(reference, temporary_path, relative_path)
self._issued_staged_objects[temporary_path] = staged
return staged
@@ -212,14 +221,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("duplicate staged source object")
seen_temporary_paths.add(item._temporary_path)
for item in staged:
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
if self._is_minio_reference(item.reference):
content = item._temporary_path.read_bytes()
get_object_storage().put_bytes(
self.object_key(item.reference),
content,
"application/octet-stream",
)
elif not item.reference.startswith("db://data-process/"):
final_path = self._path_for_relative(item._relative_path)
self._ensure_directory(final_path.parent)
if final_path.exists() or final_path.is_symlink():
raise DataProcessStorageError("source storage object already exists")
os.link(item._temporary_path, final_path, follow_symlinks=False)
self._fsync_directory(final_path.parent)
published.append(item)
item._temporary_path.unlink()
self._fsync_directory(final_path.parent)
except Exception:
for item in reversed(published):
try:
@@ -258,7 +275,13 @@ class LocalDataProcessStorage:
raise first_error
def read(self, reference: str) -> bytes | None:
"""读取 local 引用；旧 ``db://`` 对象返回 ``None`` 由数据库正文兜底。"""
"""Read a MinIO object or legacy local reference."""
if self._is_minio_reference(reference):
try:
return get_object_storage().get_bytes(self.object_key(reference))
except Exception as exc: # noqa: BLE001 - normalize object-not-found for callers
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
@@ -276,6 +299,11 @@ class LocalDataProcessStorage:
) -> int | None:
"""返回受控 local 对象大小；旧 ``db://`` 对象没有原始文件。"""
if self._is_minio_reference(reference):
try:
return int(get_object_storage().stat(self.object_key(reference)).get("byte_size") or 0)
except Exception as exc: # noqa: BLE001
raise DataProcessStorageError("source storage object does not exist") from exc
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return None
@@ -301,6 +329,15 @@ class LocalDataProcessStorage:
) -> Iterator[bytes]:
"""按范围流式读取原始文件，避免 PDF 预览把大文件整体载入内存。"""
if self._is_minio_reference(reference):
content = self.read(reference) or b""
if start < 0 or expected_size != len(content) or start > expected_size:
raise DataProcessStorageError("source object size does not match metadata")
remaining = expected_size - start if length is None else length
if remaining < 0 or start + remaining > expected_size:
raise DataProcessStorageError("invalid source byte range")
yield content[start : start + remaining]
return
relative_path = self._relative_from_reference(reference)
if relative_path is None:
raise DataProcessStorageError("original source object is not available")
@@ -337,6 +374,12 @@ class LocalDataProcessStorage:
) -> bool:
"""校验 local 引用归属；旧 ``db://`` 引用无需文件系统处理。"""
if self._is_minio_reference(reference):
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
return True
if str(reference or "").startswith("db://data-process/"):
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return True
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -382,6 +425,19 @@ class LocalDataProcessStorage:
) -> bool:
"""删除受控 local 对象；旧 ``db://`` 引用保持不变。"""
if self._is_minio_reference(reference):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_minio_owner(reference, expected_task_id, expected_source_file_id)
get_object_storage().delete(self.object_key(reference))
return True
if str(reference or "").startswith("db://data-process/"):
if (expected_task_id is None) != (expected_source_file_id is None):
raise DataProcessStorageError("both expected storage owner fields are required")
if expected_task_id is not None and expected_source_file_id is not None:
self._assert_database_owner(reference, expected_task_id, expected_source_file_id)
return False
relative_path = self._relative_from_reference(reference)
if relative_path is None:
return False
@@ -426,7 +482,7 @@ class LocalDataProcessStorage:
if reference.startswith("db://"):
return None
parsed = urlsplit(reference)
if parsed.scheme != "local" or parsed.netloc != "data-process":
if parsed.scheme not in {"local", "minio"} or parsed.netloc != "data-process":
raise DataProcessStorageError("unsupported source storage reference")
if parsed.query or parsed.fragment or "\\" in parsed.path:
raise DataProcessStorageError("unsafe source storage reference")
@@ -460,6 +516,39 @@ class LocalDataProcessStorage:
basename = _safe_basename(decoded[3])
return PurePosixPath(task_id, source_file_id, f"v{version}", basename)
@staticmethod
def _is_minio_reference(reference: str) -> bool:
return str(reference or "").startswith("minio://data-process/")
@staticmethod
def _reference(task_id: str, source_file_id: str, version: int, basename: str) -> str:
scheme = "minio" if get_settings().minio_enabled else "local"
return f"{scheme}://data-process/{task_id}/{source_file_id}/v{version}/{quote(basename, safe='')}"
@staticmethod
def object_key(reference: str) -> str:
parsed = urlsplit(reference)
if parsed.scheme != "minio" or parsed.netloc != "data-process":
raise DataProcessStorageError("reference is not a MinIO source object")
return "data-process/" + parsed.path.lstrip("/")
def _assert_minio_owner(self, reference: str, task_id: str, source_file_id: str) -> None:
relative = self._relative_from_reference(reference)
if relative is None:
raise DataProcessStorageError("invalid MinIO source reference")
self._assert_expected_owner(relative, expected_task_id=task_id, expected_source_file_id=source_file_id)
@staticmethod
def _assert_database_owner(reference: str, task_id: str, source_file_id: str) -> None:
parsed = urlsplit(reference)
parts = parsed.path.lstrip("/").split("/")
if parsed.netloc != "data-process" or len(parts) != 3:
raise DataProcessStorageError("invalid database source reference")
expected_task_id = _safe_component(task_id, "expected task id")
expected_source_file_id = _safe_component(source_file_id, "expected source file id")
if tuple(parts[:2]) != (expected_task_id, expected_source_file_id) or parts[2] != "v1":
raise DataProcessStorageError("source storage object owner mismatch")
def _path_for_relative(self, relative_path: PurePosixPath) -> Path:
if relative_path.is_absolute() or any(
part in {"", ".", ".."} for part in relative_path.parts
@@ -479,9 +568,22 @@ class LocalDataProcessStorage:
raise DataProcessStorageError("invalid staged source object")
if self._issued_staged_objects.get(item._temporary_path) is not item:
raise DataProcessStorageError("staged source object was not issued by this storage")
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
if item.reference.startswith("db://data-process/"):
parsed = urlsplit(item.reference)
parts = parsed.path.lstrip("/").split("/")
expected = item._relative_path.parts[:3]
if (
parsed.netloc != "data-process"
or parsed.query
or parsed.fragment
or len(parts) != 3
or tuple(parts) != expected
):
raise DataProcessStorageError("staged source object reference mismatch")
else:
expected_relative = self._relative_from_reference(item.reference)
if expected_relative is None or expected_relative != item._relative_path:
raise DataProcessStorageError("staged source object reference mismatch")
staging_root = self._root / ".staging"
try:
relative_temporary = item._temporary_path.relative_to(staging_root)

7
backend/app/modules/data_process/store/base.py
View File

@@ -247,14 +247,19 @@ def _source_storage_descriptor(
or f"db://data-process/{task_id}/{file_id}/v1"
)
expected_local_prefix = f"local://data-process/{task_id}/{file_id}/v1/"
expected_minio_prefix = f"minio://data-process/{task_id}/{file_id}/v1/"
expected_database_reference = f"db://data-process/{task_id}/{file_id}/v1"
if storage_object_id.startswith(expected_local_prefix) and len(storage_object_id) > len(
expected_local_prefix
):
storage_backend = "local"
elif storage_object_id.startswith(expected_minio_prefix) and len(storage_object_id) > len(
expected_minio_prefix
):
storage_backend = "minio"
elif storage_object_id == expected_database_reference:
storage_backend = "database"
elif storage_object_id.startswith(("local://data-process/", "db://data-process/")):
elif storage_object_id.startswith(("local://data-process/", "minio://data-process/", "db://data-process/")):
raise DataProcessStoreError("source storage object owner mismatch")
else:
raise DataProcessStoreError("unsupported source storage object reference")

24
backend/app/modules/data_process/store/datasets.py
View File

@@ -11,6 +11,7 @@ import psycopg
from app.core.config import get_settings
from app.modules.storage.minio_store import get_object_storage
from app.modules.storage.policy import should_store_in_minio
from .base import (
StoreBase,
@@ -168,7 +169,11 @@ class DatasetsMixin:
split_name: assignments.count(split_name) for split_name in split_order
}
split_specs: list[dict[str, Any]] = []
use_minio = bool(get_settings().minio_enabled)
# Explicit local is retained for old callers/tests that request the
# legacy backend; all normal platform requests default to MinIO.
allow_minio = bool(get_settings().minio_enabled) and str(
payload.get("storage_type") or "minio"
).lower() != "local"
for split_name in split_order:
split_records = [
(source_row, record)
@@ -193,12 +198,15 @@ class DatasetsMixin:
"storage_object_id": (
f"db://data-process/{task_id}/{file_id}/v1"
),
"store_in_minio": allow_minio and should_store_in_minio(
len(raw), content_type="application/jsonl", file_format="jsonl"
),
}
)
source_result_ids = [row["id"] for row in rows]
common_metadata = {
"source": "data_process",
"storage_backend": "minio" if use_minio else "database",
"storage_backend": "minio" if any(spec["store_in_minio"] for spec in split_specs) else "database",
"source_task_id": task_id,
"output_type": _task_output_type(task),
"reasoning_detail": _task_reasoning_detail(task),
@@ -273,7 +281,7 @@ class DatasetsMixin:
split_name = str(spec["split"])
dataset_id = dataset_ids[split_name]
storage_object_id = str(spec["storage_object_id"])
if use_minio:
if spec["store_in_minio"]:
file_name = f"{base_dataset_name}.{split_name}.jsonl"
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
uploaded = get_object_storage().put_bytes(
@@ -356,7 +364,7 @@ class DatasetsMixin:
(
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
f"{len(spec['raw'])} B",
len(spec["raw"]),
len(spec["records"]),
@@ -386,7 +394,7 @@ class DatasetsMixin:
dataset_id,
dataset_name,
dataset_types[split_name],
"minio" if use_minio else (payload.get("storage_type") or "local"),
"minio" if spec["store_in_minio"] else "database",
task_id,
task_id,
f"{len(spec['raw'])} B",
@@ -405,7 +413,11 @@ class DatasetsMixin:
),
).fetchone()
file_metadata = {**dataset_metadata, "file_split": split_name}
file_metadata = {
**dataset_metadata,
"file_split": split_name,
"storage_backend": "minio" if spec["store_in_minio"] else "database",
}
version = {
"id": spec["version_id"],
"version_no": 1,

14
backend/app/modules/data_process/store/source_files.py
View File

@@ -137,7 +137,7 @@ class SourceFilesMixin:
payload["record_count"],
payload["file_format"],
payload["checksum_sha256"],
payload["content"],
"" if str(storage_object_id or "").startswith("minio://") else payload["content"],
str(payload["content"])[:2000],
json_dumps(metadata_payload),
task.get("tenant_id"),
@@ -223,7 +223,17 @@ class SourceFilesMixin:
).fetchone()
if not row:
raise NotFoundError("source file not found")
return _decode_row(row) or {}
decoded = _decode_row(row) or {}
# New source files keep only a preview in PostgreSQL. Load the
# authoritative body from MinIO on demand for existing processing code.
reference = str(decoded.get("storage_object_id") or "")
if include_content and not decoded.get("content") and reference.startswith("minio://"):
from app.modules.data_process.storage import get_data_process_storage
decoded["content"] = (get_data_process_storage().read(reference) or b"").decode(
"utf-8", errors="replace"
)
return decoded
def source_content_window(
self, task_id: str, file_id: str, offset: int, limit: int

59
backend/app/modules/storage/minio_store.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import timedelta
from functools import lru_cache
from io import BytesIO
from collections.abc import Iterator
from typing import Any
from minio import Minio
@@ -53,6 +54,64 @@ class MinioObjectStorage:
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def get_bytes(self, object_key: str) -> bytes:
"""Read an object through the backend for small API responses and workers."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
return response.read()
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def iter_bytes(self, object_key: str, chunk_size: int = 256 * 1024) -> Iterator[bytes]:
"""Stream an object without loading the complete file into memory."""
self._ensure_enabled()
self.ensure_bucket()
response = None
try:
response = self.client.get_object(self.bucket, object_key)
while True:
chunk = response.read(chunk_size)
if not chunk:
break
yield chunk
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
finally:
if response is not None:
response.close()
response.release_conn()
def list_objects(self, prefix: str) -> list[dict[str, Any]]:
self._ensure_enabled()
self.ensure_bucket()
try:
return [
{
"object_key": item.object_name,
"byte_size": item.size or 0,
"etag": item.etag,
"last_modified": item.last_modified.isoformat() if item.last_modified else None,
}
for item in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
]
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def delete(self, object_key: str) -> None:
self._ensure_enabled()
self.ensure_bucket()
try:
self.client.remove_object(self.bucket, object_key)
except S3Error as exc:
raise ObjectStorageError(str(exc)) from exc
def put_bytes(self, object_key: str, content: bytes, content_type: str = "application/octet-stream") -> dict[str, Any]:
self._ensure_enabled()
self.ensure_bucket()

54
backend/app/modules/storage/policy.py Normal file
View File

@@ -0,0 +1,54 @@
"""Storage placement rules shared by dataset and data-processing flows."""
from __future__ import annotations
from app.core.config import get_settings
_INLINE_TEXT_FORMATS = {
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
"yaml", "yml", "xml", "html", "text/plain", "application/json",
"application/jsonl", "text/csv",
}
def should_store_in_minio(
size_bytes: int | None,
*,
content_type: str | None = None,
file_format: str | None = None,
) -> bool:
"""Return whether a file is large enough to use the shared object store.
Small files remain inline in PostgreSQL so page previews and metadata reads
do not pay an object-storage round trip. MinIO is still mandatory for
large files when it is enabled.
"""
if not get_settings().minio_enabled:
return False
try:
size = max(0, int(size_bytes or 0))
except (TypeError, ValueError):
size = 0
if size > get_settings().minio_inline_max_bytes:
return True
# Binary office/document files remain in MinIO even when small because
# their original bytes cannot be safely represented by a text DB column.
normalized_format = str(file_format or "").strip().lower().lstrip(".")
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
if normalized_format or normalized_type:
return not (
normalized_format in _INLINE_TEXT_FORMATS
or normalized_type in _INLINE_TEXT_FORMATS
or normalized_type.startswith("text/")
)
return False
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
"""Return ``minio`` or ``database`` for a managed file."""
if str(requested or "").strip().lower() == "local":
return "database"
return "minio" if should_store_in_minio(size_bytes) else "database"

10
backend/app/modules/system/router.py
View File

@@ -56,13 +56,15 @@ def audit_logs(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
current_user: dict = Depends(get_current_user),
) -> dict:
"""审计日志查询：按租户/项目/操作人/动作/目标类型/时间范围分页过滤。"""
"""审计日志查询：按组织、操作人、动作、资源、关键字和时间范围分页过滤。"""
if not is_admin(current_user):
from app.api.v1.endpoints.platform import fail
raise fail(403, "admin permission required")
@@ -73,6 +75,8 @@ def audit_logs(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=limit,
@@ -88,6 +92,8 @@ def audit_logs_export(
actor_id: str | None = Query(default=None, description="操作人 ID"),
action: str | None = Query(default=None, description="动作类型"),
target_type: str | None = Query(default=None, description="目标类型"),
target_id: str | None = Query(default=None, description="目标 ID"),
keyword: str | None = Query(default=None, description="目标 ID 或详情关键字"),
start_time: str | None = Query(default=None, description="ISO8601 起始时间"),
end_time: str | None = Query(default=None, description="ISO8601 结束时间"),
current_user: dict = Depends(get_current_user),
@@ -103,6 +109,8 @@ def audit_logs_export(
actor_id=actor_id,
action=action,
target_type=target_type,
target_id=target_id,
keyword=keyword,
start_time=start_time,
end_time=end_time,
limit=10000,

20
backend/app/schemas/data_process.py
View File

@@ -384,6 +384,26 @@ class ResultBatchRegenerateRequest(BaseModel):
return self
class ResultBatchEvaluateItem(BaseModel):
model_config = ConfigDict(extra="forbid")
result_id: str = Field(min_length=1, max_length=100)
expected_updated_at: str = Field(min_length=1, max_length=100)
class ResultBatchEvaluateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
items: list[ResultBatchEvaluateItem] = Field(min_length=1, max_length=50)
@model_validator(mode="after")
def validate_unique_results(self) -> ResultBatchEvaluateRequest:
result_ids = [item.result_id for item in self.items]
if len(result_ids) != len(set(result_ids)):
raise ValueError("result_id values must be unique")
return self
class DatasetSplit(BaseModel):
model_config = ConfigDict(extra="forbid")

71
backend/tests/test_data_process_algorithms.py
View File

@@ -9,6 +9,8 @@ from decimal import Decimal
import pytest
from docx import Document
from docx.oxml import parse_xml
from docx.oxml.ns import nsdecls, qn
from openpyxl import Workbook
from pptx import Presentation
from pptx.util import Inches
@@ -38,6 +40,7 @@ from app.modules.data_process.algorithms import (
stable_split_assignments,
structured_json_dumps,
)
from app.modules.data_process.office_preview import build_docx_preview
def _pdf_page_texts(*texts: str) -> tuple[PdfPageText, ...]:
@@ -318,6 +321,74 @@ def test_parse_pdf_docx_xlsx_and_pptx() -> None:
assert parsed_pptx.records == ()
def _docx_with_sdt_bytes() -> bytes:
"""构造带 SDT 目录内容控件的 docx，段落顺序为正文、SDT、正文。"""
document = Document()
document.add_paragraph("正文开头。")
sdt = parse_xml(
"<w:sdt %s><w:sdtPr><w:id w:val='1'/></w:sdtPr>"
"<w:sdtContent><w:p><w:r><w:t>目录条目 第一章 概述</w:t></w:r></w:p>"
"</w:sdtContent></w:sdt>" % nsdecls("w")
)
body = document.element.body
sect_pr = body.find(qn("w:sectPr"))
if sect_pr is not None:
sect_pr.addprevious(sdt)
else:
body.append(sdt)
document.add_paragraph("正文结尾。")
output = io.BytesIO()
document.save(output)
return output.getvalue()
def test_docx_extraction_and_preview_include_sdt_content() -> None:
raw = _docx_with_sdt_bytes()
parsed = parse_text_content(raw, filename="toc.docx")
assert "目录条目 第一章 概述" in parsed.text
assert (
parsed.text.index("正文开头。")
< parsed.text.index("目录条目 第一章 概述")
< parsed.text.index("正文结尾。")
)
preview = build_docx_preview(raw)
paragraph_texts = [
block["text"] for block in preview["blocks"] if block["type"] == "paragraph"
]
assert "目录条目 第一章 概述" in paragraph_texts
# 预览偏移必须与正文抽取规则一致，否则前端定位会错位。
sdt_block = next(
block
for block in preview["blocks"]
if block.get("text") == "目录条目 第一章 概述"
)
assert parsed.text[sdt_block["source_start"] : sdt_block["source_end"]] == (
"目录条目 第一章 概述"
)
def test_docx_preview_detects_outline_level_headings() -> None:
"""未套标题样式但设了大纲级别的段落（Word 目录按此收录）也按标题渲染。"""
document = Document()
document.add_heading("一级标题", level=1)
plain = document.add_paragraph("4.2.1 数据管理")
p_pr = plain._p.get_or_add_pPr()
p_pr.append(parse_xml("<w:outlineLvl %s w:val='2'/>" % nsdecls("w")))
document.add_paragraph("普通正文段落。")
output = io.BytesIO()
document.save(output)
preview = build_docx_preview(output.getvalue())
blocks = {b["text"]: b for b in preview["blocks"] if b["type"] == "paragraph"}
assert blocks["一级标题"]["heading_level"] == 1
assert blocks["4.2.1 数据管理"]["heading_level"] == 3
assert blocks["普通正文段落。"]["heading_level"] is None
def test_xlsx_record_locators_distinguish_sheets_rows_and_duplicate_records() -> None:
workbook = Workbook()
first = workbook.active

217
backend/tests/test_data_process_api.py
View File

@@ -1691,6 +1691,223 @@ def test_batch_result_regeneration_rejects_locked_tasks_before_model_call(
assert model_calls == 0
def _prepare_evaluation_task(
client: TestClient,
store: Any,
tmp_path: Path,
*,
config: dict[str, Any] | None = None,
) -> str:
task_id = client.post(
"/modelTF/data-process",
json={
"name": "数据评测",
"process_type": "structured",
"config": config or {"generation_model_id": "model-1", "output_type": "standard"},
},
).json()["data"]["id"]
store.tasks[task_id].update(
status="completed",
progress=100,
workflow_step="results",
results_confirmed=False,
)
store.models["model-1"] = {
"id": "model-1",
"online_model_name": "test-model",
"api_url": "https://model.example/v1",
"api_key": "secret",
}
store.previews[task_id] = [
{
"id": "preview-1",
"status": "original",
"original_content": "申请编号用于唯一标识一笔报销申请。",
"edited_content": "申请编号用于唯一标识一笔报销申请。",
},
{
"id": "preview-2",
"status": "original",
"original_content": "联系电话用于联系申请人。",
"edited_content": "联系电话用于联系申请人。",
},
]
store.results[task_id] = [
{
"id": "result-1",
"preview_item_id": "preview-1",
"instruction": "申请编号有什么作用？",
"input": "",
"output": "申请编号用于唯一标识一笔报销申请。",
"original_instruction": "申请编号有什么作用？",
"original_input": "",
"original_output": "申请编号用于唯一标识一笔报销申请。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:00Z",
},
{
"id": "result-2",
"preview_item_id": "preview-2",
"instruction": "联系电话有什么作用？",
"input": "",
"output": "联系电话用于联系申请人。",
"original_instruction": "联系电话有什么作用？",
"original_input": "",
"original_output": "联系电话用于联系申请人。",
"status": "valid",
"error": None,
"split": "train",
"quality_score": {},
"updated_at": "2026-08-19T09:00:01Z",
},
]
return task_id
def test_results_can_be_evaluated_in_batch_with_partial_success(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
evaluation_calls: list[dict[str, Any]] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
evaluation_calls.append({"record": deepcopy(record), "kwargs": {k: v for k, v in kwargs.items() if k != "client"}})
return {
"overall": 88.0,
"completeness": 100.0,
"length": 100.0,
"readability": 100.0,
"relevance": 90.0,
"duplicate": 100.0,
"is_valid": True,
"flags": [],
"fingerprint": "fp",
"semantic": {"question_answer": 80.0, "answer_source": 90.0, "overall": 85.0},
"judge": {"scores": {"faithfulness": 5}, "overall": 90.0},
"layers": {"rule": 92.0, "semantic": 85.0, "judge": 90.0},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={
"items": [
{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"},
# 乐观锁版本不匹配：该条应按冲突失败，另一条仍成功。
{"result_id": "result-2", "expected_updated_at": "2026-08-18T00:00:00Z"},
],
},
)
assert response.status_code == 200
data = response.json()["data"]
assert data["total"] == 2
assert data["succeeded"] == 1
assert data["failed"] == 1
assert [item["id"] for item in data["items"]] == ["result-1"]
assert data["failures"][0]["result_id"] == "result-2"
assert data["failures"][0]["code"] == "conflict"
assert len(evaluation_calls) == 1
assert evaluation_calls[0]["record"]["instruction"] == "申请编号有什么作用？"
assert evaluation_calls[0]["kwargs"]["model"]["online_model_name"] == "test-model"
assert evaluation_calls[0]["kwargs"]["source_content"] == "申请编号用于唯一标识一笔报销申请。"
stored = store.results[task_id][0]["quality_score"]
assert stored["evaluated"] is True
assert stored["layers"]["judge"] == 90.0
assert store.results[task_id][1]["quality_score"] == {}
def test_evaluation_without_generation_model_skips_judge_layer(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path, config={"output_type": "standard"})
seen_models: list[Any] = []
def fake_evaluate(record: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
seen_models.append(kwargs.get("model"))
return {
"overall": 70.0, "is_valid": True, "flags": [],
"semantic": None, "judge": None,
"layers": {"rule": 70.0, "semantic": None, "judge": None},
"evaluated": True,
}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 200
assert response.json()["data"]["succeeded"] == 1
# 任务未配置生成模型时，评审层收到的 model 必须是 None。
assert seen_models == [None]
def test_evaluation_rejects_running_task(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.tasks[task_id]["status"] = "running"
evaluation_calls = 0
def fake_evaluate(*args: Any, **kwargs: Any) -> dict[str, Any]:
nonlocal evaluation_calls
evaluation_calls += 1
return {"overall": 0, "is_valid": True, "flags": []}
monkeypatch.setattr(data_process_endpoint, "evaluate_result_record", fake_evaluate)
response = client.post(
f"/modelTF/data-process/{task_id}/results/evaluate-batch",
json={"items": [{"result_id": "result-1", "expected_updated_at": "2026-08-19T09:00:00Z"}]},
)
assert response.status_code == 409
assert evaluation_calls == 0
def test_result_update_preserves_evaluation_layers_and_drops_stale_judge(
tmp_path: Path,
) -> None:
client, store, _ = make_client(tmp_path)
task_id = _prepare_evaluation_task(client, store, tmp_path)
store.results[task_id][0]["quality_score"] = {
"overall": 90.0,
"is_valid": True,
"flags": [],
"semantic": {"overall": 85.0},
"judge": {"overall": 92.0},
"layers": {"rule": 90.0, "semantic": 85.0, "judge": 92.0},
"evaluated": True,
}
response = client.put(
f"/modelTF/data-process/{task_id}/results/result-1",
json={"output": "人工修正后的答案：申请编号唯一标识一笔报销申请。"},
)
assert response.status_code == 200
stored = store.results[task_id][0]["quality_score"]
# 手动编辑后：规则+语义重算，评审分丢弃，evaluated 标记保留。
assert stored["evaluated"] is True
assert stored["judge"] is None
assert stored["layers"]["judge"] is None
assert stored["layers"]["rule"] is not None
assert stored["overall"] >= 0
def test_preview_build_replaces_only_selected_files_and_reports_file_counts(
tmp_path: Path,
) -> None:

284
backend/tests/test_data_process_evaluation.py Normal file
View File

@@ -0,0 +1,284 @@
"""数据评测模块（三层质量评分）的单元测试。"""
from __future__ import annotations
import json
from typing import Any
import httpx
import pytest
from app.modules.data_process.algorithms.quality import (
composite_overall,
semantic_quality_scores,
)
from app.modules.data_process.evaluation import (
_JUDGE_DIMENSIONS,
_judge_system_prompt,
_validated_judge_payload,
evaluate_result_record,
reevaluate_edited_record,
)
from app.modules.data_process.generation import ModelGenerationError
RECORD = {
"instruction": "申请编号有什么作用？",
"input": "",
"output": "申请编号用于唯一标识一笔报销申请，便于跟踪审批状态。",
}
SOURCE = "报销系统中，申请编号用于唯一标识一笔报销申请，并支持跟踪审批状态。"
class _FakeEmbedModel:
"""按关键词返回固定向量，模拟语义嵌入。"""
def get_text_embedding(self, text: str) -> list[float]:
if "作用" in text or "编号" in text and "？" in text:
return [0.9, 0.1, 0.0]
if "申请编号" in text:
return [0.85, 0.2, 0.0]
return [0.0, 0.1, 0.9]
class _FailingEmbedModel:
def get_text_embedding(self, text: str) -> list[float]:
raise RuntimeError("embedding unavailable")
class _FakeResponse:
def __init__(self, payload: dict[str, Any]):
self._payload = payload
def raise_for_status(self) -> None:
return None
def json(self) -> dict[str, Any]:
return self._payload
class _FakeClient:
def __init__(self, content: str):
self._content = content
self.calls: list[dict[str, Any]] = []
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
self.calls.append({"endpoint": endpoint, "payload": json})
return _FakeResponse({
"choices": [{"message": {"content": self._content}, "finish_reason": "stop"}],
})
def close(self) -> None:
return None
class _RaisingClient:
def post(self, endpoint: str, headers: Any = None, json: Any = None) -> _FakeResponse:
raise httpx.ConnectError("model endpoint unreachable")
def close(self) -> None:
return None
def _judge_content(scores: dict[str, float], **extra: Any) -> str:
return json.dumps({"scores": scores, "reason": "总体可靠", "issues": [], **extra})
def test_judge_system_prompt_covers_rubric_dimensions() -> None:
standard = _judge_system_prompt("standard")
for name in _JUDGE_DIMENSIONS["standard"]:
assert name in standard
assert "1-5" in standard
dpo = _judge_system_prompt("dpo")
assert "chosen_quality" in dpo
assert "preference_reasonableness" in dpo
reasoning = _judge_system_prompt("reasoning")
assert "reasoning_validity" in reasoning
def test_validated_judge_payload_converts_scores_to_overall() -> None:
judged = _validated_judge_payload(
{
"scores": {
"faithfulness": 5,
"correctness": 4,
"clarity": 4,
"completeness": 3,
"alignment": 4,
},
"reason": "答案可靠",
"issues": ["回答略冗长"],
},
"standard",
)
assert judged["overall"] == round((5 + 4 + 4 + 3 + 4) / 5 * 20, 2)
assert judged["issues"] == ["回答略冗长"]
assert judged["reason"] == "答案可靠"
def test_validated_judge_payload_clamps_out_of_range_scores() -> None:
judged = _validated_judge_payload(
{
"scores": {
"faithfulness": 9,
"correctness": 4,
"clarity": 4,
"completeness": 0,
"alignment": 4,
},
},
"standard",
)
assert judged["scores"]["faithfulness"] == 5.0
assert judged["scores"]["completeness"] == 1.0
@pytest.mark.parametrize(
"scores",
[
{"faithfulness": 5, "correctness": 4, "clarity": 4, "completeness": 3},
{
"faithfulness": 5,
"correctness": 4,
"clarity": "high",
"completeness": 3,
"alignment": 4,
},
],
)
def test_validated_judge_payload_rejects_incomplete_scores(scores: dict[str, Any]) -> None:
with pytest.raises(ModelGenerationError):
_validated_judge_payload({"scores": scores}, "standard")
def test_semantic_quality_scores_uses_cosine_similarity() -> None:
scores = semantic_quality_scores(
RECORD,
source_content=SOURCE,
embed_model=_FakeEmbedModel(),
)
assert scores is not None
assert 0 < scores["question_answer"] <= 100
assert 0 < scores["answer_source"] <= 100
assert scores["overall"] == round((scores["question_answer"] + scores["answer_source"]) / 2, 2)
def test_semantic_quality_scores_degrades_to_none_on_failure() -> None:
assert (
semantic_quality_scores(
RECORD,
source_content=SOURCE,
embed_model=_FailingEmbedModel(),
)
is None
)
def test_composite_overall_weights_available_layers() -> None:
assert composite_overall(rule=80, semantic=90, judge=70) == round(80 * 0.35 + 90 * 0.20 + 70 * 0.45, 2)
assert composite_overall(rule=80, semantic=90) == round(80 * 0.6 + 90 * 0.4, 2)
assert composite_overall(rule=80) == 80.0
assert composite_overall(rule=None, judge=100) == 45.0
def test_evaluate_result_record_combines_three_layers() -> None:
client = _FakeClient(
_judge_content({
"faithfulness": 5,
"correctness": 4,
"clarity": 5,
"completeness": 4,
"alignment": 5,
})
)
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
config={"output_type": "standard", "generation_retries": 0},
client=client,
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is True
assert quality["judge"] is not None
assert quality["judge"]["model"] == "judge-model"
assert quality["semantic"] is not None
assert quality["layers"]["judge"] == quality["judge"]["overall"]
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
judge=quality["layers"]["judge"],
)
# 评审提示词必须携带来源原文作为评分锚点（正文经 NFKC 归一化）。
user_message = client.calls[0]["payload"]["messages"][1]["content"]
assert "申请编号用于唯一标识一笔报销" in user_message
def test_evaluate_result_record_degrades_when_model_fails() -> None:
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model={"api_url": "https://model.example", "online_model_name": "judge-model"},
config={"output_type": "standard", "generation_retries": 0},
client=_RaisingClient(),
embed_model=_FakeEmbedModel(),
)
assert quality["judge"] is None
assert quality["layers"]["judge"] is None
assert quality["semantic"] is not None
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
)
def test_evaluate_result_record_without_model_runs_two_layers() -> None:
quality = evaluate_result_record(
RECORD,
source_content=SOURCE,
model=None,
embed_model=_FakeEmbedModel(),
)
assert quality["judge"] is None
assert quality["evaluated"] is True
assert quality["overall"] == composite_overall(
rule=quality["layers"]["rule"],
semantic=quality["layers"]["semantic"],
)
def test_reevaluate_edited_record_drops_stale_judge() -> None:
previous = {
"evaluated": True,
"judge": {"overall": 90.0},
}
quality = reevaluate_edited_record(
{**RECORD, "output": "编辑后的新答案内容，用于验证重评逻辑。"},
source_content=SOURCE,
previous_quality=previous,
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is True
assert quality["judge"] is None
assert quality["layers"]["judge"] is None
assert quality["semantic"] is not None
def test_reevaluate_edited_record_keeps_unevaluated_state() -> None:
quality = reevaluate_edited_record(
RECORD,
source_content=SOURCE,
previous_quality={},
embed_model=_FakeEmbedModel(),
)
assert quality["evaluated"] is False
assert quality["evaluated_at"] is None

81
backend/tests/test_document_chunking.py
View File

@@ -9,6 +9,7 @@ from app.modules.data_process.document_chunking import (
DocumentChunk,
_compact_with_offsets,
_document_converter,
_nodes_to_chunks,
_project_layout_span,
chunk_fixed_text,
chunk_semantic_text,
@@ -103,6 +104,86 @@ def test_layout_projection_ignores_layout_whitespace_but_keeps_source_lines() ->
assert cursor > 0
def test_layout_projection_tolerates_list_numbers_inserted_by_serializer() -> None:
# Word 自动编号存放在 numbering.xml，python-docx 抽取的正文没有编号，
# 而 Docling 序列化切片时会补上 "1. " 前缀，投影不能因此失败。
source = "接入方式说明\n结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
compact_source, offsets = _compact_with_offsets(source)
start, end, cursor = _project_layout_span(
source,
"1. 结构化数据接入需要先配置连接地址。\n2. 非结构化接入需要上传文档。",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is not None and end is not None
assert source[start:end] == "结构化数据接入需要先配置连接地址。\n非结构化接入需要上传文档。"
assert cursor > 0
def test_layout_projection_never_moves_cursor_backwards() -> None:
source = "重复段落内容。\n中间正文。\n重复段落内容。"
compact_source, offsets = _compact_with_offsets(source)
# 重复内容回退匹配命中已消费的更早位置时，游标必须保持不退。
_, _, cursor = _project_layout_span(
source,
"重复段落内容。",
compact_source=compact_source,
source_offsets=offsets,
compact_start=compact_source.index("中间正文"),
)
assert cursor >= compact_source.index("中间正文")
def test_layout_projection_falls_back_to_line_anchors_for_inserted_content() -> None:
# 表格跨切片时 Docling 会在续片中重复表头，正文不再是连续子串；
# 按行锚点匹配仍应定位到表头所在行到末行数据之间的连续区间。
source = "表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
compact_source, offsets = _compact_with_offsets(source)
start, end, _ = _project_layout_span(
source,
"表头甲 表头乙\n第二行数据 说明二",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is not None and end is not None
assert source[start:end] == (
"表头甲\t表头乙\n第一行数据\t说明一\n第二行数据\t说明二"
)
def test_layout_projection_refuses_low_coverage_anchor_match() -> None:
source = "完全无关的正文内容甲。\n完全无关的正文内容乙。"
compact_source, offsets = _compact_with_offsets(source)
start, end, cursor = _project_layout_span(
source,
"找不到的数据行内容\n另一条找不到的数据行内容",
compact_source=compact_source,
source_offsets=offsets,
compact_start=0,
)
assert start is None
assert end is None
assert cursor == 0
def test_text_splitter_keeps_chunks_that_cannot_be_located() -> None:
class FakeNode:
def get_content(self) -> str:
return "这段文本在源文本中不存在。"
chunks = _nodes_to_chunks([FakeNode()], "完全不同的源文本。")
assert len(chunks) == 1
assert chunks[0].original_content == "这段文本在源文本中不存在。"
assert chunks[0].source_start is None
assert chunks[0].source_start_line is None
def test_short_layout_chunk_merges_with_neighbor_and_keeps_page_provenance() -> None:
source = "短标题\n这是一段足够长的正文内容，用于测试相邻切片合并。"
chunks = [