YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dataset): 统一大小与版本元数据

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-27 12:26:09 +08:00
parent e486d36a80
commit bce586697b
8 changed files with 345 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/app/api/v1/endpoints/platform.py
View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import uuid
import json
import uuid
from pathlib import Path
from typing import Any
@@ -426,6 +426,14 @@ async def dataset_preview(file_id: str) -> dict[str, Any]:
raise fail(404, "dataset file not found")
@router.get("/dataset-manage/records/{file_id}/sources")
async def dataset_record_sources(file_id: str) -> dict[str, Any]:
try:
return ok({"items": get_platform_store().dataset_file_record_sources(file_id)})
except KeyError:
raise fail(404, "dataset file not found")
@router.get("/dataset-manage/versions/{file_id}")
async def dataset_versions(file_id: str) -> dict[str, Any]:
try:

201
backend/app/db/platform_store.py
View File

@@ -1,8 +1,8 @@
from __future__ import annotations
import json
import hashlib
import hmac
import json
import math
import re
import secrets
@@ -17,7 +17,6 @@ import psycopg
from app.core.config import get_settings
ALL_PERMISSIONS = [
"dashboard",
"fine-tune",
@@ -61,6 +60,71 @@ def safe_float(value: Any, default: float = 0) -> float:
return default
_SIZE_UNIT_BYTES = {
"B": 1,
"KB": 1024,
"MB": 1024**2,
"GB": 1024**3,
"TB": 1024**4,
}
def parse_size_bytes(value: Any) -> int:
"""把历史字符串大小统一换算为字节，供接口返回稳定的数值字段。"""
if isinstance(value, bool):
return 0
if isinstance(value, (int, float)):
return max(0, int(value))
match = re.fullmatch(
r"\s*([0-9]+(?:\.[0-9]+)?)\s*(B|KB|MB|GB|TB)?\s*",
str(value or ""),
flags=re.IGNORECASE,
)
if not match:
return 0
amount = float(match.group(1))
unit = (match.group(2) or "B").upper()
return max(0, round(amount * _SIZE_UNIT_BYTES[unit]))
def version_number(value: Any, default: int = 0) -> int:
try:
number = int(value)
except (TypeError, ValueError):
return default
return number if number > 0 else default
def dataset_file_version_summary(file_row: PgRow) -> dict[str, Any]:
versions = json_loads(file_row.get("versions"), [])
versions = versions if isinstance(versions, list) else []
active_version_id = str(
file_row.get("active_version_id")
or file_row.get("current_version_id")
or ""
)
active_version = next(
(
item
for item in versions
if isinstance(item, dict) and str(item.get("id") or "") == active_version_id
),
None,
)
current_version_no = version_number(
(active_version or {}).get("version_no")
or (active_version or {}).get("version")
or file_row.get("version_no"),
default=1 if active_version_id or versions else 0,
)
return {
"active_version_id": active_version_id or None,
"current_version_id": active_version_id or None,
"current_version_no": current_version_no or None,
"version_count": len(versions),
}
def parse_training_metric_line(line: str) -> dict[str, float] | None:
if "loss" not in line and "learning_rate" not in line:
return None
@@ -1099,11 +1163,30 @@ class PlatformStore:
def _dataset(self, conn: PgConnection, row: PgRow) -> dict[str, Any]:
files = conn.execute(
"""SELECT id, name, size, active_version_id, create_time,
"""SELECT id, name, size, size_bytes, active_version_id,
current_version_id, version_no, versions, create_time,
record_count, metadata
FROM dataset_files WHERE dataset_id=? ORDER BY create_time, id""",
(row["id"],),
).fetchall()
decoded_files: list[dict[str, Any]] = []
for file_row in files:
metadata = json_loads(file_row.get("metadata"), {})
file_size_bytes = int(file_row.get("size_bytes") or 0)
if file_size_bytes <= 0:
file_size_bytes = parse_size_bytes(file_row.get("size"))
decoded_files.append(
{
"id": file_row["id"],
"name": file_row["name"],
"size": file_row["size"],
"size_bytes": file_size_bytes,
**dataset_file_version_summary(file_row),
"create_time": file_row["create_time"],
"record_count": int(file_row.get("record_count") or 0),
"split": metadata.get("file_split"),
}
)
dataset_metadata = json_loads(row.get("metadata"), {})
split_counts = dict(dataset_metadata.get("split_counts") or {})
if row.get("source") == "task" and not split_counts:
@@ -1113,26 +1196,33 @@ class PlatformStore:
(row["id"],),
).fetchall()
split_counts = {str(item["split"]): int(item["count"]) for item in split_rows}
total_size_bytes = sum(item["size_bytes"] for item in decoded_files)
if not decoded_files:
total_size_bytes = int(row.get("size_bytes") or 0)
if total_size_bytes <= 0:
total_size_bytes = parse_size_bytes(row.get("size"))
current_version_nos = sorted(
{
int(item["current_version_no"])
for item in decoded_files
if item.get("current_version_no")
}
)
return {
**dict(row),
"size_bytes": total_size_bytes,
"current_version_no": (
current_version_nos[0] if len(current_version_nos) == 1 else None
),
"current_version_nos": current_version_nos,
"version_count": sum(int(item["version_count"]) for item in decoded_files),
"metadata": dataset_metadata,
"split_counts": {
"train": int(split_counts.get("train", 0) or 0),
"validation": int(split_counts.get("validation", 0) or 0),
"test": int(split_counts.get("test", 0) or 0),
},
"files": [
{
"id": f["id"],
"name": f["name"],
"size": f["size"],
"active_version_id": f["active_version_id"],
"create_time": f["create_time"],
"record_count": int(f.get("record_count") or 0),
"split": json_loads(f.get("metadata"), {}).get("file_split"),
}
for f in files
],
"files": decoded_files,
}
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
@@ -1192,12 +1282,24 @@ class PlatformStore:
now = utcnow()
file_id = new_id("file")
version_id = f"{file_id}_v1"
size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB"
size_bytes = len(content.encode("utf-8"))
size = f"{size_bytes} B"
record_count = len([line for line in content.splitlines() if line.strip()])
version = {
"id": version_id,
"version": 1,
"version_no": 1,
"create_time": now,
"description": "uploaded",
"size_bytes": size_bytes,
"record_count": record_count,
}
conn.execute(
"""
INSERT INTO dataset_files
(id, dataset_id, name, size, content, active_version_id, versions, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
(id, dataset_id, name, size, content, active_version_id, versions, create_time,
current_version_id, size_bytes, record_count, version_no)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
""",
(
file_id,
@@ -1206,16 +1308,69 @@ class PlatformStore:
size,
content,
version_id,
json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]),
json_dumps([version]),
now,
version_id,
size_bytes,
record_count,
),
)
count = len([line for line in content.splitlines() if line.strip()])
conn.execute(
"UPDATE datasets SET count=count+?, size=? WHERE id=?",
(count, size, dataset_id),
"""UPDATE datasets
SET count=count+?, record_count=record_count+?,
size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B')
WHERE id=?""",
(record_count, record_count, size_bytes, size_bytes, dataset_id),
)
return {"id": file_id, "name": name, "size": size}
return {
"id": file_id,
"name": name,
"size": size,
"size_bytes": size_bytes,
"current_version_no": 1,
"version_count": 1,
}
def dataset_file_record_sources(self, file_id: str) -> list[dict[str, Any]]:
"""返回发布样本关联的真实原文，供数据集详情核对生成内容。"""
with self.connect() as conn:
file_row = conn.execute(
"SELECT id FROM dataset_files WHERE id=?", (file_id,)
).fetchone()
if not file_row:
raise KeyError(file_id)
rows = conn.execute(
"""
SELECT records.line_no, records.instruction, records.input, records.output,
COALESCE(
NULLIF(preview.edited_content, ''),
preview.original_content,
''
) AS source_text,
CASE
WHEN preview.edited_content IS NOT NULL
AND preview.edited_content <> ''
THEN TRUE ELSE FALSE
END AS preprocessed
FROM dataset_records AS records
LEFT JOIN data_process_preview_items AS preview
ON preview.id=records.preview_item_id
WHERE records.dataset_file_id=?
ORDER BY records.line_no NULLS LAST, records.created_at, records.id
""",
(file_id,),
).fetchall()
return [
{
"line_no": int(item.get("line_no") or index + 1),
"instruction": str(item.get("instruction") or ""),
"input": str(item.get("input") or ""),
"output": str(item.get("output") or ""),
"source_text": str(item.get("source_text") or ""),
"preprocessed": bool(item.get("preprocessed")),
}
for index, item in enumerate(rows)
]
def dataset_file(self, file_id: str) -> PgRow:
with self.connect() as conn:

46
backend/tests/test_platform_dataset_metadata.py Normal file
View File

@@ -0,0 +1,46 @@
from __future__ import annotations
from app.db.platform_store import dataset_file_version_summary, parse_size_bytes
def test_parse_size_bytes_supports_legacy_units() -> None:
assert parse_size_bytes("21563 B") == 21563
assert parse_size_bytes("1.5 KB") == 1536
assert parse_size_bytes("2 MB") == 2 * 1024**2
assert parse_size_bytes(4096) == 4096
assert parse_size_bytes("unknown") == 0
def test_dataset_file_version_summary_uses_active_version_metadata() -> None:
summary = dataset_file_version_summary(
{
"active_version_id": "file-1-v3",
"current_version_id": "file-1-v1",
"version_no": 1,
"versions": (
'[{"id":"file-1-v1","version":1},'
'{"id":"file-1-v3","version_no":3}]'
),
}
)
assert summary == {
"active_version_id": "file-1-v3",
"current_version_id": "file-1-v3",
"current_version_no": 3,
"version_count": 2,
}
def test_dataset_file_version_summary_uses_normalized_version_number_as_fallback() -> None:
summary = dataset_file_version_summary(
{
"active_version_id": "",
"current_version_id": None,
"version_no": 1,
"versions": "[]",
}
)
assert summary["current_version_no"] == 1
assert summary["version_count"] == 0