diff --git a/backend/app/api/v1/endpoints/platform.py b/backend/app/api/v1/endpoints/platform.py index d2dabfe..cc76daf 100644 --- a/backend/app/api/v1/endpoints/platform.py +++ b/backend/app/api/v1/endpoints/platform.py @@ -1,7 +1,7 @@ from __future__ import annotations -import uuid import json +import uuid from pathlib import Path from typing import Any @@ -426,6 +426,14 @@ async def dataset_preview(file_id: str) -> dict[str, Any]: raise fail(404, "dataset file not found") +@router.get("/dataset-manage/records/{file_id}/sources") +async def dataset_record_sources(file_id: str) -> dict[str, Any]: + try: + return ok({"items": get_platform_store().dataset_file_record_sources(file_id)}) + except KeyError: + raise fail(404, "dataset file not found") + + @router.get("/dataset-manage/versions/{file_id}") async def dataset_versions(file_id: str) -> dict[str, Any]: try: diff --git a/backend/app/db/platform_store.py b/backend/app/db/platform_store.py index d640ae0..c011839 100644 --- a/backend/app/db/platform_store.py +++ b/backend/app/db/platform_store.py @@ -1,8 +1,8 @@ from __future__ import annotations -import json import hashlib import hmac +import json import math import re import secrets @@ -17,7 +17,6 @@ import psycopg from app.core.config import get_settings - ALL_PERMISSIONS = [ "dashboard", "fine-tune", @@ -61,6 +60,71 @@ def safe_float(value: Any, default: float = 0) -> float: return default +_SIZE_UNIT_BYTES = { + "B": 1, + "KB": 1024, + "MB": 1024**2, + "GB": 1024**3, + "TB": 1024**4, +} + + +def parse_size_bytes(value: Any) -> int: + """把历史字符串大小统一换算为字节，供接口返回稳定的数值字段。""" + if isinstance(value, bool): + return 0 + if isinstance(value, (int, float)): + return max(0, int(value)) + match = re.fullmatch( + r"\s*([0-9]+(?:\.[0-9]+)?)\s*(B|KB|MB|GB|TB)?\s*", + str(value or ""), + flags=re.IGNORECASE, + ) + if not match: + return 0 + amount = float(match.group(1)) + unit = (match.group(2) or "B").upper() + return max(0, round(amount * _SIZE_UNIT_BYTES[unit])) + + +def version_number(value: Any, default: int = 0) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if number > 0 else default + + +def dataset_file_version_summary(file_row: PgRow) -> dict[str, Any]: + versions = json_loads(file_row.get("versions"), []) + versions = versions if isinstance(versions, list) else [] + active_version_id = str( + file_row.get("active_version_id") + or file_row.get("current_version_id") + or "" + ) + active_version = next( + ( + item + for item in versions + if isinstance(item, dict) and str(item.get("id") or "") == active_version_id + ), + None, + ) + current_version_no = version_number( + (active_version or {}).get("version_no") + or (active_version or {}).get("version") + or file_row.get("version_no"), + default=1 if active_version_id or versions else 0, + ) + return { + "active_version_id": active_version_id or None, + "current_version_id": active_version_id or None, + "current_version_no": current_version_no or None, + "version_count": len(versions), + } + + def parse_training_metric_line(line: str) -> dict[str, float] | None: if "loss" not in line and "learning_rate" not in line: return None @@ -1099,11 +1163,30 @@ class PlatformStore: def _dataset(self, conn: PgConnection, row: PgRow) -> dict[str, Any]: files = conn.execute( - """SELECT id, name, size, active_version_id, create_time, + """SELECT id, name, size, size_bytes, active_version_id, + current_version_id, version_no, versions, create_time, record_count, metadata FROM dataset_files WHERE dataset_id=? ORDER BY create_time, id""", (row["id"],), ).fetchall() + decoded_files: list[dict[str, Any]] = [] + for file_row in files: + metadata = json_loads(file_row.get("metadata"), {}) + file_size_bytes = int(file_row.get("size_bytes") or 0) + if file_size_bytes <= 0: + file_size_bytes = parse_size_bytes(file_row.get("size")) + decoded_files.append( + { + "id": file_row["id"], + "name": file_row["name"], + "size": file_row["size"], + "size_bytes": file_size_bytes, + **dataset_file_version_summary(file_row), + "create_time": file_row["create_time"], + "record_count": int(file_row.get("record_count") or 0), + "split": metadata.get("file_split"), + } + ) dataset_metadata = json_loads(row.get("metadata"), {}) split_counts = dict(dataset_metadata.get("split_counts") or {}) if row.get("source") == "task" and not split_counts: @@ -1113,26 +1196,33 @@ class PlatformStore: (row["id"],), ).fetchall() split_counts = {str(item["split"]): int(item["count"]) for item in split_rows} + total_size_bytes = sum(item["size_bytes"] for item in decoded_files) + if not decoded_files: + total_size_bytes = int(row.get("size_bytes") or 0) + if total_size_bytes <= 0: + total_size_bytes = parse_size_bytes(row.get("size")) + current_version_nos = sorted( + { + int(item["current_version_no"]) + for item in decoded_files + if item.get("current_version_no") + } + ) return { **dict(row), + "size_bytes": total_size_bytes, + "current_version_no": ( + current_version_nos[0] if len(current_version_nos) == 1 else None + ), + "current_version_nos": current_version_nos, + "version_count": sum(int(item["version_count"]) for item in decoded_files), "metadata": dataset_metadata, "split_counts": { "train": int(split_counts.get("train", 0) or 0), "validation": int(split_counts.get("validation", 0) or 0), "test": int(split_counts.get("test", 0) or 0), }, - "files": [ - { - "id": f["id"], - "name": f["name"], - "size": f["size"], - "active_version_id": f["active_version_id"], - "create_time": f["create_time"], - "record_count": int(f.get("record_count") or 0), - "split": json_loads(f.get("metadata"), {}).get("file_split"), - } - for f in files - ], + "files": decoded_files, } def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]: @@ -1192,12 +1282,24 @@ class PlatformStore: now = utcnow() file_id = new_id("file") version_id = f"{file_id}_v1" - size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB" + size_bytes = len(content.encode("utf-8")) + size = f"{size_bytes} B" + record_count = len([line for line in content.splitlines() if line.strip()]) + version = { + "id": version_id, + "version": 1, + "version_no": 1, + "create_time": now, + "description": "uploaded", + "size_bytes": size_bytes, + "record_count": record_count, + } conn.execute( """ INSERT INTO dataset_files - (id, dataset_id, name, size, content, active_version_id, versions, create_time) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + (id, dataset_id, name, size, content, active_version_id, versions, create_time, + current_version_id, size_bytes, record_count, version_no) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1) """, ( file_id, @@ -1206,16 +1308,69 @@ class PlatformStore: size, content, version_id, - json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]), + json_dumps([version]), now, + version_id, + size_bytes, + record_count, ), ) - count = len([line for line in content.splitlines() if line.strip()]) conn.execute( - "UPDATE datasets SET count=count+?, size=? WHERE id=?", - (count, size, dataset_id), + """UPDATE datasets + SET count=count+?, record_count=record_count+?, + size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B') + WHERE id=?""", + (record_count, record_count, size_bytes, size_bytes, dataset_id), ) - return {"id": file_id, "name": name, "size": size} + return { + "id": file_id, + "name": name, + "size": size, + "size_bytes": size_bytes, + "current_version_no": 1, + "version_count": 1, + } + + def dataset_file_record_sources(self, file_id: str) -> list[dict[str, Any]]: + """返回发布样本关联的真实原文，供数据集详情核对生成内容。""" + with self.connect() as conn: + file_row = conn.execute( + "SELECT id FROM dataset_files WHERE id=?", (file_id,) + ).fetchone() + if not file_row: + raise KeyError(file_id) + rows = conn.execute( + """ + SELECT records.line_no, records.instruction, records.input, records.output, + COALESCE( + NULLIF(preview.edited_content, ''), + preview.original_content, + '' + ) AS source_text, + CASE + WHEN preview.edited_content IS NOT NULL + AND preview.edited_content <> '' + THEN TRUE ELSE FALSE + END AS preprocessed + FROM dataset_records AS records + LEFT JOIN data_process_preview_items AS preview + ON preview.id=records.preview_item_id + WHERE records.dataset_file_id=? + ORDER BY records.line_no NULLS LAST, records.created_at, records.id + """, + (file_id,), + ).fetchall() + return [ + { + "line_no": int(item.get("line_no") or index + 1), + "instruction": str(item.get("instruction") or ""), + "input": str(item.get("input") or ""), + "output": str(item.get("output") or ""), + "source_text": str(item.get("source_text") or ""), + "preprocessed": bool(item.get("preprocessed")), + } + for index, item in enumerate(rows) + ] def dataset_file(self, file_id: str) -> PgRow: with self.connect() as conn: diff --git a/backend/tests/test_platform_dataset_metadata.py b/backend/tests/test_platform_dataset_metadata.py new file mode 100644 index 0000000..11a9ef2 --- /dev/null +++ b/backend/tests/test_platform_dataset_metadata.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from app.db.platform_store import dataset_file_version_summary, parse_size_bytes + + +def test_parse_size_bytes_supports_legacy_units() -> None: + assert parse_size_bytes("21563 B") == 21563 + assert parse_size_bytes("1.5 KB") == 1536 + assert parse_size_bytes("2 MB") == 2 * 1024**2 + assert parse_size_bytes(4096) == 4096 + assert parse_size_bytes("unknown") == 0 + + +def test_dataset_file_version_summary_uses_active_version_metadata() -> None: + summary = dataset_file_version_summary( + { + "active_version_id": "file-1-v3", + "current_version_id": "file-1-v1", + "version_no": 1, + "versions": ( + '[{"id":"file-1-v1","version":1},' + '{"id":"file-1-v3","version_no":3}]' + ), + } + ) + + assert summary == { + "active_version_id": "file-1-v3", + "current_version_id": "file-1-v3", + "current_version_no": 3, + "version_count": 2, + } + + +def test_dataset_file_version_summary_uses_normalized_version_number_as_fallback() -> None: + summary = dataset_file_version_summary( + { + "active_version_id": "", + "current_version_id": None, + "version_no": 1, + "versions": "[]", + } + ) + + assert summary["current_version_no"] == 1 + assert summary["version_count"] == 0 diff --git a/frontend/scripts/regression-dataset-task-tab.mjs b/frontend/scripts/regression-dataset-task-tab.mjs index 851d601..c3c47a4 100644 --- a/frontend/scripts/regression-dataset-task-tab.mjs +++ b/frontend/scripts/regression-dataset-task-tab.mjs @@ -42,7 +42,11 @@ assert.match( ) assert.match(viewSource, /return dataList\.value\.filter\(\(item\) => item\.source !== 'task'\)/) assert.doesNotMatch(viewSource, /数据任务产生的数据集[\s\S]*?return \[\]/) -assert.match(viewSource, /:search-fields="\['task_id', 'name', 'description'\]"/, '数据任务搜索应支持任务 ID') +assert.match(viewSource, /:search-fields="\['task_id', 'name'\]"/, '数据任务搜索应支持任务 ID') +assert.match(viewSource, /formatMegabytes\(row\.size_bytes, row\.size\)/, '数据集大小应统一转换为 MB') +assert.match(viewSource, /[\s\S]*?批量删除[\s\S]*?v-if="selectedCount > 0"[\s\S]*?删除（\{\{ selectedCount \}\}）/, '批量删除按钮没有按选择状态切换') diff --git a/frontend/src/api/modules/dataset.ts b/frontend/src/api/modules/dataset.ts index 546a4cf..92a7e3a 100644 --- a/frontend/src/api/modules/dataset.ts +++ b/frontend/src/api/modules/dataset.ts @@ -1,5 +1,10 @@ import { get, post, put, del } from '../request' -import type { DatasetItem, DatasetVersion, DatasetVersionList } from '@/types' +import type { + DatasetItem, + DatasetRecordSource, + DatasetVersion, + DatasetVersionList, +} from '@/types' /** 数据集列表 */ export const getDatasetList = () => get('/dataset-manage') @@ -31,6 +36,12 @@ export const uploadDatasetFiles = (datasetId: string | number, files: File[]) => export const previewDatasetFile = (fileId: string | number) => get<{ content: string }>(`/dataset-manage/preview/${encodeURIComponent(fileId)}`) +/** 获取发布样本所关联的原文参照 */ +export const getDatasetFileRecordSources = (fileId: string | number) => + get<{ items: DatasetRecordSource[] }>( + `/dataset-manage/records/${encodeURIComponent(fileId)}/sources`, + ) + /** 数据集文件版本列表 */ export const getDatasetFileVersions = (fileId: string | number) => get(`/dataset-manage/versions/${encodeURIComponent(fileId)}`) diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index e64cc66..5ab7c8c 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -63,6 +63,11 @@ export interface DatasetFile { id?: string name: string size?: string | number + size_bytes?: number + active_version_id?: string | null + current_version_id?: string | null + current_version_no?: number | null + version_count?: number record_count?: number split?: 'train' | 'validation' | 'test' | null [k: string]: any @@ -76,13 +81,26 @@ export interface DatasetItem { source?: DatasetSource task_id?: string | number size?: string | number + size_bytes?: number count?: number description?: string create_time?: string files?: DatasetFile[] + current_version_no?: number | null + current_version_nos?: number[] + version_count?: number split_counts?: { train: number; validation: number; test: number } } +export interface DatasetRecordSource { + line_no: number + instruction: string + input: string + output: string + source_text: string + preprocessed: boolean +} + export interface DatasetVersion { id: string version: number diff --git a/frontend/src/utils/fileSize.ts b/frontend/src/utils/fileSize.ts new file mode 100644 index 0000000..5f3fbcb --- /dev/null +++ b/frontend/src/utils/fileSize.ts @@ -0,0 +1,26 @@ +export function parseSizeBytes(value?: string | number | null) { + if (typeof value === 'number') return Number.isFinite(value) ? Math.max(0, value) : 0 + const match = String(value || '').trim().match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB|TB)?$/i) + if (!match) return 0 + const units: Record = { + B: 1, + KB: 1024, + MB: 1024 ** 2, + GB: 1024 ** 3, + TB: 1024 ** 4, + } + return Number(match[1]) * units[(match[2] || 'B').toUpperCase()] +} + +/** 列表统一使用 MB，非零的小文件至少展示为 0.01 MB，避免误显示为零。 */ +export function formatMegabytes( + sizeBytes?: number | null, + legacySize?: string | number | null, +) { + const numericSize = Number(sizeBytes) + const bytes = Number.isFinite(numericSize) && numericSize > 0 + ? numericSize + : parseSizeBytes(legacySize) + if (bytes <= 0) return '0.00 MB' + return `${Math.max(bytes / 1024 ** 2, 0.01).toFixed(2)} MB` +} diff --git a/frontend/src/views/dataset/DatasetListView.vue b/frontend/src/views/dataset/DatasetListView.vue index 8951055..5aa125b 100644 --- a/frontend/src/views/dataset/DatasetListView.vue +++ b/frontend/src/views/dataset/DatasetListView.vue @@ -5,6 +5,7 @@ import { ElMessage, ElMessageBox } from 'element-plus' import DataTablePage from '@/components/DataTablePage.vue' import { getDatasetList, deleteDataset, downloadDatasetUrl } from '@/api/modules/dataset' import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants' +import { formatMegabytes } from '@/utils/fileSize' import type { DatasetItem, DatasetSource } from '@/types' const router = useRouter() @@ -107,6 +108,36 @@ function handleDownload(row: any) { window.open(downloadDatasetUrl(row.id), '_blank') } +function formatSizeMb(row: DatasetItem) { + return formatMegabytes(row.size_bytes, row.size) +} + +function currentVersionNumbers(row: DatasetItem) { + const values = row.current_version_nos?.length + ? row.current_version_nos + : [ + row.current_version_no, + ...(row.files || []).map((file) => file.current_version_no), + ] + return [...new Set( + values + .map((value) => Number(value)) + .filter((value) => Number.isInteger(value) && value > 0), + )].sort((a, b) => a - b) +} + +function versionLabel(row: DatasetItem) { + const versions = currentVersionNumbers(row) + return versions.length ? versions.map((value) => `V${value}`).join(' / ') : '-' +} + +function versionDescription(row: DatasetItem) { + const fileCount = row.files?.length || 0 + const historyCount = Number(row.version_count || 0) + if (!fileCount) return '当前数据集没有文件版本' + return `${fileCount} 个文件，共 ${historyCount || fileCount} 个版本记录` +} + watch(activeTab, exitBatchMode) onMounted(loadData) @@ -118,7 +149,7 @@ onMounted(loadData) :data="filteredDataList" :loading="loading" searchable - :search-fields="['task_id', 'name', 'description']" + :search-fields="['task_id', 'name']" :multi-select="activeTab === 'task' && batchMode" :show-batch-bar="false" :create-text="activeTab === 'upload' ? '上传数据集' : ''" @@ -181,16 +212,26 @@ onMounted(loadData) - + - - + +