2 Commits

Author SHA1 Message Date
caoxiaozhu
e6bbb0bb49 feat(dataset): 补全样本原文与删除操作 2026-07-27 12:26:19 +08:00
caoxiaozhu
bce586697b fix(dataset): 统一大小与版本元数据 2026-07-27 12:26:09 +08:00
13 changed files with 722 additions and 160 deletions

View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import uuid
import json
import uuid
from pathlib import Path
from typing import Any
@@ -426,6 +426,14 @@ async def dataset_preview(file_id: str) -> dict[str, Any]:
raise fail(404, "dataset file not found")
@router.get("/dataset-manage/records/{file_id}/sources")
async def dataset_record_sources(file_id: str) -> dict[str, Any]:
try:
return ok({"items": get_platform_store().dataset_file_record_sources(file_id)})
except KeyError:
raise fail(404, "dataset file not found")
@router.get("/dataset-manage/versions/{file_id}")
async def dataset_versions(file_id: str) -> dict[str, Any]:
try:

View File

@@ -1,8 +1,8 @@
from __future__ import annotations
import json
import hashlib
import hmac
import json
import math
import re
import secrets
@@ -17,7 +17,6 @@ import psycopg
from app.core.config import get_settings
ALL_PERMISSIONS = [
"dashboard",
"fine-tune",
@@ -61,6 +60,71 @@ def safe_float(value: Any, default: float = 0) -> float:
return default
_SIZE_UNIT_BYTES = {
"B": 1,
"KB": 1024,
"MB": 1024**2,
"GB": 1024**3,
"TB": 1024**4,
}
def parse_size_bytes(value: Any) -> int:
"""把历史字符串大小统一换算为字节,供接口返回稳定的数值字段。"""
if isinstance(value, bool):
return 0
if isinstance(value, (int, float)):
return max(0, int(value))
match = re.fullmatch(
r"\s*([0-9]+(?:\.[0-9]+)?)\s*(B|KB|MB|GB|TB)?\s*",
str(value or ""),
flags=re.IGNORECASE,
)
if not match:
return 0
amount = float(match.group(1))
unit = (match.group(2) or "B").upper()
return max(0, round(amount * _SIZE_UNIT_BYTES[unit]))
def version_number(value: Any, default: int = 0) -> int:
try:
number = int(value)
except (TypeError, ValueError):
return default
return number if number > 0 else default
def dataset_file_version_summary(file_row: PgRow) -> dict[str, Any]:
versions = json_loads(file_row.get("versions"), [])
versions = versions if isinstance(versions, list) else []
active_version_id = str(
file_row.get("active_version_id")
or file_row.get("current_version_id")
or ""
)
active_version = next(
(
item
for item in versions
if isinstance(item, dict) and str(item.get("id") or "") == active_version_id
),
None,
)
current_version_no = version_number(
(active_version or {}).get("version_no")
or (active_version or {}).get("version")
or file_row.get("version_no"),
default=1 if active_version_id or versions else 0,
)
return {
"active_version_id": active_version_id or None,
"current_version_id": active_version_id or None,
"current_version_no": current_version_no or None,
"version_count": len(versions),
}
def parse_training_metric_line(line: str) -> dict[str, float] | None:
if "loss" not in line and "learning_rate" not in line:
return None
@@ -1099,11 +1163,30 @@ class PlatformStore:
def _dataset(self, conn: PgConnection, row: PgRow) -> dict[str, Any]:
files = conn.execute(
"""SELECT id, name, size, active_version_id, create_time,
"""SELECT id, name, size, size_bytes, active_version_id,
current_version_id, version_no, versions, create_time,
record_count, metadata
FROM dataset_files WHERE dataset_id=? ORDER BY create_time, id""",
(row["id"],),
).fetchall()
decoded_files: list[dict[str, Any]] = []
for file_row in files:
metadata = json_loads(file_row.get("metadata"), {})
file_size_bytes = int(file_row.get("size_bytes") or 0)
if file_size_bytes <= 0:
file_size_bytes = parse_size_bytes(file_row.get("size"))
decoded_files.append(
{
"id": file_row["id"],
"name": file_row["name"],
"size": file_row["size"],
"size_bytes": file_size_bytes,
**dataset_file_version_summary(file_row),
"create_time": file_row["create_time"],
"record_count": int(file_row.get("record_count") or 0),
"split": metadata.get("file_split"),
}
)
dataset_metadata = json_loads(row.get("metadata"), {})
split_counts = dict(dataset_metadata.get("split_counts") or {})
if row.get("source") == "task" and not split_counts:
@@ -1113,26 +1196,33 @@ class PlatformStore:
(row["id"],),
).fetchall()
split_counts = {str(item["split"]): int(item["count"]) for item in split_rows}
total_size_bytes = sum(item["size_bytes"] for item in decoded_files)
if not decoded_files:
total_size_bytes = int(row.get("size_bytes") or 0)
if total_size_bytes <= 0:
total_size_bytes = parse_size_bytes(row.get("size"))
current_version_nos = sorted(
{
int(item["current_version_no"])
for item in decoded_files
if item.get("current_version_no")
}
)
return {
**dict(row),
"size_bytes": total_size_bytes,
"current_version_no": (
current_version_nos[0] if len(current_version_nos) == 1 else None
),
"current_version_nos": current_version_nos,
"version_count": sum(int(item["version_count"]) for item in decoded_files),
"metadata": dataset_metadata,
"split_counts": {
"train": int(split_counts.get("train", 0) or 0),
"validation": int(split_counts.get("validation", 0) or 0),
"test": int(split_counts.get("test", 0) or 0),
},
"files": [
{
"id": f["id"],
"name": f["name"],
"size": f["size"],
"active_version_id": f["active_version_id"],
"create_time": f["create_time"],
"record_count": int(f.get("record_count") or 0),
"split": json_loads(f.get("metadata"), {}).get("file_split"),
}
for f in files
],
"files": decoded_files,
}
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
@@ -1192,12 +1282,24 @@ class PlatformStore:
now = utcnow()
file_id = new_id("file")
version_id = f"{file_id}_v1"
size = f"{max(1, len(content.encode('utf-8')) // 1024)} KB"
size_bytes = len(content.encode("utf-8"))
size = f"{size_bytes} B"
record_count = len([line for line in content.splitlines() if line.strip()])
version = {
"id": version_id,
"version": 1,
"version_no": 1,
"create_time": now,
"description": "uploaded",
"size_bytes": size_bytes,
"record_count": record_count,
}
conn.execute(
"""
INSERT INTO dataset_files
(id, dataset_id, name, size, content, active_version_id, versions, create_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
(id, dataset_id, name, size, content, active_version_id, versions, create_time,
current_version_id, size_bytes, record_count, version_no)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
""",
(
file_id,
@@ -1206,16 +1308,69 @@ class PlatformStore:
size,
content,
version_id,
json_dumps([{"id": version_id, "version": 1, "create_time": now, "description": "uploaded"}]),
json_dumps([version]),
now,
version_id,
size_bytes,
record_count,
),
)
count = len([line for line in content.splitlines() if line.strip()])
conn.execute(
"UPDATE datasets SET count=count+?, size=? WHERE id=?",
(count, size, dataset_id),
"""UPDATE datasets
SET count=count+?, record_count=record_count+?,
size_bytes=size_bytes+?, size=((size_bytes+?)::text || ' B')
WHERE id=?""",
(record_count, record_count, size_bytes, size_bytes, dataset_id),
)
return {"id": file_id, "name": name, "size": size}
return {
"id": file_id,
"name": name,
"size": size,
"size_bytes": size_bytes,
"current_version_no": 1,
"version_count": 1,
}
def dataset_file_record_sources(self, file_id: str) -> list[dict[str, Any]]:
"""返回发布样本关联的真实原文,供数据集详情核对生成内容。"""
with self.connect() as conn:
file_row = conn.execute(
"SELECT id FROM dataset_files WHERE id=?", (file_id,)
).fetchone()
if not file_row:
raise KeyError(file_id)
rows = conn.execute(
"""
SELECT records.line_no, records.instruction, records.input, records.output,
COALESCE(
NULLIF(preview.edited_content, ''),
preview.original_content,
''
) AS source_text,
CASE
WHEN preview.edited_content IS NOT NULL
AND preview.edited_content <> ''
THEN TRUE ELSE FALSE
END AS preprocessed
FROM dataset_records AS records
LEFT JOIN data_process_preview_items AS preview
ON preview.id=records.preview_item_id
WHERE records.dataset_file_id=?
ORDER BY records.line_no NULLS LAST, records.created_at, records.id
""",
(file_id,),
).fetchall()
return [
{
"line_no": int(item.get("line_no") or index + 1),
"instruction": str(item.get("instruction") or ""),
"input": str(item.get("input") or ""),
"output": str(item.get("output") or ""),
"source_text": str(item.get("source_text") or ""),
"preprocessed": bool(item.get("preprocessed")),
}
for index, item in enumerate(rows)
]
def dataset_file(self, file_id: str) -> PgRow:
with self.connect() as conn:

View File

@@ -0,0 +1,46 @@
from __future__ import annotations
from app.db.platform_store import dataset_file_version_summary, parse_size_bytes
def test_parse_size_bytes_supports_legacy_units() -> None:
assert parse_size_bytes("21563 B") == 21563
assert parse_size_bytes("1.5 KB") == 1536
assert parse_size_bytes("2 MB") == 2 * 1024**2
assert parse_size_bytes(4096) == 4096
assert parse_size_bytes("unknown") == 0
def test_dataset_file_version_summary_uses_active_version_metadata() -> None:
summary = dataset_file_version_summary(
{
"active_version_id": "file-1-v3",
"current_version_id": "file-1-v1",
"version_no": 1,
"versions": (
'[{"id":"file-1-v1","version":1},'
'{"id":"file-1-v3","version_no":3}]'
),
}
)
assert summary == {
"active_version_id": "file-1-v3",
"current_version_id": "file-1-v3",
"current_version_no": 3,
"version_count": 2,
}
def test_dataset_file_version_summary_uses_normalized_version_number_as_fallback() -> None:
summary = dataset_file_version_summary(
{
"active_version_id": "",
"current_version_id": None,
"version_no": 1,
"versions": "[]",
}
)
assert summary["current_version_no"] == 1
assert summary["version_count"] == 0

View File

@@ -15,6 +15,7 @@ const apiPath = path.join(root, 'src/api/modules/dataset.ts')
const adapterPath = path.join(root, 'src/mock/adapter.ts')
const mockPath = path.join(root, 'src/mock/data.ts')
const recordsPath = path.join(root, 'src/views/dataset/datasetRecords.ts')
const recordEditorPath = path.join(root, 'src/views/dataset/preview/useDatasetRecordEditor.ts')
const versionsPath = path.join(root, 'src/mock/datasetVersions.ts')
const source = fs.readFileSync(previewPath, 'utf8')
const previewLogicSource = source.slice(0, source.indexOf('<style'))
@@ -27,11 +28,12 @@ const apiSource = fs.readFileSync(apiPath, 'utf8')
const adapterSource = fs.readFileSync(adapterPath, 'utf8')
const mockSource = fs.readFileSync(mockPath, 'utf8')
const recordsSource = fs.readFileSync(recordsPath, 'utf8')
const recordEditorSource = fs.readFileSync(recordEditorPath, 'utf8')
const versionsSource = fs.readFileSync(versionsPath, 'utf8')
const recordsModuleCode = ts.transpileModule(recordsSource, {
compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
}).outputText
const { parseDatasetRecords, updateDatasetRecord } = await import(
const { deleteDatasetRecord, parseDatasetRecords, updateDatasetRecord } = await import(
`data:text/javascript;base64,${Buffer.from(recordsModuleCode).toString('base64')}`
)
const versionsModuleCode = ts.transpileModule(versionsSource, {
@@ -65,6 +67,12 @@ expect(previewSurfaceSource.includes('class="record-table"'), '样本数据应
expect(previewSurfaceSource.includes('fixed="right"'), '逐条编辑操作列应固定在表格右侧')
expect(previewSurfaceSource.includes('v-for="fieldKey in tableFieldKeys"'), '企业表格应根据数据结构动态生成字段列')
expect(previewSurfaceSource.includes("emit('edit', asDatasetRecord(row))"), '每条样本应向主页面发送编辑事件')
expect(previewSurfaceSource.includes("emit('delete', asDatasetRecord(row))"), '每条样本应提供删除操作')
expect(previewSurfaceSource.includes('label="原文参照"'), '样本列表应展示生成内容对应的原文参照')
expect(source.includes('getDatasetFileRecordSources'), '详情页应从后端读取发布样本的真实原文')
expect(recordEditorSource.includes('deleteDatasetRecord('), '删除样本应只修改当前文件草稿')
expect(recordEditorSource.includes('删除后将暂存到当前草稿'), '删除样本前应说明保存版本后才生效')
expect(source.includes('selectedFileSize'), '详情文件大小也应统一显示为 MB')
expect(previewSurfaceSource.includes('暂存修改'), '单条编辑应先暂存到当前页面')
expect(source.includes('hasPendingVersionChanges && isViewingActiveVersion'), '存在暂存修改时才应显示保存版本按钮')
expect(source.includes('保存版本'), '文件工具栏应提供保存版本按钮')
@@ -85,10 +93,10 @@ expect(source.includes('hasUnsavedChanges'), '在线编辑器应跟踪未保存
expect(source.includes('onBeforeRouteLeave'), '离开页面时应保护未保存修改')
expect(previewSurfaceSource.includes("event.key.toLowerCase() === 's'"), '在线编辑器应支持快捷键保存')
expect(source.includes('loadVersions(selectedFile)'), '版本内容加载失败后应支持重新加载')
expect(source.includes('function resetRecordEditor()'), '主页面应提供统一的编辑器状态清理函数')
const resetEditorStart = source.indexOf('function resetRecordEditor()')
const updateFieldStart = source.indexOf('function updateEditField', resetEditorStart)
const resetEditorSource = source.slice(resetEditorStart, updateFieldStart)
expect(recordEditorSource.includes('function resetRecordEditor()'), '编辑器逻辑应提供统一的状态清理函数')
const resetEditorStart = recordEditorSource.indexOf('function resetRecordEditor()')
const updateFieldStart = recordEditorSource.indexOf('function openRecordEditor', resetEditorStart)
const resetEditorSource = recordEditorSource.slice(resetEditorStart, updateFieldStart)
for (const marker of [
'editorVisible.value = false',
'editingRecord.value = null',
@@ -141,6 +149,9 @@ const afterLines = updated.split('\n')
expect(afterLines[0] === beforeLines[0] && afterLines[1] === '', '单条保存不得改写其他样本或空行')
expect(JSON.parse(afterLines[2]).instruction === '第二条已修改', '单条保存应准确更新目标样本')
expect(Array.isArray(JSON.parse(afterLines[2]).output), '单条保存不应丢失数组等非字符串字段类型')
const deleted = deleteDatasetRecord(updated, 'samples.jsonl', 2)
expect(!deleted.includes('第二条已修改'), '删除样本应准确移除目标 JSONL 记录')
expect(deleted.split('\n')[0] === beforeLines[0], '删除样本不得改写其他 JSONL 记录')
const invalid = parseDatasetRecords('{invalid json}', 'samples.jsonl')
expect(invalid.records[0].kind === 'invalid' && invalid.records[0].error, '非法 JSON 行应被单独标记并给出错误')
const repairedJson = updateDatasetRecord('{invalid json}', 'samples.json', 0, { repaired: true })

View File

@@ -42,7 +42,11 @@ assert.match(
)
assert.match(viewSource, /return dataList\.value\.filter\(\(item\) => item\.source !== 'task'\)/)
assert.doesNotMatch(viewSource, /数据任务产生的数据集[\s\S]*?return \[\]/)
assert.match(viewSource, /:search-fields="\['task_id', 'name', 'description'\]"/, '数据任务搜索应支持任务 ID')
assert.match(viewSource, /:search-fields="\['task_id', 'name'\]"/, '数据任务搜索应支持任务 ID')
assert.match(viewSource, /formatMegabytes\(row\.size_bytes, row\.size\)/, '数据集大小应统一转换为 MB')
assert.match(viewSource, /<el-table-column label="版本"/, '描述列应替换为真实版本列')
assert.doesNotMatch(viewSource, /<el-table-column label="描述"/, '列表不应继续显示描述列')
assert.match(viewSource, /current_version_nos/, '版本列应使用接口返回的真实当前版本号')
assert.match(viewSource, /:multi-select="activeTab === 'task' && batchMode"/, '多选框只能在数据任务的批量模式显示')
assert.match(viewSource, /:show-batch-bar="false"/, '数据任务应使用搜索框旁的批量删除入口')
assert.match(viewSource, /<template #toolbar-extra>[\s\S]*?批量删除[\s\S]*?v-if="selectedCount > 0"[\s\S]*?删除(\{\{ selectedCount \}\}/, '批量删除按钮没有按选择状态切换')

View File

@@ -1,5 +1,10 @@
import { get, post, put, del } from '../request'
import type { DatasetItem, DatasetVersion, DatasetVersionList } from '@/types'
import type {
DatasetItem,
DatasetRecordSource,
DatasetVersion,
DatasetVersionList,
} from '@/types'
/** 数据集列表 */
export const getDatasetList = () => get<DatasetItem[]>('/dataset-manage')
@@ -31,6 +36,12 @@ export const uploadDatasetFiles = (datasetId: string | number, files: File[]) =>
export const previewDatasetFile = (fileId: string | number) =>
get<{ content: string }>(`/dataset-manage/preview/${encodeURIComponent(fileId)}`)
/** 获取发布样本所关联的原文参照 */
export const getDatasetFileRecordSources = (fileId: string | number) =>
get<{ items: DatasetRecordSource[] }>(
`/dataset-manage/records/${encodeURIComponent(fileId)}/sources`,
)
/** 数据集文件版本列表 */
export const getDatasetFileVersions = (fileId: string | number) =>
get<DatasetVersionList>(`/dataset-manage/versions/${encodeURIComponent(fileId)}`)

View File

@@ -63,6 +63,11 @@ export interface DatasetFile {
id?: string
name: string
size?: string | number
size_bytes?: number
active_version_id?: string | null
current_version_id?: string | null
current_version_no?: number | null
version_count?: number
record_count?: number
split?: 'train' | 'validation' | 'test' | null
[k: string]: any
@@ -76,13 +81,26 @@ export interface DatasetItem {
source?: DatasetSource
task_id?: string | number
size?: string | number
size_bytes?: number
count?: number
description?: string
create_time?: string
files?: DatasetFile[]
current_version_no?: number | null
current_version_nos?: number[]
version_count?: number
split_counts?: { train: number; validation: number; test: number }
}
export interface DatasetRecordSource {
line_no: number
instruction: string
input: string
output: string
source_text: string
preprocessed: boolean
}
export interface DatasetVersion {
id: string
version: number

View File

@@ -0,0 +1,26 @@
export function parseSizeBytes(value?: string | number | null) {
if (typeof value === 'number') return Number.isFinite(value) ? Math.max(0, value) : 0
const match = String(value || '').trim().match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB|TB)?$/i)
if (!match) return 0
const units: Record<string, number> = {
B: 1,
KB: 1024,
MB: 1024 ** 2,
GB: 1024 ** 3,
TB: 1024 ** 4,
}
return Number(match[1]) * units[(match[2] || 'B').toUpperCase()]
}
/** 列表统一使用 MB非零的小文件至少展示为 0.01 MB避免误显示为零。 */
export function formatMegabytes(
sizeBytes?: number | null,
legacySize?: string | number | null,
) {
const numericSize = Number(sizeBytes)
const bytes = Number.isFinite(numericSize) && numericSize > 0
? numericSize
: parseSizeBytes(legacySize)
if (bytes <= 0) return '0.00 MB'
return `${Math.max(bytes / 1024 ** 2, 0.01).toFixed(2)} MB`
}

View File

@@ -5,6 +5,7 @@ import { ElMessage, ElMessageBox } from 'element-plus'
import DataTablePage from '@/components/DataTablePage.vue'
import { getDatasetList, deleteDataset, downloadDatasetUrl } from '@/api/modules/dataset'
import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants'
import { formatMegabytes } from '@/utils/fileSize'
import type { DatasetItem, DatasetSource } from '@/types'
const router = useRouter()
@@ -107,6 +108,36 @@ function handleDownload(row: any) {
window.open(downloadDatasetUrl(row.id), '_blank')
}
function formatSizeMb(row: DatasetItem) {
return formatMegabytes(row.size_bytes, row.size)
}
function currentVersionNumbers(row: DatasetItem) {
const values = row.current_version_nos?.length
? row.current_version_nos
: [
row.current_version_no,
...(row.files || []).map((file) => file.current_version_no),
]
return [...new Set(
values
.map((value) => Number(value))
.filter((value) => Number.isInteger(value) && value > 0),
)].sort((a, b) => a - b)
}
function versionLabel(row: DatasetItem) {
const versions = currentVersionNumbers(row)
return versions.length ? versions.map((value) => `V${value}`).join(' / ') : '-'
}
function versionDescription(row: DatasetItem) {
const fileCount = row.files?.length || 0
const historyCount = Number(row.version_count || 0)
if (!fileCount) return '当前数据集没有文件版本'
return `${fileCount} 个文件,共 ${historyCount || fileCount} 个版本记录`
}
watch(activeTab, exitBatchMode)
onMounted(loadData)
</script>
@@ -118,7 +149,7 @@ onMounted(loadData)
:data="filteredDataList"
:loading="loading"
searchable
:search-fields="['task_id', 'name', 'description']"
:search-fields="['task_id', 'name']"
:multi-select="activeTab === 'task' && batchMode"
:show-batch-bar="false"
:create-text="activeTab === 'upload' ? '上传数据集' : ''"
@@ -181,16 +212,26 @@ onMounted(loadData)
</el-tag>
</template>
</el-table-column>
<el-table-column label="大小" align="center" width="100">
<el-table-column label="大小" align="center" width="120">
<template #default="{ row }">
{{ row.size && row.size !== '0 B' && row.size !== '0' ? row.size : '-' }}
<span class="numeric-cell">{{ formatSizeMb(row as DatasetItem) }}</span>
</template>
</el-table-column>
<el-table-column label="数据条数" align="center" width="100">
<template #default="{ row }">{{ row.count || 0 }}</template>
</el-table-column>
<el-table-column label="描述" align="center" show-overflow-tooltip>
<template #default="{ row }">{{ row.description || '-' }}</template>
<el-table-column label="版本" align="center" width="120">
<template #default="{ row }">
<el-tooltip
:content="versionDescription(row as DatasetItem)"
placement="top"
:show-after="350"
>
<el-tag size="small" type="info" effect="plain">
{{ versionLabel(row as DatasetItem) }}
</el-tag>
</el-tooltip>
</template>
</el-table-column>
<el-table-column label="创建时间" align="center" width="180">
<template #default="{ row }">
@@ -257,4 +298,9 @@ onMounted(loadData)
}
}
.numeric-cell {
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
</style>

View File

@@ -10,18 +10,27 @@ import {
deleteDatasetFileVersion,
downloadFileUrl,
getDataset,
getDatasetFileRecordSources,
getDatasetFileVersionContent,
getDatasetFileVersions,
} from '@/api/modules/dataset'
import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants'
import { parseDatasetRecords, updateDatasetRecord } from './datasetRecords'
import type { DatasetRecord } from './datasetRecords'
import {
attachDatasetRecordSources,
parseDatasetRecords,
} from './datasetRecords'
import DatasetVersionBar from './preview/DatasetVersionBar.vue'
import DatasetRecordTable from './preview/DatasetRecordTable.vue'
import DatasetRecordEditorDialog from './preview/DatasetRecordEditorDialog.vue'
import DatasetRawPreview from './preview/DatasetRawPreview.vue'
import type { EditField } from './preview/types'
import type { DatasetFile, DatasetItem, DatasetVersion } from '@/types'
import { useDatasetRecordEditor } from './preview/useDatasetRecordEditor'
import { formatMegabytes } from '@/utils/fileSize'
import type {
DatasetFile,
DatasetItem,
DatasetRecordSource,
DatasetVersion,
} from '@/types'
const route = useRoute()
const datasetId = route.params.id as string
@@ -34,14 +43,9 @@ const selectedFileId = ref('')
const previewContent = ref('')
const baseVersionContent = ref('')
const previewError = ref('')
const editorVisible = ref(false)
const editingRecord = ref<DatasetRecord | null>(null)
const editFields = ref<EditField[]>([])
const rawDraft = ref('')
const originalDraft = ref('')
const savingRecord = ref(false)
const savingVersion = ref(false)
const versions = ref<DatasetVersion[]>([])
const recordSources = ref<DatasetRecordSource[]>([])
const selectedVersionId = ref('')
const activeVersionId = ref('')
const loadedVersionId = ref('')
@@ -56,15 +60,13 @@ const selectedFile = computed(() => files.value.find((file) => fileKey(file) ===
const totalLines = computed(() => previewContent.value ? previewContent.value.split('\n').length : 0)
const recordResult = computed(() => parseDatasetRecords(previewContent.value, selectedFile.value?.name || ''))
const isRecordFile = computed(() => recordResult.value.supported)
const records = computed(() => recordResult.value.records)
const records = computed(() => attachDatasetRecordSources(
recordResult.value.records,
recordSources.value,
))
const viewedVersion = computed(() => versions.value.find((item) => item.id === loadedVersionId.value))
const isViewingActiveVersion = computed(() => Boolean(loadedVersionId.value) && loadedVersionId.value === activeVersionId.value)
const currentDraft = computed(() => editingRecord.value?.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value)
const hasEditorDraftChanges = computed(() => editorVisible.value && currentDraft.value !== originalDraft.value)
const hasPendingVersionChanges = computed(() => previewContent.value !== baseVersionContent.value)
const hasUnsavedChanges = computed(() => hasEditorDraftChanges.value || hasPendingVersionChanges.value)
const datasetType = computed(() => {
const value = String(dataset.value?.type || '').toLowerCase()
return DATASET_TYPE_MAP[value] || dataset.value?.type || '-'
@@ -76,11 +78,28 @@ const storageType = computed(() => {
const createdAt = computed(() => dataset.value?.create_time
? new Date(dataset.value.create_time).toLocaleString('zh-CN')
: '-')
const datasetSize = computed(() => formatMegabytes(
dataset.value?.size_bytes,
dataset.value?.size,
))
const selectedFileSize = computed(() => formatMegabytes(
selectedFile.value?.size_bytes,
selectedFile.value?.size,
))
function fileKey(file: DatasetFile) {
return String(file.id || file.name)
}
const {
editorVisible, editingRecord, editFields, rawDraft, savingRecord,
hasEditorDraftChanges, openRecordEditor, resetRecordEditor, updateEditField,
closeRecordEditor, saveRecord, deleteRecord,
} = useDatasetRecordEditor(selectedFile, isViewingActiveVersion, previewContent)
const hasUnsavedChanges = computed(
() => hasEditorDraftChanges.value || hasPendingVersionChanges.value,
)
async function loadDataset() {
loading.value = true
try {
@@ -101,7 +120,10 @@ async function loadVersions(file: DatasetFile) {
const requestId = ++versionRequestId
try {
const result = await getDatasetFileVersions(fileKey(file))
const activeContent = await getDatasetFileVersionContent(fileKey(file), result.active_version_id)
const [activeContent, sources] = await Promise.all([
getDatasetFileVersionContent(fileKey(file), result.active_version_id),
getDatasetFileRecordSources(fileKey(file)),
])
if (requestId !== versionRequestId) return
versions.value = result.versions
nextVersionNumber.value = result.next_version_number
@@ -111,9 +133,13 @@ async function loadVersions(file: DatasetFile) {
loadedVersionId.value = result.active_version_id
previewContent.value = activeContent.content
baseVersionContent.value = activeContent.content
recordSources.value = sources.items || []
resetRecordEditor()
} catch {
if (requestId === versionRequestId) previewError.value = '版本内容加载失败,请稍后重试'
if (requestId === versionRequestId) {
recordSources.value = []
previewError.value = '版本内容加载失败,请稍后重试'
}
} finally {
if (requestId === versionRequestId) previewLoading.value = false
}
@@ -221,40 +247,6 @@ async function deleteViewedVersion() {
}
}
function openRecordEditor(record: DatasetRecord) {
if (!isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
editingRecord.value = record
if (record.kind === 'object') {
editFields.value = Object.entries(record.value as Record<string, unknown>).map(([key, value]) => ({
key,
value: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
isJson: typeof value !== 'string',
}))
rawDraft.value = ''
} else {
editFields.value = []
rawDraft.value = record.raw
}
editorVisible.value = true
originalDraft.value = record.kind === 'object' ? JSON.stringify(editFields.value) : rawDraft.value
}
function resetRecordEditor() {
editorVisible.value = false
editingRecord.value = null
editFields.value = []
rawDraft.value = ''
originalDraft.value = ''
}
function updateEditField(index: number, value: string) {
const field = editFields.value[index]
if (field) field.value = value
}
async function confirmDiscardChanges(message = '当前存在尚未保存为版本的修改,放弃后将无法恢复。') {
if (!hasUnsavedChanges.value) return true
try {
@@ -269,63 +261,6 @@ async function confirmDiscardChanges(message = '当前存在尚未保存为版
}
}
async function confirmEditorDiscardChanges() {
if (!hasEditorDraftChanges.value) return true
try {
await ElMessageBox.confirm('当前样本存在未暂存修改,放弃后将无法恢复。', '确认放弃修改?', {
type: 'warning',
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
})
return true
} catch {
return false
}
}
async function closeRecordEditor(done?: () => void) {
if (savingRecord.value) return
if (!await confirmEditorDiscardChanges()) return
resetRecordEditor()
done?.()
}
function buildNextRecordValue() {
if (editingRecord.value?.kind !== 'object') return JSON.parse(rawDraft.value)
const nextValue: Record<string, unknown> = {}
for (const field of editFields.value) {
nextValue[field.key] = field.isJson ? JSON.parse(field.value) : field.value
}
return nextValue
}
async function saveRecord() {
if (!selectedFile.value || !editingRecord.value || savingRecord.value) return
const record = editingRecord.value
let nextValue: unknown
try {
nextValue = buildNextRecordValue()
} catch (error) {
ElMessage.error(`JSON 格式校验失败:${error instanceof Error ? error.message : '内容格式不正确'}`)
return
}
savingRecord.value = true
try {
const nextContent = updateDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
nextValue,
)
previewContent.value = nextContent
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条修改已暂存`)
} finally {
savingRecord.value = false
}
}
async function saveVersion() {
if (
!selectedFile.value
@@ -396,7 +331,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
<section class="dataset-overview" aria-label="数据集概览">
<div class="overview-item"><span>数据条数</span><strong>{{ (dataset?.count || 0).toLocaleString() }}</strong></div>
<div class="overview-item"><span>数据大小</span><strong>{{ dataset?.size || '-' }}</strong></div>
<div class="overview-item"><span>数据大小</span><strong>{{ datasetSize }}</strong></div>
<div class="overview-item overview-item-wide"><span>创建时间</span><strong>{{ createdAt }}</strong></div>
</section>
@@ -433,7 +368,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
<div>
<h2>{{ selectedFile?.name }}</h2>
<p>
{{ selectedFile?.size || '未知大小' }} <span aria-hidden="true">·</span>
{{ selectedFileSize }} <span aria-hidden="true">·</span>
<template v-if="isRecordFile">{{ records.length.toLocaleString() }} 条记录</template>
<template v-else>{{ totalLines.toLocaleString() }} </template>
</p>
@@ -470,6 +405,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
:is-viewing-active-version="isViewingActiveVersion"
:version-id="loadedVersionId"
@edit="openRecordEditor"
@delete="deleteRecord"
/>
<DatasetRawPreview

View File

@@ -1,3 +1,5 @@
import type { DatasetRecordSource } from '@/types'
export type DatasetRecordKind = 'object' | 'primitive' | 'invalid'
export interface DatasetRecord {
@@ -6,6 +8,8 @@ export interface DatasetRecord {
raw: string
kind: DatasetRecordKind
value: unknown
sourceText?: string
sourceWasPreprocessed?: boolean
error?: string
}
@@ -15,6 +19,35 @@ export interface DatasetRecordResult {
error?: string
}
/** 通过内容优先、物理行兜底,把发布样本重新关联到切片原文。 */
export function attachDatasetRecordSources(
records: DatasetRecord[],
sources: DatasetRecordSource[],
) {
const used = new Set<number>()
return records.map((record) => {
const value = record.kind === 'object'
? record.value as Record<string, unknown>
: {}
let sourceIndex = sources.findIndex((source, index) => (
!used.has(index)
&& source.instruction === String(value.instruction || '')
&& source.input === String(value.input || '')
&& source.output === String(value.output || '')
))
if (sourceIndex < 0 && sources[record.sourceIndex] && !used.has(record.sourceIndex)) {
sourceIndex = record.sourceIndex
}
if (sourceIndex < 0) return record
used.add(sourceIndex)
return {
...record,
sourceText: sources[sourceIndex]?.source_text || '',
sourceWasPreprocessed: sources[sourceIndex]?.preprocessed || false,
}
})
}
function isObjectRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -119,3 +152,33 @@ export function updateDatasetRecord(
throw new Error('当前文件不支持逐条编辑')
}
/** 删除目标样本;删除只修改当前草稿,仍需由调用方保存为新版本。 */
export function deleteDatasetRecord(
content: string,
fileName: string,
sourceIndex: number,
) {
const normalizedName = fileName.toLowerCase()
if (normalizedName.endsWith('.jsonl')) {
const lines = content.split('\n')
if (sourceIndex < 0 || sourceIndex >= lines.length || !lines[sourceIndex]?.trim()) {
throw new Error('目标样本不存在')
}
lines.splice(sourceIndex, 1)
return lines.join('\n')
}
if (normalizedName.endsWith('.json')) {
const root: unknown = JSON.parse(content)
if (Array.isArray(root)) {
if (sourceIndex < 0 || sourceIndex >= root.length) throw new Error('目标样本不存在')
root.splice(sourceIndex, 1)
return JSON.stringify(root, null, 2)
}
if (sourceIndex !== 0) throw new Error('目标样本不存在')
return '[]'
}
throw new Error('当前文件不支持逐条删除')
}

View File

@@ -10,6 +10,7 @@ const props = defineProps<{
const emit = defineEmits<{
edit: [record: DatasetRecord]
delete: [record: DatasetRecord]
}>()
const pageSize = 10
@@ -18,9 +19,18 @@ const currentPage = ref(1)
const allTableFieldKeys = computed(() => {
const keys = new Set<string>()
const sourceFieldKeys = new Set([
'source_text',
'source_content',
'original_text',
'original',
'source',
])
props.records.forEach((record) => {
if (record.kind !== 'object') return
Object.keys(record.value as Record<string, unknown>).forEach((key) => keys.add(key))
Object.keys(record.value as Record<string, unknown>)
.filter((key) => !sourceFieldKeys.has(key))
.forEach((key) => keys.add(key))
})
const preferred = ['instruction', 'question', 'prompt', 'input', 'context', 'output', 'answer', 'response']
return [...keys].sort((a, b) => {
@@ -69,6 +79,16 @@ function recordFieldValue(record: DatasetRecord, key: string) {
return Object.prototype.hasOwnProperty.call(data, key) ? formatFieldValue(data[key]) : '-'
}
function recordSourceText(record: DatasetRecord) {
if (record.sourceText?.trim()) return record.sourceText
if (record.kind !== 'object') return '无关联原文'
const data = record.value as Record<string, unknown>
for (const key of ['source_text', 'source_content', 'original_text', 'original', 'source']) {
if (typeof data[key] === 'string' && data[key].trim()) return data[key]
}
return '无关联原文'
}
function extraFieldValue(record: DatasetRecord) {
if (record.kind !== 'object') return '-'
const data = record.value as Record<string, unknown>
@@ -144,6 +164,22 @@ function recordRowClassName({ row }: { row: unknown }) {
</el-tooltip>
</template>
</el-table-column>
<el-table-column label="原文参照" min-width="280">
<template #default="{ row }">
<el-tooltip
:content="recordSourceText(asDatasetRecord(row))"
placement="top"
:show-after="400"
>
<div
class="table-cell-text source-reference-cell"
:class="{ 'is-empty': recordSourceText(asDatasetRecord(row)) === '无关联原文' }"
>
{{ recordSourceText(asDatasetRecord(row)) }}
</div>
</el-tooltip>
</template>
</el-table-column>
<el-table-column v-if="hasExtraTableFields" label="其他字段" min-width="240">
<template #default="{ row }">
<el-tooltip
@@ -166,19 +202,32 @@ function recordRowClassName({ row }: { row: unknown }) {
<el-tag v-else size="small" type="success">正常</el-tag>
</template>
</el-table-column>
<el-table-column label="操作" width="96" align="center" fixed="right">
<el-table-column label="操作" width="156" align="center" fixed="right">
<template #default="{ row }">
<el-button
type="primary"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`${asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑'} ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('edit', asDatasetRecord(row))"
>
<i :class="asDatasetRecord(row).kind === 'invalid' ? 'fa fa-wrench' : 'fa fa-pencil'" />
{{ asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑' }}
</el-button>
<div class="record-actions">
<el-button
type="primary"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`${asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑'} ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('edit', asDatasetRecord(row))"
>
<i :class="asDatasetRecord(row).kind === 'invalid' ? 'fa fa-wrench' : 'fa fa-pencil'" />
{{ asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑' }}
</el-button>
<el-button
type="danger"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`删除第 ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('delete', asDatasetRecord(row))"
>
<i class="fa fa-trash-o" />
删除
</el-button>
</div>
</template>
</el-table-column>
</el-table>
@@ -287,6 +336,21 @@ function recordRowClassName({ row }: { row: unknown }) {
-webkit-line-clamp: 2;
}
.source-reference-cell {
color: #344054;
}
.source-reference-cell.is-empty {
color: #98a2b3;
}
.record-actions {
display: inline-flex;
align-items: center;
justify-content: center;
white-space: nowrap;
}
.record-table .el-button i {
margin-right: 5px;
}

View File

@@ -0,0 +1,174 @@
import { computed, ref } from 'vue'
import type { ComputedRef, Ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import {
deleteDatasetRecord,
updateDatasetRecord,
} from '../datasetRecords'
import type { DatasetRecord } from '../datasetRecords'
import type { DatasetFile } from '@/types'
import type { EditField } from './types'
export function useDatasetRecordEditor(
selectedFile: ComputedRef<DatasetFile | undefined>,
isViewingActiveVersion: ComputedRef<boolean>,
previewContent: Ref<string>,
) {
const editorVisible = ref(false)
const editingRecord = ref<DatasetRecord | null>(null)
const editFields = ref<EditField[]>([])
const rawDraft = ref('')
const originalDraft = ref('')
const savingRecord = ref(false)
const currentDraft = computed(() => editingRecord.value?.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value)
const hasEditorDraftChanges = computed(
() => editorVisible.value && currentDraft.value !== originalDraft.value,
)
function resetRecordEditor() {
editorVisible.value = false
editingRecord.value = null
editFields.value = []
rawDraft.value = ''
originalDraft.value = ''
}
function openRecordEditor(record: DatasetRecord) {
if (!isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
editingRecord.value = record
if (record.kind === 'object') {
editFields.value = Object.entries(record.value as Record<string, unknown>).map(
([key, value]) => ({
key,
value: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
isJson: typeof value !== 'string',
}),
)
rawDraft.value = ''
} else {
editFields.value = []
rawDraft.value = record.raw
}
editorVisible.value = true
originalDraft.value = record.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value
}
function updateEditField(index: number, value: string) {
const field = editFields.value[index]
if (field) field.value = value
}
async function confirmEditorDiscardChanges() {
if (!hasEditorDraftChanges.value) return true
try {
await ElMessageBox.confirm(
'当前样本存在未暂存修改,放弃后将无法恢复。',
'确认放弃修改?',
{
type: 'warning',
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
},
)
return true
} catch {
return false
}
}
async function closeRecordEditor(done?: () => void) {
if (savingRecord.value || !await confirmEditorDiscardChanges()) return
resetRecordEditor()
done?.()
}
function buildNextRecordValue() {
if (editingRecord.value?.kind !== 'object') return JSON.parse(rawDraft.value)
const nextValue: Record<string, unknown> = {}
for (const field of editFields.value) {
nextValue[field.key] = field.isJson ? JSON.parse(field.value) : field.value
}
return nextValue
}
async function saveRecord() {
if (!selectedFile.value || !editingRecord.value || savingRecord.value) return
const record = editingRecord.value
let nextValue: unknown
try {
nextValue = buildNextRecordValue()
} catch (error) {
ElMessage.error(`JSON 格式校验失败:${error instanceof Error ? error.message : '内容格式不正确'}`)
return
}
savingRecord.value = true
try {
previewContent.value = updateDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
nextValue,
)
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条修改已暂存`)
} finally {
savingRecord.value = false
}
}
async function deleteRecord(record: DatasetRecord) {
if (!selectedFile.value || !isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
try {
await ElMessageBox.confirm(
'删除后将暂存到当前草稿,点击“保存版本”后正式生效。',
`删除第 ${record.displayIndex} 条数据?`,
{
type: 'warning',
confirmButtonText: '删除',
cancelButtonText: '取消',
confirmButtonClass: 'el-button--danger',
},
)
} catch {
return
}
try {
previewContent.value = deleteDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
)
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条已删除,请保存为新版本`)
} catch (error) {
ElMessage.error(error instanceof Error ? error.message : '删除失败')
}
}
return {
editorVisible,
editingRecord,
editFields,
rawDraft,
savingRecord,
hasEditorDraftChanges,
openRecordEditor,
resetRecordEditor,
updateEditField,
closeRecordEditor,
saveRecord,
deleteRecord,
}
}