YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 数据集版本管理

Browse Source 
新增数据集版本创建、切换、激活与历史追溯能力，Mock 通过 localStorage 持久化版本状态并接入版本接口路由，预览页支持版本对比与内容回滚，列表补充版本入口，回归脚本扩充断言。
This commit is contained in:
caoxiaozhu
2026-07-13 10:31:20 +08:00
parent 4dde761348
commit 0d82930181
8 changed files with 1381 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
frontend/src/api/modules/dataset.ts
View File

@@ -1,5 +1,5 @@
import { get, post, put, del } from '../request'
import type { DatasetItem } from '@/types'
import type { DatasetItem, DatasetVersion, DatasetVersionList } from '@/types'
/** 数据集列表 */
export const getDatasetList = () => get<DatasetItem[]>('/dataset-manage')
@@ -29,11 +29,58 @@ export const uploadDatasetFiles = (datasetId: string | number, files: File[]) =>
/** 预览数据集文件内容 */
export const previewDatasetFile = (fileId: string | number) =>
get<{ content: string }>(`/dataset-manage/preview/${fileId}`)
get<{ content: string }>(`/dataset-manage/preview/${encodeURIComponent(fileId)}`)
/** 数据集文件版本列表 */
export const getDatasetFileVersions = (fileId: string | number) =>
get<DatasetVersionList>(`/dataset-manage/versions/${encodeURIComponent(fileId)}`)
/** 读取指定历史版本内容（不改变当前版本） */
export const getDatasetFileVersionContent = (fileId: string | number, versionId: string) =>
get<{ version: DatasetVersion; content: string }>(
`/dataset-manage/versions/${encodeURIComponent(fileId)}/${encodeURIComponent(versionId)}`,
)
/** 保存为新版本，并自动设为当前版本 */
export const createDatasetFileVersion = (
fileId: string | number,
content: string,
options: {
description?: string
baseVersionId: string
expectedCurrentVersionId: string
},
) =>
post<{ version: DatasetVersion; content: string }>(
`/dataset-manage/versions/${encodeURIComponent(fileId)}`,
{
content,
description: options.description || '在线编辑',
base_version_id: options.baseVersionId,
expected_current_version_id: options.expectedCurrentVersionId,
},
)
/** 切换当前使用的数据集版本 */
export const activateDatasetFileVersion = (
fileId: string | number,
versionId: string,
expectedCurrentVersionId: string,
) =>
put<{ version: DatasetVersion; content: string }>(
`/dataset-manage/versions/${encodeURIComponent(fileId)}/active`,
{ version_id: versionId, expected_current_version_id: expectedCurrentVersionId },
)
/** 下载文件 URL */
export const downloadFileUrl = (datasetId: string | number, fileId: string | number) =>
`/api/dataset-manage/download/${datasetId}/${fileId}`
export const downloadFileUrl = (
datasetId: string | number,
fileId: string | number,
versionId?: string,
) => {
const baseUrl = `/api/dataset-manage/download/${datasetId}/${fileId}`
return versionId ? `${baseUrl}?version_id=${encodeURIComponent(versionId)}` : baseUrl
}
/** 打包下载数据集 URL */
export const downloadDatasetUrl = (datasetId: string | number) =>

116
frontend/src/mock/adapter.ts
View File

@@ -22,6 +22,13 @@ import {
mockTrainingLogFiles,
mockLogContent,
} from './data'
import {
activateDatasetVersion,
appendDatasetVersion,
createInitialVersionState,
getActiveDatasetVersion,
} from './datasetVersions'
import type { StoredDatasetVersion, StoredDatasetVersionState } from './datasetVersions'
/** 模拟网络延迟 */
function delay(ms = 200): Promise<void> {
@@ -57,6 +64,41 @@ function stripBaseURL(url: string): string {
return path
}
function datasetFallbackContent(fileId: string) {
const fallbackKey = fileId.endsWith('-readme') ? 'mock-readme' : 'mock-jsonl'
return mockDatasetPreviews[fileId] ?? mockDatasetPreviews[fallbackKey] ?? ''
}
function versionStorageKey(fileId: string) {
return `mock:dataset-versions:${fileId}`
}
function persistVersionState(fileId: string, state: StoredDatasetVersionState) {
localStorage.setItem(versionStorageKey(fileId), JSON.stringify(state))
}
function getVersionState(fileId: string): StoredDatasetVersionState {
const persisted = localStorage.getItem(versionStorageKey(fileId))
if (persisted) {
try {
const state = JSON.parse(persisted) as StoredDatasetVersionState
if (state.versions?.length && state.active_version_id) return state
} catch {
// 版本数据损坏时回退到初始版本
}
}
const legacyContent = localStorage.getItem(`mock:dataset-file:${fileId}`)
const state = createInitialVersionState(legacyContent ?? datasetFallbackContent(fileId))
persistVersionState(fileId, state)
return state
}
function versionMetadata(version: StoredDatasetVersion) {
const { content: _content, ...metadata } = version
return metadata
}
/** 通过路径 + method 匹配 mock 响应 */
async function handleMock(config: AxiosRequestConfig) {
await delay(150) // 模拟网络延迟
@@ -133,9 +175,77 @@ async function handleMock(config: AxiosRequestConfig) {
m = url.match(/^\/dataset-manage\/preview\/([^/]+)$/)
if (m && method === 'get') {
const fileId = decodeURIComponent(m[1])
const fallbackKey = fileId.endsWith('-readme') ? 'mock-readme' : 'mock-jsonl'
const content = mockDatasetPreviews[fileId] ?? mockDatasetPreviews[fallbackKey]
return ok({ content })
return ok({ content: getActiveDatasetVersion(getVersionState(fileId)).content })
}
if (m && method === 'put') return fail('历史版本不可覆盖，请创建新版本', 405)
m = url.match(/^\/dataset-manage\/versions\/([^/]+)$/)
if (m && method === 'get') {
const fileId = decodeURIComponent(m[1])
const state = getVersionState(fileId)
return ok({
versions: state.versions.map(versionMetadata).sort((a, b) => b.version - a.version),
active_version_id: state.active_version_id,
})
}
if (m && method === 'post') {
const fileId = decodeURIComponent(m[1])
if (typeof body.content !== 'string') return fail('文件内容格式不正确', 400)
const currentState = getVersionState(fileId)
if (body.expected_current_version_id !== currentState.active_version_id) {
return fail('当前版本已被其他用户更新，请刷新后重试', 409)
}
if (body.base_version_id !== currentState.active_version_id) {
return fail('只能基于当前版本创建新版本', 409)
}
if (body.content === getActiveDatasetVersion(currentState).content) {
return fail('数据内容没有变化，无需创建新版本', 400)
}
const nextState = appendDatasetVersion(
currentState,
body.content,
new Date().toISOString(),
typeof body.description === 'string' ? body.description : '在线编辑',
)
try {
persistVersionState(fileId, nextState)
} catch {
return fail('文件内容过大，浏览器 Mock 存储空间不足', 413)
}
const version = getActiveDatasetVersion(nextState)
return ok({ version: versionMetadata(version), content: version.content })
}
m = url.match(/^\/dataset-manage\/versions\/([^/]+)\/active$/)
if (m && method === 'put') {
const fileId = decodeURIComponent(m[1])
const currentState = getVersionState(fileId)
if (body.expected_current_version_id !== currentState.active_version_id) {
return fail('当前版本已被其他用户更新，请刷新后重试', 409)
}
let nextState: StoredDatasetVersionState
try {
nextState = activateDatasetVersion(currentState, String(body.version_id || ''))
} catch (error) {
return fail(error instanceof Error ? error.message : '版本不存在', 404)
}
try {
persistVersionState(fileId, nextState)
} catch {
return fail('浏览器 Mock 存储空间不足', 413)
}
const version = getActiveDatasetVersion(nextState)
return ok({ version: versionMetadata(version), content: version.content })
}
m = url.match(/^\/dataset-manage\/versions\/([^/]+)\/([^/]+)$/)
if (m && method === 'get') {
const fileId = decodeURIComponent(m[1])
const versionId = decodeURIComponent(m[2])
const version = getVersionState(fileId).versions.find((item) => item.id === versionId)
return version
? ok({ version: versionMetadata(version), content: version.content })
: fail('数据集版本不存在', 404)
}
// ==================== 训练任务 ====================

173
frontend/src/mock/data.ts
View File

@@ -30,10 +30,14 @@ export const mockHealth: HealthMetrics = {
}
export const mockSystemInfo: SystemInfo = {
timestamp: '2026-07-12T10:30:00+08:00',
cpu: {
percent: 32,
cores: 8,
percents: [25, 38, 42, 30, 28, 36, 40, 18],
model: 'Intel Xeon Gold 6330',
frequency_mhz: 2500,
load_1m: 2.56,
},
memory: {
used_gb: 9.2,
@@ -46,20 +50,166 @@ export const mockSystemInfo: SystemInfo = {
used_gb: 230,
total_gb: 512,
percent: 45,
read_mb_s: 86.4,
write_mb_s: 34.8,
},
gpu: [
{ name: 'NVIDIA A800', gpu_percent: 0, memory_used_gb: 0.0, memory_total_gb: 80, temperature: 32, power_w: 38, fan_speed: 0, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 28, memory_used_gb: 22.5, memory_total_gb: 80, temperature: 52, power_w: 165, fan_speed: 32, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 73, memory_used_gb: 58.7, memory_total_gb: 80, temperature: 68, power_w: 320, fan_speed: 58, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 0, memory_used_gb: 0.0, memory_total_gb: 80, temperature: 34, power_w: 42, fan_speed: 0, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 46, memory_used_gb: 36.8, memory_total_gb: 80, temperature: 60, power_w: 210, fan_speed: 42, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 0, memory_used_gb: 0.0, memory_total_gb: 80, temperature: 33, power_w: 40, fan_speed: 0, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 85, memory_used_gb: 68.2, memory_total_gb: 80, temperature: 73, power_w: 365, fan_speed: 62, clock_mhz: 1410, driver_version: '535.86.10' },
{ name: 'NVIDIA A800', gpu_percent: 15, memory_used_gb: 12.0, memory_total_gb: 80, temperature: 45, power_w: 95, fan_speed: 22, clock_mhz: 1410, driver_version: '535.86.10' },
{
id: 0,
uuid: 'GPU-MOCK-A800-00',
name: 'NVIDIA A800',
status: 'idle',
gpu_percent: 0,
memory_used_gb: 0,
memory_total_gb: 80,
memory_percent: 0,
temperature: 32,
power_w: 38,
power_limit_w: 400,
fan_speed: 0,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [],
},
{
id: 1,
uuid: 'GPU-MOCK-A800-01',
name: 'NVIDIA A800',
status: 'busy',
gpu_percent: 28,
memory_used_gb: 22.5,
memory_total_gb: 80,
memory_percent: 28.1,
temperature: 52,
power_w: 165,
power_limit_w: 400,
fan_speed: 32,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [
{ pid: 18421, name: 'python', task_name: '指令微调任务', user: 'trainer', memory_used_gb: 18.6 },
{ pid: 18503, name: 'python', task_name: '训练指标采集', user: 'trainer', memory_used_gb: 3.9 },
],
},
{
id: 2,
uuid: 'GPU-MOCK-A800-02',
name: 'NVIDIA A800',
status: 'busy',
gpu_percent: 73,
memory_used_gb: 58.7,
memory_total_gb: 80,
memory_percent: 73.4,
temperature: 68,
power_w: 320,
power_limit_w: 400,
fan_speed: 58,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [
{ pid: 20118, name: 'python', task_name: '大模型推理服务', user: 'inference', memory_used_gb: 56.4 },
{ pid: 20207, name: 'python', task_name: '推理指标采集', user: 'inference', memory_used_gb: 2.3 },
],
},
{
id: 3,
uuid: 'GPU-MOCK-A800-03',
name: 'NVIDIA A800',
status: 'idle',
gpu_percent: 0,
memory_used_gb: 0,
memory_total_gb: 80,
memory_percent: 0,
temperature: 34,
power_w: 42,
power_limit_w: 400,
fan_speed: 0,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [],
},
{
id: 4,
uuid: 'GPU-MOCK-A800-04',
name: 'NVIDIA A800',
status: 'busy',
gpu_percent: 46,
memory_used_gb: 36.8,
memory_total_gb: 80,
memory_percent: 46,
temperature: 60,
power_w: 210,
power_limit_w: 400,
fan_speed: 42,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [
{ pid: 22680, name: 'python', task_name: '批量评测任务', user: 'evaluation', memory_used_gb: 32 },
{ pid: 22814, name: 'python', task_name: '模型裁判服务', user: 'evaluation', memory_used_gb: 4.8 },
],
},
{
id: 5,
uuid: 'GPU-MOCK-A800-05',
name: 'NVIDIA A800',
status: 'idle',
gpu_percent: 0,
memory_used_gb: 0,
memory_total_gb: 80,
memory_percent: 0,
temperature: 33,
power_w: 40,
power_limit_w: 400,
fan_speed: 0,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [],
},
{
id: 6,
uuid: 'GPU-MOCK-A800-06',
name: 'NVIDIA A800',
status: 'warning',
gpu_percent: 85,
memory_used_gb: 68.2,
memory_total_gb: 80,
memory_percent: 85.3,
temperature: 73,
power_w: 365,
power_limit_w: 400,
fan_speed: 62,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [
{ pid: 24129, name: 'python', task_name: '全参数训练任务', user: 'trainer', memory_used_gb: 63.5 },
{ pid: 24211, name: 'python', task_name: '训练检查点保存', user: 'trainer', memory_used_gb: 4.7 },
],
},
{
id: 7,
uuid: 'GPU-MOCK-A800-07',
name: 'NVIDIA A800',
status: 'busy',
gpu_percent: 15,
memory_used_gb: 12,
memory_total_gb: 80,
memory_percent: 15,
temperature: 45,
power_w: 95,
power_limit_w: 400,
fan_speed: 22,
clock_mhz: 1410,
driver_version: '535.86.10',
processes: [
{ pid: 25367, name: 'python', task_name: '在线推理服务', user: 'inference', memory_used_gb: 12 },
],
},
],
network: {
download_mb: 1024,
upload_mb: 256,
download_mb_s: 42.6,
upload_mb_s: 12.4,
},
system: {
uptime_seconds: 86400 * 3 + 3600 * 7 + 1800,
@@ -112,8 +262,6 @@ export const mockDatasets: DatasetItem[] = [
create_time: '2025-12-20T08:00:00Z',
files: [
{ id: 'finance-train-jsonl', name: 'finance_train.jsonl', size: '86.4 MB' },
{ id: 'finance-validation-jsonl', name: 'finance_validation.jsonl', size: '21.8 MB' },
{ id: 'dataset-readme', name: 'README.md', size: '4.2 KB' },
],
},
{ id: 2, name: '法律文书-训练集', type: 'train', storage_type: 'local', source: 'upload', size: '256 MB', count: 15230, description: '法律文书数据集', create_time: '2025-12-25T10:30:00Z' },
@@ -129,10 +277,7 @@ export const mockDatasets: DatasetItem[] = [
...dataset,
files: dataset.files?.length
? dataset.files
: [
{ id: `dataset-${dataset.id}-samples`, name: 'dataset_samples.jsonl', size: dataset.size || '12.8 MB' },
{ id: `dataset-${dataset.id}-readme`, name: 'README.md', size: '3.8 KB' },
],
: [{ id: `dataset-${dataset.id}-samples`, name: 'dataset_samples.jsonl', size: dataset.size || '12.8 MB' }],
}))
export const mockDatasetPreviews: Record<string, string> = {

64
frontend/src/mock/datasetVersions.ts Normal file
View File

@@ -0,0 +1,64 @@
export interface StoredDatasetVersion {
id: string
version: number
create_time: string
description?: string
content: string
}
export interface StoredDatasetVersionState {
versions: StoredDatasetVersion[]
active_version_id: string
}
export function createInitialVersionState(
content: string,
createTime = '2026-07-13T00:00:00Z',
): StoredDatasetVersionState {
const version: StoredDatasetVersion = {
id: 'v1',
version: 1,
create_time: createTime,
description: '初始版本',
content,
}
return { versions: [version], active_version_id: version.id }
}
/** 追加不可变版本快照，并自动把新版本设为当前版本。 */
export function appendDatasetVersion(
state: StoredDatasetVersionState,
content: string,
createTime: string,
description = '在线编辑',
): StoredDatasetVersionState {
const nextNumber = Math.max(0, ...state.versions.map((item) => item.version)) + 1
const version: StoredDatasetVersion = {
id: `v${nextNumber}`,
version: nextNumber,
create_time: createTime,
description,
content,
}
return {
versions: [...state.versions, version],
active_version_id: version.id,
}
}
/** 仅移动当前版本指针，不修改任何历史快照。 */
export function activateDatasetVersion(
state: StoredDatasetVersionState,
versionId: string,
): StoredDatasetVersionState {
if (!state.versions.some((item) => item.id === versionId)) {
throw new Error('数据集版本不存在')
}
return { ...state, active_version_id: versionId }
}
export function getActiveDatasetVersion(state: StoredDatasetVersionState) {
const version = state.versions.find((item) => item.id === state.active_version_id)
if (!version) throw new Error('当前数据集版本不存在')
return version
}

2
frontend/src/views/dataset/DatasetListView.vue
View File

@@ -118,7 +118,7 @@ onMounted(loadData)
<template #actions="{ row }">
<div class="action-buttons">
<el-button type="primary" link size="small" @click="handlePreview(row)">
<i class="fa fa-eye" style="margin-right: 4px" />预览
<i class="fa fa-eye" style="margin-right: 4px" />详情
</el-button>
<el-button type="success" link size="small" @click="handleDownload(row)">
<i class="fa fa-download" style="margin-right: 4px" />下载

1041
frontend/src/views/dataset/DatasetPreviewView.vue
View File

File diff suppressed because it is too large Load Diff

121
frontend/src/views/dataset/datasetRecords.ts Normal file
View File

@@ -0,0 +1,121 @@
export type DatasetRecordKind = 'object' | 'primitive' | 'invalid'
export interface DatasetRecord {
sourceIndex: number
displayIndex: number
raw: string
kind: DatasetRecordKind
value: unknown
error?: string
}
export interface DatasetRecordResult {
supported: boolean
records: DatasetRecord[]
error?: string
}
function isObjectRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function parseValue(raw: string, sourceIndex: number, displayIndex: number): DatasetRecord {
try {
const value: unknown = JSON.parse(raw)
return {
sourceIndex,
displayIndex,
raw,
kind: isObjectRecord(value) ? 'object' : 'primitive',
value,
}
} catch (error) {
return {
sourceIndex,
displayIndex,
raw,
kind: 'invalid',
value: raw,
error: error instanceof Error ? error.message : 'JSON 格式不正确',
}
}
}
/** 将 JSONL 或 JSON 文件解析为可逐条编辑的样本，保留原始物理位置。 */
export function parseDatasetRecords(content: string, fileName: string): DatasetRecordResult {
const normalizedName = fileName.toLowerCase()
if (normalizedName.endsWith('.jsonl')) {
const records: DatasetRecord[] = []
content.split('\n').forEach((line, sourceIndex) => {
if (!line.trim()) return
records.push(parseValue(line, sourceIndex, records.length + 1))
})
return { supported: true, records }
}
if (normalizedName.endsWith('.json')) {
try {
const root: unknown = JSON.parse(content)
const values = Array.isArray(root) ? root : [root]
return {
supported: true,
records: values.map((value, sourceIndex) => ({
sourceIndex,
displayIndex: sourceIndex + 1,
raw: JSON.stringify(value),
kind: isObjectRecord(value) ? 'object' : 'primitive',
value,
})),
}
} catch (error) {
return {
supported: true,
records: [{
sourceIndex: 0,
displayIndex: 1,
raw: content,
kind: 'invalid',
value: content,
error: error instanceof Error ? error.message : 'JSON 格式不正确',
}],
}
}
}
return { supported: false, records: [] }
}
/** 仅替换目标样本；JSONL 中其他行（包括空行和原格式）保持不变。 */
export function updateDatasetRecord(
content: string,
fileName: string,
sourceIndex: number,
nextValue: unknown,
) {
const normalizedName = fileName.toLowerCase()
if (normalizedName.endsWith('.jsonl')) {
const lines = content.split('\n')
if (sourceIndex < 0 || sourceIndex >= lines.length) throw new Error('目标样本不存在')
lines[sourceIndex] = JSON.stringify(nextValue)
return lines.join('\n')
}
if (normalizedName.endsWith('.json')) {
let root: unknown
try {
root = JSON.parse(content)
} catch {
if (sourceIndex !== 0) throw new Error('目标样本不存在')
return JSON.stringify(nextValue, null, 2)
}
if (Array.isArray(root)) {
if (sourceIndex < 0 || sourceIndex >= root.length) throw new Error('目标样本不存在')
root[sourceIndex] = nextValue
return JSON.stringify(root, null, 2)
}
if (sourceIndex !== 0) throw new Error('目标样本不存在')
return JSON.stringify(nextValue, null, 2)
}
throw new Error('当前文件不支持逐条编辑')
}