feat(dataset): 补全样本原文与删除操作

This commit is contained in:
caoxiaozhu
2026-07-27 12:26:19 +08:00
parent bce586697b
commit e6bbb0bb49
5 changed files with 377 additions and 129 deletions

View File

@@ -10,18 +10,27 @@ import {
deleteDatasetFileVersion,
downloadFileUrl,
getDataset,
getDatasetFileRecordSources,
getDatasetFileVersionContent,
getDatasetFileVersions,
} from '@/api/modules/dataset'
import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants'
import { parseDatasetRecords, updateDatasetRecord } from './datasetRecords'
import type { DatasetRecord } from './datasetRecords'
import {
attachDatasetRecordSources,
parseDatasetRecords,
} from './datasetRecords'
import DatasetVersionBar from './preview/DatasetVersionBar.vue'
import DatasetRecordTable from './preview/DatasetRecordTable.vue'
import DatasetRecordEditorDialog from './preview/DatasetRecordEditorDialog.vue'
import DatasetRawPreview from './preview/DatasetRawPreview.vue'
import type { EditField } from './preview/types'
import type { DatasetFile, DatasetItem, DatasetVersion } from '@/types'
import { useDatasetRecordEditor } from './preview/useDatasetRecordEditor'
import { formatMegabytes } from '@/utils/fileSize'
import type {
DatasetFile,
DatasetItem,
DatasetRecordSource,
DatasetVersion,
} from '@/types'
const route = useRoute()
const datasetId = route.params.id as string
@@ -34,14 +43,9 @@ const selectedFileId = ref('')
const previewContent = ref('')
const baseVersionContent = ref('')
const previewError = ref('')
const editorVisible = ref(false)
const editingRecord = ref<DatasetRecord | null>(null)
const editFields = ref<EditField[]>([])
const rawDraft = ref('')
const originalDraft = ref('')
const savingRecord = ref(false)
const savingVersion = ref(false)
const versions = ref<DatasetVersion[]>([])
const recordSources = ref<DatasetRecordSource[]>([])
const selectedVersionId = ref('')
const activeVersionId = ref('')
const loadedVersionId = ref('')
@@ -56,15 +60,13 @@ const selectedFile = computed(() => files.value.find((file) => fileKey(file) ===
const totalLines = computed(() => previewContent.value ? previewContent.value.split('\n').length : 0)
const recordResult = computed(() => parseDatasetRecords(previewContent.value, selectedFile.value?.name || ''))
const isRecordFile = computed(() => recordResult.value.supported)
const records = computed(() => recordResult.value.records)
const records = computed(() => attachDatasetRecordSources(
recordResult.value.records,
recordSources.value,
))
const viewedVersion = computed(() => versions.value.find((item) => item.id === loadedVersionId.value))
const isViewingActiveVersion = computed(() => Boolean(loadedVersionId.value) && loadedVersionId.value === activeVersionId.value)
const currentDraft = computed(() => editingRecord.value?.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value)
const hasEditorDraftChanges = computed(() => editorVisible.value && currentDraft.value !== originalDraft.value)
const hasPendingVersionChanges = computed(() => previewContent.value !== baseVersionContent.value)
const hasUnsavedChanges = computed(() => hasEditorDraftChanges.value || hasPendingVersionChanges.value)
const datasetType = computed(() => {
const value = String(dataset.value?.type || '').toLowerCase()
return DATASET_TYPE_MAP[value] || dataset.value?.type || '-'
@@ -76,11 +78,28 @@ const storageType = computed(() => {
const createdAt = computed(() => dataset.value?.create_time
? new Date(dataset.value.create_time).toLocaleString('zh-CN')
: '-')
const datasetSize = computed(() => formatMegabytes(
dataset.value?.size_bytes,
dataset.value?.size,
))
const selectedFileSize = computed(() => formatMegabytes(
selectedFile.value?.size_bytes,
selectedFile.value?.size,
))
function fileKey(file: DatasetFile) {
return String(file.id || file.name)
}
const {
editorVisible, editingRecord, editFields, rawDraft, savingRecord,
hasEditorDraftChanges, openRecordEditor, resetRecordEditor, updateEditField,
closeRecordEditor, saveRecord, deleteRecord,
} = useDatasetRecordEditor(selectedFile, isViewingActiveVersion, previewContent)
const hasUnsavedChanges = computed(
() => hasEditorDraftChanges.value || hasPendingVersionChanges.value,
)
async function loadDataset() {
loading.value = true
try {
@@ -101,7 +120,10 @@ async function loadVersions(file: DatasetFile) {
const requestId = ++versionRequestId
try {
const result = await getDatasetFileVersions(fileKey(file))
const activeContent = await getDatasetFileVersionContent(fileKey(file), result.active_version_id)
const [activeContent, sources] = await Promise.all([
getDatasetFileVersionContent(fileKey(file), result.active_version_id),
getDatasetFileRecordSources(fileKey(file)),
])
if (requestId !== versionRequestId) return
versions.value = result.versions
nextVersionNumber.value = result.next_version_number
@@ -111,9 +133,13 @@ async function loadVersions(file: DatasetFile) {
loadedVersionId.value = result.active_version_id
previewContent.value = activeContent.content
baseVersionContent.value = activeContent.content
recordSources.value = sources.items || []
resetRecordEditor()
} catch {
if (requestId === versionRequestId) previewError.value = '版本内容加载失败,请稍后重试'
if (requestId === versionRequestId) {
recordSources.value = []
previewError.value = '版本内容加载失败,请稍后重试'
}
} finally {
if (requestId === versionRequestId) previewLoading.value = false
}
@@ -221,40 +247,6 @@ async function deleteViewedVersion() {
}
}
function openRecordEditor(record: DatasetRecord) {
if (!isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
editingRecord.value = record
if (record.kind === 'object') {
editFields.value = Object.entries(record.value as Record<string, unknown>).map(([key, value]) => ({
key,
value: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
isJson: typeof value !== 'string',
}))
rawDraft.value = ''
} else {
editFields.value = []
rawDraft.value = record.raw
}
editorVisible.value = true
originalDraft.value = record.kind === 'object' ? JSON.stringify(editFields.value) : rawDraft.value
}
function resetRecordEditor() {
editorVisible.value = false
editingRecord.value = null
editFields.value = []
rawDraft.value = ''
originalDraft.value = ''
}
function updateEditField(index: number, value: string) {
const field = editFields.value[index]
if (field) field.value = value
}
async function confirmDiscardChanges(message = '当前存在尚未保存为版本的修改,放弃后将无法恢复。') {
if (!hasUnsavedChanges.value) return true
try {
@@ -269,63 +261,6 @@ async function confirmDiscardChanges(message = '当前存在尚未保存为版
}
}
async function confirmEditorDiscardChanges() {
if (!hasEditorDraftChanges.value) return true
try {
await ElMessageBox.confirm('当前样本存在未暂存修改,放弃后将无法恢复。', '确认放弃修改?', {
type: 'warning',
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
})
return true
} catch {
return false
}
}
async function closeRecordEditor(done?: () => void) {
if (savingRecord.value) return
if (!await confirmEditorDiscardChanges()) return
resetRecordEditor()
done?.()
}
function buildNextRecordValue() {
if (editingRecord.value?.kind !== 'object') return JSON.parse(rawDraft.value)
const nextValue: Record<string, unknown> = {}
for (const field of editFields.value) {
nextValue[field.key] = field.isJson ? JSON.parse(field.value) : field.value
}
return nextValue
}
async function saveRecord() {
if (!selectedFile.value || !editingRecord.value || savingRecord.value) return
const record = editingRecord.value
let nextValue: unknown
try {
nextValue = buildNextRecordValue()
} catch (error) {
ElMessage.error(`JSON 格式校验失败:${error instanceof Error ? error.message : '内容格式不正确'}`)
return
}
savingRecord.value = true
try {
const nextContent = updateDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
nextValue,
)
previewContent.value = nextContent
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条修改已暂存`)
} finally {
savingRecord.value = false
}
}
async function saveVersion() {
if (
!selectedFile.value
@@ -396,7 +331,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
<section class="dataset-overview" aria-label="数据集概览">
<div class="overview-item"><span>数据条数</span><strong>{{ (dataset?.count || 0).toLocaleString() }}</strong></div>
<div class="overview-item"><span>数据大小</span><strong>{{ dataset?.size || '-' }}</strong></div>
<div class="overview-item"><span>数据大小</span><strong>{{ datasetSize }}</strong></div>
<div class="overview-item overview-item-wide"><span>创建时间</span><strong>{{ createdAt }}</strong></div>
</section>
@@ -433,7 +368,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
<div>
<h2>{{ selectedFile?.name }}</h2>
<p>
{{ selectedFile?.size || '未知大小' }} <span aria-hidden="true">·</span>
{{ selectedFileSize }} <span aria-hidden="true">·</span>
<template v-if="isRecordFile">{{ records.length.toLocaleString() }} 条记录</template>
<template v-else>{{ totalLines.toLocaleString() }} </template>
</p>
@@ -470,6 +405,7 @@ onBeforeUnmount(() => window.removeEventListener('beforeunload', handleBeforeUnl
:is-viewing-active-version="isViewingActiveVersion"
:version-id="loadedVersionId"
@edit="openRecordEditor"
@delete="deleteRecord"
/>
<DatasetRawPreview

View File

@@ -1,3 +1,5 @@
import type { DatasetRecordSource } from '@/types'
export type DatasetRecordKind = 'object' | 'primitive' | 'invalid'
export interface DatasetRecord {
@@ -6,6 +8,8 @@ export interface DatasetRecord {
raw: string
kind: DatasetRecordKind
value: unknown
sourceText?: string
sourceWasPreprocessed?: boolean
error?: string
}
@@ -15,6 +19,35 @@ export interface DatasetRecordResult {
error?: string
}
/** 通过内容优先、物理行兜底,把发布样本重新关联到切片原文。 */
export function attachDatasetRecordSources(
records: DatasetRecord[],
sources: DatasetRecordSource[],
) {
const used = new Set<number>()
return records.map((record) => {
const value = record.kind === 'object'
? record.value as Record<string, unknown>
: {}
let sourceIndex = sources.findIndex((source, index) => (
!used.has(index)
&& source.instruction === String(value.instruction || '')
&& source.input === String(value.input || '')
&& source.output === String(value.output || '')
))
if (sourceIndex < 0 && sources[record.sourceIndex] && !used.has(record.sourceIndex)) {
sourceIndex = record.sourceIndex
}
if (sourceIndex < 0) return record
used.add(sourceIndex)
return {
...record,
sourceText: sources[sourceIndex]?.source_text || '',
sourceWasPreprocessed: sources[sourceIndex]?.preprocessed || false,
}
})
}
function isObjectRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -119,3 +152,33 @@ export function updateDatasetRecord(
throw new Error('当前文件不支持逐条编辑')
}
/** 删除目标样本;删除只修改当前草稿,仍需由调用方保存为新版本。 */
export function deleteDatasetRecord(
content: string,
fileName: string,
sourceIndex: number,
) {
const normalizedName = fileName.toLowerCase()
if (normalizedName.endsWith('.jsonl')) {
const lines = content.split('\n')
if (sourceIndex < 0 || sourceIndex >= lines.length || !lines[sourceIndex]?.trim()) {
throw new Error('目标样本不存在')
}
lines.splice(sourceIndex, 1)
return lines.join('\n')
}
if (normalizedName.endsWith('.json')) {
const root: unknown = JSON.parse(content)
if (Array.isArray(root)) {
if (sourceIndex < 0 || sourceIndex >= root.length) throw new Error('目标样本不存在')
root.splice(sourceIndex, 1)
return JSON.stringify(root, null, 2)
}
if (sourceIndex !== 0) throw new Error('目标样本不存在')
return '[]'
}
throw new Error('当前文件不支持逐条删除')
}

View File

@@ -10,6 +10,7 @@ const props = defineProps<{
const emit = defineEmits<{
edit: [record: DatasetRecord]
delete: [record: DatasetRecord]
}>()
const pageSize = 10
@@ -18,9 +19,18 @@ const currentPage = ref(1)
const allTableFieldKeys = computed(() => {
const keys = new Set<string>()
const sourceFieldKeys = new Set([
'source_text',
'source_content',
'original_text',
'original',
'source',
])
props.records.forEach((record) => {
if (record.kind !== 'object') return
Object.keys(record.value as Record<string, unknown>).forEach((key) => keys.add(key))
Object.keys(record.value as Record<string, unknown>)
.filter((key) => !sourceFieldKeys.has(key))
.forEach((key) => keys.add(key))
})
const preferred = ['instruction', 'question', 'prompt', 'input', 'context', 'output', 'answer', 'response']
return [...keys].sort((a, b) => {
@@ -69,6 +79,16 @@ function recordFieldValue(record: DatasetRecord, key: string) {
return Object.prototype.hasOwnProperty.call(data, key) ? formatFieldValue(data[key]) : '-'
}
function recordSourceText(record: DatasetRecord) {
if (record.sourceText?.trim()) return record.sourceText
if (record.kind !== 'object') return '无关联原文'
const data = record.value as Record<string, unknown>
for (const key of ['source_text', 'source_content', 'original_text', 'original', 'source']) {
if (typeof data[key] === 'string' && data[key].trim()) return data[key]
}
return '无关联原文'
}
function extraFieldValue(record: DatasetRecord) {
if (record.kind !== 'object') return '-'
const data = record.value as Record<string, unknown>
@@ -144,6 +164,22 @@ function recordRowClassName({ row }: { row: unknown }) {
</el-tooltip>
</template>
</el-table-column>
<el-table-column label="原文参照" min-width="280">
<template #default="{ row }">
<el-tooltip
:content="recordSourceText(asDatasetRecord(row))"
placement="top"
:show-after="400"
>
<div
class="table-cell-text source-reference-cell"
:class="{ 'is-empty': recordSourceText(asDatasetRecord(row)) === '无关联原文' }"
>
{{ recordSourceText(asDatasetRecord(row)) }}
</div>
</el-tooltip>
</template>
</el-table-column>
<el-table-column v-if="hasExtraTableFields" label="其他字段" min-width="240">
<template #default="{ row }">
<el-tooltip
@@ -166,19 +202,32 @@ function recordRowClassName({ row }: { row: unknown }) {
<el-tag v-else size="small" type="success">正常</el-tag>
</template>
</el-table-column>
<el-table-column label="操作" width="96" align="center" fixed="right">
<el-table-column label="操作" width="156" align="center" fixed="right">
<template #default="{ row }">
<el-button
type="primary"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`${asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑'} ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('edit', asDatasetRecord(row))"
>
<i :class="asDatasetRecord(row).kind === 'invalid' ? 'fa fa-wrench' : 'fa fa-pencil'" />
{{ asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑' }}
</el-button>
<div class="record-actions">
<el-button
type="primary"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`${asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑'} ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('edit', asDatasetRecord(row))"
>
<i :class="asDatasetRecord(row).kind === 'invalid' ? 'fa fa-wrench' : 'fa fa-pencil'" />
{{ asDatasetRecord(row).kind === 'invalid' ? '修复' : '编辑' }}
</el-button>
<el-button
type="danger"
link
:disabled="!isViewingActiveVersion"
:title="isViewingActiveVersion ? '' : '历史版本为只读,请先设为当前版本'"
:aria-label="`删除第 ${asDatasetRecord(row).displayIndex} 条数据`"
@click="emit('delete', asDatasetRecord(row))"
>
<i class="fa fa-trash-o" />
删除
</el-button>
</div>
</template>
</el-table-column>
</el-table>
@@ -287,6 +336,21 @@ function recordRowClassName({ row }: { row: unknown }) {
-webkit-line-clamp: 2;
}
.source-reference-cell {
color: #344054;
}
.source-reference-cell.is-empty {
color: #98a2b3;
}
.record-actions {
display: inline-flex;
align-items: center;
justify-content: center;
white-space: nowrap;
}
.record-table .el-button i {
margin-right: 5px;
}

View File

@@ -0,0 +1,174 @@
import { computed, ref } from 'vue'
import type { ComputedRef, Ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import {
deleteDatasetRecord,
updateDatasetRecord,
} from '../datasetRecords'
import type { DatasetRecord } from '../datasetRecords'
import type { DatasetFile } from '@/types'
import type { EditField } from './types'
export function useDatasetRecordEditor(
selectedFile: ComputedRef<DatasetFile | undefined>,
isViewingActiveVersion: ComputedRef<boolean>,
previewContent: Ref<string>,
) {
const editorVisible = ref(false)
const editingRecord = ref<DatasetRecord | null>(null)
const editFields = ref<EditField[]>([])
const rawDraft = ref('')
const originalDraft = ref('')
const savingRecord = ref(false)
const currentDraft = computed(() => editingRecord.value?.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value)
const hasEditorDraftChanges = computed(
() => editorVisible.value && currentDraft.value !== originalDraft.value,
)
function resetRecordEditor() {
editorVisible.value = false
editingRecord.value = null
editFields.value = []
rawDraft.value = ''
originalDraft.value = ''
}
function openRecordEditor(record: DatasetRecord) {
if (!isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
editingRecord.value = record
if (record.kind === 'object') {
editFields.value = Object.entries(record.value as Record<string, unknown>).map(
([key, value]) => ({
key,
value: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
isJson: typeof value !== 'string',
}),
)
rawDraft.value = ''
} else {
editFields.value = []
rawDraft.value = record.raw
}
editorVisible.value = true
originalDraft.value = record.kind === 'object'
? JSON.stringify(editFields.value)
: rawDraft.value
}
function updateEditField(index: number, value: string) {
const field = editFields.value[index]
if (field) field.value = value
}
async function confirmEditorDiscardChanges() {
if (!hasEditorDraftChanges.value) return true
try {
await ElMessageBox.confirm(
'当前样本存在未暂存修改,放弃后将无法恢复。',
'确认放弃修改?',
{
type: 'warning',
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
},
)
return true
} catch {
return false
}
}
async function closeRecordEditor(done?: () => void) {
if (savingRecord.value || !await confirmEditorDiscardChanges()) return
resetRecordEditor()
done?.()
}
function buildNextRecordValue() {
if (editingRecord.value?.kind !== 'object') return JSON.parse(rawDraft.value)
const nextValue: Record<string, unknown> = {}
for (const field of editFields.value) {
nextValue[field.key] = field.isJson ? JSON.parse(field.value) : field.value
}
return nextValue
}
async function saveRecord() {
if (!selectedFile.value || !editingRecord.value || savingRecord.value) return
const record = editingRecord.value
let nextValue: unknown
try {
nextValue = buildNextRecordValue()
} catch (error) {
ElMessage.error(`JSON 格式校验失败:${error instanceof Error ? error.message : '内容格式不正确'}`)
return
}
savingRecord.value = true
try {
previewContent.value = updateDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
nextValue,
)
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条修改已暂存`)
} finally {
savingRecord.value = false
}
}
async function deleteRecord(record: DatasetRecord) {
if (!selectedFile.value || !isViewingActiveVersion.value) {
ElMessage.warning('历史版本为只读,请先设为当前版本')
return
}
try {
await ElMessageBox.confirm(
'删除后将暂存到当前草稿,点击“保存版本”后正式生效。',
`删除第 ${record.displayIndex} 条数据?`,
{
type: 'warning',
confirmButtonText: '删除',
cancelButtonText: '取消',
confirmButtonClass: 'el-button--danger',
},
)
} catch {
return
}
try {
previewContent.value = deleteDatasetRecord(
previewContent.value,
selectedFile.value.name,
record.sourceIndex,
)
resetRecordEditor()
ElMessage.success(`${record.displayIndex} 条已删除,请保存为新版本`)
} catch (error) {
ElMessage.error(error instanceof Error ? error.message : '删除失败')
}
}
return {
editorVisible,
editingRecord,
editFields,
rawDraft,
savingRecord,
hasEditorDraftChanges,
openRecordEditor,
resetRecordEditor,
updateEditField,
closeRecordEditor,
saveRecord,
deleteRecord,
}
}