feat: 完成数据处理接口与前端接入
This commit is contained in:
caoxiaozhu
@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
@@ -21,6 +21,25 @@ import {
|
||||
} from './create/useDataProcessDraft'
|
||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useModelsStore } from '@/stores/models'
|
||||
import {
|
||||
buildDataProcessPreview,
|
||||
createDataProcessPreview,
|
||||
createDataProcessTask,
|
||||
deleteDataProcessPreview,
|
||||
deleteDataProcessSourceFile,
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
pullDataProcessExternalSource,
|
||||
testDataProcessExternalSource,
|
||||
updateDataProcessPreview,
|
||||
updateDataProcessTask,
|
||||
uploadDataProcessSourceFiles,
|
||||
type DataProcessExternalSourcePayload,
|
||||
type DataProcessPreviewItem,
|
||||
type DataProcessSourceFile,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { DataProcessConfig } from '@/types/dataProcess'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
GenerationControlOptions,
|
||||
@@ -35,11 +54,14 @@ import type {
|
||||
const router = useRouter()
|
||||
const modelsStore = useModelsStore()
|
||||
const { list: modelList } = storeToRefs(modelsStore)
|
||||
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
|
||||
const generationModels = computed(() => modelList.value.filter((model) => (
|
||||
model.type === 'LLM'
|
||||
&& (model.model_source === 'api' || model.model_source === 'online' || Boolean(model.api_url))
|
||||
)))
|
||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
|
||||
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
|
||||
const PREVIEW_MODEL_VERSION = 'backend-pipeline-v1'
|
||||
|
||||
const WIZARD_STEPS = [
|
||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||
@@ -52,6 +74,7 @@ const WIZARD_STEPS = [
|
||||
const currentStep = ref(0)
|
||||
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
|
||||
const task = reactive({ name: '', description: '' })
|
||||
const taskId = ref<string | null>(null)
|
||||
const processType = ref<ProcessType>('structured')
|
||||
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
|
||||
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
|
||||
@@ -61,18 +84,17 @@ const modelSelectionOptions = computed<GenerationControlOptions>(() => (
|
||||
const uploadedFiles = ref<UploadedDataFile[]>([])
|
||||
|
||||
const externalSource = reactive<ExternalDataSource>({
|
||||
type: 'mysql',
|
||||
type: 'postgresql',
|
||||
url: '',
|
||||
authMode: 'none',
|
||||
username: '',
|
||||
password: '',
|
||||
token: '',
|
||||
limit: 1000,
|
||||
query: '',
|
||||
fileName: 'external-data.jsonl',
|
||||
})
|
||||
const externalPulling = ref(false)
|
||||
const externalConnected = ref(false)
|
||||
let connectionTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let pullTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
||||
const previewSignature = ref('')
|
||||
@@ -88,6 +110,7 @@ const {
|
||||
generation,
|
||||
results,
|
||||
selectedResultId,
|
||||
persistResultChanges,
|
||||
resetDownstream,
|
||||
restoreResult,
|
||||
startGeneration,
|
||||
@@ -96,11 +119,9 @@ const {
|
||||
updateResultField,
|
||||
validateResults,
|
||||
} = useDataProcessGeneration({
|
||||
previewItems,
|
||||
processType,
|
||||
structuredOptions,
|
||||
unstructuredOptions,
|
||||
taskId,
|
||||
dirty,
|
||||
beforeGenerate: syncPreviewChanges,
|
||||
})
|
||||
|
||||
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
|
||||
@@ -160,7 +181,101 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
|
||||
structuredOptions.value = { ...structuredOptions.value, ...value }
|
||||
}
|
||||
|
||||
function toBackendConfig(): DataProcessConfig {
|
||||
const options = processType.value === 'unstructured'
|
||||
? unstructuredOptions.value
|
||||
: structuredOptions.value
|
||||
|
||||
const common = {
|
||||
preprocess_options: [...options.preprocessOptions],
|
||||
semantic_enrichment: options.semanticEnrichment,
|
||||
dataset_split: { ...options.datasetSplit },
|
||||
generation_model_id: options.generationModelId,
|
||||
generation_prompt: options.generationPrompt,
|
||||
temperature: options.temperature,
|
||||
max_tokens: options.maxTokens,
|
||||
json_mode: options.jsonMode,
|
||||
quality_filter_enabled: options.qualityFilterEnabled,
|
||||
filter_low_quality: options.filterLowQuality,
|
||||
filter_short_content: options.filterShortContent,
|
||||
min_output_length: options.minOutputLength,
|
||||
}
|
||||
|
||||
if (processType.value === 'unstructured') {
|
||||
return {
|
||||
...common,
|
||||
chunk_method: unstructuredOptions.value.chunkMethod,
|
||||
chunk_size: unstructuredOptions.value.chunkSize,
|
||||
chunk_overlap: unstructuredOptions.value.chunkOverlap,
|
||||
min_chunk_size: unstructuredOptions.value.minChunkSize,
|
||||
custom_delimiter: unstructuredOptions.value.customDelimiter,
|
||||
preserve_tables: unstructuredOptions.value.preserveTables,
|
||||
preserve_code_blocks: unstructuredOptions.value.preserveCodeBlocks,
|
||||
preserve_lists: unstructuredOptions.value.preserveLists,
|
||||
qa_pairs_per_chunk: unstructuredOptions.value.qaPairsPerChunk,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...common,
|
||||
qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
|
||||
}
|
||||
}
|
||||
|
||||
function taskPayload() {
|
||||
return {
|
||||
name: task.name.trim(),
|
||||
description: task.description.trim(),
|
||||
process_type: processType.value,
|
||||
config: toBackendConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
function externalPayload(): DataProcessExternalSourcePayload {
|
||||
return {
|
||||
type: externalSource.type,
|
||||
url: externalSource.url.trim(),
|
||||
auth_mode: externalSource.authMode,
|
||||
username: externalSource.username || undefined,
|
||||
password: externalSource.password || undefined,
|
||||
limit: externalSource.limit,
|
||||
query: externalSource.query?.trim() || undefined,
|
||||
file_name: externalSource.fileName || 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
return {
|
||||
id: String(item.id),
|
||||
sourceFileId: String(item.source_file_id),
|
||||
originalContent: item.original_content,
|
||||
editedContent: item.edited_content,
|
||||
sourceStart: item.source_start,
|
||||
sourceEnd: item.source_end,
|
||||
sourceStartLine: item.source_start_line,
|
||||
sourceEndLine: item.source_end_line,
|
||||
tokenCount: item.token_count,
|
||||
status: item.status,
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
|
||||
function mapSourceFile(file: DataProcessSourceFile, content = ''): UploadedDataFile {
|
||||
return {
|
||||
uid: String(file.id),
|
||||
sourceFileId: String(file.id),
|
||||
name: file.name,
|
||||
size: file.size_bytes,
|
||||
count: file.record_count,
|
||||
content,
|
||||
fileFormat: file.file_format,
|
||||
checksumSha256: file.checksum_sha256,
|
||||
status: 'ready',
|
||||
}
|
||||
}
|
||||
|
||||
const { persistDraft, restoreDraft } = useDataProcessDraft({
|
||||
taskId,
|
||||
currentStepId,
|
||||
task,
|
||||
processType,
|
||||
@@ -255,7 +370,7 @@ const generationOptionsSignature = computed(() => JSON.stringify(generationAffec
|
||||
|
||||
function buildPreviewSignature() {
|
||||
const filesSignature = uploadedFiles.value
|
||||
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
|
||||
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.checksumSha256 || file.count}`)
|
||||
.join('|')
|
||||
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
|
||||
}
|
||||
@@ -280,7 +395,7 @@ watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||
})
|
||||
|
||||
watch(
|
||||
[task, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
[taskId, task, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
persistDraft,
|
||||
{ deep: true },
|
||||
)
|
||||
@@ -294,48 +409,85 @@ watch(currentStep, () => nextTick(scrollToStepTop))
|
||||
async function handleFileChange(uploadFile: UploadFile) {
|
||||
const raw = uploadFile.raw
|
||||
if (!raw) return
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
if (raw.size > 200 * 1024 * 1024) {
|
||||
ElMessage.warning('单文件不能超过 200MB')
|
||||
return
|
||||
}
|
||||
|
||||
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
|
||||
const textExtensions = ['txt', 'md', 'json', 'jsonl', 'csv']
|
||||
const textExtensions = new Set(['txt', 'md', 'json', 'jsonl', 'csv'])
|
||||
if (!textExtensions.has(extension)) {
|
||||
ElMessage.error('当前仅支持 TXT、Markdown、JSON、JSONL 和 CSV；不会用示例内容替代无法解析的文件')
|
||||
return
|
||||
}
|
||||
|
||||
if (uploadedFiles.value.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
ElMessage.warning('同名且同大小的文件已经上传')
|
||||
return
|
||||
}
|
||||
|
||||
let content = ''
|
||||
if (textExtensions.includes(extension)) {
|
||||
try {
|
||||
content = await raw.text()
|
||||
} catch {
|
||||
content = ''
|
||||
}
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await raw.arrayBuffer())
|
||||
} catch {
|
||||
ElMessage.error('文件不是有效的 UTF-8 文本，请转换编码后重试')
|
||||
return
|
||||
}
|
||||
if (!content.trim()) {
|
||||
ElMessage.warning('不能上传空文件')
|
||||
return
|
||||
}
|
||||
|
||||
const fileContent = content.trim() ? content : DEFAULT_SOURCE_TEXT
|
||||
const linesCount = fileContent.split('\n').filter((line) => line.trim()).length
|
||||
|
||||
// Prevent duplicate upload of the same file
|
||||
if (!uploadedFiles.value.some(f => f.name === raw.name && f.size === raw.size)) {
|
||||
uploadedFiles.value.push({
|
||||
uid: uploadFile.uid || Date.now() + Math.random(),
|
||||
name: raw.name,
|
||||
size: raw.size,
|
||||
count: linesCount,
|
||||
content: fileContent
|
||||
})
|
||||
try {
|
||||
const uploaded = await uploadDataProcessSourceFiles(taskId.value, [raw])
|
||||
const source = uploaded.files[0]
|
||||
if (!source) throw new Error('后端未返回源文件记录')
|
||||
uploadedFiles.value.push(mapSourceFile(source, content))
|
||||
previewSignature.value = ''
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
ElMessage.success(`文件 ${source.name} 上传成功`)
|
||||
} catch {
|
||||
// 请求层已展示后端的解析或格式错误。
|
||||
}
|
||||
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function useSampleFile() {
|
||||
uploadedFiles.value = [{
|
||||
uid: 'sample-1',
|
||||
name: 'finance_qa.jsonl',
|
||||
size: 128 * 1024 * 1024,
|
||||
count: DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length,
|
||||
content: DEFAULT_SOURCE_TEXT
|
||||
}]
|
||||
dirty.value = true
|
||||
async function useSampleFile() {
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
const sample = new File([DEFAULT_SOURCE_TEXT], 'finance_qa.jsonl', { type: 'application/x-ndjson' })
|
||||
await handleFileChange({ raw: sample, uid: Date.now(), name: sample.name } as UploadFile)
|
||||
}
|
||||
|
||||
async function restoreRegisteredSources() {
|
||||
if (!taskId.value) return
|
||||
try {
|
||||
const savedTask = await getDataProcessTask(taskId.value)
|
||||
const sources = savedTask.source_files || []
|
||||
const restoredFiles = await Promise.all(sources.map(async (file) => {
|
||||
try {
|
||||
const source = await getDataProcessSourceContent(taskId.value!, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
return mapSourceFile(file, source.content)
|
||||
} catch {
|
||||
return mapSourceFile(file)
|
||||
}
|
||||
}))
|
||||
uploadedFiles.value = restoredFiles
|
||||
if (restoredFiles.length) {
|
||||
ElMessage.success(`已同步 ${restoredFiles.length} 个已登记源文件`)
|
||||
}
|
||||
} catch {
|
||||
ElMessage.warning('草稿任务暂时无法从后端同步，请检查服务后重试')
|
||||
}
|
||||
}
|
||||
|
||||
function updateExternalSource(value: ExternalDataSource) {
|
||||
@@ -343,58 +495,81 @@ function updateExternalSource(value: ExternalDataSource) {
|
||||
externalConnected.value = false
|
||||
}
|
||||
|
||||
function handleTestConnection() {
|
||||
async function handleTestConnection() {
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
if (!externalSource.url.trim()) {
|
||||
ElMessage.warning('请先填写数据源地址')
|
||||
return
|
||||
}
|
||||
if (connectionTimer) clearTimeout(connectionTimer)
|
||||
externalPulling.value = true
|
||||
connectionTimer = setTimeout(() => {
|
||||
connectionTimer = null
|
||||
try {
|
||||
const result = await testDataProcessExternalSource(taskId.value, externalPayload())
|
||||
externalConnected.value = result.connected
|
||||
if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
|
||||
else ElMessage.warning(result.message || '数据源连接失败')
|
||||
} catch {
|
||||
externalConnected.value = false
|
||||
} finally {
|
||||
externalPulling.value = false
|
||||
externalConnected.value = true
|
||||
ElMessage.success('数据源连接测试成功')
|
||||
}, 1500)
|
||||
}
|
||||
}
|
||||
|
||||
function handlePullData() {
|
||||
async function handlePullData() {
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
if (!externalSource.url.trim()) {
|
||||
ElMessage.warning('请先填写数据源地址')
|
||||
return
|
||||
}
|
||||
if (pullTimer) clearTimeout(pullTimer)
|
||||
if (!externalSource.query?.trim()) {
|
||||
ElMessage.warning('请先填写只读 SELECT 查询语句')
|
||||
return
|
||||
}
|
||||
externalPulling.value = true
|
||||
pullTimer = setTimeout(() => {
|
||||
pullTimer = null
|
||||
externalPulling.value = false
|
||||
try {
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
|
||||
const newFiles: UploadedDataFile[] = []
|
||||
for (const file of response.files) {
|
||||
const source = await getDataProcessSourceContent(taskId.value, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
newFiles.push(mapSourceFile(file, source.content))
|
||||
}
|
||||
uploadedFiles.value.push(...newFiles)
|
||||
externalConnected.value = true
|
||||
const typeName = externalSource.type.toUpperCase()
|
||||
const id = `external-${Date.now()}`
|
||||
uploadedFiles.value.push({
|
||||
uid: id,
|
||||
name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
|
||||
size: Math.min(externalSource.limit, 5000) * 64,
|
||||
count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
|
||||
content: DEFAULT_SOURCE_TEXT,
|
||||
})
|
||||
dirty.value = true
|
||||
ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
|
||||
}, 2000)
|
||||
}
|
||||
|
||||
function handleRemoveFile(uid: string | number) {
|
||||
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
|
||||
if (index > -1) {
|
||||
uploadedFiles.value.splice(index, 1)
|
||||
previewSignature.value = ''
|
||||
previewItems.value = []
|
||||
selectedPreviewId.value = null
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
ElMessage.success(`已成功登记 ${newFiles.length} 个外部源文件`)
|
||||
} catch {
|
||||
externalConnected.value = false
|
||||
} finally {
|
||||
externalPulling.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRemoveFile(uid: string | number) {
|
||||
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
|
||||
if (index < 0 || !taskId.value) return
|
||||
try {
|
||||
await deleteDataProcessSourceFile(taskId.value, uid)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
uploadedFiles.value.splice(index, 1)
|
||||
previewSignature.value = ''
|
||||
previewItems.value = []
|
||||
selectedPreviewId.value = null
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function resetSourceDataForProcessTypeChange() {
|
||||
uploadedFiles.value = []
|
||||
previewSignature.value = ''
|
||||
@@ -415,10 +590,24 @@ async function nextFromCreate() {
|
||||
async function nextFromModel() {
|
||||
const valid = await modelSelectionRef.value?.validate()
|
||||
if (!valid) return
|
||||
goToStep('upload')
|
||||
try {
|
||||
const saved = taskId.value
|
||||
? await updateDataProcessTask(taskId.value, taskPayload())
|
||||
: await createDataProcessTask(taskPayload())
|
||||
taskId.value = String(saved.id)
|
||||
dirty.value = true
|
||||
persistDraft()
|
||||
goToStep('upload')
|
||||
} catch {
|
||||
// 请求层已展示名称冲突或配置非法等具体原因。
|
||||
}
|
||||
}
|
||||
|
||||
function nextFromUpload() {
|
||||
async function nextFromUpload() {
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
|
||||
return
|
||||
}
|
||||
if (uploadedFiles.value.length === 0) {
|
||||
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
||||
return
|
||||
@@ -426,14 +615,25 @@ function nextFromUpload() {
|
||||
|
||||
const signature = buildPreviewSignature()
|
||||
if (signature !== previewSignature.value) {
|
||||
previewItems.value = uploadedFiles.value.flatMap((file) =>
|
||||
buildPreviewItems(
|
||||
file.content,
|
||||
processType.value,
|
||||
String(file.uid),
|
||||
processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
|
||||
),
|
||||
)
|
||||
try {
|
||||
await buildDataProcessPreview(taskId.value, {
|
||||
source_file_ids: uploadedFiles.value.map((file) => file.sourceFileId || file.uid),
|
||||
})
|
||||
const first = await getDataProcessPreview(taskId.value, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const next = await getDataProcessPreview(taskId.value, { page, page_size: 500 })
|
||||
items.push(...next.items)
|
||||
}
|
||||
previewItems.value = items.map(mapPreviewItem)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
if (!previewItems.value.length) {
|
||||
ElMessage.warning('源文件没有生成可用的预览条目，请检查文件内容和预处理配置')
|
||||
return
|
||||
}
|
||||
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
|
||||
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
|
||||
selectedPreviewIdsByFile.value = selectedPreviewId.value && selectedPreviewFileId.value
|
||||
@@ -464,40 +664,50 @@ function updatePreviewContent(id: string, value: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item.editedContent = value
|
||||
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
|
||||
item.tokenCount = estimateTokenCount(value)
|
||||
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
async function syncPreviewChanges() {
|
||||
if (!taskId.value) throw new Error('任务尚未创建')
|
||||
const changedItems = previewItems.value.filter((item) => item.status === 'modified' || item.status === 'manual')
|
||||
for (const item of changedItems) {
|
||||
const saved = await updateDataProcessPreview(taskId.value, item.id, {
|
||||
edited_content: item.editedContent,
|
||||
expected_updated_at: item.updatedAt,
|
||||
})
|
||||
const index = previewItems.value.findIndex((entry) => entry.id === item.id)
|
||||
if (index >= 0) previewItems.value[index] = mapPreviewItem(saved)
|
||||
}
|
||||
}
|
||||
|
||||
function restorePreviewItem(id: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item || item.sourceStart == null) return
|
||||
item.editedContent = item.originalContent
|
||||
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
|
||||
item.tokenCount = estimateTokenCount(item.originalContent)
|
||||
item.status = 'original'
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function addPreviewItem() {
|
||||
if (!selectedPreviewFileId.value) return
|
||||
const id = `manual-${Date.now()}`
|
||||
previewItems.value.push({
|
||||
id,
|
||||
sourceFileId: selectedPreviewFileId.value,
|
||||
originalContent: '',
|
||||
editedContent: '',
|
||||
sourceStart: null,
|
||||
sourceEnd: null,
|
||||
sourceStartLine: null,
|
||||
sourceEndLine: null,
|
||||
tokenCount: 1,
|
||||
status: 'manual',
|
||||
})
|
||||
selectPreviewItem(id)
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
async function addPreviewItem() {
|
||||
if (!selectedPreviewFileId.value || !taskId.value) return
|
||||
try {
|
||||
const created = await createDataProcessPreview(taskId.value, {
|
||||
source_file_id: selectedPreviewFileId.value,
|
||||
edited_content: '',
|
||||
})
|
||||
const item = mapPreviewItem(created)
|
||||
previewItems.value.push(item)
|
||||
selectPreviewItem(item.id)
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
} catch {
|
||||
// 请求层已展示错误。
|
||||
}
|
||||
}
|
||||
|
||||
async function removePreviewItem(id: string) {
|
||||
@@ -512,6 +722,12 @@ async function removePreviewItem(id: string) {
|
||||
|
||||
const index = previewItems.value.findIndex((item) => item.id === id)
|
||||
if (index < 0) return
|
||||
if (!taskId.value) return
|
||||
try {
|
||||
await deleteDataProcessPreview(taskId.value, id)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
previewItems.value.splice(index, 1)
|
||||
selectedPreviewId.value = activePreviewItems.value[Math.min(index, activePreviewItems.value.length - 1)]?.id ?? null
|
||||
if (selectedPreviewFileId.value && selectedPreviewId.value) {
|
||||
@@ -531,7 +747,7 @@ async function handlePrimaryAction() {
|
||||
return
|
||||
}
|
||||
if (currentStepId.value === 'upload') {
|
||||
nextFromUpload()
|
||||
await nextFromUpload()
|
||||
return
|
||||
}
|
||||
if (currentStepId.value === 'preview') {
|
||||
@@ -546,7 +762,7 @@ async function handlePrimaryAction() {
|
||||
if (generation.status === 'success') {
|
||||
goToStep('results')
|
||||
} else if (generation.status !== 'running') {
|
||||
startGeneration()
|
||||
await startGeneration()
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -567,6 +783,15 @@ async function saveTask() {
|
||||
ElMessage.warning('请先修正校验失败的结果')
|
||||
return
|
||||
}
|
||||
try {
|
||||
await persistResultChanges()
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
if (!validateResults()) {
|
||||
ElMessage.warning('仍有结果未通过后端质量校验，请继续修正')
|
||||
return
|
||||
}
|
||||
dirty.value = false
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
allowLeave = true
|
||||
@@ -607,12 +832,10 @@ onBeforeRouteLeave(async () => {
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
stopGenerationTimer()
|
||||
if (connectionTimer) clearTimeout(connectionTimer)
|
||||
if (pullTimer) clearTimeout(pullTimer)
|
||||
})
|
||||
onMounted(() => {
|
||||
onMounted(async () => {
|
||||
restoreDraft()
|
||||
modelsStore.load()
|
||||
await Promise.all([modelsStore.load(), restoreRegisteredSources()])
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
@@ -1,228 +1,451 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref } from 'vue'
|
||||
import { computed, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import PageCard from '@/components/PageCard.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
|
||||
interface ResultRow {
|
||||
id: string
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
status: 'valid' | 'modified' | 'invalid'
|
||||
}
|
||||
|
||||
interface ProcessDetail {
|
||||
id: string
|
||||
name: string
|
||||
status: string
|
||||
description: string
|
||||
processType: string
|
||||
sourceDataset: string
|
||||
outputDataset?: string
|
||||
outputDatasetId?: number
|
||||
creator: string
|
||||
createTime: string
|
||||
startTime?: string
|
||||
completeTime?: string
|
||||
duration?: string
|
||||
progress: number
|
||||
inputCount: number
|
||||
outputCount: number
|
||||
filteredCount: number
|
||||
duplicateCount: number
|
||||
errorCount: number
|
||||
config: Array<{ label: string; value: string }>
|
||||
results: ResultRow[]
|
||||
failureReason?: string
|
||||
}
|
||||
|
||||
const completedResults: ResultRow[] = [
|
||||
{
|
||||
id: 'result-001',
|
||||
instruction: '用户询问如何修改订单收货地址，应如何回复？',
|
||||
input: '订单已经提交，但还没有发货。',
|
||||
output: '您好，订单发货前可以在订单详情中申请修改收货地址，提交后请等待系统审核。',
|
||||
status: 'valid',
|
||||
},
|
||||
{
|
||||
id: 'result-002',
|
||||
instruction: '概括用户的退款诉求。',
|
||||
input: '商品收到后发现破损，希望尽快退货退款。',
|
||||
output: '用户因商品破损申请退货退款，并希望尽快处理。',
|
||||
status: 'modified',
|
||||
},
|
||||
{
|
||||
id: 'result-003',
|
||||
instruction: '判断咨询所属业务类型。',
|
||||
input: '会员积分什么时候到账？',
|
||||
output: '会员权益 / 积分到账咨询',
|
||||
status: 'valid',
|
||||
},
|
||||
{
|
||||
id: 'result-004',
|
||||
instruction: '生成简洁的客服回复。',
|
||||
input: '优惠券显示已过期，但昨天还能使用。',
|
||||
output: '您好，请提供优惠券名称和订单信息，我们将为您核实有效期及使用记录。',
|
||||
status: 'valid',
|
||||
},
|
||||
]
|
||||
|
||||
const detailMap: Record<string, ProcessDetail> = {
|
||||
'183921': {
|
||||
id: '183921',
|
||||
name: '客服问答数据清洗',
|
||||
status: 'completed',
|
||||
description: '清洗客服对话中的无效记录、重复问答和格式异常内容，生成可用于模型训练的标准数据集。',
|
||||
processType: '结构化数据',
|
||||
sourceDataset: '客服对话原始集',
|
||||
outputDataset: '客服对话清洗集',
|
||||
outputDatasetId: 7,
|
||||
creator: '管理员',
|
||||
createTime: '2026-07-08 14:23:00',
|
||||
startTime: '2026-07-08 14:23:18',
|
||||
completeTime: '2026-07-08 14:28:42',
|
||||
duration: '5 分 24 秒',
|
||||
progress: 100,
|
||||
inputCount: 19068,
|
||||
outputCount: 18240,
|
||||
filteredCount: 186,
|
||||
duplicateCount: 642,
|
||||
errorCount: 0,
|
||||
config: [
|
||||
{ label: '预处理规则', value: '清理无效数据、结构检测、内容去重、格式标准化' },
|
||||
{ label: '输出格式', value: 'Alpaca JSONL' },
|
||||
{ label: '数据集划分', value: '训练集 80% / 验证集 10% / 测试集 10%' },
|
||||
{ label: '处理引擎', value: 'DataFlow Engine v2.3' },
|
||||
],
|
||||
results: completedResults,
|
||||
},
|
||||
'492015': {
|
||||
id: '492015', name: '指令微调数据构造', status: 'running',
|
||||
description: '从通用语料中构造指令微调训练样本。', processType: '非结构化数据',
|
||||
sourceDataset: '通用语料库', outputDataset: 'SFT 指令集', outputDatasetId: 8, creator: '管理员',
|
||||
createTime: '2026-07-09 09:10:00', startTime: '2026-07-09 09:10:21', duration: '处理中', progress: 68,
|
||||
inputCount: 18500, outputCount: 8568, filteredCount: 425, duplicateCount: 192, errorCount: 8,
|
||||
config: [
|
||||
{ label: '切分方式', value: '语义切分' }, { label: '切片长度', value: '800 Tokens，重叠 80 Tokens' },
|
||||
{ label: '生成数量', value: '每个切片生成 2 组问答' }, { label: '处理引擎', value: 'DataFlow Engine v2.3' },
|
||||
], results: [],
|
||||
},
|
||||
'731948': {
|
||||
id: '731948', name: '敏感信息脱敏处理', status: 'pending', description: '识别并脱敏用户反馈数据中的敏感字段。',
|
||||
processType: '结构化数据', sourceDataset: '用户反馈数据', outputDataset: '用户反馈脱敏集',
|
||||
outputDatasetId: 9, creator: '管理员', createTime: '2026-07-09 16:45:00', duration: '等待执行', progress: 0,
|
||||
inputCount: 9340, outputCount: 0, filteredCount: 0, duplicateCount: 0, errorCount: 0,
|
||||
config: [
|
||||
{ label: '脱敏范围', value: '姓名、手机号、身份证号、地址' }, { label: '替换方式', value: '掩码替换' },
|
||||
{ label: '输出格式', value: 'JSONL' }, { label: '处理引擎', value: 'DataFlow Engine v2.3' },
|
||||
], results: [],
|
||||
},
|
||||
'582012': {
|
||||
id: '582012', name: '多轮对话拼接', status: 'failed', description: '将单轮问答按会话标识拼接为多轮对话数据。',
|
||||
processType: '结构化数据', sourceDataset: '单轮问答集', creator: '管理员',
|
||||
createTime: '2026-07-10 08:30:00', startTime: '2026-07-10 08:30:16', completeTime: '2026-07-10 08:31:04',
|
||||
duration: '48 秒', progress: 37, inputCount: 7520, outputCount: 2780, filteredCount: 24, duplicateCount: 0, errorCount: 1,
|
||||
config: [
|
||||
{ label: '会话字段', value: 'conversation_id' }, { label: '排序字段', value: 'message_time' },
|
||||
{ label: '最大轮次', value: '20 轮' }, { label: '处理引擎', value: 'DataFlow Engine v2.3' },
|
||||
], results: [], failureReason: '第 2 个源文件缺少 conversation_id 字段，无法继续执行会话拼接。',
|
||||
},
|
||||
}
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
import {
|
||||
getDataProcessProgress,
|
||||
getDataProcessResults,
|
||||
getDataProcessTask,
|
||||
publishDataProcess,
|
||||
restoreDataProcessResult,
|
||||
updateDataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type {
|
||||
DataProcessDatasetSplit,
|
||||
DataProcessPublishPayload,
|
||||
DataProcessResult,
|
||||
DataProcessResultStatus,
|
||||
DataProcessTask,
|
||||
DataProcessType,
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
const taskId = computed(() => String(route.params.id || ''))
|
||||
const detail = ref<DataProcessTask | null>(null)
|
||||
const loading = ref(true)
|
||||
const loadError = ref('')
|
||||
const results = ref<DataProcessResult[]>([])
|
||||
const resultTotal = ref(0)
|
||||
const resultLoading = ref(false)
|
||||
const resultError = ref('')
|
||||
const keyword = ref('')
|
||||
const statusFilter = ref('')
|
||||
const currentPage = ref(1)
|
||||
const pageSize = ref(10)
|
||||
const taskId = computed(() => String(route.params.id || ''))
|
||||
const detail = computed(() => detailMap[taskId.value])
|
||||
const editingResult = ref<DataProcessResult | null>(null)
|
||||
const editDialogVisible = ref(false)
|
||||
const savingResult = ref(false)
|
||||
const restoringResultId = ref<string | number | null>(null)
|
||||
const publishDialogVisible = ref(false)
|
||||
const publishing = ref(false)
|
||||
let resultFilterTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const editForm = reactive({ instruction: '', input: '', output: '' })
|
||||
const publishForm = reactive<DataProcessPublishPayload>({
|
||||
dataset_name: '',
|
||||
dataset_type: 'train',
|
||||
storage_type: 'local',
|
||||
split: { train: 80, validation: 10, test: 10 },
|
||||
format: 'alpaca_jsonl',
|
||||
})
|
||||
|
||||
const processTypeMap: Record<DataProcessType, string> = {
|
||||
structured: '结构化数据',
|
||||
unstructured: '非结构化数据',
|
||||
external: '外来数据源拉取',
|
||||
}
|
||||
|
||||
const configLabelMap: Record<string, string> = {
|
||||
preprocess_options: '预处理规则',
|
||||
dataset_split: '数据集划分',
|
||||
generation_model_id: '数据生成模型',
|
||||
generation_prompt: '生成提示语',
|
||||
temperature: '生成温度',
|
||||
max_tokens: '最大输出长度',
|
||||
json_mode: 'JSON 输出',
|
||||
quality_filter_enabled: '质量筛选',
|
||||
filter_low_quality: '过滤低质量内容',
|
||||
filter_short_content: '过滤过短内容',
|
||||
min_output_length: '最少输出字数',
|
||||
semantic_enrichment: '语义增强',
|
||||
qa_pairs_per_row: '每行生成数量',
|
||||
qa_pairs_per_chunk: '每切片生成数量',
|
||||
chunk_method: '切分方式',
|
||||
chunk_size: '切片长度',
|
||||
chunk_overlap: '重叠长度',
|
||||
min_chunk_size: '最小切片长度',
|
||||
custom_delimiter: '自定义分隔符',
|
||||
preserve_tables: '保留表格',
|
||||
preserve_code_blocks: '保留代码块',
|
||||
preserve_lists: '保留列表',
|
||||
}
|
||||
|
||||
function numeric(value: number | undefined) {
|
||||
return Number.isFinite(value) ? Number(value) : 0
|
||||
}
|
||||
|
||||
function formatDateTime(value?: string | null) {
|
||||
if (!value) return '-'
|
||||
const date = new Date(value)
|
||||
return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false })
|
||||
}
|
||||
|
||||
const retentionRate = computed(() => {
|
||||
if (!detail.value?.inputCount) return 0
|
||||
return Number(((detail.value.outputCount / detail.value.inputCount) * 100).toFixed(1))
|
||||
const inputCount = numeric(detail.value?.input_count)
|
||||
return inputCount
|
||||
? Number(((numeric(detail.value?.output_count) / inputCount) * 100).toFixed(1))
|
||||
: 0
|
||||
})
|
||||
|
||||
const filteredResults = computed(() => {
|
||||
const normalizedKeyword = keyword.value.trim().toLowerCase()
|
||||
return (detail.value?.results || []).filter((item) => {
|
||||
const matchesStatus = !statusFilter.value || item.status === statusFilter.value
|
||||
const matchesKeyword = !normalizedKeyword
|
||||
|| [item.instruction, item.input, item.output].some((text) => text.toLowerCase().includes(normalizedKeyword))
|
||||
return matchesStatus && matchesKeyword
|
||||
})
|
||||
const progressPercentage = computed(() => Math.min(100, Math.max(0, numeric(detail.value?.progress))))
|
||||
const sourceDatasetName = computed(() => (
|
||||
detail.value?.source_dataset_name
|
||||
|| detail.value?.source_dataset
|
||||
|| detail.value?.source_files?.map((file) => file.name).join('、')
|
||||
|| '源文件上传'
|
||||
))
|
||||
const outputDatasetName = computed(() => (
|
||||
detail.value?.output_dataset_name || detail.value?.output_dataset || ''
|
||||
))
|
||||
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
|
||||
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
|
||||
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
|
||||
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
|
||||
const completeTime = computed(() => detail.value?.complete_time || detail.value?.completed_at)
|
||||
|
||||
const durationText = computed(() => {
|
||||
if (detail.value?.duration) return detail.value.duration
|
||||
const seconds = detail.value?.duration_seconds
|
||||
if (!Number.isFinite(seconds)) return detail.value?.status === 'running' ? '处理中' : '-'
|
||||
const safeSeconds = Math.max(0, Math.round(Number(seconds)))
|
||||
const minutes = Math.floor(safeSeconds / 60)
|
||||
const restSeconds = safeSeconds % 60
|
||||
return minutes ? `${minutes} 分 ${restSeconds} 秒` : `${restSeconds} 秒`
|
||||
})
|
||||
|
||||
const paginatedResults = computed(() => {
|
||||
const start = (currentPage.value - 1) * pageSize.value
|
||||
return filteredResults.value.slice(start, start + pageSize.value)
|
||||
const configRows = computed(() => Object.entries(detail.value?.config || {})
|
||||
.filter(([key]) => !/(?:password|secret|token|api_key)/i.test(key))
|
||||
.map(([key, value]) => ({
|
||||
label: configLabelMap[key] || key.split('_').join(' '),
|
||||
value: formatConfigValue(key, value),
|
||||
})))
|
||||
|
||||
function formatConfigValue(key: string, value: unknown) {
|
||||
if (key === 'dataset_split' && value && typeof value === 'object') {
|
||||
const split = value as Partial<DataProcessDatasetSplit>
|
||||
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
|
||||
}
|
||||
if (Array.isArray(value)) return value.length ? value.join('、') : '-'
|
||||
if (typeof value === 'boolean') return value ? '是' : '否'
|
||||
if (value && typeof value === 'object') return JSON.stringify(value)
|
||||
return value == null || value === '' ? '-' : String(value)
|
||||
}
|
||||
|
||||
function processTypeLabel(value?: DataProcessType) {
|
||||
return value ? processTypeMap[value] || value : '-'
|
||||
}
|
||||
|
||||
function isActiveStatus(status?: DataProcessTask['status']) {
|
||||
return status === 'running'
|
||||
}
|
||||
|
||||
async function loadTask(silent = false) {
|
||||
if (!silent) loading.value = true
|
||||
loadError.value = ''
|
||||
try {
|
||||
detail.value = await getDataProcessTask(taskId.value)
|
||||
} catch {
|
||||
detail.value = null
|
||||
loadError.value = '数据处理任务加载失败，任务可能已删除或当前无权访问。'
|
||||
} finally {
|
||||
if (!silent) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadResults() {
|
||||
resultLoading.value = true
|
||||
resultError.value = ''
|
||||
try {
|
||||
const response = await getDataProcessResults(taskId.value, {
|
||||
page: currentPage.value,
|
||||
page_size: pageSize.value,
|
||||
keyword: keyword.value.trim() || undefined,
|
||||
status: statusFilter.value || undefined,
|
||||
})
|
||||
results.value = response.items
|
||||
resultTotal.value = response.total
|
||||
} catch {
|
||||
results.value = []
|
||||
resultTotal.value = 0
|
||||
resultError.value = '结果明细加载失败，请稍后重试。'
|
||||
} finally {
|
||||
resultLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadPage() {
|
||||
loading.value = true
|
||||
await loadTask(true)
|
||||
if (detail.value) {
|
||||
await loadResults()
|
||||
} else {
|
||||
results.value = []
|
||||
resultTotal.value = 0
|
||||
resultError.value = ''
|
||||
}
|
||||
loading.value = false
|
||||
if (isActiveStatus(detail.value?.status)) startPolling()
|
||||
else stopPolling()
|
||||
}
|
||||
|
||||
async function refreshRuntime() {
|
||||
if (!detail.value || !isActiveStatus(detail.value.status)) {
|
||||
stopPolling()
|
||||
return
|
||||
}
|
||||
|
||||
const progress = await getDataProcessProgress(taskId.value)
|
||||
detail.value = {
|
||||
...detail.value,
|
||||
status: progress.status,
|
||||
progress: progress.progress,
|
||||
input_count: progress.input_count ?? detail.value.input_count,
|
||||
output_count: progress.output_count ?? detail.value.output_count,
|
||||
filtered_count: progress.filtered_count ?? detail.value.filtered_count,
|
||||
duplicate_count: progress.duplicate_count ?? detail.value.duplicate_count,
|
||||
error_count: progress.error_count ?? detail.value.error_count,
|
||||
failure_reason: progress.failure_reason ?? detail.value.failure_reason,
|
||||
}
|
||||
|
||||
if (!isActiveStatus(progress.status)) {
|
||||
stopPolling()
|
||||
await Promise.all([loadTask(true), loadResults()])
|
||||
}
|
||||
}
|
||||
|
||||
const { start: startPolling, stop: stopPolling } = usePolling(refreshRuntime, 3000, {
|
||||
immediate: false,
|
||||
})
|
||||
|
||||
function resultStatusLabel(status: ResultRow['status']) {
|
||||
function scheduleResultReload() {
|
||||
if (currentPage.value !== 1) {
|
||||
currentPage.value = 1
|
||||
return
|
||||
}
|
||||
if (resultFilterTimer) clearTimeout(resultFilterTimer)
|
||||
resultFilterTimer = setTimeout(() => void loadResults(), 300)
|
||||
}
|
||||
|
||||
function resultStatusLabel(status: DataProcessResultStatus) {
|
||||
return status === 'valid' ? '有效' : status === 'modified' ? '已修改' : '无效'
|
||||
}
|
||||
|
||||
function resultStatusType(status: ResultRow['status']) {
|
||||
function resultStatusType(status: DataProcessResultStatus) {
|
||||
return status === 'valid' ? 'success' : status === 'modified' ? 'warning' : 'danger'
|
||||
}
|
||||
|
||||
function resetPage() {
|
||||
currentPage.value = 1
|
||||
function qualityScoreLabel(value: DataProcessResult['quality_score']) {
|
||||
if (value == null) return '-'
|
||||
const score = value.overall
|
||||
return Number.isFinite(score) ? Number(score).toFixed(1) : '-'
|
||||
}
|
||||
|
||||
function qualityFlagsLabel(value: DataProcessResult['quality_score']) {
|
||||
return value?.flags?.length ? value.flags.join('、') : '未命中质量规则'
|
||||
}
|
||||
|
||||
function replaceResult(updated: DataProcessResult) {
|
||||
const index = results.value.findIndex((item) => item.id === updated.id)
|
||||
if (index >= 0) results.value.splice(index, 1, updated)
|
||||
}
|
||||
|
||||
function openResultEditor(result: DataProcessResult) {
|
||||
editingResult.value = result
|
||||
Object.assign(editForm, {
|
||||
instruction: result.instruction,
|
||||
input: result.input,
|
||||
output: result.output,
|
||||
})
|
||||
editDialogVisible.value = true
|
||||
}
|
||||
|
||||
async function saveResult() {
|
||||
if (!editingResult.value) return
|
||||
if (!editForm.instruction.trim() || !editForm.output.trim()) {
|
||||
ElMessage.warning('Instruction 和 Output 不能为空')
|
||||
return
|
||||
}
|
||||
|
||||
savingResult.value = true
|
||||
try {
|
||||
const updated = await updateDataProcessResult(taskId.value, editingResult.value.id, {
|
||||
instruction: editForm.instruction,
|
||||
input: editForm.input,
|
||||
output: editForm.output,
|
||||
expected_updated_at: editingResult.value.updated_at,
|
||||
})
|
||||
replaceResult(updated)
|
||||
await loadTask(true)
|
||||
editDialogVisible.value = false
|
||||
ElMessage.success('结果已保存')
|
||||
} catch {
|
||||
// 统一请求层已展示后端返回的失败原因。
|
||||
} finally {
|
||||
savingResult.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreResult(result: DataProcessResult) {
|
||||
try {
|
||||
await ElMessageBox.confirm('确定恢复为模型最初生成的内容吗？', '恢复生成结果', {
|
||||
type: 'warning',
|
||||
confirmButtonText: '恢复',
|
||||
cancelButtonText: '取消',
|
||||
})
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
restoringResultId.value = result.id
|
||||
try {
|
||||
const restored = await restoreDataProcessResult(taskId.value, result.id)
|
||||
replaceResult(restored)
|
||||
await loadTask(true)
|
||||
ElMessage.success('已恢复生成结果')
|
||||
} catch {
|
||||
// 统一请求层已展示后端返回的失败原因。
|
||||
} finally {
|
||||
restoringResultId.value = null
|
||||
}
|
||||
}
|
||||
|
||||
function configuredSplit(): DataProcessDatasetSplit {
|
||||
const split = detail.value?.config?.dataset_split
|
||||
if (!split || typeof split !== 'object') return { train: 80, validation: 10, test: 10 }
|
||||
const value = split as Partial<DataProcessDatasetSplit>
|
||||
return {
|
||||
train: Number(value.train) || 0,
|
||||
validation: Number(value.validation) || 0,
|
||||
test: Number(value.test) || 0,
|
||||
}
|
||||
}
|
||||
|
||||
function openPublishDialog() {
|
||||
if (!detail.value) return
|
||||
if (outputDatasetId.value) {
|
||||
void router.push(`/dataset/${outputDatasetId.value}/preview`)
|
||||
return
|
||||
}
|
||||
publishForm.dataset_name = `${detail.value.name}-数据集`
|
||||
publishForm.split = configuredSplit()
|
||||
publishDialogVisible.value = true
|
||||
}
|
||||
|
||||
async function publishDataset() {
|
||||
if (!publishForm.dataset_name.trim()) {
|
||||
ElMessage.warning('请输入数据集名称')
|
||||
return
|
||||
}
|
||||
|
||||
publishing.value = true
|
||||
try {
|
||||
const published = await publishDataProcess(taskId.value, {
|
||||
...publishForm,
|
||||
dataset_name: publishForm.dataset_name.trim(),
|
||||
split: { ...publishForm.split },
|
||||
})
|
||||
const datasetId = published.dataset_id || published.output_dataset_id || published.dataset?.id
|
||||
if (!datasetId) {
|
||||
ElMessage.success('数据集发布成功')
|
||||
publishDialogVisible.value = false
|
||||
await loadTask(true)
|
||||
return
|
||||
}
|
||||
ElMessage.success('数据集发布成功')
|
||||
publishDialogVisible.value = false
|
||||
await router.push(`/dataset/${datasetId}/preview`)
|
||||
} catch {
|
||||
// 统一请求层已展示后端返回的失败原因。
|
||||
} finally {
|
||||
publishing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch([currentPage, pageSize], () => void loadResults())
|
||||
|
||||
onMounted(loadPage)
|
||||
onBeforeUnmount(() => {
|
||||
if (resultFilterTimer) clearTimeout(resultFilterTimer)
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<PageCard class="data-process-detail-page">
|
||||
<PageCard class="data-process-detail-page" v-loading="loading">
|
||||
<template v-if="detail" #header>
|
||||
<div class="detail-heading">
|
||||
<div class="heading-row">
|
||||
<h1>{{ detail.name }}</h1>
|
||||
<ModelStatusTag :status="detail.status" />
|
||||
<el-button
|
||||
v-if="detail.status === 'completed'"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa" :class="outputDatasetId ? 'fa-external-link' : 'fa-database'" />
|
||||
{{ outputDatasetId ? '查看输出数据集' : '发布为数据集' }}
|
||||
</el-button>
|
||||
</div>
|
||||
<p>{{ detail.description }}</p>
|
||||
<p>{{ detail.description || '暂无任务描述' }}</p>
|
||||
<dl class="heading-meta">
|
||||
<div><dt>任务 ID</dt><dd>{{ detail.id }}</dd></div>
|
||||
<div><dt>处理类型</dt><dd>{{ detail.processType }}</dd></div>
|
||||
<div><dt>创建人</dt><dd>{{ detail.creator }}</dd></div>
|
||||
<div><dt>处理类型</dt><dd>{{ processTypeLabel(detail.process_type) }}</dd></div>
|
||||
<div><dt>创建人</dt><dd>{{ creatorName }}</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<div v-if="!detail" class="not-found-state">
|
||||
<div v-if="loadError" class="not-found-state" role="alert">
|
||||
<i class="fa fa-exclamation-circle" aria-hidden="true" />
|
||||
<h2>未找到数据处理任务</h2>
|
||||
<p>任务可能已被删除，或当前链接已失效。</p>
|
||||
<el-button type="primary" @click="router.push('/data-process')">返回任务列表</el-button>
|
||||
<h2>无法加载数据处理任务</h2>
|
||||
<p>{{ loadError }}</p>
|
||||
<div class="load-state-actions">
|
||||
<el-button @click="router.push('/data-process')">返回任务列表</el-button>
|
||||
<el-button type="primary" @click="loadPage">重新加载</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template v-else>
|
||||
<template v-else-if="detail">
|
||||
<section class="metric-grid" aria-label="处理结果概览">
|
||||
<div class="metric-card">
|
||||
<span>处理耗时</span>
|
||||
<strong>{{ detail.duration || '尚未开始' }}</strong>
|
||||
<small>{{ detail.completeTime ? `完成于 ${detail.completeTime}` : `当前进度 ${detail.progress}%` }}</small>
|
||||
<strong>{{ durationText }}</strong>
|
||||
<small>{{ completeTime ? `完成于 ${formatDateTime(completeTime)}` : `当前进度 ${progressPercentage}%` }}</small>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<span>输入数据</span>
|
||||
<strong>{{ detail.inputCount.toLocaleString() }}</strong>
|
||||
<small>来源：{{ detail.sourceDataset }}</small>
|
||||
<strong>{{ numeric(detail.input_count).toLocaleString() }}</strong>
|
||||
<small>来源：{{ sourceDatasetName }}</small>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<span>输出结果</span>
|
||||
<strong>{{ detail.outputCount.toLocaleString() }}</strong>
|
||||
<small>{{ detail.outputDataset || '尚未生成输出数据集' }}</small>
|
||||
<strong>{{ numeric(detail.output_count).toLocaleString() }}</strong>
|
||||
<small>{{ outputDatasetName || '尚未生成输出数据集' }}</small>
|
||||
</div>
|
||||
<div class="metric-card is-primary">
|
||||
<span>数据保留率</span>
|
||||
<strong>{{ detail.inputCount ? `${retentionRate}%` : '-' }}</strong>
|
||||
<el-progress :percentage="detail.progress" :show-text="false" :stroke-width="5" />
|
||||
<strong>{{ numeric(detail.input_count) ? `${retentionRate}%` : '-' }}</strong>
|
||||
<el-progress :percentage="progressPercentage" :show-text="false" :stroke-width="5" />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div v-if="detail.failureReason" class="failure-alert" role="alert">
|
||||
<div v-if="detail.failure_reason" class="failure-alert" role="alert">
|
||||
<i class="fa fa-exclamation-triangle" aria-hidden="true" />
|
||||
<div><strong>处理任务执行失败</strong><p>{{ detail.failureReason }}</p></div>
|
||||
<div><strong>处理任务执行失败</strong><p>{{ detail.failure_reason }}</p></div>
|
||||
</div>
|
||||
|
||||
<div class="detail-grid">
|
||||
@@ -231,21 +454,21 @@ function resetPage() {
|
||||
<div><h2 id="runtime-title">运行信息</h2><p>查看任务执行时间与数据流向</p></div>
|
||||
</div>
|
||||
<dl class="info-list">
|
||||
<div><dt>创建时间</dt><dd>{{ detail.createTime }}</dd></div>
|
||||
<div><dt>开始时间</dt><dd>{{ detail.startTime || '尚未开始' }}</dd></div>
|
||||
<div><dt>完成时间</dt><dd>{{ detail.completeTime || '尚未完成' }}</dd></div>
|
||||
<div><dt>处理耗时</dt><dd>{{ detail.duration || '-' }}</dd></div>
|
||||
<div><dt>源数据集</dt><dd>{{ detail.sourceDataset }}</dd></div>
|
||||
<div><dt>创建时间</dt><dd>{{ formatDateTime(createTime) }}</dd></div>
|
||||
<div><dt>开始时间</dt><dd>{{ formatDateTime(startTime) }}</dd></div>
|
||||
<div><dt>完成时间</dt><dd>{{ formatDateTime(completeTime) }}</dd></div>
|
||||
<div><dt>处理耗时</dt><dd>{{ durationText }}</dd></div>
|
||||
<div><dt>源数据集</dt><dd>{{ sourceDatasetName }}</dd></div>
|
||||
<div>
|
||||
<dt>输出数据集</dt>
|
||||
<dd>
|
||||
<el-button
|
||||
v-if="detail.outputDataset && detail.status === 'completed'"
|
||||
v-if="outputDatasetId && outputDatasetName"
|
||||
type="primary"
|
||||
link
|
||||
@click="router.push(`/dataset/${detail.outputDatasetId}/preview`)"
|
||||
>{{ detail.outputDataset }} <i class="fa fa-external-link" /></el-button>
|
||||
<span v-else>{{ detail.outputDataset || '尚未生成' }}</span>
|
||||
@click="router.push(`/dataset/${outputDatasetId}/preview`)"
|
||||
>{{ outputDatasetName }} <i class="fa fa-external-link" /></el-button>
|
||||
<span v-else>{{ outputDatasetName || '尚未生成' }}</span>
|
||||
</dd>
|
||||
</div>
|
||||
</dl>
|
||||
@@ -256,12 +479,12 @@ function resetPage() {
|
||||
<div><h2 id="statistics-title">处理统计</h2><p>查看数据清洗、过滤和输出情况</p></div>
|
||||
</div>
|
||||
<div class="statistics-grid">
|
||||
<div><span>原始数据</span><strong>{{ detail.inputCount.toLocaleString() }}</strong></div>
|
||||
<div><span>成功输出</span><strong>{{ detail.outputCount.toLocaleString() }}</strong></div>
|
||||
<div><span>过滤数据</span><strong>{{ detail.filteredCount.toLocaleString() }}</strong></div>
|
||||
<div><span>重复数据</span><strong>{{ detail.duplicateCount.toLocaleString() }}</strong></div>
|
||||
<div><span>异常数据</span><strong>{{ detail.errorCount.toLocaleString() }}</strong></div>
|
||||
<div><span>执行进度</span><strong>{{ detail.progress }}%</strong></div>
|
||||
<div><span>原始数据</span><strong>{{ numeric(detail.input_count).toLocaleString() }}</strong></div>
|
||||
<div><span>成功输出</span><strong>{{ numeric(detail.output_count).toLocaleString() }}</strong></div>
|
||||
<div><span>过滤数据</span><strong>{{ numeric(detail.filtered_count).toLocaleString() }}</strong></div>
|
||||
<div><span>重复数据</span><strong>{{ numeric(detail.duplicate_count).toLocaleString() }}</strong></div>
|
||||
<div><span>异常数据</span><strong>{{ numeric(detail.error_count).toLocaleString() }}</strong></div>
|
||||
<div><span>执行进度</span><strong>{{ progressPercentage }}%</strong></div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
@@ -270,58 +493,148 @@ function resetPage() {
|
||||
<div class="section-heading">
|
||||
<div><h2 id="config-title">处理配置</h2><p>任务执行时使用的规则与参数</p></div>
|
||||
</div>
|
||||
<dl class="config-grid">
|
||||
<div v-for="item in detail.config" :key="item.label"><dt>{{ item.label }}</dt><dd>{{ item.value }}</dd></div>
|
||||
<dl v-if="configRows.length" class="config-grid">
|
||||
<div v-for="item in configRows" :key="item.label"><dt>{{ item.label }}</dt><dd>{{ item.value }}</dd></div>
|
||||
</dl>
|
||||
<div v-else class="compact-empty">暂无处理配置</div>
|
||||
</section>
|
||||
|
||||
<section class="detail-section result-section" aria-labelledby="result-title">
|
||||
<section class="detail-section result-section" aria-labelledby="result-title" v-loading="resultLoading">
|
||||
<div class="result-toolbar">
|
||||
<div class="section-heading">
|
||||
<div><h2 id="result-title">结果明细</h2><p>查看处理完成后的数据内容与校验状态</p></div>
|
||||
<div><h2 id="result-title">结果明细</h2><p>查看、编辑或恢复处理结果</p></div>
|
||||
</div>
|
||||
<div v-if="detail.results.length" class="result-filters">
|
||||
<el-input v-model="keyword" clearable placeholder="搜索指令、输入或输出" @input="resetPage">
|
||||
<div class="result-filters">
|
||||
<el-input
|
||||
v-model="keyword"
|
||||
clearable
|
||||
placeholder="搜索指令、输入或输出"
|
||||
@input="scheduleResultReload"
|
||||
@clear="scheduleResultReload"
|
||||
>
|
||||
<template #prefix><i class="fa fa-search" aria-hidden="true" /></template>
|
||||
</el-input>
|
||||
<el-select v-model="statusFilter" clearable placeholder="全部状态" @change="resetPage">
|
||||
<el-select v-model="statusFilter" clearable placeholder="全部状态" @change="scheduleResultReload">
|
||||
<el-option label="有效" value="valid" />
|
||||
<el-option label="已修改" value="modified" />
|
||||
<el-option label="无效" value="invalid" />
|
||||
</el-select>
|
||||
<el-button :loading="resultLoading" @click="loadResults"><i class="fa fa-refresh" /></el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<el-table v-if="filteredResults.length" :data="paginatedResults" row-key="id" table-layout="fixed">
|
||||
<el-table v-if="results.length" :data="results" row-key="id" table-layout="fixed">
|
||||
<el-table-column type="index" label="#" width="56" align="center" />
|
||||
<el-table-column label="指令" min-width="210" show-overflow-tooltip prop="instruction" />
|
||||
<el-table-column label="输入" min-width="190" show-overflow-tooltip prop="input" />
|
||||
<el-table-column label="输出" min-width="260" show-overflow-tooltip prop="output" />
|
||||
<el-table-column label="指令" min-width="190" show-overflow-tooltip prop="instruction" />
|
||||
<el-table-column label="输入" min-width="160" show-overflow-tooltip prop="input" />
|
||||
<el-table-column label="输出" min-width="230" show-overflow-tooltip prop="output" />
|
||||
<el-table-column label="质量分" width="88" align="center">
|
||||
<template #default="{ row }">
|
||||
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
|
||||
<span>{{ qualityScoreLabel((row as DataProcessResult).quality_score) }}</span>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="状态" width="90" align="center">
|
||||
<template #default="{ row }">
|
||||
<el-tag :type="resultStatusType(row.status)" size="small">{{ resultStatusLabel(row.status) }}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="操作" width="130" align="center" fixed="right">
|
||||
<template #default="{ row }">
|
||||
<el-button
|
||||
link
|
||||
type="primary"
|
||||
:disabled="Boolean(outputDatasetId)"
|
||||
@click="openResultEditor(row as DataProcessResult)"
|
||||
>编辑</el-button>
|
||||
<el-button
|
||||
link
|
||||
:disabled="Boolean(outputDatasetId)"
|
||||
:loading="restoringResultId === row.id"
|
||||
@click="restoreResult(row as DataProcessResult)"
|
||||
>恢复</el-button>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
|
||||
<div v-else class="result-empty">
|
||||
<i class="fa" :class="detail.status === 'failed' ? 'fa-exclamation-circle' : 'fa-hourglass-half'" aria-hidden="true" />
|
||||
<strong>{{ detail.status === 'completed' ? '没有符合条件的结果' : detail.status === 'failed' ? '任务失败，未生成结果明细' : '处理完成后将在这里展示结果明细' }}</strong>
|
||||
<span>{{ detail.status === 'completed' ? '请调整搜索或筛选条件' : `当前任务状态：${detail.status === 'running' ? '运行中' : '等待中'}` }}</span>
|
||||
<i class="fa" :class="resultError || detail.status === 'failed' ? 'fa-exclamation-circle' : 'fa-hourglass-half'" aria-hidden="true" />
|
||||
<strong>{{ resultError || (detail.status === 'completed' ? '没有符合条件的结果' : detail.status === 'failed' ? '任务失败，未生成结果明细' : '处理完成后将在这里展示结果明细') }}</strong>
|
||||
<span v-if="!resultError">{{ detail.status === 'completed' ? '请调整搜索或筛选条件' : `当前任务状态：${detail.status === 'running' ? '运行中' : '等待中'}` }}</span>
|
||||
<el-button v-else type="primary" link @click="loadResults">重新加载</el-button>
|
||||
</div>
|
||||
|
||||
<div v-if="filteredResults.length" class="result-pagination">
|
||||
<span>共 {{ filteredResults.length }} 条结果</span>
|
||||
<div v-if="resultTotal" class="result-pagination">
|
||||
<span>共 {{ resultTotal }} 条结果</span>
|
||||
<el-pagination
|
||||
v-model:current-page="currentPage"
|
||||
v-model:page-size="pageSize"
|
||||
layout="prev, pager, next"
|
||||
:total="filteredResults.length"
|
||||
layout="sizes, prev, pager, next"
|
||||
:page-sizes="[10, 20, 50]"
|
||||
:total="resultTotal"
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
</PageCard>
|
||||
|
||||
<el-dialog v-model="editDialogVisible" title="编辑处理结果" width="680px" destroy-on-close>
|
||||
<el-form label-position="top">
|
||||
<el-form-item label="Instruction" required>
|
||||
<el-input v-model="editForm.instruction" type="textarea" :rows="3" maxlength="4000" show-word-limit />
|
||||
</el-form-item>
|
||||
<el-form-item label="Input">
|
||||
<el-input v-model="editForm.input" type="textarea" :rows="3" maxlength="10000" show-word-limit />
|
||||
</el-form-item>
|
||||
<el-form-item label="Output" required>
|
||||
<el-input v-model="editForm.output" type="textarea" :rows="6" maxlength="20000" show-word-limit />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="editDialogVisible = false">取消</el-button>
|
||||
<el-button type="primary" :loading="savingResult" @click="saveResult">保存</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
|
||||
<el-dialog v-model="publishDialogVisible" title="发布为数据集" width="560px" destroy-on-close>
|
||||
<el-form label-position="top">
|
||||
<el-form-item label="数据集名称" required>
|
||||
<el-input v-model="publishForm.dataset_name" maxlength="150" show-word-limit />
|
||||
</el-form-item>
|
||||
<div class="publish-form-grid">
|
||||
<el-form-item label="数据集类型">
|
||||
<el-select v-model="publishForm.dataset_type">
|
||||
<el-option label="训练数据" value="train" />
|
||||
<el-option label="测试数据" value="test" />
|
||||
<el-option label="评测数据" value="eval" />
|
||||
<el-option label="验证数据" value="val" />
|
||||
<el-option label="其他" value="other" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="存储位置">
|
||||
<el-select v-model="publishForm.storage_type">
|
||||
<el-option label="平台本地存储" value="local" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="输出格式">
|
||||
<el-select v-model="publishForm.format">
|
||||
<el-option label="Alpaca JSONL" value="alpaca_jsonl" />
|
||||
<el-option label="JSONL" value="jsonl" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</div>
|
||||
<el-alert
|
||||
type="info"
|
||||
:closable="false"
|
||||
:title="`数据集划分：训练集 ${publishForm.split.train}% / 验证集 ${publishForm.split.validation}% / 测试集 ${publishForm.split.test}%`"
|
||||
/>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="publishDialogVisible = false">取消</el-button>
|
||||
<el-button type="primary" :loading="publishing" @click="publishDataset">发布并查看</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
@@ -335,6 +648,12 @@ function resetPage() {
|
||||
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
|
||||
}
|
||||
|
||||
.publish-button { margin-left: auto; }
|
||||
.load-state-actions { display: flex; gap: 10px; }
|
||||
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
|
||||
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
|
||||
.publish-form-grid :deep(.el-select) { width: 100%; }
|
||||
|
||||
.heading-meta {
|
||||
margin: 16px 0 0; display: flex; flex-wrap: wrap; gap: 10px 30px;
|
||||
div { display: flex; gap: 7px; font-size: 12px; }
|
||||
@@ -417,6 +736,9 @@ function resetPage() {
|
||||
|
||||
@media (max-width: 720px) {
|
||||
.metric-grid, .config-grid { grid-template-columns: 1fr; }
|
||||
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
|
||||
.publish-button { width: 100%; margin-left: 0; }
|
||||
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
|
||||
.result-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.result-filters { padding: 0 16px 16px; flex-direction: column; }
|
||||
.result-filters :deep(.el-input), .result-filters :deep(.el-select) { width: 100%; }
|
||||
|
||||
@@ -1,71 +1,35 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import DataTablePage from '@/components/DataTablePage.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { deleteDataProcessTask, getDataProcessTasks } from '@/api/modules/dataProcess'
|
||||
import type { DataProcessTask, DataProcessType } from '@/types/dataProcess'
|
||||
|
||||
import type { ProcessType } from './create/types'
|
||||
|
||||
/** 数据处理任务类型 */
|
||||
interface DataProcessTask {
|
||||
id: number | string
|
||||
name: string
|
||||
status: string
|
||||
process_type: ProcessType
|
||||
source_dataset: string
|
||||
output_dataset?: string
|
||||
create_time?: string
|
||||
}
|
||||
|
||||
const processTypeMap: Record<ProcessType, string> = {
|
||||
const processTypeMap: Record<DataProcessType, string> = {
|
||||
structured: '结构化数据',
|
||||
unstructured: '非结构化数据',
|
||||
external: '外来数据源拉取',
|
||||
}
|
||||
|
||||
// TODO: 接入真实接口前，先用本地 mock 数据
|
||||
const router = useRouter()
|
||||
const dataList = ref<DataProcessTask[]>([
|
||||
{
|
||||
id: 183921,
|
||||
name: '客服问答数据清洗',
|
||||
status: 'completed',
|
||||
process_type: 'structured',
|
||||
source_dataset: '客服对话原始集',
|
||||
output_dataset: '客服对话清洗集',
|
||||
create_time: '2026-07-08 14:23:00',
|
||||
},
|
||||
{
|
||||
id: 492015,
|
||||
name: '指令微调数据构造',
|
||||
status: 'running',
|
||||
process_type: 'unstructured',
|
||||
source_dataset: '通用语料库',
|
||||
output_dataset: 'SFT 指令集',
|
||||
create_time: '2026-07-09 09:10:00',
|
||||
},
|
||||
{
|
||||
id: 731948,
|
||||
name: '敏感信息脱敏处理',
|
||||
status: 'pending',
|
||||
process_type: 'structured',
|
||||
source_dataset: '用户反馈数据',
|
||||
create_time: '2026-07-09 16:45:00',
|
||||
},
|
||||
{
|
||||
id: 582012,
|
||||
name: '多轮对话拼接',
|
||||
status: 'failed',
|
||||
process_type: 'structured',
|
||||
source_dataset: '单轮问答集',
|
||||
create_time: '2026-07-10 08:30:00',
|
||||
},
|
||||
])
|
||||
const dataList = ref<DataProcessTask[]>([])
|
||||
const loading = ref(false)
|
||||
const deletingId = ref<string | number | null>(null)
|
||||
const loadError = ref('')
|
||||
|
||||
/** 新建数据处理任务 */
|
||||
function handleCreate() {
|
||||
router.push('/data-process/create')
|
||||
async function loadData(silent = false) {
|
||||
if (!silent) loading.value = true
|
||||
loadError.value = ''
|
||||
try {
|
||||
const response = await getDataProcessTasks({ page: 1, page_size: 200 })
|
||||
dataList.value = response.items
|
||||
} catch {
|
||||
loadError.value = '数据处理任务加载失败，请稍后重试。'
|
||||
} finally {
|
||||
if (!silent) loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
/** 查看任务详情 */
|
||||
@@ -74,15 +38,50 @@ function viewDetail(row: unknown) {
|
||||
router.push({ name: 'data-process-detail', params: { id: taskId } })
|
||||
}
|
||||
|
||||
/** 删除任务（功能开发中） */
|
||||
function handleDelete(_row: unknown) {
|
||||
ElMessage.info('删除功能开发中...')
|
||||
/** 删除由后端再次校验任务状态以及发布锁。 */
|
||||
async function handleDelete(row: DataProcessTask) {
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
`确定删除数据处理任务“${row.name}”吗？删除后无法恢复。`,
|
||||
'确认删除',
|
||||
{
|
||||
type: 'warning',
|
||||
confirmButtonText: '删除',
|
||||
cancelButtonText: '取消',
|
||||
confirmButtonClass: 'el-button--danger',
|
||||
},
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
deletingId.value = row.id
|
||||
try {
|
||||
await deleteDataProcessTask(row.id)
|
||||
dataList.value = dataList.value.filter((item) => item.id !== row.id)
|
||||
ElMessage.success('数据处理任务已删除')
|
||||
} catch {
|
||||
// 统一请求层已展示后端返回的失败原因。
|
||||
} finally {
|
||||
deletingId.value = null
|
||||
}
|
||||
}
|
||||
|
||||
function formatDateTime(value?: string) {
|
||||
if (!value) return '-'
|
||||
return new Date(value).toLocaleString('zh-CN', { hour12: false })
|
||||
const date = new Date(value)
|
||||
return Number.isNaN(date.getTime()) ? value : date.toLocaleString('zh-CN', { hour12: false })
|
||||
}
|
||||
|
||||
function sourceDatasetName(task: DataProcessTask) {
|
||||
return task.source_dataset_name || task.source_dataset || '-'
|
||||
}
|
||||
|
||||
function outputDatasetName(task: DataProcessTask) {
|
||||
return task.output_dataset_name || task.output_dataset || '-'
|
||||
}
|
||||
|
||||
onMounted(loadData)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -90,12 +89,14 @@ function formatDateTime(value?: string) {
|
||||
<DataTablePage
|
||||
title=""
|
||||
:data="dataList"
|
||||
:loading="loading"
|
||||
searchable
|
||||
:search-fields="['name']"
|
||||
create-text="新建数据处理"
|
||||
create-to="/data-process/create"
|
||||
row-key="id"
|
||||
:page-size="10"
|
||||
:empty-text="loadError || '暂无数据处理任务'"
|
||||
>
|
||||
<template #columns>
|
||||
<el-table-column label="任务ID" prop="id" align="center" width="100" />
|
||||
@@ -108,19 +109,19 @@ function formatDateTime(value?: string) {
|
||||
<el-table-column label="处理类型" align="center" width="140">
|
||||
<template #default="{ row }">
|
||||
<el-tag v-if="row.process_type" size="small" type="info" effect="plain">
|
||||
{{ processTypeMap[row.process_type as ProcessType] || row.process_type }}
|
||||
{{ processTypeMap[row.process_type as DataProcessType] || row.process_type }}
|
||||
</el-tag>
|
||||
<span v-else>-</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="源数据集" align="center" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.source_dataset || '-' }}</template>
|
||||
<template #default="{ row }">{{ sourceDatasetName(row as DataProcessTask) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="输出数据集" align="center" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.output_dataset || '-' }}</template>
|
||||
<template #default="{ row }">{{ outputDatasetName(row as DataProcessTask) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="创建时间" align="center" width="190">
|
||||
<template #default="{ row }">{{ formatDateTime(row.create_time) }}</template>
|
||||
<template #default="{ row }">{{ formatDateTime(row.create_time || row.created_at) }}</template>
|
||||
</el-table-column>
|
||||
</template>
|
||||
|
||||
@@ -129,7 +130,13 @@ function formatDateTime(value?: string) {
|
||||
<el-button type="primary" link size="small" @click="viewDetail(row)">
|
||||
<i class="fa fa-file-text-o" style="margin-right: 4px" />详情
|
||||
</el-button>
|
||||
<el-button type="danger" link size="small" @click="handleDelete(row)">
|
||||
<el-button
|
||||
type="danger"
|
||||
link
|
||||
size="small"
|
||||
:loading="deletingId === row.id"
|
||||
@click="handleDelete(row as DataProcessTask)"
|
||||
>
|
||||
<i class="fa fa-trash-o" style="margin-right: 4px" />删除
|
||||
</el-button>
|
||||
</div>
|
||||
|
||||
@@ -17,6 +17,8 @@ const emit = defineEmits<{
|
||||
'update:selectedId': [value: string]
|
||||
'update:selectedFileId': [value: string]
|
||||
'update:item-content': [id: string, value: string]
|
||||
'restore:item': [id: string]
|
||||
'add:item': []
|
||||
'remove:item': [id: string]
|
||||
}>()
|
||||
|
||||
@@ -71,6 +73,12 @@ function saveEditor() {
|
||||
closeEditor()
|
||||
}
|
||||
|
||||
function restoreItem() {
|
||||
if (!editingItem.value) return
|
||||
emit('restore:item', editingItem.value.id)
|
||||
closeEditor()
|
||||
}
|
||||
|
||||
function removeItem(item: PreviewItem) {
|
||||
selectItem(item.id)
|
||||
emit('remove:item', item.id)
|
||||
@@ -169,7 +177,12 @@ function lineRange(item: PreviewItem) {
|
||||
<div class="preview-pane">
|
||||
<div class="pane-header">
|
||||
<strong>{{ processType === 'unstructured' ? '切片内容' : '记录内容' }}</strong>
|
||||
<span>共 {{ items.length.toLocaleString() }} 条</span>
|
||||
<div class="pane-header-actions">
|
||||
<span>共 {{ items.length.toLocaleString() }} 条</span>
|
||||
<el-button link type="primary" @click="emit('add:item')">
|
||||
<i class="fa fa-plus" /> 手动新增
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template v-if="!editingItem">
|
||||
@@ -234,6 +247,13 @@ function lineRange(item: PreviewItem) {
|
||||
resize="none"
|
||||
/>
|
||||
<div class="editor-actions">
|
||||
<el-button
|
||||
v-if="editingItem.sourceStart != null"
|
||||
link
|
||||
@click="restoreItem"
|
||||
>
|
||||
<i class="fa fa-undo" /> 恢复原始内容
|
||||
</el-button>
|
||||
<div>
|
||||
<el-button @click="closeEditor">取消</el-button>
|
||||
<el-button type="primary" @click="saveEditor">保存修改</el-button>
|
||||
@@ -288,6 +308,17 @@ function lineRange(item: PreviewItem) {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.pane-header-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
|
||||
> span {
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
}
|
||||
}
|
||||
|
||||
.file-option {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto auto;
|
||||
|
||||
@@ -69,6 +69,12 @@ function selectRelative(offset: number) {
|
||||
<div>
|
||||
<strong>结果 #{{ String(selectedIndex + 1).padStart(3, '0') }}</strong>
|
||||
<span v-if="selectedItem.status === 'modified'" class="modified-label">已修改</span>
|
||||
<el-tag v-if="selectedItem.split" size="small" effect="plain">{{ selectedItem.split }}</el-tag>
|
||||
<el-tag
|
||||
v-if="selectedItem.qualityScore != null"
|
||||
size="small"
|
||||
:type="selectedItem.qualityScore >= 80 ? 'success' : selectedItem.qualityScore >= 60 ? 'warning' : 'danger'"
|
||||
>质量 {{ selectedItem.qualityScore.toFixed(1) }}</el-tag>
|
||||
</div>
|
||||
<el-button link @click="emit('restore:item', selectedItem.id)"><i class="fa fa-undo" /> 恢复生成结果</el-button>
|
||||
</div>
|
||||
@@ -106,6 +112,15 @@ function selectRelative(offset: number) {
|
||||
<div v-else class="validation-success">
|
||||
<i class="fa fa-check-circle" /> 字段校验通过
|
||||
</div>
|
||||
<div v-if="selectedItem.qualityFlags?.length" class="quality-flags">
|
||||
<el-tag
|
||||
v-for="flag in selectedItem.qualityFlags"
|
||||
:key="flag"
|
||||
size="small"
|
||||
type="warning"
|
||||
effect="plain"
|
||||
>{{ flag }}</el-tag>
|
||||
</div>
|
||||
<div class="editor-pagination">
|
||||
<el-button :disabled="selectedIndex <= 0" @click="selectRelative(-1)">上一条</el-button>
|
||||
<span>{{ selectedIndex + 1 }} / {{ items.length }}</span>
|
||||
@@ -275,6 +290,12 @@ function selectRelative(offset: number) {
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.quality-flags {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.validation-error {
|
||||
color: #b45309;
|
||||
background: #fff7e8;
|
||||
|
||||
@@ -21,16 +21,12 @@ const emit = defineEmits<{
|
||||
}>()
|
||||
|
||||
const DATA_SOURCE_TYPES = [
|
||||
{ value: 'mysql', label: 'MySQL' },
|
||||
{ value: 'postgresql', label: 'PostgreSQL' },
|
||||
{ value: 'mongodb', label: 'MongoDB' },
|
||||
{ value: 'api', label: 'REST API' },
|
||||
]
|
||||
|
||||
const AUTH_MODES = [
|
||||
{ value: 'none', label: '免鉴权' },
|
||||
{ value: 'basic', label: '账号密码' },
|
||||
{ value: 'token', label: 'Token' },
|
||||
]
|
||||
|
||||
const FILE_PAGE_SIZE = 10
|
||||
@@ -38,9 +34,11 @@ const currentFilePage = ref(1)
|
||||
|
||||
const isExternal = computed(() => props.processType === 'external')
|
||||
|
||||
// 后端首版严格支持这些可验证的文本格式；不要把无法解析的二进制文档
|
||||
// 静默替换成示例正文。
|
||||
const uploadAccept = computed(() => props.processType === 'unstructured'
|
||||
? '.txt,.md,.pdf,.docx,.doc,.json,.jsonl'
|
||||
: '.json,.jsonl,.csv,.xlsx,.xls')
|
||||
? '.txt,.md,.json,.jsonl'
|
||||
: '.json,.jsonl,.csv,.txt,.md')
|
||||
|
||||
const pagedUploadedFiles = computed(() => {
|
||||
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
|
||||
@@ -101,7 +99,7 @@ function formatSize(size: number) {
|
||||
<el-form-item label="地址 / URL">
|
||||
<el-input
|
||||
:model-value="externalSource.url"
|
||||
placeholder="例如：mysql://host:3306/db 或 https://api.example.com/data"
|
||||
placeholder="例如：postgresql://db.example.com:5432/my_database"
|
||||
aria-label="数据源地址或 URL"
|
||||
@update:model-value="updateExternalField('url', $event)"
|
||||
/>
|
||||
@@ -140,17 +138,6 @@ function formatSize(size: number) {
|
||||
@update:model-value="updateExternalField('password', $event)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="externalSource.authMode === 'token'" label="Token">
|
||||
<el-input
|
||||
:model-value="externalSource.token"
|
||||
type="password"
|
||||
show-password
|
||||
autocomplete="off"
|
||||
placeholder="请输入访问 Token"
|
||||
aria-label="数据源访问 Token"
|
||||
@update:model-value="updateExternalField('token', $event)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="拉取条数">
|
||||
<el-input-number
|
||||
:model-value="externalSource.limit"
|
||||
@@ -162,6 +149,19 @@ function formatSize(size: number) {
|
||||
@update:model-value="updateExternalField('limit', Number($event) || 0)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="只读查询语句" class="external-query-field">
|
||||
<el-input
|
||||
:model-value="externalSource.query"
|
||||
type="textarea"
|
||||
:rows="4"
|
||||
maxlength="20000"
|
||||
show-word-limit
|
||||
placeholder="例如：SELECT question, answer FROM qa_data ORDER BY id"
|
||||
aria-label="外部数据源只读查询语句"
|
||||
@update:model-value="updateExternalField('query', $event)"
|
||||
/>
|
||||
<small>只允许单条 SELECT 或 WITH 查询；后端会拒绝写入、DDL 和多语句。</small>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
|
||||
<div class="external-actions">
|
||||
|
||||
@@ -25,7 +25,7 @@ const PREPROCESS_OPTIONS: Array<{
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
||||
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一编码、空白、字段名和 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
||||
]
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
SourceLine,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import type { SourceLine } from './types'
|
||||
|
||||
/** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
'问：如何看待当前的通货膨胀风险？',
|
||||
'答：当前通胀水平总体可控，但仍需关注能源价格与供给扰动。',
|
||||
@@ -14,26 +8,13 @@ export const DEFAULT_SOURCE_TEXT = [
|
||||
'答：会议时间以美联储官方日历为准，市场会重点关注利率路径指引。',
|
||||
'问：人民币汇率未来走势如何？',
|
||||
'答：人民币汇率取决于中美利差、经济基本面与政策预期。',
|
||||
'问：银行理财产品收益率为何持续走低？',
|
||||
'答：主要与市场利率下行、资产端收益下降以及风险偏好变化有关。',
|
||||
'问：什么是复利？',
|
||||
'答：复利是指在计算利息时，将上一期利息加入本金，再计算下一期利息。',
|
||||
'问：如何评估股票的投资价值？',
|
||||
'答：评估股票投资价值可以从以下几个方面进行：',
|
||||
'1. 公司基本面：分析公司的财务状况、盈利能力、成长性等。',
|
||||
'2. 行业前景：考察公司所处行业的发展趋势和竞争格局。',
|
||||
'3. 估值水平：通过市盈率、市净率等指标判断估值是否合理。',
|
||||
'4. 财务健康：关注公司的负债情况、现金流状况等。',
|
||||
'5. 管理团队：评估管理层的能力和过往业绩。',
|
||||
'此外，还需要关注宏观经济环境、政策变化等因素对股票市场的影响。',
|
||||
'问：债券和股票的主要区别是什么？',
|
||||
'答：债券收益相对稳定但上行有限，股票波动更大且承担更高风险。',
|
||||
'问：什么是市盈率？',
|
||||
'答：市盈率是股票价格与每股收益的比值，常用于衡量估值水平。',
|
||||
'问：如何进行资产配置？',
|
||||
'答：应根据投资目标、风险承受能力和市场环境合理分配资产。',
|
||||
'答：复利是将上一期利息加入本金，再计算下一期利息。',
|
||||
].join('\n')
|
||||
|
||||
/**
|
||||
* 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。
|
||||
*/
|
||||
export function sourceLines(sourceText: string): SourceLine[] {
|
||||
const rawLines = sourceText.split('\n')
|
||||
let cursor = 0
|
||||
@@ -46,495 +27,7 @@ export function sourceLines(sourceText: string): SourceLine[] {
|
||||
})
|
||||
}
|
||||
|
||||
interface SourceRange {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
interface ProtectedRange extends SourceRange {
|
||||
kind: 'code' | 'table' | 'list'
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 800
|
||||
const DEFAULT_CHUNK_OVERLAP = 100
|
||||
const DEFAULT_MIN_CHUNK_SIZE = 100
|
||||
|
||||
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
|
||||
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
|
||||
}
|
||||
|
||||
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
|
||||
let nextStart = Math.max(0, start)
|
||||
let nextEnd = Math.min(sourceText.length, end)
|
||||
|
||||
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
|
||||
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
|
||||
|
||||
return { start: nextStart, end: nextEnd }
|
||||
}
|
||||
|
||||
function normalizeDelimiter(delimiter: string | undefined): string {
|
||||
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
|
||||
}
|
||||
|
||||
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
|
||||
return line.start < range.end && line.end > range.start
|
||||
}
|
||||
|
||||
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
|
||||
return ranges.some((range) => overlapsRange(line, range))
|
||||
}
|
||||
|
||||
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
let openFence: { start: number; marker: string; length: number } | null = null
|
||||
|
||||
for (const line of lines) {
|
||||
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
|
||||
if (!fence) continue
|
||||
|
||||
const marker = fence[1][0]
|
||||
if (!openFence) {
|
||||
openFence = { start: line.start, marker, length: fence[1].length }
|
||||
continue
|
||||
}
|
||||
|
||||
if (marker === openFence.marker && fence[1].length >= openFence.length) {
|
||||
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
|
||||
openFence = null
|
||||
}
|
||||
}
|
||||
|
||||
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isTableSeparator(content: string): boolean {
|
||||
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
|
||||
const cells = normalized.split('|').map((cell) => cell.trim())
|
||||
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||||
}
|
||||
|
||||
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length - 1; index += 1) {
|
||||
const header = lines[index]
|
||||
const separator = lines[index + 1]
|
||||
if (
|
||||
isLineProtected(header, codeRanges)
|
||||
|| isLineProtected(separator, codeRanges)
|
||||
|| !header.content.includes('|')
|
||||
|| !isTableSeparator(separator.content)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
let endIndex = index + 1
|
||||
while (
|
||||
endIndex + 1 < lines.length
|
||||
&& !isLineProtected(lines[endIndex + 1], codeRanges)
|
||||
&& lines[endIndex + 1].content.trim()
|
||||
&& lines[endIndex + 1].content.includes('|')
|
||||
) {
|
||||
endIndex += 1
|
||||
}
|
||||
|
||||
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
|
||||
index = endIndex
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isListItem(content: string): boolean {
|
||||
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
|
||||
}
|
||||
|
||||
function isListContinuation(content: string): boolean {
|
||||
return /^\s{2,}\S/.test(content)
|
||||
}
|
||||
|
||||
function detectListRanges(
|
||||
lines: SourceLine[],
|
||||
excludedRanges: SourceRange[],
|
||||
): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
|
||||
|
||||
let endIndex = index
|
||||
let itemCount = 1
|
||||
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
|
||||
const nextContent = lines[endIndex + 1].content
|
||||
if (isListItem(nextContent)) {
|
||||
itemCount += 1
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
if (isListContinuation(nextContent)) {
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (itemCount >= 2) {
|
||||
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
|
||||
index = endIndex
|
||||
}
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
|
||||
return ranges
|
||||
.sort((left, right) => left.start - right.start || left.end - right.end)
|
||||
.reduce<ProtectedRange[]>((merged, range) => {
|
||||
const previous = merged[merged.length - 1]
|
||||
if (previous && range.start < previous.end) {
|
||||
previous.end = Math.max(previous.end, range.end)
|
||||
return merged
|
||||
}
|
||||
merged.push({ ...range })
|
||||
return merged
|
||||
}, [])
|
||||
}
|
||||
|
||||
function protectedRangesForOptions(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): ProtectedRange[] {
|
||||
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
|
||||
|
||||
const lines = sourceLines(sourceText)
|
||||
const codeRanges = detectCodeBlockRanges(sourceText, lines)
|
||||
const tableRanges = detectTableRanges(lines, codeRanges)
|
||||
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
|
||||
const enabledRanges = [
|
||||
...(options?.preserveCodeBlocks ? codeRanges : []),
|
||||
...(options?.preserveTables ? tableRanges : []),
|
||||
...(options?.preserveLists ? listRanges : []),
|
||||
]
|
||||
|
||||
return mergeProtectedRanges(enabledRanges)
|
||||
}
|
||||
|
||||
function protectedRangeContaining(
|
||||
ranges: ProtectedRange[],
|
||||
offset: number,
|
||||
): ProtectedRange | undefined {
|
||||
return ranges.find((range) => range.start < offset && offset < range.end)
|
||||
}
|
||||
|
||||
function normalizeChunkStart(
|
||||
sourceText: string,
|
||||
cursor: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
let start = Math.max(0, Math.min(cursor, sourceText.length))
|
||||
const overlapBlock = protectedRangeContaining(protectedRanges, start)
|
||||
if (overlapBlock) start = overlapBlock.end
|
||||
|
||||
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
|
||||
|
||||
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
|
||||
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
|
||||
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
function protectChunkEnd(
|
||||
proposedEnd: number,
|
||||
start: number,
|
||||
minimumEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
|
||||
if (!splitBlock) return proposedEnd
|
||||
|
||||
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
|
||||
return splitBlock.start > start && splitBlock.start >= minimumEnd
|
||||
? splitBlock.start
|
||||
: splitBlock.end
|
||||
}
|
||||
|
||||
function restoreProtectedEdges(
|
||||
range: SourceRange,
|
||||
rawStart: number,
|
||||
rawEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): SourceRange {
|
||||
const nextRange = { ...range }
|
||||
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
|
||||
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
|
||||
|
||||
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
|
||||
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
|
||||
return nextRange
|
||||
}
|
||||
|
||||
function lastBoundaryInRange(
|
||||
sourceText: string,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const candidates: number[] = []
|
||||
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
|
||||
|
||||
boundaryTokens.forEach((token) => {
|
||||
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
|
||||
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
|
||||
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
|
||||
})
|
||||
|
||||
return candidates.length ? Math.max(...candidates) : null
|
||||
}
|
||||
|
||||
function lastHeadingBoundary(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const section = sourceText.slice(start, idealEnd)
|
||||
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
|
||||
let boundary: number | null = null
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = headingPattern.exec(section))) {
|
||||
const absoluteStart = start + match.index
|
||||
if (absoluteStart >= minimumEnd) boundary = absoluteStart
|
||||
}
|
||||
|
||||
return boundary
|
||||
}
|
||||
|
||||
function resolveChunkEnd(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
options: UnstructuredProcessOptions | undefined,
|
||||
): number {
|
||||
const method = options?.chunkMethod ?? 'semantic'
|
||||
|
||||
if (method === 'fixed') return idealEnd
|
||||
|
||||
if (method === 'custom') {
|
||||
const delimiter = normalizeDelimiter(options?.customDelimiter)
|
||||
if (!delimiter) return idealEnd
|
||||
|
||||
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
|
||||
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
|
||||
return boundary >= minimumEnd ? boundary : idealEnd
|
||||
}
|
||||
|
||||
if (method === 'heading') {
|
||||
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
|
||||
if (headingBoundary !== null) return headingBoundary
|
||||
}
|
||||
|
||||
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
|
||||
}
|
||||
|
||||
function buildUnstructuredRanges(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): SourceRange[] {
|
||||
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
|
||||
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
|
||||
const minimumCharacters = Math.min(
|
||||
targetCharacters,
|
||||
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
|
||||
)
|
||||
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
|
||||
const protectedRanges = protectedRangesForOptions(sourceText, options)
|
||||
const ranges: SourceRange[] = []
|
||||
let cursor = 0
|
||||
|
||||
while (cursor < sourceText.length) {
|
||||
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
|
||||
if (start >= sourceText.length) break
|
||||
|
||||
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
|
||||
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
|
||||
let end = idealEnd === sourceText.length
|
||||
? idealEnd
|
||||
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
|
||||
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
|
||||
|
||||
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
|
||||
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
|
||||
|
||||
let range = trimSourceRange(sourceText, start, end)
|
||||
range = restoreProtectedEdges(range, start, end, protectedRanges)
|
||||
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
|
||||
range.end = Math.min(end, range.start + minimumCharacters)
|
||||
}
|
||||
if (range.end <= range.start) {
|
||||
cursor = Math.max(cursor + 1, end)
|
||||
continue
|
||||
}
|
||||
|
||||
const isLastRange = end >= sourceText.length
|
||||
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
|
||||
ranges[ranges.length - 1].end = range.end
|
||||
break
|
||||
}
|
||||
|
||||
ranges.push(range)
|
||||
if (isLastRange) break
|
||||
|
||||
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
|
||||
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
|
||||
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
|
||||
const nextCursor = range.end - actualOverlap
|
||||
cursor = nextCursor > start ? nextCursor : range.end
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
|
||||
if (!lines.length) return null
|
||||
|
||||
let low = 0
|
||||
let high = lines.length - 1
|
||||
let result = 0
|
||||
|
||||
while (low <= high) {
|
||||
const middle = Math.floor((low + high) / 2)
|
||||
if (lines[middle].start <= offset) {
|
||||
result = middle
|
||||
low = middle + 1
|
||||
} else {
|
||||
high = middle - 1
|
||||
}
|
||||
}
|
||||
|
||||
return lines[result].number
|
||||
}
|
||||
|
||||
function previewItemFromRange(
|
||||
sourceText: string,
|
||||
lines: SourceLine[],
|
||||
range: SourceRange,
|
||||
sourceFileId: string,
|
||||
index: number,
|
||||
): PreviewItem {
|
||||
const content = sourceText.slice(range.start, range.end)
|
||||
|
||||
return {
|
||||
id: `preview-${sourceFileId}-${index + 1}`,
|
||||
sourceFileId,
|
||||
originalContent: content,
|
||||
editedContent: content,
|
||||
sourceStart: range.start,
|
||||
sourceEnd: range.end,
|
||||
sourceStartLine: lineNumberAtOffset(lines, range.start),
|
||||
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
|
||||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||
status: 'original',
|
||||
}
|
||||
}
|
||||
|
||||
export function buildPreviewItems(
|
||||
sourceText: string,
|
||||
processType: ProcessType,
|
||||
sourceFileId = 'default-source',
|
||||
unstructuredOptions?: UnstructuredProcessOptions,
|
||||
): PreviewItem[] {
|
||||
const lines = sourceLines(sourceText)
|
||||
|
||||
if (processType === 'unstructured') {
|
||||
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
|
||||
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
|
||||
))
|
||||
}
|
||||
|
||||
const meaningfulLines = lines.filter((line) => line.content.trim())
|
||||
const groupSize = processType === 'structured' ? 1 : 3
|
||||
const items: PreviewItem[] = []
|
||||
|
||||
for (let index = 0; index < meaningfulLines.length; index += groupSize) {
|
||||
const group = meaningfulLines.slice(index, index + groupSize)
|
||||
if (!group.length) continue
|
||||
|
||||
const sourceStart = group[0].start
|
||||
const sourceEnd = group[group.length - 1].end
|
||||
const content = sourceText.slice(sourceStart, sourceEnd)
|
||||
|
||||
items.push({
|
||||
id: `preview-${sourceFileId}-${items.length + 1}`,
|
||||
sourceFileId,
|
||||
originalContent: content,
|
||||
editedContent: content,
|
||||
sourceStart,
|
||||
sourceEnd,
|
||||
sourceStartLine: group[0].number,
|
||||
sourceEndLine: group[group.length - 1].number,
|
||||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||
status: 'original',
|
||||
})
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
const SEMANTIC_PREFIXES = [
|
||||
'请结合实际情况，说明一下：',
|
||||
'如果方便的话，请详细解答：',
|
||||
'请用通俗易懂的方式说明：',
|
||||
'请从实际应用角度说明：',
|
||||
'请简洁、自然地说明：',
|
||||
]
|
||||
|
||||
export function createResults(
|
||||
items: PreviewItem[],
|
||||
options?: StructuredProcessOptions | UnstructuredProcessOptions,
|
||||
): ResultItem[] {
|
||||
const resultCount = options && 'qaPairsPerChunk' in options
|
||||
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||||
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||||
|
||||
const generatedResults = items.flatMap((item, index) => {
|
||||
if (options?.qualityFilterEnabled && options.filterLowQuality) {
|
||||
if (item.status === 'invalid' || !item.editedContent.trim()) return []
|
||||
}
|
||||
|
||||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||||
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||
|
||||
return Array.from({ length: resultCount }, (_, variantIndex) => {
|
||||
const instruction = options?.semanticEnrichment
|
||||
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
|
||||
: variantIndex === 0
|
||||
? baseInstruction
|
||||
: `${baseInstruction}（问法 ${variantIndex + 1}）`
|
||||
|
||||
return {
|
||||
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
|
||||
instruction,
|
||||
input: '',
|
||||
output,
|
||||
originalInstruction: instruction,
|
||||
originalInput: '',
|
||||
originalOutput: output,
|
||||
status: 'valid' as const,
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
if (!options?.qualityFilterEnabled) return generatedResults
|
||||
|
||||
return generatedResults.filter((result) => {
|
||||
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
|
||||
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
|
||||
return true
|
||||
})
|
||||
/** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */
|
||||
export function estimateTokenCount(text: string): number {
|
||||
return text.match(/[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)?.length ?? 0
|
||||
}
|
||||
|
||||
@@ -64,19 +64,25 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
|
||||
export interface ExternalDataSource {
|
||||
type: string
|
||||
url: string
|
||||
authMode: string
|
||||
authMode: 'none' | 'basic'
|
||||
username?: string
|
||||
password?: string
|
||||
token?: string
|
||||
limit: number
|
||||
query?: string
|
||||
fileName?: string
|
||||
}
|
||||
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
content: string
|
||||
fileFormat?: string
|
||||
checksumSha256?: string
|
||||
status?: 'uploading' | 'ready' | 'failed'
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface SourceLine {
|
||||
@@ -97,6 +103,10 @@ export interface PreviewItem {
|
||||
sourceEndLine: number | null
|
||||
tokenCount: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
qualityScore?: number
|
||||
qualityDetails?: Record<string, number>
|
||||
piiStats?: Record<string, number>
|
||||
updatedAt?: string
|
||||
}
|
||||
|
||||
export interface GenerationState {
|
||||
@@ -115,4 +125,9 @@ export interface ResultItem {
|
||||
originalOutput: string
|
||||
status: 'valid' | 'modified' | 'invalid'
|
||||
error?: string
|
||||
split?: 'train' | 'validation' | 'test'
|
||||
qualityScore?: number
|
||||
qualityDetails?: Record<string, number>
|
||||
qualityFlags?: string[]
|
||||
updatedAt?: string
|
||||
}
|
||||
|
||||
@@ -9,10 +9,11 @@ import type {
|
||||
} from './types'
|
||||
|
||||
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6
|
||||
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 7
|
||||
|
||||
interface DraftSnapshot {
|
||||
schemaVersion?: number
|
||||
taskId?: string
|
||||
currentStepId?: StepId
|
||||
task?: { name?: string; description?: string }
|
||||
processType?: ProcessType
|
||||
@@ -22,6 +23,7 @@ interface DraftSnapshot {
|
||||
}
|
||||
|
||||
interface DraftBindings {
|
||||
taskId: Ref<string | null>
|
||||
currentStepId: Readonly<Ref<StepId>>
|
||||
task: Reactive<{ name: string; description: string }>
|
||||
processType: Ref<ProcessType>
|
||||
@@ -35,11 +37,13 @@ interface DraftBindings {
|
||||
|
||||
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
|
||||
return {
|
||||
type: typeof source.type === 'string' ? source.type : 'mysql',
|
||||
type: typeof source.type === 'string' ? source.type : 'postgresql',
|
||||
url: typeof source.url === 'string' ? source.url : '',
|
||||
authMode: typeof source.authMode === 'string' ? source.authMode : 'none',
|
||||
authMode: source.authMode === 'basic' ? 'basic' as const : 'none' as const,
|
||||
username: typeof source.username === 'string' ? source.username : '',
|
||||
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
|
||||
query: typeof source.query === 'string' ? source.query : '',
|
||||
fileName: typeof source.fileName === 'string' ? source.fileName : 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +58,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
function draftSnapshot(): DraftSnapshot {
|
||||
return {
|
||||
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
|
||||
taskId: bindings.taskId.value || undefined,
|
||||
currentStepId: bindings.currentStepId.value,
|
||||
task: { ...bindings.task },
|
||||
processType: bindings.processType.value,
|
||||
@@ -92,6 +97,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
|
||||
bindings.restoringDraft.value = true
|
||||
bindings.goToStep('create')
|
||||
bindings.taskId.value = typeof snapshot.taskId === 'string' ? snapshot.taskId : null
|
||||
bindings.task.name = snapshot.task?.name || ''
|
||||
bindings.task.description = snapshot.task?.description || ''
|
||||
bindings.processType.value = snapshot.processType === 'unstructured' || snapshot.processType === 'external'
|
||||
@@ -128,7 +134,6 @@ export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
|
||||
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
|
||||
password: '',
|
||||
token: '',
|
||||
})
|
||||
bindings.dirty.value = false
|
||||
|
||||
@@ -137,7 +142,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
// 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。
|
||||
writeDraft(false)
|
||||
})
|
||||
ElMessage.info('已恢复上次的任务配置，请重新上传或拉取源数据')
|
||||
ElMessage.info('已恢复上次的任务配置')
|
||||
} catch {
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
}
|
||||
|
||||
@@ -1,21 +1,42 @@
|
||||
import { reactive, ref, type Ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { createResults } from './previewModel'
|
||||
import type {
|
||||
GenerationState,
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import {
|
||||
generateDataProcess,
|
||||
getDataProcessProgress,
|
||||
getDataProcessResults,
|
||||
restoreDataProcessResult,
|
||||
stopDataProcess,
|
||||
updateDataProcessResult,
|
||||
type DataProcessProgress,
|
||||
type DataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { GenerationState, ResultItem } from './types'
|
||||
|
||||
interface GenerationBindings {
|
||||
previewItems: Ref<PreviewItem[]>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
taskId: Ref<string | null>
|
||||
dirty: Ref<boolean>
|
||||
beforeGenerate?: () => Promise<void>
|
||||
}
|
||||
|
||||
const RESULT_PAGE_SIZE = 500
|
||||
const POLL_INTERVAL_MS = 1500
|
||||
|
||||
function mapResult(item: DataProcessResult): ResultItem {
|
||||
return {
|
||||
id: String(item.id),
|
||||
instruction: item.instruction,
|
||||
input: item.input || '',
|
||||
output: item.output,
|
||||
originalInstruction: item.original_instruction ?? item.instruction,
|
||||
originalInput: item.original_input ?? item.input ?? '',
|
||||
originalOutput: item.original_output ?? item.output,
|
||||
status: item.status,
|
||||
error: item.error || undefined,
|
||||
split: item.split || undefined,
|
||||
qualityScore: item.quality_score?.overall,
|
||||
qualityFlags: item.quality_score?.flags || [],
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
|
||||
export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
@@ -26,10 +47,13 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
progress: 0,
|
||||
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
|
||||
})
|
||||
let generationTimer: ReturnType<typeof setInterval> | null = null
|
||||
let generationTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let generationRun = 0
|
||||
let pollFailureCount = 0
|
||||
|
||||
function stopGenerationTimer() {
|
||||
if (generationTimer) clearInterval(generationTimer)
|
||||
generationRun += 1
|
||||
if (generationTimer) clearTimeout(generationTimer)
|
||||
generationTimer = null
|
||||
}
|
||||
|
||||
@@ -42,35 +66,123 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
selectedResultId.value = null
|
||||
}
|
||||
|
||||
function startGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'running'
|
||||
generation.progress = 0
|
||||
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
|
||||
|
||||
generationTimer = setInterval(() => {
|
||||
generation.progress = Math.min(100, generation.progress + 8)
|
||||
if (generation.progress < 100) return
|
||||
|
||||
stopGenerationTimer()
|
||||
generation.status = 'success'
|
||||
results.value = createResults(
|
||||
bindings.previewItems.value,
|
||||
bindings.processType.value === 'structured'
|
||||
? bindings.structuredOptions.value
|
||||
: bindings.processType.value === 'unstructured'
|
||||
? bindings.unstructuredOptions.value
|
||||
: undefined,
|
||||
)
|
||||
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
bindings.dirty.value = true
|
||||
ElMessage.success('数据处理完成')
|
||||
}, 180)
|
||||
function applyProgress(progress: DataProcessProgress) {
|
||||
generation.progress = Math.max(0, Math.min(100, Number(progress.progress) || 0))
|
||||
generation.message = progress.message || (
|
||||
progress.status === 'running'
|
||||
? '后端正在生成标准化结果并进行质量评分。'
|
||||
: progress.status === 'completed'
|
||||
? '数据处理已完成。'
|
||||
: progress.failure_reason || '任务已停止。'
|
||||
)
|
||||
}
|
||||
|
||||
function stopGeneration() {
|
||||
async function loadAllResults(taskId: string) {
|
||||
const first = await getDataProcessResults(taskId, { page: 1, page_size: RESULT_PAGE_SIZE })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const next = await getDataProcessResults(taskId, { page, page_size: RESULT_PAGE_SIZE })
|
||||
items.push(...next.items)
|
||||
}
|
||||
results.value = items.map(mapResult)
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
}
|
||||
|
||||
async function finishFromProgress(progress: DataProcessProgress) {
|
||||
pollFailureCount = 0
|
||||
applyProgress(progress)
|
||||
if (progress.status === 'completed') {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
await loadAllResults(taskId)
|
||||
generation.status = 'success'
|
||||
generation.progress = 100
|
||||
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
|
||||
bindings.dirty.value = true
|
||||
ElMessage.success('数据处理完成')
|
||||
return
|
||||
}
|
||||
if (progress.status === 'failed' || progress.status === 'stopped') {
|
||||
generation.status = 'failed'
|
||||
generation.message = progress.failure_reason || progress.message || (
|
||||
progress.status === 'stopped' ? '任务已停止，可以重新生成。' : '数据处理失败，请检查配置后重试。'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function pollGeneration(runId: number) {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId || runId !== generationRun || generation.status !== 'running') return
|
||||
|
||||
try {
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (runId !== generationRun) return
|
||||
pollFailureCount = 0
|
||||
if (progress.status === 'running' || progress.status === 'pending') {
|
||||
applyProgress(progress)
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
await finishFromProgress(progress)
|
||||
} catch (error) {
|
||||
if (runId !== generationRun) return
|
||||
pollFailureCount += 1
|
||||
if (pollFailureCount <= 3) {
|
||||
generation.message = `进度查询暂时失败，正在重试（${pollFailureCount}/3）…`
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。'
|
||||
}
|
||||
}
|
||||
|
||||
async function startGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) {
|
||||
ElMessage.error('任务尚未创建，请返回上一步重试')
|
||||
return
|
||||
}
|
||||
|
||||
stopGenerationTimer()
|
||||
const runId = generationRun
|
||||
generation.status = 'running'
|
||||
pollFailureCount = 0
|
||||
generation.progress = 0
|
||||
generation.message = '正在同步预览修改并启动后端处理，请稍候。'
|
||||
|
||||
try {
|
||||
await bindings.beforeGenerate?.()
|
||||
const progress = await generateDataProcess(taskId)
|
||||
if (runId !== generationRun) return
|
||||
if (progress.status === 'completed' || progress.status === 'failed' || progress.status === 'stopped') {
|
||||
await finishFromProgress(progress)
|
||||
return
|
||||
}
|
||||
applyProgress(progress)
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
} catch (error) {
|
||||
if (runId !== generationRun) return
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '启动数据处理失败，请重试。'
|
||||
}
|
||||
}
|
||||
|
||||
async function stopGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
stopGenerationTimer()
|
||||
try {
|
||||
const progress = await stopDataProcess(taskId)
|
||||
applyProgress(progress)
|
||||
} catch {
|
||||
generation.status = 'running'
|
||||
generation.message = '停止请求失败，继续查询后端任务状态。'
|
||||
const runId = generationRun
|
||||
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
|
||||
return
|
||||
}
|
||||
generation.status = 'failed'
|
||||
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
|
||||
}
|
||||
@@ -88,22 +200,41 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
|
||||
function restoreResult(id: string) {
|
||||
async function restoreResult(id: string) {
|
||||
const taskId = bindings.taskId.value
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item.instruction = item.originalInstruction
|
||||
item.input = item.originalInput
|
||||
item.output = item.originalOutput
|
||||
item.error = undefined
|
||||
item.status = 'valid'
|
||||
if (!taskId || !item) return
|
||||
const restored = await restoreDataProcessResult(taskId, id)
|
||||
const index = results.value.indexOf(item)
|
||||
results.value[index] = mapResult(restored)
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
|
||||
async function persistResultChanges() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) throw new Error('任务尚未创建')
|
||||
const changed = results.value.filter((item) => (
|
||||
item.instruction !== item.originalInstruction
|
||||
|| item.input !== item.originalInput
|
||||
|| item.output !== item.originalOutput
|
||||
))
|
||||
for (const item of changed) {
|
||||
const saved = await updateDataProcessResult(taskId, item.id, {
|
||||
instruction: item.instruction,
|
||||
input: item.input,
|
||||
output: item.output,
|
||||
expected_updated_at: item.updatedAt,
|
||||
})
|
||||
const index = results.value.findIndex((entry) => entry.id === item.id)
|
||||
if (index >= 0) results.value[index] = mapResult(saved)
|
||||
}
|
||||
}
|
||||
|
||||
function validateResults() {
|
||||
let firstInvalidId: string | null = null
|
||||
for (const item of results.value) {
|
||||
if (!item.instruction.trim() || !item.output.trim()) {
|
||||
item.error = 'Instruction 和 Output 不能为空'
|
||||
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
|
||||
item.error ||= '结果未通过后端质量校验，请修改后重新保存'
|
||||
item.status = 'invalid'
|
||||
firstInvalidId ??= item.id
|
||||
}
|
||||
@@ -116,6 +247,7 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
generation,
|
||||
results,
|
||||
selectedResultId,
|
||||
persistResultChanges,
|
||||
resetDownstream,
|
||||
restoreResult,
|
||||
startGeneration,
|
||||
|
||||
Reference in New Issue
Block a user